Compare commits
No commits in common. "develop" and "v0.1.0" have entirely different histories.
1
.env.template
Normal file
1
.env.template
Normal file
@ -0,0 +1 @@
|
||||
AUTH_COOKIE=
|
||||
5
.gitignore
vendored
5
.gitignore
vendored
@ -1,6 +1 @@
|
||||
.env
|
||||
.vscode
|
||||
|
||||
# go executable
|
||||
hnbot
|
||||
hnbot.sqlite3
|
||||
|
||||
10
README.md
10
README.md
@ -1,11 +1,7 @@
|
||||
# hnbot
|
||||
|
||||
> Hello, I am a bot posting top stories from HN.
|
||||
> Hello, I am a bot crossposting top posts from HN.
|
||||
>
|
||||
> My original mission was to orange-pill HN by offering the OPs on HN to claim the sats their stories received here.
|
||||
However, my comments were shadowbanned and ultimately not approved by dang, the site admin.
|
||||
See this thread: [#164155](https://stacker.news/items/164155)
|
||||
>
|
||||
> If you are one of these OPs and want to claim your sats, reply to this bio and we will find a solution!
|
||||
> I curate content to only post stuff which could be interesting for the SN community on a best-efforts basis.
|
||||
|
||||
-- https://stacker.news/hn
|
||||
-- https://stacker.news/items/161788
|
||||
|
||||
101
db/db.go
101
db/db.go
@ -1,101 +0,0 @@
|
||||
package db
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
|
||||
"github.com/ekzyis/hnbot/hn"
|
||||
_ "github.com/mattn/go-sqlite3"
|
||||
)
|
||||
|
||||
var (
|
||||
_db *sql.DB
|
||||
)
|
||||
|
||||
func init() {
|
||||
var err error
|
||||
_db, err = sql.Open("sqlite3", "hnbot.sqlite3")
|
||||
if err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// fix 'database is locked' error
|
||||
// see https://github.com/mattn/go-sqlite3/issues/274#issuecomment-191597862
|
||||
_db.SetMaxOpenConns(1)
|
||||
|
||||
migrate(_db)
|
||||
}
|
||||
|
||||
func Query(query string, args ...interface{}) (*sql.Rows, error) {
|
||||
return _db.Query(query, args...)
|
||||
}
|
||||
|
||||
func migrate(db *sql.DB) {
|
||||
if _, err := db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS hn_items (
|
||||
id INTEGER NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
time TIMESTAMP WITH TIMEZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
title TEXT NOT NULL,
|
||||
url TEXT,
|
||||
author TEXT NOT NULL,
|
||||
ndescendants INTEGER NOT NULL,
|
||||
score INTEGER NOT NULL,
|
||||
rank INTEGER NOT NULL,
|
||||
PRIMARY KEY (id, created_at)
|
||||
);
|
||||
`); err != nil {
|
||||
err = fmt.Errorf("error during migration: %w", err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
if _, err := db.Exec(`
|
||||
CREATE TABLE IF NOT EXISTS sn_items (
|
||||
id INTEGER NOT NULL,
|
||||
created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
hn_id INTEGER NOT NULL REFERENCES hn_items(id),
|
||||
PRIMARY KEY (id, hn_id)
|
||||
);
|
||||
`); err != nil {
|
||||
err = fmt.Errorf("error during migration: %w", err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func ItemHasComment(parentId int) bool {
|
||||
var count int
|
||||
err := _db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error during item check: %w", err)
|
||||
log.Fatal(err)
|
||||
}
|
||||
return count > 0
|
||||
}
|
||||
|
||||
func SaveHnItems(story *[]hn.Item) error {
|
||||
for i, s := range *story {
|
||||
if err := SaveHnItem(&s, i+1); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveHnItem(s *hn.Item, rank int) error {
|
||||
if _, err := _db.Exec(`
|
||||
INSERT INTO hn_items(id, time, title, url, author, ndescendants, score, rank)
|
||||
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
|
||||
s.ID, s.Time, s.Title, s.Url, s.By, s.Descendants, s.Score, rank); err != nil {
|
||||
err = fmt.Errorf("error during item insert: %w", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func SaveSnItem(id int, hnId int) error {
|
||||
if _, err := _db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil {
|
||||
err = fmt.Errorf("error during sn item insert: %w", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
10
go.mod
10
go.mod
@ -1,12 +1,8 @@
|
||||
module github.com/ekzyis/hnbot
|
||||
module gitlab.com/ekzyis/hnbot
|
||||
|
||||
go 1.20
|
||||
|
||||
require (
|
||||
github.com/dustin/go-humanize v1.0.1
|
||||
github.com/ekzyis/snappy v0.8.2
|
||||
github.com/joho/godotenv v1.5.1
|
||||
github.com/mattn/go-sqlite3 v1.14.22
|
||||
github.com/joho/godotenv v1.5.1 // indirect
|
||||
github.com/namsral/flag v1.7.4-pre // indirect
|
||||
)
|
||||
|
||||
require gopkg.in/guregu/null.v4 v4.0.0 // indirect
|
||||
|
||||
10
go.sum
10
go.sum
@ -1,10 +1,4 @@
|
||||
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
|
||||
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
|
||||
github.com/ekzyis/snappy v0.8.2 h1:fT+IekT9QMg82waxfo5Qtutb5pZIJDTiw/iK0kYhRKs=
|
||||
github.com/ekzyis/snappy v0.8.2/go.mod h1:UksYI0dU0+cnzz0LQjWB1P0QQP/ghx47e4atP99a5Lk=
|
||||
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
|
||||
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
|
||||
github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
|
||||
github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
|
||||
gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg=
|
||||
gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI=
|
||||
github.com/namsral/flag v1.7.4-pre h1:b2ScHhoCUkbsq0d2C15Mv+VU8bl8hAXV8arnWiOHNZs=
|
||||
github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo=
|
||||
|
||||
105
hn/hn.go
105
hn/hn.go
@ -1,105 +0,0 @@
|
||||
package hn
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"log"
|
||||
"net/http"
|
||||
"regexp"
|
||||
"strconv"
|
||||
)
|
||||
|
||||
type Item struct {
|
||||
ID int
|
||||
By string // username of author
|
||||
Time int // UNIX timestamp
|
||||
Descendants int // number of comments
|
||||
Kids []int
|
||||
Score int
|
||||
Title string
|
||||
Url string
|
||||
}
|
||||
|
||||
var (
|
||||
hnUrl = "https://news.ycombinator.com"
|
||||
hnFirebaseUrl = "https://hacker-news.firebaseio.com/v0"
|
||||
hnLinkRegexp = regexp.MustCompile(`(?:https?:\/\/)?news\.ycombinator\.com\/item\?id=([0-9]+)`)
|
||||
)
|
||||
|
||||
func FetchTopItems() ([]Item, error) {
|
||||
log.Println("[hn] fetch top items ...")
|
||||
|
||||
// API docs: https://github.com/HackerNews/API
|
||||
url := fmt.Sprintf("%s/topstories.json", hnFirebaseUrl)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error fetching HN top stories %w:", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
var ids []int
|
||||
err = json.NewDecoder(resp.Body).Decode(&ids)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("error decoding HN top stories JSON: %w", err)
|
||||
}
|
||||
|
||||
// we are only interested in the first page of top stories
|
||||
const limit = 30
|
||||
ids = ids[:limit]
|
||||
|
||||
var stories [limit]Item
|
||||
for i, id := range ids {
|
||||
var item Item
|
||||
err := FetchItemById(id, &item)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
stories[i] = item
|
||||
}
|
||||
|
||||
log.Println("[hn] fetch top items ... OK")
|
||||
// Can't return [30]Item as []Item so we copy the array
|
||||
return stories[:], nil
|
||||
}
|
||||
|
||||
func FetchItemById(id int, hnItem *Item) error {
|
||||
// log.Printf("[hn] fetch HN item %d ...\n", id)
|
||||
|
||||
url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
err = fmt.Errorf("error fetching HN item %d: %w", id, err)
|
||||
return err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
err = json.NewDecoder(resp.Body).Decode(&hnItem)
|
||||
if err != nil {
|
||||
err := fmt.Errorf("error decoding JSON for HN item %d: %w", id, err)
|
||||
return err
|
||||
}
|
||||
|
||||
// log.Printf("[hn] fetch HN item %d ... OK\n", id)
|
||||
return nil
|
||||
}
|
||||
|
||||
func ParseLink(link string) (int, error) {
|
||||
match := hnLinkRegexp.FindStringSubmatch(link)
|
||||
if len(match) == 0 {
|
||||
return -1, errors.New("not a hacker news link")
|
||||
}
|
||||
id, err := strconv.Atoi(match[1])
|
||||
if err != nil {
|
||||
return -1, errors.New("integer conversion to string failed")
|
||||
}
|
||||
return id, nil
|
||||
}
|
||||
|
||||
func UserLink(user string) string {
|
||||
return fmt.Sprintf("%s/user?id=%s", hnUrl, user)
|
||||
}
|
||||
|
||||
func ItemLink(id int) string {
|
||||
return fmt.Sprintf("%s/item?id=%d", hnUrl, id)
|
||||
}
|
||||
191
main.go
191
main.go
@ -1,77 +1,150 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
"net/http"
|
||||
|
||||
"github.com/ekzyis/hnbot/db"
|
||||
"github.com/ekzyis/hnbot/hn"
|
||||
sn "github.com/ekzyis/hnbot/sn"
|
||||
"github.com/joho/godotenv"
|
||||
"github.com/namsral/flag"
|
||||
)
|
||||
|
||||
func SyncHnItemsToDb() {
|
||||
for {
|
||||
now := time.Now()
|
||||
dur := now.Truncate(time.Minute).Add(time.Minute).Sub(now)
|
||||
log.Println("[hn] sleeping for", dur.Round(time.Second))
|
||||
time.Sleep(dur)
|
||||
var (
|
||||
AuthCookie string
|
||||
)
|
||||
|
||||
stories, err := hn.FetchTopItems()
|
||||
func init() {
|
||||
err := godotenv.Load()
|
||||
if err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
log.Fatal("Error loading .env file")
|
||||
}
|
||||
if err := db.SaveHnItems(&stories); err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
flag.StringVar(&AuthCookie, "auth_cookie", "", "Cookie required for authorization")
|
||||
flag.Parse()
|
||||
if AuthCookie == "" {
|
||||
log.Fatal("auth cookie not set")
|
||||
}
|
||||
}
|
||||
|
||||
type ItemID = int
|
||||
|
||||
type Story struct {
|
||||
ID ItemID
|
||||
By string // username of author
|
||||
Time int // UNIX timestamp
|
||||
Descendants int // number of comments
|
||||
Kids []ItemID
|
||||
Score int
|
||||
Title string
|
||||
Url string
|
||||
}
|
||||
|
||||
type GraphQLPayload struct {
|
||||
Query string `json:"query"`
|
||||
Variables map[string]interface{} `json:"variables,omitempty"`
|
||||
}
|
||||
|
||||
func fetchTopStoriesFromHN() []Story {
|
||||
// API docs: https://github.com/HackerNews/API
|
||||
|
||||
url := "https://hacker-news.firebaseio.com/v0/topstories.json"
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
log.Fatal("Error fetching top stories:", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
log.Printf("GET %s %d\n", url, resp.StatusCode)
|
||||
|
||||
var ids []int
|
||||
err = json.NewDecoder(resp.Body).Decode(&ids)
|
||||
if err != nil {
|
||||
log.Fatal("Error decoding top stories JSON:", err)
|
||||
}
|
||||
|
||||
// we are only interested in the first page of top stories
|
||||
const limit = 30
|
||||
ids = ids[:limit]
|
||||
|
||||
var stories [limit]Story
|
||||
for i, id := range ids {
|
||||
story := fetchStoryByID(id)
|
||||
stories[i] = story
|
||||
}
|
||||
|
||||
// Can't return [30]Story as []Story so we copy the array
|
||||
return stories[:]
|
||||
}
|
||||
|
||||
func fetchStoryByID(id ItemID) Story {
|
||||
url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
log.Fatal("Error fetching story:", err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
log.Printf("GET %s %d\n", url, resp.StatusCode)
|
||||
|
||||
var story Story
|
||||
err = json.NewDecoder(resp.Body).Decode(&story)
|
||||
if err != nil {
|
||||
log.Fatal("Error decoding story JSON:", err)
|
||||
}
|
||||
|
||||
return story
|
||||
}
|
||||
|
||||
func filterByRelevanceForSN(stories *[]Story) *[]Story {
|
||||
// TODO: filter by relevance
|
||||
|
||||
slice := (*stories)[0:1]
|
||||
return &slice
|
||||
}
|
||||
|
||||
func postToSN(story *Story) {
|
||||
// TODO: check for dupes first
|
||||
|
||||
body := GraphQLPayload{
|
||||
Query: `
|
||||
mutation upsertLink($url: String!, $title: String!) {
|
||||
upsertLink(url: $url, title: $title) {
|
||||
id
|
||||
}
|
||||
}
|
||||
`,
|
||||
Variables: map[string]interface{}{
|
||||
"url": story.Url,
|
||||
"title": story.Title,
|
||||
},
|
||||
}
|
||||
|
||||
bodyJSON, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
log.Fatal("Error during json.Marshal:", err)
|
||||
}
|
||||
|
||||
url := "https://stacker.news/api/graphql"
|
||||
req, err := http.NewRequest("POST", url, bytes.NewBuffer(bodyJSON))
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
req.Header.Set("Content-Type", "application/json")
|
||||
req.Header.Set("Cookie", AuthCookie)
|
||||
|
||||
client := http.DefaultClient
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
panic(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
|
||||
log.Printf("POST %s %d\n", url, resp.StatusCode)
|
||||
}
|
||||
|
||||
func main() {
|
||||
if err := godotenv.Load(); err != nil {
|
||||
log.Fatal(err)
|
||||
}
|
||||
|
||||
// fetch HN front page every minute in the background and store state in db
|
||||
go SyncHnItemsToDb()
|
||||
|
||||
// check every 15 minutes if there is now a HN item that is worth posting to SN
|
||||
for {
|
||||
var (
|
||||
filtered *[]hn.Item
|
||||
err error
|
||||
)
|
||||
|
||||
now := time.Now()
|
||||
dur := now.Truncate(time.Minute).Add(15 * time.Minute).Sub(now)
|
||||
log.Println("[sn] sleeping for", dur.Round(time.Second))
|
||||
time.Sleep(dur)
|
||||
|
||||
if filtered, err = sn.CurateContent(); err != nil {
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
|
||||
log.Printf("[sn] found %d item(s) to post\n", len(*filtered))
|
||||
|
||||
for _, item := range *filtered {
|
||||
_, err := sn.Post(&item, sn.PostOptions{SkipDupes: false})
|
||||
if err != nil {
|
||||
var dupesErr *sn.DupesError
|
||||
if errors.As(err, &dupesErr) {
|
||||
log.Println(dupesErr)
|
||||
parentId := dupesErr.Dupes[0].Id
|
||||
if err := db.SaveSnItem(parentId, item.ID); err != nil {
|
||||
log.Println(err)
|
||||
}
|
||||
continue
|
||||
}
|
||||
log.Println(err)
|
||||
continue
|
||||
}
|
||||
}
|
||||
stories := fetchTopStoriesFromHN()
|
||||
filtered := filterByRelevanceForSN(&stories)
|
||||
for _, story := range *filtered {
|
||||
postToSN(&story)
|
||||
}
|
||||
}
|
||||
|
||||
102
sn/sn.go
102
sn/sn.go
@ -1,102 +0,0 @@
|
||||
package sn
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"log"
|
||||
"time"
|
||||
|
||||
"github.com/dustin/go-humanize"
|
||||
"github.com/ekzyis/hnbot/db"
|
||||
"github.com/ekzyis/hnbot/hn"
|
||||
sn "github.com/ekzyis/snappy"
|
||||
)
|
||||
|
||||
type DupesError = sn.DupesError
|
||||
|
||||
func CurateContent() (*[]hn.Item, error) {
|
||||
var (
|
||||
rows *sql.Rows
|
||||
err error
|
||||
)
|
||||
if rows, err = db.Query(`
|
||||
SELECT t.id, time, title, url, author, score, ndescendants
|
||||
FROM (
|
||||
SELECT id, MIN(created_at) AS start, MAX(created_at) AS end
|
||||
FROM hn_items
|
||||
WHERE rank = 1 AND id NOT IN (SELECT hn_id FROM sn_items) AND length(title) >= 5
|
||||
GROUP BY id
|
||||
HAVING unixepoch(end) - unixepoch(start) >= 3600
|
||||
ORDER BY time ASC
|
||||
LIMIT 1
|
||||
) t JOIN hn_items ON t.id = hn_items.id AND t.end = hn_items.created_at;
|
||||
`); err != nil {
|
||||
err = fmt.Errorf("error querying hn_items: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
defer rows.Close()
|
||||
|
||||
var items []hn.Item
|
||||
for rows.Next() {
|
||||
var item hn.Item
|
||||
if err = rows.Scan(&item.ID, &item.Time, &item.Title, &item.Url, &item.By, &item.Score, &item.Descendants); err != nil {
|
||||
err = fmt.Errorf("error scanning hn_items: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
items = append(items, item)
|
||||
}
|
||||
if err = rows.Err(); err != nil {
|
||||
err = fmt.Errorf("error iterating hn_items: %w", err)
|
||||
return nil, err
|
||||
}
|
||||
return &items, nil
|
||||
}
|
||||
|
||||
type PostOptions struct {
|
||||
SkipDupes bool
|
||||
}
|
||||
|
||||
func Post(item *hn.Item, options PostOptions) (int, error) {
|
||||
c := sn.NewClient()
|
||||
url := item.Url
|
||||
if url == "" {
|
||||
url = hn.ItemLink(item.ID)
|
||||
}
|
||||
log.Printf("post to SN: %s ...\n", url)
|
||||
|
||||
if !options.SkipDupes {
|
||||
dupes, err := c.Dupes(url)
|
||||
if err != nil {
|
||||
return -1, err
|
||||
}
|
||||
if len(*dupes) > 0 {
|
||||
return -1, &sn.DupesError{Url: url, Dupes: *dupes}
|
||||
}
|
||||
}
|
||||
|
||||
title := item.Title
|
||||
if len(title) > 80 {
|
||||
title = title[0:80]
|
||||
}
|
||||
|
||||
comment := fmt.Sprintf(
|
||||
"This link was posted by [%s](%s) %s on [HN](%s). It received %d points and %d comments.",
|
||||
item.By,
|
||||
hn.UserLink(item.By),
|
||||
humanize.Time(time.Unix(int64(item.Time), 0)),
|
||||
hn.ItemLink(item.ID),
|
||||
item.Score, item.Descendants,
|
||||
)
|
||||
|
||||
parentId, err := c.PostLink(url, title, comment, "tech")
|
||||
if err != nil {
|
||||
return -1, fmt.Errorf("error posting link: %w", err)
|
||||
}
|
||||
|
||||
log.Printf("post to SN: %s ... OK \n", url)
|
||||
if err := db.SaveSnItem(parentId, item.ID); err != nil {
|
||||
return -1, err
|
||||
}
|
||||
|
||||
return parentId, nil
|
||||
}
|
||||
Loading…
x
Reference in New Issue
Block a user