diff --git a/db.go b/db/db.go similarity index 74% rename from db.go rename to db/db.go index 734921b..d283084 100644 --- a/db.go +++ b/db/db.go @@ -1,4 +1,4 @@ -package main +package db import ( "database/sql" @@ -6,19 +6,24 @@ import ( "log" _ "github.com/mattn/go-sqlite3" + "gitlab.com/ekzyis/hnbot/hn" ) var ( - db *sql.DB + _db *sql.DB ) func init() { var err error - db, err = sql.Open("sqlite3", "hnbot.sqlite3") + _db, err = sql.Open("sqlite3", "hnbot.sqlite3") if err != nil { log.Fatal(err) } - migrate(db) + migrate(_db) +} + +func Query(query string, args ...interface{}) (*sql.Rows, error) { + return _db.Query(query, args...) } func migrate(db *sql.DB) { @@ -54,7 +59,7 @@ func migrate(db *sql.DB) { func ItemHasComment(parentId int) bool { var count int - err := db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count) + err := _db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count) if err != nil { err = fmt.Errorf("error during item check: %w", err) log.Fatal(err) @@ -62,17 +67,17 @@ func ItemHasComment(parentId int) bool { return count > 0 } -func SaveStories(story *[]Story) error { +func SaveHnItems(story *[]hn.Item) error { for i, s := range *story { - if err := SaveStory(&s, i+1); err != nil { + if err := SaveHnItem(&s, i+1); err != nil { return err } } return nil } -func SaveStory(s *Story, rank int) error { - if _, err := db.Exec(` +func SaveHnItem(s *hn.Item, rank int) error { + if _, err := _db.Exec(` INSERT INTO hn_items(id, time, title, url, author, ndescendants, score, rank) VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, s.ID, s.Time, s.Title, s.Url, s.By, s.Descendants, s.Score, rank); err != nil { @@ -83,7 +88,7 @@ func SaveStory(s *Story, rank int) error { } func SaveSnItem(id int, hnId int) error { - if _, err := db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil { + if _, err := _db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil { err = fmt.Errorf("error during sn item insert: %w", err) return err } diff --git a/hn.go b/hn.go deleted file mode 100644 index 2b60056..0000000 --- a/hn.go +++ /dev/null @@ -1,107 +0,0 @@ -package main - -import ( - "encoding/json" - "errors" - "fmt" - "log" - "net/http" - "regexp" - "strconv" -) - -type Story struct { - ID int - By string // username of author - Time int // UNIX timestamp - Descendants int // number of comments - Kids []int - Score int - Title string - Url string -} - -var ( - HackerNewsUrl = "https://news.ycombinator.com" - HackerNewsFirebaseUrl = "https://hacker-news.firebaseio.com/v0" - HackerNewsLinkRegexp = regexp.MustCompile(`(?:https?:\/\/)?news\.ycombinator\.com\/item\?id=([0-9]+)`) -) - -func FetchHackerNewsTopStories() ([]Story, error) { - log.Println("Fetching HN top stories ...") - - // API docs: https://github.com/HackerNews/API - url := fmt.Sprintf("%s/topstories.json", HackerNewsFirebaseUrl) - resp, err := http.Get(url) - if err != nil { - err = fmt.Errorf("error fetching HN top stories %w:", err) - return nil, err - } - defer resp.Body.Close() - - var ids []int - err = json.NewDecoder(resp.Body).Decode(&ids) - if err != nil { - err = fmt.Errorf("error decoding HN top stories JSON: %w", err) - return nil, err - } - - // we are only interested in the first page of top stories - const limit = 30 - ids = ids[:limit] - - var stories [limit]Story - for i, id := range ids { - story, err := FetchStoryById(id) - if err != nil { - return nil, err - } - stories[i] = story - } - - log.Println("Fetching HN top stories ... OK") - // Can't return [30]Story as []Story so we copy the array - return stories[:], nil -} - -func FetchStoryById(id int) (Story, error) { - log.Printf("Fetching HN story (id=%d) ...\n", id) - - url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id) - resp, err := http.Get(url) - if err != nil { - err = fmt.Errorf("error fetching HN story (id=%d): %w", id, err) - return Story{}, err - } - defer resp.Body.Close() - - var story Story - err = json.NewDecoder(resp.Body).Decode(&story) - if err != nil { - err := fmt.Errorf("error decoding HN story JSON (id=%d): %w", id, err) - return Story{}, err - } - - log.Printf("Fetching HN story (id=%d) ... OK\n", id) - return story, nil -} - -func ParseHackerNewsLink(link string) (int, error) { - match := HackerNewsLinkRegexp.FindStringSubmatch(link) - if len(match) == 0 { - return -1, errors.New("input is not a hacker news link") - } - id, err := strconv.Atoi(match[1]) - if err != nil { - return -1, errors.New("integer conversion to string failed") - } - return id, nil -} - -func HackerNewsUserLink(user string) string { - return fmt.Sprintf("%s/user?id=%s", HackerNewsUrl, user) -} - -func HackerNewsItemLink(id int) string { - return fmt.Sprintf("%s/item?id=%d", HackerNewsUrl, id) -} diff --git a/hn/hn.go b/hn/hn.go new file mode 100644 index 0000000..85937e9 --- /dev/null +++ b/hn/hn.go @@ -0,0 +1,105 @@ +package hn + +import ( + "encoding/json" + "errors" + "fmt" + "log" + "net/http" + "regexp" + "strconv" +) + +type Item struct { + ID int + By string // username of author + Time int // UNIX timestamp + Descendants int // number of comments + Kids []int + Score int + Title string + Url string +} + +var ( + hnUrl = "https://news.ycombinator.com" + hnFirebaseUrl = "https://hacker-news.firebaseio.com/v0" + hnLinkRegexp = regexp.MustCompile(`(?:https?:\/\/)?news\.ycombinator\.com\/item\?id=([0-9]+)`) +) + +func FetchTopItems() ([]Item, error) { + log.Println("[hn] fetch top items ...") + + // API docs: https://github.com/HackerNews/API + url := fmt.Sprintf("%s/topstories.json", hnFirebaseUrl) + resp, err := http.Get(url) + if err != nil { + return nil, fmt.Errorf("error fetching HN top stories %w:", err) + } + defer resp.Body.Close() + + var ids []int + err = json.NewDecoder(resp.Body).Decode(&ids) + if err != nil { + return nil, fmt.Errorf("error decoding HN top stories JSON: %w", err) + } + + // we are only interested in the first page of top stories + const limit = 30 + ids = ids[:limit] + + var stories [limit]Item + for i, id := range ids { + var item Item + err := FetchItemById(id, &item) + if err != nil { + return nil, err + } + stories[i] = item + } + + log.Println("[hn] fetch top items ... OK") + // Can't return [30]Item as []Item so we copy the array + return stories[:], nil +} + +func FetchItemById(id int, hnItem *Item) error { + // log.Printf("[hn] fetch HN item %d ...\n", id) + + url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id) + resp, err := http.Get(url) + if err != nil { + err = fmt.Errorf("error fetching HN item %d: %w", id, err) + return err + } + defer resp.Body.Close() + + err = json.NewDecoder(resp.Body).Decode(&hnItem) + if err != nil { + err := fmt.Errorf("error decoding JSON for HN item %d: %w", id, err) + return err + } + + // log.Printf("[hn] fetch HN item %d ... OK\n", id) + return nil +} + +func ParseLink(link string) (int, error) { + match := hnLinkRegexp.FindStringSubmatch(link) + if len(match) == 0 { + return -1, errors.New("not a hacker news link") + } + id, err := strconv.Atoi(match[1]) + if err != nil { + return -1, errors.New("integer conversion to string failed") + } + return id, nil +} + +func UserLink(user string) string { + return fmt.Sprintf("%s/user?id=%s", hnUrl, user) +} + +func ItemLink(id int) string { + return fmt.Sprintf("%s/item?id=%d", hnUrl, id) +} diff --git a/main.go b/main.go index b7fcea2..2a3d459 100644 --- a/main.go +++ b/main.go @@ -5,22 +5,24 @@ import ( "log" "time" - sn "github.com/ekzyis/snappy" + "gitlab.com/ekzyis/hnbot/db" + "gitlab.com/ekzyis/hnbot/hn" + sn "gitlab.com/ekzyis/hnbot/sn" ) -func SyncStories() { +func SyncHnItemsToDb() { for { now := time.Now() dur := now.Truncate(time.Minute).Add(time.Minute).Sub(now) log.Println("[hn] sleeping for", dur.Round(time.Second)) time.Sleep(dur) - stories, err := FetchHackerNewsTopStories() + stories, err := hn.FetchTopItems() if err != nil { log.Println(err) continue } - if err := SaveStories(&stories); err != nil { + if err := db.SaveHnItems(&stories); err != nil { log.Println(err) continue } @@ -28,10 +30,13 @@ func SyncStories() { } func main() { - go SyncStories() + // fetch HN front page every minute in the background and store state in db + go SyncHnItemsToDb() + + // check every 15 minutes if there is now a HN item that is worth posting to SN for { var ( - filtered *[]Story + filtered *[]hn.Item err error ) @@ -40,21 +45,21 @@ func main() { log.Println("[sn] sleeping for", dur.Round(time.Second)) time.Sleep(dur) - if filtered, err = CurateContentForStackerNews(); err != nil { + if filtered, err = sn.CurateContent(); err != nil { log.Println(err) continue } - for _, story := range *filtered { - _, err := PostStoryToStackerNews(&story, PostStoryOptions{SkipDupes: false}) + log.Printf("[sn] found %d item(s) to post\n", len(*filtered)) + + for _, item := range *filtered { + _, err := sn.Post(&item, sn.PostOptions{SkipDupes: false}) if err != nil { var dupesErr *sn.DupesError if errors.As(err, &dupesErr) { - // SendDupesErrorToDiscord(story.ID, dupesErr) log.Println(dupesErr) - // save dupe in db to prevent retries parentId := dupesErr.Dupes[0].Id - if err := SaveSnItem(parentId, story.ID); err != nil { + if err := db.SaveSnItem(parentId, item.ID); err != nil { log.Println(err) } continue diff --git a/sn.go b/sn/sn.go similarity index 65% rename from sn.go rename to sn/sn.go index 37ea84f..a7da590 100644 --- a/sn.go +++ b/sn/sn.go @@ -1,4 +1,4 @@ -package main +package sn import ( "database/sql" @@ -8,9 +8,13 @@ import ( "github.com/dustin/go-humanize" sn "github.com/ekzyis/snappy" + "gitlab.com/ekzyis/hnbot/db" + "gitlab.com/ekzyis/hnbot/hn" ) -func CurateContentForStackerNews() (*[]Story, error) { +type DupesError = sn.DupesError + +func CurateContent() (*[]hn.Item, error) { var ( rows *sql.Rows err error @@ -32,33 +36,33 @@ func CurateContentForStackerNews() (*[]Story, error) { } defer rows.Close() - var stories []Story + var items []hn.Item for rows.Next() { - var story Story - if err = rows.Scan(&story.ID, &story.Time, &story.Title, &story.Url, &story.By, &story.Score, &story.Descendants); err != nil { + var item hn.Item + if err = rows.Scan(&item.ID, &item.Time, &item.Title, &item.Url, &item.By, &item.Score, &item.Descendants); err != nil { err = fmt.Errorf("error scanning hn_items: %w", err) return nil, err } - stories = append(stories, story) + items = append(items, item) } if err = rows.Err(); err != nil { err = fmt.Errorf("error iterating hn_items: %w", err) return nil, err } - return &stories, nil + return &items, nil } -type PostStoryOptions struct { +type PostOptions struct { SkipDupes bool } -func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error) { +func Post(item *hn.Item, options PostOptions) (int, error) { c := sn.NewClient() - url := story.Url + url := item.Url if url == "" { - url = HackerNewsItemLink(story.ID) + url = hn.ItemLink(item.ID) } - log.Printf("Posting to SN (url=%s) ...\n", url) + log.Printf("post to SN: %s ...\n", url) if !options.SkipDupes { dupes, err := c.Dupes(url) @@ -70,18 +74,18 @@ func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error) } } - title := story.Title + title := item.Title if len(title) > 80 { title = title[0:80] } comment := fmt.Sprintf( "This link was posted by [%s](%s) %s on [HN](%s). It received %d points and %d comments.", - story.By, - HackerNewsUserLink(story.By), - humanize.Time(time.Unix(int64(story.Time), 0)), - HackerNewsItemLink(story.ID), - story.Score, story.Descendants, + item.By, + hn.UserLink(item.By), + humanize.Time(time.Unix(int64(item.Time), 0)), + hn.ItemLink(item.ID), + item.Score, item.Descendants, ) parentId, err := c.PostLink(url, title, comment, "tech") @@ -89,8 +93,8 @@ func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error) return -1, fmt.Errorf("error posting link: %w", err) } - log.Printf("Posting to SN (url=%s) ... OK \n", url) - if err := SaveSnItem(parentId, story.ID); err != nil { + log.Printf("post to SN: %s ... OK \n", url) + if err := db.SaveSnItem(parentId, item.ID); err != nil { return -1, err }