refactor into individual packages

This commit is contained in:
ekzyis 2025-02-03 22:47:39 +01:00
parent 7641414693
commit 2cfb0eaa31
5 changed files with 161 additions and 149 deletions

View File

@ -1,4 +1,4 @@
package main package db
import ( import (
"database/sql" "database/sql"
@ -6,19 +6,24 @@ import (
"log" "log"
_ "github.com/mattn/go-sqlite3" _ "github.com/mattn/go-sqlite3"
"gitlab.com/ekzyis/hnbot/hn"
) )
var ( var (
db *sql.DB _db *sql.DB
) )
func init() { func init() {
var err error var err error
db, err = sql.Open("sqlite3", "hnbot.sqlite3") _db, err = sql.Open("sqlite3", "hnbot.sqlite3")
if err != nil { if err != nil {
log.Fatal(err) log.Fatal(err)
} }
migrate(db) migrate(_db)
}
func Query(query string, args ...interface{}) (*sql.Rows, error) {
return _db.Query(query, args...)
} }
func migrate(db *sql.DB) { func migrate(db *sql.DB) {
@ -54,7 +59,7 @@ func migrate(db *sql.DB) {
func ItemHasComment(parentId int) bool { func ItemHasComment(parentId int) bool {
var count int var count int
err := db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count) err := _db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count)
if err != nil { if err != nil {
err = fmt.Errorf("error during item check: %w", err) err = fmt.Errorf("error during item check: %w", err)
log.Fatal(err) log.Fatal(err)
@ -62,17 +67,17 @@ func ItemHasComment(parentId int) bool {
return count > 0 return count > 0
} }
func SaveStories(story *[]Story) error { func SaveHnItems(story *[]hn.Item) error {
for i, s := range *story { for i, s := range *story {
if err := SaveStory(&s, i+1); err != nil { if err := SaveHnItem(&s, i+1); err != nil {
return err return err
} }
} }
return nil return nil
} }
func SaveStory(s *Story, rank int) error { func SaveHnItem(s *hn.Item, rank int) error {
if _, err := db.Exec(` if _, err := _db.Exec(`
INSERT INTO hn_items(id, time, title, url, author, ndescendants, score, rank) INSERT INTO hn_items(id, time, title, url, author, ndescendants, score, rank)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
s.ID, s.Time, s.Title, s.Url, s.By, s.Descendants, s.Score, rank); err != nil { s.ID, s.Time, s.Title, s.Url, s.By, s.Descendants, s.Score, rank); err != nil {
@ -83,7 +88,7 @@ func SaveStory(s *Story, rank int) error {
} }
func SaveSnItem(id int, hnId int) error { func SaveSnItem(id int, hnId int) error {
if _, err := db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil { if _, err := _db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil {
err = fmt.Errorf("error during sn item insert: %w", err) err = fmt.Errorf("error during sn item insert: %w", err)
return err return err
} }

107
hn.go
View File

@ -1,107 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"regexp"
"strconv"
)
type Story struct {
ID int
By string // username of author
Time int // UNIX timestamp
Descendants int // number of comments
Kids []int
Score int
Title string
Url string
}
var (
HackerNewsUrl = "https://news.ycombinator.com"
HackerNewsFirebaseUrl = "https://hacker-news.firebaseio.com/v0"
HackerNewsLinkRegexp = regexp.MustCompile(`(?:https?:\/\/)?news\.ycombinator\.com\/item\?id=([0-9]+)`)
)
func FetchHackerNewsTopStories() ([]Story, error) {
log.Println("Fetching HN top stories ...")
// API docs: https://github.com/HackerNews/API
url := fmt.Sprintf("%s/topstories.json", HackerNewsFirebaseUrl)
resp, err := http.Get(url)
if err != nil {
err = fmt.Errorf("error fetching HN top stories %w:", err)
return nil, err
}
defer resp.Body.Close()
var ids []int
err = json.NewDecoder(resp.Body).Decode(&ids)
if err != nil {
err = fmt.Errorf("error decoding HN top stories JSON: %w", err)
return nil, err
}
// we are only interested in the first page of top stories
const limit = 30
ids = ids[:limit]
var stories [limit]Story
for i, id := range ids {
story, err := FetchStoryById(id)
if err != nil {
return nil, err
}
stories[i] = story
}
log.Println("Fetching HN top stories ... OK")
// Can't return [30]Story as []Story so we copy the array
return stories[:], nil
}
func FetchStoryById(id int) (Story, error) {
log.Printf("Fetching HN story (id=%d) ...\n", id)
url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
resp, err := http.Get(url)
if err != nil {
err = fmt.Errorf("error fetching HN story (id=%d): %w", id, err)
return Story{}, err
}
defer resp.Body.Close()
var story Story
err = json.NewDecoder(resp.Body).Decode(&story)
if err != nil {
err := fmt.Errorf("error decoding HN story JSON (id=%d): %w", id, err)
return Story{}, err
}
log.Printf("Fetching HN story (id=%d) ... OK\n", id)
return story, nil
}
func ParseHackerNewsLink(link string) (int, error) {
match := HackerNewsLinkRegexp.FindStringSubmatch(link)
if len(match) == 0 {
return -1, errors.New("input is not a hacker news link")
}
id, err := strconv.Atoi(match[1])
if err != nil {
return -1, errors.New("integer conversion to string failed")
}
return id, nil
}
func HackerNewsUserLink(user string) string {
return fmt.Sprintf("%s/user?id=%s", HackerNewsUrl, user)
}
func HackerNewsItemLink(id int) string {
return fmt.Sprintf("%s/item?id=%d", HackerNewsUrl, id)
}

105
hn/hn.go Normal file
View File

@ -0,0 +1,105 @@
package hn
import (
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"regexp"
"strconv"
)
type Item struct {
ID int
By string // username of author
Time int // UNIX timestamp
Descendants int // number of comments
Kids []int
Score int
Title string
Url string
}
var (
hnUrl = "https://news.ycombinator.com"
hnFirebaseUrl = "https://hacker-news.firebaseio.com/v0"
hnLinkRegexp = regexp.MustCompile(`(?:https?:\/\/)?news\.ycombinator\.com\/item\?id=([0-9]+)`)
)
func FetchTopItems() ([]Item, error) {
log.Println("[hn] fetch top items ...")
// API docs: https://github.com/HackerNews/API
url := fmt.Sprintf("%s/topstories.json", hnFirebaseUrl)
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("error fetching HN top stories %w:", err)
}
defer resp.Body.Close()
var ids []int
err = json.NewDecoder(resp.Body).Decode(&ids)
if err != nil {
return nil, fmt.Errorf("error decoding HN top stories JSON: %w", err)
}
// we are only interested in the first page of top stories
const limit = 30
ids = ids[:limit]
var stories [limit]Item
for i, id := range ids {
var item Item
err := FetchItemById(id, &item)
if err != nil {
return nil, err
}
stories[i] = item
}
log.Println("[hn] fetch top items ... OK")
// Can't return [30]Item as []Item so we copy the array
return stories[:], nil
}
func FetchItemById(id int, hnItem *Item) error {
// log.Printf("[hn] fetch HN item %d ...\n", id)
url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
resp, err := http.Get(url)
if err != nil {
err = fmt.Errorf("error fetching HN item %d: %w", id, err)
return err
}
defer resp.Body.Close()
err = json.NewDecoder(resp.Body).Decode(&hnItem)
if err != nil {
err := fmt.Errorf("error decoding JSON for HN item %d: %w", id, err)
return err
}
// log.Printf("[hn] fetch HN item %d ... OK\n", id)
return nil
}
func ParseLink(link string) (int, error) {
match := hnLinkRegexp.FindStringSubmatch(link)
if len(match) == 0 {
return -1, errors.New("not a hacker news link")
}
id, err := strconv.Atoi(match[1])
if err != nil {
return -1, errors.New("integer conversion to string failed")
}
return id, nil
}
func UserLink(user string) string {
return fmt.Sprintf("%s/user?id=%s", hnUrl, user)
}
func ItemLink(id int) string {
return fmt.Sprintf("%s/item?id=%d", hnUrl, id)
}

29
main.go
View File

@ -5,22 +5,24 @@ import (
"log" "log"
"time" "time"
sn "github.com/ekzyis/snappy" "gitlab.com/ekzyis/hnbot/db"
"gitlab.com/ekzyis/hnbot/hn"
sn "gitlab.com/ekzyis/hnbot/sn"
) )
func SyncStories() { func SyncHnItemsToDb() {
for { for {
now := time.Now() now := time.Now()
dur := now.Truncate(time.Minute).Add(time.Minute).Sub(now) dur := now.Truncate(time.Minute).Add(time.Minute).Sub(now)
log.Println("[hn] sleeping for", dur.Round(time.Second)) log.Println("[hn] sleeping for", dur.Round(time.Second))
time.Sleep(dur) time.Sleep(dur)
stories, err := FetchHackerNewsTopStories() stories, err := hn.FetchTopItems()
if err != nil { if err != nil {
log.Println(err) log.Println(err)
continue continue
} }
if err := SaveStories(&stories); err != nil { if err := db.SaveHnItems(&stories); err != nil {
log.Println(err) log.Println(err)
continue continue
} }
@ -28,10 +30,13 @@ func SyncStories() {
} }
func main() { func main() {
go SyncStories() // fetch HN front page every minute in the background and store state in db
go SyncHnItemsToDb()
// check every 15 minutes if there is now a HN item that is worth posting to SN
for { for {
var ( var (
filtered *[]Story filtered *[]hn.Item
err error err error
) )
@ -40,21 +45,21 @@ func main() {
log.Println("[sn] sleeping for", dur.Round(time.Second)) log.Println("[sn] sleeping for", dur.Round(time.Second))
time.Sleep(dur) time.Sleep(dur)
if filtered, err = CurateContentForStackerNews(); err != nil { if filtered, err = sn.CurateContent(); err != nil {
log.Println(err) log.Println(err)
continue continue
} }
for _, story := range *filtered { log.Printf("[sn] found %d item(s) to post\n", len(*filtered))
_, err := PostStoryToStackerNews(&story, PostStoryOptions{SkipDupes: false})
for _, item := range *filtered {
_, err := sn.Post(&item, sn.PostOptions{SkipDupes: false})
if err != nil { if err != nil {
var dupesErr *sn.DupesError var dupesErr *sn.DupesError
if errors.As(err, &dupesErr) { if errors.As(err, &dupesErr) {
// SendDupesErrorToDiscord(story.ID, dupesErr)
log.Println(dupesErr) log.Println(dupesErr)
// save dupe in db to prevent retries
parentId := dupesErr.Dupes[0].Id parentId := dupesErr.Dupes[0].Id
if err := SaveSnItem(parentId, story.ID); err != nil { if err := db.SaveSnItem(parentId, item.ID); err != nil {
log.Println(err) log.Println(err)
} }
continue continue

View File

@ -1,4 +1,4 @@
package main package sn
import ( import (
"database/sql" "database/sql"
@ -8,9 +8,13 @@ import (
"github.com/dustin/go-humanize" "github.com/dustin/go-humanize"
sn "github.com/ekzyis/snappy" sn "github.com/ekzyis/snappy"
"gitlab.com/ekzyis/hnbot/db"
"gitlab.com/ekzyis/hnbot/hn"
) )
func CurateContentForStackerNews() (*[]Story, error) { type DupesError = sn.DupesError
func CurateContent() (*[]hn.Item, error) {
var ( var (
rows *sql.Rows rows *sql.Rows
err error err error
@ -32,33 +36,33 @@ func CurateContentForStackerNews() (*[]Story, error) {
} }
defer rows.Close() defer rows.Close()
var stories []Story var items []hn.Item
for rows.Next() { for rows.Next() {
var story Story var item hn.Item
if err = rows.Scan(&story.ID, &story.Time, &story.Title, &story.Url, &story.By, &story.Score, &story.Descendants); err != nil { if err = rows.Scan(&item.ID, &item.Time, &item.Title, &item.Url, &item.By, &item.Score, &item.Descendants); err != nil {
err = fmt.Errorf("error scanning hn_items: %w", err) err = fmt.Errorf("error scanning hn_items: %w", err)
return nil, err return nil, err
} }
stories = append(stories, story) items = append(items, item)
} }
if err = rows.Err(); err != nil { if err = rows.Err(); err != nil {
err = fmt.Errorf("error iterating hn_items: %w", err) err = fmt.Errorf("error iterating hn_items: %w", err)
return nil, err return nil, err
} }
return &stories, nil return &items, nil
} }
type PostStoryOptions struct { type PostOptions struct {
SkipDupes bool SkipDupes bool
} }
func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error) { func Post(item *hn.Item, options PostOptions) (int, error) {
c := sn.NewClient() c := sn.NewClient()
url := story.Url url := item.Url
if url == "" { if url == "" {
url = HackerNewsItemLink(story.ID) url = hn.ItemLink(item.ID)
} }
log.Printf("Posting to SN (url=%s) ...\n", url) log.Printf("post to SN: %s ...\n", url)
if !options.SkipDupes { if !options.SkipDupes {
dupes, err := c.Dupes(url) dupes, err := c.Dupes(url)
@ -70,18 +74,18 @@ func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error)
} }
} }
title := story.Title title := item.Title
if len(title) > 80 { if len(title) > 80 {
title = title[0:80] title = title[0:80]
} }
comment := fmt.Sprintf( comment := fmt.Sprintf(
"This link was posted by [%s](%s) %s on [HN](%s). It received %d points and %d comments.", "This link was posted by [%s](%s) %s on [HN](%s). It received %d points and %d comments.",
story.By, item.By,
HackerNewsUserLink(story.By), hn.UserLink(item.By),
humanize.Time(time.Unix(int64(story.Time), 0)), humanize.Time(time.Unix(int64(item.Time), 0)),
HackerNewsItemLink(story.ID), hn.ItemLink(item.ID),
story.Score, story.Descendants, item.Score, item.Descendants,
) )
parentId, err := c.PostLink(url, title, comment, "tech") parentId, err := c.PostLink(url, title, comment, "tech")
@ -89,8 +93,8 @@ func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error)
return -1, fmt.Errorf("error posting link: %w", err) return -1, fmt.Errorf("error posting link: %w", err)
} }
log.Printf("Posting to SN (url=%s) ... OK \n", url) log.Printf("post to SN: %s ... OK \n", url)
if err := SaveSnItem(parentId, story.ID); err != nil { if err := db.SaveSnItem(parentId, item.ID); err != nil {
return -1, err return -1, err
} }