ekzyis/hnbot
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor into individual packages

Browse Source
This commit is contained in:
ekzyis 2025-02-03 22:47:39 +01:00
parent 7641414693
commit 2cfb0eaa31
5 changed files with 161 additions and 149 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

25
db.go → db/db.go
View File

@ -1,4 +1,4 @@
package main
package db
import (
"database/sql"
@ -6,19 +6,24 @@ import (
"log"
_ "github.com/mattn/go-sqlite3"
"gitlab.com/ekzyis/hnbot/hn"
)
var (
db *sql.DB
_db *sql.DB
)
func init() {
var err error
db, err = sql.Open("sqlite3", "hnbot.sqlite3")
_db, err = sql.Open("sqlite3", "hnbot.sqlite3")
if err != nil {
log.Fatal(err)
}
migrate(db)
migrate(_db)
}
func Query(query string, args ...interface{}) (*sql.Rows, error) {
return _db.Query(query, args...)
}
func migrate(db *sql.DB) {
@ -54,7 +59,7 @@ func migrate(db *sql.DB) {
func ItemHasComment(parentId int) bool {
var count int
err := db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count)
err := _db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count)
if err != nil {
err = fmt.Errorf("error during item check: %w", err)
log.Fatal(err)
@ -62,17 +67,17 @@ func ItemHasComment(parentId int) bool {
return count > 0
}
func SaveStories(story *[]Story) error {
func SaveHnItems(story *[]hn.Item) error {
for i, s := range *story {
if err := SaveStory(&s, i+1); err != nil {
if err := SaveHnItem(&s, i+1); err != nil {
return err
}
}
return nil
}
func SaveStory(s *Story, rank int) error {
if _, err := db.Exec(`
func SaveHnItem(s *hn.Item, rank int) error {
if _, err := _db.Exec(`
INSERT INTO hn_items(id, time, title, url, author, ndescendants, score, rank)
VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
s.ID, s.Time, s.Title, s.Url, s.By, s.Descendants, s.Score, rank); err != nil {
@ -83,7 +88,7 @@ func SaveStory(s *Story, rank int) error {
}
func SaveSnItem(id int, hnId int) error {
if _, err := db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil {
if _, err := _db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil {
err = fmt.Errorf("error during sn item insert: %w", err)
return err
}

107
hn.go
View File

@ -1,107 +0,0 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"regexp"
"strconv"
)
type Story struct {
ID int
By string // username of author
Time int // UNIX timestamp
Descendants int // number of comments
Kids []int
Score int
Title string
Url string
}
var (
HackerNewsUrl = "https://news.ycombinator.com"
HackerNewsFirebaseUrl = "https://hacker-news.firebaseio.com/v0"
HackerNewsLinkRegexp = regexp.MustCompile(`(?:https?:\/\/)?news\.ycombinator\.com\/item\?id=([0-9]+)`)
)
func FetchHackerNewsTopStories() ([]Story, error) {
log.Println("Fetching HN top stories ...")
// API docs: https://github.com/HackerNews/API
url := fmt.Sprintf("%s/topstories.json", HackerNewsFirebaseUrl)
resp, err := http.Get(url)
if err != nil {
err = fmt.Errorf("error fetching HN top stories %w:", err)
return nil, err
}
defer resp.Body.Close()
var ids []int
err = json.NewDecoder(resp.Body).Decode(&ids)
if err != nil {
err = fmt.Errorf("error decoding HN top stories JSON: %w", err)
return nil, err
}
// we are only interested in the first page of top stories
const limit = 30
ids = ids[:limit]
var stories [limit]Story
for i, id := range ids {
story, err := FetchStoryById(id)
if err != nil {
return nil, err
}
stories[i] = story
}
log.Println("Fetching HN top stories ... OK")
// Can't return [30]Story as []Story so we copy the array
return stories[:], nil
}
func FetchStoryById(id int) (Story, error) {
log.Printf("Fetching HN story (id=%d) ...\n", id)
url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
resp, err := http.Get(url)
if err != nil {
err = fmt.Errorf("error fetching HN story (id=%d): %w", id, err)
return Story{}, err
}
defer resp.Body.Close()
var story Story
err = json.NewDecoder(resp.Body).Decode(&story)
if err != nil {
err := fmt.Errorf("error decoding HN story JSON (id=%d): %w", id, err)
return Story{}, err
}
log.Printf("Fetching HN story (id=%d) ... OK\n", id)
return story, nil
}
func ParseHackerNewsLink(link string) (int, error) {
match := HackerNewsLinkRegexp.FindStringSubmatch(link)
if len(match) == 0 {
return -1, errors.New("input is not a hacker news link")
}
id, err := strconv.Atoi(match[1])
if err != nil {
return -1, errors.New("integer conversion to string failed")
}
return id, nil
}
func HackerNewsUserLink(user string) string {
return fmt.Sprintf("%s/user?id=%s", HackerNewsUrl, user)
}
func HackerNewsItemLink(id int) string {
return fmt.Sprintf("%s/item?id=%d", HackerNewsUrl, id)
}

105
hn/hn.go Normal file
View File

@ -0,0 +1,105 @@
package hn
import (
"encoding/json"
"errors"
"fmt"
"log"
"net/http"
"regexp"
"strconv"
)
type Item struct {
ID int
By string // username of author
Time int // UNIX timestamp
Descendants int // number of comments
Kids []int
Score int
Title string
Url string
}
var (
hnUrl = "https://news.ycombinator.com"
hnFirebaseUrl = "https://hacker-news.firebaseio.com/v0"
hnLinkRegexp = regexp.MustCompile(`(?:https?:\/\/)?news\.ycombinator\.com\/item\?id=([0-9]+)`)
)
func FetchTopItems() ([]Item, error) {
log.Println("[hn] fetch top items ...")
// API docs: https://github.com/HackerNews/API
url := fmt.Sprintf("%s/topstories.json", hnFirebaseUrl)
resp, err := http.Get(url)
if err != nil {
return nil, fmt.Errorf("error fetching HN top stories %w:", err)
}
defer resp.Body.Close()
var ids []int
err = json.NewDecoder(resp.Body).Decode(&ids)
if err != nil {
return nil, fmt.Errorf("error decoding HN top stories JSON: %w", err)
}
// we are only interested in the first page of top stories
const limit = 30
ids = ids[:limit]
var stories [limit]Item
for i, id := range ids {
var item Item
err := FetchItemById(id, &item)
if err != nil {
return nil, err
}
stories[i] = item
}
log.Println("[hn] fetch top items ... OK")
// Can't return [30]Item as []Item so we copy the array
return stories[:], nil
}
func FetchItemById(id int, hnItem *Item) error {
// log.Printf("[hn] fetch HN item %d ...\n", id)
url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
resp, err := http.Get(url)
if err != nil {
err = fmt.Errorf("error fetching HN item %d: %w", id, err)
return err
}
defer resp.Body.Close()
err = json.NewDecoder(resp.Body).Decode(&hnItem)
if err != nil {
err := fmt.Errorf("error decoding JSON for HN item %d: %w", id, err)
return err
}
// log.Printf("[hn] fetch HN item %d ... OK\n", id)
return nil
}
func ParseLink(link string) (int, error) {
match := hnLinkRegexp.FindStringSubmatch(link)
if len(match) == 0 {
return -1, errors.New("not a hacker news link")
}
id, err := strconv.Atoi(match[1])
if err != nil {
return -1, errors.New("integer conversion to string failed")
}
return id, nil
}
func UserLink(user string) string {
return fmt.Sprintf("%s/user?id=%s", hnUrl, user)
}
func ItemLink(id int) string {
return fmt.Sprintf("%s/item?id=%d", hnUrl, id)
}

29
main.go
View File

@ -5,22 +5,24 @@ import (
"log"
"time"
sn "github.com/ekzyis/snappy"
"gitlab.com/ekzyis/hnbot/db"
"gitlab.com/ekzyis/hnbot/hn"
sn "gitlab.com/ekzyis/hnbot/sn"
)
func SyncStories() {
func SyncHnItemsToDb() {
for {
now := time.Now()
dur := now.Truncate(time.Minute).Add(time.Minute).Sub(now)
log.Println("[hn] sleeping for", dur.Round(time.Second))
time.Sleep(dur)
stories, err := FetchHackerNewsTopStories()
stories, err := hn.FetchTopItems()
if err != nil {
log.Println(err)
continue
}
if err := SaveStories(&stories); err != nil {
if err := db.SaveHnItems(&stories); err != nil {
log.Println(err)
continue
}
@ -28,10 +30,13 @@ func SyncStories() {
}
func main() {
go SyncStories()
// fetch HN front page every minute in the background and store state in db
go SyncHnItemsToDb()
// check every 15 minutes if there is now a HN item that is worth posting to SN
for {
var (
filtered *[]Story
filtered *[]hn.Item
err error
)
@ -40,21 +45,21 @@ func main() {
log.Println("[sn] sleeping for", dur.Round(time.Second))
time.Sleep(dur)
if filtered, err = CurateContentForStackerNews(); err != nil {
if filtered, err = sn.CurateContent(); err != nil {
log.Println(err)
continue
}
for _, story := range *filtered {
_, err := PostStoryToStackerNews(&story, PostStoryOptions{SkipDupes: false})
log.Printf("[sn] found %d item(s) to post\n", len(*filtered))
for _, item := range *filtered {
_, err := sn.Post(&item, sn.PostOptions{SkipDupes: false})
if err != nil {
var dupesErr *sn.DupesError
if errors.As(err, &dupesErr) {
// SendDupesErrorToDiscord(story.ID, dupesErr)
log.Println(dupesErr)
// save dupe in db to prevent retries
parentId := dupesErr.Dupes[0].Id
if err := SaveSnItem(parentId, story.ID); err != nil {
if err := db.SaveSnItem(parentId, item.ID); err != nil {
log.Println(err)
}
continue

44
sn.go → sn/sn.go
View File

@ -1,4 +1,4 @@
package main
package sn
import (
"database/sql"
@ -8,9 +8,13 @@ import (
"github.com/dustin/go-humanize"
sn "github.com/ekzyis/snappy"
"gitlab.com/ekzyis/hnbot/db"
"gitlab.com/ekzyis/hnbot/hn"
)
func CurateContentForStackerNews() (*[]Story, error) {
type DupesError = sn.DupesError
func CurateContent() (*[]hn.Item, error) {
var (
rows *sql.Rows
err error
@ -32,33 +36,33 @@ func CurateContentForStackerNews() (*[]Story, error) {
}
defer rows.Close()
var stories []Story
var items []hn.Item
for rows.Next() {
var story Story
if err = rows.Scan(&story.ID, &story.Time, &story.Title, &story.Url, &story.By, &story.Score, &story.Descendants); err != nil {
var item hn.Item
if err = rows.Scan(&item.ID, &item.Time, &item.Title, &item.Url, &item.By, &item.Score, &item.Descendants); err != nil {
err = fmt.Errorf("error scanning hn_items: %w", err)
return nil, err
}
stories = append(stories, story)
items = append(items, item)
}
if err = rows.Err(); err != nil {
err = fmt.Errorf("error iterating hn_items: %w", err)
return nil, err
}
return &stories, nil
return &items, nil
}
type PostStoryOptions struct {
type PostOptions struct {
SkipDupes bool
}
func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error) {
func Post(item *hn.Item, options PostOptions) (int, error) {
c := sn.NewClient()
url := story.Url
url := item.Url
if url == "" {
url = HackerNewsItemLink(story.ID)
url = hn.ItemLink(item.ID)
}
log.Printf("Posting to SN (url=%s) ...\n", url)
log.Printf("post to SN: %s ...\n", url)
if !options.SkipDupes {
dupes, err := c.Dupes(url)
@ -70,18 +74,18 @@ func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error)
}
}
title := story.Title
title := item.Title
if len(title) > 80 {
title = title[0:80]
}
comment := fmt.Sprintf(
"This link was posted by [%s](%s) %s on [HN](%s). It received %d points and %d comments.",
story.By,
HackerNewsUserLink(story.By),
humanize.Time(time.Unix(int64(story.Time), 0)),
HackerNewsItemLink(story.ID),
story.Score, story.Descendants,
item.By,
hn.UserLink(item.By),
humanize.Time(time.Unix(int64(item.Time), 0)),
hn.ItemLink(item.ID),
item.Score, item.Descendants,
)
parentId, err := c.PostLink(url, title, comment, "tech")
@ -89,8 +93,8 @@ func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error)
return -1, fmt.Errorf("error posting link: %w", err)
}
log.Printf("Posting to SN (url=%s) ... OK \n", url)
if err := SaveSnItem(parentId, story.ID); err != nil {
log.Printf("post to SN: %s ... OK \n", url)
if err := db.SaveSnItem(parentId, item.ID); err != nil {
return -1, err
}