db.go

@@ -1,29 +1,24 @@
-package db
+package main
 
 import (
 	"database/sql"
 	"fmt"
 	"log"
 
-	"github.com/ekzyis/hnbot/hn"
 	_ "github.com/mattn/go-sqlite3"
 )
 
 var (
-	_db *sql.DB
+	db *sql.DB
 )
 
 func init() {
 	var err error
-	_db, err = sql.Open("sqlite3", "hnbot.sqlite3")
+	db, err = sql.Open("sqlite3", "hnbot.sqlite3")
 	if err != nil {
 		log.Fatal(err)
 	}
-	migrate(_db)
+	migrate(db)
-}
-
-func Query(query string, args ...interface{}) (*sql.Rows, error) {
-	return _db.Query(query, args...)
 }
 
 func migrate(db *sql.DB) {
@@ -59,7 +54,7 @@ func migrate(db *sql.DB) {
 
 func ItemHasComment(parentId int) bool {
 	var count int
-	err := _db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count)
+	err := db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count)
 	if err != nil {
 		err = fmt.Errorf("error during item check: %w", err)
 		log.Fatal(err)
@@ -67,17 +62,17 @@ func ItemHasComment(parentId int) bool {
 	return count > 0
 }
 
-func SaveHnItems(story *[]hn.Item) error {
+func SaveStories(story *[]Story) error {
 	for i, s := range *story {
-		if err := SaveHnItem(&s, i+1); err != nil {
+		if err := SaveStory(&s, i+1); err != nil {
 			return err
 		}
 	}
 	return nil
 }
 
-func SaveHnItem(s *hn.Item, rank int) error {
+func SaveStory(s *Story, rank int) error {
-	if _, err := _db.Exec(`
+	if _, err := db.Exec(`
 			INSERT INTO hn_items(id, time, title, url, author, ndescendants, score, rank)
 			VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
 		s.ID, s.Time, s.Title, s.Url, s.By, s.Descendants, s.Score, rank); err != nil {
@@ -88,7 +83,7 @@ func SaveHnItem(s *hn.Item, rank int) error {
 }
 
 func SaveSnItem(id int, hnId int) error {
-	if _, err := _db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil {
+	if _, err := db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil {
 		err = fmt.Errorf("error during sn item insert: %w", err)
 		return err
 	}

go.mod

@@ -1,12 +1,9 @@
-module github.com/ekzyis/hnbot
+module gitlab.com/ekzyis/hnbot
 
 go 1.20
 
 require (
 	github.com/dustin/go-humanize v1.0.1
-	github.com/ekzyis/snappy v0.8.0
+	github.com/ekzyis/snappy v0.4.2
-	github.com/joho/godotenv v1.5.1
 	github.com/mattn/go-sqlite3 v1.14.22
 )
-
-require gopkg.in/guregu/null.v4 v4.0.0 // indirect

go.sum

@@ -1,10 +1,6 @@
 github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
-github.com/ekzyis/snappy v0.8.0 h1:e7dRR384XJgNYa1FWNIZmqITSHOSanteBFXQJPfcQwg=
+github.com/ekzyis/snappy v0.4.2 h1:5Tw13OIm+w4jTmO3O4c7HzyoiAhbxKl1WzqSoi6x8GQ=
-github.com/ekzyis/snappy v0.8.0/go.mod h1:UksYI0dU0+cnzz0LQjWB1P0QQP/ghx47e4atP99a5Lk=
+github.com/ekzyis/snappy v0.4.2/go.mod h1:BxJwdGlCwUw0Q5pQzBr59weAIS6pkVdivBBaZkkWTSo=
-github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
-github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
 github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
 github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
-gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg=
-gopkg.in/guregu/null.v4 v4.0.0/go.mod h1:YoQhUrADuG3i9WqesrCmpNRwm1ypAgSHYqoOcTu/JrI=

hn.go

@@ -0,0 +1,107 @@
+package main
+
+import (
+	"encoding/json"
+	"errors"
+	"fmt"
+	"log"
+	"net/http"
+	"regexp"
+	"strconv"
+)
+
+type Story struct {
+	ID          int
+	By          string // username of author
+	Time        int    // UNIX timestamp
+	Descendants int    // number of comments
+	Kids        []int
+	Score       int
+	Title       string
+	Url         string
+}
+
+var (
+	HackerNewsUrl         = "https://news.ycombinator.com"
+	HackerNewsFirebaseUrl = "https://hacker-news.firebaseio.com/v0"
+	HackerNewsLinkRegexp  = regexp.MustCompile(`(?:https?:\/\/)?news\.ycombinator\.com\/item\?id=([0-9]+)`)
+)
+
+func FetchHackerNewsTopStories() ([]Story, error) {
+	log.Println("Fetching HN top stories ...")
+
+	// API docs: https://github.com/HackerNews/API
+	url := fmt.Sprintf("%s/topstories.json", HackerNewsFirebaseUrl)
+	resp, err := http.Get(url)
+	if err != nil {
+		err = fmt.Errorf("error fetching HN top stories %w:", err)
+		return nil, err
+	}
+	defer resp.Body.Close()
+
+	var ids []int
+	err = json.NewDecoder(resp.Body).Decode(&ids)
+	if err != nil {
+		err = fmt.Errorf("error decoding HN top stories JSON: %w", err)
+		return nil, err
+	}
+
+	// we are only interested in the first page of top stories
+	const limit = 30
+	ids = ids[:limit]
+
+	var stories [limit]Story
+	for i, id := range ids {
+		story, err := FetchStoryById(id)
+		if err != nil {
+			return nil, err
+		}
+		stories[i] = story
+	}
+
+	log.Println("Fetching HN top stories ... OK")
+	// Can't return [30]Story as []Story so we copy the array
+	return stories[:], nil
+}
+
+func FetchStoryById(id int) (Story, error) {
+	log.Printf("Fetching HN story (id=%d) ...\n", id)
+
+	url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
+	resp, err := http.Get(url)
+	if err != nil {
+		err = fmt.Errorf("error fetching HN story (id=%d): %w", id, err)
+		return Story{}, err
+	}
+	defer resp.Body.Close()
+
+	var story Story
+	err = json.NewDecoder(resp.Body).Decode(&story)
+	if err != nil {
+		err := fmt.Errorf("error decoding HN story JSON (id=%d): %w", id, err)
+		return Story{}, err
+	}
+
+	log.Printf("Fetching HN story (id=%d) ... OK\n", id)
+	return story, nil
+}
+
+func ParseHackerNewsLink(link string) (int, error) {
+	match := HackerNewsLinkRegexp.FindStringSubmatch(link)
+	if len(match) == 0 {
+		return -1, errors.New("input is not a hacker news link")
+	}
+	id, err := strconv.Atoi(match[1])
+	if err != nil {
+		return -1, errors.New("integer conversion to string failed")
+	}
+	return id, nil
+}
+
+func HackerNewsUserLink(user string) string {
+	return fmt.Sprintf("%s/user?id=%s", HackerNewsUrl, user)
+}
+
+func HackerNewsItemLink(id int) string {
+	return fmt.Sprintf("%s/item?id=%d", HackerNewsUrl, id)
+}

hn/hn.go

@@ -1,105 +0,0 @@
-package hn
-
-import (
-	"encoding/json"
-	"errors"
-	"fmt"
-	"log"
-	"net/http"
-	"regexp"
-	"strconv"
-)
-
-type Item struct {
-	ID          int
-	By          string // username of author
-	Time        int    // UNIX timestamp
-	Descendants int    // number of comments
-	Kids        []int
-	Score       int
-	Title       string
-	Url         string
-}
-
-var (
-	hnUrl         = "https://news.ycombinator.com"
-	hnFirebaseUrl = "https://hacker-news.firebaseio.com/v0"
-	hnLinkRegexp  = regexp.MustCompile(`(?:https?:\/\/)?news\.ycombinator\.com\/item\?id=([0-9]+)`)
-)
-
-func FetchTopItems() ([]Item, error) {
-	log.Println("[hn] fetch top items ...")
-
-	// API docs: https://github.com/HackerNews/API
-	url := fmt.Sprintf("%s/topstories.json", hnFirebaseUrl)
-	resp, err := http.Get(url)
-	if err != nil {
-		return nil, fmt.Errorf("error fetching HN top stories %w:", err)
-	}
-	defer resp.Body.Close()
-
-	var ids []int
-	err = json.NewDecoder(resp.Body).Decode(&ids)
-	if err != nil {
-		return nil, fmt.Errorf("error decoding HN top stories JSON: %w", err)
-	}
-
-	// we are only interested in the first page of top stories
-	const limit = 30
-	ids = ids[:limit]
-
-	var stories [limit]Item
-	for i, id := range ids {
-		var item Item
-		err := FetchItemById(id, &item)
-		if err != nil {
-			return nil, err
-		}
-		stories[i] = item
-	}
-
-	log.Println("[hn] fetch top items ... OK")
-	// Can't return [30]Item as []Item so we copy the array
-	return stories[:], nil
-}
-
-func FetchItemById(id int, hnItem *Item) error {
-	// log.Printf("[hn] fetch HN item %d ...\n", id)
-
-	url := fmt.Sprintf("https://hacker-news.firebaseio.com/v0/item/%d.json", id)
-	resp, err := http.Get(url)
-	if err != nil {
-		err = fmt.Errorf("error fetching HN item %d: %w", id, err)
-		return err
-	}
-	defer resp.Body.Close()
-
-	err = json.NewDecoder(resp.Body).Decode(&hnItem)
-	if err != nil {
-		err := fmt.Errorf("error decoding JSON for HN item %d: %w", id, err)
-		return err
-	}
-
-	// log.Printf("[hn] fetch HN item %d ... OK\n", id)
-	return nil
-}
-
-func ParseLink(link string) (int, error) {
-	match := hnLinkRegexp.FindStringSubmatch(link)
-	if len(match) == 0 {
-		return -1, errors.New("not a hacker news link")
-	}
-	id, err := strconv.Atoi(match[1])
-	if err != nil {
-		return -1, errors.New("integer conversion to string failed")
-	}
-	return id, nil
-}
-
-func UserLink(user string) string {
-	return fmt.Sprintf("%s/user?id=%s", hnUrl, user)
-}
-
-func ItemLink(id int) string {
-	return fmt.Sprintf("%s/item?id=%d", hnUrl, id)
-}

main.go

@@ -5,25 +5,22 @@ import (
 	"log"
 	"time"
 
-	"github.com/ekzyis/hnbot/db"
+	sn "github.com/ekzyis/snappy"
-	"github.com/ekzyis/hnbot/hn"
-	sn "github.com/ekzyis/hnbot/sn"
-	"github.com/joho/godotenv"
 )
 
-func SyncHnItemsToDb() {
+func SyncStories() {
 	for {
 		now := time.Now()
 		dur := now.Truncate(time.Minute).Add(time.Minute).Sub(now)
 		log.Println("[hn] sleeping for", dur.Round(time.Second))
 		time.Sleep(dur)
 
-		stories, err := hn.FetchTopItems()
+		stories, err := FetchHackerNewsTopStories()
 		if err != nil {
 			log.Println(err)
 			continue
 		}
-		if err := db.SaveHnItems(&stories); err != nil {
+		if err := SaveStories(&stories); err != nil {
 			log.Println(err)
 			continue
 		}
@@ -31,17 +28,10 @@ func SyncHnItemsToDb() {
 }
 
 func main() {
-	if err := godotenv.Load(); err != nil {
+	go SyncStories()
-		log.Fatal(err)
-	}
-
-	// fetch HN front page every minute in the background and store state in db
-	go SyncHnItemsToDb()
-
-	// check every 15 minutes if there is now a HN item that is worth posting to SN
 	for {
 		var (
-			filtered *[]hn.Item
+			filtered *[]Story
 			err      error
 		)
 
@@ -50,21 +40,21 @@ func main() {
 		log.Println("[sn] sleeping for", dur.Round(time.Second))
 		time.Sleep(dur)
 
-		if filtered, err = sn.CurateContent(); err != nil {
+		if filtered, err = CurateContentForStackerNews(); err != nil {
 			log.Println(err)
 			continue
 		}
 
-		log.Printf("[sn] found %d item(s) to post\n", len(*filtered))
+		for _, story := range *filtered {
-
+			_, err := PostStoryToStackerNews(&story, PostStoryOptions{SkipDupes: false})
-		for _, item := range *filtered {
-			_, err := sn.Post(&item, sn.PostOptions{SkipDupes: false})
 			if err != nil {
 				var dupesErr *sn.DupesError
 				if errors.As(err, &dupesErr) {
+					// SendDupesErrorToDiscord(story.ID, dupesErr)
 					log.Println(dupesErr)
+					// save dupe in db to prevent retries
 					parentId := dupesErr.Dupes[0].Id
-					if err := db.SaveSnItem(parentId, item.ID); err != nil {
+					if err := SaveSnItem(parentId, story.ID); err != nil {
 						log.Println(err)
 					}
 					continue

sn.go

@@ -1,4 +1,4 @@
-package sn
+package main
 
 import (
 	"database/sql"
@@ -7,14 +7,10 @@ import (
 	"time"
 
 	"github.com/dustin/go-humanize"
-	"github.com/ekzyis/hnbot/db"
-	"github.com/ekzyis/hnbot/hn"
 	sn "github.com/ekzyis/snappy"
 )
 
-type DupesError = sn.DupesError
+func CurateContentForStackerNews() (*[]Story, error) {
-
-func CurateContent() (*[]hn.Item, error) {
 	var (
 		rows *sql.Rows
 		err  error
@@ -36,33 +32,33 @@ func CurateContent() (*[]hn.Item, error) {
 	}
 	defer rows.Close()
 
-	var items []hn.Item
+	var stories []Story
 	for rows.Next() {
-		var item hn.Item
+		var story Story
-		if err = rows.Scan(&item.ID, &item.Time, &item.Title, &item.Url, &item.By, &item.Score, &item.Descendants); err != nil {
+		if err = rows.Scan(&story.ID, &story.Time, &story.Title, &story.Url, &story.By, &story.Score, &story.Descendants); err != nil {
 			err = fmt.Errorf("error scanning hn_items: %w", err)
 			return nil, err
 		}
-		items = append(items, item)
+		stories = append(stories, story)
 	}
 	if err = rows.Err(); err != nil {
 		err = fmt.Errorf("error iterating hn_items: %w", err)
 		return nil, err
 	}
-	return &items, nil
+	return &stories, nil
 }
 
-type PostOptions struct {
+type PostStoryOptions struct {
 	SkipDupes bool
 }
 
-func Post(item *hn.Item, options PostOptions) (int, error) {
+func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error) {
 	c := sn.NewClient()
-	url := item.Url
+	url := story.Url
 	if url == "" {
-		url = hn.ItemLink(item.ID)
+		url = HackerNewsItemLink(story.ID)
 	}
-	log.Printf("post to SN: %s ...\n", url)
+	log.Printf("Posting to SN (url=%s) ...\n", url)
 
 	if !options.SkipDupes {
 		dupes, err := c.Dupes(url)
@@ -74,18 +70,18 @@ func Post(item *hn.Item, options PostOptions) (int, error) {
 		}
 	}
 
-	title := item.Title
+	title := story.Title
 	if len(title) > 80 {
 		title = title[0:80]
 	}
 
 	comment := fmt.Sprintf(
 		"This link was posted by [%s](%s) %s on [HN](%s). It received %d points and %d comments.",
-		item.By,
+		story.By,
-		hn.UserLink(item.By),
+		HackerNewsUserLink(story.By),
-		humanize.Time(time.Unix(int64(item.Time), 0)),
+		humanize.Time(time.Unix(int64(story.Time), 0)),
-		hn.ItemLink(item.ID),
+		HackerNewsItemLink(story.ID),
-		item.Score, item.Descendants,
+		story.Score, story.Descendants,
 	)
 
 	parentId, err := c.PostLink(url, title, comment, "tech")
@@ -93,8 +89,8 @@ func Post(item *hn.Item, options PostOptions) (int, error) {
 		return -1, fmt.Errorf("error posting link: %w", err)
 	}
 
-	log.Printf("post to SN: %s ... OK \n", url)
+	log.Printf("Posting to SN (url=%s) ... OK \n", url)
-	if err := db.SaveSnItem(parentId, item.ID); err != nil {
+	if err := SaveSnItem(parentId, story.ID); err != nil {
 		return -1, err
 	}