Store time series of HN data in SQLite3

This can be used in the future for better content curation and generating charts.

Currently, it still posts stories when they hit rank 1.

.gitignore

@@ -3,3 +3,4 @@
 
 # go executable
 hnbot
+hnbot.sqlite3

db.go

@@ -0,0 +1,90 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+var (
+	db *sql.DB
+)
+
+func init() {
+	var err error
+	db, err = sql.Open("sqlite3", "hnbot.sqlite3")
+	if err != nil {
+		log.Fatal(err)
+	}
+	migrate(db)
+}
+
+func migrate(db *sql.DB) {
+	if _, err := db.Exec(`
+		CREATE TABLE IF NOT EXISTS hn_items (
+			id INTEGER NOT NULL,
+			created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+			time TIMESTAMP WITH TIMEZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+			title TEXT NOT NULL,
+			url TEXT,
+			author TEXT NOT NULL,
+			ndescendants INTEGER NOT NULL,
+			score INTEGER NOT NULL,
+			rank INTEGER NOT NULL,
+			PRIMARY KEY (id, created_at)
+		);
+	`); err != nil {
+		err = fmt.Errorf("error during migration: %w", err)
+		log.Fatal(err)
+	}
+	if _, err := db.Exec(`
+		CREATE TABLE IF NOT EXISTS sn_items (
+			id INTEGER PRIMARY KEY,
+			created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP,
+			hn_id INTEGER NOT NULL REFERENCES hn_items(id)
+		);
+	`); err != nil {
+		err = fmt.Errorf("error during migration: %w", err)
+		log.Fatal(err)
+	}
+}
+
+func ItemHasComment(parentId int) bool {
+	var count int
+	err := db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count)
+	if err != nil {
+		err = fmt.Errorf("error during item check: %w", err)
+		log.Fatal(err)
+	}
+	return count > 0
+}
+
+func SaveStories(story *[]Story) error {
+	for i, s := range *story {
+		if err := SaveStory(&s, i+1); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func SaveStory(s *Story, rank int) error {
+	if _, err := db.Exec(`
+			INSERT INTO hn_items(id, time, title, url, author, ndescendants, score, rank)
+			VALUES (?, ?, ?, ?, ?, ?, ?, ?)`,
+		s.ID, s.Time, s.Title, s.Url, s.By, s.Descendants, s.Score, rank); err != nil {
+		err = fmt.Errorf("error during item insert: %w", err)
+		return err
+	}
+	return nil
+}
+
+func SaveSnItem(id int, hnId int) error {
+	if _, err := db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil {
+		err = fmt.Errorf("error during sn item insert: %w", err)
+		return err
+	}
+	return nil
+}

go.mod

@@ -14,6 +14,7 @@ require (
 	github.com/davecgh/go-spew v1.1.1 // indirect
 	github.com/ekzyis/sn-goapi v0.3.1 // indirect
 	github.com/gorilla/websocket v1.4.2 // indirect
+	github.com/mattn/go-sqlite3 v1.14.22 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
 	golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect
 	golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect

go.sum

@@ -19,6 +19,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U
 github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU=
+github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
 github.com/namsral/flag v1.7.4-pre h1:b2ScHhoCUkbsq0d2C15Mv+VU8bl8hAXV8arnWiOHNZs=
 github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=

main.go

@@ -59,11 +59,22 @@ func main() {
 		stories, err := FetchHackerNewsTopStories()
 		if err != nil {
 			SendErrorToDiscord(err)
-			WaitUntilNextHour()
+			WaitUntilNextMinute()
 			continue
 		}
 
-		filtered := CurateContentForStackerNews(&stories)
+		if err := SaveStories(&stories); err != nil {
+			SendErrorToDiscord(err)
+			WaitUntilNextMinute()
+			continue
+		}
+
+		var filtered *[]Story
+		if filtered, err = CurateContentForStackerNews(); err != nil {
+			SendErrorToDiscord(err)
+			WaitUntilNextMinute()
+			continue
+		}
 
 		for _, story := range *filtered {
 			_, err := PostStoryToStackerNews(&story, PostStoryOptions{SkipDupes: false})
@@ -78,6 +89,6 @@ func main() {
 				continue
 			}
 		}
-		WaitUntilNextHour()
+		WaitUntilNextMinute()
 	}
 }

sn.go

@@ -1,6 +1,7 @@
 package main
 
 import (
+	"database/sql"
 	"fmt"
 	"log"
 	"time"
@@ -10,11 +11,38 @@ import (
 	"github.com/ekzyis/sn-goapi"
 )
 
-func CurateContentForStackerNews(stories *[]Story) *[]Story {
+func CurateContentForStackerNews() (*[]Story, error) {
-	// TODO: filter by relevance
+	var (
+		rows *sql.Rows
+		err  error
+	)
+	if rows, err = db.Query(`
+		SELECT t.id, time, title, url, author, score, ndescendants
+		FROM (
+			SELECT id, MAX(created_at) AS created_at FROM hn_items
+			WHERE rank = 1 AND id NOT IN (SELECT hn_id FROM sn_items)
+			GROUP BY id
+		) t JOIN hn_items ON t.id = hn_items.id AND t.created_at = hn_items.created_at;
+	`); err != nil {
+		err = fmt.Errorf("error querying hn_items: %w", err)
+		return nil, err
+	}
+	defer rows.Close()
 
-	slice := (*stories)[0:1]
+	var stories []Story
-	return &slice
+	for rows.Next() {
+		var story Story
+		if err = rows.Scan(&story.ID, &story.Time, &story.Title, &story.Url, &story.By, &story.Score, &story.Descendants); err != nil {
+			err = fmt.Errorf("error scanning hn_items: %w", err)
+			return nil, err
+		}
+		stories = append(stories, story)
+	}
+	if err = rows.Err(); err != nil {
+		err = fmt.Errorf("error iterating hn_items: %w", err)
+		return nil, err
+	}
+	return &stories, nil
 }
 
 type PostStoryOptions struct {
@@ -49,6 +77,10 @@ func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error)
 	}
 
 	log.Printf("Posting to SN (url=%s) ... OK \n", url)
+	if err := SaveSnItem(parentId, story.ID); err != nil {
+		return -1, err
+	}
+
 	SendStackerNewsEmbedToDiscord(story.Title, parentId)
 
 	comment := fmt.Sprintf(