From 7e4744503fcf394db4aeca365a9f4831ddbf6aa3 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Wed, 13 Mar 2024 13:13:11 +0100 Subject: [PATCH] Store time series of HN data in SQLite3 This can be used in the future for better content curation and generating charts. Currently, it still posts stories when they hit rank 1. --- .gitignore | 1 + db.go | 90 ++++++++++++++++++++++++++++++++++++++++++++++++++++++ go.mod | 1 + go.sum | 2 ++ main.go | 17 +++++++++-- sn.go | 40 +++++++++++++++++++++--- 6 files changed, 144 insertions(+), 7 deletions(-) create mode 100644 db.go diff --git a/.gitignore b/.gitignore index 5d9a769..e261421 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ # go executable hnbot +hnbot.sqlite3 \ No newline at end of file diff --git a/db.go b/db.go new file mode 100644 index 0000000..ac79081 --- /dev/null +++ b/db.go @@ -0,0 +1,90 @@ +package main + +import ( + "database/sql" + "fmt" + "log" + + _ "github.com/mattn/go-sqlite3" +) + +var ( + db *sql.DB +) + +func init() { + var err error + db, err = sql.Open("sqlite3", "hnbot.sqlite3") + if err != nil { + log.Fatal(err) + } + migrate(db) +} + +func migrate(db *sql.DB) { + if _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS hn_items ( + id INTEGER NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + time TIMESTAMP WITH TIMEZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + title TEXT NOT NULL, + url TEXT, + author TEXT NOT NULL, + ndescendants INTEGER NOT NULL, + score INTEGER NOT NULL, + rank INTEGER NOT NULL, + PRIMARY KEY (id, created_at) + ); + `); err != nil { + err = fmt.Errorf("error during migration: %w", err) + log.Fatal(err) + } + if _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS sn_items ( + id INTEGER PRIMARY KEY, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP, + hn_id INTEGER NOT NULL REFERENCES hn_items(id) + ); + `); err != nil { + err = fmt.Errorf("error during migration: %w", err) + log.Fatal(err) + } +} + +func ItemHasComment(parentId int) bool { + var count int + err := db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count) + if err != nil { + err = fmt.Errorf("error during item check: %w", err) + log.Fatal(err) + } + return count > 0 +} + +func SaveStories(story *[]Story) error { + for i, s := range *story { + if err := SaveStory(&s, i+1); err != nil { + return err + } + } + return nil +} + +func SaveStory(s *Story, rank int) error { + if _, err := db.Exec(` + INSERT INTO hn_items(id, time, title, url, author, ndescendants, score, rank) + VALUES (?, ?, ?, ?, ?, ?, ?, ?)`, + s.ID, s.Time, s.Title, s.Url, s.By, s.Descendants, s.Score, rank); err != nil { + err = fmt.Errorf("error during item insert: %w", err) + return err + } + return nil +} + +func SaveSnItem(id int, hnId int) error { + if _, err := db.Exec(`INSERT INTO sn_items(id, hn_id) VALUES (?, ?)`, id, hnId); err != nil { + err = fmt.Errorf("error during sn item insert: %w", err) + return err + } + return nil +} diff --git a/go.mod b/go.mod index 6057165..c01de62 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/ekzyis/sn-goapi v0.3.1 // indirect github.com/gorilla/websocket v1.4.2 // indirect + github.com/mattn/go-sqlite3 v1.14.22 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b // indirect golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect diff --git a/go.sum b/go.sum index 2f2ba2d..24a72cb 100644 --- a/go.sum +++ b/go.sum @@ -19,6 +19,8 @@ github.com/gorilla/websocket v1.4.2 h1:+/TMaTYc4QFitKJxsQ7Yye35DkWvkdLcvGKqM+x0U github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/mattn/go-sqlite3 v1.14.22 h1:2gZY6PC6kBnID23Tichd1K+Z0oS6nE/XwU+Vz/5o4kU= +github.com/mattn/go-sqlite3 v1.14.22/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y= github.com/namsral/flag v1.7.4-pre h1:b2ScHhoCUkbsq0d2C15Mv+VU8bl8hAXV8arnWiOHNZs= github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= diff --git a/main.go b/main.go index 2aab953..18e5033 100644 --- a/main.go +++ b/main.go @@ -59,11 +59,22 @@ func main() { stories, err := FetchHackerNewsTopStories() if err != nil { SendErrorToDiscord(err) - WaitUntilNextHour() + WaitUntilNextMinute() continue } - filtered := CurateContentForStackerNews(&stories) + if err := SaveStories(&stories); err != nil { + SendErrorToDiscord(err) + WaitUntilNextMinute() + continue + } + + var filtered *[]Story + if filtered, err = CurateContentForStackerNews(); err != nil { + SendErrorToDiscord(err) + WaitUntilNextMinute() + continue + } for _, story := range *filtered { _, err := PostStoryToStackerNews(&story, PostStoryOptions{SkipDupes: false}) @@ -78,6 +89,6 @@ func main() { continue } } - WaitUntilNextHour() + WaitUntilNextMinute() } } diff --git a/sn.go b/sn.go index b397ed9..9a48ad1 100644 --- a/sn.go +++ b/sn.go @@ -1,6 +1,7 @@ package main import ( + "database/sql" "fmt" "log" "time" @@ -10,11 +11,38 @@ import ( "github.com/ekzyis/sn-goapi" ) -func CurateContentForStackerNews(stories *[]Story) *[]Story { - // TODO: filter by relevance +func CurateContentForStackerNews() (*[]Story, error) { + var ( + rows *sql.Rows + err error + ) + if rows, err = db.Query(` + SELECT t.id, time, title, url, author, score, ndescendants + FROM ( + SELECT id, MAX(created_at) AS created_at FROM hn_items + WHERE rank = 1 AND id NOT IN (SELECT hn_id FROM sn_items) + GROUP BY id + ) t JOIN hn_items ON t.id = hn_items.id AND t.created_at = hn_items.created_at; + `); err != nil { + err = fmt.Errorf("error querying hn_items: %w", err) + return nil, err + } + defer rows.Close() - slice := (*stories)[0:1] - return &slice + var stories []Story + for rows.Next() { + var story Story + if err = rows.Scan(&story.ID, &story.Time, &story.Title, &story.Url, &story.By, &story.Score, &story.Descendants); err != nil { + err = fmt.Errorf("error scanning hn_items: %w", err) + return nil, err + } + stories = append(stories, story) + } + if err = rows.Err(); err != nil { + err = fmt.Errorf("error iterating hn_items: %w", err) + return nil, err + } + return &stories, nil } type PostStoryOptions struct { @@ -49,6 +77,10 @@ func PostStoryToStackerNews(story *Story, options PostStoryOptions) (int, error) } log.Printf("Posting to SN (url=%s) ... OK \n", url) + if err := SaveSnItem(parentId, story.ID); err != nil { + return -1, err + } + SendStackerNewsEmbedToDiscord(story.Title, parentId) comment := fmt.Sprintf(