From d4347238ed1c87fee19e71631e5fc4749b10ce5c Mon Sep 17 00:00:00 2001 From: ekzyis Date: Wed, 23 Aug 2023 04:06:37 +0200 Subject: [PATCH] initial release This is a Stacker News bot that fetches every minute all recent posts. If a post is a twitter link, it adds a comment with nitter links and stores the comment + item id in a sqlite3 database. If the post already has a comment from the bot (by checking the database), it does not post a comment to prevent duplicates. --- .env.template | 1 + .gitignore | 3 ++ README.md | 7 ++++ db.go | 56 ++++++++++++++++++++++++++++++ go.mod | 10 ++++++ go.sum | 14 ++++++++ main.go | 94 +++++++++++++++++++++++++++++++++++++++++++++++++++ nostr.go | 5 +++ 8 files changed, 190 insertions(+) create mode 100644 .env.template create mode 100644 .gitignore create mode 100644 README.md create mode 100644 db.go create mode 100644 go.mod create mode 100644 go.sum create mode 100644 main.go create mode 100644 nostr.go diff --git a/.env.template b/.env.template new file mode 100644 index 0000000..f487789 --- /dev/null +++ b/.env.template @@ -0,0 +1 @@ +SN_AUTH_COOKIE= \ No newline at end of file diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..bc5c84a --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +.env +*.sqlite3 +unpaywall diff --git a/README.md b/README.md new file mode 100644 index 0000000..000d2a2 --- /dev/null +++ b/README.md @@ -0,0 +1,7 @@ +# unpaywall + +This is a [Stacker News bot](https://stacker.news/unpaywall) that fetches every minute all recent posts. + +If a post is a twitter link, it adds a comment with nitter links and stores the comment + item id in a sqlite3 database. + +If the post already has a comment from the bot (by checking the database), it does not post a comment to prevent duplicates. diff --git a/db.go b/db.go new file mode 100644 index 0000000..7cfd8a5 --- /dev/null +++ b/db.go @@ -0,0 +1,56 @@ +package main + +import ( + "database/sql" + "fmt" + "log" + + "github.com/ekzyis/sn-goapi" + _ "github.com/mattn/go-sqlite3" +) + +var ( + db *sql.DB +) + +func init() { + var err error + db, err = sql.Open("sqlite3", "unpaywall.sqlite3") + if err != nil { + log.Fatal(err) + } + migrate(db) +} + +func migrate(db *sql.DB) { + _, err := db.Exec(` + CREATE TABLE IF NOT EXISTS comments ( + id INTEGER PRIMARY KEY, + text TEXT NOT NULL, + parent_id INTEGER NOT NULL, + created_at TIMESTAMP WITH TIME ZONE NOT NULL DEFAULT CURRENT_TIMESTAMP + ) + `) + if err != nil { + err = fmt.Errorf("error during migration: %w", err) + log.Fatal(err) + } +} + +func ItemHasComment(parentId int) bool { + var count int + err := db.QueryRow(`SELECT COUNT(1) FROM comments WHERE parent_id = ?`, parentId).Scan(&count) + if err != nil { + err = fmt.Errorf("error during item check: %w", err) + log.Fatal(err) + } + return count > 0 +} + +func SaveComment(comment *sn.Comment) { + _, err := db.Exec(`INSERT INTO comments(id, text, parent_id) VALUES (?, ?, ?)`, comment.Id, comment.Text, comment.ParentId) + if err != nil { + err = fmt.Errorf("error during item insert: %w", err) + log.Fatal(err) + } +} diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..e1b3bc8 --- /dev/null +++ b/go.mod @@ -0,0 +1,10 @@ +module git.ekzyis.com/ekzyis/unpaywall + +go 1.21.0 + +require ( + github.com/ekzyis/sn-goapi v0.3.0 // indirect + github.com/joho/godotenv v1.5.1 // indirect + github.com/mattn/go-sqlite3 v1.14.17 // indirect + github.com/namsral/flag v1.7.4-pre // indirect +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..79f75ec --- /dev/null +++ b/go.sum @@ -0,0 +1,14 @@ +github.com/ekzyis/sn-goapi v0.2.0 h1:k5YJlG+OguQEQRlbS5KghUttbfJaEkfvIfqB8B133FY= +github.com/ekzyis/sn-goapi v0.2.0/go.mod h1:FObbYr/NXgnXNWU+EwiWKoWQy+wAaRS6AoW3NgsJ/Oo= +github.com/ekzyis/sn-goapi v0.2.1-0.20230823005215-5fb6668b6270 h1:aVDLDhrRsxP+if+ujCOpydFapeei2YL47Cp1ghgbJ2I= +github.com/ekzyis/sn-goapi v0.2.1-0.20230823005215-5fb6668b6270/go.mod h1:FObbYr/NXgnXNWU+EwiWKoWQy+wAaRS6AoW3NgsJ/Oo= +github.com/ekzyis/sn-goapi v0.2.1-0.20230823015232-6a259372b331 h1:NpCmANlADOcRYxKe3GfaYcfra2fjTpW65M36Dq65LX8= +github.com/ekzyis/sn-goapi v0.2.1-0.20230823015232-6a259372b331/go.mod h1:FObbYr/NXgnXNWU+EwiWKoWQy+wAaRS6AoW3NgsJ/Oo= +github.com/ekzyis/sn-goapi v0.3.0 h1:pXRbD2iAMHvUC/+gZKXPRwInP4PuABEC1qQc+DWARlE= +github.com/ekzyis/sn-goapi v0.3.0/go.mod h1:FObbYr/NXgnXNWU+EwiWKoWQy+wAaRS6AoW3NgsJ/Oo= +github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= +github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= +github.com/mattn/go-sqlite3 v1.14.17 h1:mCRHCLDUBXgpKAqIKsaAaAsrAlbkeomtRFKXh2L6YIM= +github.com/mattn/go-sqlite3 v1.14.17/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= +github.com/namsral/flag v1.7.4-pre h1:b2ScHhoCUkbsq0d2C15Mv+VU8bl8hAXV8arnWiOHNZs= +github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo= diff --git a/main.go b/main.go new file mode 100644 index 0000000..b068f69 --- /dev/null +++ b/main.go @@ -0,0 +1,94 @@ +package main + +import ( + "fmt" + "log" + "regexp" + "strings" + "time" + + "github.com/ekzyis/sn-goapi" +) + +var ( + TwitterUrlRegexp = regexp.MustCompile(`^(?:https?:\/\/)?((www\.)?(twitter|x)\.com)\/`) + NitterUrls = []string{"nitter.net", "nitter.it"} +) + +func WaitUntilNext(d time.Duration) { + now := time.Now() + dur := now.Truncate(d).Add(d).Sub(now) + log.Println("sleeping for", dur.Round(time.Second)) + time.Sleep(dur) +} + +func CheckNotifications() { + var prevHasNewNotes bool + for { + log.Println("Checking notifications ...") + hasNewNotes, err := sn.CheckNotifications() + if err != nil { + SendToNostr(fmt.Sprint(err)) + } else { + if !prevHasNewNotes && hasNewNotes { + // only send on "rising edge" + SendToNostr("new notifications") + log.Println("Forwarded notifications to monitoring") + } else if hasNewNotes { + log.Println("Notifications already forwarded") + } + } + prevHasNewNotes = hasNewNotes + WaitUntilNext(time.Hour) + } +} + +func SessionKeepAlive() { + for { + log.Println("Refresh session using GET /api/auth/session ...") + sn.RefreshSession() + WaitUntilNext(time.Hour) + } +} + +func main() { + go CheckNotifications() + go SessionKeepAlive() + for { + log.Println("fetching items ...") + r, err := sn.Items(&sn.ItemsQuery{Sort: "recent", Limit: 21}) + if err != nil { + SendToNostr(fmt.Sprint(err)) + WaitUntilNext(time.Minute) + continue + } + + for _, item := range r.Items { + if m := TwitterUrlRegexp.FindStringSubmatch(item.Url); m != nil { + log.Printf("item %d is twitter link\n", item.Id) + if ItemHasComment(item.Id) { + log.Printf("item %d already has nitter links comment\n", item.Id) + continue + } + comment := "Nitter links:\n\n" + for _, nUrl := range NitterUrls { + nitterLink := strings.Replace(item.Url, m[1], nUrl, 1) + comment += nitterLink + "\n" + } + comment += "\n_Nitter is a free and open source alternative Twitter front-end focused on privacy and performance. " + comment += "Click [here](https://github.com/zedeus/nitter) for more information._" + cId, err := sn.CreateComment(item.Id, comment) + if err != nil { + SendToNostr(fmt.Sprint(err)) + continue + } + log.Printf("created comment %d\n", cId) + SaveComment(&sn.Comment{Id: cId, Text: comment, ParentId: item.Id}) + } else { + log.Printf("item %d is not twitter link\n", item.Id) + } + } + + WaitUntilNext(time.Minute) + } +} diff --git a/nostr.go b/nostr.go new file mode 100644 index 0000000..8d7125a --- /dev/null +++ b/nostr.go @@ -0,0 +1,5 @@ +package main + +func SendToNostr(message string) { + // TODO send message to me on nostr +}