Use sqlite3 to fix duplicate TG posts

.gitignore

@@ -1,2 +1,3 @@
 .env
 sn-rss2tg
+sn-rss2tg.sqlite3

db.go

@@ -0,0 +1,69 @@
+package main
+
+import (
+	"database/sql"
+	"fmt"
+	"log"
+	"strconv"
+	"strings"
+
+	_ "github.com/mattn/go-sqlite3"
+)
+
+var (
+	db *sql.DB
+)
+
+func init() {
+	var err error
+	db, err = sql.Open("sqlite3", "sn-rss2tg.sqlite3")
+	if err != nil {
+		log.Fatal(err)
+	}
+	migrate(db)
+}
+
+func migrate(db *sql.DB) {
+	_, err := db.Exec(`
+		CREATE TABLE IF NOT EXISTS items (
+			id INTEGER PRIMARY KEY,
+			title TEXT NOT NULL,
+			link TEXT NOT NULL,
+			pub_date TIMESTAMP WITH TIME ZONE NOT NULL
+		)
+	`)
+	if err != nil {
+		err = fmt.Errorf("error during migration: %w", err)
+		log.Fatal(err)
+	}
+}
+
+func extractIdFromItem(item Item) int {
+	parts := strings.Split(item.Guid, "/")
+	id, err := strconv.Atoi(parts[len(parts)-1])
+	if err != nil {
+		err = fmt.Errorf("error during item id extraction: %w", err)
+		log.Fatal(err)
+	}
+	return id
+}
+
+func ItemExists(item Item) bool {
+	id := extractIdFromItem(item)
+	var count int
+	err := db.QueryRow(`SELECT COUNT(1) FROM items WHERE id = ?`, id).Scan(&count)
+	if err != nil {
+		err = fmt.Errorf("error during item check: %w", err)
+		log.Fatal(err)
+	}
+	return count > 0
+}
+
+func SaveItem(item Item) {
+	id := extractIdFromItem(item)
+	_, err := db.Exec(`INSERT INTO items(id, title, link, pub_date) VALUES (?, ?, ?, ?)`, id, item.Title, item.Link, item.PubDate.Time)
+	if err != nil {
+		err = fmt.Errorf("error during item insert: %w", err)
+		log.Fatal(err)
+	}
+}

go.mod

@@ -5,5 +5,6 @@ go 1.20
 require (
 	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/joho/godotenv v1.5.1 // indirect
+	github.com/mattn/go-sqlite3 v1.14.16 // indirect
 	github.com/namsral/flag v1.7.4-pre // indirect
 )

go.sum

@@ -2,5 +2,7 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
 github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
 github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
+github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
+github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
 github.com/namsral/flag v1.7.4-pre h1:b2ScHhoCUkbsq0d2C15Mv+VU8bl8hAXV8arnWiOHNZs=
 github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo=

main.go

@@ -25,7 +25,7 @@ func contains(rss *Rss, a Item) bool {
 }
 
 func main() {
-	var oldRss *Rss
+	defer db.Close()
 	for {
 		rss, err := FetchStackerNewsRssFeed()
 		if err != nil {
@@ -35,9 +35,8 @@ func main() {
 		}
 
 		for _, item := range rss.Channel.Items {
-			// TODO: We ignore errors during sending items to telegram.
+			if ItemExists(item) {
-			//   This means these items will be missed since they will be skipped in the next run
+				log.Printf("item %s already exists. skipping.\n", item.Guid)
-			if contains(oldRss, item) {
 				continue
 			}
 			err := SendItemToTelegram(&item)
@@ -45,9 +44,9 @@ func main() {
 				log.Println(err)
 				continue
 			}
+			SaveItem(item)
 		}
 
-		oldRss = rss
 		WaitUntilNextUpdate()
 	}
 }