Use sqlite3 to fix duplicate TG posts

This commit is contained in:
ekzyis 2023-05-07 18:30:02 +02:00
parent 0b64e64ceb
commit c96d4a8be2
5 changed files with 77 additions and 5 deletions

1
.gitignore vendored
View File

@ -1,2 +1,3 @@
.env .env
sn-rss2tg sn-rss2tg
sn-rss2tg.sqlite3

69
db.go Normal file
View File

@ -0,0 +1,69 @@
package main
import (
"database/sql"
"fmt"
"log"
"strconv"
"strings"
_ "github.com/mattn/go-sqlite3"
)
var (
db *sql.DB
)
func init() {
var err error
db, err = sql.Open("sqlite3", "sn-rss2tg.sqlite3")
if err != nil {
log.Fatal(err)
}
migrate(db)
}
func migrate(db *sql.DB) {
_, err := db.Exec(`
CREATE TABLE IF NOT EXISTS items (
id INTEGER PRIMARY KEY,
title TEXT NOT NULL,
link TEXT NOT NULL,
pub_date TIMESTAMP WITH TIME ZONE NOT NULL
)
`)
if err != nil {
err = fmt.Errorf("error during migration: %w", err)
log.Fatal(err)
}
}
func extractIdFromItem(item Item) int {
parts := strings.Split(item.Guid, "/")
id, err := strconv.Atoi(parts[len(parts)-1])
if err != nil {
err = fmt.Errorf("error during item id extraction: %w", err)
log.Fatal(err)
}
return id
}
func ItemExists(item Item) bool {
id := extractIdFromItem(item)
var count int
err := db.QueryRow(`SELECT COUNT(1) FROM items WHERE id = ?`, id).Scan(&count)
if err != nil {
err = fmt.Errorf("error during item check: %w", err)
log.Fatal(err)
}
return count > 0
}
func SaveItem(item Item) {
id := extractIdFromItem(item)
_, err := db.Exec(`INSERT INTO items(id, title, link, pub_date) VALUES (?, ?, ?, ?)`, id, item.Title, item.Link, item.PubDate.Time)
if err != nil {
err = fmt.Errorf("error during item insert: %w", err)
log.Fatal(err)
}
}

1
go.mod
View File

@ -5,5 +5,6 @@ go 1.20
require ( require (
github.com/dustin/go-humanize v1.0.1 // indirect github.com/dustin/go-humanize v1.0.1 // indirect
github.com/joho/godotenv v1.5.1 // indirect github.com/joho/godotenv v1.5.1 // indirect
github.com/mattn/go-sqlite3 v1.14.16 // indirect
github.com/namsral/flag v1.7.4-pre // indirect github.com/namsral/flag v1.7.4-pre // indirect
) )

2
go.sum
View File

@ -2,5 +2,7 @@ github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkp
github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0= github.com/joho/godotenv v1.5.1 h1:7eLL/+HRGLY0ldzfGMeQkb7vMd0as4CfYvUVzLqw0N0=
github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4= github.com/joho/godotenv v1.5.1/go.mod h1:f4LDr5Voq0i2e/R5DDNOoa2zzDfwtkZa6DnEwAbqwq4=
github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y=
github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg=
github.com/namsral/flag v1.7.4-pre h1:b2ScHhoCUkbsq0d2C15Mv+VU8bl8hAXV8arnWiOHNZs= github.com/namsral/flag v1.7.4-pre h1:b2ScHhoCUkbsq0d2C15Mv+VU8bl8hAXV8arnWiOHNZs=
github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo= github.com/namsral/flag v1.7.4-pre/go.mod h1:OXldTctbM6SWH1K899kPZcf65KxJiD7MsceFUpB5yDo=

View File

@ -25,7 +25,7 @@ func contains(rss *Rss, a Item) bool {
} }
func main() { func main() {
var oldRss *Rss defer db.Close()
for { for {
rss, err := FetchStackerNewsRssFeed() rss, err := FetchStackerNewsRssFeed()
if err != nil { if err != nil {
@ -35,9 +35,8 @@ func main() {
} }
for _, item := range rss.Channel.Items { for _, item := range rss.Channel.Items {
// TODO: We ignore errors during sending items to telegram. if ItemExists(item) {
// This means these items will be missed since they will be skipped in the next run log.Printf("item %s already exists. skipping.\n", item.Guid)
if contains(oldRss, item) {
continue continue
} }
err := SendItemToTelegram(&item) err := SendItemToTelegram(&item)
@ -45,9 +44,9 @@ func main() {
log.Println(err) log.Println(err)
continue continue
} }
SaveItem(item)
} }
oldRss = rss
WaitUntilNextUpdate() WaitUntilNextUpdate()
} }
} }