ekzyis/stacker.news
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

moar link extraction vibe coding: config file, batching, log level

Browse Source
This commit is contained in:
k00b 2025-02-28 11:35:18 -06:00
parent d1bbfd5339
commit 4e113c267b
2 changed files with 244 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
.gitignore vendored
View File

@ -61,4 +61,7 @@ scripts/nwc-keys.json
docker/lnbits/data docker/lnbits/data
# lndk # lndk
!docker/lndk/tls-*.pem !docker/lndk/tls-*.pem
# nostr link extract
scripts/nostr-link-extract.config.json

341
scripts/nostr-link-extract.js
View File

@ -1,5 +1,7 @@
const WebSocket = require('ws') // You might need to install this: npm install ws const WebSocket = require('ws') // You might need to install this: npm install ws
const { nip19 } = require('nostr-tools') // Keep this for formatting const { nip19 } = require('nostr-tools') // Keep this for formatting
const fs = require('fs')
const path = require('path')
// ANSI color codes // ANSI color codes
const colors = { const colors = {
@ -37,33 +39,108 @@ const colors = {
} }
} }
// Default configuration
let config = {
userPubkeys: [],
ignorePubkeys: [],
timeIntervalHours: 12,
verbosity: 'normal', // Can be 'minimal', 'normal', or 'debug'
relayUrls: [
'wss://relay.nostr.band',
'wss://relay.primal.net',
'wss://relay.damus.io'
],
batchSize: 100,
mediaPatterns: [
{
type: 'extensions',
patterns: ['\\.jpg$', '\\.jpeg$', '\\.png$', '\\.gif$', '\\.bmp$', '\\.webp$', '\\.tiff$', '\\.ico$',
'\\.mp4$', '\\.webm$', '\\.mov$', '\\.avi$', '\\.mkv$', '\\.flv$', '\\.wmv$',
'\\.mp3$', '\\.wav$', '\\.ogg$', '\\.flac$', '\\.aac$', '\\.m4a$']
},
{
type: 'domains',
patterns: [
'nostr\\.build\\/[ai]\\/\\w+',
'i\\.imgur\\.com\\/\\w+',
'i\\.ibb\\.co\\/\\w+\\/',
'tenor\\.com\\/view\\/',
'giphy\\.com\\/gifs\\/',
'soundcloud\\.com\\/',
'spotify\\.com\\/',
'fountain\\.fm\\/'
]
}
]
}
/** /**
* Checks if a URL is a media file or hosted on a media platform * Logger utility that respects the configured verbosity level
*/
const logger = {
// Always show error messages
error: (message) => {
console.error(`${colors.fg.red}Error: ${message}${colors.reset}`)
},
// Minimal essential info - always show regardless of verbosity
info: (message) => {
console.log(`${colors.fg.green}${message}${colors.reset}`)
},
// Progress updates - show in normal and debug modes
progress: (message) => {
if (config.verbosity !== 'minimal') {
console.log(`${colors.fg.blue}${message}${colors.reset}`)
}
},
// Detailed debug info - only show in debug mode
debug: (message) => {
if (config.verbosity === 'debug') {
console.log(`${colors.fg.gray}${message}${colors.reset}`)
}
},
// Results info - formatted differently for clarity
result: (message) => {
console.log(`${colors.bright}${colors.fg.green}${message}${colors.reset}`)
}
}
/**
* Load configuration from a JSON file
* @param {String} configPath - Path to the config file
* @returns {Object} - Configuration object
*/
function loadConfig (configPath) {
try {
const configData = fs.readFileSync(configPath, 'utf8')
const loadedConfig = JSON.parse(configData)
// Merge with default config to ensure all properties exist
return { ...config, ...loadedConfig }
} catch (error) {
logger.error(`Error loading config file: ${error.message}`)
logger.info('Using default configuration')
return config
}
}
/**
* Checks if a URL is a media file or hosted on a media platform based on configured patterns
* @param {String} url - URL to check * @param {String} url - URL to check
* @returns {Boolean} - true if it's likely a media URL * @returns {Boolean} - true if it's likely a media URL
*/ */
function isMediaUrl (url) { function isMediaUrl (url) {
// Check for common media file extensions // Check for media patterns from config
const mediaExtensions = /\.(jpg|jpeg|png|gif|bmp|webp|tiff|ico|mp4|webm|mov|avi|mkv|flv|wmv|mp3|wav|ogg|flac|aac|m4a)($|\?)/i if (config.mediaPatterns) {
if (mediaExtensions.test(url)) return true for (const patternGroup of config.mediaPatterns) {
for (const pattern of patternGroup.patterns) {
// Check for common media hosting platforms const regex = new RegExp(pattern, 'i')
const mediaHostingPatterns = [ if (regex.test(url)) return true
// Image hosting }
/nostr\.build\/[ai]\/\w+/i, }
/i\.imgur\.com\/\w+/i,
/i\.ibb\.co\/\w+\//i,
// Video hosting
/tenor\.com\/view\//i,
/giphy\.com\/gifs\//i,
// Audio hosting
/soundcloud\.com\//i,
/spotify\.com\//i
]
// Check each pattern
for (const pattern of mediaHostingPatterns) {
if (pattern.test(url)) return true
} }
return false return false
@ -77,7 +154,7 @@ function isMediaUrl (url) {
* @returns {Promise<Array>} - Array of events matching the filter * @returns {Promise<Array>} - Array of events matching the filter
*/ */
async function fetchEvents (relayUrls, filter, timeoutMs = 10000) { async function fetchEvents (relayUrls, filter, timeoutMs = 10000) {
console.log(`Fetching events with filter: ${JSON.stringify(filter)}`) logger.debug(`Fetching events with filter: ${JSON.stringify(filter)}`)
const events = [] const events = []
for (const url of relayUrls) { for (const url of relayUrls) {
@ -113,13 +190,13 @@ async function fetchEvents (relayUrls, filter, timeoutMs = 10000) {
resolve(localEvents) resolve(localEvents)
} }
} catch (error) { } catch (error) {
console.error(`Error parsing message: ${error.message}`) logger.debug(`Error parsing message: ${error.message}`)
} }
}) })
}) })
ws.on('error', (error) => { ws.on('error', (error) => {
console.error(`WebSocket error for ${url}: ${error.message}`) logger.debug(`WebSocket error for ${url}: ${error.message}`)
clearTimeout(timeout) clearTimeout(timeout)
resolve([]) // Resolve with empty array on error resolve([]) // Resolve with empty array on error
}) })
@ -130,10 +207,10 @@ async function fetchEvents (relayUrls, filter, timeoutMs = 10000) {
}) })
}) })
console.log(`Got ${relayEvents.length} events from ${url}`) logger.debug(`Got ${relayEvents.length} events from ${url}`)
events.push(...relayEvents) events.push(...relayEvents)
} catch (error) { } catch (error) {
console.error(`Error connecting to ${url}: ${error.message}`) logger.debug(`Error connecting to ${url}: ${error.message}`)
} }
} }
@ -155,20 +232,33 @@ async function fetchEvents (relayUrls, filter, timeoutMs = 10000) {
* @param {Array} userPubkeys - Array of Nostr user public keys * @param {Array} userPubkeys - Array of Nostr user public keys
* @param {Number} timeIntervalHours - Number of hours to look back from now * @param {Number} timeIntervalHours - Number of hours to look back from now
* @param {Array} relayUrls - Array of Nostr relay URLs * @param {Array} relayUrls - Array of Nostr relay URLs
* @param {Array} ignorePubkeys - Array of pubkeys to ignore (optional)
* @returns {Promise<Array>} - Array of note objects containing external links within the time interval * @returns {Promise<Array>} - Array of note objects containing external links within the time interval
*/ */
async function getNotesWithLinks (userPubkeys, timeIntervalHours, relayUrls) { async function getNotesWithLinks (userPubkeys, timeIntervalHours, relayUrls, ignorePubkeys = []) {
// Calculate the cutoff time in seconds (Nostr uses UNIX timestamp) // Calculate the cutoff time in seconds (Nostr uses UNIX timestamp)
const now = Math.floor(Date.now() / 1000) const now = Math.floor(Date.now() / 1000)
const cutoffTime = now - (timeIntervalHours * 60 * 60) const cutoffTime = now - (timeIntervalHours * 60 * 60)
const allNotesWithLinks = [] const allNotesWithLinks = []
const allFollowedPubkeys = new Set() // To collect all followed pubkeys const allFollowedPubkeys = new Set() // To collect all followed pubkeys
const ignoreSet = new Set(ignorePubkeys) // Convert ignore list to Set for efficient lookups
if (ignoreSet.size > 0) {
logger.debug(`Ignoring ${ignoreSet.size} author(s) as requested`)
}
logger.info(`Fetching follow lists for ${userPubkeys.length} users...`)
// First get the followings for each user // First get the followings for each user
for (const pubkey of userPubkeys) { for (const pubkey of userPubkeys) {
try { try {
console.log(`Fetching follow list for ${pubkey} from ${relayUrls.length} relays...`) // Skip if this pubkey is in the ignore list
if (ignoreSet.has(pubkey)) {
logger.debug(`Skipping user ${pubkey} as it's in the ignore list`)
continue
}
logger.debug(`Fetching follow list for ${pubkey} from ${relayUrls.length} relays...`)
// Get the most recent contact list (kind 3) // Get the most recent contact list (kind 3)
const followListEvents = await fetchEvents(relayUrls, { const followListEvents = await fetchEvents(relayUrls, {
@ -177,7 +267,7 @@ async function getNotesWithLinks (userPubkeys, timeIntervalHours, relayUrls) {
}) })
if (followListEvents.length === 0) { if (followListEvents.length === 0) {
console.log(`No follow list found for user ${pubkey}. Verify this pubkey has contacts on these relays.`) logger.debug(`No follow list found for user ${pubkey}. Verify this pubkey has contacts on these relays.`)
continue continue
} }
@ -186,16 +276,16 @@ async function getNotesWithLinks (userPubkeys, timeIntervalHours, relayUrls) {
!latest || event.created_at > latest.created_at ? event : latest, null) !latest || event.created_at > latest.created_at ? event : latest, null)
if (!latestFollowList) { if (!latestFollowList) {
console.log(`No valid follow list found for user ${pubkey}`) logger.debug(`No valid follow list found for user ${pubkey}`)
continue continue
} }
console.log(`Found follow list created at: ${new Date(latestFollowList.created_at * 1000).toISOString()}`) logger.debug(`Found follow list created at: ${new Date(latestFollowList.created_at * 1000).toISOString()}`)
// Check if tags property exists // Check if tags property exists
if (!latestFollowList.tags) { if (!latestFollowList.tags) {
console.log(`No tags found in follow list for user ${pubkey}`) logger.debug(`No tags found in follow list for user ${pubkey}`)
console.log('Follow list data:', JSON.stringify(latestFollowList, null, 2)) logger.debug('Follow list data:', JSON.stringify(latestFollowList, null, 2))
continue continue
} }
@ -203,88 +293,106 @@ async function getNotesWithLinks (userPubkeys, timeIntervalHours, relayUrls) {
const followedPubkeys = latestFollowList.tags const followedPubkeys = latestFollowList.tags
.filter(tag => tag[0] === 'p') .filter(tag => tag[0] === 'p')
.map(tag => tag[1]) .map(tag => tag[1])
.filter(pk => !ignoreSet.has(pk)) // Filter out pubkeys from the ignore list
if (!followedPubkeys || followedPubkeys.length === 0) { if (!followedPubkeys || followedPubkeys.length === 0) {
console.log(`No followed users found for user ${pubkey}`) logger.debug(`No followed users found for user ${pubkey} (after filtering ignore list)`)
continue continue
} }
// Add all followed pubkeys to our set // Add all followed pubkeys to our set
followedPubkeys.forEach(pk => allFollowedPubkeys.add(pk)) followedPubkeys.forEach(pk => allFollowedPubkeys.add(pk))
console.log(`Added ${followedPubkeys.length} followed users for ${pubkey} (total: ${allFollowedPubkeys.size})`) logger.debug(`Added ${followedPubkeys.length} followed users for ${pubkey} (total: ${allFollowedPubkeys.size})`)
} catch (error) { } catch (error) {
console.error(`Error processing user ${pubkey}:`, error) logger.error(`Error processing user ${pubkey}: ${error}`)
} }
} }
// If we found any followed pubkeys, fetch their notes in a single batch // If we found any followed pubkeys, fetch their notes in batches
if (allFollowedPubkeys.size > 0) { if (allFollowedPubkeys.size > 0) {
console.log(`Fetching notes from ${allFollowedPubkeys.size} followed users in a single batch...`)
// Convert Set to Array for the filter // Convert Set to Array for the filter
const followedPubkeysArray = Array.from(allFollowedPubkeys) const followedPubkeysArray = Array.from(allFollowedPubkeys)
const batchSize = config.batchSize || 100 // Use config batch size or default to 100
const totalBatches = Math.ceil(followedPubkeysArray.length / batchSize)
// Fetch notes from all followed users at once logger.progress(`Processing ${followedPubkeysArray.length} followed users in ${totalBatches} batches...`)
const notes = await fetchEvents(relayUrls, {
kinds: [1],
authors: followedPubkeysArray,
since: cutoffTime
}, 30000) // Use a longer timeout for this larger query
console.log(`Retrieved ${notes.length} total notes from followed users`) // Process in batches
for (let batchNum = 0; batchNum < totalBatches; batchNum++) {
const start = batchNum * batchSize
const end = Math.min(start + batchSize, followedPubkeysArray.length)
const batch = followedPubkeysArray.slice(start, end)
// Filter notes that have URLs (excluding notes with only media URLs) logger.progress(`Fetching batch ${batchNum + 1}/${totalBatches} (${batch.length} authors)...`)
const notesWithUrls = notes.filter(note => {
// Extract all URLs from content
const urlRegex = /(https?:\/\/[^\s]+)/g
const matches = note.content.match(urlRegex) || []
if (matches.length === 0) return false // No URLs at all // Fetch notes from the current batch of users
const notes = await fetchEvents(relayUrls, {
kinds: [1],
authors: batch,
since: cutoffTime
}, 30000) // Use a longer timeout for this larger query
// Check if any URL is not a media file logger.debug(`Retrieved ${notes.length} notes from batch ${batchNum + 1}`)
const hasNonMediaUrl = matches.some(url => !isMediaUrl(url))
return hasNonMediaUrl // Filter notes that have URLs (excluding notes with only media URLs)
}) const notesWithUrls = notes.filter(note => {
// Extract all URLs from content
const urlRegex = /(https?:\/\/[^\s]+)/g
const matches = note.content.match(urlRegex) || []
console.log(`Found ${notesWithUrls.length} notes containing non-media URLs`) if (matches.length === 0) return false // No URLs at all
// Get all unique authors from the filtered notes // Check if any URL is not a media file
const authorsWithUrls = new Set(notesWithUrls.map(note => note.pubkey)) const hasNonMediaUrl = matches.some(url => !isMediaUrl(url))
console.log(`Notes with URLs came from ${authorsWithUrls.size} unique authors`)
// Fetch metadata for all relevant authors in a single batch return hasNonMediaUrl
if (authorsWithUrls.size > 0) {
console.log(`Fetching metadata for ${authorsWithUrls.size} authors...`)
const allMetadata = await fetchEvents(relayUrls, {
kinds: [0],
authors: Array.from(authorsWithUrls)
}) })
// Create a map of author pubkey to their latest metadata logger.debug(`Found ${notesWithUrls.length} notes containing non-media URLs in batch ${batchNum + 1}`)
const metadataByAuthor = {}
allMetadata.forEach(meta => {
if (!metadataByAuthor[meta.pubkey] || meta.created_at > metadataByAuthor[meta.pubkey].created_at) {
metadataByAuthor[meta.pubkey] = meta
}
})
// Attach metadata to notes // Get all unique authors from the filtered notes in this batch
for (const note of notesWithUrls) { const authorsWithUrls = new Set(notesWithUrls.map(note => note.pubkey))
if (metadataByAuthor[note.pubkey]) {
try { // Fetch metadata for all relevant authors in this batch
const metadata = JSON.parse(metadataByAuthor[note.pubkey].content) if (authorsWithUrls.size > 0) {
note.userMetadata = metadata logger.debug(`Fetching metadata for ${authorsWithUrls.size} authors from batch ${batchNum + 1}...`)
} catch (e) { const allMetadata = await fetchEvents(relayUrls, {
console.error(`Error parsing metadata for ${note.pubkey}: ${e.message}`) kinds: [0],
authors: Array.from(authorsWithUrls)
})
// Create a map of author pubkey to their latest metadata
const metadataByAuthor = {}
allMetadata.forEach(meta => {
if (!metadataByAuthor[meta.pubkey] || meta.created_at > metadataByAuthor[meta.pubkey].created_at) {
metadataByAuthor[meta.pubkey] = meta
}
})
// Attach metadata to notes
for (const note of notesWithUrls) {
if (metadataByAuthor[note.pubkey]) {
try {
const metadata = JSON.parse(metadataByAuthor[note.pubkey].content)
note.userMetadata = metadata
} catch (e) {
logger.debug(`Error parsing metadata for ${note.pubkey}: ${e.message}`)
}
} }
} }
} }
// Add all notes with URLs from this batch to our results
allNotesWithLinks.push(...notesWithUrls)
// Show incremental progress during batch processing
if (allNotesWithLinks.length > 0 && batchNum < totalBatches - 1) {
logger.progress(`Found ${allNotesWithLinks.length} notes with links so far...`)
}
} }
// Add all notes with URLs to our results logger.progress(`Completed processing all ${totalBatches} batches`)
allNotesWithLinks.push(...notesWithUrls)
} }
return allNotesWithLinks return allNotesWithLinks
@ -367,36 +475,67 @@ function escapeRegExp (string) {
return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&') return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
} }
/**
* Convert a pubkey from npub to hex format if needed
* @param {String} key - Pubkey in either npub or hex format
* @returns {String} - Pubkey in hex format
*/
function normalizeToHexPubkey (key) {
// If it's an npub, decode it
if (typeof key === 'string' && key.startsWith('npub1')) {
try {
const { type, data } = nip19.decode(key)
if (type === 'npub') {
return data
}
} catch (e) {
logger.error(`Error decoding npub ${key}: ${e.message}`)
}
}
// Otherwise assume it's already in hex format
return key
}
/** /**
* Main function to execute the script * Main function to execute the script
*/ */
async function main () { async function main () {
// Example usage // Load configuration from file
const userPubkey = '05933d8782d155d10cf8a06f37962f329855188063903d332714fbd881bac46e' const configPath = path.join(__dirname, 'nostr-link-extract.config.json')
logger.info(`Loading configuration from ${configPath}`)
// List of relays that were working in our test config = loadConfig(configPath)
const relayUrls = [
'wss://relay.damus.io',
'wss://relay.nostr.band',
'wss://nos.lol',
'wss://nostr.wine',
'wss://relay.snort.social',
'wss://relay.primal.net'
]
try { try {
console.log(`${colors.bright}${colors.fg.green}Fetching notes with links...${colors.reset}`) logger.info(`Starting Nostr link extraction (time interval: ${config.timeIntervalHours} hours)`)
const notesWithLinks = await getNotesWithLinks([userPubkey], 24, relayUrls)
// Convert any npub format keys to hex
const hexUserPubkeys = config.userPubkeys.map(normalizeToHexPubkey)
const hexIgnorePubkeys = config.ignorePubkeys.map(normalizeToHexPubkey)
// Log the conversion for clarity (helpful for debugging)
if (config.userPubkeys.some(key => key.startsWith('npub1'))) {
logger.debug('Converted user npubs to hex format for Nostr protocol')
}
if (config.ignorePubkeys.some(key => key.startsWith('npub1'))) {
logger.debug('Converted ignore list npubs to hex format for Nostr protocol')
}
const notesWithLinks = await getNotesWithLinks(
hexUserPubkeys,
config.timeIntervalHours,
config.relayUrls,
hexIgnorePubkeys
)
if (notesWithLinks.length > 0) { if (notesWithLinks.length > 0) {
const formattedOutput = formatNoteOutput(notesWithLinks) const formattedOutput = formatNoteOutput(notesWithLinks)
console.log(formattedOutput) console.log(formattedOutput)
console.log(`${colors.bright}${colors.fg.green}Total notes with links: ${notesWithLinks.length}${colors.reset}`) logger.result(`Total notes with links: ${notesWithLinks.length}`)
} else { } else {
console.log(`${colors.fg.yellow}No notes with links found in the specified time interval.${colors.reset}`) logger.info('No notes with links found in the specified time interval.')
} }
} catch (error) { } catch (error) {
console.error(`${colors.fg.red}Error: ${error}${colors.reset}`) logger.error(`${error}`)
} }
} }