Fix footnotes and overflow (#1940)

* Fix missing uncollapse on footnote click

* Add comments to variables

.github/workflows/extend-awards.yml

@@ -0,0 +1,32 @@
+name: extend-awards
+run-name: Extending awards
+on:
+  pull_request:
+    types: [ closed ]
+    branches:
+      - master
+jobs:
+  if_merged:
+    if: |
+      github.event_name == 'pull_request' &&
+      github.event.action == 'closed' &&
+      github.event.pull_request.merged == true &&
+      github.event.pull_request.head.ref != 'extend-awards/patch'
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13' 
+      - run: pip install requests
+      - run: python extend-awards.py
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+          GITHUB_CONTEXT: ${{ toJson(github) }}
+      - uses: peter-evans/create-pull-request@v7
+        with:
+          add-paths: awards.csv
+          branch: extend-awards/patch
+          commit-message: Extending awards.csv
+          title: Extending awards.csv
+          body: A PR was merged that solves an issue and awards.csv should be extended.

.gitignore

@@ -62,3 +62,6 @@ docker/lnbits/data
 
 # lndk
 !docker/lndk/tls-*.pem
+
+# nostr link extract
+scripts/nostr-link-extract.config.json

README.md

@@ -31,6 +31,7 @@ Go to [localhost:3000](http://localhost:3000).
    - ssh: `git clone git@github.com:stackernews/stacker.news.git`
    - https: `git clone https://github.com/stackernews/stacker.news.git`
 - Install [docker](https://docs.docker.com/compose/install/)
+    - If you're running MacOS or Windows, I ***highly recommend***  using [OrbStack](https://orbstack.dev/) instead of Docker Desktop
 - Please make sure that at least 10 GB of free space is available, otherwise you may encounter issues while setting up the development environment.
 
 <br>

api/paidAction/index.js

@@ -3,7 +3,7 @@ import { datePivot } from '@/lib/time'
 import { PAID_ACTION_PAYMENT_METHODS, USER_ID } from '@/lib/constants'
 import { createHmac } from '@/api/resolvers/wallet'
 import { Prisma } from '@prisma/client'
-import { createWrappedInvoice, createInvoice as createUserInvoice } from '@/wallets/server'
+import { createWrappedInvoice, createUserInvoice } from '@/wallets/server'
 import { assertBelowMaxPendingInvoices, assertBelowMaxPendingDirectPayments } from './lib/assert'
 
 import * as ITEM_CREATE from './itemCreate'
@@ -264,26 +264,28 @@ async function performDirectAction (actionType, args, incomingContext) {
     throw new NonInvoiceablePeerError()
   }
 
-  let invoiceObject
-
   try {
     await assertBelowMaxPendingDirectPayments(userId, incomingContext)
 
     const description = actionDescription ?? await paidActions[actionType].describe(args, incomingContext)
-    invoiceObject = await createUserInvoice(userId, {
+
+    for await (const { invoice, logger, wallet } of createUserInvoice(userId, {
       msats: cost,
       description,
       expiry: INVOICE_EXPIRE_SECS
-    }, { models, lnd })
+    }, { models, lnd })) {
+      let hash
+      try {
+        hash = parsePaymentRequest({ request: invoice }).id
       } catch (e) {
-    console.error('failed to create outside invoice', e)
-    throw new NonInvoiceablePeerError()
+        console.error('failed to parse invoice', e)
+        logger?.error('failed to parse invoice: ' + e.message, { bolt11: invoice })
+        continue
       }
 
-  const { invoice, wallet } = invoiceObject
-  const hash = parsePaymentRequest({ request: invoice }).id
-
-  const payment = await models.directPayment.create({
+      try {
+        return {
+          invoice: await models.directPayment.create({
             data: {
               comment,
               lud18Data,
@@ -294,12 +296,19 @@ async function performDirectAction (actionType, args, incomingContext) {
               walletId: wallet.id,
               receiverId: userId
             }
-  })
-
-  return {
-    invoice: payment,
+          }),
           paymentMethod: 'DIRECT'
         }
+      } catch (e) {
+        console.error('failed to create direct payment', e)
+        logger?.error('failed to create direct payment: ' + e.message, { bolt11: invoice })
+      }
+    }
+  } catch (e) {
+    console.error('failed to create user invoice', e)
+  }
+
+  throw new NonInvoiceablePeerError()
 }
 
 export async function retryPaidAction (actionType, args, incomingContext) {
@@ -419,7 +428,7 @@ async function createSNInvoice (actionType, args, context) {
 }
 
 async function createDbInvoice (actionType, args, context) {
-  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context
+  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, paymentAttempt, predecessorId } = context
   const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs
 
   const db = tx ?? models
@@ -445,6 +454,7 @@ async function createDbInvoice (actionType, args, context) {
     actionArgs: args,
     expiresAt,
     actionId,
+    paymentAttempt,
     predecessorId
   }
 

api/resolvers/growth.js

@@ -121,6 +121,39 @@ export default {
         FROM ${viewGroup(range, 'stacking_growth')}
         GROUP BY time
         ORDER BY time ASC`, ...range)
+    },
+    itemGrowthSubs: async (parent, { when, to, from, sub }, { models }) => {
+      const range = whenRange(when, from, to)
+
+      const subExists = await models.sub.findUnique({ where: { name: sub } })
+      if (!subExists) throw new Error('Sub not found')
+
+      return await models.$queryRawUnsafe(`
+        SELECT date_trunc('${timeUnitForRange(range)}', t) at time zone 'America/Chicago' as time, json_build_array(
+          json_build_object('name', 'posts', 'value', coalesce(sum(posts),0)),
+          json_build_object('name', 'comments', 'value', coalesce(sum(comments),0))
+        ) AS data
+        FROM ${viewGroup(range, 'sub_stats')}
+        WHERE sub_name = $3
+        GROUP BY time
+        ORDER BY time ASC`, ...range, sub)
+    },
+    revenueGrowthSubs: async (parent, { when, to, from, sub }, { models }) => {
+      const range = whenRange(when, from, to)
+
+      const subExists = await models.sub.findUnique({ where: { name: sub } })
+      if (!subExists) throw new Error('Sub not found')
+
+      return await models.$queryRawUnsafe(`
+        SELECT date_trunc('${timeUnitForRange(range)}', t) at time zone 'America/Chicago' as time, json_build_array(
+          json_build_object('name', 'revenue', 'value', coalesce(sum(msats_revenue/1000),0)),
+          json_build_object('name', 'stacking', 'value', coalesce(sum(msats_stacked/1000),0)),
+          json_build_object('name', 'spending', 'value', coalesce(sum(msats_spent/1000),0))
+        ) AS data
+        FROM ${viewGroup(range, 'sub_stats')}
+        WHERE sub_name = $3
+        GROUP BY time
+        ORDER BY time ASC`, ...range, sub)
     }
   }
 }

api/resolvers/notifications.js

@@ -5,6 +5,7 @@ import { pushSubscriptionSchema, validateSchema } from '@/lib/validate'
 import { replyToSubscription } from '@/lib/webPush'
 import { getSub } from './sub'
 import { GqlAuthenticationError, GqlInputError } from '@/lib/error'
+import { WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants'
 
 export default {
   Query: {
@@ -345,11 +346,25 @@ export default {
       )
 
       queries.push(
-        `(SELECT "Invoice".id::text, "Invoice"."updated_at" AS "sortTime", NULL as "earnedSats", 'Invoicification' AS type
+        `(SELECT "Invoice".id::text,
+          CASE
+            WHEN
+              "Invoice"."paymentAttempt" < ${WALLET_MAX_RETRIES}
+              AND "Invoice"."userCancel" = false
+              AND "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}'
+            THEN "Invoice"."cancelledAt" + interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}'
+            ELSE "Invoice"."updated_at"
+          END AS "sortTime", NULL as "earnedSats", 'Invoicification' AS type
         FROM "Invoice"
         WHERE "Invoice"."userId" = $1
         AND "Invoice"."updated_at" < $2
         AND "Invoice"."actionState" = 'FAILED'
+        AND (
+          -- this is the inverse of the filter for automated retries
+          "Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES}
+          OR "Invoice"."userCancel" = true
+          OR "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}'
+        )
         AND (
           "Invoice"."actionType" = 'ITEM_CREATE' OR
           "Invoice"."actionType" = 'ZAP' OR

api/resolvers/paidAction.js

@@ -1,5 +1,5 @@
 import { retryPaidAction } from '../paidAction'
-import { USER_ID } from '@/lib/constants'
+import { USER_ID, WALLET_MAX_RETRIES, WALLET_RETRY_TIMEOUT_MS } from '@/lib/constants'
 
 function paidActionType (actionType) {
   switch (actionType) {
@@ -50,24 +50,32 @@ export default {
     }
   },
   Mutation: {
-    retryPaidAction: async (parent, { invoiceId }, { models, me, lnd }) => {
+    retryPaidAction: async (parent, { invoiceId, newAttempt }, { models, me, lnd }) => {
       if (!me) {
         throw new Error('You must be logged in')
       }
 
-      const invoice = await models.invoice.findUnique({ where: { id: invoiceId, userId: me.id } })
+      // make sure only one client at a time can retry by acquiring a lock that expires
+      const [invoice] = await models.$queryRaw`
+          UPDATE "Invoice"
+          SET "retryPendingSince" = now()
+          WHERE
+            id = ${invoiceId} AND
+            "userId" = ${me.id} AND
+            "actionState" = 'FAILED' AND
+            ("retryPendingSince" IS NULL OR "retryPendingSince" < now() - ${`${WALLET_RETRY_TIMEOUT_MS} milliseconds`}::interval)
+          RETURNING *`
       if (!invoice) {
-        throw new Error('Invoice not found')
+        throw new Error('Invoice not found or retry pending')
       }
 
-      if (invoice.actionState !== 'FAILED') {
-        if (invoice.actionState === 'PAID') {
-          throw new Error('Invoice is already paid')
-        }
-        throw new Error(`Invoice is not in failed state: ${invoice.actionState}`)
+      // do we want to retry a payment from the beginning with all sender and receiver wallets?
+      const paymentAttempt = newAttempt ? invoice.paymentAttempt + 1 : invoice.paymentAttempt
+      if (paymentAttempt > WALLET_MAX_RETRIES) {
+        throw new Error('Payment has been retried too many times')
       }
 
-      const result = await retryPaidAction(invoice.actionType, { invoice }, { models, me, lnd })
+      const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd })
 
       return {
         ...result,

api/resolvers/user.js

@@ -4,9 +4,9 @@ import { decodeCursor, LIMIT, nextCursorEncoded } from '@/lib/cursor'
 import { msatsToSats } from '@/lib/format'
 import { bioSchema, emailSchema, settingsSchema, validateSchema, userSchema } from '@/lib/validate'
 import { getItem, updateItem, filterClause, createItem, whereClause, muteClause, activeOrMine } from './item'
-import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES } from '@/lib/constants'
+import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES, WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants'
 import { viewGroup } from './growth'
-import { timeUnitForRange, whenRange } from '@/lib/time'
+import { datePivot, timeUnitForRange, whenRange } from '@/lib/time'
 import assertApiKeyNotPermitted from './apiKey'
 import { hashEmail } from '@/lib/crypto'
 import { isMuted } from '@/lib/user'
@@ -543,7 +543,17 @@ export default {
           actionType: {
             in: INVOICE_ACTION_NOTIFICATION_TYPES
           },
-          actionState: 'FAILED'
+          actionState: 'FAILED',
+          OR: [
+            {
+              paymentAttempt: {
+                gte: WALLET_MAX_RETRIES
+              }
+            },
+            {
+              userCancel: true
+            }
+          ]
         }
       })
 
@@ -552,6 +562,31 @@ export default {
         return true
       }
 
+      const invoiceActionFailed2 = await models.invoice.findFirst({
+        where: {
+          userId: me.id,
+          updatedAt: {
+            gt: datePivot(lastChecked, { milliseconds: -WALLET_RETRY_BEFORE_MS })
+          },
+          actionType: {
+            in: INVOICE_ACTION_NOTIFICATION_TYPES
+          },
+          actionState: 'FAILED',
+          paymentAttempt: {
+            lt: WALLET_MAX_RETRIES
+          },
+          userCancel: false,
+          cancelledAt: {
+            lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS })
+          }
+        }
+      })
+
+      if (invoiceActionFailed2) {
+        foundNotes()
+        return true
+      }
+
       // update checkedNotesAt to prevent rechecking same time period
       models.user.update({
         where: { id: me.id },

api/resolvers/wallet.js

@@ -9,7 +9,10 @@ import { formatMsats, msatsToSats, msatsToSatsDecimal, satsToMsats } from '@/lib
 import {
   USER_ID, INVOICE_RETENTION_DAYS,
   PAID_ACTION_PAYMENT_METHODS,
-  WALLET_CREATE_INVOICE_TIMEOUT_MS
+  WALLET_CREATE_INVOICE_TIMEOUT_MS,
+  WALLET_RETRY_AFTER_MS,
+  WALLET_RETRY_BEFORE_MS,
+  WALLET_MAX_RETRIES
 } from '@/lib/constants'
 import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate'
 import assertGofacYourself from './ofac'
@@ -456,6 +459,21 @@ const resolvers = {
         cursor: nextCursor,
         entries: logs
       }
+    },
+    failedInvoices: async (parent, args, { me, models }) => {
+      if (!me) {
+        throw new GqlAuthenticationError()
+      }
+      return await models.$queryRaw`
+        SELECT * FROM "Invoice"
+        WHERE "userId" = ${me.id}
+        AND "actionState" = 'FAILED'
+        -- never retry if user has cancelled the invoice manually
+        AND "userCancel" = false
+        AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval
+        AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval
+        AND "paymentAttempt" < ${WALLET_MAX_RETRIES}
+        ORDER BY id DESC`
     }
   },
   Wallet: {

api/typeDefs/growth.js

@@ -13,6 +13,8 @@ export default gql`
     spenderGrowth(when: String, from: String, to: String): [TimeData!]!
     stackingGrowth(when: String, from: String, to: String): [TimeData!]!
     stackerGrowth(when: String, from: String, to: String): [TimeData!]!
+    itemGrowthSubs(when: String, from: String, to: String, sub: String): [TimeData!]!
+    revenueGrowthSubs(when: String, from: String, to: String, sub: String): [TimeData!]!
   }
 
   type TimeData {

api/typeDefs/item.js

@@ -159,7 +159,7 @@ export default gql`
     remote: Boolean
     sub: Sub
     subName: String
-    status: String
+    status: String!
     uploadId: Int
     otsHash: String
     parentOtsHash: String

api/typeDefs/paidAction.js

@@ -7,7 +7,7 @@ extend type Query {
 }
 
 extend type Mutation {
-  retryPaidAction(invoiceId: Int!): PaidAction!
+  retryPaidAction(invoiceId: Int!, newAttempt: Boolean): PaidAction!
 }
 
 enum PaymentMethod {

api/typeDefs/sub.js

@@ -31,7 +31,7 @@ export default gql`
   }
 
   type Sub {
-    name: ID!
+    name: String!
     createdAt: Date!
     userId: Int!
     user: User!

api/typeDefs/user.js

@@ -49,7 +49,7 @@ export default gql`
   type User {
     id: ID!
     createdAt: Date!
-    name: String
+    name: String!
     nitems(when: String, from: String, to: String): Int!
     nposts(when: String, from: String, to: String): Int!
     nterritories(when: String, from: String, to: String): Int!

api/typeDefs/wallet.js

@@ -72,6 +72,7 @@ const typeDefs = `
     wallet(id: ID!): Wallet
     walletByType(type: String!): Wallet
     walletLogs(type: String, from: String, to: String, cursor: String): WalletLog!
+    failedInvoices: [Invoice!]!
   }
 
   extend type Mutation {

awards.csv

@@ -174,3 +174,14 @@ Soxasora,pr,#1820,#1819,easy,,,1,90k,soxasora@blink.sv,2025-01-27
 SatsAllDay,issue,#1820,#1819,easy,,,1,9k,weareallsatoshi@getalby.com,2025-01-27
 Soxasora,pr,#1814,#1736,easy,,,,100k,soxasora@blink.sv,2025-01-27
 jason-me,pr,#1857,,easy,,,,100k,rrbtc@vlt.ge,2025-02-08
+ed-kung,pr,#1901,#323,good-first-issue,,,,20k,simplestacker@getalby.com,2025-02-14
+Scroogey-SN,pr,#1911,#1905,good-first-issue,,,1,18k,???,???
+Scroogey-SN,pr,#1928,#1924,good-first-issue,,,,20k,???,???
+dtonon,issue,#1928,#1924,good-first-issue,,,,2k,???,???
+ed-kung,pr,#1926,#1914,medium-hard,,,,500k,simplestacker@getalby.com,???
+ed-kung,issue,#1926,#1914,medium-hard,,,,50k,simplestacker@getalby.com,???
+ed-kung,pr,#1926,#1927,easy,,,,100k,simplestacker@getalby.com,???
+ed-kung,issue,#1926,#1927,easy,,,,10k,simplestacker@getalby.com,???
+ed-kung,issue,#1913,#1890,good-first-issue,,,,2k,simplestacker@getalby.com,???
+Scroogey-SN,pr,#1930,#1167,good-first-issue,,,,20k,???,???
+itsrealfake,issue,#1930,#1167,good-first-issue,,,,2k,???,???

components/dark-mode.js

@@ -34,20 +34,23 @@ const setTheme = (dark) => {
 
 const listenForThemeChange = (onChange) => {
   const mql = window.matchMedia(PREFER_DARK_QUERY)
-  mql.onchange = mql => {
+  const onMqlChange = () => {
     const { user, dark } = getTheme()
     if (!user) {
       handleThemeChange(dark)
       onChange({ user, dark })
     }
   }
-  window.onstorage = e => {
+  mql.addEventListener('change', onMqlChange)
+
+  const onStorage = (e) => {
     if (e.key === STORAGE_KEY) {
       const dark = JSON.parse(e.newValue)
       setTheme(dark)
       onChange({ user: true, dark })
     }
   }
+  window.addEventListener('storage', onStorage)
 
   const root = window.document.documentElement
   const observer = new window.MutationObserver(() => {
@@ -56,7 +59,11 @@ const listenForThemeChange = (onChange) => {
   })
   observer.observe(root, { attributes: true, attributeFilter: ['data-bs-theme'] })
 
-  return () => observer.disconnect()
+  return () => {
+    observer.disconnect()
+    mql.removeEventListener('change', onMqlChange)
+    window.removeEventListener('storage', onStorage)
+  }
 }
 
 export default function useDarkMode () {
@@ -65,7 +72,7 @@ export default function useDarkMode () {
   useEffect(() => {
     const { user, dark } = getTheme()
     setDark({ user, dark })
-    listenForThemeChange(setDark)
+    return listenForThemeChange(setDark)
   }, [])
 
   return [dark?.dark, () => {

components/footer.js

@@ -173,7 +173,7 @@ export default function Footer ({ links = true }) {
               <Rewards />
             </div>
             <div className='mb-0' style={{ fontWeight: 500 }}>
-              <Link href='/stackers/day' className='nav-link p-0 p-0 d-inline-flex'>
+              <Link href='/stackers/all/day' className='nav-link p-0 p-0 d-inline-flex'>
                 analytics
               </Link>
               <span className='mx-2 text-muted'> \ </span>

components/form.js

@@ -20,7 +20,6 @@ import TextareaAutosize from 'react-textarea-autosize'
 import { useToast } from './toast'
 import { numWithUnits } from '@/lib/format'
 import textAreaCaret from 'textarea-caret'
-import ReactDatePicker from 'react-datepicker'
 import 'react-datepicker/dist/react-datepicker.css'
 import useDebounceCallback, { debounce } from './use-debounce-callback'
 import { FileUpload } from './file-upload'
@@ -38,9 +37,10 @@ import QrIcon from '@/svgs/qr-code-line.svg'
 import QrScanIcon from '@/svgs/qr-scan-line.svg'
 import { useShowModal } from './modal'
 import { QRCodeSVG } from 'qrcode.react'
-import { Scanner } from '@yudiel/react-qr-scanner'
+import dynamic from 'next/dynamic'
 import { qrImageSettings } from './qr'
 import { useIsClient } from './use-client'
+import PageLoading from './page-loading'
 
 export class SessionRequiredError extends Error {
   constructor () {
@@ -971,6 +971,19 @@ export function Select ({ label, items, info, groupClassName, onChange, noForm,
   )
 }
 
+function DatePickerSkeleton () {
+  return (
+    <div className='react-datepicker-wrapper'>
+      <input className='form-control clouds fade-out p-0 px-2 mb-0' />
+    </div>
+  )
+}
+
+const ReactDatePicker = dynamic(() => import('react-datepicker').then(mod => mod.default), {
+  ssr: false,
+  loading: () => <DatePickerSkeleton />
+})
+
 export function DatePicker ({ fromName, toName, noForm, onChange, when, from, to, className, ...props }) {
   const formik = noForm ? null : useFormikContext()
   const [,, fromHelpers] = noForm ? [{}, {}, {}] : useField({ ...props, name: fromName })
@@ -1038,6 +1051,8 @@ export function DatePicker ({ fromName, toName, noForm, onChange, when, from, to
   }
 
   return (
+    <>
+      {ReactDatePicker && (
         <ReactDatePicker
           className={`form-control text-center ${className}`}
           selectsRange
@@ -1051,6 +1066,8 @@ export function DatePicker ({ fromName, toName, noForm, onChange, when, from, to
           onChangeRaw={onChangeRawHandler}
           onChange={innerOnChange}
         />
+      )}
+    </>
   )
 }
 
@@ -1070,7 +1087,13 @@ export function DateTimeInput ({ label, groupClassName, name, ...props }) {
 
 function DateTimePicker ({ name, className, ...props }) {
   const [field, , helpers] = useField({ ...props, name })
+  const ReactDatePicker = dynamic(() => import('react-datepicker').then(mod => mod.default), {
+    ssr: false,
+    loading: () => <span>loading date picker</span>
+  })
   return (
+    <>
+      {ReactDatePicker && (
         <ReactDatePicker
           {...field}
           {...props}
@@ -1083,6 +1106,8 @@ function DateTimePicker ({ name, className, ...props }) {
             helpers.setValue(val)
           }}
         />
+      )}
+    </>
   )
 }
 
@@ -1149,6 +1174,10 @@ function QrPassword ({ value }) {
 function PasswordScanner ({ onScan, text }) {
   const showModal = useShowModal()
   const toaster = useToast()
+  const Scanner = dynamic(() => import('@yudiel/react-qr-scanner').then(mod => mod.Scanner), {
+    ssr: false,
+    loading: () => <PageLoading />
+  })
 
   return (
     <InputGroup.Text
@@ -1158,6 +1187,7 @@ function PasswordScanner ({ onScan, text }) {
           return (
             <div>
               {text && <h5 className='line-height-md mb-4 text-center'>{text}</h5>}
+              {Scanner && (
                 <Scanner
                   formats={['qr_code']}
                   onScan={([{ rawValue: result }]) => {
@@ -1178,6 +1208,7 @@ function PasswordScanner ({ onScan, text }) {
                     onClose()
                   }}
                 />
+              )}
             </div>
           )
         })

components/link-to-context.module.css

@@ -31,6 +31,7 @@
 .linkBoxParent input,
 .linkBoxParent iframe,
 .linkBoxParent video,
+.linkBoxParent pre,
 .linkBoxParent img {
     pointer-events: auto !important;
 }

components/sub-analytics-header.js

@@ -0,0 +1,79 @@
+import { useRouter } from 'next/router'
+import { Select, DatePicker } from './form'
+import { useSubs } from './sub-select'
+import { WHENS } from '@/lib/constants'
+import { whenToFrom } from '@/lib/time'
+import styles from './sub-select.module.css'
+import classNames from 'classnames'
+
+export function SubAnalyticsHeader ({ pathname = null }) {
+  const router = useRouter()
+
+  const path = pathname || 'stackers'
+
+  const select = async values => {
+    const { sub, when, ...query } = values
+
+    if (when !== 'custom') { delete query.from; delete query.to }
+    if (query.from && !query.to) return
+
+    await router.push({
+
+      pathname: `/${path}/${sub}/${when}`,
+      query
+    })
+  }
+
+  const when = router.query.when || 'day'
+  const sub = router.query.sub || 'all'
+
+  const subs = useSubs({ prependSubs: ['all'], sub, appendSubs: [], filterSubs: () => true })
+
+  return (
+    <div className='text-muted fw-bold my-0 d-flex align-items-center flex-wrap'>
+      <div className='text-muted fw-bold mb-2 d-flex align-items-center'>
+        stacker analytics in
+        <Select
+          groupClassName='mb-0 mx-2'
+          className={classNames(styles.subSelect, styles.subSelectSmall)}
+          name='sub'
+          size='sm'
+          items={subs}
+          value={sub}
+          noForm
+          onChange={(formik, e) => {
+            const range = when === 'custom' ? { from: router.query.from, to: router.query.to } : {}
+            select({ sub: e.target.value, when, ...range })
+          }}
+        />
+        for
+        <Select
+          groupClassName='mb-0 mx-2'
+          className='w-auto'
+          name='when'
+          size='sm'
+          items={WHENS}
+          value={when}
+          noForm
+          onChange={(formik, e) => {
+            const range = e.target.value === 'custom' ? { from: whenToFrom(when), to: Date.now() } : {}
+            select({ sub, when: e.target.value, ...range })
+          }}
+        />
+      </div>
+      {when === 'custom' &&
+        <DatePicker
+          noForm
+          fromName='from'
+          toName='to'
+          className='p-0 px-2 mb-0'
+          onChange={(formik, [from, to], e) => {
+            select({ sub, when, from: from.getTime(), to: to.getTime() })
+          }}
+          from={router.query.from}
+          to={router.query.to}
+          when={when}
+        />}
+    </div>
+  )
+}

components/text.js

@@ -1,8 +1,7 @@
 import styles from './text.module.css'
 import ReactMarkdown from 'react-markdown'
 import gfm from 'remark-gfm'
-import { LightAsync as SyntaxHighlighter } from 'react-syntax-highlighter'
-import atomDark from 'react-syntax-highlighter/dist/cjs/styles/prism/atom-dark'
+import dynamic from 'next/dynamic'
 import React, { useState, memo, useRef, useCallback, useMemo, useEffect } from 'react'
 import MediaOrLink from './media-or-link'
 import { IMGPROXY_URL_REGEXP, decodeProxyUrl } from '@/lib/url'
@@ -21,7 +20,6 @@ import rehypeSN from '@/lib/rehype-sn'
 import remarkUnicode from '@/lib/remark-unicode'
 import Embed from './embed'
 import remarkMath from 'remark-math'
-import rehypeMathjax from 'rehype-mathjax'
 
 const rehypeSNStyled = () => rehypeSN({
   stylers: [{
@@ -36,7 +34,6 @@ const rehypeSNStyled = () => rehypeSN({
 })
 
 const remarkPlugins = [gfm, remarkUnicode, [remarkMath, { singleDollarTextMath: false }]]
-const rehypePlugins = [rehypeSNStyled, rehypeMathjax]
 
 export function SearchText ({ text }) {
   return (
@@ -52,16 +49,32 @@ export function SearchText ({ text }) {
 
 // this is one of the slowest components to render
 export default memo(function Text ({ rel = UNKNOWN_LINK_REL, imgproxyUrls, children, tab, itemId, outlawed, topLevel }) {
+  // would the text overflow on the current screen size?
   const [overflowing, setOverflowing] = useState(false)
-  const router = useRouter()
+  // should we show the full text?
   const [show, setShow] = useState(false)
   const containerRef = useRef(null)
 
+  const router = useRouter()
+  const [mathJaxPlugin, setMathJaxPlugin] = useState(null)
+
+  // we only need mathjax if there's math content between $$ tags
+  useEffect(() => {
+    if (/\$\$(.|\n)+\$\$/g.test(children)) {
+      import('rehype-mathjax').then(mod => {
+        setMathJaxPlugin(() => mod.default)
+      }).catch(err => {
+        console.error('error loading mathjax', err)
+        setMathJaxPlugin(null)
+      })
+    }
+  }, [children])
+
   // if we are navigating to a hash, show the full text
   useEffect(() => {
-    setShow(router.asPath.includes('#') && !router.asPath.includes('#itemfn-'))
+    setShow(router.asPath.includes('#'))
     const handleRouteChange = (url, { shallow }) => {
-      setShow(url.includes('#') && !url.includes('#itemfn-'))
+      setShow(url.includes('#'))
     }
 
     router.events.on('hashChangeStart', handleRouteChange)
@@ -134,12 +147,12 @@ export default memo(function Text ({ rel = UNKNOWN_LINK_REL, imgproxyUrls, child
     <ReactMarkdown
       components={components}
       remarkPlugins={remarkPlugins}
-      rehypePlugins={rehypePlugins}
+      rehypePlugins={[rehypeSNStyled, mathJaxPlugin].filter(Boolean)}
       remarkRehypeOptions={{ clobberPrefix: `itemfn-${itemId}-` }}
     >
       {children}
     </ReactMarkdown>
-  ), [components, remarkPlugins, rehypePlugins, children, itemId])
+  ), [components, remarkPlugins, mathJaxPlugin, children, itemId])
 
   const showOverflow = useCallback(() => setShow(true), [setShow])
 
@@ -228,17 +241,58 @@ function Table ({ node, ...props }) {
   )
 }
 
+// prevent layout shifting when the code block is loading
+function CodeSkeleton ({ className, children, ...props }) {
+  return (
+    <div className='rounded' style={{ padding: '0.5em' }}>
+      <code className={`${className}`} {...props}>
+        {children}
+      </code>
+    </div>
+  )
+}
+
 function Code ({ node, inline, className, children, style, ...props }) {
-  return inline
-    ? (
+  const [ReactSyntaxHighlighter, setReactSyntaxHighlighter] = useState(null)
+  const [syntaxTheme, setSyntaxTheme] = useState(null)
+  const language = className?.match(/language-(\w+)/)?.[1] || 'text'
+
+  const loadHighlighter = useCallback(() =>
+    Promise.all([
+      dynamic(() => import('react-syntax-highlighter').then(mod => mod.LightAsync), {
+        ssr: false,
+        loading: () => <CodeSkeleton className={className} {...props}>{children}</CodeSkeleton>
+      }),
+      import('react-syntax-highlighter/dist/cjs/styles/hljs/atom-one-dark').then(mod => mod.default)
+    ]), []
+  )
+
+  useEffect(() => {
+    if (!inline && language !== 'math') { // MathJax should handle math
+      // loading the syntax highlighter and theme only when needed
+      loadHighlighter().then(([highlighter, theme]) => {
+        setReactSyntaxHighlighter(() => highlighter)
+        setSyntaxTheme(() => theme)
+      })
+    }
+  }, [inline])
+
+  if (inline || !ReactSyntaxHighlighter) { // inline code doesn't have a border radius
+    return (
       <code className={className} {...props}>
         {children}
       </code>
     )
-    : (
-      <SyntaxHighlighter style={atomDark} language='text' PreTag='div' {...props}>
+  }
+
+  return (
+    <>
+      {ReactSyntaxHighlighter && syntaxTheme && (
+        <ReactSyntaxHighlighter style={syntaxTheme} language={language} PreTag='div' customStyle={{ borderRadius: '0.3rem' }} {...props}>
           {children}
-      </SyntaxHighlighter>
+        </ReactSyntaxHighlighter>
+      )}
+    </>
   )
 }
 

components/use-invoice.js

@@ -1,5 +1,5 @@
 import { useApolloClient, useMutation } from '@apollo/client'
-import { useCallback } from 'react'
+import { useCallback, useMemo } from 'react'
 import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors'
 import { RETRY_PAID_ACTION } from '@/fragments/paidAction'
 import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet'
@@ -42,9 +42,9 @@ export default function useInvoice () {
     return data.cancelInvoice
   }, [cancelInvoice])
 
-  const retry = useCallback(async ({ id, hash, hmac }, { update }) => {
+  const retry = useCallback(async ({ id, hash, hmac, newAttempt = false }, { update } = {}) => {
     console.log('retrying invoice:', hash)
-    const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id) }, update })
+    const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id), newAttempt }, update })
     if (error) throw error
 
     const newInvoice = data.retryPaidAction.invoice
@@ -53,5 +53,5 @@ export default function useInvoice () {
     return newInvoice
   }, [retryPaidAction])
 
-  return { cancel, retry, isInvoice }
+  return useMemo(() => ({ cancel, retry, isInvoice }), [cancel, retry, isInvoice])
 }

components/use-qr-payment.js

@@ -1,8 +1,9 @@
 import { useCallback } from 'react'
 import Invoice from '@/components/invoice'
-import { InvoiceCanceledError, InvoiceExpiredError } from '@/wallets/errors'
+import { InvoiceCanceledError, InvoiceExpiredError, AnonWalletError } from '@/wallets/errors'
 import { useShowModal } from '@/components/modal'
 import useInvoice from '@/components/use-invoice'
+import { sendPayment } from '@/wallets/webln/client'
 
 export default function useQrPayment () {
   const invoice = useInvoice()
@@ -16,6 +17,10 @@ export default function useQrPayment () {
       waitFor = inv => inv?.satsReceived > 0
     } = {}
   ) => {
+    // if anon user and webln is available, try to pay with webln
+    if (typeof window.webln !== 'undefined' && (walletError instanceof AnonWalletError)) {
+      sendPayment(inv.bolt11).catch(e => { console.error('WebLN payment failed:', e) })
+    }
     return await new Promise((resolve, reject) => {
       let paid
       const cancelAndReject = async (onClose) => {

components/user-analytics-header.js

@@ -3,10 +3,10 @@ import { Select, DatePicker } from './form'
 import { WHENS } from '@/lib/constants'
 import { whenToFrom } from '@/lib/time'
 
-export function UsageHeader ({ pathname = null }) {
+export function UserAnalyticsHeader ({ pathname = null }) {
   const router = useRouter()
 
-  const path = pathname || 'stackers'
+  const path = pathname || 'satistics/graph'
 
   const select = async values => {
     const { when, ...query } = values

contributors.txt

@@ -11,3 +11,5 @@ btcbagehot
 felipe
 benalleng
 rblb
+Scroogey
+SimpleStacker

docs/dev/extend-awards.md

@@ -0,0 +1,54 @@
+# Automatically extend awards.csv
+
+## Overview
+
+Whenever a pull request (PR) is merged in the [stacker.news](https://github.com/stackernews/stacker.news) repository, a [GitHub Action](https://docs.github.com/en/actions) is triggered:
+
+If the merged PR solves an issue with [award tags](https://github.com/stackernews/stacker.news?tab=readme-ov-file#contributing),
+the amounts due to the PR and issue authors are calculated and corresponding lines are added to the [awards.csv](https://github.com/stackernews/stacker.news/blob/master/awards.csv) file,
+and a PR is opened for this change.
+
+## Action
+
+The action is defined in [.github/workflows/extend-awards.yml](.github/workflows/extend-awards.yml).
+
+Filters on the event type and parameters ensure the action is [triggered only on merged PRs](https://stackoverflow.com/questions/60710209/trigger-github-actions-only-when-pr-is-merged).
+
+The primary job consists of several steps:
+  - [checkout](https://github.com/actions/checkout) checks out the repository
+  - [setup-python](https://github.com/actions/setup-python) installs [Python](https://en.wikipedia.org/wiki/Python_(programming_language))
+  - [pip](https://en.wikipedia.org/wiki/Pip_%28package_manager%29) installs the [requests](https://docs.python-requests.org/en/latest/index.html) module
+  - a script (see below) is executed, which appends lines to [awards.csv](awards.csv) if needed
+  - [create-pull-request](https://github.com/peter-evans/create-pull-request) looks for modified files and creates (or updates) a PR
+
+## Script
+
+The script is [extend-awards.py](extend-awards.py).
+
+The script extracts from the [environment](https://en.wikipedia.org/wiki/Environment_variable) an authentication token needed for the [GitHub REST API](https://docs.github.com/en/rest/about-the-rest-api/about-the-rest-api) and the [context](https://docs.github.com/en/actions/writing-workflows/choosing-what-your-workflow-does/accessing-contextual-information-about-workflow-runs) containing the event details including the merged PR (formatted in [JSON](https://en.wikipedia.org/wiki/JSON)).
+
+In the merged PR's title and body it searches for the first [GitHub issue URL](https://github.com/stackernews/stacker.news/issues/) or any number with a hash symbol (#) prefix, and takes this as the issue being solved by the PR.
+
+Using the GitHub REST API it fetches the issue and analyzes its tags for difficulty and priority.
+
+It fetches the issue's timeline and counts the number of reviews completed with status 'changes requested' to calculate the amount reduction.
+
+It calculates the amounts due to the PR author and the issue author.
+
+It reads the existing awards.csv file to suppress appending redundant lines (same user, PR, and issue) and fill known receive methods (same user).
+
+Finally, it appends zero, one, or two lines to the awards.csv file.
+
+## Diagnostics
+
+In the GitHub web interface under 'Actions' each invokation of the action can be viewed, including environment and [output and errors](https://en.wikipedia.org/wiki/Standard_streams) of the script. First, the specific invokation is selected, then the job 'if_merged', then the step 'Run python extend-awards.py'. The environment is found by expanding the inner 'Run python extended-awards.py' on the first line.
+
+The normal output includes details about the issue number found, the amount calculation, or the reason for not appending lines.
+
+The error output may include a [Python traceback](https://realpython.com/python-traceback/) which helps to explain the error.
+
+The environment contains in GITHUB_CONTEXT the event details, which may be required to understand the error.
+
+## Security considerations
+
+The create-pull-request step requires [workflow permissions](https://github.com/peter-evans/create-pull-request#workflow-permissions).

extend-awards.py

@@ -0,0 +1,104 @@
+import json, os, re, requests
+
+difficulties = {'good-first-issue':20000,'easy':100000,'medium':250000,'medium-hard':500000,'hard':1000000}
+priorities = {'low':0.5,'medium':1.5,'high':2,'urgent':3}
+ignored = ['huumn', 'ekzyis']
+fn = 'awards.csv'
+
+sess = requests.Session()
+headers = {'Authorization':'Bearer %s' % os.getenv('GITHUB_TOKEN') }
+awards = []
+
+def getIssue(n):
+	url = 'https://api.github.com/repos/stackernews/stacker.news/issues/' + n
+	r = sess.get(url, headers=headers)
+	j = json.loads(r.text)
+	return j
+
+def findIssueInPR(j):
+	p = re.compile('(#|https://github.com/stackernews/stacker.news/issues/)([0-9]+)')
+	for m in p.finditer(j['title']):
+		return m.group(2)
+	if not 'body' in j or j['body'] is None:
+		return
+	for s in j['body'].split('\n'):
+		for m in p.finditer(s):
+			return m.group(2)
+
+def addAward(user, kind, pr, issue, difficulty, priority, count, amount):
+	if amount >= 1000000 and amount % 1000000 == 0:
+		amount = str(int(amount / 1000000)) + 'm'
+	elif amount >= 1000 and amount % 1000 == 0:
+		amount = str(int(amount / 1000)) + 'k'
+	for a in awards:
+		if a[0] == user and a[1] == kind and a[2] == pr:
+			print('found existing entry %s' % a)
+			if a[8] != amount:
+				print('warning: amount %s != %s' % (a[8], amount))
+			return
+	if count < 1:
+		count = ''
+	addr = '???'
+	for a in awards:
+		if a[0] == user and a[9] != '???':
+			addr = a[9]
+	print('adding %s,%s,%s,%s,%s,%s,%s,,%s,%s,???' % (user, kind, pr, issue, difficulty, priority, count, amount, addr))
+	with open(fn, 'a') as f:
+		print('%s,%s,%s,%s,%s,%s,%s,,%s,%s,???' % (user, kind, pr, issue, difficulty, priority, count, amount, addr), file=f)
+
+def countReviews(pr):
+	url = 'https://api.github.com/repos/stackernews/stacker.news/issues/%s/timeline' % pr
+	r = sess.get(url, headers=headers)
+	j = json.loads(r.text)
+	count = 0
+	for e in j:
+		if e['event'] == 'reviewed' and e['state'] == 'changes_requested':
+			count += 1
+	return count
+
+def checkPR(i):
+	pr = str(i['number'])
+	print('pr %s' % pr)
+	n = findIssueInPR(i)
+	if not n:
+		print('pr %s does not solve an issue' % pr)
+		return
+	print('solves issue %s' % n)
+	j = getIssue(n)
+	difficulty = ''
+	amount = 0
+	priority = ''
+	multiplier = 1
+	for l in j['labels']:
+		for d in difficulties:
+			if l['name'] == 'difficulty:' + d:
+				difficulty = d
+				amount = difficulties[d]
+		for p in priorities:
+			if l['name'] == 'priority:' + p:
+				priority = p
+				multiplier = priorities[p]
+	if amount * multiplier <= 0:
+		print('issue gives no award')
+		return
+	count = countReviews(pr)
+	if count >= 10:
+		print('too many reviews, no award')
+		return
+	if count > 0:
+		print('%d reviews, %d%% reduction' % (count, count * 10))
+	award = amount * multiplier * (10 - count) / 10
+	print('award is %d' % award)
+	if i['user']['login'] not in ignored:
+		addAward(i['user']['login'], 'pr', '#' + pr, '#' + n, difficulty, priority, count, award)
+	if j['user']['login'] not in ignored:
+		count = 0
+		addAward(j['user']['login'], 'issue', '#' + pr, '#' + n, difficulty, priority, count, int(award / 10))
+
+with open(fn, 'r') as f:
+	for s in f:
+		s = s.split('\n')[0]
+		awards.append(s.split(','))
+
+j = json.loads(os.getenv('GITHUB_CONTEXT'))
+checkPR(j['event']['pull_request'])

fragments/paidAction.js

@@ -91,8 +91,8 @@ export const RETRY_PAID_ACTION = gql`
   ${PAID_ACTION}
   ${ITEM_PAID_ACTION_FIELDS}
   ${ITEM_ACT_PAID_ACTION_FIELDS}
-  mutation retryPaidAction($invoiceId: Int!) {
-    retryPaidAction(invoiceId: $invoiceId) {
+  mutation retryPaidAction($invoiceId: Int!, $newAttempt: Boolean) {
+    retryPaidAction(invoiceId: $invoiceId, newAttempt: $newAttempt) {
       __typename
       ...PaidActionFields
       ... on ItemPaidAction {

fragments/wallet.js

@@ -231,3 +231,12 @@ export const CANCEL_INVOICE = gql`
     }
   }
 `
+
+export const FAILED_INVOICES = gql`
+  ${INVOICE_FIELDS}
+  query FailedInvoices {
+    failedInvoices {
+      ...InvoiceFields
+    }
+  }
+`

lib/apollo.js

@@ -283,6 +283,12 @@ function getClient (uri) {
                   facts: [...(existing?.facts || []), ...incoming.facts]
                 }
               }
+            },
+            failedInvoices: {
+              keyArgs: [],
+              merge (existing, incoming) {
+                return incoming
+              }
             }
           }
         },

lib/constants.js

@@ -2,6 +2,7 @@
 // to be loaded from the server
 export const DEFAULT_SUBS = ['bitcoin', 'nostr', 'tech', 'meta', 'jobs']
 export const DEFAULT_SUBS_NO_JOBS = DEFAULT_SUBS.filter(s => s !== 'jobs')
+export const RESERVED_SUB_NAMES = ['all', 'home']
 
 export const PAID_ACTION_PAYMENT_METHODS = {
   FEE_CREDIT: 'FEE_CREDIT',
@@ -12,7 +13,7 @@ export const PAID_ACTION_PAYMENT_METHODS = {
   REWARD_SATS: 'REWARD_SATS'
 }
 export const PAID_ACTION_TERMINAL_STATES = ['FAILED', 'PAID', 'RETRYING']
-export const NOFOLLOW_LIMIT = 1000
+export const NOFOLLOW_LIMIT = 250
 export const UNKNOWN_LINK_REL = 'noreferrer nofollow noopener'
 export const UPLOAD_SIZE_MAX = 50 * 1024 * 1024
 export const UPLOAD_SIZE_MAX_AVATAR = 5 * 1024 * 1024
@@ -198,4 +199,14 @@ export const ZAP_UNDO_DELAY_MS = 5_000
 export const WALLET_SEND_PAYMENT_TIMEOUT_MS = 150_000
 export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000
 
+// interval between which failed invoices are returned to a client for automated retries.
+// retry-after must be high enough such that intermediate failed invoices that will already
+// be retried by the client due to sender or receiver fallbacks are not returned to the client.
+export const WALLET_RETRY_AFTER_MS = 60_000 // 1 minute
+export const WALLET_RETRY_BEFORE_MS = 3_600_000 // 1 hour
+// we want to attempt a payment three times so we retry two times
+export const WALLET_MAX_RETRIES = 2
+// when a pending retry for an invoice should be considered expired and can be attempted again
+export const WALLET_RETRY_TIMEOUT_MS = 60_000 // 1 minute
+
 export const BECH32_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l'

lib/rehype-sn.js

@@ -59,10 +59,10 @@ export default function rehypeSN (options = {}) {
             if (node.properties.href.includes('#itemfn-')) {
               node.tagName = 'footnote'
             } else {
-              const { itemId, linkText } = parseInternalLinks(node.properties.href)
-              if (itemId) {
+              const { itemId, commentId, linkText } = parseInternalLinks(node.properties.href)
+              if (itemId || commentId) {
                 node.tagName = 'item'
-                node.properties.id = itemId
+                node.properties.id = commentId || itemId
                 if (node.properties.href === toString(node)) {
                   node.children[0].value = linkText
                 }

lib/url.js

@@ -93,7 +93,7 @@ export function parseEmbedUrl (href) {
     const { hostname, pathname, searchParams } = new URL(href)
 
     // nostr prefixes: [npub1, nevent1, nprofile1, note1]
-    const nostr = href.match(/(?<id>(?<type>npub1|nevent1|nprofile1|note1|naddr1)[02-9ac-hj-np-z]+)/)
+    const nostr = href.match(/\/(?<id>(?<type>npub1|nevent1|nprofile1|note1|naddr1)[02-9ac-hj-np-z]+)/)
     if (nostr?.groups?.id) {
       let id = nostr.groups.id
       if (nostr.groups.type === 'npub1') {

lib/validate.js

@@ -2,7 +2,8 @@ import { string, ValidationError, number, object, array, boolean, date } from '.
 import {
   BOOST_MIN, MAX_POLL_CHOICE_LENGTH, MAX_TITLE_LENGTH, MAX_POLL_NUM_CHOICES,
   MIN_POLL_NUM_CHOICES, MAX_FORWARDS, BOOST_MULT, MAX_TERRITORY_DESC_LENGTH, POST_TYPES,
-  TERRITORY_BILLING_TYPES, MAX_COMMENT_TEXT_LENGTH, MAX_POST_TEXT_LENGTH, MIN_TITLE_LENGTH, BOUNTY_MIN, BOUNTY_MAX
+  TERRITORY_BILLING_TYPES, MAX_COMMENT_TEXT_LENGTH, MAX_POST_TEXT_LENGTH, MIN_TITLE_LENGTH, BOUNTY_MIN, BOUNTY_MAX,
+  RESERVED_SUB_NAMES
 } from './constants'
 import { SUPPORTED_CURRENCIES } from './currency'
 import { NOSTR_MAX_RELAY_NUM, NOSTR_PUBKEY_BECH32, NOSTR_PUBKEY_HEX } from './nostr'
@@ -306,7 +307,7 @@ export function territorySchema (args) {
           const isArchived = sub => sub.status === 'STOPPED'
           const filter = sub => editing ? !isEdit(sub) : !isArchived(sub)
           const exists = await subExists(name, { ...args, filter })
-          return !exists
+          return !exists & !RESERVED_SUB_NAMES.includes(name)
         },
         message: 'taken'
       }),

next.config.js

@@ -189,6 +189,11 @@ module.exports = withPlausibleProxy()({
         source: '/statistics',
         destination: '/satistics?inc=invoice,withdrawal',
         permanent: true
+      },
+      {
+        source: '/top/cowboys/:when',
+        destination: '/top/cowboys',
+        permanent: true
       }
     ]
   },

pages/email.js

@@ -22,7 +22,7 @@ export default function Email () {
     const params = new URLSearchParams()
     if (callback.callbackUrl) params.set('callbackUrl', callback.callbackUrl)
     params.set('token', token)
-    params.set('email', callback.email)
+    params.set('email', callback.email.toLowerCase())
     const url = `/api/auth/callback/email?${params.toString()}`
     router.push(url)
   }, [callback, router])

pages/satistics/graphs/[when].js

@@ -6,7 +6,7 @@ import { useRouter } from 'next/router'
 import PageLoading from '@/components/page-loading'
 import dynamic from 'next/dynamic'
 import { numWithUnits } from '@/lib/format'
-import { UsageHeader } from '@/components/usage-header'
+import { UserAnalyticsHeader } from '@/components/user-analytics-header'
 import { SatisticsHeader } from '..'
 import { WhenComposedChartSkeleton, WhenAreaChartSkeleton } from '@/components/charts-skeletons'
 import OverlayTrigger from 'react-bootstrap/OverlayTrigger'
@@ -55,7 +55,7 @@ export default function Satistics ({ ssrData }) {
         <SatisticsHeader />
         <div className='tab-content' id='myTabContent'>
           <div className='tab-pane fade show active text-muted' id='statistics' role='tabpanel' aria-labelledby='statistics-tab'>
-            <UsageHeader pathname='satistics/graphs' />
+            <UserAnalyticsHeader pathname='satistics/graphs' />
             <div className='mt-3'>
               <div className='d-flex row justify-content-between'>
                 <div className='col-md-6 mb-2'>

pages/stackers/[sub]/[when].js

@@ -0,0 +1,157 @@
+import { gql, useQuery } from '@apollo/client'
+import { getGetServerSideProps } from '@/api/ssrApollo'
+import Layout from '@/components/layout'
+import Col from 'react-bootstrap/Col'
+import Row from 'react-bootstrap/Row'
+import { SubAnalyticsHeader } from '@/components/sub-analytics-header'
+import { useRouter } from 'next/router'
+import dynamic from 'next/dynamic'
+import PageLoading from '@/components/page-loading'
+import { WhenAreaChartSkeleton, WhenComposedChartSkeleton, WhenLineChartSkeleton } from '@/components/charts-skeletons'
+
+const WhenAreaChart = dynamic(() => import('@/components/charts').then(mod => mod.WhenAreaChart), {
+  loading: () => <WhenAreaChartSkeleton />
+})
+const WhenLineChart = dynamic(() => import('@/components/charts').then(mod => mod.WhenLineChart), {
+  loading: () => <WhenLineChartSkeleton />
+})
+const WhenComposedChart = dynamic(() => import('@/components/charts').then(mod => mod.WhenComposedChart), {
+  loading: () => <WhenComposedChartSkeleton />
+})
+
+const GROWTH_QUERY = gql`
+  query Growth($when: String!, $from: String, $to: String, $sub: String, $subSelect: Boolean = false)
+  {
+    registrationGrowth(when: $when, from: $from, to: $to) @skip(if: $subSelect) {
+      time
+      data {
+        name
+        value
+      }
+    }
+    itemGrowth(when: $when, from: $from, to: $to) @skip(if: $subSelect) {
+      time
+      data {
+        name
+        value
+      }
+    }
+    spendingGrowth(when: $when, from: $from, to: $to) @skip(if: $subSelect) {
+      time
+      data {
+        name
+        value
+      }
+    }
+    spenderGrowth(when: $when, from: $from, to: $to) @skip(if: $subSelect) {
+      time
+      data {
+        name
+        value
+      }
+    }
+    stackingGrowth(when: $when, from: $from, to: $to) @skip(if: $subSelect) {
+      time
+      data {
+        name
+        value
+      }
+    }
+    stackerGrowth(when: $when, from: $from, to: $to) @skip(if: $subSelect) {
+      time
+      data {
+        name
+        value
+      }
+    }
+    itemGrowthSubs(when: $when, from: $from, to: $to, sub: $sub) @include(if: $subSelect) {
+      time
+      data {
+        name
+        value
+      }
+    }
+    revenueGrowthSubs(when: $when, from: $from, to: $to, sub: $sub) @include(if: $subSelect) {
+      time
+      data {
+        name
+        value
+      }
+    }
+  }`
+
+const variablesFunc = vars => ({ ...vars, subSelect: vars.sub !== 'all' })
+export const getServerSideProps = getGetServerSideProps({ query: GROWTH_QUERY, variables: variablesFunc })
+
+export default function Growth ({ ssrData }) {
+  const router = useRouter()
+  const { when, from, to, sub } = router.query
+
+  const { data } = useQuery(GROWTH_QUERY, { variables: { when, from, to, sub, subSelect: sub !== 'all' } })
+  if (!data && !ssrData) return <PageLoading />
+
+  const {
+    registrationGrowth,
+    itemGrowth,
+    spendingGrowth,
+    spenderGrowth,
+    stackingGrowth,
+    stackerGrowth,
+    itemGrowthSubs,
+    revenueGrowthSubs
+  } = data || ssrData
+
+  if (sub === 'all') {
+    return (
+      <Layout>
+        <SubAnalyticsHeader />
+        <Row>
+          <Col className='mt-3'>
+            <div className='text-center text-muted fw-bold'>stackers</div>
+            <WhenLineChart data={stackerGrowth} />
+          </Col>
+          <Col className='mt-3'>
+            <div className='text-center text-muted fw-bold'>stacking</div>
+            <WhenAreaChart data={stackingGrowth} />
+          </Col>
+        </Row>
+        <Row>
+          <Col className='mt-3'>
+            <div className='text-center text-muted fw-bold'>spenders</div>
+            <WhenLineChart data={spenderGrowth} />
+          </Col>
+          <Col className='mt-3'>
+            <div className='text-center text-muted fw-bold'>spending</div>
+            <WhenAreaChart data={spendingGrowth} />
+          </Col>
+        </Row>
+        <Row>
+          <Col className='mt-3'>
+            <div className='text-center text-muted fw-bold'>registrations</div>
+            <WhenAreaChart data={registrationGrowth} />
+          </Col>
+          <Col className='mt-3'>
+            <div className='text-center text-muted fw-bold'>items</div>
+            <WhenComposedChart data={itemGrowth} areaNames={['posts', 'comments', 'jobs']} areaAxis='left' lineNames={['comments/posts', 'territories']} lineAxis='right' barNames={['zaps']} />
+          </Col>
+        </Row>
+      </Layout>
+    )
+  } else {
+    return (
+      <Layout>
+        <SubAnalyticsHeader />
+        <Row>
+          <Col className='mt-3'>
+            <div className='text-center text-muted fw-bold'>items</div>
+            <WhenLineChart data={itemGrowthSubs} />
+          </Col>
+          <Col className='mt-3'>
+            <div className='text-center text-muted fw-bold'>sats</div>
+            <WhenLineChart data={revenueGrowthSubs} />
+          </Col>
+        </Row>
+      </Layout>
+    )
+  }
+}

pages/stackers/[when].js

@@ -1,115 +0,0 @@
-import { gql, useQuery } from '@apollo/client'
-import { getGetServerSideProps } from '@/api/ssrApollo'
-import Layout from '@/components/layout'
-import Col from 'react-bootstrap/Col'
-import Row from 'react-bootstrap/Row'
-import { UsageHeader } from '@/components/usage-header'
-import { useRouter } from 'next/router'
-import dynamic from 'next/dynamic'
-import PageLoading from '@/components/page-loading'
-import { WhenAreaChartSkeleton, WhenComposedChartSkeleton, WhenLineChartSkeleton } from '@/components/charts-skeletons'
-
-const WhenAreaChart = dynamic(() => import('@/components/charts').then(mod => mod.WhenAreaChart), {
-  loading: () => <WhenAreaChartSkeleton />
-})
-const WhenLineChart = dynamic(() => import('@/components/charts').then(mod => mod.WhenLineChart), {
-  loading: () => <WhenLineChartSkeleton />
-})
-const WhenComposedChart = dynamic(() => import('@/components/charts').then(mod => mod.WhenComposedChart), {
-  loading: () => <WhenComposedChartSkeleton />
-})
-
-const GROWTH_QUERY = gql`
-  query Growth($when: String!, $from: String, $to: String)
-  {
-    registrationGrowth(when: $when, from: $from, to: $to) {
-      time
-      data {
-        name
-        value
-      }
-    }
-    itemGrowth(when: $when, from: $from, to: $to) {
-      time
-      data {
-        name
-        value
-      }
-    }
-    spendingGrowth(when: $when, from: $from, to: $to) {
-      time
-      data {
-        name
-        value
-      }
-    }
-    spenderGrowth(when: $when, from: $from, to: $to) {
-      time
-      data {
-        name
-        value
-      }
-    }
-    stackingGrowth(when: $when, from: $from, to: $to) {
-      time
-      data {
-        name
-        value
-      }
-    }
-    stackerGrowth(when: $when, from: $from, to: $to) {
-      time
-      data {
-        name
-        value
-      }
-    }
-  }`
-
-export const getServerSideProps = getGetServerSideProps({ query: GROWTH_QUERY })
-
-export default function Growth ({ ssrData }) {
-  const router = useRouter()
-  const { when, from, to } = router.query
-
-  const { data } = useQuery(GROWTH_QUERY, { variables: { when, from, to } })
-  if (!data && !ssrData) return <PageLoading />
-
-  const { registrationGrowth, itemGrowth, spendingGrowth, spenderGrowth, stackingGrowth, stackerGrowth } = data || ssrData
-
-  return (
-    <Layout>
-      <UsageHeader />
-      <Row>
-        <Col className='mt-3'>
-          <div className='text-center text-muted fw-bold'>stackers</div>
-          <WhenLineChart data={stackerGrowth} />
-        </Col>
-        <Col className='mt-3'>
-          <div className='text-center text-muted fw-bold'>stacking</div>
-          <WhenAreaChart data={stackingGrowth} />
-        </Col>
-      </Row>
-      <Row>
-        <Col className='mt-3'>
-          <div className='text-center text-muted fw-bold'>spenders</div>
-          <WhenLineChart data={spenderGrowth} />
-        </Col>
-        <Col className='mt-3'>
-          <div className='text-center text-muted fw-bold'>spending</div>
-          <WhenAreaChart data={spendingGrowth} />
-        </Col>
-      </Row>
-      <Row>
-        <Col className='mt-3'>
-          <div className='text-center text-muted fw-bold'>registrations</div>
-          <WhenAreaChart data={registrationGrowth} />
-        </Col>
-        <Col className='mt-3'>
-          <div className='text-center text-muted fw-bold'>items</div>
-          <WhenComposedChart data={itemGrowth} areaNames={['posts', 'comments', 'jobs']} areaAxis='left' lineNames={['comments/posts', 'territories']} lineAxis='right' barNames={['zaps']} />
-        </Col>
-      </Row>
-    </Layout>
-  )
-}

pages/withdraw.js

@@ -15,7 +15,6 @@ import { lnAddrSchema, withdrawlSchema } from '@/lib/validate'
 import { useShowModal } from '@/components/modal'
 import { useField } from 'formik'
 import { useToast } from '@/components/toast'
-import { Scanner } from '@yudiel/react-qr-scanner'
 import { decode } from 'bolt11'
 import CameraIcon from '@/svgs/camera-line.svg'
 import { FAST_POLL_INTERVAL, SSR } from '@/lib/constants'
@@ -24,6 +23,8 @@ import useDebounceCallback from '@/components/use-debounce-callback'
 import { lnAddrOptions } from '@/lib/lnurl'
 import AccordianItem from '@/components/accordian-item'
 import { numWithUnits } from '@/lib/format'
+import PageLoading from '@/components/page-loading'
+import dynamic from 'next/dynamic'
 
 export const getServerSideProps = getGetServerSideProps({ authRequired: true })
 
@@ -153,12 +154,19 @@ function InvoiceScanner ({ fieldName }) {
   const showModal = useShowModal()
   const [,, helpers] = useField(fieldName)
   const toaster = useToast()
+  const Scanner = dynamic(() => import('@yudiel/react-qr-scanner').then(mod => mod.Scanner), {
+    ssr: false,
+    loading: () => <PageLoading />
+  })
+
   return (
     <InputGroup.Text
       style={{ cursor: 'pointer' }}
       onClick={() => {
         showModal(onClose => {
           return (
+            <>
+              {Scanner && (
                 <Scanner
                   formats={['qr_code']}
                   onScan={([{ rawValue: result }]) => {
@@ -185,7 +193,8 @@ function InvoiceScanner ({ fieldName }) {
                     }
                     onClose()
                   }}
-            />
+                />)}
+            </>
           )
         })
       }}

pages/~/top/cowboys.js

@@ -6,6 +6,7 @@ import UserList from '@/components/user-list'
 
 export const getServerSideProps = getGetServerSideProps({ query: TOP_COWBOYS })
 
+// if a when descriptor is provided, it redirects here; see next.config.js
 export default function Index ({ ssrData }) {
   return (
     <Layout>

prisma/migrations/20250107084543_automated_retries/migration.sql

@@ -0,0 +1,4 @@
+-- AlterTable
+ALTER TABLE "Invoice" ADD COLUMN     "paymentAttempt" INTEGER NOT NULL DEFAULT 0;
+ALTER TABLE "Invoice" ADD COLUMN     "retryPendingSince" TIMESTAMP(3);
+CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt");

prisma/schema.prisma

@@ -928,6 +928,8 @@ model Invoice {
   cancelled          Boolean             @default(false)
   cancelledAt        DateTime?
   userCancel         Boolean?
+  paymentAttempt     Int                 @default(0)
+  retryPendingSince  DateTime?
   msatsRequested     BigInt
   msatsReceived      BigInt?
   desc               String?
@@ -956,6 +958,7 @@ model Invoice {
   @@index([confirmedIndex], map: "Invoice.confirmedIndex_index")
   @@index([isHeld])
   @@index([confirmedAt])
+  @@index([cancelledAt])
   @@index([actionType])
   @@index([actionState])
 }

scripts/nostr-link-extract.js

@@ -0,0 +1,543 @@
+const WebSocket = require('ws') // You might need to install this: npm install ws
+const { nip19 } = require('nostr-tools') // Keep this for formatting
+const fs = require('fs')
+const path = require('path')
+
+// ANSI color codes
+const colors = {
+  reset: '\x1b[0m',
+  bright: '\x1b[1m',
+  dim: '\x1b[2m',
+  underscore: '\x1b[4m',
+  blink: '\x1b[5m',
+  reverse: '\x1b[7m',
+  hidden: '\x1b[8m',
+
+  fg: {
+    black: '\x1b[30m',
+    red: '\x1b[31m',
+    green: '\x1b[32m',
+    yellow: '\x1b[33m',
+    blue: '\x1b[34m',
+    magenta: '\x1b[35m',
+    cyan: '\x1b[36m',
+    white: '\x1b[37m',
+    gray: '\x1b[90m',
+    crimson: '\x1b[38m'
+  },
+  bg: {
+    black: '\x1b[40m',
+    red: '\x1b[41m',
+    green: '\x1b[42m',
+    yellow: '\x1b[43m',
+    blue: '\x1b[44m',
+    magenta: '\x1b[45m',
+    cyan: '\x1b[46m',
+    white: '\x1b[47m',
+    gray: '\x1b[100m',
+    crimson: '\x1b[48m'
+  }
+}
+
+// Default configuration
+let config = {
+  userPubkeys: [],
+  ignorePubkeys: [],
+  timeIntervalHours: 12,
+  verbosity: 'normal', // Can be 'minimal', 'normal', or 'debug'
+  relayUrls: [
+    'wss://relay.nostr.band',
+    'wss://relay.primal.net',
+    'wss://relay.damus.io'
+  ],
+  batchSize: 100,
+  mediaPatterns: [
+    {
+      type: 'extensions',
+      patterns: ['\\.jpg$', '\\.jpeg$', '\\.png$', '\\.gif$', '\\.bmp$', '\\.webp$', '\\.tiff$', '\\.ico$',
+        '\\.mp4$', '\\.webm$', '\\.mov$', '\\.avi$', '\\.mkv$', '\\.flv$', '\\.wmv$',
+        '\\.mp3$', '\\.wav$', '\\.ogg$', '\\.flac$', '\\.aac$', '\\.m4a$']
+    },
+    {
+      type: 'domains',
+      patterns: [
+        'nostr\\.build\\/[ai]\\/\\w+',
+        'i\\.imgur\\.com\\/\\w+',
+        'i\\.ibb\\.co\\/\\w+\\/',
+        'tenor\\.com\\/view\\/',
+        'giphy\\.com\\/gifs\\/',
+        'soundcloud\\.com\\/',
+        'spotify\\.com\\/',
+        'fountain\\.fm\\/'
+      ]
+    }
+  ]
+}
+
+/**
+ * Logger utility that respects the configured verbosity level
+ */
+const logger = {
+  // Always show error messages
+  error: (message) => {
+    console.error(`${colors.fg.red}Error: ${message}${colors.reset}`)
+  },
+
+  // Minimal essential info - always show regardless of verbosity
+  info: (message) => {
+    console.log(`${colors.fg.green}${message}${colors.reset}`)
+  },
+
+  // Progress updates - show in normal and debug modes
+  progress: (message) => {
+    if (config.verbosity !== 'minimal') {
+      console.log(`${colors.fg.blue}${message}${colors.reset}`)
+    }
+  },
+
+  // Detailed debug info - only show in debug mode
+  debug: (message) => {
+    if (config.verbosity === 'debug') {
+      console.log(`${colors.fg.gray}${message}${colors.reset}`)
+    }
+  },
+
+  // Results info - formatted differently for clarity
+  result: (message) => {
+    console.log(`${colors.bright}${colors.fg.green}${message}${colors.reset}`)
+  }
+}
+
+/**
+ * Load configuration from a JSON file
+ * @param {String} configPath - Path to the config file
+ * @returns {Object} - Configuration object
+ */
+function loadConfig (configPath) {
+  try {
+    const configData = fs.readFileSync(configPath, 'utf8')
+    const loadedConfig = JSON.parse(configData)
+
+    // Merge with default config to ensure all properties exist
+    return { ...config, ...loadedConfig }
+  } catch (error) {
+    logger.error(`Error loading config file: ${error.message}`)
+    logger.info('Using default configuration')
+    return config
+  }
+}
+
+/**
+ * Checks if a URL is a media file or hosted on a media platform based on configured patterns
+ * @param {String} url - URL to check
+ * @returns {Boolean} - true if it's likely a media URL
+ */
+function isMediaUrl (url) {
+  // Check for media patterns from config
+  if (config.mediaPatterns) {
+    for (const patternGroup of config.mediaPatterns) {
+      for (const pattern of patternGroup.patterns) {
+        const regex = new RegExp(pattern, 'i')
+        if (regex.test(url)) return true
+      }
+    }
+  }
+
+  return false
+}
+
+/**
+ * Fetches events from Nostr relays using WebSockets
+ * @param {Array} relayUrls - Array of relay URLs
+ * @param {Object} filter - Nostr filter object
+ * @param {Number} timeoutMs - Timeout in milliseconds
+ * @returns {Promise<Array>} - Array of events matching the filter
+ */
+async function fetchEvents (relayUrls, filter, timeoutMs = 10000) {
+  logger.debug(`Fetching events with filter: ${JSON.stringify(filter)}`)
+  const events = []
+
+  for (const url of relayUrls) {
+    try {
+      const ws = new WebSocket(url)
+
+      const relayEvents = await new Promise((resolve, reject) => {
+        const timeout = setTimeout(() => {
+          ws.close()
+          resolve([]) // Resolve with empty array on timeout
+        }, timeoutMs)
+
+        const localEvents = []
+
+        ws.on('open', () => {
+          // Create a unique request ID
+          const requestId = `req${Math.floor(Math.random() * 10000)}`
+
+          // Format and send the request
+          const request = JSON.stringify(['REQ', requestId, filter])
+          ws.send(request)
+
+          ws.on('message', (data) => {
+            try {
+              const message = JSON.parse(data.toString())
+
+              // Check if it's an EVENT message
+              if (message[0] === 'EVENT' && message[2]) {
+                localEvents.push(message[2])
+              } else if (message[0] === 'EOSE') {
+                clearTimeout(timeout)
+                ws.close()
+                resolve(localEvents)
+              }
+            } catch (error) {
+              logger.debug(`Error parsing message: ${error.message}`)
+            }
+          })
+        })
+
+        ws.on('error', (error) => {
+          logger.debug(`WebSocket error for ${url}: ${error.message}`)
+          clearTimeout(timeout)
+          resolve([]) // Resolve with empty array on error
+        })
+
+        ws.on('close', () => {
+          clearTimeout(timeout)
+          resolve(localEvents)
+        })
+      })
+
+      logger.debug(`Got ${relayEvents.length} events from ${url}`)
+      events.push(...relayEvents)
+    } catch (error) {
+      logger.debug(`Error connecting to ${url}: ${error.message}`)
+    }
+  }
+
+  // Remove duplicates based on event ID
+  const uniqueEvents = {}
+  events.forEach(event => {
+    if (!uniqueEvents[event.id]) {
+      uniqueEvents[event.id] = event
+    }
+  })
+
+  return Object.values(uniqueEvents)
+}
+
+/**
+ * Get Nostr notes from followings of specified users that contain external links
+ * and were posted within the specified time interval.
+ *
+ * @param {Array} userPubkeys - Array of Nostr user public keys
+ * @param {Number} timeIntervalHours - Number of hours to look back from now
+ * @param {Array} relayUrls - Array of Nostr relay URLs
+ * @param {Array} ignorePubkeys - Array of pubkeys to ignore (optional)
+ * @returns {Promise<Array>} - Array of note objects containing external links within the time interval
+ */
+async function getNotesWithLinks (userPubkeys, timeIntervalHours, relayUrls, ignorePubkeys = []) {
+  // Calculate the cutoff time in seconds (Nostr uses UNIX timestamp)
+  const now = Math.floor(Date.now() / 1000)
+  const cutoffTime = now - (timeIntervalHours * 60 * 60)
+
+  const allNotesWithLinks = []
+  const allFollowedPubkeys = new Set() // To collect all followed pubkeys
+  const ignoreSet = new Set(ignorePubkeys) // Convert ignore list to Set for efficient lookups
+
+  if (ignoreSet.size > 0) {
+    logger.debug(`Ignoring ${ignoreSet.size} author(s) as requested`)
+  }
+
+  logger.info(`Fetching follow lists for ${userPubkeys.length} users...`)
+  // First get the followings for each user
+  for (const pubkey of userPubkeys) {
+    try {
+      // Skip if this pubkey is in the ignore list
+      if (ignoreSet.has(pubkey)) {
+        logger.debug(`Skipping user ${pubkey} as it's in the ignore list`)
+        continue
+      }
+
+      logger.debug(`Fetching follow list for ${pubkey} from ${relayUrls.length} relays...`)
+
+      // Get the most recent contact list (kind 3)
+      const followListEvents = await fetchEvents(relayUrls, {
+        kinds: [3],
+        authors: [pubkey]
+      })
+
+      if (followListEvents.length === 0) {
+        logger.debug(`No follow list found for user ${pubkey}. Verify this pubkey has contacts on these relays.`)
+        continue
+      }
+
+      // Find the most recent follow list event
+      const latestFollowList = followListEvents.reduce((latest, event) =>
+        !latest || event.created_at > latest.created_at ? event : latest, null)
+
+      if (!latestFollowList) {
+        logger.debug(`No valid follow list found for user ${pubkey}`)
+        continue
+      }
+
+      logger.debug(`Found follow list created at: ${new Date(latestFollowList.created_at * 1000).toISOString()}`)
+
+      // Check if tags property exists
+      if (!latestFollowList.tags) {
+        logger.debug(`No tags found in follow list for user ${pubkey}`)
+        logger.debug('Follow list data:', JSON.stringify(latestFollowList, null, 2))
+        continue
+      }
+
+      // Extract followed pubkeys from the follow list (tag type 'p')
+      const followedPubkeys = latestFollowList.tags
+        .filter(tag => tag[0] === 'p')
+        .map(tag => tag[1])
+        .filter(pk => !ignoreSet.has(pk)) // Filter out pubkeys from the ignore list
+
+      if (!followedPubkeys || followedPubkeys.length === 0) {
+        logger.debug(`No followed users found for user ${pubkey} (after filtering ignore list)`)
+        continue
+      }
+
+      // Add all followed pubkeys to our set
+      followedPubkeys.forEach(pk => allFollowedPubkeys.add(pk))
+
+      logger.debug(`Added ${followedPubkeys.length} followed users for ${pubkey} (total: ${allFollowedPubkeys.size})`)
+    } catch (error) {
+      logger.error(`Error processing user ${pubkey}: ${error}`)
+    }
+  }
+
+  // If we found any followed pubkeys, fetch their notes in batches
+  if (allFollowedPubkeys.size > 0) {
+    // Convert Set to Array for the filter
+    const followedPubkeysArray = Array.from(allFollowedPubkeys)
+    const batchSize = config.batchSize || 100 // Use config batch size or default to 100
+    const totalBatches = Math.ceil(followedPubkeysArray.length / batchSize)
+
+    logger.progress(`Processing ${followedPubkeysArray.length} followed users in ${totalBatches} batches...`)
+
+    // Process in batches
+    for (let batchNum = 0; batchNum < totalBatches; batchNum++) {
+      const start = batchNum * batchSize
+      const end = Math.min(start + batchSize, followedPubkeysArray.length)
+      const batch = followedPubkeysArray.slice(start, end)
+
+      logger.progress(`Fetching batch ${batchNum + 1}/${totalBatches} (${batch.length} authors)...`)
+
+      // Fetch notes from the current batch of users
+      const notes = await fetchEvents(relayUrls, {
+        kinds: [1],
+        authors: batch,
+        since: cutoffTime
+      }, 30000) // Use a longer timeout for this larger query
+
+      logger.debug(`Retrieved ${notes.length} notes from batch ${batchNum + 1}`)
+
+      // Filter notes that have URLs (excluding notes with only media URLs)
+      const notesWithUrls = notes.filter(note => {
+        // Extract all URLs from content
+        const urlRegex = /(https?:\/\/[^\s]+)/g
+        const matches = note.content.match(urlRegex) || []
+
+        if (matches.length === 0) return false // No URLs at all
+
+        // Check if any URL is not a media file
+        const hasNonMediaUrl = matches.some(url => !isMediaUrl(url))
+
+        return hasNonMediaUrl
+      })
+
+      logger.debug(`Found ${notesWithUrls.length} notes containing non-media URLs in batch ${batchNum + 1}`)
+
+      // Get all unique authors from the filtered notes in this batch
+      const authorsWithUrls = new Set(notesWithUrls.map(note => note.pubkey))
+
+      // Fetch metadata for all relevant authors in this batch
+      if (authorsWithUrls.size > 0) {
+        logger.debug(`Fetching metadata for ${authorsWithUrls.size} authors from batch ${batchNum + 1}...`)
+        const allMetadata = await fetchEvents(relayUrls, {
+          kinds: [0],
+          authors: Array.from(authorsWithUrls)
+        })
+
+        // Create a map of author pubkey to their latest metadata
+        const metadataByAuthor = {}
+        allMetadata.forEach(meta => {
+          if (!metadataByAuthor[meta.pubkey] || meta.created_at > metadataByAuthor[meta.pubkey].created_at) {
+            metadataByAuthor[meta.pubkey] = meta
+          }
+        })
+
+        // Attach metadata to notes
+        for (const note of notesWithUrls) {
+          if (metadataByAuthor[note.pubkey]) {
+            try {
+              const metadata = JSON.parse(metadataByAuthor[note.pubkey].content)
+              note.userMetadata = metadata
+            } catch (e) {
+              logger.debug(`Error parsing metadata for ${note.pubkey}: ${e.message}`)
+            }
+          }
+        }
+      }
+
+      // Add all notes with URLs from this batch to our results
+      allNotesWithLinks.push(...notesWithUrls)
+
+      // Show incremental progress during batch processing
+      if (allNotesWithLinks.length > 0 && batchNum < totalBatches - 1) {
+        logger.progress(`Found ${allNotesWithLinks.length} notes with links so far...`)
+      }
+    }
+
+    logger.progress(`Completed processing all ${totalBatches} batches`)
+  }
+
+  return allNotesWithLinks
+}
+
+/**
+ * Format the notes for display with colorful output
+ *
+ * @param {Array} notes - Array of note objects
+ * @returns {String} - Formatted string with note information
+ */
+function formatNoteOutput (notes) {
+  const output = []
+
+  for (const note of notes) {
+    // Get note ID as npub
+    const noteId = nip19.noteEncode(note.id)
+    const pubkey = nip19.npubEncode(note.pubkey)
+
+    // Get user display name or fall back to npub
+    const userName = note.userMetadata
+      ? (note.userMetadata.display_name || note.userMetadata.name || pubkey)
+      : pubkey
+
+    // Get timestamp as readable date
+    const timestamp = new Date(note.created_at * 1000).toISOString()
+
+    // Extract URLs from content, marking media URLs with colors
+    const urlRegex = /(https?:\/\/[^\s]+)/g
+    const matches = note.content.match(urlRegex) || []
+
+    // Format URLs with colors
+    const markedUrls = matches.map(url => {
+      const isMedia = isMediaUrl(url)
+      if (isMedia) {
+        return `${colors.fg.gray}${url}${colors.reset} (media)`
+      } else {
+        return `${colors.bright}${colors.fg.cyan}${url}${colors.reset}`
+      }
+    })
+
+    // Format output with colors
+    output.push(`${colors.bright}${colors.fg.yellow}Note by ${colors.fg.magenta}${userName}${colors.fg.yellow} at ${timestamp}${colors.reset}`)
+    output.push(`${colors.fg.green}Note ID: ${colors.reset}${noteId}`)
+    output.push(`${colors.fg.green}Pubkey: ${colors.reset}${pubkey}`)
+
+    // Add links with a heading
+    output.push(`${colors.bright}${colors.fg.blue}External URLs:${colors.reset}`)
+    markedUrls.forEach(url => {
+      output.push(`  • ${url}`)
+    })
+
+    // Add content with a heading
+    output.push(`${colors.bright}${colors.fg.blue}Note content:${colors.reset}`)
+
+    // Colorize any links in content when displaying
+    let coloredContent = note.content
+    for (const url of matches) {
+      const isMedia = isMediaUrl(url)
+      const colorCode = isMedia ? colors.fg.gray : colors.bright + colors.fg.cyan
+      coloredContent = coloredContent.replace(
+        new RegExp(escapeRegExp(url), 'g'),
+        `${colorCode}${url}${colors.reset}`
+      )
+    }
+    output.push(coloredContent)
+
+    output.push(`${colors.fg.yellow}${'-'.repeat(50)}${colors.reset}`)
+  }
+
+  return output.join('\n')
+}
+
+/**
+ * Escape special characters for use in a regular expression
+ * @param {String} string - String to escape
+ * @returns {String} - Escaped string
+ */
+function escapeRegExp (string) {
+  return string.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
+}
+
+/**
+ * Convert a pubkey from npub to hex format if needed
+ * @param {String} key - Pubkey in either npub or hex format
+ * @returns {String} - Pubkey in hex format
+ */
+function normalizeToHexPubkey (key) {
+  // If it's an npub, decode it
+  if (typeof key === 'string' && key.startsWith('npub1')) {
+    try {
+      const { type, data } = nip19.decode(key)
+      if (type === 'npub') {
+        return data
+      }
+    } catch (e) {
+      logger.error(`Error decoding npub ${key}: ${e.message}`)
+    }
+  }
+  // Otherwise assume it's already in hex format
+  return key
+}
+
+/**
+ * Main function to execute the script
+ */
+async function main () {
+  // Load configuration from file
+  const configPath = path.join(__dirname, 'nostr-link-extract.config.json')
+  logger.info(`Loading configuration from ${configPath}`)
+  config = loadConfig(configPath)
+
+  try {
+    logger.info(`Starting Nostr link extraction (time interval: ${config.timeIntervalHours} hours)`)
+
+    // Convert any npub format keys to hex
+    const hexUserPubkeys = config.userPubkeys.map(normalizeToHexPubkey)
+    const hexIgnorePubkeys = config.ignorePubkeys.map(normalizeToHexPubkey)
+
+    // Log the conversion for clarity (helpful for debugging)
+    if (config.userPubkeys.some(key => key.startsWith('npub1'))) {
+      logger.debug('Converted user npubs to hex format for Nostr protocol')
+    }
+    if (config.ignorePubkeys.some(key => key.startsWith('npub1'))) {
+      logger.debug('Converted ignore list npubs to hex format for Nostr protocol')
+    }
+
+    const notesWithLinks = await getNotesWithLinks(
+      hexUserPubkeys,
+      config.timeIntervalHours,
+      config.relayUrls,
+      hexIgnorePubkeys
+    )
+
+    if (notesWithLinks.length > 0) {
+      const formattedOutput = formatNoteOutput(notesWithLinks)
+      console.log(formattedOutput)
+      logger.result(`Total notes with links: ${notesWithLinks.length}`)
+    } else {
+      logger.info('No notes with links found in the specified time interval.')
+    }
+  } catch (error) {
+    logger.error(`${error}`)
+  }
+}
+
+// Execute the script
+main()

wallets/errors.js

@@ -62,6 +62,13 @@ export class WalletsNotAvailableError extends WalletConfigurationError {
   }
 }
 
+export class AnonWalletError extends WalletConfigurationError {
+  constructor () {
+    super('anon cannot pay with wallets')
+    this.name = 'AnonWalletError'
+  }
+}
+
 export class WalletAggregateError extends WalletError {
   constructor (errors, invoice) {
     super('WalletAggregateError')

wallets/index.js

@@ -1,12 +1,15 @@
 import { useMe } from '@/components/me'
-import { SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet'
-import { SSR } from '@/lib/constants'
-import { useApolloClient, useMutation, useQuery } from '@apollo/client'
+import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet'
+import { NORMAL_POLL_INTERVAL, SSR } from '@/lib/constants'
+import { useApolloClient, useLazyQuery, useMutation, useQuery } from '@apollo/client'
 import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react'
 import { getStorageKey, getWalletByType, walletPrioritySort, canSend, isConfigured, upsertWalletVariables, siftConfig, saveWalletLocally } from './common'
 import useVault from '@/components/vault/use-vault'
 import walletDefs from '@/wallets/client'
 import { generateMutation } from './graphql'
+import { useWalletPayment } from './payment'
+import useInvoice from '@/components/use-invoice'
+import { WalletConfigurationError } from './errors'
 
 const WalletsContext = createContext({
   wallets: []
@@ -204,7 +207,9 @@ export function WalletsProvider ({ children }) {
         removeLocalWallets
       }}
     >
+      <RetryHandler>
         {children}
+      </RetryHandler>
     </WalletsContext.Provider>
   )
 }
@@ -221,7 +226,79 @@ export function useWallet (name) {
 export function useSendWallets () {
   const { wallets } = useWallets()
   // return all enabled wallets that are available and can send
-  return wallets
+  return useMemo(() => wallets
     .filter(w => !w.def.isAvailable || w.def.isAvailable())
-    .filter(w => w.config?.enabled && canSend(w))
+    .filter(w => w.config?.enabled && canSend(w)), [wallets])
+}
+
+function RetryHandler ({ children }) {
+  const wallets = useSendWallets()
+  const waitForWalletPayment = useWalletPayment()
+  const invoiceHelper = useInvoice()
+  const [getFailedInvoices] = useLazyQuery(FAILED_INVOICES, { fetchPolicy: 'network-only', nextFetchPolicy: 'network-only' })
+
+  const retry = useCallback(async (invoice) => {
+    const newInvoice = await invoiceHelper.retry({ ...invoice, newAttempt: true })
+
+    try {
+      await waitForWalletPayment(newInvoice)
+    } catch (err) {
+      if (err instanceof WalletConfigurationError) {
+        // consume attempt by canceling invoice
+        await invoiceHelper.cancel(newInvoice)
+      }
+      throw err
+    }
+  }, [invoiceHelper, waitForWalletPayment])
+
+  useEffect(() => {
+    // we always retry failed invoices, even if the user has no wallets on any client
+    // to make sure that failed payments will always show up in notifications eventually
+
+    const retryPoll = async () => {
+      let failedInvoices
+      try {
+        const { data, error } = await getFailedInvoices()
+        if (error) throw error
+        failedInvoices = data.failedInvoices
+      } catch (err) {
+        console.error('failed to fetch invoices to retry:', err)
+        return
+      }
+
+      for (const inv of failedInvoices) {
+        try {
+          await retry(inv)
+        } catch (err) {
+          // some retries are expected to fail since only one client at a time is allowed to retry
+          // these should show up as 'invoice not found' errors
+          console.error('retry failed:', err)
+        }
+      }
+    }
+
+    let timeout, stopped
+    const queuePoll = () => {
+      timeout = setTimeout(async () => {
+        try {
+          await retryPoll()
+        } catch (err) {
+          // every error should already be handled in retryPoll
+          // but this catch is a safety net to not trigger an unhandled promise rejection
+          console.error('retry poll failed:', err)
+        }
+        if (!stopped) queuePoll()
+      }, NORMAL_POLL_INTERVAL)
+    }
+
+    const stopPolling = () => {
+      stopped = true
+      clearTimeout(timeout)
+    }
+
+    queuePoll()
+    return stopPolling
+  }, [wallets, getFailedInvoices, retry])
+
+  return children
 }

wallets/payment.js

@@ -4,23 +4,30 @@ import { formatSats } from '@/lib/format'
 import useInvoice from '@/components/use-invoice'
 import { FAST_POLL_INTERVAL, WALLET_SEND_PAYMENT_TIMEOUT_MS } from '@/lib/constants'
 import {
-  WalletsNotAvailableError, WalletSenderError, WalletAggregateError, WalletPaymentAggregateError,
+  AnonWalletError, WalletsNotAvailableError, WalletSenderError, WalletAggregateError, WalletPaymentAggregateError,
   WalletNotEnabledError, WalletSendNotConfiguredError, WalletPaymentError, WalletError, WalletReceiverError
 } from '@/wallets/errors'
 import { canSend } from './common'
 import { useWalletLoggerFactory } from './logger'
 import { timeoutSignal, withTimeout } from '@/lib/time'
+import { useMe } from '@/components/me'
 
 export function useWalletPayment () {
   const wallets = useSendWallets()
   const sendPayment = useSendPayment()
   const loggerFactory = useWalletLoggerFactory()
   const invoiceHelper = useInvoice()
+  const { me } = useMe()
 
-  return useCallback(async (invoice, { waitFor, updateOnFallback }) => {
+  return useCallback(async (invoice, { waitFor, updateOnFallback } = {}) => {
     let aggregateError = new WalletAggregateError([])
     let latestInvoice = invoice
 
+    // anon user cannot pay with wallets
+    if (!me) {
+      throw new AnonWalletError()
+    }
+
     // throw a special error that caller can handle separately if no payment was attempted
     if (wallets.length === 0) {
       throw new WalletsNotAvailableError()

wallets/server.js

@@ -24,9 +24,13 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln]
 
 const MAX_PENDING_INVOICES_PER_WALLET = 25
 
-export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { predecessorId, models }) {
+export async function * createUserInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { paymentAttempt, predecessorId, models }) {
   // get the wallets in order of priority
-  const wallets = await getInvoiceableWallets(userId, { predecessorId, models })
+  const wallets = await getInvoiceableWallets(userId, {
+    paymentAttempt,
+    predecessorId,
+    models
+  })
 
   msats = toPositiveNumber(msats)
 
@@ -68,34 +72,29 @@ export async function createInvoice (userId, { msats, description, descriptionHa
         }
       }
 
-      return { invoice, wallet, logger }
+      yield { invoice, wallet, logger }
     } catch (err) {
+      console.error('failed to create user invoice:', err)
       logger.error(err.message, { status: true })
     }
   }
-
-  throw new Error('no wallet to receive available')
 }
 
 export async function createWrappedInvoice (userId,
   { msats, feePercent, description, descriptionHash, expiry = 360 },
-  { predecessorId, models, me, lnd }) {
-  let logger, bolt11
-  try {
-    const { invoice, wallet } = await createInvoice(userId, {
+  { paymentAttempt, predecessorId, models, me, lnd }) {
+  // loop over all receiver wallet invoices until we successfully wrapped one
+  for await (const { invoice, logger, wallet } of createUserInvoice(userId, {
     // this is the amount the stacker will receive, the other (feePercent)% is our fee
     msats: toPositiveBigInt(msats) * (100n - feePercent) / 100n,
     description,
     descriptionHash,
     expiry
-    }, { predecessorId, models })
-
-    logger = walletLogger({ wallet, models })
+  }, { paymentAttempt, predecessorId, models })) {
+    let bolt11
+    try {
       bolt11 = invoice
-
-    const { invoice: wrappedInvoice, maxFee } =
-      await wrapInvoice({ bolt11, feePercent }, { msats, description, descriptionHash }, { me, lnd })
-
+      const { invoice: wrappedInvoice, maxFee } = await wrapInvoice({ bolt11, feePercent }, { msats, description, descriptionHash }, { me, lnd })
       return {
         invoice,
         wrappedInvoice: wrappedInvoice.request,
@@ -103,12 +102,15 @@ export async function createWrappedInvoice (userId,
         maxFee
       }
     } catch (e) {
-    logger?.error('invalid invoice: ' + e.message, { bolt11 })
-    throw e
+      console.error('failed to wrap invoice:', e)
+      logger?.error('failed to wrap invoice: ' + e.message, { bolt11 })
     }
   }
 
-export async function getInvoiceableWallets (userId, { predecessorId, models }) {
+  throw new Error('no wallet to receive available')
+}
+
+export async function getInvoiceableWallets (userId, { paymentAttempt, predecessorId, models }) {
   // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices.
   // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it
   // so it has not been updated yet.
@@ -141,6 +143,7 @@ export async function getInvoiceableWallets (userId, { predecessorId, models })
           FROM "Invoice"
           JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId"
           WHERE "Invoice"."actionState" = 'RETRYING'
+          AND "Invoice"."paymentAttempt" = ${paymentAttempt}
         )
         SELECT
           "InvoiceForward"."walletId"

wallets/webln/client.js

@@ -1,14 +1,19 @@
 import { useEffect } from 'react'
 import { SSR } from '@/lib/constants'
+import { WalletError } from '../errors'
 export * from '@/wallets/webln'
 
 export const sendPayment = async (bolt11) => {
   if (typeof window.webln === 'undefined') {
-    throw new Error('WebLN provider not found')
+    throw new WalletError('WebLN provider not found')
   }
 
   // this will prompt the user to unlock the wallet if it's locked
+  try {
     await window.webln.enable()
+  } catch (err) {
+    throw new WalletError(err.message)
+  }
 
   // this will prompt for payment if no budget is set
   const response = await window.webln.sendPayment(bolt11)
@@ -16,7 +21,7 @@ export const sendPayment = async (bolt11) => {
     // sendPayment returns nothing if WebLN was enabled
     // but browser extension that provides WebLN was then disabled
     // without reloading the page
-    throw new Error('sendPayment returned no response')
+    throw new WalletError('sendPayment returned no response')
   }
 
   return response.preimage

worker/autowithdraw.js

@@ -1,6 +1,6 @@
 import { msatsSatsFloor, msatsToSats, satsToMsats } from '@/lib/format'
 import { createWithdrawal } from '@/api/resolvers/wallet'
-import { createInvoice } from '@/wallets/server'
+import { createUserInvoice } from '@/wallets/server'
 
 export async function autoWithdraw ({ data: { id }, models, lnd }) {
   const user = await models.user.findUnique({ where: { id } })
@@ -42,14 +42,20 @@ export async function autoWithdraw ({ data: { id }, models, lnd }) {
 
   if (pendingOrFailed.exists) return
 
-  const { invoice, wallet, logger } = await createInvoice(id, { msats, description: 'SN: autowithdrawal', expiry: 360 }, { models })
-
+  for await (const { invoice, wallet, logger } of createUserInvoice(id, {
+    msats,
+    description: 'SN: autowithdrawal',
+    expiry: 360
+  }, { models })) {
     try {
       return await createWithdrawal(null,
         { invoice, maxFee: msatsToSats(maxFeeMsats) },
         { me: { id }, models, lnd, wallet, logger })
     } catch (err) {
-    logger.error(`incoming payment failed: ${err}`, { bolt11: invoice })
-    throw err
+      console.error('failed to create autowithdrawal:', err)
+      logger?.error('incoming payment failed: ' + err.message, { bolt11: invoice })
     }
   }
+
+  throw new Error('no wallet to receive available')
+}