ekzyis/stacker.news
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Automated retries (#1776)

Browse Source 
* Poll failed invoices with visibility timeout

* Don't return intermediate failed invoices

* Don't retry too old invoices

* Retry invoices on client

* Only attempt payment 3 times

* Fix fallbacks during last retry

* Rename retry column to paymentAttempt

* Fix no index used

* Resolve TODOs

* Use expiring locks

* Better comments for constants

* Acquire lock during retry

* Use expiring lock in retry mutation

* Use now() instead of CURRENT_TIMESTAMP

* Cosmetic changes

* Immediately show failed post payments in notifications

* Update hasNewNotes

* Never retry on user cancel

For a consistent UX and less mental overhead, I decided to remove the exception for ITEM_CREATE where it would still retry in the background even though we want to show the payment failure immediately in notifications.

* Fix notifications without pending retries missing if no send wallets

If a stacker has no send wallets, they would miss notifications about failed payments because they would never get retried.

This commit fixes this by making the notifications query aware if the stacker has send wallets. This way, it can tell if a notification will be retried or not.

* Stop hiding userCancel in notifications

As mentioned in a previous commit, I want to show anything that will not be attempted anymore in notifications.

Before, I wanted to hide manually cancelled invoices but to not change experience unnecessarily and to decrease mental overhead, I changed my mind.

* Also consider invoice.cancelledAt in notifications

* Always retry failed payments, even without send wallets

* Fix notification indicator on retry timeout

* Set invoice.updated_at to date slightly in the future

* Use default job priority

* Stop retrying after one hour

* Remove special case for ITEM_CREATE

* Replace retryTimeout job with notification indicator query

* Fix sortTime

---------

Co-authored-by: Keyan <34140557+huumn@users.noreply.github.com>
This commit is contained in:
ekzyis 2025-02-15 02:25:11 +01:00 committed by GitHub
parent f4040756b3
commit 0032e064b2
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
17 changed files with 227 additions and 35 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
api/paidAction/index.js
View File

@ -419,7 +419,7 @@ async function createSNInvoice (actionType, args, context) {
}
async function createDbInvoice (actionType, args, context) {
const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context
const { me, models, tx, cost, optimistic, actionId, invoiceArgs, paymentAttempt, predecessorId } = context
const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs
const db = tx ?? models
@ -445,6 +445,7 @@ async function createDbInvoice (actionType, args, context) {
actionArgs: args,
expiresAt,
actionId,
paymentAttempt,
predecessorId
}

17
api/resolvers/notifications.js
View File

@ -5,6 +5,7 @@ import { pushSubscriptionSchema, validateSchema } from '@/lib/validate'
import { replyToSubscription } from '@/lib/webPush'
import { getSub } from './sub'
import { GqlAuthenticationError, GqlInputError } from '@/lib/error'
import { WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants'
export default {
Query: {
@ -345,11 +346,25 @@ export default {
)
queries.push(
`(SELECT "Invoice".id::text, "Invoice"."updated_at" AS "sortTime", NULL as "earnedSats", 'Invoicification' AS type
`(SELECT "Invoice".id::text,
CASE
WHEN
"Invoice"."paymentAttempt" < ${WALLET_MAX_RETRIES}
AND "Invoice"."userCancel" = false
AND "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}'
THEN "Invoice"."cancelledAt" + interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}'
ELSE "Invoice"."updated_at"
END AS "sortTime", NULL as "earnedSats", 'Invoicification' AS type
FROM "Invoice"
WHERE "Invoice"."userId" = $1
AND "Invoice"."updated_at" < $2
AND "Invoice"."actionState" = 'FAILED'
AND (
-- this is the inverse of the filter for automated retries
"Invoice"."paymentAttempt" >= ${WALLET_MAX_RETRIES}
OR "Invoice"."userCancel" = true
OR "Invoice"."cancelledAt" <= now() - interval '${`${WALLET_RETRY_BEFORE_MS} milliseconds`}'
)
AND (
"Invoice"."actionType" = 'ITEM_CREATE' OR
"Invoice"."actionType" = 'ZAP' OR

28
api/resolvers/paidAction.js
View File

@ -1,5 +1,5 @@
import { retryPaidAction } from '../paidAction'
import { USER_ID } from '@/lib/constants'
import { USER_ID, WALLET_MAX_RETRIES, WALLET_RETRY_TIMEOUT_MS } from '@/lib/constants'
function paidActionType (actionType) {
switch (actionType) {
@ -50,24 +50,32 @@ export default {
}
},
Mutation: {
retryPaidAction: async (parent, { invoiceId }, { models, me, lnd }) => {
retryPaidAction: async (parent, { invoiceId, newAttempt }, { models, me, lnd }) => {
if (!me) {
throw new Error('You must be logged in')
}
const invoice = await models.invoice.findUnique({ where: { id: invoiceId, userId: me.id } })
// make sure only one client at a time can retry by acquiring a lock that expires
const [invoice] = await models.$queryRaw`
UPDATE "Invoice"
SET "retryPendingSince" = now()
WHERE
id = ${invoiceId} AND
"userId" = ${me.id} AND
"actionState" = 'FAILED' AND
("retryPendingSince" IS NULL OR "retryPendingSince" < now() - ${`${WALLET_RETRY_TIMEOUT_MS} milliseconds`}::interval)
RETURNING *`
if (!invoice) {
throw new Error('Invoice not found')
throw new Error('Invoice not found or retry pending')
}
if (invoice.actionState !== 'FAILED') {
if (invoice.actionState === 'PAID') {
throw new Error('Invoice is already paid')
}
throw new Error(`Invoice is not in failed state: ${invoice.actionState}`)
// do we want to retry a payment from the beginning with all sender and receiver wallets?
const paymentAttempt = newAttempt ? invoice.paymentAttempt + 1 : invoice.paymentAttempt
if (paymentAttempt > WALLET_MAX_RETRIES) {
throw new Error('Payment has been retried too many times')
}
const result = await retryPaidAction(invoice.actionType, { invoice }, { models, me, lnd })
const result = await retryPaidAction(invoice.actionType, { invoice }, { paymentAttempt, models, me, lnd })
return {
...result,

41
api/resolvers/user.js
View File

@ -4,9 +4,9 @@ import { decodeCursor, LIMIT, nextCursorEncoded } from '@/lib/cursor'
import { msatsToSats } from '@/lib/format'
import { bioSchema, emailSchema, settingsSchema, validateSchema, userSchema } from '@/lib/validate'
import { getItem, updateItem, filterClause, createItem, whereClause, muteClause, activeOrMine } from './item'
import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES } from '@/lib/constants'
import { USER_ID, RESERVED_MAX_USER_ID, SN_NO_REWARDS_IDS, INVOICE_ACTION_NOTIFICATION_TYPES, WALLET_MAX_RETRIES, WALLET_RETRY_BEFORE_MS } from '@/lib/constants'
import { viewGroup } from './growth'
import { timeUnitForRange, whenRange } from '@/lib/time'
import { datePivot, timeUnitForRange, whenRange } from '@/lib/time'
import assertApiKeyNotPermitted from './apiKey'
import { hashEmail } from '@/lib/crypto'
import { isMuted } from '@/lib/user'
@ -543,7 +543,17 @@ export default {
actionType: {
in: INVOICE_ACTION_NOTIFICATION_TYPES
},
actionState: 'FAILED'
actionState: 'FAILED',
OR: [
{
paymentAttempt: {
gte: WALLET_MAX_RETRIES
}
},
{
userCancel: true
}
]
}
})
@ -552,6 +562,31 @@ export default {
return true
}
const invoiceActionFailed2 = await models.invoice.findFirst({
where: {
userId: me.id,
updatedAt: {
gt: datePivot(lastChecked, { milliseconds: -WALLET_RETRY_BEFORE_MS })
},
actionType: {
in: INVOICE_ACTION_NOTIFICATION_TYPES
},
actionState: 'FAILED',
paymentAttempt: {
lt: WALLET_MAX_RETRIES
},
userCancel: false,
cancelledAt: {
lte: datePivot(new Date(), { milliseconds: -WALLET_RETRY_BEFORE_MS })
}
}
})
if (invoiceActionFailed2) {
foundNotes()
return true
}
// update checkedNotesAt to prevent rechecking same time period
models.user.update({
where: { id: me.id },

20
api/resolvers/wallet.js
View File

@ -9,7 +9,10 @@ import { formatMsats, msatsToSats, msatsToSatsDecimal, satsToMsats } from '@/lib
import {
USER_ID, INVOICE_RETENTION_DAYS,
PAID_ACTION_PAYMENT_METHODS,
WALLET_CREATE_INVOICE_TIMEOUT_MS
WALLET_CREATE_INVOICE_TIMEOUT_MS,
WALLET_RETRY_AFTER_MS,
WALLET_RETRY_BEFORE_MS,
WALLET_MAX_RETRIES
} from '@/lib/constants'
import { amountSchema, validateSchema, withdrawlSchema, lnAddrSchema } from '@/lib/validate'
import assertGofacYourself from './ofac'
@ -456,6 +459,21 @@ const resolvers = {
cursor: nextCursor,
entries: logs
}
},
failedInvoices: async (parent, args, { me, models }) => {
if (!me) {
throw new GqlAuthenticationError()
}
return await models.$queryRaw`
SELECT * FROM "Invoice"
WHERE "userId" = ${me.id}
AND "actionState" = 'FAILED'
-- never retry if user has cancelled the invoice manually
AND "userCancel" = false
AND "cancelledAt" < now() - ${`${WALLET_RETRY_AFTER_MS} milliseconds`}::interval
AND "cancelledAt" > now() - ${`${WALLET_RETRY_BEFORE_MS} milliseconds`}::interval
AND "paymentAttempt" < ${WALLET_MAX_RETRIES}
ORDER BY id DESC`
}
},
Wallet: {

2
api/typeDefs/paidAction.js
View File

@ -7,7 +7,7 @@ extend type Query {
}
extend type Mutation {
retryPaidAction(invoiceId: Int!): PaidAction!
retryPaidAction(invoiceId: Int!, newAttempt: Boolean): PaidAction!
}
enum PaymentMethod {

1
api/typeDefs/wallet.js
View File

@ -72,6 +72,7 @@ const typeDefs = `
wallet(id: ID!): Wallet
walletByType(type: String!): Wallet
walletLogs(type: String, from: String, to: String, cursor: String): WalletLog!
failedInvoices: [Invoice!]!
}
extend type Mutation {

8
components/use-invoice.js
View File

@ -1,5 +1,5 @@
import { useApolloClient, useMutation } from '@apollo/client'
import { useCallback } from 'react'
import { useCallback, useMemo } from 'react'
import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors'
import { RETRY_PAID_ACTION } from '@/fragments/paidAction'
import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet'
@ -42,9 +42,9 @@ export default function useInvoice () {
return data.cancelInvoice
}, [cancelInvoice])
const retry = useCallback(async ({ id, hash, hmac }, { update }) => {
const retry = useCallback(async ({ id, hash, hmac, newAttempt = false }, { update } = {}) => {
console.log('retrying invoice:', hash)
const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id) }, update })
const { data, error } = await retryPaidAction({ variables: { invoiceId: Number(id), newAttempt }, update })
if (error) throw error
const newInvoice = data.retryPaidAction.invoice
@ -53,5 +53,5 @@ export default function useInvoice () {
return newInvoice
}, [retryPaidAction])
return { cancel, retry, isInvoice }
return useMemo(() => ({ cancel, retry, isInvoice }), [cancel, retry, isInvoice])
}

4
fragments/paidAction.js
View File

@ -91,8 +91,8 @@ export const RETRY_PAID_ACTION = gql`
${PAID_ACTION}
${ITEM_PAID_ACTION_FIELDS}
${ITEM_ACT_PAID_ACTION_FIELDS}
mutation retryPaidAction($invoiceId: Int!) {
retryPaidAction(invoiceId: $invoiceId) {
mutation retryPaidAction($invoiceId: Int!, $newAttempt: Boolean) {
retryPaidAction(invoiceId: $invoiceId, newAttempt: $newAttempt) {
__typename
...PaidActionFields
... on ItemPaidAction {

9
fragments/wallet.js
View File

@ -231,3 +231,12 @@ export const CANCEL_INVOICE = gql`
}
}
`
export const FAILED_INVOICES = gql`
${INVOICE_FIELDS}
query FailedInvoices {
failedInvoices {
...InvoiceFields
}
}
`

6
lib/apollo.js
View File

@ -283,6 +283,12 @@ function getClient (uri) {
facts: [...(existing?.facts || []), ...incoming.facts]
}
}
},
failedInvoices: {
keyArgs: [],
merge (existing, incoming) {
return incoming
}
}
}
},

10
lib/constants.js
View File

@ -198,4 +198,14 @@ export const ZAP_UNDO_DELAY_MS = 5_000
export const WALLET_SEND_PAYMENT_TIMEOUT_MS = 150_000
export const WALLET_CREATE_INVOICE_TIMEOUT_MS = 45_000
// interval between which failed invoices are returned to a client for automated retries.
// retry-after must be high enough such that intermediate failed invoices that will already
// be retried by the client due to sender or receiver fallbacks are not returned to the client.
export const WALLET_RETRY_AFTER_MS = 60_000 // 1 minute
export const WALLET_RETRY_BEFORE_MS = 3_600_000 // 1 hour
// we want to attempt a payment three times so we retry two times
export const WALLET_MAX_RETRIES = 2
// when a pending retry for an invoice should be considered expired and can be attempted again
export const WALLET_RETRY_TIMEOUT_MS = 60_000 // 1 minute
export const BECH32_CHARSET = 'qpzry9x8gf2tvdw0s3jn54khce6mua7l'

4
prisma/migrations/20250107084543_automated_retries/migration.sql Normal file
View File

@ -0,0 +1,4 @@
-- AlterTable
ALTER TABLE "Invoice" ADD COLUMN "paymentAttempt" INTEGER NOT NULL DEFAULT 0;
ALTER TABLE "Invoice" ADD COLUMN "retryPendingSince" TIMESTAMP(3);
CREATE INDEX "Invoice_cancelledAt_idx" ON "Invoice"("cancelledAt");

3
prisma/schema.prisma
View File

@ -928,6 +928,8 @@ model Invoice {
cancelled Boolean @default(false)
cancelledAt DateTime?
userCancel Boolean?
paymentAttempt Int @default(0)
retryPendingSince DateTime?
msatsRequested BigInt
msatsReceived BigInt?
desc String?
@ -956,6 +958,7 @@ model Invoice {
@@index([confirmedIndex], map: "Invoice.confirmedIndex_index")
@@index([isHeld])
@@index([confirmedAt])
@@index([cancelledAt])
@@index([actionType])
@@index([actionState])
}

89
wallets/index.js
View File

@ -1,12 +1,15 @@
import { useMe } from '@/components/me'
import { SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet'
import { SSR } from '@/lib/constants'
import { useApolloClient, useMutation, useQuery } from '@apollo/client'
import { FAILED_INVOICES, SET_WALLET_PRIORITY, WALLETS } from '@/fragments/wallet'
import { NORMAL_POLL_INTERVAL, SSR } from '@/lib/constants'
import { useApolloClient, useLazyQuery, useMutation, useQuery } from '@apollo/client'
import { createContext, useCallback, useContext, useEffect, useMemo, useState } from 'react'
import { getStorageKey, getWalletByType, walletPrioritySort, canSend, isConfigured, upsertWalletVariables, siftConfig, saveWalletLocally } from './common'
import useVault from '@/components/vault/use-vault'
import walletDefs from '@/wallets/client'
import { generateMutation } from './graphql'
import { useWalletPayment } from './payment'
import useInvoice from '@/components/use-invoice'
import { WalletConfigurationError } from './errors'
const WalletsContext = createContext({
wallets: []
@ -204,7 +207,9 @@ export function WalletsProvider ({ children }) {
removeLocalWallets
}}
>
{children}
<RetryHandler>
{children}
</RetryHandler>
</WalletsContext.Provider>
)
}
@ -221,7 +226,79 @@ export function useWallet (name) {
export function useSendWallets () {
const { wallets } = useWallets()
// return all enabled wallets that are available and can send
return wallets
return useMemo(() => wallets
.filter(w => !w.def.isAvailable || w.def.isAvailable())
.filter(w => w.config?.enabled && canSend(w))
.filter(w => w.config?.enabled && canSend(w)), [wallets])
}
function RetryHandler ({ children }) {
const wallets = useSendWallets()
const waitForWalletPayment = useWalletPayment()
const invoiceHelper = useInvoice()
const [getFailedInvoices] = useLazyQuery(FAILED_INVOICES, { fetchPolicy: 'network-only', nextFetchPolicy: 'network-only' })
const retry = useCallback(async (invoice) => {
const newInvoice = await invoiceHelper.retry({ ...invoice, newAttempt: true })
try {
await waitForWalletPayment(newInvoice)
} catch (err) {
if (err instanceof WalletConfigurationError) {
// consume attempt by canceling invoice
await invoiceHelper.cancel(newInvoice)
}
throw err
}
}, [invoiceHelper, waitForWalletPayment])
useEffect(() => {
// we always retry failed invoices, even if the user has no wallets on any client
// to make sure that failed payments will always show up in notifications eventually
const retryPoll = async () => {
let failedInvoices
try {
const { data, error } = await getFailedInvoices()
if (error) throw error
failedInvoices = data.failedInvoices
} catch (err) {
console.error('failed to fetch invoices to retry:', err)
return
}
for (const inv of failedInvoices) {
try {
await retry(inv)
} catch (err) {
// some retries are expected to fail since only one client at a time is allowed to retry
// these should show up as 'invoice not found' errors
console.error('retry failed:', err)
}
}
}
let timeout, stopped
const queuePoll = () => {
timeout = setTimeout(async () => {
try {
await retryPoll()
} catch (err) {
// every error should already be handled in retryPoll
// but this catch is a safety net to not trigger an unhandled promise rejection
console.error('retry poll failed:', err)
}
if (!stopped) queuePoll()
}, NORMAL_POLL_INTERVAL)
}
const stopPolling = () => {
stopped = true
clearTimeout(timeout)
}
queuePoll()
return stopPolling
}, [wallets, getFailedInvoices, retry])
return children
}

2
wallets/payment.js
View File

@ -17,7 +17,7 @@ export function useWalletPayment () {
const loggerFactory = useWalletLoggerFactory()
const invoiceHelper = useInvoice()
return useCallback(async (invoice, { waitFor, updateOnFallback }) => {
return useCallback(async (invoice, { waitFor, updateOnFallback } = {}) => {
let aggregateError = new WalletAggregateError([])
let latestInvoice = invoice

15
wallets/server.js
View File

@ -24,9 +24,13 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln]
const MAX_PENDING_INVOICES_PER_WALLET = 25
export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { predecessorId, models }) {
export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { paymentAttempt, predecessorId, models }) {
// get the wallets in order of priority
const wallets = await getInvoiceableWallets(userId, { predecessorId, models })
const wallets = await getInvoiceableWallets(userId, {
paymentAttempt,
predecessorId,
models
})
msats = toPositiveNumber(msats)
@ -79,7 +83,7 @@ export async function createInvoice (userId, { msats, description, descriptionHa
export async function createWrappedInvoice (userId,
{ msats, feePercent, description, descriptionHash, expiry = 360 },
{ predecessorId, models, me, lnd }) {
{ paymentAttempt, predecessorId, models, me, lnd }) {
let logger, bolt11
try {
const { invoice, wallet } = await createInvoice(userId, {
@ -88,7 +92,7 @@ export async function createWrappedInvoice (userId,
description,
descriptionHash,
expiry
}, { predecessorId, models })
}, { paymentAttempt, predecessorId, models })
logger = walletLogger({ wallet, models })
bolt11 = invoice
@ -108,7 +112,7 @@ export async function createWrappedInvoice (userId,
}
}
export async function getInvoiceableWallets (userId, { predecessorId, models }) {
export async function getInvoiceableWallets (userId, { paymentAttempt, predecessorId, models }) {
// filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices.
// the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it
// so it has not been updated yet.
@ -141,6 +145,7 @@ export async function getInvoiceableWallets (userId, { predecessorId, models })
FROM "Invoice"
JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId"
WHERE "Invoice"."actionState" = 'RETRYING'
AND "Invoice"."paymentAttempt" = ${paymentAttempt}
)
SELECT
"InvoiceForward"."walletId"