From a46f81f1e18b8668836908e3bb56a626e37eb566 Mon Sep 17 00:00:00 2001 From: ekzyis Date: Tue, 10 Dec 2024 21:15:29 +0100 Subject: [PATCH] Receiver fallbacks (#1688) * Use same naming scheme between ln containers and env vars * Add router_lnd container * Only open channels to router_lnd * Use 1sat base fee and 0ppm fee rate * Add script to test routing * Also fund router_lnd wallet * Receiver fallbacks * Rename to predecessorId * Remove useless wallet table join * Missing renaming to predecessor * Fix payment stuck on sender error We want to await the invoice poll promise so we can check for receiver errors, but in case of sender errors, the promise will never settle. * Don't log failed forwards as sender errors * fix check for receiver error --------- Co-authored-by: Keyan <34140557+huumn@users.noreply.github.com> Co-authored-by: k00b --- api/paidAction/index.js | 48 +++++----- api/resolvers/wallet.js | 9 ++ api/typeDefs/wallet.js | 1 + components/use-invoice.js | 15 +-- fragments/wallet.js | 1 + .../migration.sql | 14 +++ prisma/schema.prisma | 68 +++++++------- wallets/errors.js | 8 ++ wallets/payment.js | 92 ++++++++++++------- wallets/server.js | 61 ++++++++---- 10 files changed, 207 insertions(+), 110 deletions(-) create mode 100644 prisma/migrations/20241206155927_invoice_predecessors/migration.sql diff --git a/api/paidAction/index.js b/api/paidAction/index.js index cc9ced4a..7e65c4eb 100644 --- a/api/paidAction/index.js +++ b/api/paidAction/index.js @@ -317,34 +317,39 @@ export async function retryPaidAction (actionType, args, incomingContext) { optimistic: actionOptimistic, me: await models.user.findUnique({ where: { id: parseInt(me.id) } }), cost: BigInt(msatsRequested), - actionId + actionId, + predecessorId: failedInvoice.id } let invoiceArgs const invoiceForward = await models.invoiceForward.findUnique({ - where: { invoiceId: failedInvoice.id }, + where: { + invoiceId: failedInvoice.id + }, include: { - wallet: true, - invoice: true, - withdrawl: true + wallet: true } }) - // TODO: receiver fallbacks - // use next receiver wallet if forward failed (we currently immediately fallback to SN) - const failedForward = invoiceForward?.withdrawl && invoiceForward.withdrawl.actionState !== 'CONFIRMED' - if (invoiceForward && !failedForward) { - const { userId } = invoiceForward.wallet - const { invoice: bolt11, wrappedInvoice: wrappedBolt11, wallet, maxFee } = await createWrappedInvoice(userId, { - msats: failedInvoice.msatsRequested, - feePercent: await action.getSybilFeePercent?.(actionArgs, retryContext), - description: await action.describe?.(actionArgs, retryContext), - expiry: INVOICE_EXPIRE_SECS - }, retryContext) - invoiceArgs = { bolt11, wrappedBolt11, wallet, maxFee } - } else { - invoiceArgs = await createSNInvoice(actionType, actionArgs, retryContext) + + if (invoiceForward) { + // this is a wrapped invoice, we need to retry it with receiver fallbacks + try { + const { userId } = invoiceForward.wallet + // this will return an invoice from the first receiver wallet that didn't fail yet and throw if none is available + const { invoice: bolt11, wrappedInvoice: wrappedBolt11, wallet, maxFee } = await createWrappedInvoice(userId, { + msats: failedInvoice.msatsRequested, + feePercent: await action.getSybilFeePercent?.(actionArgs, retryContext), + description: await action.describe?.(actionArgs, retryContext), + expiry: INVOICE_EXPIRE_SECS + }, retryContext) + invoiceArgs = { bolt11, wrappedBolt11, wallet, maxFee } + } catch (err) { + console.log('failed to retry wrapped invoice, falling back to SN:', err) + } } + invoiceArgs ??= await createSNInvoice(actionType, actionArgs, retryContext) + return await models.$transaction(async tx => { const context = { ...retryContext, tx, invoiceArgs } @@ -404,7 +409,7 @@ async function createSNInvoice (actionType, args, context) { } async function createDbInvoice (actionType, args, context) { - const { me, models, tx, cost, optimistic, actionId, invoiceArgs } = context + const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs const db = tx ?? models @@ -429,7 +434,8 @@ async function createDbInvoice (actionType, args, context) { actionOptimistic: optimistic, actionArgs: args, expiresAt, - actionId + actionId, + predecessorId } let invoice diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index b0f9f779..7d578cb7 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -606,6 +606,15 @@ const resolvers = { satsReceived: i => msatsToSats(i.msatsReceived), satsRequested: i => msatsToSats(i.msatsRequested), // we never want to fetch the sensitive data full monty in nested resolvers + forwardStatus: async (invoice, args, { models }) => { + const forward = await models.invoiceForward.findUnique({ + where: { invoiceId: Number(invoice.id) }, + include: { + withdrawl: true + } + }) + return forward?.withdrawl?.status + }, forwardedSats: async (invoice, args, { models }) => { const msats = (await models.invoiceForward.findUnique({ where: { invoiceId: Number(invoice.id) }, diff --git a/api/typeDefs/wallet.js b/api/typeDefs/wallet.js index 97f4f69e..932b67bc 100644 --- a/api/typeDefs/wallet.js +++ b/api/typeDefs/wallet.js @@ -129,6 +129,7 @@ const typeDefs = ` item: Item itemAct: ItemAct forwardedSats: Int + forwardStatus: String } type Withdrawl { diff --git a/components/use-invoice.js b/components/use-invoice.js index 977aa2d7..f5af94da 100644 --- a/components/use-invoice.js +++ b/components/use-invoice.js @@ -1,8 +1,8 @@ import { useApolloClient, useMutation } from '@apollo/client' import { useCallback } from 'react' +import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors' import { RETRY_PAID_ACTION } from '@/fragments/paidAction' import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet' -import { InvoiceExpiredError, InvoiceCanceledError } from '@/wallets/errors' export default function useInvoice () { const client = useApolloClient() @@ -16,20 +16,21 @@ export default function useInvoice () { throw error } - const { cancelled, cancelledAt, actionError, actionState, expiresAt } = data.invoice + const { cancelled, cancelledAt, actionError, expiresAt, forwardStatus } = data.invoice const expired = cancelledAt && new Date(expiresAt) < new Date(cancelledAt) if (expired) { throw new InvoiceExpiredError(data.invoice) } - if (cancelled || actionError) { - throw new InvoiceCanceledError(data.invoice, actionError) + const failed = cancelled || actionError + + if (failed && (forwardStatus && forwardStatus !== 'CONFIRMED')) { + throw new WalletReceiverError(data.invoice) } - // write to cache if paid - if (actionState === 'PAID') { - client.writeQuery({ query: INVOICE, variables: { id }, data: { invoice: data.invoice } }) + if (failed) { + throw new InvoiceCanceledError(data.invoice, actionError) } return { invoice: data.invoice, check: that(data.invoice) } diff --git a/fragments/wallet.js b/fragments/wallet.js index 7fe3fc49..6f84f4af 100644 --- a/fragments/wallet.js +++ b/fragments/wallet.js @@ -23,6 +23,7 @@ export const INVOICE_FIELDS = gql` actionError confirmedPreimage forwardedSats + forwardStatus }` export const INVOICE_FULL = gql` diff --git a/prisma/migrations/20241206155927_invoice_predecessors/migration.sql b/prisma/migrations/20241206155927_invoice_predecessors/migration.sql new file mode 100644 index 00000000..20260a45 --- /dev/null +++ b/prisma/migrations/20241206155927_invoice_predecessors/migration.sql @@ -0,0 +1,14 @@ +/* + Warnings: + + - A unique constraint covering the columns `[predecessorId]` on the table `Invoice` will be added. If there are existing duplicate values, this will fail. + +*/ +-- AlterTable +ALTER TABLE "Invoice" ADD COLUMN "predecessorId" INTEGER; + +-- CreateIndex +CREATE UNIQUE INDEX "Invoice.predecessorId_unique" ON "Invoice"("predecessorId"); + +-- AddForeignKey +ALTER TABLE "Invoice" ADD CONSTRAINT "Invoice_predecessorId_fkey" FOREIGN KEY ("predecessorId") REFERENCES "Invoice"("id") ON DELETE CASCADE ON UPDATE CASCADE; diff --git a/prisma/schema.prisma b/prisma/schema.prisma index 65b6b7a3..59685b93 100644 --- a/prisma/schema.prisma +++ b/prisma/schema.prisma @@ -904,39 +904,41 @@ model ItemMention { } model Invoice { - id Int @id @default(autoincrement()) - createdAt DateTime @default(now()) @map("created_at") - updatedAt DateTime @default(now()) @updatedAt @map("updated_at") - userId Int - hash String @unique(map: "Invoice.hash_unique") - preimage String? @unique(map: "Invoice.preimage_unique") - isHeld Boolean? - bolt11 String - expiresAt DateTime - confirmedAt DateTime? - confirmedIndex BigInt? - cancelled Boolean @default(false) - cancelledAt DateTime? - msatsRequested BigInt - msatsReceived BigInt? - desc String? - comment String? - lud18Data Json? - user User @relation(fields: [userId], references: [id], onDelete: Cascade) - invoiceForward InvoiceForward? - - actionState InvoiceActionState? - actionType InvoiceActionType? - actionOptimistic Boolean? - actionId Int? - actionArgs Json? @db.JsonB - actionError String? - actionResult Json? @db.JsonB - ItemAct ItemAct[] - Item Item[] - Upload Upload[] - PollVote PollVote[] - PollBlindVote PollBlindVote[] + id Int @id @default(autoincrement()) + createdAt DateTime @default(now()) @map("created_at") + updatedAt DateTime @default(now()) @updatedAt @map("updated_at") + userId Int + hash String @unique(map: "Invoice.hash_unique") + preimage String? @unique(map: "Invoice.preimage_unique") + isHeld Boolean? + bolt11 String + expiresAt DateTime + confirmedAt DateTime? + confirmedIndex BigInt? + cancelled Boolean @default(false) + cancelledAt DateTime? + msatsRequested BigInt + msatsReceived BigInt? + desc String? + comment String? + lud18Data Json? + user User @relation(fields: [userId], references: [id], onDelete: Cascade) + invoiceForward InvoiceForward? + predecessorId Int? @unique(map: "Invoice.predecessorId_unique") + predecessorInvoice Invoice? @relation("PredecessorInvoice", fields: [predecessorId], references: [id], onDelete: Cascade) + successorInvoice Invoice? @relation("PredecessorInvoice") + actionState InvoiceActionState? + actionType InvoiceActionType? + actionOptimistic Boolean? + actionId Int? + actionArgs Json? @db.JsonB + actionError String? + actionResult Json? @db.JsonB + ItemAct ItemAct[] + Item Item[] + Upload Upload[] + PollVote PollVote[] + PollBlindVote PollBlindVote[] @@index([createdAt], map: "Invoice.created_at_index") @@index([userId], map: "Invoice.userId_index") diff --git a/wallets/errors.js b/wallets/errors.js index 510d8e78..13c9ca18 100644 --- a/wallets/errors.js +++ b/wallets/errors.js @@ -47,6 +47,14 @@ export class WalletSenderError extends WalletPaymentError { } } +export class WalletReceiverError extends WalletPaymentError { + constructor (invoice) { + super(`payment forwarding failed for invoice ${invoice.hash}`) + this.name = 'WalletReceiverError' + this.invoice = invoice + } +} + export class WalletsNotAvailableError extends WalletConfigurationError { constructor () { super('no wallet available') diff --git a/wallets/payment.js b/wallets/payment.js index 157e06ea..33baa717 100644 --- a/wallets/payment.js +++ b/wallets/payment.js @@ -5,14 +5,16 @@ import useInvoice from '@/components/use-invoice' import { FAST_POLL_INTERVAL } from '@/lib/constants' import { WalletsNotAvailableError, WalletSenderError, WalletAggregateError, WalletPaymentAggregateError, - WalletNotEnabledError, WalletSendNotConfiguredError, WalletPaymentError, WalletError + WalletNotEnabledError, WalletSendNotConfiguredError, WalletPaymentError, WalletError, WalletReceiverError } from '@/wallets/errors' import { canSend } from './common' import { useWalletLoggerFactory } from './logger' +import { withTimeout } from '@/lib/time' export function useWalletPayment () { const wallets = useSendWallets() const sendPayment = useSendPayment() + const loggerFactory = useWalletLoggerFactory() const invoiceHelper = useInvoice() return useCallback(async (invoice, { waitFor, updateOnFallback }) => { @@ -24,44 +26,72 @@ export function useWalletPayment () { throw new WalletsNotAvailableError() } - for (const [i, wallet] of wallets.entries()) { + for (let i = 0; i < wallets.length; i++) { + const wallet = wallets[i] + const logger = loggerFactory(wallet) + + const { bolt11 } = latestInvoice const controller = invoiceController(latestInvoice, invoiceHelper.isInvoice) + + const walletPromise = sendPayment(wallet, logger, latestInvoice) + const pollPromise = controller.wait(waitFor) + try { return await new Promise((resolve, reject) => { // can't await wallet payments since we might pay hold invoices and thus payments might not settle immediately. // that's why we separately check if we received the payment with the invoice controller. - sendPayment(wallet, latestInvoice).catch(reject) - controller.wait(waitFor) - .then(resolve) - .catch(reject) + walletPromise.catch(reject) + pollPromise.then(resolve).catch(reject) }) } catch (err) { - // cancel invoice to make sure it cannot be paid later and create new invoice to retry. - // we only need to do this if payment was attempted which is not the case if the wallet is not enabled. - if (err instanceof WalletPaymentError) { - await invoiceHelper.cancel(latestInvoice) + let paymentError = err + const message = `payment failed: ${paymentError.reason ?? paymentError.message}` - // is there another wallet to try? - const lastAttempt = i === wallets.length - 1 - if (!lastAttempt) { - latestInvoice = await invoiceHelper.retry(latestInvoice, { update: updateOnFallback }) + if (!(paymentError instanceof WalletError)) { + // payment failed for some reason unrelated to wallets (ie invoice expired or was canceled). + // bail out of attempting wallets. + logger.error(message, { bolt11 }) + throw paymentError + } + + // at this point, paymentError is always a wallet error, + // we just need to distinguish between receiver and sender errors + + try { + // we always await the poll promise here to check for failed forwards since sender wallet errors + // can be caused by them which we want to handle as receiver errors, not sender errors. + // but we don't wait forever because real sender errors will cause the poll promise to never settle. + await withTimeout(pollPromise, FAST_POLL_INTERVAL * 2.5) + } catch (err) { + if (err instanceof WalletError) { + paymentError = err } } - // TODO: receiver fallbacks - // - // if payment failed because of the receiver, we should use the same wallet again. - // if (err instanceof ReceiverError) { ... } - - // try next wallet if the payment failed because of the wallet - // and not because it expired or was canceled - if (err instanceof WalletError) { - aggregateError = new WalletAggregateError([aggregateError, err], latestInvoice) - continue + if (paymentError instanceof WalletReceiverError) { + // if payment failed because of the receiver, use the same wallet again + // and log this as info, not error + logger.info('failed to forward payment to receiver, retrying with new invoice', { bolt11 }) + i -= 1 + } else if (paymentError instanceof WalletPaymentError) { + // only log payment errors, not configuration errors + logger.error(message, { bolt11 }) } - // payment failed not because of the sender or receiver wallet. bail out of attemping wallets. - throw err + if (paymentError instanceof WalletPaymentError) { + // if a payment was attempted, cancel invoice to make sure it cannot be paid later and create new invoice to retry. + await invoiceHelper.cancel(latestInvoice) + } + + // only create a new invoice if we will try to pay with a wallet again + const retry = paymentError instanceof WalletReceiverError || i < wallets.length - 1 + if (retry) { + latestInvoice = await invoiceHelper.retry(latestInvoice, { update: updateOnFallback }) + } + + aggregateError = new WalletAggregateError([aggregateError, paymentError], latestInvoice) + + continue } finally { controller.stop() } @@ -111,11 +141,7 @@ function invoiceController (inv, isInvoice) { } function useSendPayment () { - const factory = useWalletLoggerFactory() - - return useCallback(async (wallet, invoice) => { - const logger = factory(wallet) - + return useCallback(async (wallet, logger, invoice) => { if (!wallet.config.enabled) { throw new WalletNotEnabledError(wallet.def.name) } @@ -131,9 +157,9 @@ function useSendPayment () { const preimage = await wallet.def.sendPayment(bolt11, wallet.config, { logger }) logger.ok(`↗ payment sent: ${formatSats(satsRequested)}`, { bolt11, preimage }) } catch (err) { + // we don't log the error here since we want to handle receiver errors separately const message = err.message || err.toString?.() - logger.error(`payment failed: ${message}`, { bolt11 }) throw new WalletSenderError(wallet.def.name, invoice, message) } - }, [factory]) + }, []) } diff --git a/wallets/server.js b/wallets/server.js index c329d767..b0da2aa7 100644 --- a/wallets/server.js +++ b/wallets/server.js @@ -24,9 +24,9 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln] const MAX_PENDING_INVOICES_PER_WALLET = 25 -export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { models }) { +export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { predecessorId, models }) { // get the wallets in order of priority - const wallets = await getInvoiceableWallets(userId, { models }) + const wallets = await getInvoiceableWallets(userId, { predecessorId, models }) msats = toPositiveNumber(msats) @@ -81,7 +81,7 @@ export async function createInvoice (userId, { msats, description, descriptionHa export async function createWrappedInvoice (userId, { msats, feePercent, description, descriptionHash, expiry = 360 }, - { models, me, lnd }) { + { predecessorId, models, me, lnd }) { let logger, bolt11 try { const { invoice, wallet } = await createInvoice(userId, { @@ -90,7 +90,7 @@ export async function createWrappedInvoice (userId, description, descriptionHash, expiry - }, { models }) + }, { predecessorId, models }) logger = walletLogger({ wallet, models }) bolt11 = invoice @@ -110,18 +110,47 @@ export async function createWrappedInvoice (userId, } } -export async function getInvoiceableWallets (userId, { models }) { - const wallets = await models.wallet.findMany({ - where: { userId, enabled: true }, - include: { - user: true - }, - orderBy: [ - { priority: 'asc' }, - // use id as tie breaker (older wallet first) - { id: 'asc' } - ] - }) +export async function getInvoiceableWallets (userId, { predecessorId, models }) { + // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices. + // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it + // so it has not been updated yet. + // if predecessorId is not provided, the subquery will be empty and thus no wallets are filtered out. + const wallets = await models.$queryRaw` + SELECT + "Wallet".*, + jsonb_build_object( + 'id', "users"."id", + 'hideInvoiceDesc', "users"."hideInvoiceDesc" + ) AS "user" + FROM "Wallet" + JOIN "users" ON "users"."id" = "Wallet"."userId" + WHERE + "Wallet"."userId" = ${userId} + AND "Wallet"."enabled" = true + AND "Wallet"."id" NOT IN ( + WITH RECURSIVE "Retries" AS ( + -- select the current failed invoice that we are currently retrying + -- this failed invoice will be used to start the recursion + SELECT "Invoice"."id", "Invoice"."predecessorId" + FROM "Invoice" + WHERE "Invoice"."id" = ${predecessorId} AND "Invoice"."actionState" = 'FAILED' + + UNION ALL + + -- recursive part: use predecessorId to select the previous invoice that failed in the chain + -- until there is no more previous invoice + SELECT "Invoice"."id", "Invoice"."predecessorId" + FROM "Invoice" + JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId" + WHERE "Invoice"."actionState" = 'RETRYING' + ) + SELECT + "InvoiceForward"."walletId" + FROM "Retries" + JOIN "InvoiceForward" ON "InvoiceForward"."invoiceId" = "Retries"."id" + WHERE "InvoiceForward"."withdrawlId" IS NOT NULL + ) + ORDER BY "Wallet"."priority" ASC, "Wallet"."id" ASC` const walletsWithDefs = wallets.map(wallet => { const w = walletDefs.find(w => w.walletType === wallet.type)