Receiver fallbacks (#1688)

* Use same naming scheme between ln containers and env vars * Add router_lnd container * Only open channels to router_lnd * Use 1sat base fee and 0ppm fee rate * Add script to test routing * Also fund router_lnd wallet * Receiver fallbacks * Rename to predecessorId * Remove useless wallet table join * Missing renaming to predecessor * Fix payment stuck on sender error We want to await the invoice poll promise so we can check for receiver errors, but in case of sender errors, the promise will never settle. * Don't log failed forwards as sender errors * fix check for receiver error --------- Co-authored-by: Keyan <34140557+huumn@users.noreply.github.com> Co-authored-by: k00b <k00b@stacker.news>
2024-12-10 21:15:29 +01:00 · 2024-12-10 21:15:29 +01:00 · a46f81f1e1
parent e6c74c965b
commit a46f81f1e1
10 changed files with 207 additions and 110 deletions
--- a/api/paidAction/index.js
+++ b/api/paidAction/index.js
@ -317,34 +317,39 @@ export async function retryPaidAction (actionType, args, incomingContext) {
    optimistic: actionOptimistic,
    me: await models.user.findUnique({ where: { id: parseInt(me.id) } }),
    cost: BigInt(msatsRequested),
-    actionId
+    actionId,
+    predecessorId: failedInvoice.id
  }

  let invoiceArgs
  const invoiceForward = await models.invoiceForward.findUnique({
-    where: { invoiceId: failedInvoice.id },
+    where: {
+      invoiceId: failedInvoice.id
+    },
    include: {
-      wallet: true,
-      invoice: true,
-      withdrawl: true
+      wallet: true
    }
  })
-  // TODO: receiver fallbacks
-  // use next receiver wallet if forward failed (we currently immediately fallback to SN)
-  const failedForward = invoiceForward?.withdrawl && invoiceForward.withdrawl.actionState !== 'CONFIRMED'
-  if (invoiceForward && !failedForward) {
-    const { userId } = invoiceForward.wallet
-    const { invoice: bolt11, wrappedInvoice: wrappedBolt11, wallet, maxFee } = await createWrappedInvoice(userId, {
-      msats: failedInvoice.msatsRequested,
-      feePercent: await action.getSybilFeePercent?.(actionArgs, retryContext),
-      description: await action.describe?.(actionArgs, retryContext),
-      expiry: INVOICE_EXPIRE_SECS
-    }, retryContext)
-    invoiceArgs = { bolt11, wrappedBolt11, wallet, maxFee }
-  } else {
-    invoiceArgs = await createSNInvoice(actionType, actionArgs, retryContext)
+
+  if (invoiceForward) {
+    // this is a wrapped invoice, we need to retry it with receiver fallbacks
+    try {
+      const { userId } = invoiceForward.wallet
+      // this will return an invoice from the first receiver wallet that didn't fail yet and throw if none is available
+      const { invoice: bolt11, wrappedInvoice: wrappedBolt11, wallet, maxFee } = await createWrappedInvoice(userId, {
+        msats: failedInvoice.msatsRequested,
+        feePercent: await action.getSybilFeePercent?.(actionArgs, retryContext),
+        description: await action.describe?.(actionArgs, retryContext),
+        expiry: INVOICE_EXPIRE_SECS
+      }, retryContext)
+      invoiceArgs = { bolt11, wrappedBolt11, wallet, maxFee }
+    } catch (err) {
+      console.log('failed to retry wrapped invoice, falling back to SN:', err)
+    }
  }

+  invoiceArgs ??= await createSNInvoice(actionType, actionArgs, retryContext)
+
  return await models.$transaction(async tx => {
    const context = { ...retryContext, tx, invoiceArgs }

@ -404,7 +409,7 @@ async function createSNInvoice (actionType, args, context) {
 }

 async function createDbInvoice (actionType, args, context) {
-  const { me, models, tx, cost, optimistic, actionId, invoiceArgs } = context
+  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context
  const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs

  const db = tx ?? models
@ -429,7 +434,8 @@ async function createDbInvoice (actionType, args, context) {
    actionOptimistic: optimistic,
    actionArgs: args,
    expiresAt,
-    actionId
+    actionId,
+    predecessorId
  }

  let invoice
--- a/api/resolvers/wallet.js
+++ b/api/resolvers/wallet.js
@ -606,6 +606,15 @@ const resolvers = {
    satsReceived: i => msatsToSats(i.msatsReceived),
    satsRequested: i => msatsToSats(i.msatsRequested),
    // we never want to fetch the sensitive data full monty in nested resolvers
+    forwardStatus: async (invoice, args, { models }) => {
+      const forward = await models.invoiceForward.findUnique({
+        where: { invoiceId: Number(invoice.id) },
+        include: {
+          withdrawl: true
+        }
+      })
+      return forward?.withdrawl?.status
+    },
    forwardedSats: async (invoice, args, { models }) => {
      const msats = (await models.invoiceForward.findUnique({
        where: { invoiceId: Number(invoice.id) },
--- a/api/typeDefs/wallet.js
+++ b/api/typeDefs/wallet.js
@ -129,6 +129,7 @@ const typeDefs = `
    item: Item
    itemAct: ItemAct
    forwardedSats: Int
+    forwardStatus: String
  }

  type Withdrawl {
--- a/components/use-invoice.js
+++ b/components/use-invoice.js
@ -1,8 +1,8 @@
 import { useApolloClient, useMutation } from '@apollo/client'
 import { useCallback } from 'react'
+import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors'
 import { RETRY_PAID_ACTION } from '@/fragments/paidAction'
 import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet'
-import { InvoiceExpiredError, InvoiceCanceledError } from '@/wallets/errors'

 export default function useInvoice () {
  const client = useApolloClient()
@ -16,20 +16,21 @@ export default function useInvoice () {
      throw error
    }

-    const { cancelled, cancelledAt, actionError, actionState, expiresAt } = data.invoice
+    const { cancelled, cancelledAt, actionError, expiresAt, forwardStatus } = data.invoice

    const expired = cancelledAt && new Date(expiresAt) < new Date(cancelledAt)
    if (expired) {
      throw new InvoiceExpiredError(data.invoice)
    }

-    if (cancelled || actionError) {
-      throw new InvoiceCanceledError(data.invoice, actionError)
+    const failed = cancelled || actionError
+
+    if (failed && (forwardStatus && forwardStatus !== 'CONFIRMED')) {
+      throw new WalletReceiverError(data.invoice)
    }

-    // write to cache if paid
-    if (actionState === 'PAID') {
-      client.writeQuery({ query: INVOICE, variables: { id }, data: { invoice: data.invoice } })
+    if (failed) {
+      throw new InvoiceCanceledError(data.invoice, actionError)
    }

    return { invoice: data.invoice, check: that(data.invoice) }
--- a/fragments/wallet.js
+++ b/fragments/wallet.js
@ -23,6 +23,7 @@ export const INVOICE_FIELDS = gql`
    actionError
    confirmedPreimage
    forwardedSats
+    forwardStatus
  }`

 export const INVOICE_FULL = gql`
--- a/prisma/migrations/20241206155927_invoice_predecessors/migration.sql
+++ b/prisma/migrations/20241206155927_invoice_predecessors/migration.sql
@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[predecessorId]` on the table `Invoice` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- AlterTable
+ALTER TABLE "Invoice" ADD COLUMN     "predecessorId" INTEGER;
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Invoice.predecessorId_unique" ON "Invoice"("predecessorId");
+
+-- AddForeignKey
+ALTER TABLE "Invoice" ADD CONSTRAINT "Invoice_predecessorId_fkey" FOREIGN KEY ("predecessorId") REFERENCES "Invoice"("id") ON DELETE CASCADE ON UPDATE CASCADE;
--- a/prisma/schema.prisma
+++ b/prisma/schema.prisma
@ -904,39 +904,41 @@ model ItemMention {
 }

 model Invoice {
-  id             Int             @id @default(autoincrement())
-  createdAt      DateTime        @default(now()) @map("created_at")
-  updatedAt      DateTime        @default(now()) @updatedAt @map("updated_at")
-  userId         Int
-  hash           String          @unique(map: "Invoice.hash_unique")
-  preimage       String?         @unique(map: "Invoice.preimage_unique")
-  isHeld         Boolean?
-  bolt11         String
-  expiresAt      DateTime
-  confirmedAt    DateTime?
-  confirmedIndex BigInt?
-  cancelled      Boolean         @default(false)
-  cancelledAt    DateTime?
-  msatsRequested BigInt
-  msatsReceived  BigInt?
-  desc           String?
-  comment        String?
-  lud18Data      Json?
-  user           User            @relation(fields: [userId], references: [id], onDelete: Cascade)
-  invoiceForward InvoiceForward?
-
-  actionState      InvoiceActionState?
-  actionType       InvoiceActionType?
-  actionOptimistic Boolean?
-  actionId         Int?
-  actionArgs       Json?               @db.JsonB
-  actionError      String?
-  actionResult     Json?               @db.JsonB
-  ItemAct          ItemAct[]
-  Item             Item[]
-  Upload           Upload[]
-  PollVote         PollVote[]
-  PollBlindVote    PollBlindVote[]
+  id                 Int                 @id @default(autoincrement())
+  createdAt          DateTime            @default(now()) @map("created_at")
+  updatedAt          DateTime            @default(now()) @updatedAt @map("updated_at")
+  userId             Int
+  hash               String              @unique(map: "Invoice.hash_unique")
+  preimage           String?             @unique(map: "Invoice.preimage_unique")
+  isHeld             Boolean?
+  bolt11             String
+  expiresAt          DateTime
+  confirmedAt        DateTime?
+  confirmedIndex     BigInt?
+  cancelled          Boolean             @default(false)
+  cancelledAt        DateTime?
+  msatsRequested     BigInt
+  msatsReceived      BigInt?
+  desc               String?
+  comment            String?
+  lud18Data          Json?
+  user               User                @relation(fields: [userId], references: [id], onDelete: Cascade)
+  invoiceForward     InvoiceForward?
+  predecessorId      Int?                @unique(map: "Invoice.predecessorId_unique")
+  predecessorInvoice Invoice?            @relation("PredecessorInvoice", fields: [predecessorId], references: [id], onDelete: Cascade)
+  successorInvoice   Invoice?            @relation("PredecessorInvoice")
+  actionState        InvoiceActionState?
+  actionType         InvoiceActionType?
+  actionOptimistic   Boolean?
+  actionId           Int?
+  actionArgs         Json?               @db.JsonB
+  actionError        String?
+  actionResult       Json?               @db.JsonB
+  ItemAct            ItemAct[]
+  Item               Item[]
+  Upload             Upload[]
+  PollVote           PollVote[]
+  PollBlindVote      PollBlindVote[]

  @@index([createdAt], map: "Invoice.created_at_index")
  @@index([userId], map: "Invoice.userId_index")
--- a/wallets/errors.js
+++ b/wallets/errors.js
@ -47,6 +47,14 @@ export class WalletSenderError extends WalletPaymentError {
  }
 }

+export class WalletReceiverError extends WalletPaymentError {
+  constructor (invoice) {
+    super(`payment forwarding failed for invoice ${invoice.hash}`)
+    this.name = 'WalletReceiverError'
+    this.invoice = invoice
+  }
+}
+
 export class WalletsNotAvailableError extends WalletConfigurationError {
  constructor () {
    super('no wallet available')
--- a/wallets/payment.js
+++ b/wallets/payment.js
@ -5,14 +5,16 @@ import useInvoice from '@/components/use-invoice'
 import { FAST_POLL_INTERVAL } from '@/lib/constants'
 import {
  WalletsNotAvailableError, WalletSenderError, WalletAggregateError, WalletPaymentAggregateError,
-  WalletNotEnabledError, WalletSendNotConfiguredError, WalletPaymentError, WalletError
+  WalletNotEnabledError, WalletSendNotConfiguredError, WalletPaymentError, WalletError, WalletReceiverError
 } from '@/wallets/errors'
 import { canSend } from './common'
 import { useWalletLoggerFactory } from './logger'
+import { withTimeout } from '@/lib/time'

 export function useWalletPayment () {
  const wallets = useSendWallets()
  const sendPayment = useSendPayment()
+  const loggerFactory = useWalletLoggerFactory()
  const invoiceHelper = useInvoice()

  return useCallback(async (invoice, { waitFor, updateOnFallback }) => {
@ -24,44 +26,72 @@ export function useWalletPayment () {
      throw new WalletsNotAvailableError()
    }

-    for (const [i, wallet] of wallets.entries()) {
+    for (let i = 0; i < wallets.length; i++) {
+      const wallet = wallets[i]
+      const logger = loggerFactory(wallet)
+
+      const { bolt11 } = latestInvoice
      const controller = invoiceController(latestInvoice, invoiceHelper.isInvoice)
+
+      const walletPromise = sendPayment(wallet, logger, latestInvoice)
+      const pollPromise = controller.wait(waitFor)
+
      try {
        return await new Promise((resolve, reject) => {
          // can't await wallet payments since we might pay hold invoices and thus payments might not settle immediately.
          // that's why we separately check if we received the payment with the invoice controller.
-          sendPayment(wallet, latestInvoice).catch(reject)
-          controller.wait(waitFor)
-            .then(resolve)
-            .catch(reject)
+          walletPromise.catch(reject)
+          pollPromise.then(resolve).catch(reject)
        })
      } catch (err) {
-        // cancel invoice to make sure it cannot be paid later and create new invoice to retry.
-        // we only need to do this if payment was attempted which is not the case if the wallet is not enabled.
-        if (err instanceof WalletPaymentError) {
-          await invoiceHelper.cancel(latestInvoice)
+        let paymentError = err
+        const message = `payment failed: ${paymentError.reason ?? paymentError.message}`

-          // is there another wallet to try?
-          const lastAttempt = i === wallets.length - 1
-          if (!lastAttempt) {
-            latestInvoice = await invoiceHelper.retry(latestInvoice, { update: updateOnFallback })
+        if (!(paymentError instanceof WalletError)) {
+          // payment failed for some reason unrelated to wallets (ie invoice expired or was canceled).
+          // bail out of attempting wallets.
+          logger.error(message, { bolt11 })
+          throw paymentError
+        }
+
+        // at this point, paymentError is always a wallet error,
+        // we just need to distinguish between receiver and sender errors
+
+        try {
+          // we always await the poll promise here to check for failed forwards since sender wallet errors
+          // can be caused by them which we want to handle as receiver errors, not sender errors.
+          // but we don't wait forever because real sender errors will cause the poll promise to never settle.
+          await withTimeout(pollPromise, FAST_POLL_INTERVAL * 2.5)
+        } catch (err) {
+          if (err instanceof WalletError) {
+            paymentError = err
          }
        }

-        // TODO: receiver fallbacks
-        //
-        // if payment failed because of the receiver, we should use the same wallet again.
-        // if (err instanceof ReceiverError) { ... }
-
-        // try next wallet if the payment failed because of the wallet
-        // and not because it expired or was canceled
-        if (err instanceof WalletError) {
-          aggregateError = new WalletAggregateError([aggregateError, err], latestInvoice)
-          continue
+        if (paymentError instanceof WalletReceiverError) {
+          // if payment failed because of the receiver, use the same wallet again
+          // and log this as info, not error
+          logger.info('failed to forward payment to receiver, retrying with new invoice', { bolt11 })
+          i -= 1
+        } else if (paymentError instanceof WalletPaymentError) {
+          // only log payment errors, not configuration errors
+          logger.error(message, { bolt11 })
        }

-        // payment failed not because of the sender or receiver wallet. bail out of attemping wallets.
-        throw err
+        if (paymentError instanceof WalletPaymentError) {
+          // if a payment was attempted, cancel invoice to make sure it cannot be paid later and create new invoice to retry.
+          await invoiceHelper.cancel(latestInvoice)
+        }
+
+        // only create a new invoice if we will try to pay with a wallet again
+        const retry = paymentError instanceof WalletReceiverError || i < wallets.length - 1
+        if (retry) {
+          latestInvoice = await invoiceHelper.retry(latestInvoice, { update: updateOnFallback })
+        }
+
+        aggregateError = new WalletAggregateError([aggregateError, paymentError], latestInvoice)
+
+        continue
      } finally {
        controller.stop()
      }
@ -111,11 +141,7 @@ function invoiceController (inv, isInvoice) {
 }

 function useSendPayment () {
-  const factory = useWalletLoggerFactory()
-
-  return useCallback(async (wallet, invoice) => {
-    const logger = factory(wallet)
-
+  return useCallback(async (wallet, logger, invoice) => {
    if (!wallet.config.enabled) {
      throw new WalletNotEnabledError(wallet.def.name)
    }
@ -131,9 +157,9 @@ function useSendPayment () {
      const preimage = await wallet.def.sendPayment(bolt11, wallet.config, { logger })
      logger.ok(`↗ payment sent: ${formatSats(satsRequested)}`, { bolt11, preimage })
    } catch (err) {
+      // we don't log the error here since we want to handle receiver errors separately
      const message = err.message || err.toString?.()
-      logger.error(`payment failed: ${message}`, { bolt11 })
      throw new WalletSenderError(wallet.def.name, invoice, message)
    }
-  }, [factory])
+  }, [])
 }
--- a/wallets/server.js
+++ b/wallets/server.js
@ -24,9 +24,9 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln]

 const MAX_PENDING_INVOICES_PER_WALLET = 25

-export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { models }) {
+export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { predecessorId, models }) {
  // get the wallets in order of priority
-  const wallets = await getInvoiceableWallets(userId, { models })
+  const wallets = await getInvoiceableWallets(userId, { predecessorId, models })

  msats = toPositiveNumber(msats)

@ -81,7 +81,7 @@ export async function createInvoice (userId, { msats, description, descriptionHa

 export async function createWrappedInvoice (userId,
  { msats, feePercent, description, descriptionHash, expiry = 360 },
-  { models, me, lnd }) {
+  { predecessorId, models, me, lnd }) {
  let logger, bolt11
  try {
    const { invoice, wallet } = await createInvoice(userId, {
@ -90,7 +90,7 @@ export async function createWrappedInvoice (userId,
      description,
      descriptionHash,
      expiry
-    }, { models })
+    }, { predecessorId, models })

    logger = walletLogger({ wallet, models })
    bolt11 = invoice
@ -110,18 +110,47 @@ export async function createWrappedInvoice (userId,
  }
 }

-export async function getInvoiceableWallets (userId, { models }) {
-  const wallets = await models.wallet.findMany({
-    where: { userId, enabled: true },
-    include: {
-      user: true
-    },
-    orderBy: [
-      { priority: 'asc' },
-      // use id as tie breaker (older wallet first)
-      { id: 'asc' }
-    ]
-  })
+export async function getInvoiceableWallets (userId, { predecessorId, models }) {
+  // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices.
+  // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it
+  // so it has not been updated yet.
+  // if predecessorId is not provided, the subquery will be empty and thus no wallets are filtered out.
+  const wallets = await models.$queryRaw`
+    SELECT
+      "Wallet".*,
+      jsonb_build_object(
+        'id', "users"."id",
+        'hideInvoiceDesc', "users"."hideInvoiceDesc"
+      ) AS "user"
+    FROM "Wallet"
+    JOIN "users" ON "users"."id" = "Wallet"."userId"
+    WHERE
+      "Wallet"."userId" = ${userId}
+      AND "Wallet"."enabled" = true
+      AND "Wallet"."id" NOT IN (
+        WITH RECURSIVE "Retries" AS (
+          -- select the current failed invoice that we are currently retrying
+          -- this failed invoice will be used to start the recursion
+          SELECT "Invoice"."id", "Invoice"."predecessorId"
+          FROM "Invoice"
+          WHERE "Invoice"."id" = ${predecessorId} AND "Invoice"."actionState" = 'FAILED'
+
+          UNION ALL
+
+          -- recursive part: use predecessorId to select the previous invoice that failed in the chain
+          -- until there is no more previous invoice
+          SELECT "Invoice"."id", "Invoice"."predecessorId"
+          FROM "Invoice"
+          JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId"
+          WHERE "Invoice"."actionState" = 'RETRYING'
+        )
+        SELECT
+          "InvoiceForward"."walletId"
+        FROM "Retries"
+        JOIN "InvoiceForward" ON "InvoiceForward"."invoiceId" = "Retries"."id"
+        WHERE "InvoiceForward"."withdrawlId" IS NOT NULL
+      )
+    ORDER BY "Wallet"."priority" ASC, "Wallet"."id" ASC`

  const walletsWithDefs = wallets.map(wallet => {
    const w = walletDefs.find(w => w.walletType === wallet.type)