Receiver fallbacks (#1688)

* Use same naming scheme between ln containers and env vars

* Add router_lnd container

* Only open channels to router_lnd

* Use 1sat base fee and 0ppm fee rate

* Add script to test routing

* Also fund router_lnd wallet

* Receiver fallbacks

* Rename to predecessorId

* Remove useless wallet table join

* Missing renaming to predecessor

* Fix payment stuck on sender error

We want to await the invoice poll promise so we can check for receiver errors, but in case of sender errors, the promise will never settle.

* Don't log failed forwards as sender errors

* fix check for receiver error

---------

Co-authored-by: Keyan <34140557+huumn@users.noreply.github.com>
Co-authored-by: k00b <k00b@stacker.news>

api/paidAction/index.js

@@ -317,23 +317,25 @@ export async function retryPaidAction (actionType, args, incomingContext) {
     optimistic: actionOptimistic,
     me: await models.user.findUnique({ where: { id: parseInt(me.id) } }),
     cost: BigInt(msatsRequested),
-    actionId
+    actionId,
+    predecessorId: failedInvoice.id
   }
 
   let invoiceArgs
   const invoiceForward = await models.invoiceForward.findUnique({
-    where: { invoiceId: failedInvoice.id },
+    where: {
+      invoiceId: failedInvoice.id
+    },
     include: {
-      wallet: true,
+      wallet: true
-      invoice: true,
-      withdrawl: true
     }
   })
-  // TODO: receiver fallbacks
+
-  // use next receiver wallet if forward failed (we currently immediately fallback to SN)
+  if (invoiceForward) {
-  const failedForward = invoiceForward?.withdrawl && invoiceForward.withdrawl.actionState !== 'CONFIRMED'
+    // this is a wrapped invoice, we need to retry it with receiver fallbacks
-  if (invoiceForward && !failedForward) {
+    try {
       const { userId } = invoiceForward.wallet
+      // this will return an invoice from the first receiver wallet that didn't fail yet and throw if none is available
       const { invoice: bolt11, wrappedInvoice: wrappedBolt11, wallet, maxFee } = await createWrappedInvoice(userId, {
         msats: failedInvoice.msatsRequested,
         feePercent: await action.getSybilFeePercent?.(actionArgs, retryContext),
@@ -341,9 +343,12 @@ export async function retryPaidAction (actionType, args, incomingContext) {
         expiry: INVOICE_EXPIRE_SECS
       }, retryContext)
       invoiceArgs = { bolt11, wrappedBolt11, wallet, maxFee }
-  } else {
+    } catch (err) {
-    invoiceArgs = await createSNInvoice(actionType, actionArgs, retryContext)
+      console.log('failed to retry wrapped invoice, falling back to SN:', err)
     }
+  }
+
+  invoiceArgs ??= await createSNInvoice(actionType, actionArgs, retryContext)
 
   return await models.$transaction(async tx => {
     const context = { ...retryContext, tx, invoiceArgs }
@@ -404,7 +409,7 @@ async function createSNInvoice (actionType, args, context) {
 }
 
 async function createDbInvoice (actionType, args, context) {
-  const { me, models, tx, cost, optimistic, actionId, invoiceArgs } = context
+  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context
   const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs
 
   const db = tx ?? models
@@ -429,7 +434,8 @@ async function createDbInvoice (actionType, args, context) {
     actionOptimistic: optimistic,
     actionArgs: args,
     expiresAt,
-    actionId
+    actionId,
+    predecessorId
   }
 
   let invoice

api/resolvers/wallet.js

@@ -606,6 +606,15 @@ const resolvers = {
     satsReceived: i => msatsToSats(i.msatsReceived),
     satsRequested: i => msatsToSats(i.msatsRequested),
     // we never want to fetch the sensitive data full monty in nested resolvers
+    forwardStatus: async (invoice, args, { models }) => {
+      const forward = await models.invoiceForward.findUnique({
+        where: { invoiceId: Number(invoice.id) },
+        include: {
+          withdrawl: true
+        }
+      })
+      return forward?.withdrawl?.status
+    },
     forwardedSats: async (invoice, args, { models }) => {
       const msats = (await models.invoiceForward.findUnique({
         where: { invoiceId: Number(invoice.id) },

api/typeDefs/wallet.js

@@ -129,6 +129,7 @@ const typeDefs = `
     item: Item
     itemAct: ItemAct
     forwardedSats: Int
+    forwardStatus: String
   }
 
   type Withdrawl {

components/use-invoice.js

@@ -1,8 +1,8 @@
 import { useApolloClient, useMutation } from '@apollo/client'
 import { useCallback } from 'react'
+import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors'
 import { RETRY_PAID_ACTION } from '@/fragments/paidAction'
 import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet'
-import { InvoiceExpiredError, InvoiceCanceledError } from '@/wallets/errors'
 
 export default function useInvoice () {
   const client = useApolloClient()
@@ -16,20 +16,21 @@ export default function useInvoice () {
       throw error
     }
 
-    const { cancelled, cancelledAt, actionError, actionState, expiresAt } = data.invoice
+    const { cancelled, cancelledAt, actionError, expiresAt, forwardStatus } = data.invoice
 
     const expired = cancelledAt && new Date(expiresAt) < new Date(cancelledAt)
     if (expired) {
       throw new InvoiceExpiredError(data.invoice)
     }
 
-    if (cancelled || actionError) {
+    const failed = cancelled || actionError
-      throw new InvoiceCanceledError(data.invoice, actionError)
+
+    if (failed && (forwardStatus && forwardStatus !== 'CONFIRMED')) {
+      throw new WalletReceiverError(data.invoice)
     }
 
-    // write to cache if paid
+    if (failed) {
-    if (actionState === 'PAID') {
+      throw new InvoiceCanceledError(data.invoice, actionError)
-      client.writeQuery({ query: INVOICE, variables: { id }, data: { invoice: data.invoice } })
     }
 
     return { invoice: data.invoice, check: that(data.invoice) }

fragments/wallet.js

@@ -23,6 +23,7 @@ export const INVOICE_FIELDS = gql`
     actionError
     confirmedPreimage
     forwardedSats
+    forwardStatus
   }`
 
 export const INVOICE_FULL = gql`

prisma/migrations/20241206155927_invoice_predecessors/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[predecessorId]` on the table `Invoice` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- AlterTable
+ALTER TABLE "Invoice" ADD COLUMN     "predecessorId" INTEGER;
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Invoice.predecessorId_unique" ON "Invoice"("predecessorId");
+
+-- AddForeignKey
+ALTER TABLE "Invoice" ADD CONSTRAINT "Invoice_predecessorId_fkey" FOREIGN KEY ("predecessorId") REFERENCES "Invoice"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/schema.prisma

@@ -924,7 +924,9 @@ model Invoice {
   lud18Data          Json?
   user               User                @relation(fields: [userId], references: [id], onDelete: Cascade)
   invoiceForward     InvoiceForward?
-
+  predecessorId      Int?                @unique(map: "Invoice.predecessorId_unique")
+  predecessorInvoice Invoice?            @relation("PredecessorInvoice", fields: [predecessorId], references: [id], onDelete: Cascade)
+  successorInvoice   Invoice?            @relation("PredecessorInvoice")
   actionState        InvoiceActionState?
   actionType         InvoiceActionType?
   actionOptimistic   Boolean?

wallets/errors.js

@@ -47,6 +47,14 @@ export class WalletSenderError extends WalletPaymentError {
   }
 }
 
+export class WalletReceiverError extends WalletPaymentError {
+  constructor (invoice) {
+    super(`payment forwarding failed for invoice ${invoice.hash}`)
+    this.name = 'WalletReceiverError'
+    this.invoice = invoice
+  }
+}
+
 export class WalletsNotAvailableError extends WalletConfigurationError {
   constructor () {
     super('no wallet available')

wallets/payment.js

@@ -5,14 +5,16 @@ import useInvoice from '@/components/use-invoice'
 import { FAST_POLL_INTERVAL } from '@/lib/constants'
 import {
   WalletsNotAvailableError, WalletSenderError, WalletAggregateError, WalletPaymentAggregateError,
-  WalletNotEnabledError, WalletSendNotConfiguredError, WalletPaymentError, WalletError
+  WalletNotEnabledError, WalletSendNotConfiguredError, WalletPaymentError, WalletError, WalletReceiverError
 } from '@/wallets/errors'
 import { canSend } from './common'
 import { useWalletLoggerFactory } from './logger'
+import { withTimeout } from '@/lib/time'
 
 export function useWalletPayment () {
   const wallets = useSendWallets()
   const sendPayment = useSendPayment()
+  const loggerFactory = useWalletLoggerFactory()
   const invoiceHelper = useInvoice()
 
   return useCallback(async (invoice, { waitFor, updateOnFallback }) => {
@@ -24,44 +26,72 @@ export function useWalletPayment () {
       throw new WalletsNotAvailableError()
     }
 
-    for (const [i, wallet] of wallets.entries()) {
+    for (let i = 0; i < wallets.length; i++) {
+      const wallet = wallets[i]
+      const logger = loggerFactory(wallet)
+
+      const { bolt11 } = latestInvoice
       const controller = invoiceController(latestInvoice, invoiceHelper.isInvoice)
+
+      const walletPromise = sendPayment(wallet, logger, latestInvoice)
+      const pollPromise = controller.wait(waitFor)
+
       try {
         return await new Promise((resolve, reject) => {
           // can't await wallet payments since we might pay hold invoices and thus payments might not settle immediately.
           // that's why we separately check if we received the payment with the invoice controller.
-          sendPayment(wallet, latestInvoice).catch(reject)
+          walletPromise.catch(reject)
-          controller.wait(waitFor)
+          pollPromise.then(resolve).catch(reject)
-            .then(resolve)
-            .catch(reject)
         })
       } catch (err) {
-        // cancel invoice to make sure it cannot be paid later and create new invoice to retry.
+        let paymentError = err
-        // we only need to do this if payment was attempted which is not the case if the wallet is not enabled.
+        const message = `payment failed: ${paymentError.reason ?? paymentError.message}`
-        if (err instanceof WalletPaymentError) {
-          await invoiceHelper.cancel(latestInvoice)
 
-          // is there another wallet to try?
+        if (!(paymentError instanceof WalletError)) {
-          const lastAttempt = i === wallets.length - 1
+          // payment failed for some reason unrelated to wallets (ie invoice expired or was canceled).
-          if (!lastAttempt) {
+          // bail out of attempting wallets.
+          logger.error(message, { bolt11 })
+          throw paymentError
+        }
+
+        // at this point, paymentError is always a wallet error,
+        // we just need to distinguish between receiver and sender errors
+
+        try {
+          // we always await the poll promise here to check for failed forwards since sender wallet errors
+          // can be caused by them which we want to handle as receiver errors, not sender errors.
+          // but we don't wait forever because real sender errors will cause the poll promise to never settle.
+          await withTimeout(pollPromise, FAST_POLL_INTERVAL * 2.5)
+        } catch (err) {
+          if (err instanceof WalletError) {
+            paymentError = err
+          }
+        }
+
+        if (paymentError instanceof WalletReceiverError) {
+          // if payment failed because of the receiver, use the same wallet again
+          // and log this as info, not error
+          logger.info('failed to forward payment to receiver, retrying with new invoice', { bolt11 })
+          i -= 1
+        } else if (paymentError instanceof WalletPaymentError) {
+          // only log payment errors, not configuration errors
+          logger.error(message, { bolt11 })
+        }
+
+        if (paymentError instanceof WalletPaymentError) {
+          // if a payment was attempted, cancel invoice to make sure it cannot be paid later and create new invoice to retry.
+          await invoiceHelper.cancel(latestInvoice)
+        }
+
+        // only create a new invoice if we will try to pay with a wallet again
+        const retry = paymentError instanceof WalletReceiverError || i < wallets.length - 1
+        if (retry) {
           latestInvoice = await invoiceHelper.retry(latestInvoice, { update: updateOnFallback })
         }
-        }
 
-        // TODO: receiver fallbacks
+        aggregateError = new WalletAggregateError([aggregateError, paymentError], latestInvoice)
-        //
-        // if payment failed because of the receiver, we should use the same wallet again.
-        // if (err instanceof ReceiverError) { ... }
 
-        // try next wallet if the payment failed because of the wallet
-        // and not because it expired or was canceled
-        if (err instanceof WalletError) {
-          aggregateError = new WalletAggregateError([aggregateError, err], latestInvoice)
         continue
-        }
-
-        // payment failed not because of the sender or receiver wallet. bail out of attemping wallets.
-        throw err
       } finally {
         controller.stop()
       }
@@ -111,11 +141,7 @@ function invoiceController (inv, isInvoice) {
 }
 
 function useSendPayment () {
-  const factory = useWalletLoggerFactory()
+  return useCallback(async (wallet, logger, invoice) => {
-
-  return useCallback(async (wallet, invoice) => {
-    const logger = factory(wallet)
-
     if (!wallet.config.enabled) {
       throw new WalletNotEnabledError(wallet.def.name)
     }
@@ -131,9 +157,9 @@ function useSendPayment () {
       const preimage = await wallet.def.sendPayment(bolt11, wallet.config, { logger })
       logger.ok(`↗ payment sent: ${formatSats(satsRequested)}`, { bolt11, preimage })
     } catch (err) {
+      // we don't log the error here since we want to handle receiver errors separately
       const message = err.message || err.toString?.()
-      logger.error(`payment failed: ${message}`, { bolt11 })
       throw new WalletSenderError(wallet.def.name, invoice, message)
     }
-  }, [factory])
+  }, [])
 }

wallets/server.js

@@ -24,9 +24,9 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln]
 
 const MAX_PENDING_INVOICES_PER_WALLET = 25
 
-export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { models }) {
+export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { predecessorId, models }) {
   // get the wallets in order of priority
-  const wallets = await getInvoiceableWallets(userId, { models })
+  const wallets = await getInvoiceableWallets(userId, { predecessorId, models })
 
   msats = toPositiveNumber(msats)
 
@@ -81,7 +81,7 @@ export async function createInvoice (userId, { msats, description, descriptionHa
 
 export async function createWrappedInvoice (userId,
   { msats, feePercent, description, descriptionHash, expiry = 360 },
-  { models, me, lnd }) {
+  { predecessorId, models, me, lnd }) {
   let logger, bolt11
   try {
     const { invoice, wallet } = await createInvoice(userId, {
@@ -90,7 +90,7 @@ export async function createWrappedInvoice (userId,
       description,
       descriptionHash,
       expiry
-    }, { models })
+    }, { predecessorId, models })
 
     logger = walletLogger({ wallet, models })
     bolt11 = invoice
@@ -110,18 +110,47 @@ export async function createWrappedInvoice (userId,
   }
 }
 
-export async function getInvoiceableWallets (userId, { models }) {
+export async function getInvoiceableWallets (userId, { predecessorId, models }) {
-  const wallets = await models.wallet.findMany({
+  // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices.
-    where: { userId, enabled: true },
+  // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it
-    include: {
+  // so it has not been updated yet.
-      user: true
+  // if predecessorId is not provided, the subquery will be empty and thus no wallets are filtered out.
-    },
+  const wallets = await models.$queryRaw`
-    orderBy: [
+    SELECT
-      { priority: 'asc' },
+      "Wallet".*,
-      // use id as tie breaker (older wallet first)
+      jsonb_build_object(
-      { id: 'asc' }
+        'id', "users"."id",
-    ]
+        'hideInvoiceDesc', "users"."hideInvoiceDesc"
-  })
+      ) AS "user"
+    FROM "Wallet"
+    JOIN "users" ON "users"."id" = "Wallet"."userId"
+    WHERE
+      "Wallet"."userId" = ${userId}
+      AND "Wallet"."enabled" = true
+      AND "Wallet"."id" NOT IN (
+        WITH RECURSIVE "Retries" AS (
+          -- select the current failed invoice that we are currently retrying
+          -- this failed invoice will be used to start the recursion
+          SELECT "Invoice"."id", "Invoice"."predecessorId"
+          FROM "Invoice"
+          WHERE "Invoice"."id" = ${predecessorId} AND "Invoice"."actionState" = 'FAILED'
+
+          UNION ALL
+
+          -- recursive part: use predecessorId to select the previous invoice that failed in the chain
+          -- until there is no more previous invoice
+          SELECT "Invoice"."id", "Invoice"."predecessorId"
+          FROM "Invoice"
+          JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId"
+          WHERE "Invoice"."actionState" = 'RETRYING'
+        )
+        SELECT
+          "InvoiceForward"."walletId"
+        FROM "Retries"
+        JOIN "InvoiceForward" ON "InvoiceForward"."invoiceId" = "Retries"."id"
+        WHERE "InvoiceForward"."withdrawlId" IS NOT NULL
+      )
+    ORDER BY "Wallet"."priority" ASC, "Wallet"."id" ASC`
 
   const walletsWithDefs = wallets.map(wallet => {
     const w = walletDefs.find(w => w.walletType === wallet.type)