Receiver fallbacks (#1688)

* Use same naming scheme between ln containers and env vars

* Add router_lnd container

* Only open channels to router_lnd

* Use 1sat base fee and 0ppm fee rate

* Add script to test routing

* Also fund router_lnd wallet

* Receiver fallbacks

* Rename to predecessorId

* Remove useless wallet table join

* Missing renaming to predecessor

* Fix payment stuck on sender error

We want to await the invoice poll promise so we can check for receiver errors, but in case of sender errors, the promise will never settle.

* Don't log failed forwards as sender errors

* fix check for receiver error

---------

Co-authored-by: Keyan <34140557+huumn@users.noreply.github.com>
Co-authored-by: k00b <k00b@stacker.news>

api/paidAction/index.js

@@ -317,34 +317,39 @@ export async function retryPaidAction (actionType, args, incomingContext) {
     optimistic: actionOptimistic,
     me: await models.user.findUnique({ where: { id: parseInt(me.id) } }),
     cost: BigInt(msatsRequested),
-    actionId
+    actionId,
+    predecessorId: failedInvoice.id
   }
 
   let invoiceArgs
   const invoiceForward = await models.invoiceForward.findUnique({
-    where: { invoiceId: failedInvoice.id },
+    where: {
+      invoiceId: failedInvoice.id
+    },
     include: {
-      wallet: true,
+      wallet: true
-      invoice: true,
-      withdrawl: true
     }
   })
-  // TODO: receiver fallbacks
+
-  // use next receiver wallet if forward failed (we currently immediately fallback to SN)
+  if (invoiceForward) {
-  const failedForward = invoiceForward?.withdrawl && invoiceForward.withdrawl.actionState !== 'CONFIRMED'
+    // this is a wrapped invoice, we need to retry it with receiver fallbacks
-  if (invoiceForward && !failedForward) {
+    try {
-    const { userId } = invoiceForward.wallet
+      const { userId } = invoiceForward.wallet
-    const { invoice: bolt11, wrappedInvoice: wrappedBolt11, wallet, maxFee } = await createWrappedInvoice(userId, {
+      // this will return an invoice from the first receiver wallet that didn't fail yet and throw if none is available
-      msats: failedInvoice.msatsRequested,
+      const { invoice: bolt11, wrappedInvoice: wrappedBolt11, wallet, maxFee } = await createWrappedInvoice(userId, {
-      feePercent: await action.getSybilFeePercent?.(actionArgs, retryContext),
+        msats: failedInvoice.msatsRequested,
-      description: await action.describe?.(actionArgs, retryContext),
+        feePercent: await action.getSybilFeePercent?.(actionArgs, retryContext),
-      expiry: INVOICE_EXPIRE_SECS
+        description: await action.describe?.(actionArgs, retryContext),
-    }, retryContext)
+        expiry: INVOICE_EXPIRE_SECS
-    invoiceArgs = { bolt11, wrappedBolt11, wallet, maxFee }
+      }, retryContext)
-  } else {
+      invoiceArgs = { bolt11, wrappedBolt11, wallet, maxFee }
-    invoiceArgs = await createSNInvoice(actionType, actionArgs, retryContext)
+    } catch (err) {
+      console.log('failed to retry wrapped invoice, falling back to SN:', err)
+    }
   }
 
+  invoiceArgs ??= await createSNInvoice(actionType, actionArgs, retryContext)
+
   return await models.$transaction(async tx => {
     const context = { ...retryContext, tx, invoiceArgs }
 
@@ -404,7 +409,7 @@ async function createSNInvoice (actionType, args, context) {
 }
 
 async function createDbInvoice (actionType, args, context) {
-  const { me, models, tx, cost, optimistic, actionId, invoiceArgs } = context
+  const { me, models, tx, cost, optimistic, actionId, invoiceArgs, predecessorId } = context
   const { bolt11, wrappedBolt11, preimage, wallet, maxFee } = invoiceArgs
 
   const db = tx ?? models
@@ -429,7 +434,8 @@ async function createDbInvoice (actionType, args, context) {
     actionOptimistic: optimistic,
     actionArgs: args,
     expiresAt,
-    actionId
+    actionId,
+    predecessorId
   }
 
   let invoice

api/resolvers/wallet.js

@@ -606,6 +606,15 @@ const resolvers = {
     satsReceived: i => msatsToSats(i.msatsReceived),
     satsRequested: i => msatsToSats(i.msatsRequested),
     // we never want to fetch the sensitive data full monty in nested resolvers
+    forwardStatus: async (invoice, args, { models }) => {
+      const forward = await models.invoiceForward.findUnique({
+        where: { invoiceId: Number(invoice.id) },
+        include: {
+          withdrawl: true
+        }
+      })
+      return forward?.withdrawl?.status
+    },
     forwardedSats: async (invoice, args, { models }) => {
       const msats = (await models.invoiceForward.findUnique({
         where: { invoiceId: Number(invoice.id) },

api/typeDefs/wallet.js

@@ -129,6 +129,7 @@ const typeDefs = `
     item: Item
     itemAct: ItemAct
     forwardedSats: Int
+    forwardStatus: String
   }
 
   type Withdrawl {

components/use-invoice.js

@@ -1,8 +1,8 @@
 import { useApolloClient, useMutation } from '@apollo/client'
 import { useCallback } from 'react'
+import { InvoiceCanceledError, InvoiceExpiredError, WalletReceiverError } from '@/wallets/errors'
 import { RETRY_PAID_ACTION } from '@/fragments/paidAction'
 import { INVOICE, CANCEL_INVOICE } from '@/fragments/wallet'
-import { InvoiceExpiredError, InvoiceCanceledError } from '@/wallets/errors'
 
 export default function useInvoice () {
   const client = useApolloClient()
@@ -16,20 +16,21 @@ export default function useInvoice () {
       throw error
     }
 
-    const { cancelled, cancelledAt, actionError, actionState, expiresAt } = data.invoice
+    const { cancelled, cancelledAt, actionError, expiresAt, forwardStatus } = data.invoice
 
     const expired = cancelledAt && new Date(expiresAt) < new Date(cancelledAt)
     if (expired) {
       throw new InvoiceExpiredError(data.invoice)
     }
 
-    if (cancelled || actionError) {
+    const failed = cancelled || actionError
-      throw new InvoiceCanceledError(data.invoice, actionError)
+
+    if (failed && (forwardStatus && forwardStatus !== 'CONFIRMED')) {
+      throw new WalletReceiverError(data.invoice)
     }
 
-    // write to cache if paid
+    if (failed) {
-    if (actionState === 'PAID') {
+      throw new InvoiceCanceledError(data.invoice, actionError)
-      client.writeQuery({ query: INVOICE, variables: { id }, data: { invoice: data.invoice } })
     }
 
     return { invoice: data.invoice, check: that(data.invoice) }

fragments/wallet.js

@@ -23,6 +23,7 @@ export const INVOICE_FIELDS = gql`
     actionError
     confirmedPreimage
     forwardedSats
+    forwardStatus
   }`
 
 export const INVOICE_FULL = gql`

prisma/migrations/20241206155927_invoice_predecessors/migration.sql

@@ -0,0 +1,14 @@
+/*
+  Warnings:
+
+  - A unique constraint covering the columns `[predecessorId]` on the table `Invoice` will be added. If there are existing duplicate values, this will fail.
+
+*/
+-- AlterTable
+ALTER TABLE "Invoice" ADD COLUMN     "predecessorId" INTEGER;
+
+-- CreateIndex
+CREATE UNIQUE INDEX "Invoice.predecessorId_unique" ON "Invoice"("predecessorId");
+
+-- AddForeignKey
+ALTER TABLE "Invoice" ADD CONSTRAINT "Invoice_predecessorId_fkey" FOREIGN KEY ("predecessorId") REFERENCES "Invoice"("id") ON DELETE CASCADE ON UPDATE CASCADE;

prisma/schema.prisma

@@ -904,39 +904,41 @@ model ItemMention {
 }
 
 model Invoice {
-  id             Int             @id @default(autoincrement())
+  id                 Int                 @id @default(autoincrement())
-  createdAt      DateTime        @default(now()) @map("created_at")
+  createdAt          DateTime            @default(now()) @map("created_at")
-  updatedAt      DateTime        @default(now()) @updatedAt @map("updated_at")
+  updatedAt          DateTime            @default(now()) @updatedAt @map("updated_at")
-  userId         Int
+  userId             Int
-  hash           String          @unique(map: "Invoice.hash_unique")
+  hash               String              @unique(map: "Invoice.hash_unique")
-  preimage       String?         @unique(map: "Invoice.preimage_unique")
+  preimage           String?             @unique(map: "Invoice.preimage_unique")
-  isHeld         Boolean?
+  isHeld             Boolean?
-  bolt11         String
+  bolt11             String
-  expiresAt      DateTime
+  expiresAt          DateTime
-  confirmedAt    DateTime?
+  confirmedAt        DateTime?
-  confirmedIndex BigInt?
+  confirmedIndex     BigInt?
-  cancelled      Boolean         @default(false)
+  cancelled          Boolean             @default(false)
-  cancelledAt    DateTime?
+  cancelledAt        DateTime?
-  msatsRequested BigInt
+  msatsRequested     BigInt
-  msatsReceived  BigInt?
+  msatsReceived      BigInt?
-  desc           String?
+  desc               String?
-  comment        String?
+  comment            String?
-  lud18Data      Json?
+  lud18Data          Json?
-  user           User            @relation(fields: [userId], references: [id], onDelete: Cascade)
+  user               User                @relation(fields: [userId], references: [id], onDelete: Cascade)
-  invoiceForward InvoiceForward?
+  invoiceForward     InvoiceForward?
-
+  predecessorId      Int?                @unique(map: "Invoice.predecessorId_unique")
-  actionState      InvoiceActionState?
+  predecessorInvoice Invoice?            @relation("PredecessorInvoice", fields: [predecessorId], references: [id], onDelete: Cascade)
-  actionType       InvoiceActionType?
+  successorInvoice   Invoice?            @relation("PredecessorInvoice")
-  actionOptimistic Boolean?
+  actionState        InvoiceActionState?
-  actionId         Int?
+  actionType         InvoiceActionType?
-  actionArgs       Json?               @db.JsonB
+  actionOptimistic   Boolean?
-  actionError      String?
+  actionId           Int?
-  actionResult     Json?               @db.JsonB
+  actionArgs         Json?               @db.JsonB
-  ItemAct          ItemAct[]
+  actionError        String?
-  Item             Item[]
+  actionResult       Json?               @db.JsonB
-  Upload           Upload[]
+  ItemAct            ItemAct[]
-  PollVote         PollVote[]
+  Item               Item[]
-  PollBlindVote    PollBlindVote[]
+  Upload             Upload[]
+  PollVote           PollVote[]
+  PollBlindVote      PollBlindVote[]
 
   @@index([createdAt], map: "Invoice.created_at_index")
   @@index([userId], map: "Invoice.userId_index")

wallets/errors.js

@@ -47,6 +47,14 @@ export class WalletSenderError extends WalletPaymentError {
   }
 }
 
+export class WalletReceiverError extends WalletPaymentError {
+  constructor (invoice) {
+    super(`payment forwarding failed for invoice ${invoice.hash}`)
+    this.name = 'WalletReceiverError'
+    this.invoice = invoice
+  }
+}
+
 export class WalletsNotAvailableError extends WalletConfigurationError {
   constructor () {
     super('no wallet available')

wallets/payment.js

@@ -5,14 +5,16 @@ import useInvoice from '@/components/use-invoice'
 import { FAST_POLL_INTERVAL } from '@/lib/constants'
 import {
   WalletsNotAvailableError, WalletSenderError, WalletAggregateError, WalletPaymentAggregateError,
-  WalletNotEnabledError, WalletSendNotConfiguredError, WalletPaymentError, WalletError
+  WalletNotEnabledError, WalletSendNotConfiguredError, WalletPaymentError, WalletError, WalletReceiverError
 } from '@/wallets/errors'
 import { canSend } from './common'
 import { useWalletLoggerFactory } from './logger'
+import { withTimeout } from '@/lib/time'
 
 export function useWalletPayment () {
   const wallets = useSendWallets()
   const sendPayment = useSendPayment()
+  const loggerFactory = useWalletLoggerFactory()
   const invoiceHelper = useInvoice()
 
   return useCallback(async (invoice, { waitFor, updateOnFallback }) => {
@@ -24,44 +26,72 @@ export function useWalletPayment () {
       throw new WalletsNotAvailableError()
     }
 
-    for (const [i, wallet] of wallets.entries()) {
+    for (let i = 0; i < wallets.length; i++) {
+      const wallet = wallets[i]
+      const logger = loggerFactory(wallet)
+
+      const { bolt11 } = latestInvoice
       const controller = invoiceController(latestInvoice, invoiceHelper.isInvoice)
+
+      const walletPromise = sendPayment(wallet, logger, latestInvoice)
+      const pollPromise = controller.wait(waitFor)
+
       try {
         return await new Promise((resolve, reject) => {
           // can't await wallet payments since we might pay hold invoices and thus payments might not settle immediately.
           // that's why we separately check if we received the payment with the invoice controller.
-          sendPayment(wallet, latestInvoice).catch(reject)
+          walletPromise.catch(reject)
-          controller.wait(waitFor)
+          pollPromise.then(resolve).catch(reject)
-            .then(resolve)
-            .catch(reject)
         })
       } catch (err) {
-        // cancel invoice to make sure it cannot be paid later and create new invoice to retry.
+        let paymentError = err
-        // we only need to do this if payment was attempted which is not the case if the wallet is not enabled.
+        const message = `payment failed: ${paymentError.reason ?? paymentError.message}`
-        if (err instanceof WalletPaymentError) {
-          await invoiceHelper.cancel(latestInvoice)
 
-          // is there another wallet to try?
+        if (!(paymentError instanceof WalletError)) {
-          const lastAttempt = i === wallets.length - 1
+          // payment failed for some reason unrelated to wallets (ie invoice expired or was canceled).
-          if (!lastAttempt) {
+          // bail out of attempting wallets.
-            latestInvoice = await invoiceHelper.retry(latestInvoice, { update: updateOnFallback })
+          logger.error(message, { bolt11 })
+          throw paymentError
+        }
+
+        // at this point, paymentError is always a wallet error,
+        // we just need to distinguish between receiver and sender errors
+
+        try {
+          // we always await the poll promise here to check for failed forwards since sender wallet errors
+          // can be caused by them which we want to handle as receiver errors, not sender errors.
+          // but we don't wait forever because real sender errors will cause the poll promise to never settle.
+          await withTimeout(pollPromise, FAST_POLL_INTERVAL * 2.5)
+        } catch (err) {
+          if (err instanceof WalletError) {
+            paymentError = err
           }
         }
 
-        // TODO: receiver fallbacks
+        if (paymentError instanceof WalletReceiverError) {
-        //
+          // if payment failed because of the receiver, use the same wallet again
-        // if payment failed because of the receiver, we should use the same wallet again.
+          // and log this as info, not error
-        // if (err instanceof ReceiverError) { ... }
+          logger.info('failed to forward payment to receiver, retrying with new invoice', { bolt11 })
-
+          i -= 1
-        // try next wallet if the payment failed because of the wallet
+        } else if (paymentError instanceof WalletPaymentError) {
-        // and not because it expired or was canceled
+          // only log payment errors, not configuration errors
-        if (err instanceof WalletError) {
+          logger.error(message, { bolt11 })
-          aggregateError = new WalletAggregateError([aggregateError, err], latestInvoice)
-          continue
         }
 
-        // payment failed not because of the sender or receiver wallet. bail out of attemping wallets.
+        if (paymentError instanceof WalletPaymentError) {
-        throw err
+          // if a payment was attempted, cancel invoice to make sure it cannot be paid later and create new invoice to retry.
+          await invoiceHelper.cancel(latestInvoice)
+        }
+
+        // only create a new invoice if we will try to pay with a wallet again
+        const retry = paymentError instanceof WalletReceiverError || i < wallets.length - 1
+        if (retry) {
+          latestInvoice = await invoiceHelper.retry(latestInvoice, { update: updateOnFallback })
+        }
+
+        aggregateError = new WalletAggregateError([aggregateError, paymentError], latestInvoice)
+
+        continue
       } finally {
         controller.stop()
       }
@@ -111,11 +141,7 @@ function invoiceController (inv, isInvoice) {
 }
 
 function useSendPayment () {
-  const factory = useWalletLoggerFactory()
+  return useCallback(async (wallet, logger, invoice) => {
-
-  return useCallback(async (wallet, invoice) => {
-    const logger = factory(wallet)
-
     if (!wallet.config.enabled) {
       throw new WalletNotEnabledError(wallet.def.name)
     }
@@ -131,9 +157,9 @@ function useSendPayment () {
       const preimage = await wallet.def.sendPayment(bolt11, wallet.config, { logger })
       logger.ok(`↗ payment sent: ${formatSats(satsRequested)}`, { bolt11, preimage })
     } catch (err) {
+      // we don't log the error here since we want to handle receiver errors separately
       const message = err.message || err.toString?.()
-      logger.error(`payment failed: ${message}`, { bolt11 })
       throw new WalletSenderError(wallet.def.name, invoice, message)
     }
-  }, [factory])
+  }, [])
 }

wallets/server.js

@@ -24,9 +24,9 @@ export default [lnd, cln, lnAddr, lnbits, nwc, phoenixd, blink, lnc, webln]
 
 const MAX_PENDING_INVOICES_PER_WALLET = 25
 
-export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { models }) {
+export async function createInvoice (userId, { msats, description, descriptionHash, expiry = 360 }, { predecessorId, models }) {
   // get the wallets in order of priority
-  const wallets = await getInvoiceableWallets(userId, { models })
+  const wallets = await getInvoiceableWallets(userId, { predecessorId, models })
 
   msats = toPositiveNumber(msats)
 
@@ -81,7 +81,7 @@ export async function createInvoice (userId, { msats, description, descriptionHa
 
 export async function createWrappedInvoice (userId,
   { msats, feePercent, description, descriptionHash, expiry = 360 },
-  { models, me, lnd }) {
+  { predecessorId, models, me, lnd }) {
   let logger, bolt11
   try {
     const { invoice, wallet } = await createInvoice(userId, {
@@ -90,7 +90,7 @@ export async function createWrappedInvoice (userId,
       description,
       descriptionHash,
       expiry
-    }, { models })
+    }, { predecessorId, models })
 
     logger = walletLogger({ wallet, models })
     bolt11 = invoice
@@ -110,18 +110,47 @@ export async function createWrappedInvoice (userId,
   }
 }
 
-export async function getInvoiceableWallets (userId, { models }) {
+export async function getInvoiceableWallets (userId, { predecessorId, models }) {
-  const wallets = await models.wallet.findMany({
+  // filter out all wallets that have already been tried by recursively following the retry chain of predecessor invoices.
-    where: { userId, enabled: true },
+  // the current predecessor invoice is in state 'FAILED' and not in state 'RETRYING' because we are currently retrying it
-    include: {
+  // so it has not been updated yet.
-      user: true
+  // if predecessorId is not provided, the subquery will be empty and thus no wallets are filtered out.
-    },
+  const wallets = await models.$queryRaw`
-    orderBy: [
+    SELECT
-      { priority: 'asc' },
+      "Wallet".*,
-      // use id as tie breaker (older wallet first)
+      jsonb_build_object(
-      { id: 'asc' }
+        'id', "users"."id",
-    ]
+        'hideInvoiceDesc', "users"."hideInvoiceDesc"
-  })
+      ) AS "user"
+    FROM "Wallet"
+    JOIN "users" ON "users"."id" = "Wallet"."userId"
+    WHERE
+      "Wallet"."userId" = ${userId}
+      AND "Wallet"."enabled" = true
+      AND "Wallet"."id" NOT IN (
+        WITH RECURSIVE "Retries" AS (
+          -- select the current failed invoice that we are currently retrying
+          -- this failed invoice will be used to start the recursion
+          SELECT "Invoice"."id", "Invoice"."predecessorId"
+          FROM "Invoice"
+          WHERE "Invoice"."id" = ${predecessorId} AND "Invoice"."actionState" = 'FAILED'
+
+          UNION ALL
+
+          -- recursive part: use predecessorId to select the previous invoice that failed in the chain
+          -- until there is no more previous invoice
+          SELECT "Invoice"."id", "Invoice"."predecessorId"
+          FROM "Invoice"
+          JOIN "Retries" ON "Invoice"."id" = "Retries"."predecessorId"
+          WHERE "Invoice"."actionState" = 'RETRYING'
+        )
+        SELECT
+          "InvoiceForward"."walletId"
+        FROM "Retries"
+        JOIN "InvoiceForward" ON "InvoiceForward"."invoiceId" = "Retries"."id"
+        WHERE "InvoiceForward"."withdrawlId" IS NOT NULL
+      )
+    ORDER BY "Wallet"."priority" ASC, "Wallet"."id" ASC`
 
   const walletsWithDefs = wallets.map(wallet => {
     const w = walletDefs.find(w => w.walletType === wallet.type)