From 4ce395889da8ba3c477f46ea5fc76315b839ab45 Mon Sep 17 00:00:00 2001 From: Keyan <34140557+huumn@users.noreply.github.com> Date: Wed, 2 Oct 2024 15:03:30 -0500 Subject: [PATCH] Be kind to lnd (#1448) * cache or remove unecessary calls to lnd * avoid redundant grpc calls in state machine * store preimage whenever available * enhancements post self-code review * small refinements * fixes * fix lnurl-verify * prevent wallet logger throwing on idb close * fix promise in race while waiting for payment --- api/lnd/index.js | 70 +++++++++++++++-- api/paidAction/index.js | 2 +- api/resolvers/blockHeight.js | 43 ++++------- api/resolvers/chainFee.js | 41 ++++------ api/resolvers/price.js | 43 ++++------- api/resolvers/wallet.js | 30 +++++--- components/payment.js | 16 ++-- components/use-paid-mutation.js | 5 +- components/wallet-logger.js | 36 +++++---- fragments/wallet.js | 2 +- lib/fetch.js | 70 +++++++++++++++++ pages/api/lnurlp/[username]/pay.js | 2 +- pages/api/lnurlp/[username]/verify/[hash].js | 14 ++-- .../migration.sql | 44 +++++++++++ wallets/wrap.js | 8 +- worker/imgproxy.js | 14 +--- worker/nostr.js | 11 ++- worker/paidAction.js | 46 ++++++----- worker/wallet.js | 77 +++++++++---------- 19 files changed, 363 insertions(+), 211 deletions(-) create mode 100644 lib/fetch.js create mode 100644 prisma/migrations/20241002145114_create_invoice_update/migration.sql diff --git a/api/lnd/index.js b/api/lnd/index.js index 0e53bd70..035afcf9 100644 --- a/api/lnd/index.js +++ b/api/lnd/index.js @@ -1,14 +1,15 @@ +import { cachedFetcher } from '@/lib/fetch' import { toPositiveNumber } from '@/lib/validate' -import lndService from 'ln-service' +import { authenticatedLndGrpc, getIdentity, getHeight, getWalletInfo, getNode } from 'ln-service' -const { lnd } = lndService.authenticatedLndGrpc({ +const { lnd } = authenticatedLndGrpc({ cert: process.env.LND_CERT, macaroon: process.env.LND_MACAROON, socket: process.env.LND_SOCKET }) // Check LND GRPC connection -lndService.getWalletInfo({ lnd }, (err, result) => { +getWalletInfo({ lnd }, (err, result) => { if (err) { console.error('LND GRPC connection error') return @@ -80,16 +81,69 @@ export function getPaymentFailureStatus (withdrawal) { } if (withdrawal?.failed.is_insufficient_balance) { - return 'INSUFFICIENT_BALANCE' + return { + status: 'INSUFFICIENT_BALANCE', + message: 'you didn\'t have enough sats' + } } else if (withdrawal?.failed.is_invalid_payment) { - return 'INVALID_PAYMENT' + return { + status: 'INVALID_PAYMENT', + message: 'invalid payment' + } } else if (withdrawal?.failed.is_pathfinding_timeout) { - return 'PATHFINDING_TIMEOUT' + return { + status: 'PATHFINDING_TIMEOUT', + message: 'no route found' + } } else if (withdrawal?.failed.is_route_not_found) { - return 'ROUTE_NOT_FOUND' + return { + status: 'ROUTE_NOT_FOUND', + message: 'no route found' + } } - return 'UNKNOWN_FAILURE' + return { + status: 'UNKNOWN_FAILURE', + message: 'unknown failure' + } } +export const getBlockHeight = cachedFetcher(async () => { + try { + const { current_block_height: height } = await getHeight({ lnd }) + return height + } catch (err) { + throw new Error(`Unable to fetch block height: ${err.message}`) + } +}, { + maxSize: 1, + cacheExpiry: 60 * 1000, // 1 minute + forceRefreshThreshold: 5 * 60 * 1000 // 5 minutes +}) + +export const getOurPubkey = cachedFetcher(async () => { + try { + const { identity } = await getIdentity({ lnd }) + return identity.public_key + } catch (err) { + throw new Error(`Unable to fetch identity: ${err.message}`) + } +}, { + maxSize: 1, + cacheExpiry: 0, // never expire + forceRefreshThreshold: 0 // never force refresh +}) + +export const getNodeInfo = cachedFetcher(async (args) => { + try { + return await getNode({ lnd, ...args }) + } catch (err) { + throw new Error(`Unable to fetch node info: ${err.message}`) + } +}, { + maxSize: 1000, + cacheExpiry: 1000 * 60 * 60 * 24, // 1 day + forceRefreshThreshold: 1000 * 60 * 60 * 24 * 7 // 1 week +}) + export default lnd diff --git a/api/paidAction/index.js b/api/paidAction/index.js index 3ae9cfb3..946d6725 100644 --- a/api/paidAction/index.js +++ b/api/paidAction/index.js @@ -309,7 +309,7 @@ async function createDbInvoice (actionType, args, context, const invoiceData = { hash: servedInvoice.id, msatsRequested: BigInt(servedInvoice.mtokens), - preimage: optimistic ? undefined : preimage, + preimage, bolt11: servedBolt11, userId: me?.id ?? USER_ID.anon, actionType, diff --git a/api/resolvers/blockHeight.js b/api/resolvers/blockHeight.js index 9f73f898..5739b634 100644 --- a/api/resolvers/blockHeight.js +++ b/api/resolvers/blockHeight.js @@ -1,39 +1,26 @@ -import lndService from 'ln-service' -import lnd from '@/api/lnd' import { isServiceEnabled } from '@/lib/sndev' +import { cachedFetcher } from '@/lib/fetch' +import { getHeight } from 'ln-service' -const cache = new Map() -const expiresIn = 1000 * 30 // 30 seconds in milliseconds - -async function fetchBlockHeight () { - let blockHeight = 0 - if (!isServiceEnabled('payments')) return blockHeight +const getBlockHeight = cachedFetcher(async ({ lnd }) => { try { - const height = await lndService.getHeight({ lnd }) - blockHeight = height.current_block_height - cache.set('block', { height: blockHeight, createdAt: Date.now() }) + const { current_block_height: height } = await getHeight({ lnd }) + return height } catch (err) { - console.error('fetchBlockHeight', err) + console.error('getBlockHeight', err) + return 0 } - return blockHeight -} - -async function getBlockHeight () { - if (cache.has('block')) { - const { height, createdAt } = cache.get('block') - const expired = createdAt + expiresIn < Date.now() - if (expired) fetchBlockHeight().catch(console.error) // update cache - return height // serve stale block height (this on the SSR critical path) - } else { - fetchBlockHeight().catch(console.error) - } - return 0 -} +}, { + maxSize: 1, + cacheExpiry: 60 * 1000, // 1 minute + forceRefreshThreshold: 0 +}) export default { Query: { - blockHeight: async (parent, opts, ctx) => { - return await getBlockHeight() + blockHeight: async (parent, opts, { lnd }) => { + if (!isServiceEnabled('payments')) return 0 + return await getBlockHeight({ lnd }) || 0 } } } diff --git a/api/resolvers/chainFee.js b/api/resolvers/chainFee.js index b47dd9c5..d66e7a09 100644 --- a/api/resolvers/chainFee.js +++ b/api/resolvers/chainFee.js @@ -1,36 +1,25 @@ -const cache = new Map() -const expiresIn = 1000 * 30 // 30 seconds in milliseconds +import { cachedFetcher } from '@/lib/fetch' -async function fetchChainFeeRate () { +const getChainFeeRate = cachedFetcher(async () => { const url = 'https://mempool.space/api/v1/fees/recommended' - const chainFee = await fetch(url) - .then((res) => res.json()) - .then((body) => body.hourFee) - .catch((err) => { - console.error('fetchChainFee', err) - return 0 - }) - - cache.set('fee', { fee: chainFee, createdAt: Date.now() }) - return chainFee -} - -async function getChainFeeRate () { - if (cache.has('fee')) { - const { fee, createdAt } = cache.get('fee') - const expired = createdAt + expiresIn < Date.now() - if (expired) fetchChainFeeRate().catch(console.error) // update cache - return fee - } else { - fetchChainFeeRate().catch(console.error) + try { + const res = await fetch(url) + const body = await res.json() + return body.hourFee + } catch (err) { + console.error('fetchChainFee', err) + return 0 } - return 0 -} +}, { + maxSize: 1, + cacheExpiry: 60 * 1000, // 1 minute + forceRefreshThreshold: 0 // never force refresh +}) export default { Query: { chainFee: async (parent, opts, ctx) => { - return await getChainFeeRate() + return await getChainFeeRate() || 0 } } } diff --git a/api/resolvers/price.js b/api/resolvers/price.js index c04e7df9..d261d137 100644 --- a/api/resolvers/price.js +++ b/api/resolvers/price.js @@ -1,36 +1,27 @@ -const cache = new Map() -const expiresIn = 30000 // in milliseconds +import { SUPPORTED_CURRENCIES } from '@/lib/currency' +import { cachedFetcher } from '@/lib/fetch' -async function fetchPrice (fiat) { - const url = `https://api.coinbase.com/v2/prices/BTC-${fiat}/spot` - const price = await fetch(url) - .then((res) => res.json()) - .then((body) => parseFloat(body.data.amount)) - .catch((err) => { - console.error(err) - return -1 - }) - cache.set(fiat, { price, createdAt: Date.now() }) - return price -} - -async function getPrice (fiat) { +const getPrice = cachedFetcher(async (fiat) => { fiat ??= 'USD' - if (cache.has(fiat)) { - const { price, createdAt } = cache.get(fiat) - const expired = createdAt + expiresIn < Date.now() - if (expired) fetchPrice(fiat).catch(console.error) // update cache - return price // serve stale price (this on the SSR critical path) - } else { - fetchPrice(fiat).catch(console.error) + const url = `https://api.coinbase.com/v2/prices/BTC-${fiat}/spot` + try { + const res = await fetch(url) + const body = await res.json() + return parseFloat(body.data.amount) + } catch (err) { + console.error(err) + return -1 } - return null -} +}, { + maxSize: SUPPORTED_CURRENCIES.length, + cacheExpiry: 60 * 1000, // 1 minute + forceRefreshThreshold: 0 // never force refresh +}) export default { Query: { price: async (parent, { fiatCurrency }, ctx) => { - return await getPrice(fiatCurrency) + return await getPrice(fiatCurrency) || -1 } } } diff --git a/api/resolvers/wallet.js b/api/resolvers/wallet.js index ab2390ff..820c08f1 100644 --- a/api/resolvers/wallet.js +++ b/api/resolvers/wallet.js @@ -1,10 +1,17 @@ -import { createHodlInvoice, createInvoice, decodePaymentRequest, payViaPaymentRequest, getInvoice as getInvoiceFromLnd, getNode, deletePayment, getPayment, getIdentity } from 'ln-service' +import { + createHodlInvoice, createInvoice, payViaPaymentRequest, + getInvoice as getInvoiceFromLnd, deletePayment, getPayment, + parsePaymentRequest +} from 'ln-service' import crypto, { timingSafeEqual } from 'crypto' import serialize from './serial' import { decodeCursor, LIMIT, nextCursorEncoded } from '@/lib/cursor' import { SELECT, itemQueryWithMeta } from './item' import { msatsToSats, msatsToSatsDecimal } from '@/lib/format' -import { ANON_BALANCE_LIMIT_MSATS, ANON_INV_PENDING_LIMIT, USER_ID, BALANCE_LIMIT_MSATS, INVOICE_RETENTION_DAYS, INV_PENDING_LIMIT, USER_IDS_BALANCE_NO_LIMIT, LND_PATHFINDING_TIMEOUT_MS } from '@/lib/constants' +import { + ANON_BALANCE_LIMIT_MSATS, ANON_INV_PENDING_LIMIT, USER_ID, BALANCE_LIMIT_MSATS, + INVOICE_RETENTION_DAYS, INV_PENDING_LIMIT, USER_IDS_BALANCE_NO_LIMIT, LND_PATHFINDING_TIMEOUT_MS +} from '@/lib/constants' import { amountSchema, ssValidate, withdrawlSchema, lnAddrSchema, walletValidate } from '@/lib/validate' import { datePivot } from '@/lib/time' import assertGofacYourself from './ofac' @@ -15,6 +22,7 @@ import walletDefs from 'wallets/server' import { generateResolverName, generateTypeDefName } from '@/lib/wallet' import { lnAddrOptions } from '@/lib/lnurl' import { GqlAuthenticationError, GqlAuthorizationError, GqlInputError } from '@/lib/error' +import { getNodeInfo, getOurPubkey } from '../lnd' function injectResolvers (resolvers) { console.group('injected GraphQL resolvers:') @@ -74,7 +82,7 @@ export async function getInvoice (parent, { id }, { me, models, lnd }) { try { if (inv.confirmedAt) { - inv.confirmedPreimage = (await getInvoiceFromLnd({ id: inv.hash, lnd })).secret + inv.confirmedPreimage = inv.preimage ?? (await getInvoiceFromLnd({ id: inv.hash, lnd })).secret } } catch (err) { console.error('error fetching invoice from LND', err) @@ -399,7 +407,7 @@ const resolvers = { }) const [inv] = await serialize( - models.$queryRaw`SELECT * FROM create_invoice(${invoice.id}, ${hodlInvoice ? invoice.secret : null}::TEXT, ${invoice.request}, + models.$queryRaw`SELECT * FROM create_invoice(${invoice.id}, ${invoice.secret}::TEXT, ${invoice.request}, ${expiresAt}::timestamp, ${amount * 1000}, ${user.id}::INTEGER, ${description}, NULL, NULL, ${invLimit}::INTEGER, ${balanceLimit})`, { models } @@ -498,7 +506,7 @@ const resolvers = { preimage: async (withdrawl, args, { lnd }) => { try { if (withdrawl.status === 'CONFIRMED') { - return (await getPayment({ id: withdrawl.hash, lnd })).payment.secret + return withdrawl.preimage ?? (await getPayment({ id: withdrawl.hash, lnd })).payment.secret } } catch (err) { console.error('error fetching payment from LND', err) @@ -679,14 +687,14 @@ export async function createWithdrawal (parent, { invoice, maxFee }, { me, model // decode invoice to get amount let decoded, node try { - decoded = await decodePaymentRequest({ lnd, request: invoice }) + decoded = await parsePaymentRequest({ request: invoice }) } catch (error) { console.log(error) throw new GqlInputError('could not decode invoice') } try { - node = await getNode({ lnd, public_key: decoded.destination, is_omitting_channels: true }) + node = await getNodeInfo({ public_key: decoded.destination, is_omitting_channels: true }) } catch (error) { // likely not found if it's an unannounced channel, e.g. phoenix console.log(error) @@ -703,6 +711,10 @@ export async function createWithdrawal (parent, { invoice, maxFee }, { me, model throw new GqlInputError('your invoice must specify an amount') } + if (decoded.mtokens > Number.MAX_SAFE_INTEGER) { + throw new GqlInputError('your invoice amount is too large') + } + const msatsFee = Number(maxFee) * 1000 const user = await models.user.findUnique({ where: { id: me.id } }) @@ -784,8 +796,8 @@ export async function fetchLnAddrInvoice ( // decode invoice try { - const decoded = await decodePaymentRequest({ lnd, request: res.pr }) - const ourPubkey = (await getIdentity({ lnd })).public_key + const decoded = await parsePaymentRequest({ request: res.pr }) + const ourPubkey = await getOurPubkey() if (autoWithdraw && decoded.destination === ourPubkey && process.env.NODE_ENV === 'production') { // unset lnaddr so we don't trigger another withdrawal with same destination await models.wallet.deleteMany({ diff --git a/components/payment.js b/components/payment.js index 67a3265e..253cc6df 100644 --- a/components/payment.js +++ b/components/payment.js @@ -67,19 +67,25 @@ export const useInvoice = () => { if (error) { throw error } - const { hash, cancelled, actionError } = data.invoice + + const { hash, cancelled, actionError, actionState } = data.invoice if (cancelled || actionError) { throw new InvoiceCanceledError(hash, actionError) } + // write to cache if paid + if (actionState === 'PAID') { + client.writeQuery({ query: INVOICE, variables: { id }, data: { invoice: data.invoice } }) + } + return that(data.invoice) }, [client]) const waitController = useMemo(() => { const controller = new AbortController() const signal = controller.signal - controller.wait = async ({ id }, waitFor = inv => (inv.satsReceived > 0)) => { + controller.wait = async ({ id }, waitFor = inv => inv?.actionState === 'PAID') => { return await new Promise((resolve, reject) => { const interval = setInterval(async () => { try { @@ -138,11 +144,7 @@ export const useWalletPayment = () => { return await new Promise((resolve, reject) => { // can't use await here since we might pay JIT invoices and sendPaymentAsync is not supported yet. // see https://www.webln.guide/building-lightning-apps/webln-reference/webln.sendpaymentasync - wallet.sendPayment(bolt11) - // JIT invoice payments will never resolve here - // since they only get resolved after settlement which can't happen here - .then(resolve) - .catch(reject) + wallet.sendPayment(bolt11).catch(reject) invoice.waitUntilPaid({ id }, waitFor) .then(resolve) .catch(reject) diff --git a/components/use-paid-mutation.js b/components/use-paid-mutation.js index e4d14045..93009609 100644 --- a/components/use-paid-mutation.js +++ b/components/use-paid-mutation.js @@ -85,7 +85,7 @@ export function usePaidMutation (mutation, // onCompleted is called before the invoice is paid for optimistic updates ourOnCompleted?.(data) // don't wait to pay the invoice - waitForPayment(invoice, { persistOnNavigate }).then(() => { + waitForPayment(invoice, { persistOnNavigate, waitFor }).then(() => { onPaid?.(client.cache, { data }) }).catch(e => { console.error('usePaidMutation: failed to pay invoice', e) @@ -178,7 +178,8 @@ export const paidActionCacheMods = { id: `Invoice:${invoice.id}`, fields: { actionState: () => 'PAID', - confirmedAt: () => new Date().toISOString() + confirmedAt: () => new Date().toISOString(), + satsReceived: () => invoice.satsRequested } }) } diff --git a/components/wallet-logger.js b/components/wallet-logger.js index e695eb97..9e623e4f 100644 --- a/components/wallet-logger.js +++ b/components/wallet-logger.js @@ -163,9 +163,13 @@ export const WalletLoggerProvider = ({ children }) => { // IDB may not be ready yet return logQueue.current.push(log) } - const tx = idb.current.transaction(idbStoreName, 'readwrite') - const request = tx.objectStore(idbStoreName).add(log) - request.onerror = () => console.error('failed to save log:', log) + try { + const tx = idb.current.transaction(idbStoreName, 'readwrite') + const request = tx.objectStore(idbStoreName).add(log) + request.onerror = () => console.error('failed to save log:', log) + } catch (e) { + console.error('failed to save log:', log, e) + } }, []) useEffect(() => { @@ -214,19 +218,23 @@ export const WalletLoggerProvider = ({ children }) => { await deleteServerWalletLogs({ variables: { wallet: wallet?.walletType } }) } if (!wallet || wallet.sendPayment) { - const tx = idb.current.transaction(idbStoreName, 'readwrite') - const objectStore = tx.objectStore(idbStoreName) - const idx = objectStore.index('wallet_ts') - const request = wallet ? idx.openCursor(window.IDBKeyRange.bound([tag(wallet), -Infinity], [tag(wallet), Infinity])) : idx.openCursor() - request.onsuccess = function (event) { - const cursor = event.target.result - if (cursor) { - cursor.delete() - cursor.continue() - } else { + try { + const tx = idb.current.transaction(idbStoreName, 'readwrite') + const objectStore = tx.objectStore(idbStoreName) + const idx = objectStore.index('wallet_ts') + const request = wallet ? idx.openCursor(window.IDBKeyRange.bound([tag(wallet), -Infinity], [tag(wallet), Infinity])) : idx.openCursor() + request.onsuccess = function (event) { + const cursor = event.target.result + if (cursor) { + cursor.delete() + cursor.continue() + } else { // finished - setLogs((logs) => logs.filter(l => wallet ? l.wallet !== tag(wallet) : false)) + setLogs((logs) => logs.filter(l => wallet ? l.wallet !== tag(wallet) : false)) + } } + } catch (e) { + console.error('failed to delete logs', e) } } }, [me, setLogs]) diff --git a/fragments/wallet.js b/fragments/wallet.js index fc0f4c66..140c0b12 100644 --- a/fragments/wallet.js +++ b/fragments/wallet.js @@ -16,10 +16,10 @@ export const INVOICE_FIELDS = gql` isHeld comment lud18Data - confirmedPreimage actionState actionType actionError + confirmedPreimage }` export const INVOICE_FULL = gql` diff --git a/lib/fetch.js b/lib/fetch.js new file mode 100644 index 00000000..88ede993 --- /dev/null +++ b/lib/fetch.js @@ -0,0 +1,70 @@ +export async function fetchWithTimeout (resource, { timeout = 1000, ...options } = {}) { + const controller = new AbortController() + const id = setTimeout(() => controller.abort(), timeout) + + const response = await fetch(resource, { + ...options, + signal: controller.signal + }) + clearTimeout(id) + + return response +} + +class LRUCache { + constructor (maxSize = 100) { + this.maxSize = maxSize + this.cache = new Map() + } + + get (key) { + if (!this.cache.has(key)) return undefined + const value = this.cache.get(key) + // refresh the entry + this.cache.delete(key) + this.cache.set(key, value) + return value + } + + set (key, value) { + if (this.cache.has(key)) this.cache.delete(key) + else if (this.cache.size >= this.maxSize) { + // Remove the least recently used item + this.cache.delete(this.cache.keys().next().value) + } + this.cache.set(key, value) + } +} + +export function cachedFetcher (fetcher, { maxSize = 100, cacheExpiry, forceRefreshThreshold }) { + const cache = new LRUCache(maxSize) + + return async function cachedFetch (...args) { + const key = JSON.stringify(args) + const now = Date.now() + + async function fetchAndCache () { + const result = await fetcher(...args) + cache.set(key, { data: result, createdAt: now }) + return result + } + + const cached = cache.get(key) + + if (cached) { + const age = now - cached.createdAt + + if (cacheExpiry === 0 || age < cacheExpiry) { + return cached.data + } else if (forceRefreshThreshold === 0 || age < forceRefreshThreshold) { + fetchAndCache().catch(console.error) + return cached.data + } + } else if (forceRefreshThreshold === 0) { + fetchAndCache().catch(console.error) + return null + } + + return await fetchAndCache() + } +} diff --git a/pages/api/lnurlp/[username]/pay.js b/pages/api/lnurlp/[username]/pay.js index 10c82468..b153a2ee 100644 --- a/pages/api/lnurlp/[username]/pay.js +++ b/pages/api/lnurlp/[username]/pay.js @@ -81,7 +81,7 @@ export default async ({ query: { username, amount, nostr, comment, payerdata: pa }) await serialize( - models.$queryRaw`SELECT * FROM create_invoice(${invoice.id}, NULL, ${invoice.request}, + models.$queryRaw`SELECT * FROM create_invoice(${invoice.id}, ${invoice.secret}::TEXT, ${invoice.request}, ${expiresAt}::timestamp, ${Number(amount)}, ${user.id}::INTEGER, ${noteStr || description}, ${comment || null}, ${parsedPayerData || null}::JSONB, ${INV_PENDING_LIMIT}::INTEGER, ${USER_IDS_BALANCE_NO_LIMIT.includes(Number(user.id)) ? 0 : BALANCE_LIMIT_MSATS})`, diff --git a/pages/api/lnurlp/[username]/verify/[hash].js b/pages/api/lnurlp/[username]/verify/[hash].js index a8887b41..6168fcd6 100644 --- a/pages/api/lnurlp/[username]/verify/[hash].js +++ b/pages/api/lnurlp/[username]/verify/[hash].js @@ -1,15 +1,15 @@ -import lnd from '@/api/lnd' -import { getInvoice } from 'ln-service' +import models from '@/api/models' export default async ({ query: { hash } }, res) => { try { - const inv = await getInvoice({ id: hash, lnd }) - const settled = inv.is_confirmed - return res.status(200).json({ status: 'OK', settled, preimage: settled ? inv.secret : null, pr: inv.request }) - } catch (err) { - if (err[1] === 'UnexpectedLookupInvoiceErr') { + const inv = await models.invoice.findUnique({ where: { hash } }) + if (!inv) { return res.status(404).json({ status: 'ERROR', reason: 'not found' }) } + const settled = inv.confirmedAt + return res.status(200).json({ status: 'OK', settled: !!settled, preimage: settled ? inv.preimage : null, pr: inv.bolt11 }) + } catch (err) { + console.log('error', err) return res.status(500).json({ status: 'ERROR', reason: 'internal server error' }) } } diff --git a/prisma/migrations/20241002145114_create_invoice_update/migration.sql b/prisma/migrations/20241002145114_create_invoice_update/migration.sql new file mode 100644 index 00000000..fa0a808c --- /dev/null +++ b/prisma/migrations/20241002145114_create_invoice_update/migration.sql @@ -0,0 +1,44 @@ +CREATE OR REPLACE FUNCTION create_invoice(hash TEXT, preimage TEXT, bolt11 TEXT, expires_at timestamp(3) without time zone, + msats_req BIGINT, user_id INTEGER, idesc TEXT, comment TEXT, lud18_data JSONB, inv_limit INTEGER, balance_limit_msats BIGINT) +RETURNS "Invoice" +LANGUAGE plpgsql +AS $$ +DECLARE + invoice "Invoice"; + inv_limit_reached BOOLEAN; + balance_limit_reached BOOLEAN; + inv_pending_msats BIGINT; + wdwl_pending_msats BIGINT; +BEGIN + PERFORM ASSERT_SERIALIZED(); + + -- prevent too many pending invoices + SELECT inv_limit > 0 AND count(*) >= inv_limit, COALESCE(sum("msatsRequested"), 0) INTO inv_limit_reached, inv_pending_msats + FROM "Invoice" + WHERE "userId" = user_id AND "expiresAt" > now_utc() AND "confirmedAt" IS NULL AND cancelled = false; + + IF inv_limit_reached THEN + RAISE EXCEPTION 'SN_INV_PENDING_LIMIT'; + END IF; + + -- account for pending withdrawals + SELECT COALESCE(sum("msatsPaying"), 0) + COALESCE(sum("msatsFeePaying"), 0) INTO wdwl_pending_msats + FROM "Withdrawl" + WHERE "userId" = user_id AND status IS NULL; + + -- prevent pending invoices + msats from exceeding the limit + SELECT balance_limit_msats > 0 AND inv_pending_msats+wdwl_pending_msats+msats_req+msats > balance_limit_msats INTO balance_limit_reached + FROM users + WHERE id = user_id; + + IF balance_limit_reached THEN + RAISE EXCEPTION 'SN_INV_EXCEED_BALANCE'; + END IF; + + -- we good, proceed frens + INSERT INTO "Invoice" (hash, preimage, bolt11, "expiresAt", "msatsRequested", "userId", created_at, updated_at, "desc", comment, "lud18Data") + VALUES (hash, preimage, bolt11, expires_at, msats_req, user_id, now_utc(), now_utc(), idesc, comment, lud18_data) RETURNING * INTO invoice; + + RETURN invoice; +END; +$$; \ No newline at end of file diff --git a/wallets/wrap.js b/wallets/wrap.js index fa7822ca..91b7eafc 100644 --- a/wallets/wrap.js +++ b/wallets/wrap.js @@ -1,5 +1,5 @@ -import { createHodlInvoice, getHeight, parsePaymentRequest } from 'ln-service' -import { estimateRouteFee } from '../api/lnd' +import { createHodlInvoice, parsePaymentRequest } from 'ln-service' +import { estimateRouteFee, getBlockHeight } from '../api/lnd' import { toPositiveNumber } from '@/lib/validate' const MIN_OUTGOING_MSATS = BigInt(900) // the minimum msats we'll allow for the outgoing invoice @@ -44,7 +44,7 @@ export default async function wrapInvoice (bolt11, { msats, description, descrip if (outgoingMsat < MIN_OUTGOING_MSATS) { throw new Error(`Invoice amount is too low: ${outgoingMsat}`) } - if (inv.mtokens > MAX_OUTGOING_MSATS) { + if (outgoingMsat > MAX_OUTGOING_MSATS) { throw new Error(`Invoice amount is too high: ${outgoingMsat}`) } } else { @@ -131,7 +131,7 @@ export default async function wrapInvoice (bolt11, { msats, description, descrip timeout: FEE_ESTIMATE_TIMEOUT_SECS }) - const { current_block_height: blockHeight } = await getHeight({ lnd }) + const blockHeight = await getBlockHeight() /* we want the incoming invoice to have MIN_SETTLEMENT_CLTV_DELTA higher final cltv delta than the expected ctlv_delta of the outgoing invoice's entire route diff --git a/worker/imgproxy.js b/worker/imgproxy.js index 7ad4df52..49788098 100644 --- a/worker/imgproxy.js +++ b/worker/imgproxy.js @@ -3,6 +3,7 @@ import { extractUrls } from '@/lib/md.js' import { isJob } from '@/lib/item.js' import path from 'node:path' import { decodeProxyUrl } from '@/lib/url' +import { fetchWithTimeout } from '@/lib/fetch' const imgProxyEnabled = process.env.NODE_ENV === 'production' || (process.env.NEXT_PUBLIC_IMGPROXY_URL && process.env.IMGPROXY_SALT && process.env.IMGPROXY_KEY) @@ -133,19 +134,6 @@ const createImgproxyPath = ({ url, pathname = '/', options }) => { return path.join(pathname, signature, target) } -async function fetchWithTimeout (resource, { timeout = 1000, ...options } = {}) { - const controller = new AbortController() - const id = setTimeout(() => controller.abort(), timeout) - - const response = await fetch(resource, { - ...options, - signal: controller.signal - }) - clearTimeout(id) - - return response -} - const isMediaURL = async (url, { forceFetch }) => { if (cache.has(url)) return cache.get(url) diff --git a/worker/nostr.js b/worker/nostr.js index a84a2851..7dd932c9 100644 --- a/worker/nostr.js +++ b/worker/nostr.js @@ -1,18 +1,17 @@ -import { getInvoice } from 'ln-service' import { signId, calculateId, getPublicKey } from 'nostr' import { Relay } from '@/lib/nostr' const nostrOptions = { startAfter: 5, retryLimit: 21, retryBackoff: true } export async function nip57 ({ data: { hash }, boss, lnd, models }) { - let inv, lnInv + let inv try { - lnInv = await getInvoice({ id: hash, lnd }) inv = await models.invoice.findUnique({ where: { hash } }) + if (!inv || !inv.confirmedAt) return } catch (err) { console.log(err) // on lnd related errors, we manually retry which so we don't exponentially backoff @@ -35,14 +34,14 @@ export async function nip57 ({ data: { hash }, boss, lnd, models }) { const tags = [ptag] if (etag) tags.push(etag) if (atag) tags.push(atag) - tags.push(['bolt11', lnInv.request]) + tags.push(['bolt11', inv.bolt11]) tags.push(['description', inv.desc]) - tags.push(['preimage', lnInv.secret]) + tags.push(['preimage', inv.preimage]) const e = { kind: 9735, pubkey: getPublicKey(process.env.NOSTR_PRIVATE_KEY), - created_at: Math.floor(new Date(lnInv.confirmed_at).getTime() / 1000), + created_at: Math.floor(new Date(inv.confirmedAt).getTime() / 1000), content: '', tags } diff --git a/worker/paidAction.js b/worker/paidAction.js index 52820693..4bd19bcd 100644 --- a/worker/paidAction.js +++ b/worker/paidAction.js @@ -14,7 +14,7 @@ import { MIN_SETTLEMENT_CLTV_DELTA } from 'wallets/wrap' // aggressive finalization retry options const FINALIZE_OPTIONS = { retryLimit: 2 ** 31 - 1, retryBackoff: false, retryDelay: 5, priority: 1000 } -async function transitionInvoice (jobName, { invoiceId, fromState, toState, transition }, { models, lnd, boss }) { +async function transitionInvoice (jobName, { invoiceId, fromState, toState, transition, invoice }, { models, lnd, boss }) { console.group(`${jobName}: transitioning invoice ${invoiceId} from ${fromState} to ${toState}`) try { @@ -30,7 +30,7 @@ async function transitionInvoice (jobName, { invoiceId, fromState, toState, tran fromState = [fromState] } - const lndInvoice = await getInvoice({ id: currentDbInvoice.hash, lnd }) + const lndInvoice = invoice ?? await getInvoice({ id: currentDbInvoice.hash, lnd }) const transitionedInvoice = await models.$transaction(async tx => { const include = { @@ -133,7 +133,7 @@ async function performPessimisticAction ({ lndInvoice, dbInvoice, tx, models, ln } } -export async function paidActionPaid ({ data: { invoiceId }, models, lnd, boss }) { +export async function paidActionPaid ({ data: { invoiceId, ...args }, models, lnd, boss }) { return await transitionInvoice('paidActionPaid', { invoiceId, fromState: ['HELD', 'PENDING', 'FORWARDED'], @@ -153,12 +153,13 @@ export async function paidActionPaid ({ data: { invoiceId }, models, lnd, boss } confirmedIndex: lndInvoice.confirmed_index, msatsReceived: BigInt(lndInvoice.received_mtokens) } - } + }, + ...args }, { models, lnd, boss }) } // this performs forward creating the outgoing payment -export async function paidActionForwarding ({ data: { invoiceId }, models, lnd, boss }) { +export async function paidActionForwarding ({ data: { invoiceId, ...args }, models, lnd, boss }) { const transitionedInvoice = await transitionInvoice('paidActionForwarding', { invoiceId, fromState: 'PENDING_HELD', @@ -213,7 +214,8 @@ export async function paidActionForwarding ({ data: { invoiceId }, models, lnd, } } } - } + }, + ...args }, { models, lnd, boss }) // only pay if we successfully transitioned which can only happen once @@ -238,7 +240,7 @@ export async function paidActionForwarding ({ data: { invoiceId }, models, lnd, } // this finalizes the forward by settling the incoming invoice after the outgoing payment is confirmed -export async function paidActionForwarded ({ data: { invoiceId }, models, lnd, boss }) { +export async function paidActionForwarded ({ data: { invoiceId, withdrawal, ...args }, models, lnd, boss }) { return await transitionInvoice('paidActionForwarded', { invoiceId, fromState: 'FORWARDING', @@ -249,7 +251,7 @@ export async function paidActionForwarded ({ data: { invoiceId }, models, lnd, b } const { hash, msatsPaying } = dbInvoice.invoiceForward.withdrawl - const { payment, is_confirmed: isConfirmed } = await getPayment({ id: hash, lnd }) + const { payment, is_confirmed: isConfirmed } = withdrawal ?? await getPayment({ id: hash, lnd }) if (!isConfirmed) { throw new Error('payment is not confirmed') } @@ -258,6 +260,7 @@ export async function paidActionForwarded ({ data: { invoiceId }, models, lnd, b await settleHodlInvoice({ secret: payment.secret, lnd }) return { + preimage: payment.secret, invoiceForward: { update: { withdrawl: { @@ -271,12 +274,13 @@ export async function paidActionForwarded ({ data: { invoiceId }, models, lnd, b } } } - } + }, + ...args }, { models, lnd, boss }) } // when the pending forward fails, we need to cancel the incoming invoice -export async function paidActionFailedForward ({ data: { invoiceId }, models, lnd, boss }) { +export async function paidActionFailedForward ({ data: { invoiceId, withdrawal: pWithdrawal, ...args }, models, lnd, boss }) { return await transitionInvoice('paidActionFailedForward', { invoiceId, fromState: 'FORWARDING', @@ -289,7 +293,7 @@ export async function paidActionFailedForward ({ data: { invoiceId }, models, ln let withdrawal let notSent = false try { - withdrawal = await getPayment({ id: dbInvoice.invoiceForward.withdrawl.hash, lnd }) + withdrawal = pWithdrawal ?? await getPayment({ id: dbInvoice.invoiceForward.withdrawl.hash, lnd }) } catch (err) { if (err[1] === 'SentPaymentNotFound' && dbInvoice.invoiceForward.withdrawl.createdAt < datePivot(new Date(), { milliseconds: -LND_PATHFINDING_TIMEOUT_MS * 2 })) { @@ -313,17 +317,18 @@ export async function paidActionFailedForward ({ data: { invoiceId }, models, ln update: { withdrawl: { update: { - status: getPaymentFailureStatus(withdrawal) + status: getPaymentFailureStatus(withdrawal).status } } } } } - } + }, + ...args }, { models, lnd, boss }) } -export async function paidActionHeld ({ data: { invoiceId }, models, lnd, boss }) { +export async function paidActionHeld ({ data: { invoiceId, ...args }, models, lnd, boss }) { return await transitionInvoice('paidActionHeld', { invoiceId, fromState: 'PENDING_HELD', @@ -355,11 +360,12 @@ export async function paidActionHeld ({ data: { invoiceId }, models, lnd, boss } isHeld: true, msatsReceived: BigInt(lndInvoice.received_mtokens) } - } + }, + ...args }, { models, lnd, boss }) } -export async function paidActionCanceling ({ data: { invoiceId }, models, lnd, boss }) { +export async function paidActionCanceling ({ data: { invoiceId, ...args }, models, lnd, boss }) { return await transitionInvoice('paidActionCanceling', { invoiceId, fromState: ['HELD', 'PENDING', 'PENDING_HELD', 'FAILED_FORWARD'], @@ -370,11 +376,12 @@ export async function paidActionCanceling ({ data: { invoiceId }, models, lnd, b } await cancelHodlInvoice({ id: dbInvoice.hash, lnd }) - } + }, + ...args }, { models, lnd, boss }) } -export async function paidActionFailed ({ data: { invoiceId }, models, lnd, boss }) { +export async function paidActionFailed ({ data: { invoiceId, ...args }, models, lnd, boss }) { return await transitionInvoice('paidActionFailed', { invoiceId, // any of these states can transition to FAILED @@ -390,6 +397,7 @@ export async function paidActionFailed ({ data: { invoiceId }, models, lnd, boss return { cancelled: true } - } + }, + ...args }, { models, lnd, boss }) } diff --git a/worker/wallet.js b/worker/wallet.js index 4ffdaf88..ac749e48 100644 --- a/worker/wallet.js +++ b/worker/wallet.js @@ -15,6 +15,7 @@ import { paidActionForwarding, paidActionCanceling } from './paidAction.js' +import { getPaymentFailureStatus } from '@/api/lnd/index.js' export async function subscribeToWallet (args) { await subscribeToDeposits(args) @@ -67,10 +68,11 @@ async function subscribeToDeposits (args) { try { logEvent('invoice_updated', inv) if (inv.secret) { - await checkInvoice({ data: { hash: inv.id }, ...args }) + // subscribeToInvoices only returns when added or settled + await checkInvoice({ data: { hash: inv.id, invoice: inv }, ...args }) } else { // this is a HODL invoice. We need to use SubscribeToInvoice which has is_held transitions - // https://api.lightning.community/api/lnd/invoices/subscribe-single-invoice + // and is_canceled transitions https://api.lightning.community/api/lnd/invoices/subscribe-single-invoice // SubscribeToInvoices is only for invoice creation and settlement transitions // https://api.lightning.community/api/lnd/lightning/subscribe-invoices subscribeToHodlInvoice({ hash: inv.id, ...args }) @@ -97,7 +99,7 @@ function subscribeToHodlInvoice (args) { sub.on('invoice_updated', async (inv) => { logEvent('hodl_invoice_updated', inv) try { - await checkInvoice({ data: { hash: inv.id }, ...args }) + await checkInvoice({ data: { hash: inv.id, invoice: inv }, ...args }) // after settle or confirm we can stop listening for updates if (inv.is_confirmed || inv.is_canceled) { resolve() @@ -112,8 +114,10 @@ function subscribeToHodlInvoice (args) { }) } -export async function checkInvoice ({ data: { hash }, boss, models, lnd }) { - const inv = await getInvoice({ id: hash, lnd }) +// if we already have the invoice from a subscription event or previous call, +// we can skip a getInvoice call +export async function checkInvoice ({ data: { hash, invoice }, boss, models, lnd }) { + const inv = invoice ?? await getInvoice({ id: hash, lnd }) // invoice could be created by LND but wasn't inserted into the database yet // this is expected and the function will be called again with the updates @@ -134,7 +138,7 @@ export async function checkInvoice ({ data: { hash }, boss, models, lnd }) { if (inv.is_confirmed) { if (dbInv.actionType) { - return await paidActionPaid({ data: { invoiceId: dbInv.id }, models, lnd, boss }) + return await paidActionPaid({ data: { invoiceId: dbInv.id, invoice: inv }, models, lnd, boss }) } // NOTE: confirm invoice prevents double confirmations (idempotent) @@ -146,8 +150,6 @@ export async function checkInvoice ({ data: { hash }, boss, models, lnd }) { models.invoice.update({ where: { hash }, data: { confirmedIndex: inv.confirmed_index } }) ], { models }) - // don't send notifications for JIT invoices - if (dbInv.preimage) return if (code === 0) { notifyDeposit(dbInv.userId, { comment: dbInv.comment, ...inv }) } @@ -160,11 +162,11 @@ export async function checkInvoice ({ data: { hash }, boss, models, lnd }) { if (dbInv.invoiceForward) { if (dbInv.invoiceForward.withdrawl) { // transitions when held are dependent on the withdrawl status - return await checkWithdrawal({ data: { hash: dbInv.invoiceForward.withdrawl.hash }, models, lnd, boss }) + return await checkWithdrawal({ data: { hash: dbInv.invoiceForward.withdrawl.hash, invoice: inv }, models, lnd, boss }) } - return await paidActionForwarding({ data: { invoiceId: dbInv.id }, models, lnd, boss }) + return await paidActionForwarding({ data: { invoiceId: dbInv.id, invoice: inv }, models, lnd, boss }) } - return await paidActionHeld({ data: { invoiceId: dbInv.id }, models, lnd, boss }) + return await paidActionHeld({ data: { invoiceId: dbInv.id, invoice: inv }, models, lnd, boss }) } // First query makes sure that after payment, JIT invoices are settled // within 60 seconds or they will be canceled to minimize risk of @@ -190,7 +192,7 @@ export async function checkInvoice ({ data: { hash }, boss, models, lnd }) { if (inv.is_canceled) { if (dbInv.actionType) { - return await paidActionFailed({ data: { invoiceId: dbInv.id }, models, lnd, boss }) + return await paidActionFailed({ data: { invoiceId: dbInv.id, invoice: inv }, models, lnd, boss }) } return await serialize( @@ -216,7 +218,9 @@ async function subscribeToWithdrawals (args) { sub.on('confirmed', async (payment) => { logEvent('confirmed', payment) try { - await checkWithdrawal({ data: { hash: payment.id }, ...args }) + // see https://github.com/alexbosworth/lightning/blob/ddf1f214ebddf62e9e19fd32a57fbeeba713340d/lnd_methods/offchain/subscribe_to_payments.js + const withdrawal = { payment, is_confirmed: true } + await checkWithdrawal({ data: { hash: payment.id, withdrawal }, ...args }) } catch (error) { logEventError('confirmed', error) } @@ -225,7 +229,9 @@ async function subscribeToWithdrawals (args) { sub.on('failed', async (payment) => { logEvent('failed', payment) try { - await checkWithdrawal({ data: { hash: payment.id }, ...args }) + // see https://github.com/alexbosworth/lightning/blob/ddf1f214ebddf62e9e19fd32a57fbeeba713340d/lnd_methods/offchain/subscribe_to_payments.js + const withdrawal = { failed: payment, is_failed: true } + await checkWithdrawal({ data: { hash: payment.id, withdrawal }, ...args }) } catch (error) { logEventError('failed', error) } @@ -238,7 +244,9 @@ async function subscribeToWithdrawals (args) { await checkPendingWithdrawals(args) } -export async function checkWithdrawal ({ data: { hash }, boss, models, lnd }) { +// if we already have the payment from a subscription event or previous call, +// we can skip a getPayment call +export async function checkWithdrawal ({ data: { hash, withdrawal, invoice }, boss, models, lnd }) { // get the withdrawl if pending or it's an invoiceForward const dbWdrwl = await models.withdrawl.findFirst({ where: { @@ -265,7 +273,7 @@ export async function checkWithdrawal ({ data: { hash }, boss, models, lnd }) { let wdrwl let notSent = false try { - wdrwl = await getPayment({ id: hash, lnd }) + wdrwl = withdrawal ?? await getPayment({ id: hash, lnd }) } catch (err) { if (err[1] === 'SentPaymentNotFound' && dbWdrwl.createdAt < datePivot(new Date(), { milliseconds: -LND_PATHFINDING_TIMEOUT_MS * 2 })) { @@ -278,15 +286,20 @@ export async function checkWithdrawal ({ data: { hash }, boss, models, lnd }) { if (wdrwl?.is_confirmed) { if (dbWdrwl.invoiceForward.length > 0) { - return await paidActionForwarded({ data: { invoiceId: dbWdrwl.invoiceForward[0].invoice.id }, models, lnd, boss }) + return await paidActionForwarded({ data: { invoiceId: dbWdrwl.invoiceForward[0].invoice.id, withdrawal: wdrwl, invoice }, models, lnd, boss }) } const fee = Number(wdrwl.payment.fee_mtokens) const paid = Number(wdrwl.payment.mtokens) - fee - const [{ confirm_withdrawl: code }] = await serialize( + const [{ confirm_withdrawl: code }] = await serialize([ models.$queryRaw`SELECT confirm_withdrawl(${dbWdrwl.id}::INTEGER, ${paid}, ${fee})`, - { models } - ) + models.withdrawl.update({ + where: { id: dbWdrwl.id }, + data: { + preimage: wdrwl.payment.secret + } + }) + ], { models }) if (code === 0) { notifyWithdrawal(dbWdrwl.userId, wdrwl) if (dbWdrwl.wallet) { @@ -299,23 +312,10 @@ export async function checkWithdrawal ({ data: { hash }, boss, models, lnd }) { } } else if (wdrwl?.is_failed || notSent) { if (dbWdrwl.invoiceForward.length > 0) { - return await paidActionFailedForward({ data: { invoiceId: dbWdrwl.invoiceForward[0].invoice.id }, models, lnd, boss }) + return await paidActionFailedForward({ data: { invoiceId: dbWdrwl.invoiceForward[0].invoice.id, withdrawal: wdrwl, invoice }, models, lnd, boss }) } - let status = 'UNKNOWN_FAILURE'; let message = 'unknown failure' - if (wdrwl?.failed.is_insufficient_balance) { - status = 'INSUFFICIENT_BALANCE' - message = "you didn't have enough sats" - } else if (wdrwl?.failed.is_invalid_payment) { - status = 'INVALID_PAYMENT' - message = 'invalid payment' - } else if (wdrwl?.failed.is_pathfinding_timeout) { - status = 'PATHFINDING_TIMEOUT' - message = 'no route found' - } else if (wdrwl?.failed.is_route_not_found) { - status = 'ROUTE_NOT_FOUND' - message = 'no route found' - } + const { status, message } = getPaymentFailureStatus(wdrwl) const [{ reverse_withdrawl: code }] = await serialize( models.$queryRaw` @@ -393,12 +393,11 @@ export async function finalizeHodlInvoice ({ data: { hash }, models, lnd, boss, // if this is an actionType we need to cancel conditionally if (dbInv.actionType) { - await paidActionCanceling({ data: { invoiceId: dbInv.id }, models, lnd, boss }) - await checkInvoice({ data: { hash }, models, lnd, ...args }) - return + await paidActionCanceling({ data: { invoiceId: dbInv.id, invoice: inv }, models, lnd, boss }) + } else { + await cancelHodlInvoice({ id: hash, lnd }) } - await cancelHodlInvoice({ id: hash, lnd }) // sync LND invoice status with invoice status in database await checkInvoice({ data: { hash }, models, lnd, ...args }) }