Add image fees during item creation/update

* we calculate image fees during item creation and update now
* function imageFees returns queries which deduct fees from user and mark images as paid + fees
* queries need to be run inside same transaction as item creation/update

api/resolvers/item.js

@@ -1,6 +1,6 @@
 import { GraphQLError } from 'graphql'
 import { ensureProtocol, removeTracking } from '../../lib/url'
-import { serializeInvoicable } from './serial'
+import serialize, { serializeInvoicable } from './serial'
 import { decodeCursor, LIMIT, nextCursorEncoded } from '../../lib/cursor'
 import { getMetadata, metadataRuleSets } from 'page-metadata-parser'
 import { ruleSet as publicationDateRuleSet } from '../../lib/timedate-scraper'
@@ -1090,10 +1090,14 @@ export const updateItem = async (parent, { sub: subName, forward, options, ...it
   item = { subName, userId: me.id, ...item }
   const fwdUsers = await getForwardUsers(models, forward)
 
+  const [imgQueriesFn, imgFees] = await imageFees(item.text, { models, me })
   item = await serializeInvoicable(
-    models.$queryRawUnsafe(`${SELECT} FROM update_item($1::JSONB, $2::JSONB, $3::JSONB) AS "Item"`,
-      JSON.stringify(item), JSON.stringify(fwdUsers), JSON.stringify(options)),
-    { models, lnd, hash, hmac, me }
+    [
+      models.$queryRawUnsafe(`${SELECT} FROM update_item($1::JSONB, $2::JSONB, $3::JSONB) AS "Item"`,
+        JSON.stringify(item), JSON.stringify(fwdUsers), JSON.stringify(options)),
+      ...imgQueriesFn(Number(item.id))
+    ],
+    { models, lnd, hash, hmac, me, enforceFee: imgFees }
   )
 
   await createMentions(item, models)
@@ -1123,7 +1127,8 @@ export const createItem = async (parent, { forward, options, ...item }, { me, mo
     item.url = removeTracking(item.url)
   }
 
-  const enforceFee = me ? undefined : (item.parentId ? ANON_COMMENT_FEE : (ANON_POST_FEE + (item.boost || 0)))
+  const [imgQueriesFn, imgFees] = await imageFees(item.text, { models, me })
+  const enforceFee = (me ? undefined : (item.parentId ? ANON_COMMENT_FEE : (ANON_POST_FEE + (item.boost || 0)))) + imgFees
   item = await serializeInvoicable(
     models.$queryRawUnsafe(
       `${SELECT} FROM create_item($1::JSONB, $2::JSONB, $3::JSONB, '${spamInterval}'::INTERVAL) AS "Item"`,
@@ -1131,6 +1136,10 @@ export const createItem = async (parent, { forward, options, ...item }, { me, mo
     { models, lnd, hash, hmac, me, enforceFee }
   )
 
+  // TODO run image queries in same transaction as create_item
+  const imgQueries = imgQueriesFn(item.id)
+  if (imgQueries.length > 0) await serialize(models, ...imgQueries)
+
   await createMentions(item, models)
 
   await enqueueDeletionJob(item, models)
@@ -1141,6 +1150,81 @@ export const createItem = async (parent, { forward, options, ...item }, { me, mo
   return item
 }
 
+const AWS_S3_URL_REGEXP = new RegExp(`https://${process.env.NEXT_PUBLIC_AWS_UPLOAD_BUCKET}.s3.amazonaws.com/([0-9]+)`, 'g')
+async function imageFees (text, { models, me }) {
+  // To apply image fees, we return queries which need to be run, preferably in the same transaction as creating or updating an item.
+  function queries (userId, imgIds, imgFees) {
+    return itemId => {
+      return [
+        // pay fees
+        models.$queryRawUnsafe('SELECT * FROM user_fee($1::INTEGER, $2::INTEGER, $3::BIGINT)', userId, itemId, imgFees),
+        // mark images as paid
+        models.upload.updateMany({ where: { id: { in: imgIds } }, data: { paid: true } })
+      ]
+    }
+  }
+
+  // no text means no image fees
+  if (!text) return [itemId => [], 0]
+
+  // parse all s3 keys (= image ids) from text
+  const textS3Keys = [...text.matchAll(AWS_S3_URL_REGEXP)].map(m => Number(m[1]))
+  if (!textS3Keys.length) return [itemId => [], 0]
+
+  // we want to ignore image ids in text for which someone already paid during fee calculation
+  // to make sure that every image is only paid once
+  const unpaidS3Keys = (await models.upload.findMany({ select: { id: true }, where: { id: { in: textS3Keys }, paid: false } })).map(({ id }) => id)
+  const unpaid = unpaidS3Keys.length
+
+  if (!unpaid) return [itemId => [], 0]
+
+  if (!me) {
+    // anons pay for every new image 100 sats
+    const fees = unpaid * 100
+    return [queries(ANON_USER_ID, unpaidS3Keys, fees), fees]
+  }
+
+  // we want to ignore avatars during fee calculation
+  const { photoId } = await models.user.findUnique({ where: { id: me.id } })
+
+  // check how much stacker uploaded in last 24 hours (excluding avatar)
+  const { _sum: { size: size24h } } = await models.upload.aggregate({
+    _sum: { size: true },
+    where: {
+      userId: me.id,
+      createdAt: { gt: datePivot(new Date(), { days: -1 }) },
+      paid: true,
+      id: photoId ? { not: photoId } : undefined
+    }
+  })
+
+  // check how much stacker uploaded now in size
+  const { _sum: { size: sizeNow } } = await models.upload.aggregate({
+    _count: { id: true },
+    _sum: { size: true },
+    where: { id: { in: unpaidS3Keys } }
+  })
+
+  // total size that we consider to calculate fees includes size of images within last 24 hours and size of incoming images
+  const size = size24h + sizeNow
+  const MB = 1024 * 1024 // factor for bytes -> megabytes
+
+  // 10 MB per 24 hours are free. fee is also 0 if there are no incoming images (obviously)
+  let fees
+  if (!sizeNow || size <= 10 * MB) {
+    fees = 0
+  } else if (size <= 25 * MB) {
+    fees = 10 * unpaid
+  } else if (size <= 50 * MB) {
+    fees = 100 * unpaid
+  } else if (size <= 100 * MB) {
+    fees = 1000 * unpaid
+  } else {
+    fees = 10000 * unpaid
+  }
+  return [queries(me.id, unpaidS3Keys, fees), fees]
+}
+
 const clearDeletionJobs = async (item, models) => {
   await models.$queryRawUnsafe(`DELETE FROM pgboss.job WHERE name = 'deleteItem' AND data->>'id' = '${item.id}';`)
 }

api/resolvers/serial.js

@@ -59,25 +59,29 @@ export default async function serialize (models, ...calls) {
   })
 }
 
-export async function serializeInvoicable (query, { models, lnd, hash, hmac, me, enforceFee }) {
+export async function serializeInvoicable (queries, { models, lnd, hash, hmac, me, enforceFee }) {
   if (!me && !hash) {
     throw new Error('you must be logged in or pay')
   }
 
-  let trx = [query]
+  let trx = Array.isArray(queries) ? queries : [queries]
+  // save offset to the first query in arguments relative to queries that we will run
+  let queryOffset = 0
 
   let invoice
   if (hash) {
     invoice = await checkInvoice(models, hash, hmac, enforceFee)
     trx = [
       models.$queryRaw`UPDATE users SET msats = msats + ${invoice.msatsReceived} WHERE id = ${invoice.user.id}`,
-      query,
+      ...trx,
       models.invoice.update({ where: { hash: invoice.hash }, data: { confirmedAt: new Date() } })
     ]
+    // first query in arguments is now at index 1
+    queryOffset = 1
   }
 
   const results = await serialize(models, ...trx)
-  const result = trx.length > 1 ? results[1][0] : results[0]
+  const result = trx.length > 1 ? results[queryOffset][0] : results[0]
 
   if (invoice?.isHeld) await settleHodlInvoice({ secret: invoice.preimage, lnd })
 

api/resolvers/upload.js

@@ -28,7 +28,8 @@ export default {
         size,
         width,
         height,
-        userId: me.id
+        userId: me.id,
+        paid: avatar ? undefined : false
       }
 
       let uploadId

prisma/migrations/20231024155646_user_fee/migration.sql

@@ -0,0 +1,24 @@
+-- function to manually deduct fees from user, for example for images fees
+CREATE OR REPLACE FUNCTION user_fee(user_id INTEGER, item_id INTEGER, cost_msats BIGINT)
+RETURNS users
+LANGUAGE plpgsql
+AS $$
+DECLARE
+    user users;
+    user_msats BIGINT;
+BEGIN
+    PERFORM ASSERT_SERIALIZED();
+
+    SELECT msats INTO user_msats FROM users WHERE id = user_id;
+    IF cost_msats > user_msats THEN
+        RAISE EXCEPTION 'SN_INSUFFICIENT_FUNDS';
+    END IF;
+
+    UPDATE users SET msats = msats - cost_msats WHERE id = user_id RETURNING * INTO user;
+
+    INSERT INTO "ItemAct" (msats, "itemId", "userId", act)
+    VALUES (cost_msats, item_id, user_id, 'FEE');
+
+    RETURN user;
+END;
+$$;

worker/imgproxy.js

@@ -12,7 +12,6 @@ if (!imgProxyEnabled) {
 const IMGPROXY_URL = process.env.NEXT_PUBLIC_IMGPROXY_URL
 const IMGPROXY_SALT = process.env.IMGPROXY_SALT
 const IMGPROXY_KEY = process.env.IMGPROXY_KEY
-const AWS_S3_URL = `https://${process.env.NEXT_PUBLIC_AWS_UPLOAD_BUCKET}.s3.amazonaws.com/`
 
 const cache = new Map()
 
@@ -107,10 +106,6 @@ export const createImgproxyUrls = async (id, text, { models, forceFetch }) => {
       url = decodeOriginalUrl(url)
       console.log('[imgproxy] id:', id, '-- original url:', url)
     }
-    if (url.startsWith(AWS_S3_URL)) {
-      const s3Key = url.split('/').pop()
-      await models.upload.update({ data: { itemId: Number(id) }, where: { id: Number(s3Key) } })
-    }
     if (!(await isImageURL(url, { forceFetch }))) {
       console.log('[imgproxy] id:', id, '-- not image url:', url)
       continue