Add schedule to delete unused images

api/s3/index.js

@@ -24,12 +24,14 @@ export function createPresignedPost ({ key, type, size }) {
   })
 }
 
-export function deleteObject (key) {
+export function deleteObjects (keys) {
   const s3 = new AWS.S3({ apiVersion: '2006-03-01' })
   return new Promise((resolve, reject) => {
-    s3.deleteObject({
+    s3.deleteObjects({
       Bucket,
-      Key: key
-    }, (err, data) => { err ? reject(err) : resolve(key) })
+      Delete: {
+        Objects: keys.map(key => ({ Key: String(key) }))
+      }
+    }, (err, data) => { err ? reject(err) : resolve(keys) })
   })
 }

prisma/migrations/20231025203103_delete_unused_images_schedule/migration.sql

@@ -0,0 +1,23 @@
+-- add 'deleteUnusedImages' job to the prisma hack
+-- see migration 20230522153900_schedule_jobs
+CREATE OR REPLACE FUNCTION create_scheduled_jobs()
+RETURNS INTEGER
+LANGUAGE plpgsql
+AS $$
+DECLARE
+BEGIN
+    INSERT INTO pgboss.schedule (name, cron, timezone) VALUES ('trust', '0 2 * * *', 'America/Chicago') ON CONFLICT DO NOTHING;
+    INSERT INTO pgboss.schedule (name, cron, timezone) VALUES ('auction', '* * * * *', 'America/Chicago') ON CONFLICT DO NOTHING;
+    INSERT INTO pgboss.schedule (name, cron, timezone) VALUES ('earn', '0 0 * * *', 'America/Chicago') ON CONFLICT DO NOTHING;
+    INSERT INTO pgboss.schedule (name, cron, timezone) VALUES ('streak', '15 0 * * *','America/Chicago') ON CONFLICT DO NOTHING;
+    INSERT INTO pgboss.schedule (name, cron, timezone) VALUES ('views', '0 0 * * *', 'America/Chicago') ON CONFLICT DO NOTHING;
+    INSERT INTO pgboss.schedule (name, cron, timezone) VALUES ('rankViews', '* * * * *', 'America/Chicago') ON CONFLICT DO NOTHING;
+    INSERT INTO pgboss.schedule (name, cron, timezone) VALUES ('rankViews', '* * * * *', 'America/Chicago') ON CONFLICT DO NOTHING;
+    INSERT INTO pgboss.schedule (name, cron, timezone) VALUES ('deleteUnusedImages', '* * * * *', 'America/Chicago') ON CONFLICT DO NOTHING;
+    return 0;
+EXCEPTION WHEN OTHERS THEN
+    return 0;
+END;
+$$;
+
+SELECT create_scheduled_jobs();

worker/deleteUnusedImages.js

@@ -0,0 +1,18 @@
+import { deleteObjects } from '../api/s3'
+
+export function deleteUnusedImages ({ models }) {
+  return async function ({ name }) {
+    console.log('running', name)
+
+    // delete all images in database and S3 which weren't paid in the last 24 hours
+    const unpaidImages = await models.$queryRaw`
+    SELECT id
+    FROM "Upload"
+    WHERE paid = 'f' AND created_at < date_trunc('hour', now() - interval '24 hours')
+    `
+    const s3Keys = unpaidImages.map(({ id }) => id)
+    console.log('deleting images:', s3Keys)
+    await deleteObjects(s3Keys)
+    await models.upload.deleteMany({ where: { id: { in: s3Keys } } })
+  }
+}

worker/index.js

@@ -16,6 +16,7 @@ import { authenticatedLndGrpc } from 'ln-service'
 import { views, rankViews } from './views.js'
 import { imgproxy } from './imgproxy.js'
 import { deleteItem } from './ephemeralItems.js'
+import { deleteUnusedImages } from './deleteUnusedImages.js'
 
 const { loadEnvConfig } = nextEnv
 const { ApolloClient, HttpLink, InMemoryCache } = apolloClient
@@ -70,6 +71,7 @@ async function work () {
   await boss.work('rankViews', rankViews(args))
   await boss.work('imgproxy', imgproxy(args))
   await boss.work('deleteItem', deleteItem(args))
+  await boss.work('deleteUnusedImages', deleteUnusedImages(args))
 
   console.log('working jobs')
 }