Fix image deletion batch size (#769)

api/s3/index.js

@@ -24,14 +24,29 @@ export function createPresignedPost ({ key, type, size }) {
   })
 }
 
-export function deleteObjects (keys) {
+export async function deleteObjects (keys) {
   const s3 = new AWS.S3({ apiVersion: '2006-03-01' })
-  return new Promise((resolve, reject) => {
+  // max 1000 keys per request
-    s3.deleteObjects({
+  // see https://docs.aws.amazon.com/cli/latest/reference/s3api/delete-objects.html
-      Bucket,
+  const batchSize = 1000
-      Delete: {
+  const deleted = []
-        Objects: keys.map(key => ({ Key: String(key) }))
+  for (let i = 0; i < keys.length; i += batchSize) {
+    const batch = keys.slice(i, i + batchSize)
+    await new Promise((resolve, reject) => {
+      const params = {
+        Bucket,
+        Delete: {
+          Objects: batch.map(key => ({ Key: String(key) }))
+        }
       }
-    }, (err, data) => { err ? reject(err) : resolve(keys) })
+      s3.deleteObjects(params, (err, data) => {
-  })
+        if (err) return reject(err)
+        const deleted = data.Deleted?.map(({ Key }) => parseInt(Key)) || []
+        resolve(deleted)
+      })
+    }).then((deleteConfirmed) => {
+      deleted.push(...deleteConfirmed)
+    }).catch(console.error)
+  }
+  return deleted
 }

worker/deleteUnusedImages.js

@@ -17,7 +17,12 @@ export async function deleteUnusedImages ({ models }) {
     AND created_at < date_trunc('hour', now() - CASE WHEN "userId" = ${ANON_USER_ID} THEN interval '1 hour' ELSE interval '24 hours' END)`
 
   const s3Keys = unpaidImages.map(({ id }) => id)
+  if (s3Keys.length === 0) {
+    console.log('no images to delete.')
+    return
+  }
   console.log('deleting images:', s3Keys)
-  await deleteObjects(s3Keys)
+  const deleted = await deleteObjects(s3Keys)
-  await models.upload.deleteMany({ where: { id: { in: s3Keys } } })
+  console.log('deleted images:', deleted)
+  await models.upload.deleteMany({ where: { id: { in: deleted } } })
 }