From 3d94ef32acded4164bd6f4cfe94dff0d18f09033 Mon Sep 17 00:00:00 2001 From: keyan Date: Wed, 12 Jan 2022 17:13:09 -0600 Subject: [PATCH] improve screen capture: 2x resolution, caching, exit on exception --- package-lock.json | 202 +++++++++++++++++++++++++++++++ package.json | 4 +- pages/api/capture/[[...path]].js | 73 ++++++++--- spawn/capture.js | 13 +- 4 files changed, 270 insertions(+), 22 deletions(-) diff --git a/package-lock.json b/package-lock.json index b82029b1..b69993fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -12,6 +12,7 @@ "@prisma/client": "^2.25.0", "apollo-server-micro": "^2.21.2", "async-retry": "^1.3.1", + "aws-sdk": "^2.1056.0", "babel-plugin-inline-react-svg": "^2.0.1", "bech32": "^2.0.0", "bolt11": "^1.3.4", @@ -46,6 +47,7 @@ "secp256k1": "^4.0.2", "swr": "^0.5.4", "use-dark-mode": "^2.3.1", + "uuid": "^8.3.2", "webln": "^0.2.2", "yup": "^0.32.9" }, @@ -1819,6 +1821,107 @@ "url": "https://github.com/sponsors/ljharb" } }, + "node_modules/aws-sdk": { + "version": "2.1056.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1056.0.tgz", + "integrity": "sha512-ocpc4Sy9Lclt+v5bIRuvFq0WwJoLIU26ikdCQn+ke9lIDPC9+hGZbkFK7TiqTu3noEekgIubGHFGEkd/5V0HhQ==", + "dependencies": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "engines": { + "node": ">= 10.0.0" + } + }, + "node_modules/aws-sdk/node_modules/buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "dependencies": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "node_modules/aws-sdk/node_modules/events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/aws-sdk/node_modules/ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "node_modules/aws-sdk/node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/aws-sdk/node_modules/punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "node_modules/aws-sdk/node_modules/querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=", + "deprecated": "The querystring API is considered Legacy. new code should use the URLSearchParams API instead.", + "engines": { + "node": ">=0.4.x" + } + }, + "node_modules/aws-sdk/node_modules/sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "node_modules/aws-sdk/node_modules/url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "dependencies": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "node_modules/aws-sdk/node_modules/uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==", + "deprecated": "Please upgrade to version 7 or higher. Older versions may use Math.random() in certain circumstances, which is known to be problematic. See https://v8.dev/blog/math-random for details.", + "bin": { + "uuid": "bin/uuid" + } + }, + "node_modules/aws-sdk/node_modules/xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "dependencies": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "node_modules/aws-sdk/node_modules/xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=", + "engines": { + "node": ">=4.0" + } + }, "node_modules/babel-eslint": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", @@ -5594,6 +5697,14 @@ "url": "https://github.com/chalk/supports-color?sponsor=1" } }, + "node_modules/jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=", + "engines": { + "node": ">= 0.6.0" + } + }, "node_modules/jose": { "version": "1.28.1", "resolved": "https://registry.npmjs.org/jose/-/jose-1.28.1.tgz", @@ -13226,6 +13337,92 @@ "resolved": "https://registry.npmjs.org/available-typed-arrays/-/available-typed-arrays-1.0.5.tgz", "integrity": "sha512-DMD0KiN46eipeziST1LPP/STfDU0sufISXmjSgvVsoU2tqxctQeASejWcfNtxYKqETM1UxQ8sp2OrSBWpHY6sw==" }, + "aws-sdk": { + "version": "2.1056.0", + "resolved": "https://registry.npmjs.org/aws-sdk/-/aws-sdk-2.1056.0.tgz", + "integrity": "sha512-ocpc4Sy9Lclt+v5bIRuvFq0WwJoLIU26ikdCQn+ke9lIDPC9+hGZbkFK7TiqTu3noEekgIubGHFGEkd/5V0HhQ==", + "requires": { + "buffer": "4.9.2", + "events": "1.1.1", + "ieee754": "1.1.13", + "jmespath": "0.15.0", + "querystring": "0.2.0", + "sax": "1.2.1", + "url": "0.10.3", + "uuid": "3.3.2", + "xml2js": "0.4.19" + }, + "dependencies": { + "buffer": { + "version": "4.9.2", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-4.9.2.tgz", + "integrity": "sha512-xq+q3SRMOxGivLhBNaUdC64hDTQwejJ+H0T/NB1XMtTVEwNTrfFF3gAxiyW0Bu/xWEGhjVKgUcMhCrUy2+uCWg==", + "requires": { + "base64-js": "^1.0.2", + "ieee754": "^1.1.4", + "isarray": "^1.0.0" + } + }, + "events": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/events/-/events-1.1.1.tgz", + "integrity": "sha1-nr23Y1rQmccNzEwqH1AEKI6L2SQ=" + }, + "ieee754": { + "version": "1.1.13", + "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", + "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" + }, + "isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "punycode": { + "version": "1.3.2", + "resolved": "https://registry.npmjs.org/punycode/-/punycode-1.3.2.tgz", + "integrity": "sha1-llOgNvt8HuQjQvIyXM7v6jkmxI0=" + }, + "querystring": { + "version": "0.2.0", + "resolved": "https://registry.npmjs.org/querystring/-/querystring-0.2.0.tgz", + "integrity": "sha1-sgmEkgO7Jd+CDadW50cAWHhSFiA=" + }, + "sax": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.2.1.tgz", + "integrity": "sha1-e45lYZCyKOgaZq6nSEgNgozS03o=" + }, + "url": { + "version": "0.10.3", + "resolved": "https://registry.npmjs.org/url/-/url-0.10.3.tgz", + "integrity": "sha1-Ah5NnHcF8hu/N9A861h2dAJ3TGQ=", + "requires": { + "punycode": "1.3.2", + "querystring": "0.2.0" + } + }, + "uuid": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/uuid/-/uuid-3.3.2.tgz", + "integrity": "sha512-yXJmeNaw3DnnKAOKJE51sL/ZaYfWJRl1pK9dr19YFCu0ObS231AB1/LbqTKRAQ5kw8A90rA6fr4riOUpTZvQZA==" + }, + "xml2js": { + "version": "0.4.19", + "resolved": "https://registry.npmjs.org/xml2js/-/xml2js-0.4.19.tgz", + "integrity": "sha512-esZnJZJOiJR9wWKMyuvSE1y6Dq5LCuJanqhxslH2bxM6duahNZ+HMpCLhBQGZkbX6xRf8x1Y2eJlgt2q3qo49Q==", + "requires": { + "sax": ">=0.6.0", + "xmlbuilder": "~9.0.1" + } + }, + "xmlbuilder": { + "version": "9.0.7", + "resolved": "https://registry.npmjs.org/xmlbuilder/-/xmlbuilder-9.0.7.tgz", + "integrity": "sha1-Ey7mPS7FVlxVfiD0wi35rKaGsQ0=" + } + } + }, "babel-eslint": { "version": "10.1.0", "resolved": "https://registry.npmjs.org/babel-eslint/-/babel-eslint-10.1.0.tgz", @@ -16064,6 +16261,11 @@ } } }, + "jmespath": { + "version": "0.15.0", + "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.15.0.tgz", + "integrity": "sha1-o/Iiqarp+Wb10nx5ZRDigJF2Qhc=" + }, "jose": { "version": "1.28.1", "resolved": "https://registry.npmjs.org/jose/-/jose-1.28.1.tgz", diff --git a/package.json b/package.json index 9b0f23f0..feb2fa13 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,7 @@ "@prisma/client": "^2.25.0", "apollo-server-micro": "^2.21.2", "async-retry": "^1.3.1", + "aws-sdk": "^2.1056.0", "babel-plugin-inline-react-svg": "^2.0.1", "bech32": "^2.0.0", "bolt11": "^1.3.4", @@ -47,6 +48,7 @@ "secp256k1": "^4.0.2", "swr": "^0.5.4", "use-dark-mode": "^2.3.1", + "uuid": "^8.3.2", "webln": "^0.2.2", "yup": "^0.32.9" }, @@ -71,4 +73,4 @@ "eslint-plugin-compat": "^3.9.0", "standard": "^16.0.3" } -} \ No newline at end of file +} diff --git a/pages/api/capture/[[...path]].js b/pages/api/capture/[[...path]].js index ef6df88e..a1aa01d2 100644 --- a/pages/api/capture/[[...path]].js +++ b/pages/api/capture/[[...path]].js @@ -1,8 +1,19 @@ import path from 'path' +import AWS from 'aws-sdk' +import {PassThrough} from 'stream' const { spawn } = require('child_process') +const bucketName = 'sn-capture' +const bucketRegion = 'us-east-1' +const contentType = 'image/png' +const bucketUrl = 'https://sn-capture.s3.amazonaws.com/' +const s3PathPrefix = process.env.NODE_ENV === 'development' ? 'dev/' : '' var capturing = false +AWS.config.update({ + region: bucketRegion +}) + export default async function handler (req, res) { if (capturing) { return res.writeHead(503, { @@ -11,25 +22,53 @@ export default async function handler (req, res) { } return new Promise(resolve => { - capturing = true - const url = process.env.SELF_URL + '/' + path.join(...(req.query.path || [])) - res.setHeader('Content-Type', 'image/png') + const joinedPath = path.join(...(req.query.path || [])) + const s3Path = s3PathPrefix + (joinedPath === '.' ? '_' : joinedPath) + const url = process.env.SELF_URL + '/' + joinedPath + const aws = new AWS.S3({apiVersion: '2006-03-01'}) - const capture = spawn( - 'node', ['./spawn/capture.js', url], {maxBuffer: 1024*1024*5}) - - capture.on('close', code => { - if (code !== 0) { - res.status(500).end() - } else { - res.status(200).end() - } - capture.removeAllListeners() - capturing = false + // check to see if we have a recent version of the object + aws.headObject({ + Bucket: bucketName, + Key: s3Path, + IfModifiedSince : new Date(new Date().getTime() - 15*60000) + }).promise().then(() => { + // this path is cached so return it + res.writeHead(302, { Location: bucketUrl + s3Path }).end() resolve() + }).catch(() => { + // we don't have it cached, so capture it and cache it + capturing = true + const pass = new PassThrough() + aws.upload({ + Bucket: bucketName, + Key: s3Path, + ACL: 'public-read', + Body: pass, + ContentType: contentType + }).promise().catch(console.log) + + res.setHeader('Content-Type', contentType) + const capture = spawn( + 'node', ['./spawn/capture.js', url], {maxBuffer: 1024*1024*5}) + + capture.on('close', code => { + if (code !== 0) { + res.status(500).end() + } else { + res.status(200).end() + } + pass.end() + capture.removeAllListeners() + capturing = false + resolve() + }) + capture.on('error', err => console.log('error', err)) + capture.stderr.on('data', data => console.log('error stderr', data.toString())) + capture.stdout.on('data', data => { + res.write(data) + pass.write(data) + }) }) - capture.on('error', err => console.log('error', err)) - capture.stderr.on('data', data => console.log('error stderr', data.toString())) - capture.stdout.on('data', data => res.write(data)) }) } \ No newline at end of file diff --git a/spawn/capture.js b/spawn/capture.js index 90fc7661..71631605 100755 --- a/spawn/capture.js +++ b/spawn/capture.js @@ -3,10 +3,15 @@ const Pageres = require('pageres') async function captureUrl () { - const streams = await new Pageres({ crop: true, timeout: 5 }) - .src(process.argv[2], ['600x314']) - .run() - process.stdout.write(streams[0], () => process.exit(0)) + try { + const streams = await new Pageres({ crop: true, scale: 2, timeout: 10, launchOptions: { args: ['--single-process'] } }) + .src(process.argv[2], ['600x315']) + .run() + process.stdout.write(streams[0], () => process.exit(0)) + } catch (e) { + console.log(e) + process.exit(1) + } } captureUrl()