stacker.news/worker/imgproxy.js

194 lines
5.9 KiB
JavaScript
Raw Normal View History

2023-10-01 23:03:52 +00:00
import { createHmac } from 'node:crypto'
import { extractUrls } from '../lib/md.js'
import { isJob } from '../lib/item.js'
2023-10-01 23:03:52 +00:00
const imgProxyEnabled = process.env.NODE_ENV === 'production' ||
(process.env.NEXT_PUBLIC_IMGPROXY_URL && process.env.IMGPROXY_SALT && process.env.IMGPROXY_KEY)
if (!imgProxyEnabled) {
console.warn('IMGPROXY_* env vars not set, imgproxy calls are no-ops now')
}
const IMGPROXY_URL = process.env.NEXT_PUBLIC_IMGPROXY_URL
const IMGPROXY_SALT = process.env.IMGPROXY_SALT
const IMGPROXY_KEY = process.env.IMGPROXY_KEY
const cache = new Map()
// based on heuristics. see https://stacker.news/items/266838
const imageUrlMatchers = [
u => u.host === 'i.postimg.cc',
u => u.host === 'pbs.twimg.com',
u => u.host === 'i.ibb.co',
u => u.host === 'nostr.build' || u.host === 'cdn.nostr.build',
u => u.host === 'www.zapread.com' && u.pathname.startsWith('/i'),
u => u.host === 'i.imgflip.com',
u => u.host === 'i.redd.it',
u => u.host === 'media.tenor.com',
u => u.host === 'i.imgur.com'
]
const exclude = [
u => u.protocol === 'mailto:',
u => u.host.endsWith('.onion') || u.host.endsWith('.b32.ip') || u.host.endsWith('.loki'),
u => ['twitter.com', 'x.com', 'nitter.it', 'nitter.at'].some(h => h === u.host),
u => u.host === 'stacker.news',
u => u.host === 'news.ycombinator.com',
u => u.host === 'www.youtube.com' || u.host === 'youtu.be',
u => u.host === 'github.com'
]
function matchUrl (matchers, url) {
try {
return matchers.some(matcher => matcher(new URL(url)))
} catch (err) {
console.log(url, err)
return false
}
}
function decodeOriginalUrl (imgproxyUrl) {
const parts = imgproxyUrl.split('/')
const b64Url = parts[parts.length - 1]
const originalUrl = Buffer.from(b64Url, 'base64url').toString('utf-8')
return originalUrl
}
export function imgproxy ({ models }) {
return async function ({ data: { id, forceFetch = false } }) {
if (!imgProxyEnabled) return
console.log('running imgproxy job', id)
const item = await models.item.findUnique({ where: { id } })
let imgproxyUrls = {}
try {
if (item.text) {
Image uploads (#576) * Add icon to add images * Open file explorer to select image * Upload images to S3 on selection * Show uploaded images below text input * Link and remove image * Fetch unsubmitted images from database * Mark S3 images as submitted in imgproxy job * Add margin-top * Mark images as submitted on client after successful mutation * Also delete objects in S3 * Allow items to have multiple uploads linked * Overwrite old avatar * Add fees for presigned URLs * Use Github style upload * removed upfront fees * removed images provider since we no longer need to keep track of unsubmitted images on the client * removed User.images resolver * removed deleteImage mutation * use Github style upload where it shows ![Uploading <filename>...]() first and then replaces that with ![<filename>](<url>) after successful upload * Add Upload.paid boolean column One item can have multiple images linked to it, but an image can also be used in multiple items (many-to-many relation). Since we don't really care to which item an image is linked and vice versa, we just use a boolean column to mark if an image was already paid for. This makes fee calculation easier since no JOINs are required. * Add image fees during item creation/update * we calculate image fees during item creation and update now * function imageFees returns queries which deduct fees from user and mark images as paid + fees * queries need to be run inside same transaction as item creation/update * Allow anons to get presigned URLs * Add comments regarding avatar upload * Use megabytes in error message * Remove unnecessary avatar check during image fees calculation * Show image fees in frontend * Also update image fees on blur This makes sure that the images fees reflect the current state. For example, if an image was removed. We could also add debounced requests. * Show amount of unpaid images in receipt * Fix fees in sats deducted from msats * Fix algebraic order of fees Spam fees must come immediately after the base fee since it multiplies the base fee. * Fix image fees in edit receipt * Fix stale fees shown If we pay for an image and then want to edit the comment, the cache might return stale date; suggesting we didn't pay for the existing image yet. * Add 0 base fee in edit receipt * Remove 's' from 'image fees' in receipts * Remove unnecessary async * Remove 'Uploading <name>...' from text input on error * Support upload of multiple files at once * Add schedule to delete unused images * Fix image fee display in receipts * Use Drag and Drop API for image upload * Remove dragOver style on drop * Increase max upload size to 10MB to allow HQ camera pictures * Fix free upload quota * Fix stale image fees served * Fix bad image fee return statements * Fix multiplication with feesPerImage * Fix NULL returned for size24h, sizeNow * Remove unnecessary text field in query * refactor: Unify <ImageUpload> and <Upload> component * Add avatar cache busting using random query param * Calculate image fee info in postgres function * we now calculate image fee info in a postgres function which is much cleaner * we use this function inside `create_item` and `update_item`: image fees are now deducted in the same transaction as creating/updating the item! * reversed changes in `serializeInvoiceable` * Fix line break in receipt * Update upload limits * Add comment about `e.target.value = null` * Use debounce instead of onBlur to update image fees info * Fix invoice amount * Refactor avatar upload control flow * Update image fees in onChange * Fix rescheduling of other jobs * also update schedule from every minute to every hour * Add image fees in calling context * keep item ids on uploads * Fix incompatible onSubmit signature * Revert "keep item ids on uploads" This reverts commit 4688962abcd54fdc5850109372a7ad054cf9b2e4. * many2many item uploads * pretty subdomain for images * handle upload conditions for profile images and job logos --------- Co-authored-by: ekzyis <ek@ekzyis.com> Co-authored-by: ekzyis <ek@stacker.news>
2023-11-06 20:53:33 +00:00
imgproxyUrls = await createImgproxyUrls(id, item.text, { models, forceFetch })
2023-10-01 23:03:52 +00:00
}
if (item.url && !isJob(item)) {
Image uploads (#576) * Add icon to add images * Open file explorer to select image * Upload images to S3 on selection * Show uploaded images below text input * Link and remove image * Fetch unsubmitted images from database * Mark S3 images as submitted in imgproxy job * Add margin-top * Mark images as submitted on client after successful mutation * Also delete objects in S3 * Allow items to have multiple uploads linked * Overwrite old avatar * Add fees for presigned URLs * Use Github style upload * removed upfront fees * removed images provider since we no longer need to keep track of unsubmitted images on the client * removed User.images resolver * removed deleteImage mutation * use Github style upload where it shows ![Uploading <filename>...]() first and then replaces that with ![<filename>](<url>) after successful upload * Add Upload.paid boolean column One item can have multiple images linked to it, but an image can also be used in multiple items (many-to-many relation). Since we don't really care to which item an image is linked and vice versa, we just use a boolean column to mark if an image was already paid for. This makes fee calculation easier since no JOINs are required. * Add image fees during item creation/update * we calculate image fees during item creation and update now * function imageFees returns queries which deduct fees from user and mark images as paid + fees * queries need to be run inside same transaction as item creation/update * Allow anons to get presigned URLs * Add comments regarding avatar upload * Use megabytes in error message * Remove unnecessary avatar check during image fees calculation * Show image fees in frontend * Also update image fees on blur This makes sure that the images fees reflect the current state. For example, if an image was removed. We could also add debounced requests. * Show amount of unpaid images in receipt * Fix fees in sats deducted from msats * Fix algebraic order of fees Spam fees must come immediately after the base fee since it multiplies the base fee. * Fix image fees in edit receipt * Fix stale fees shown If we pay for an image and then want to edit the comment, the cache might return stale date; suggesting we didn't pay for the existing image yet. * Add 0 base fee in edit receipt * Remove 's' from 'image fees' in receipts * Remove unnecessary async * Remove 'Uploading <name>...' from text input on error * Support upload of multiple files at once * Add schedule to delete unused images * Fix image fee display in receipts * Use Drag and Drop API for image upload * Remove dragOver style on drop * Increase max upload size to 10MB to allow HQ camera pictures * Fix free upload quota * Fix stale image fees served * Fix bad image fee return statements * Fix multiplication with feesPerImage * Fix NULL returned for size24h, sizeNow * Remove unnecessary text field in query * refactor: Unify <ImageUpload> and <Upload> component * Add avatar cache busting using random query param * Calculate image fee info in postgres function * we now calculate image fee info in a postgres function which is much cleaner * we use this function inside `create_item` and `update_item`: image fees are now deducted in the same transaction as creating/updating the item! * reversed changes in `serializeInvoiceable` * Fix line break in receipt * Update upload limits * Add comment about `e.target.value = null` * Use debounce instead of onBlur to update image fees info * Fix invoice amount * Refactor avatar upload control flow * Update image fees in onChange * Fix rescheduling of other jobs * also update schedule from every minute to every hour * Add image fees in calling context * keep item ids on uploads * Fix incompatible onSubmit signature * Revert "keep item ids on uploads" This reverts commit 4688962abcd54fdc5850109372a7ad054cf9b2e4. * many2many item uploads * pretty subdomain for images * handle upload conditions for profile images and job logos --------- Co-authored-by: ekzyis <ek@ekzyis.com> Co-authored-by: ekzyis <ek@stacker.news>
2023-11-06 20:53:33 +00:00
imgproxyUrls = { ...imgproxyUrls, ...(await createImgproxyUrls(id, item.url, { models, forceFetch })) }
2023-10-01 23:03:52 +00:00
}
} catch (err) {
console.log('[imgproxy] error:', err)
// rethrow for retry
throw err
}
console.log('[imgproxy] updating item', id, 'with urls', imgproxyUrls)
await models.item.update({ where: { id }, data: { imgproxyUrls } })
}
}
Image uploads (#576) * Add icon to add images * Open file explorer to select image * Upload images to S3 on selection * Show uploaded images below text input * Link and remove image * Fetch unsubmitted images from database * Mark S3 images as submitted in imgproxy job * Add margin-top * Mark images as submitted on client after successful mutation * Also delete objects in S3 * Allow items to have multiple uploads linked * Overwrite old avatar * Add fees for presigned URLs * Use Github style upload * removed upfront fees * removed images provider since we no longer need to keep track of unsubmitted images on the client * removed User.images resolver * removed deleteImage mutation * use Github style upload where it shows ![Uploading <filename>...]() first and then replaces that with ![<filename>](<url>) after successful upload * Add Upload.paid boolean column One item can have multiple images linked to it, but an image can also be used in multiple items (many-to-many relation). Since we don't really care to which item an image is linked and vice versa, we just use a boolean column to mark if an image was already paid for. This makes fee calculation easier since no JOINs are required. * Add image fees during item creation/update * we calculate image fees during item creation and update now * function imageFees returns queries which deduct fees from user and mark images as paid + fees * queries need to be run inside same transaction as item creation/update * Allow anons to get presigned URLs * Add comments regarding avatar upload * Use megabytes in error message * Remove unnecessary avatar check during image fees calculation * Show image fees in frontend * Also update image fees on blur This makes sure that the images fees reflect the current state. For example, if an image was removed. We could also add debounced requests. * Show amount of unpaid images in receipt * Fix fees in sats deducted from msats * Fix algebraic order of fees Spam fees must come immediately after the base fee since it multiplies the base fee. * Fix image fees in edit receipt * Fix stale fees shown If we pay for an image and then want to edit the comment, the cache might return stale date; suggesting we didn't pay for the existing image yet. * Add 0 base fee in edit receipt * Remove 's' from 'image fees' in receipts * Remove unnecessary async * Remove 'Uploading <name>...' from text input on error * Support upload of multiple files at once * Add schedule to delete unused images * Fix image fee display in receipts * Use Drag and Drop API for image upload * Remove dragOver style on drop * Increase max upload size to 10MB to allow HQ camera pictures * Fix free upload quota * Fix stale image fees served * Fix bad image fee return statements * Fix multiplication with feesPerImage * Fix NULL returned for size24h, sizeNow * Remove unnecessary text field in query * refactor: Unify <ImageUpload> and <Upload> component * Add avatar cache busting using random query param * Calculate image fee info in postgres function * we now calculate image fee info in a postgres function which is much cleaner * we use this function inside `create_item` and `update_item`: image fees are now deducted in the same transaction as creating/updating the item! * reversed changes in `serializeInvoiceable` * Fix line break in receipt * Update upload limits * Add comment about `e.target.value = null` * Use debounce instead of onBlur to update image fees info * Fix invoice amount * Refactor avatar upload control flow * Update image fees in onChange * Fix rescheduling of other jobs * also update schedule from every minute to every hour * Add image fees in calling context * keep item ids on uploads * Fix incompatible onSubmit signature * Revert "keep item ids on uploads" This reverts commit 4688962abcd54fdc5850109372a7ad054cf9b2e4. * many2many item uploads * pretty subdomain for images * handle upload conditions for profile images and job logos --------- Co-authored-by: ekzyis <ek@ekzyis.com> Co-authored-by: ekzyis <ek@stacker.news>
2023-11-06 20:53:33 +00:00
export const createImgproxyUrls = async (id, text, { models, forceFetch }) => {
2023-10-01 23:03:52 +00:00
const urls = extractUrls(text)
console.log('[imgproxy] id:', id, '-- extracted urls:', urls)
// resolutions that we target:
// - nHD: 640x 360
// - qHD: 960x 540
// - HD: 1280x 720
// - HD+: 1600x 900
// - FHD: 1920x1080
// - QHD: 2560x1440
// reference:
// - https://en.wikipedia.org/wiki/Graphics_display_resolution#High-definition_(HD_and_derivatives)
// - https://www.browserstack.com/guide/ideal-screen-sizes-for-responsive-design
const resolutions = ['640x360', '960x540', '1280x720', '1600x900', '1920x1080', '2560x1440']
const imgproxyUrls = {}
for (let url of urls) {
if (!url) continue
console.log('[imgproxy] id:', id, '-- processing url:', url)
if (url.startsWith(IMGPROXY_URL)) {
console.log('[imgproxy] id:', id, '-- proxy url, decoding original url:', url)
// backwards compatibility: we used to replace image urls with imgproxy urls
url = decodeOriginalUrl(url)
console.log('[imgproxy] id:', id, '-- original url:', url)
}
if (!(await isImageURL(url, { forceFetch }))) {
console.log('[imgproxy] id:', id, '-- not image url:', url)
continue
}
imgproxyUrls[url] = {}
for (const res of resolutions) {
const [w, h] = res.split('x')
const processingOptions = `/rs:fit:${w}:${h}`
2023-10-01 23:03:52 +00:00
imgproxyUrls[url][`${w}w`] = createImgproxyUrl(url, processingOptions)
}
}
return imgproxyUrls
}
const createImgproxyUrl = (url, processingOptions) => {
const b64Url = Buffer.from(url, 'utf-8').toString('base64url')
const target = `${processingOptions}/${b64Url}`
const signature = sign(target)
return `${IMGPROXY_URL}${signature}${target}`
}
async function fetchWithTimeout (resource, { timeout = 1000, ...options } = {}) {
const controller = new AbortController()
const id = setTimeout(() => controller.abort(), timeout)
const response = await fetch(resource, {
...options,
signal: controller.signal
})
clearTimeout(id)
return response
}
const isImageURL = async (url, { forceFetch }) => {
if (cache.has(url)) return cache.get(url)
if (!forceFetch && matchUrl(imageUrlMatchers, url)) {
return true
}
if (!forceFetch && matchUrl(exclude, url)) {
return false
}
let isImage
// first run HEAD with small timeout
try {
// https://stackoverflow.com/a/68118683
const res = await fetchWithTimeout(url, { timeout: 1000, method: 'HEAD' })
const buf = await res.blob()
isImage = buf.type.startsWith('image/')
} catch (err) {
console.log(url, err)
}
// For HEAD requests, positives are most likely true positives.
// However, negatives may be false negatives
if (isImage) {
cache.set(url, true)
return true
}
// if not known yet, run GET request with longer timeout
try {
const res = await fetchWithTimeout(url, { timeout: 10000 })
const buf = await res.blob()
isImage = buf.type.startsWith('image/')
} catch (err) {
console.log(url, err)
}
cache.set(url, isImage)
return isImage
}
const hexDecode = (hex) => Buffer.from(hex, 'hex')
const sign = (target) => {
// https://github.com/imgproxy/imgproxy/blob/master/examples/signature.js
const hmac = createHmac('sha256', hexDecode(IMGPROXY_KEY))
hmac.update(hexDecode(IMGPROXY_SALT))
hmac.update(target)
return hmac.digest('base64url')
}