From 72ecc7b26692adb1173670ee54aa3ebcc7904c87 Mon Sep 17 00:00:00 2001 From: keyan Date: Tue, 12 Mar 2024 18:05:10 -0500 Subject: [PATCH] s3 and image proxy with broken name resolution --- .env.sample | 47 ++++++++++++++++++------------- .gitignore | 8 ++---- api/s3/index.js | 7 ++++- components/image.js | 4 +-- components/item-job.js | 3 +- components/job-form.js | 4 +-- components/user-header.js | 4 +-- components/user-list.js | 3 +- docker-compose.yml | 59 ++++++++++++++++++++++++--------------- docker/s3/cors.json | 19 +++++++++++++ docker/s3/init-s3.sh | 5 ++++ lib/constants.js | 4 ++- middleware.js | 9 ++++-- sndev | 31 ++++++++++++++++---- 14 files changed, 140 insertions(+), 67 deletions(-) create mode 100644 docker/s3/cors.json create mode 100755 docker/s3/init-s3.sh diff --git a/.env.sample b/.env.sample index cf81c9c0..d6958a53 100644 --- a/.env.sample +++ b/.env.sample @@ -16,10 +16,10 @@ LOGIN_EMAIL_SERVER= LOGIN_EMAIL_FROM= LIST_MONK_AUTH= -##################################################################### -# OTHER / OPTIONAL # -# configuration for push notifications, slack and imgproxy are here # -##################################################################### +######################################################## +# OTHER / OPTIONAL # +# configuration for push notifications, slack are here # +######################################################## # VAPID for Web Push VAPID_MAILTO= @@ -30,18 +30,6 @@ VAPID_PRIVKEY= SLACK_BOT_TOKEN= SLACK_CHANNEL_ID= -# imgproxy -NEXT_PUBLIC_IMGPROXY_URL=localhost:3001 -IMGPROXY_KEY= -IMGPROXY_SALT= - -# search -OPENSEARCH_URL=http://opensearch:9200 -OPENSEARCH_USERNAME= -OPENSEARCH_PASSWORD= -OPENSEARCH_INDEX=item -OPENSEARCH_MODEL_ID= - # lnurl ... you'll need a tunnel to localhost:3000 for these LNAUTH_URL= LNWITH_URL= @@ -69,7 +57,11 @@ LND_SOCKET=sn_lnd:10009 # openssl rand -hex 32 NOSTR_PRIVATE_KEY=5f30b7e7714360f51f2be2e30c1d93b7fdf67366e730658e85777dfcc4e4245f -# imgproxy options +# imgproxy +NEXT_PUBLIC_IMGPROXY_URL=http://imgproxy:3001 +IMGPROXY_KEY=9c273e803fd5d444bf8883f8c3000de57bee7995222370cab7f2d218dd9a4bbff6ca11cbf902e61eeef4358616f231da51e183aee6841e3a797a5c9a9530ba67 +IMGPROXY_SALT=47b802be2c9250a66b998f411fc63912ab0bc1c6b47d99b8d37c61019d1312a984b98745eac83db9791b01bb8c93ecbc9b2ef9f2981d66061c7d0a4528ff6465 + IMGPROXY_ENABLE_WEBP_DETECTION=1 IMGPROXY_ENABLE_AVIF_DETECTION=1 IMGPROXY_MAX_ANIMATION_FRAMES=2000 @@ -81,6 +73,17 @@ IMGPROXY_DOWNLOAD_TIMEOUT=9 # IMGPROXY_DEVELOPMENT_ERRORS_MODE=1 # IMGPROXY_ENABLE_DEBUG_HEADERS=true +NEXT_PUBLIC_AWS_UPLOAD_BUCKET=uploads +NEXT_PUBLIC_MEDIA_DOMAIN=s3:4566 +NEXT_PUBLIC_MEDIA_URL=http://s3:4566/uploads + +# search +OPENSEARCH_URL=http://opensearch:9200 +OPENSEARCH_USERNAME= +OPENSEARCH_PASSWORD= +OPENSEARCH_INDEX=item +OPENSEARCH_MODEL_ID= + # prisma db url DATABASE_URL="postgresql://sn:password@db:5432/stackernews?schema=public" @@ -117,10 +120,16 @@ LND_ADDR=bcrt1q7q06n5st4vqq3lssn0rtkrn2qqypghv9xg2xnl LND_PUBKEY=02cb2e2d5a6c5b17fa67b1a883e2973c82e328fb9bd08b2b156a9e23820c87a490 # stacker lnd container stuff -STACKER_LND_REST_PORT=9090 +STACKER_LND_REST_PORT=8081 STACKER_LND_GRPC_PORT=10010 # docker exec -u lnd stacker_lnd lncli newaddress p2wkh --unused STACKER_LND_ADDR=bcrt1qfqau4ug9e6rtrvxrgclg58e0r93wshucumm9vu STACKER_LND_PUBKEY=028093ae52e011d45b3e67f2e0f2cb6c3a1d7f88d2920d408f3ac6db3a56dc4b35 -LNCLI_NETWORK=regtest \ No newline at end of file +LNCLI_NETWORK=regtest + +# localstack container stuff +AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE +AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY +PERSISTENCE=1 +SKIP_SSL_CERT_DOWNLOAD=1 diff --git a/.gitignore b/.gitignore index 831f0446..3604baeb 100644 --- a/.gitignore +++ b/.gitignore @@ -28,13 +28,9 @@ yarn-debug.log* yarn-error.log* # local env files -.env envbak -.env.local -.env.development.local -.env.test.local -.env.production.local -.env.sndev +.env* +!.env.sample # vercel .vercel diff --git a/api/s3/index.js b/api/s3/index.js index e2f999e0..ee2a8f36 100644 --- a/api/s3/index.js +++ b/api/s3/index.js @@ -1,4 +1,5 @@ import AWS from 'aws-sdk' +import { MEDIA_URL } from '../../lib/constants' const bucketRegion = 'us-east-1' const Bucket = process.env.NEXT_PUBLIC_AWS_UPLOAD_BUCKET @@ -8,7 +9,11 @@ AWS.config.update({ }) export function createPresignedPost ({ key, type, size }) { - const s3 = new AWS.S3({ apiVersion: '2006-03-01' }) + const s3 = new AWS.S3({ + apiVersion: '2006-03-01', + endpoint: process.env.NODE_ENV === 'development' ? `${MEDIA_URL}/${Bucket}` : undefined, + s3ForcePathStyle: process.env.NODE_ENV === 'development' + }) return new Promise((resolve, reject) => { s3.createPresignedPost({ Bucket, diff --git a/components/image.js b/components/image.js index a0f5e01f..f0ab9a5f 100644 --- a/components/image.js +++ b/components/image.js @@ -4,7 +4,7 @@ import { IMGPROXY_URL_REGEXP } from '../lib/url' import { useShowModal } from './modal' import { useMe } from './me' import { Dropdown } from 'react-bootstrap' -import { UNKNOWN_LINK_REL, UPLOAD_TYPES_ALLOW } from '../lib/constants' +import { UNKNOWN_LINK_REL, UPLOAD_TYPES_ALLOW, MEDIA_URL } from '../lib/constants' import { useToast } from './toast' import gql from 'graphql-tag' import { useMutation } from '@apollo/client' @@ -224,7 +224,7 @@ export const ImageUpload = forwardRef(({ children, className, onSelect, onUpload return } - const url = `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${data.getSignedPOST.fields.key}` + const url = `${MEDIA_URL}/${data.getSignedPOST.fields.key}` // key is upload id in database const id = data.getSignedPOST.fields.key onSuccess?.({ ...variables, id, name: file.name, url, file }) diff --git a/components/item-job.js b/components/item-job.js index ab38ba43..c59190c9 100644 --- a/components/item-job.js +++ b/components/item-job.js @@ -10,6 +10,7 @@ import { timeSince } from '../lib/time' import EmailIcon from '../svgs/mail-open-line.svg' import Share from './share' import Hat from './hat' +import { MEDIA_URL } from '../lib/constants' export default function ItemJob ({ item, toc, rank, children }) { const isEmail = string().email().isValidSync(item.url) @@ -25,7 +26,7 @@ export default function ItemJob ({ item, toc, rank, children }) {
diff --git a/components/job-form.js b/components/job-form.js index 5d2e835d..0dbf2325 100644 --- a/components/job-form.js +++ b/components/job-form.js @@ -15,7 +15,7 @@ import Link from 'next/link' import { usePrice } from './price' import Avatar from './avatar' import { jobSchema } from '../lib/validate' -import { MAX_TITLE_LENGTH } from '../lib/constants' +import { MAX_TITLE_LENGTH, MEDIA_URL } from '../lib/constants' import { useToast } from './toast' import { toastDeleteScheduled } from '../lib/form' import { ItemButtonBar } from './post' @@ -110,7 +110,7 @@ export default function JobForm ({ item, sub }) {
diff --git a/components/user-header.js b/components/user-header.js index 805a7ea5..3a916cfe 100644 --- a/components/user-header.js +++ b/components/user-header.js @@ -28,7 +28,7 @@ import { hexToBech32 } from '../lib/nostr' import NostrIcon from '../svgs/nostr.svg' import GithubIcon from '../svgs/github-fill.svg' import TwitterIcon from '../svgs/twitter-fill.svg' -import { UNKNOWN_LINK_REL } from '../lib/constants' +import { UNKNOWN_LINK_REL, MEDIA_URL } from '../lib/constants' export default function UserHeader ({ user }) { const router = useRouter() @@ -96,7 +96,7 @@ function HeaderPhoto ({ user, isMe }) { } } ) - const src = user.photoId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${user.photoId}` : '/dorian400.jpg' + const src = user.photoId ? `${MEDIA_URL}/${user.photoId}` : '/dorian400.jpg' return (
diff --git a/components/user-list.js b/components/user-list.js index fca1ad89..98f895db 100644 --- a/components/user-list.js +++ b/components/user-list.js @@ -8,6 +8,7 @@ import { useQuery } from '@apollo/client' import MoreFooter from './more-footer' import { useData } from './use-data' import Hat from './hat' +import { MEDIA_URL } from '../lib/constants' // all of this nonsense is to show the stat we are sorting by first const Stacked = ({ user }) => (user.optional.stacked !== null && {abbrNum(user.optional.stacked)} stacked) @@ -48,7 +49,7 @@ function User ({ user, rank, statComps, Embellish }) {
diff --git a/docker-compose.yml b/docker-compose.yml index fb66848d..8f5fcbd2 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,7 +15,7 @@ services: ports: - "5431:5432" env_file: - - .env.sndev + - .env.development volumes: - ./docker/db/seed.sql:/docker-entrypoint-initdb.d/seed.sql - db:/var/lib/postgresql/data @@ -45,18 +45,17 @@ services: sn_lnd: condition: service_healthy restart: true + # s3: + # condition: service_healthy + # restart: true env_file: - - .env.sndev + - .env.development expose: - "3000" ports: - "3000:3000" volumes: - ./:/app - links: - - db - - opensearch - - sn_lnd labels: CONNECT: "localhost:3000" worker: @@ -81,20 +80,15 @@ services: condition: service_healthy restart: true env_file: - - .env.sndev + - .env.development volumes: - ./:/app - links: - - db - - app - - opensearch - - sn_lnd entrypoint: ["/bin/sh", "-c"] command: - npm run worker:dev imgproxy: container_name: imgproxy - image: darthsim/imgproxy:v3.18.1 + image: docker.imgproxy.pro/imgproxy:v3.23.0 healthcheck: test: [ "CMD", "imgproxy", "health" ] interval: 10s @@ -103,13 +97,33 @@ services: start_period: 1m restart: unless-stopped env_file: - - .env.sndev + - .env.development ports: - "3001:8080" - links: - - app labels: - CONNECT: "localhost:3001" + - "CONNECT=localhost:3001" + s3: + container_name: s3 + image: localstack/localstack:s3-latest + # healthcheck: + # test: ["CMD-SHELL", "awslocal", "s3", "ls", "s3://uploads"] + # interval: 10s + # timeout: 10s + # retries: 10 + # start_period: 1m + restart: unless-stopped + env_file: + - .env.development + environment: + - DEBUG=1 + ports: + - "4566:4566" + volumes: + - 's3:/var/lib/localstack' + - './docker/s3/init-s3.sh:/etc/localstack/init/ready.d/init-s3.sh' + - './docker/s3/cors.json:/etc/localstack/init/ready.d/cors.json' + labels: + - "CONNECT=localhost:4566" opensearch: image: opensearchproject/opensearch:2.12.0 container_name: opensearch @@ -121,7 +135,7 @@ services: start_period: 1m restart: unless-stopped env_file: - - .env.sndev + - .env.development environment: - OPENSEARCH_INITIAL_ADMIN_PASSWORD=mVchg1T5oA9wudUh ports: @@ -153,15 +167,13 @@ services: condition: service_healthy restart: true env_file: - - .env.sndev + - .env.development environment: - opensearch.ssl.verificationMode=none - OPENSEARCH_HOSTS=http://opensearch:9200 - server.ssl.enabled=false ports: - 5601:5601 - expose: - - "5601" links: - opensearch labels: @@ -238,7 +250,7 @@ services: condition: service_healthy restart: true env_file: - - .env.sndev + - .env.development command: - 'lnd' - '--noseedbackup' @@ -300,7 +312,7 @@ services: condition: service_healthy restart: true env_file: - - .env.sndev + - .env.development command: - 'lnd' - '--noseedbackup' @@ -361,3 +373,4 @@ volumes: bitcoin: sn_lnd: stacker_lnd: + s3: diff --git a/docker/s3/cors.json b/docker/s3/cors.json new file mode 100644 index 00000000..f02ddd3a --- /dev/null +++ b/docker/s3/cors.json @@ -0,0 +1,19 @@ +{ + "CORSRules": [ + { + "AllowedHeaders": [ + "*" + ], + "AllowedMethods": [ + "PUT", + "POST", + "GET", + "HEAD" + ], + "AllowedOrigins": [ + "http://localhost:3000" + ], + "ExposeHeaders": [] + } + ] +} \ No newline at end of file diff --git a/docker/s3/init-s3.sh b/docker/s3/init-s3.sh new file mode 100755 index 00000000..9e6ed148 --- /dev/null +++ b/docker/s3/init-s3.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +pip3 install --upgrade virtualenv awscli awscli-local requests +awslocal s3 mb s3://uploads +awslocal s3api put-bucket-cors --bucket uploads --cors-configuration file:///etc/localstack/init/ready.d/cors.json \ No newline at end of file diff --git a/lib/constants.js b/lib/constants.js index a629b472..5962447a 100644 --- a/lib/constants.js +++ b/lib/constants.js @@ -10,7 +10,9 @@ export const BOOST_MIN = BOOST_MULT * 5 export const UPLOAD_SIZE_MAX = 25 * 1024 * 1024 export const UPLOAD_SIZE_MAX_AVATAR = 5 * 1024 * 1024 export const IMAGE_PIXELS_MAX = 35000000 -export const AWS_S3_URL_REGEXP = new RegExp(`https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/([0-9]+)`, 'g') +// backwards compatibile with old media domain env var +export const MEDIA_URL = process.env.NEXT_PUBLIC_MEDIA_URL || `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}` +export const AWS_S3_URL_REGEXP = new RegExp(`${MEDIA_URL}/([0-9]+)`, 'g') export const UPLOAD_TYPES_ALLOW = [ 'image/gif', 'image/heic', diff --git a/middleware.js b/middleware.js index 858b80e4..1433f80b 100644 --- a/middleware.js +++ b/middleware.js @@ -20,13 +20,16 @@ export function middleware (request) { } const nonce = Buffer.from(crypto.randomUUID()).toString('base64') + // we want to load media from other localhost ports during development + const devSrc = process.env.NODE_ENV === 'development' ? 'localhost:* ' : '' + const cspHeader = [ // if something is not explicitly allowed, we don't allow it. "default-src 'none'", "font-src 'self' a.stacker.news", // we want to load images from everywhere but we can limit to HTTPS at least - "img-src 'self' a.stacker.news m.stacker.news https: data: blob:", - "media-src 'self' a.stacker.news m.stacker.news", + `img-src 'self' ${devSrc}a.stacker.news m.stacker.news https: data: blob:`, + `media-src 'self' ${devSrc}a.stacker.news m.stacker.news`, // Using nonces and strict-dynamic deploys a strict CSP. // see https://cheatsheetseries.owasp.org/cheatsheets/Content_Security_Policy_Cheat_Sheet.html#strict-policy. // Old browsers will ignore nonce and strict-dynamic and fallback to host-based matching and unsafe-inline @@ -39,7 +42,7 @@ export function middleware (request) { "style-src 'self' a.stacker.news 'unsafe-inline'", "manifest-src 'self'", 'frame-src www.youtube.com platform.twitter.com', - "connect-src 'self' https: wss:", + `connect-src 'self' ${devSrc}https: wss:`, // disable dangerous plugins like Flash "object-src 'none'", // blocks injection of tags diff --git a/sndev b/sndev index 192afea6..53bee80f 100755 --- a/sndev +++ b/sndev @@ -9,7 +9,7 @@ docker__compose() { exit 0 fi - CURRENT_UID=$(id -u) CURRENT_GID=$(id -g) command docker compose --env-file .env.sndev "$@" + CURRENT_UID=$(id -u) CURRENT_GID=$(id -g) command docker compose --env-file .env.development "$@" } docker__exec() { @@ -55,9 +55,23 @@ docker__stacker_lnd() { sndev__start() { shift - if ! [ -f .env.sndev ]; then - echo ".env.sndev does not exist ... creating from .env.sample" - cp .env.sample .env.sndev + if ! [ -f .env.development ]; then + echo ".env.development does not exist ... creating from .env.sample" + cp .env.sample .env.development + elif ! git diff --exit-code .env.sample .env.development; then + echo ".env.development is different from .env.sample ..." + echo "do you want to merge .env.sample into .env.development? [y/N]" + read -r answer + if [ "$answer" = "y" ]; then + # merge .env.sample into .env.development in a posix compliant way + git merge-file --theirs .env.development /dev/fd/3 3<<-EOF /dev/fd/4 4<<-EOF +$(git show HEAD:.env.sample) +EOF +$(cat .env.sample) +EOF + else + echo "merge cancelled" + fi fi if [ $# -eq 0 ]; then @@ -140,8 +154,13 @@ OPTIONS" } sndev__delete() { - # todo: add a confirmation prompt - docker__compose down --volumes --remove-orphans + echo "this will delete the containers, volumes, and orphans - are you sure? [y/N]" + read -r answer + if [ "$answer" = "y" ]; then + docker__compose down --volumes --remove-orphans + else + echo "delete cancelled" + fi } sndev__help_delete() {