s3 and image proxy with broken name resolution

.env.sample

@@ -16,10 +16,10 @@ LOGIN_EMAIL_SERVER=
 LOGIN_EMAIL_FROM=
 LIST_MONK_AUTH=
 
-#####################################################################
+########################################################
-# OTHER / OPTIONAL                                                  #
+# OTHER / OPTIONAL                                     #
-# configuration for push notifications, slack and imgproxy are here #
+# configuration for push notifications, slack are here #
-#####################################################################
+########################################################
 
 # VAPID for Web Push
 VAPID_MAILTO=
@@ -30,18 +30,6 @@ VAPID_PRIVKEY=
 SLACK_BOT_TOKEN=
 SLACK_CHANNEL_ID=
 
-# imgproxy
-NEXT_PUBLIC_IMGPROXY_URL=localhost:3001
-IMGPROXY_KEY=
-IMGPROXY_SALT=
-
-# search
-OPENSEARCH_URL=http://opensearch:9200
-OPENSEARCH_USERNAME=
-OPENSEARCH_PASSWORD=
-OPENSEARCH_INDEX=item
-OPENSEARCH_MODEL_ID=
-
 # lnurl ... you'll need a tunnel to localhost:3000 for these
 LNAUTH_URL=
 LNWITH_URL=
@@ -69,7 +57,11 @@ LND_SOCKET=sn_lnd:10009
 # openssl rand -hex 32
 NOSTR_PRIVATE_KEY=5f30b7e7714360f51f2be2e30c1d93b7fdf67366e730658e85777dfcc4e4245f
 
-# imgproxy options
+# imgproxy
+NEXT_PUBLIC_IMGPROXY_URL=http://imgproxy:3001
+IMGPROXY_KEY=9c273e803fd5d444bf8883f8c3000de57bee7995222370cab7f2d218dd9a4bbff6ca11cbf902e61eeef4358616f231da51e183aee6841e3a797a5c9a9530ba67
+IMGPROXY_SALT=47b802be2c9250a66b998f411fc63912ab0bc1c6b47d99b8d37c61019d1312a984b98745eac83db9791b01bb8c93ecbc9b2ef9f2981d66061c7d0a4528ff6465
+
 IMGPROXY_ENABLE_WEBP_DETECTION=1
 IMGPROXY_ENABLE_AVIF_DETECTION=1
 IMGPROXY_MAX_ANIMATION_FRAMES=2000
@@ -81,6 +73,17 @@ IMGPROXY_DOWNLOAD_TIMEOUT=9
 # IMGPROXY_DEVELOPMENT_ERRORS_MODE=1
 # IMGPROXY_ENABLE_DEBUG_HEADERS=true
 
+NEXT_PUBLIC_AWS_UPLOAD_BUCKET=uploads
+NEXT_PUBLIC_MEDIA_DOMAIN=s3:4566
+NEXT_PUBLIC_MEDIA_URL=http://s3:4566/uploads
+
+# search
+OPENSEARCH_URL=http://opensearch:9200
+OPENSEARCH_USERNAME=
+OPENSEARCH_PASSWORD=
+OPENSEARCH_INDEX=item
+OPENSEARCH_MODEL_ID=
+
 # prisma db url
 DATABASE_URL="postgresql://sn:password@db:5432/stackernews?schema=public"
 
@@ -117,10 +120,16 @@ LND_ADDR=bcrt1q7q06n5st4vqq3lssn0rtkrn2qqypghv9xg2xnl
 LND_PUBKEY=02cb2e2d5a6c5b17fa67b1a883e2973c82e328fb9bd08b2b156a9e23820c87a490
 
 # stacker lnd container stuff
-STACKER_LND_REST_PORT=9090
+STACKER_LND_REST_PORT=8081
 STACKER_LND_GRPC_PORT=10010
 # docker exec -u lnd stacker_lnd lncli newaddress p2wkh --unused
 STACKER_LND_ADDR=bcrt1qfqau4ug9e6rtrvxrgclg58e0r93wshucumm9vu
 STACKER_LND_PUBKEY=028093ae52e011d45b3e67f2e0f2cb6c3a1d7f88d2920d408f3ac6db3a56dc4b35
 
-LNCLI_NETWORK=regtest
+LNCLI_NETWORK=regtest
+
+# localstack container stuff
+AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
+AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
+PERSISTENCE=1
+SKIP_SSL_CERT_DOWNLOAD=1

.gitignore

@@ -28,13 +28,9 @@ yarn-debug.log*
 yarn-error.log*
 
 # local env files
-.env
 envbak
-.env.local
+.env*
-.env.development.local
+!.env.sample
-.env.test.local
-.env.production.local
-.env.sndev
 
 # vercel
 .vercel

api/s3/index.js

@@ -1,4 +1,5 @@
 import AWS from 'aws-sdk'
+import { MEDIA_URL } from '../../lib/constants'
 
 const bucketRegion = 'us-east-1'
 const Bucket = process.env.NEXT_PUBLIC_AWS_UPLOAD_BUCKET
@@ -8,7 +9,11 @@ AWS.config.update({
 })
 
 export function createPresignedPost ({ key, type, size }) {
-  const s3 = new AWS.S3({ apiVersion: '2006-03-01' })
+  const s3 = new AWS.S3({
+    apiVersion: '2006-03-01',
+    endpoint: process.env.NODE_ENV === 'development' ? `${MEDIA_URL}/${Bucket}` : undefined,
+    s3ForcePathStyle: process.env.NODE_ENV === 'development'
+  })
   return new Promise((resolve, reject) => {
     s3.createPresignedPost({
       Bucket,

components/image.js

@@ -4,7 +4,7 @@ import { IMGPROXY_URL_REGEXP } from '../lib/url'
 import { useShowModal } from './modal'
 import { useMe } from './me'
 import { Dropdown } from 'react-bootstrap'
-import { UNKNOWN_LINK_REL, UPLOAD_TYPES_ALLOW } from '../lib/constants'
+import { UNKNOWN_LINK_REL, UPLOAD_TYPES_ALLOW, MEDIA_URL } from '../lib/constants'
 import { useToast } from './toast'
 import gql from 'graphql-tag'
 import { useMutation } from '@apollo/client'
@@ -224,7 +224,7 @@ export const ImageUpload = forwardRef(({ children, className, onSelect, onUpload
           return
         }
 
-        const url = `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${data.getSignedPOST.fields.key}`
+        const url = `${MEDIA_URL}/${data.getSignedPOST.fields.key}`
         // key is upload id in database
         const id = data.getSignedPOST.fields.key
         onSuccess?.({ ...variables, id, name: file.name, url, file })

components/item-job.js

@@ -10,6 +10,7 @@ import { timeSince } from '../lib/time'
 import EmailIcon from '../svgs/mail-open-line.svg'
 import Share from './share'
 import Hat from './hat'
+import { MEDIA_URL } from '../lib/constants'
 
 export default function ItemJob ({ item, toc, rank, children }) {
   const isEmail = string().email().isValidSync(item.url)
@@ -25,7 +26,7 @@ export default function ItemJob ({ item, toc, rank, children }) {
       <div className={styles.item}>
         <Link href={`/items/${item.id}`}>
           <Image
-            src={item.uploadId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${item.uploadId}` : '/jobs-default.png'} width='42' height='42' className={styles.companyImage}
+            src={item.uploadId ? `${MEDIA_URL}/${item.uploadId}` : '/jobs-default.png'} width='42' height='42' className={styles.companyImage}
           />
         </Link>
         <div className={`${styles.hunk} align-self-center mb-0`}>

components/job-form.js

@@ -15,7 +15,7 @@ import Link from 'next/link'
 import { usePrice } from './price'
 import Avatar from './avatar'
 import { jobSchema } from '../lib/validate'
-import { MAX_TITLE_LENGTH } from '../lib/constants'
+import { MAX_TITLE_LENGTH, MEDIA_URL } from '../lib/constants'
 import { useToast } from './toast'
 import { toastDeleteScheduled } from '../lib/form'
 import { ItemButtonBar } from './post'
@@ -110,7 +110,7 @@ export default function JobForm ({ item, sub }) {
           <label className='form-label'>logo</label>
           <div className='position-relative' style={{ width: 'fit-content' }}>
             <Image
-              src={logoId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${logoId}` : '/jobs-default.png'} width='135' height='135' roundedCircle
+              src={logoId ? `${MEDIA_URL}/${logoId}` : '/jobs-default.png'} width='135' height='135' roundedCircle
             />
             <Avatar onSuccess={setLogoId} />
           </div>

components/user-header.js

@@ -28,7 +28,7 @@ import { hexToBech32 } from '../lib/nostr'
 import NostrIcon from '../svgs/nostr.svg'
 import GithubIcon from '../svgs/github-fill.svg'
 import TwitterIcon from '../svgs/twitter-fill.svg'
-import { UNKNOWN_LINK_REL } from '../lib/constants'
+import { UNKNOWN_LINK_REL, MEDIA_URL } from '../lib/constants'
 
 export default function UserHeader ({ user }) {
   const router = useRouter()
@@ -96,7 +96,7 @@ function HeaderPhoto ({ user, isMe }) {
       }
     }
   )
-  const src = user.photoId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${user.photoId}` : '/dorian400.jpg'
+  const src = user.photoId ? `${MEDIA_URL}/${user.photoId}` : '/dorian400.jpg'
 
   return (
     <div className='position-relative align-self-start' style={{ width: 'fit-content' }}>

components/user-list.js

@@ -8,6 +8,7 @@ import { useQuery } from '@apollo/client'
 import MoreFooter from './more-footer'
 import { useData } from './use-data'
 import Hat from './hat'
+import { MEDIA_URL } from '../lib/constants'
 
 // all of this nonsense is to show the stat we are sorting by first
 const Stacked = ({ user }) => (user.optional.stacked !== null && <span>{abbrNum(user.optional.stacked)} stacked</span>)
@@ -48,7 +49,7 @@ function User ({ user, rank, statComps, Embellish }) {
       <div className={`${styles.item} mb-2`}>
         <Link href={`/${user.name}`}>
           <Image
-            src={user.photoId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${user.photoId}` : '/dorian400.jpg'} width='32' height='32'
+            src={user.photoId ? `${MEDIA_URL}/${user.photoId}` : '/dorian400.jpg'} width='32' height='32'
             className={`${userStyles.userimg} me-2`}
           />
         </Link>

docker-compose.yml

@@ -15,7 +15,7 @@ services:
     ports:
       - "5431:5432"
     env_file:
-      - .env.sndev
+      - .env.development
     volumes:
       - ./docker/db/seed.sql:/docker-entrypoint-initdb.d/seed.sql
       - db:/var/lib/postgresql/data
@@ -45,18 +45,17 @@ services:
       sn_lnd:
         condition: service_healthy
         restart: true
+      # s3:
+      #   condition: service_healthy
+      #   restart: true
     env_file:
-      - .env.sndev
+      - .env.development
     expose:
       - "3000"
     ports:
       - "3000:3000"
     volumes:
       - ./:/app
-    links:
-      - db
-      - opensearch
-      - sn_lnd
     labels:
       CONNECT: "localhost:3000"
   worker:
@@ -81,20 +80,15 @@ services:
         condition: service_healthy
         restart: true
     env_file:
-      - .env.sndev
+      - .env.development
     volumes:
       - ./:/app
-    links:
-      - db
-      - app
-      - opensearch
-      - sn_lnd
     entrypoint: ["/bin/sh", "-c"]
     command:
       - npm run worker:dev
   imgproxy:
     container_name: imgproxy
-    image: darthsim/imgproxy:v3.18.1
+    image: docker.imgproxy.pro/imgproxy:v3.23.0
     healthcheck:
       test: [ "CMD", "imgproxy", "health" ]
       interval: 10s
@@ -103,13 +97,33 @@ services:
       start_period: 1m
     restart: unless-stopped
     env_file:
-      - .env.sndev
+      - .env.development
     ports:
       - "3001:8080"
-    links:
-      - app
     labels:
-      CONNECT: "localhost:3001"
+      - "CONNECT=localhost:3001"
+  s3:
+    container_name: s3
+    image: localstack/localstack:s3-latest
+    # healthcheck:
+    #   test: ["CMD-SHELL", "awslocal", "s3", "ls", "s3://uploads"]
+    #   interval: 10s
+    #   timeout: 10s
+    #   retries: 10
+    #   start_period: 1m
+    restart: unless-stopped
+    env_file:
+      - .env.development
+    environment:
+      - DEBUG=1
+    ports:
+      - "4566:4566"
+    volumes:
+      - 's3:/var/lib/localstack'
+      - './docker/s3/init-s3.sh:/etc/localstack/init/ready.d/init-s3.sh'
+      - './docker/s3/cors.json:/etc/localstack/init/ready.d/cors.json'
+    labels:
+      - "CONNECT=localhost:4566"
   opensearch:
     image: opensearchproject/opensearch:2.12.0
     container_name: opensearch
@@ -121,7 +135,7 @@ services:
       start_period: 1m
     restart: unless-stopped
     env_file:
-      - .env.sndev
+      - .env.development
     environment:
       - OPENSEARCH_INITIAL_ADMIN_PASSWORD=mVchg1T5oA9wudUh
     ports:
@@ -153,15 +167,13 @@ services:
         condition: service_healthy
         restart: true
     env_file:
-      - .env.sndev
+      - .env.development
     environment:
       - opensearch.ssl.verificationMode=none
       - OPENSEARCH_HOSTS=http://opensearch:9200
       - server.ssl.enabled=false
     ports:
       - 5601:5601
-    expose:
-      - "5601"
     links:
       - opensearch
     labels:
@@ -238,7 +250,7 @@ services:
         condition: service_healthy
         restart: true
     env_file:
-      - .env.sndev
+      - .env.development
     command:
       - 'lnd'
       - '--noseedbackup'
@@ -300,7 +312,7 @@ services:
         condition: service_healthy
         restart: true
     env_file:
-      - .env.sndev
+      - .env.development
     command:
       - 'lnd'
       - '--noseedbackup'
@@ -361,3 +373,4 @@ volumes:
   bitcoin:
   sn_lnd:
   stacker_lnd:
+  s3:

docker/s3/cors.json

@@ -0,0 +1,19 @@
+{
+  "CORSRules": [
+      {
+          "AllowedHeaders": [
+              "*"
+          ],
+          "AllowedMethods": [
+              "PUT",
+              "POST",
+              "GET",
+              "HEAD"
+          ],
+          "AllowedOrigins": [
+              "http://localhost:3000"
+          ],
+          "ExposeHeaders": []
+      }
+  ]
+}

docker/s3/init-s3.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+
+pip3 install --upgrade virtualenv awscli awscli-local requests
+awslocal s3 mb s3://uploads
+awslocal s3api put-bucket-cors --bucket uploads --cors-configuration file:///etc/localstack/init/ready.d/cors.json

lib/constants.js

@@ -10,7 +10,9 @@ export const BOOST_MIN = BOOST_MULT * 5
 export const UPLOAD_SIZE_MAX = 25 * 1024 * 1024
 export const UPLOAD_SIZE_MAX_AVATAR = 5 * 1024 * 1024
 export const IMAGE_PIXELS_MAX = 35000000
-export const AWS_S3_URL_REGEXP = new RegExp(`https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/([0-9]+)`, 'g')
+// backwards compatibile with old media domain env var
+export const MEDIA_URL = process.env.NEXT_PUBLIC_MEDIA_URL || `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}`
+export const AWS_S3_URL_REGEXP = new RegExp(`${MEDIA_URL}/([0-9]+)`, 'g')
 export const UPLOAD_TYPES_ALLOW = [
   'image/gif',
   'image/heic',

middleware.js

@@ -20,13 +20,16 @@ export function middleware (request) {
   }
 
   const nonce = Buffer.from(crypto.randomUUID()).toString('base64')
+  // we want to load media from other localhost ports during development
+  const devSrc = process.env.NODE_ENV === 'development' ? 'localhost:* ' : ''
+
   const cspHeader = [
     // if something is not explicitly allowed, we don't allow it.
     "default-src 'none'",
     "font-src 'self' a.stacker.news",
     // we want to load images from everywhere but we can limit to HTTPS at least
-    "img-src 'self' a.stacker.news m.stacker.news https: data: blob:",
+    `img-src 'self' ${devSrc}a.stacker.news m.stacker.news https: data: blob:`,
-    "media-src 'self' a.stacker.news m.stacker.news",
+    `media-src 'self' ${devSrc}a.stacker.news m.stacker.news`,
     // Using nonces and strict-dynamic deploys a strict CSP.
     // see https://cheatsheetseries.owasp.org/cheatsheets/Content_Security_Policy_Cheat_Sheet.html#strict-policy.
     // Old browsers will ignore nonce and strict-dynamic and fallback to host-based matching and unsafe-inline
@@ -39,7 +42,7 @@ export function middleware (request) {
     "style-src 'self' a.stacker.news 'unsafe-inline'",
     "manifest-src 'self'",
     'frame-src www.youtube.com platform.twitter.com',
-    "connect-src 'self' https: wss:",
+    `connect-src 'self' ${devSrc}https: wss:`,
     // disable dangerous plugins like Flash
     "object-src 'none'",
     // blocks injection of <base> tags

sndev

@@ -9,7 +9,7 @@ docker__compose() {
     exit 0
   fi
 
-  CURRENT_UID=$(id -u) CURRENT_GID=$(id -g) command docker compose --env-file .env.sndev "$@"
+  CURRENT_UID=$(id -u) CURRENT_GID=$(id -g) command docker compose --env-file .env.development "$@"
 }
 
 docker__exec() {
@@ -55,9 +55,23 @@ docker__stacker_lnd() {
 sndev__start() {
   shift
 
-  if ! [ -f .env.sndev ]; then
+  if ! [ -f .env.development ]; then
-    echo ".env.sndev does not exist ... creating from .env.sample"
+    echo ".env.development does not exist ... creating from .env.sample"
-    cp .env.sample .env.sndev
+    cp .env.sample .env.development
+  elif ! git diff --exit-code .env.sample .env.development; then
+    echo ".env.development is different from .env.sample ..."
+    echo "do you want to merge .env.sample into .env.development? [y/N]"
+    read -r answer
+    if [ "$answer" = "y" ]; then
+      # merge .env.sample into .env.development in a posix compliant way
+      git merge-file --theirs .env.development /dev/fd/3 3<<-EOF /dev/fd/4 4<<-EOF
+$(git show HEAD:.env.sample)
+EOF
+$(cat .env.sample)
+EOF
+    else
+      echo "merge cancelled"
+    fi
   fi
 
   if [ $# -eq 0 ]; then
@@ -140,8 +154,13 @@ OPTIONS"
 }
 
 sndev__delete() {
-  # todo: add a confirmation prompt
+  echo "this will delete the containers, volumes, and orphans - are you sure? [y/N]"
-  docker__compose down --volumes --remove-orphans
+  read -r answer
+  if [ "$answer" = "y" ]; then
+    docker__compose down --volumes --remove-orphans
+  else
+    echo "delete cancelled"
+  fi
 }
 
 sndev__help_delete() {