s3 and image proxy with broken name resolution
This commit is contained in:
keyan
parent
bb41692acc
commit
72ecc7b266
47
.env.sample
47
.env.sample
|
|
@ -16,10 +16,10 @@ LOGIN_EMAIL_SERVER=
|
|||
LOGIN_EMAIL_FROM=
|
||||
LIST_MONK_AUTH=
|
||||
|
||||
#####################################################################
|
||||
# OTHER / OPTIONAL #
|
||||
# configuration for push notifications, slack and imgproxy are here #
|
||||
#####################################################################
|
||||
########################################################
|
||||
# OTHER / OPTIONAL #
|
||||
# configuration for push notifications, slack are here #
|
||||
########################################################
|
||||
|
||||
# VAPID for Web Push
|
||||
VAPID_MAILTO=
|
||||
|
|
@ -30,18 +30,6 @@ VAPID_PRIVKEY=
|
|||
SLACK_BOT_TOKEN=
|
||||
SLACK_CHANNEL_ID=
|
||||
|
||||
# imgproxy
|
||||
NEXT_PUBLIC_IMGPROXY_URL=localhost:3001
|
||||
IMGPROXY_KEY=
|
||||
IMGPROXY_SALT=
|
||||
|
||||
# search
|
||||
OPENSEARCH_URL=http://opensearch:9200
|
||||
OPENSEARCH_USERNAME=
|
||||
OPENSEARCH_PASSWORD=
|
||||
OPENSEARCH_INDEX=item
|
||||
OPENSEARCH_MODEL_ID=
|
||||
|
||||
# lnurl ... you'll need a tunnel to localhost:3000 for these
|
||||
LNAUTH_URL=
|
||||
LNWITH_URL=
|
||||
|
|
@ -69,7 +57,11 @@ LND_SOCKET=sn_lnd:10009
|
|||
# openssl rand -hex 32
|
||||
NOSTR_PRIVATE_KEY=5f30b7e7714360f51f2be2e30c1d93b7fdf67366e730658e85777dfcc4e4245f
|
||||
|
||||
# imgproxy options
|
||||
# imgproxy
|
||||
NEXT_PUBLIC_IMGPROXY_URL=http://imgproxy:3001
|
||||
IMGPROXY_KEY=9c273e803fd5d444bf8883f8c3000de57bee7995222370cab7f2d218dd9a4bbff6ca11cbf902e61eeef4358616f231da51e183aee6841e3a797a5c9a9530ba67
|
||||
IMGPROXY_SALT=47b802be2c9250a66b998f411fc63912ab0bc1c6b47d99b8d37c61019d1312a984b98745eac83db9791b01bb8c93ecbc9b2ef9f2981d66061c7d0a4528ff6465
|
||||
|
||||
IMGPROXY_ENABLE_WEBP_DETECTION=1
|
||||
IMGPROXY_ENABLE_AVIF_DETECTION=1
|
||||
IMGPROXY_MAX_ANIMATION_FRAMES=2000
|
||||
|
|
@ -81,6 +73,17 @@ IMGPROXY_DOWNLOAD_TIMEOUT=9
|
|||
# IMGPROXY_DEVELOPMENT_ERRORS_MODE=1
|
||||
# IMGPROXY_ENABLE_DEBUG_HEADERS=true
|
||||
|
||||
NEXT_PUBLIC_AWS_UPLOAD_BUCKET=uploads
|
||||
NEXT_PUBLIC_MEDIA_DOMAIN=s3:4566
|
||||
NEXT_PUBLIC_MEDIA_URL=http://s3:4566/uploads
|
||||
|
||||
# search
|
||||
OPENSEARCH_URL=http://opensearch:9200
|
||||
OPENSEARCH_USERNAME=
|
||||
OPENSEARCH_PASSWORD=
|
||||
OPENSEARCH_INDEX=item
|
||||
OPENSEARCH_MODEL_ID=
|
||||
|
||||
# prisma db url
|
||||
DATABASE_URL="postgresql://sn:password@db:5432/stackernews?schema=public"
|
||||
|
||||
|
|
@ -117,10 +120,16 @@ LND_ADDR=bcrt1q7q06n5st4vqq3lssn0rtkrn2qqypghv9xg2xnl
|
|||
LND_PUBKEY=02cb2e2d5a6c5b17fa67b1a883e2973c82e328fb9bd08b2b156a9e23820c87a490
|
||||
|
||||
# stacker lnd container stuff
|
||||
STACKER_LND_REST_PORT=9090
|
||||
STACKER_LND_REST_PORT=8081
|
||||
STACKER_LND_GRPC_PORT=10010
|
||||
# docker exec -u lnd stacker_lnd lncli newaddress p2wkh --unused
|
||||
STACKER_LND_ADDR=bcrt1qfqau4ug9e6rtrvxrgclg58e0r93wshucumm9vu
|
||||
STACKER_LND_PUBKEY=028093ae52e011d45b3e67f2e0f2cb6c3a1d7f88d2920d408f3ac6db3a56dc4b35
|
||||
|
||||
LNCLI_NETWORK=regtest
|
||||
LNCLI_NETWORK=regtest
|
||||
|
||||
# localstack container stuff
|
||||
AWS_ACCESS_KEY_ID=AKIAIOSFODNN7EXAMPLE
|
||||
AWS_SECRET_ACCESS_KEY=wJalrXUtnFEMI/K7MDENG/bPxRfiCYEXAMPLEKEY
|
||||
PERSISTENCE=1
|
||||
SKIP_SSL_CERT_DOWNLOAD=1
|
||||
|
|
|
|||
|
|
@ -28,13 +28,9 @@ yarn-debug.log*
|
|||
yarn-error.log*
|
||||
|
||||
# local env files
|
||||
.env
|
||||
envbak
|
||||
.env.local
|
||||
.env.development.local
|
||||
.env.test.local
|
||||
.env.production.local
|
||||
.env.sndev
|
||||
.env*
|
||||
!.env.sample
|
||||
|
||||
# vercel
|
||||
.vercel
|
||||
|
|
|
|||
|
|
@ -1,4 +1,5 @@
|
|||
import AWS from 'aws-sdk'
|
||||
import { MEDIA_URL } from '../../lib/constants'
|
||||
|
||||
const bucketRegion = 'us-east-1'
|
||||
const Bucket = process.env.NEXT_PUBLIC_AWS_UPLOAD_BUCKET
|
||||
|
|
@ -8,7 +9,11 @@ AWS.config.update({
|
|||
})
|
||||
|
||||
export function createPresignedPost ({ key, type, size }) {
|
||||
const s3 = new AWS.S3({ apiVersion: '2006-03-01' })
|
||||
const s3 = new AWS.S3({
|
||||
apiVersion: '2006-03-01',
|
||||
endpoint: process.env.NODE_ENV === 'development' ? `${MEDIA_URL}/${Bucket}` : undefined,
|
||||
s3ForcePathStyle: process.env.NODE_ENV === 'development'
|
||||
})
|
||||
return new Promise((resolve, reject) => {
|
||||
s3.createPresignedPost({
|
||||
Bucket,
|
||||
|
|
|
|||
|
|
@ -4,7 +4,7 @@ import { IMGPROXY_URL_REGEXP } from '../lib/url'
|
|||
import { useShowModal } from './modal'
|
||||
import { useMe } from './me'
|
||||
import { Dropdown } from 'react-bootstrap'
|
||||
import { UNKNOWN_LINK_REL, UPLOAD_TYPES_ALLOW } from '../lib/constants'
|
||||
import { UNKNOWN_LINK_REL, UPLOAD_TYPES_ALLOW, MEDIA_URL } from '../lib/constants'
|
||||
import { useToast } from './toast'
|
||||
import gql from 'graphql-tag'
|
||||
import { useMutation } from '@apollo/client'
|
||||
|
|
@ -224,7 +224,7 @@ export const ImageUpload = forwardRef(({ children, className, onSelect, onUpload
|
|||
return
|
||||
}
|
||||
|
||||
const url = `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${data.getSignedPOST.fields.key}`
|
||||
const url = `${MEDIA_URL}/${data.getSignedPOST.fields.key}`
|
||||
// key is upload id in database
|
||||
const id = data.getSignedPOST.fields.key
|
||||
onSuccess?.({ ...variables, id, name: file.name, url, file })
|
||||
|
|
|
|||
|
|
@ -10,6 +10,7 @@ import { timeSince } from '../lib/time'
|
|||
import EmailIcon from '../svgs/mail-open-line.svg'
|
||||
import Share from './share'
|
||||
import Hat from './hat'
|
||||
import { MEDIA_URL } from '../lib/constants'
|
||||
|
||||
export default function ItemJob ({ item, toc, rank, children }) {
|
||||
const isEmail = string().email().isValidSync(item.url)
|
||||
|
|
@ -25,7 +26,7 @@ export default function ItemJob ({ item, toc, rank, children }) {
|
|||
<div className={styles.item}>
|
||||
<Link href={`/items/${item.id}`}>
|
||||
<Image
|
||||
src={item.uploadId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${item.uploadId}` : '/jobs-default.png'} width='42' height='42' className={styles.companyImage}
|
||||
src={item.uploadId ? `${MEDIA_URL}/${item.uploadId}` : '/jobs-default.png'} width='42' height='42' className={styles.companyImage}
|
||||
/>
|
||||
</Link>
|
||||
<div className={`${styles.hunk} align-self-center mb-0`}>
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ import Link from 'next/link'
|
|||
import { usePrice } from './price'
|
||||
import Avatar from './avatar'
|
||||
import { jobSchema } from '../lib/validate'
|
||||
import { MAX_TITLE_LENGTH } from '../lib/constants'
|
||||
import { MAX_TITLE_LENGTH, MEDIA_URL } from '../lib/constants'
|
||||
import { useToast } from './toast'
|
||||
import { toastDeleteScheduled } from '../lib/form'
|
||||
import { ItemButtonBar } from './post'
|
||||
|
|
@ -110,7 +110,7 @@ export default function JobForm ({ item, sub }) {
|
|||
<label className='form-label'>logo</label>
|
||||
<div className='position-relative' style={{ width: 'fit-content' }}>
|
||||
<Image
|
||||
src={logoId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${logoId}` : '/jobs-default.png'} width='135' height='135' roundedCircle
|
||||
src={logoId ? `${MEDIA_URL}/${logoId}` : '/jobs-default.png'} width='135' height='135' roundedCircle
|
||||
/>
|
||||
<Avatar onSuccess={setLogoId} />
|
||||
</div>
|
||||
|
|
|
|||
|
|
@ -28,7 +28,7 @@ import { hexToBech32 } from '../lib/nostr'
|
|||
import NostrIcon from '../svgs/nostr.svg'
|
||||
import GithubIcon from '../svgs/github-fill.svg'
|
||||
import TwitterIcon from '../svgs/twitter-fill.svg'
|
||||
import { UNKNOWN_LINK_REL } from '../lib/constants'
|
||||
import { UNKNOWN_LINK_REL, MEDIA_URL } from '../lib/constants'
|
||||
|
||||
export default function UserHeader ({ user }) {
|
||||
const router = useRouter()
|
||||
|
|
@ -96,7 +96,7 @@ function HeaderPhoto ({ user, isMe }) {
|
|||
}
|
||||
}
|
||||
)
|
||||
const src = user.photoId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${user.photoId}` : '/dorian400.jpg'
|
||||
const src = user.photoId ? `${MEDIA_URL}/${user.photoId}` : '/dorian400.jpg'
|
||||
|
||||
return (
|
||||
<div className='position-relative align-self-start' style={{ width: 'fit-content' }}>
|
||||
|
|
|
|||
|
|
@ -8,6 +8,7 @@ import { useQuery } from '@apollo/client'
|
|||
import MoreFooter from './more-footer'
|
||||
import { useData } from './use-data'
|
||||
import Hat from './hat'
|
||||
import { MEDIA_URL } from '../lib/constants'
|
||||
|
||||
// all of this nonsense is to show the stat we are sorting by first
|
||||
const Stacked = ({ user }) => (user.optional.stacked !== null && <span>{abbrNum(user.optional.stacked)} stacked</span>)
|
||||
|
|
@ -48,7 +49,7 @@ function User ({ user, rank, statComps, Embellish }) {
|
|||
<div className={`${styles.item} mb-2`}>
|
||||
<Link href={`/${user.name}`}>
|
||||
<Image
|
||||
src={user.photoId ? `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/${user.photoId}` : '/dorian400.jpg'} width='32' height='32'
|
||||
src={user.photoId ? `${MEDIA_URL}/${user.photoId}` : '/dorian400.jpg'} width='32' height='32'
|
||||
className={`${userStyles.userimg} me-2`}
|
||||
/>
|
||||
</Link>
|
||||
|
|
|
|||
|
|
@ -15,7 +15,7 @@ services:
|
|||
ports:
|
||||
- "5431:5432"
|
||||
env_file:
|
||||
- .env.sndev
|
||||
- .env.development
|
||||
volumes:
|
||||
- ./docker/db/seed.sql:/docker-entrypoint-initdb.d/seed.sql
|
||||
- db:/var/lib/postgresql/data
|
||||
|
|
@ -45,18 +45,17 @@ services:
|
|||
sn_lnd:
|
||||
condition: service_healthy
|
||||
restart: true
|
||||
# s3:
|
||||
# condition: service_healthy
|
||||
# restart: true
|
||||
env_file:
|
||||
- .env.sndev
|
||||
- .env.development
|
||||
expose:
|
||||
- "3000"
|
||||
ports:
|
||||
- "3000:3000"
|
||||
volumes:
|
||||
- ./:/app
|
||||
links:
|
||||
- db
|
||||
- opensearch
|
||||
- sn_lnd
|
||||
labels:
|
||||
CONNECT: "localhost:3000"
|
||||
worker:
|
||||
|
|
@ -81,20 +80,15 @@ services:
|
|||
condition: service_healthy
|
||||
restart: true
|
||||
env_file:
|
||||
- .env.sndev
|
||||
- .env.development
|
||||
volumes:
|
||||
- ./:/app
|
||||
links:
|
||||
- db
|
||||
- app
|
||||
- opensearch
|
||||
- sn_lnd
|
||||
entrypoint: ["/bin/sh", "-c"]
|
||||
command:
|
||||
- npm run worker:dev
|
||||
imgproxy:
|
||||
container_name: imgproxy
|
||||
image: darthsim/imgproxy:v3.18.1
|
||||
image: docker.imgproxy.pro/imgproxy:v3.23.0
|
||||
healthcheck:
|
||||
test: [ "CMD", "imgproxy", "health" ]
|
||||
interval: 10s
|
||||
|
|
@ -103,13 +97,33 @@ services:
|
|||
start_period: 1m
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- .env.sndev
|
||||
- .env.development
|
||||
ports:
|
||||
- "3001:8080"
|
||||
links:
|
||||
- app
|
||||
labels:
|
||||
CONNECT: "localhost:3001"
|
||||
- "CONNECT=localhost:3001"
|
||||
s3:
|
||||
container_name: s3
|
||||
image: localstack/localstack:s3-latest
|
||||
# healthcheck:
|
||||
# test: ["CMD-SHELL", "awslocal", "s3", "ls", "s3://uploads"]
|
||||
# interval: 10s
|
||||
# timeout: 10s
|
||||
# retries: 10
|
||||
# start_period: 1m
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- .env.development
|
||||
environment:
|
||||
- DEBUG=1
|
||||
ports:
|
||||
- "4566:4566"
|
||||
volumes:
|
||||
- 's3:/var/lib/localstack'
|
||||
- './docker/s3/init-s3.sh:/etc/localstack/init/ready.d/init-s3.sh'
|
||||
- './docker/s3/cors.json:/etc/localstack/init/ready.d/cors.json'
|
||||
labels:
|
||||
- "CONNECT=localhost:4566"
|
||||
opensearch:
|
||||
image: opensearchproject/opensearch:2.12.0
|
||||
container_name: opensearch
|
||||
|
|
@ -121,7 +135,7 @@ services:
|
|||
start_period: 1m
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- .env.sndev
|
||||
- .env.development
|
||||
environment:
|
||||
- OPENSEARCH_INITIAL_ADMIN_PASSWORD=mVchg1T5oA9wudUh
|
||||
ports:
|
||||
|
|
@ -153,15 +167,13 @@ services:
|
|||
condition: service_healthy
|
||||
restart: true
|
||||
env_file:
|
||||
- .env.sndev
|
||||
- .env.development
|
||||
environment:
|
||||
- opensearch.ssl.verificationMode=none
|
||||
- OPENSEARCH_HOSTS=http://opensearch:9200
|
||||
- server.ssl.enabled=false
|
||||
ports:
|
||||
- 5601:5601
|
||||
expose:
|
||||
- "5601"
|
||||
links:
|
||||
- opensearch
|
||||
labels:
|
||||
|
|
@ -238,7 +250,7 @@ services:
|
|||
condition: service_healthy
|
||||
restart: true
|
||||
env_file:
|
||||
- .env.sndev
|
||||
- .env.development
|
||||
command:
|
||||
- 'lnd'
|
||||
- '--noseedbackup'
|
||||
|
|
@ -300,7 +312,7 @@ services:
|
|||
condition: service_healthy
|
||||
restart: true
|
||||
env_file:
|
||||
- .env.sndev
|
||||
- .env.development
|
||||
command:
|
||||
- 'lnd'
|
||||
- '--noseedbackup'
|
||||
|
|
@ -361,3 +373,4 @@ volumes:
|
|||
bitcoin:
|
||||
sn_lnd:
|
||||
stacker_lnd:
|
||||
s3:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,19 @@
|
|||
{
|
||||
"CORSRules": [
|
||||
{
|
||||
"AllowedHeaders": [
|
||||
"*"
|
||||
],
|
||||
"AllowedMethods": [
|
||||
"PUT",
|
||||
"POST",
|
||||
"GET",
|
||||
"HEAD"
|
||||
],
|
||||
"AllowedOrigins": [
|
||||
"http://localhost:3000"
|
||||
],
|
||||
"ExposeHeaders": []
|
||||
}
|
||||
]
|
||||
}
|
||||
|
|
@ -0,0 +1,5 @@
|
|||
#!/bin/bash
|
||||
|
||||
pip3 install --upgrade virtualenv awscli awscli-local requests
|
||||
awslocal s3 mb s3://uploads
|
||||
awslocal s3api put-bucket-cors --bucket uploads --cors-configuration file:///etc/localstack/init/ready.d/cors.json
|
||||
|
|
@ -10,7 +10,9 @@ export const BOOST_MIN = BOOST_MULT * 5
|
|||
export const UPLOAD_SIZE_MAX = 25 * 1024 * 1024
|
||||
export const UPLOAD_SIZE_MAX_AVATAR = 5 * 1024 * 1024
|
||||
export const IMAGE_PIXELS_MAX = 35000000
|
||||
export const AWS_S3_URL_REGEXP = new RegExp(`https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}/([0-9]+)`, 'g')
|
||||
// backwards compatibile with old media domain env var
|
||||
export const MEDIA_URL = process.env.NEXT_PUBLIC_MEDIA_URL || `https://${process.env.NEXT_PUBLIC_MEDIA_DOMAIN}`
|
||||
export const AWS_S3_URL_REGEXP = new RegExp(`${MEDIA_URL}/([0-9]+)`, 'g')
|
||||
export const UPLOAD_TYPES_ALLOW = [
|
||||
'image/gif',
|
||||
'image/heic',
|
||||
|
|
|
|||
|
|
@ -20,13 +20,16 @@ export function middleware (request) {
|
|||
}
|
||||
|
||||
const nonce = Buffer.from(crypto.randomUUID()).toString('base64')
|
||||
// we want to load media from other localhost ports during development
|
||||
const devSrc = process.env.NODE_ENV === 'development' ? 'localhost:* ' : ''
|
||||
|
||||
const cspHeader = [
|
||||
// if something is not explicitly allowed, we don't allow it.
|
||||
"default-src 'none'",
|
||||
"font-src 'self' a.stacker.news",
|
||||
// we want to load images from everywhere but we can limit to HTTPS at least
|
||||
"img-src 'self' a.stacker.news m.stacker.news https: data: blob:",
|
||||
"media-src 'self' a.stacker.news m.stacker.news",
|
||||
`img-src 'self' ${devSrc}a.stacker.news m.stacker.news https: data: blob:`,
|
||||
`media-src 'self' ${devSrc}a.stacker.news m.stacker.news`,
|
||||
// Using nonces and strict-dynamic deploys a strict CSP.
|
||||
// see https://cheatsheetseries.owasp.org/cheatsheets/Content_Security_Policy_Cheat_Sheet.html#strict-policy.
|
||||
// Old browsers will ignore nonce and strict-dynamic and fallback to host-based matching and unsafe-inline
|
||||
|
|
@ -39,7 +42,7 @@ export function middleware (request) {
|
|||
"style-src 'self' a.stacker.news 'unsafe-inline'",
|
||||
"manifest-src 'self'",
|
||||
'frame-src www.youtube.com platform.twitter.com',
|
||||
"connect-src 'self' https: wss:",
|
||||
`connect-src 'self' ${devSrc}https: wss:`,
|
||||
// disable dangerous plugins like Flash
|
||||
"object-src 'none'",
|
||||
// blocks injection of <base> tags
|
||||
|
|
|
|||
31
sndev
31
sndev
|
|
@ -9,7 +9,7 @@ docker__compose() {
|
|||
exit 0
|
||||
fi
|
||||
|
||||
CURRENT_UID=$(id -u) CURRENT_GID=$(id -g) command docker compose --env-file .env.sndev "$@"
|
||||
CURRENT_UID=$(id -u) CURRENT_GID=$(id -g) command docker compose --env-file .env.development "$@"
|
||||
}
|
||||
|
||||
docker__exec() {
|
||||
|
|
@ -55,9 +55,23 @@ docker__stacker_lnd() {
|
|||
sndev__start() {
|
||||
shift
|
||||
|
||||
if ! [ -f .env.sndev ]; then
|
||||
echo ".env.sndev does not exist ... creating from .env.sample"
|
||||
cp .env.sample .env.sndev
|
||||
if ! [ -f .env.development ]; then
|
||||
echo ".env.development does not exist ... creating from .env.sample"
|
||||
cp .env.sample .env.development
|
||||
elif ! git diff --exit-code .env.sample .env.development; then
|
||||
echo ".env.development is different from .env.sample ..."
|
||||
echo "do you want to merge .env.sample into .env.development? [y/N]"
|
||||
read -r answer
|
||||
if [ "$answer" = "y" ]; then
|
||||
# merge .env.sample into .env.development in a posix compliant way
|
||||
git merge-file --theirs .env.development /dev/fd/3 3<<-EOF /dev/fd/4 4<<-EOF
|
||||
$(git show HEAD:.env.sample)
|
||||
EOF
|
||||
$(cat .env.sample)
|
||||
EOF
|
||||
else
|
||||
echo "merge cancelled"
|
||||
fi
|
||||
fi
|
||||
|
||||
if [ $# -eq 0 ]; then
|
||||
|
|
@ -140,8 +154,13 @@ OPTIONS"
|
|||
}
|
||||
|
||||
sndev__delete() {
|
||||
# todo: add a confirmation prompt
|
||||
docker__compose down --volumes --remove-orphans
|
||||
echo "this will delete the containers, volumes, and orphans - are you sure? [y/N]"
|
||||
read -r answer
|
||||
if [ "$answer" = "y" ]; then
|
||||
docker__compose down --volumes --remove-orphans
|
||||
else
|
||||
echo "delete cancelled"
|
||||
fi
|
||||
}
|
||||
|
||||
sndev__help_delete() {
|
||||
|
|
|
|||
Loading…
Reference in New Issue