diff --git a/.env.development b/.env.development index 7572da56..745a944e 100644 --- a/.env.development +++ b/.env.development @@ -190,4 +190,9 @@ CPU_SHARES_IMPORTANT=1024 CPU_SHARES_MODERATE=512 CPU_SHARES_LOW=256 -NEXT_TELEMETRY_DISABLED=1 \ No newline at end of file +NEXT_TELEMETRY_DISABLED=1 + +# custom domains stuff +# local DNS server for custom domain verification, by default it's dnsmasq. +# reachable by containers on 172.30.0.2(:53), outside of docker with 0.0.0.0:5353 +DOMAINS_DNS_SERVER=172.30.0.2 \ No newline at end of file diff --git a/.gitignore b/.gitignore index a8dbb48c..a01a1b4d 100644 --- a/.gitignore +++ b/.gitignore @@ -70,4 +70,7 @@ scripts/twitter-link-extract.config.json scripts/twitter-links.db # pay-awards -scripts/pay-awards.config.json \ No newline at end of file +scripts/pay-awards.config.json + +# dnsmasq +docker/dnsmasq/dnsmasq.d/* \ No newline at end of file diff --git a/README.md b/README.md index c607b056..6ba46641 100644 --- a/README.md +++ b/README.md @@ -87,6 +87,9 @@ COMMANDS psql open psql on db prisma run prisma commands + domains: + domains custom domains dev management + dev: pr fetch and checkout a pr lint run linters @@ -150,6 +153,17 @@ After `nlp-setup` is done, restart your containers to enable semantic search: > ./sndev restart ``` +#### Local DNS via dnsmasq + +To enable dnsmasq: + +- domains should be enabled in `COMPOSE_PROFILES`: + + ```.env + COMPOSE_PROFILES=...,domains,... + ``` + +To add/remove DNS records you can now use `./sndev domains dns`. More on this [here](#add-or-remove-dns-records-in-local).
@@ -449,6 +463,25 @@ To enable Web Push locally, you will need to set the `VAPID_*` env vars. `VAPID_
+## Custom domains + +### Add or remove DNS records in local + +A worker dedicated to verifying custom domains, checks, among other things, if a domain has the correct DNS records and values. This would normally require a real domain and access to its DNS configuration. Therefore we use dnsmasq to have local DNS, make sure you have [enabled it](#local-dns-via-dnsmasq). + +To add a DNS record the syntax is the following: + +`./sndev domains dns add|remove cname|txt ` + +For TXT records, you can also use `""` quoted strings on `value`. + +To list all DNS records present in the dnsmasq config: `./sndev domains dns list` + +#### Access a local custom domain added via dnsmasq +sndev will use the dnsmasq DNS server by default, but chances are that you might want to access the domain via your browser. + +For every edit on dnsmasq, it will give you the option to either edit the `/etc/hosts` file or use the dnsmasq DNS server which can be reached on `127.0.0.1:5353`. You can avoid getting asked to edit the `/etc/hosts` file by adding the `--no-hosts` parameter. + # Internals
diff --git a/docker-compose.yml b/docker-compose.yml index e1965d26..35b6e232 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -119,6 +119,9 @@ services: command: - npm run worker:dev cpu_shares: "${CPU_SHARES_IMPORTANT}" + networks: + - default + - domains-network imgproxy: container_name: imgproxy image: darthsim/imgproxy:v3.23.0 @@ -806,6 +809,28 @@ services: CONNECT: "localhost:${LNBITS_WEB_PORT}" TORDIR: "/app/.tor" cpu_shares: "${CPU_SHARES_LOW}" + dnsmasq: + image: 4km3/dnsmasq:2.90-r3 + profiles: + - domains + container_name: dnsmasq + restart: unless-stopped + ports: + - "5353:53/tcp" + - "5353:53/udp" + command: + - --no-daemon + - --address=/.sndev/127.0.0.1 + - --conf-file=/etc/dnsmasq.conf + - --conf-dir=/etc/dnsmasq.d + volumes: + - ./docker/dnsmasq/dnsmasq.conf:/etc/dnsmasq.conf + - ./docker/dnsmasq/dnsmasq.d:/etc/dnsmasq.d + cpu_shares: "${CPU_SHARES_LOW}" + networks: + domains-network: + ipv4_address: 172.30.0.2 + volumes: db: os: @@ -819,3 +844,13 @@ volumes: nwc_recv: tordata: eclair: + dnsmasq: + +networks: + default: {} + domains-network: + name: domains-network + driver: bridge + ipam: + config: + - subnet: 172.30.0.0/24 \ No newline at end of file diff --git a/docker/dnsmasq/dnsmasq.conf b/docker/dnsmasq/dnsmasq.conf new file mode 100644 index 00000000..32b91543 --- /dev/null +++ b/docker/dnsmasq/dnsmasq.conf @@ -0,0 +1,12 @@ +server=1.1.1.1 +no-resolv +bind-interfaces +listen-address=0.0.0.0 + +log-queries +log-facility=/var/log/dnsmasq.log + +# example of cname and txt for custom domains verification +# this is to be edited by sndev cli or manually +cname=www.pizza.sndev,sn.sndev +txt-record=_snverify.www.pizza.sndev,"EXAMPLE_TXT_VALUE" diff --git a/scripts/set-dnsmasq b/scripts/set-dnsmasq new file mode 100755 index 00000000..2587b781 --- /dev/null +++ b/scripts/set-dnsmasq @@ -0,0 +1,267 @@ +#!/bin/sh +# Script to handle local DNS testing via dnsmasq +# supports add/remove/list of CNAME and TXT records +# on dnsmasq config changes it will restart the dnsmasq container +# it also asks to add the record to the /etc/hosts file if --no-hosts is not used + +# prep +set -e + +# ensure directory exists before using the file +mkdir -p ./docker/dnsmasq/dnsmasq.d +# dedicated sndev.conf file +DNSMASQ_CONF_PATH="./docker/dnsmasq/dnsmasq.d/sndev.conf" + +# check if running on Windows or macOS +# this script doesn't support Windows /etc/hosts editing +# sed -i has different syntax on macOS/BSD and Linux +IS_WINDOWS=false +IS_DARWIN=false + +OS_NAME=$(uname -s) +case "$OS_NAME" in + MINGW*|CYGWIN*|MSYS*) + IS_WINDOWS=true + ;; + Darwin*) + IS_DARWIN=true + ;; +esac + +# general usage +usage() { + cat < [--no-hosts] + remove [--no-hosts] + list + +FLAGS + --no-hosts Skip asking to add/remove record to /etc/hosts + Useful if you're using dnsmasq [127.0.0.1:5353] as your DNS server. + +EXAMPLES + $ sndev domains dns add cname www.pizza.sndev sn.sndev + $ sndev domains dns remove txt _snverify.www.pizza.sndev "7vfyvQO...vMALqvqkTQ" + $ sndev domains dns list cname|txt +EOF +exit 1 +} + +# handle flags +while [ $# -gt 0 ]; do + case "$1" in + -h|--help) + usage + ;; + *) + break + ;; + esac +done + +INTENT=$1 # add, remove, list +TYPE=$2 # cname, txt +NAME=$3 # www.pizza.com +VALUE=$4 # stacker.news or "7vfyvQO...vMALqvqkTQ=" +NO_HOSTS=false # handled via --no-hosts flag + +# creates a line compatible with dnsmasq config file +prepare_line() { + if [ "$TYPE" = "cname" ]; then + LINE="cname=${NAME},${VALUE}" + elif [ "$TYPE" = "txt" ]; then + escaped_quotes=$(printf '%s' "$VALUE" | sed 's/"/\\"/g') + LINE="txt-record=${NAME},\"${escaped_quotes}\"" + else + echo "Invalid record type: $TYPE" + usage + fi +} + +# if we're adding or removing a record, we need to check for required args +if [ "$INTENT" = "add" ] || [ "$INTENT" = "remove" ]; then + if [ $# -lt 4 ]; then + echo "Not enough arguments" + usage + else + prepare_line # prepare the line for the dnsmasq config file + shift 4 # 4 args: intent, type, name, value + # we need to get the --no-hosts flag if it's present + if [ "$1" = "--no-hosts" ]; then + NO_HOSTS=true + shift + fi + fi +# if we're listing records, we need to have at least the TYPE arg +elif [ "$INTENT" = "list" ] && [ $# -lt 2 ]; then + echo "No type provided" + usage +fi + +# add a record to the dnsmasq config +add_record() { + # check if the record already exists + if grep -Fxq "$LINE" "$DNSMASQ_CONF_PATH"; then + echo "Record already exists: $LINE" + exit 1 + fi + + # add the record to the dnsmasq config + echo "Adding record: $LINE" + printf "%s\n" "$LINE" >> "$DNSMASQ_CONF_PATH" + + # if we're adding a CNAME record and --no-hosts is not used, we need to ask to add the record to the /etc/hosts file + if [ "$TYPE" = "cname" ] && [ "$NO_HOSTS" = false ]; then + # dnsmasq pamphlet + printf " + While sndev will use dnsmasq DNS server, your system won't use it by default. + You can either manually point DNS to 127.0.0.1:5353 to access it system-wide, + or add this record to /etc/hosts to access it via browser.\n\n" + + # ask to add the record to the /etc/hosts file + printf "[sudo] Do you want to add '127.0.0.1 %s' to /etc/hosts? [y/N] " "$NAME" + read -r response + case "$response" in + [Yy]*) + # add the record to the /etc/hosts file + if ! add_record_to_hosts "$NAME"; then + echo "/etc/hosts hasn't been touched." + fi + ;; + esac + fi + echo "Done." + exit 0 +} + +# remove a record from the dnsmasq config +remove_record() { + # check if the record exists + if ! grep -Fxq "$LINE" "$DNSMASQ_CONF_PATH"; then + echo "Can't find record: $LINE" + echo "The record may have been removed or the name/value is incorrect or incomplete." + echo "Use 'sndev domains dns list' to see all records." + exit 1 + fi + + # remove the record from the dnsmasq config + echo "Removing record: $LINE" + if [ "$IS_DARWIN" = true ]; then + sed -i '' "/^$LINE$/d" "$DNSMASQ_CONF_PATH" + else + sed -i "/^$LINE$/d" "$DNSMASQ_CONF_PATH" + fi + + # if we're removing a CNAME record and --no-hosts is not used, we need to ask to remove the record from the /etc/hosts file + if [ "$TYPE" = "cname" ] && [ "$NO_HOSTS" = false ]; then + # ask to remove the record from the /etc/hosts file + printf "[sudo] Do you want to remove this record from /etc/hosts? [y/N] " + read -r response + case "$response" in + [Yy]*) + # remove the record from the /etc/hosts file + if ! remove_record_from_hosts "$NAME"; then + echo "/etc/hosts hasn't been touched." + fi + ;; + esac + fi + echo "Done." + exit 0 +} + +# list all records of a given type +list_records() { + if [ "$TYPE" = "txt" ]; then + TYPE="txt-record" + fi + grep "^$TYPE=" "$DNSMASQ_CONF_PATH" || echo "No $TYPE records found." +} + +# add a record to the /etc/hosts file +add_record_to_hosts() { + domain="$1" + + # this script doesn't support Windows /etc/hosts editing + if [ "$IS_WINDOWS" = true ]; then + echo "Adding records to /etc/hosts via this script is not supported on Windows" + return 1 + fi + + # check if the record already exists in the /etc/hosts file + if check_record_hosts_exists "$domain"; then + echo "Record already exists in /etc/hosts: $domain" + return 1 + fi + + # add the record to the /etc/hosts file + echo "Adding record to /etc/hosts: $domain" + echo "This operation will require sudo privileges" + if ! echo "127.0.0.1 $domain" | sudo tee -a "/etc/hosts" > /dev/null; then + echo "Failed to add record to /etc/hosts" + return 1 + fi + echo "$domain added to /etc/hosts." + echo "You can now access http://$domain:3000 via browser." + return 0 +} + +remove_record_from_hosts() { + domain="$1" + + # this script doesn't support Windows /etc/hosts editing + if [ "$IS_WINDOWS" = true ]; then + echo "Removing records from /etc/hosts via this script is not supported on Windows" + return 1 + fi + + # check if the record exists in the /etc/hosts file + if ! check_record_hosts_exists "$domain"; then + echo "Record not found in /etc/hosts: $domain" + return 1 + fi + + # remove the record from the /etc/hosts file + echo "Removing record from /etc/hosts: $domain" + echo "This operation will require sudo privileges" + if [ "$IS_DARWIN" = true ]; then + if ! sudo sed -i '' "/^127.0.0.1 $domain$/d" "/etc/hosts" 2>/dev/null; then + echo "Failed to remove record from /etc/hosts." + return 1 + fi + else + if ! sudo sed -i "/^127.0.0.1 $domain$/d" "/etc/hosts" 2>/dev/null; then + echo "Failed to remove record from /etc/hosts." + return 1 + fi + fi + echo "$domain removed from /etc/hosts." + return 0 +} + +# check if a record exists in the /etc/hosts file +check_record_hosts_exists() { + domain="$1" + + # grep for the record + if grep -Fxq "127.0.0.1 $domain" "/etc/hosts"; then + return 0 + fi + return 1 +} + +# switch intents +case "$INTENT" in + add) add_record ;; + remove) remove_record ;; + list) list_records ;; + *) usage ;; +esac + diff --git a/sndev b/sndev index a0fc1dcc..cc0a646c 100755 --- a/sndev +++ b/sndev @@ -322,6 +322,43 @@ COMMANDS" sndev__prisma --help | awk '/Commands/{y=1;next}y' | awk '!/^([\t ]+init)|([\t ]+studio)/' | sed -n '/Flags/q;p' } +sndev__domains() { + shift + case $1 in + dns) + shift + if ./scripts/set-dnsmasq "$@" && { [ "$1" = "add" ] || [ "$1" = "remove" ]; }; then + echo "restarting dnsmasq to apply changes" + if docker ps | grep -q dnsmasq; then + docker__compose restart dnsmasq + exit 0 + else + echo "dnsmasq is not running, you may need to start it manually" + exit 1 + fi + fi + ;; + # PLACEHOLDER for domain verification management + *) + sndev__help_domains + exit 0 + esac +} + +sndev__help_domains() { + help=" +manage custom domains + +USAGE + $ sndev domains [COMMAND] + +COMMANDS + dns [add|remove|list] +" + + echo "$help" +} + sndev__lint() { shift docker__exec -t -u apprunner app npm run lint @@ -607,6 +644,9 @@ COMMANDS psql open psql on db prisma run prisma commands + domains: + domains custom domains dev management + dev: pr fetch and checkout a pr lint run linters