local DNS server via dnsmasq (#2168)

* Use dnsmasq to create virtual hosts and mock DNS management for custom domains

- dnsmasq docker image
- dnsmasq network bridge
- point *.sndev to 127.0.0.1
- set-dnsmasq script
- -- add/remove/list dns records in dnsmasq.conf
- add 'domains' to sndev
- 'sndev domains dns' referencing set-dnsmasq script

* restart dnsmasq if add/remove succeeded

* add domain to /etc/hosts; cleanup

* tell if the command needs sudo permission

* add directions for dnsmasq DNS server usage

* add --no-hosts flag to skip asking to edit /etc/hosts

* add domains command to README.md

* add dnsmasq instructions to README.md

* correct exit on usage function; final cleanup and comments

* portable bash; use default network for dnsmasq; set a version for dnsmasq image

* POSIX compliance, add env var to .env.development, adjust README

* ignore dnsmasq.conf edits, use template instead

* use extra configs for dnsmasq, more POSIX compliance

* fix --no-hosts flag recognition, light cleanup

* shift 4 only if the command has enough args; more error messages; adjust TXT type only on list

* different sed syntax for macOS

.env.development

@@ -190,4 +190,9 @@ CPU_SHARES_IMPORTANT=1024
 CPU_SHARES_MODERATE=512
 CPU_SHARES_LOW=256
 
-NEXT_TELEMETRY_DISABLED=1
+NEXT_TELEMETRY_DISABLED=1
+
+# custom domains stuff
+# local DNS server for custom domain verification, by default it's dnsmasq.
+# reachable by containers on 172.30.0.2(:53), outside of docker with 0.0.0.0:5353
+DOMAINS_DNS_SERVER=172.30.0.2

.gitignore

@@ -70,4 +70,7 @@ scripts/twitter-link-extract.config.json
 scripts/twitter-links.db
 
 # pay-awards
-scripts/pay-awards.config.json
+scripts/pay-awards.config.json
+
+# dnsmasq
+docker/dnsmasq/dnsmasq.d/*

README.md

@@ -87,6 +87,9 @@ COMMANDS
     psql                   open psql on db
     prisma                 run prisma commands
 
+  domains:
+    domains                custom domains dev management
+
   dev:
     pr                     fetch and checkout a pr
     lint                   run linters
@@ -150,6 +153,17 @@ After `nlp-setup` is done, restart your containers to enable semantic search:
 > ./sndev restart
 ```
 
+#### Local DNS via dnsmasq
+
+To enable dnsmasq:
+
+- domains should be enabled in `COMPOSE_PROFILES`:
+
+    ```.env
+    COMPOSE_PROFILES=...,domains,...
+    ```
+
+To add/remove DNS records you can now use `./sndev domains dns`. More on this [here](#add-or-remove-dns-records-in-local).
 
 <br>
 
@@ -449,6 +463,25 @@ To enable Web Push locally, you will need to set the `VAPID_*` env vars. `VAPID_
 
 <br>
 
+## Custom domains
+
+### Add or remove DNS records in local
+
+A worker dedicated to verifying custom domains, checks, among other things, if a domain has the correct DNS records and values. This would normally require a real domain and access to its DNS configuration. Therefore we use dnsmasq to have local DNS, make sure you have [enabled it](#local-dns-via-dnsmasq).
+
+To add a DNS record the syntax is the following:
+
+`./sndev domains dns add|remove cname|txt <name/domain> <value>`
+
+For TXT records, you can also use `""` quoted strings on `value`.
+
+To list all DNS records present in the dnsmasq config: `./sndev domains dns list`
+
+#### Access a local custom domain added via dnsmasq
+sndev will use the dnsmasq DNS server by default, but chances are that you might want to access the domain via your browser.
+
+For every edit on dnsmasq, it will give you the option to either edit the `/etc/hosts` file or use the dnsmasq DNS server which can be reached on `127.0.0.1:5353`. You can avoid getting asked to edit the `/etc/hosts` file by adding the `--no-hosts` parameter.
+
 # Internals
 
 <br>

docker-compose.yml

@@ -119,6 +119,9 @@ services:
     command:
       - npm run worker:dev
     cpu_shares: "${CPU_SHARES_IMPORTANT}"
+    networks:
+      - default
+      - domains-network
   imgproxy:
     container_name: imgproxy
     image: darthsim/imgproxy:v3.23.0
@@ -806,6 +809,28 @@ services:
       CONNECT: "localhost:${LNBITS_WEB_PORT}"
       TORDIR: "/app/.tor"
     cpu_shares: "${CPU_SHARES_LOW}"
+  dnsmasq:
+    image: 4km3/dnsmasq:2.90-r3
+    profiles:
+      - domains
+    container_name: dnsmasq
+    restart: unless-stopped
+    ports:
+      - "5353:53/tcp"
+      - "5353:53/udp"
+    command:
+      - --no-daemon
+      - --address=/.sndev/127.0.0.1
+      - --conf-file=/etc/dnsmasq.conf
+      - --conf-dir=/etc/dnsmasq.d
+    volumes:
+      - ./docker/dnsmasq/dnsmasq.conf:/etc/dnsmasq.conf
+      - ./docker/dnsmasq/dnsmasq.d:/etc/dnsmasq.d
+    cpu_shares: "${CPU_SHARES_LOW}"
+    networks:
+      domains-network:
+        ipv4_address: 172.30.0.2
+
 volumes:
   db:
   os:
@@ -819,3 +844,13 @@ volumes:
   nwc_recv:
   tordata:
   eclair:
+  dnsmasq:
+
+networks:
+  default: {}
+  domains-network:
+    name: domains-network
+    driver: bridge
+    ipam:
+      config:
+        - subnet: 172.30.0.0/24

docker/dnsmasq/dnsmasq.conf

@@ -0,0 +1,12 @@
+server=1.1.1.1
+no-resolv
+bind-interfaces
+listen-address=0.0.0.0
+
+log-queries
+log-facility=/var/log/dnsmasq.log
+
+# example of cname and txt for custom domains verification
+# this is to be edited by sndev cli or manually
+cname=www.pizza.sndev,sn.sndev
+txt-record=_snverify.www.pizza.sndev,"EXAMPLE_TXT_VALUE"

scripts/set-dnsmasq

@@ -0,0 +1,267 @@
+#!/bin/sh
+# Script to handle local DNS testing via dnsmasq
+# supports add/remove/list of CNAME and TXT records
+# on dnsmasq config changes it will restart the dnsmasq container
+# it also asks to add the record to the /etc/hosts file if --no-hosts is not used
+
+# prep
+set -e
+
+# ensure directory exists before using the file
+mkdir -p ./docker/dnsmasq/dnsmasq.d
+# dedicated sndev.conf file
+DNSMASQ_CONF_PATH="./docker/dnsmasq/dnsmasq.d/sndev.conf"
+
+# check if running on Windows or macOS
+# this script doesn't support Windows /etc/hosts editing
+# sed -i has different syntax on macOS/BSD and Linux
+IS_WINDOWS=false
+IS_DARWIN=false
+
+OS_NAME=$(uname -s)
+case "$OS_NAME" in
+  MINGW*|CYGWIN*|MSYS*)
+    IS_WINDOWS=true
+    ;;
+  Darwin*)
+    IS_DARWIN=true
+    ;;
+esac
+
+# general usage
+usage() {
+  cat <<EOF
+Set mock DNS records for custom domains in dnsmasq conf.
+Use .sndev domains as they automatically resolve to 127.0.0.1.
+
+USAGE
+  $ sndev domains dns [COMMAND]
+
+COMMANDS
+  add    <cname|txt> <name> <value> [--no-hosts]
+  remove <cname|txt> <name> <value> [--no-hosts]
+  list   <cname|txt>
+
+FLAGS
+  --no-hosts  Skip asking to add/remove record to /etc/hosts
+              Useful if you're using dnsmasq [127.0.0.1:5353] as your DNS server.
+
+EXAMPLES
+  $ sndev domains dns add cname www.pizza.sndev sn.sndev
+  $ sndev domains dns remove txt _snverify.www.pizza.sndev "7vfyvQO...vMALqvqkTQ"
+  $ sndev domains dns list cname|txt
+EOF
+exit 1
+}
+
+# handle flags
+while [ $# -gt 0 ]; do
+  case "$1" in
+    -h|--help)
+      usage
+    ;;
+    *)
+      break
+    ;;
+  esac
+done
+
+INTENT=$1       # add, remove, list
+TYPE=$2         # cname, txt
+NAME=$3         # www.pizza.com
+VALUE=$4        # stacker.news or "7vfyvQO...vMALqvqkTQ="
+NO_HOSTS=false  # handled via --no-hosts flag
+
+# creates a line compatible with dnsmasq config file
+prepare_line() {
+  if [ "$TYPE" = "cname" ]; then
+    LINE="cname=${NAME},${VALUE}"
+  elif [ "$TYPE" = "txt" ]; then
+    escaped_quotes=$(printf '%s' "$VALUE" | sed 's/"/\\"/g')
+    LINE="txt-record=${NAME},\"${escaped_quotes}\""
+  else
+    echo "Invalid record type: $TYPE"
+    usage
+  fi
+}
+
+# if we're adding or removing a record, we need to check for required args
+if [ "$INTENT" = "add" ] || [ "$INTENT" = "remove" ]; then
+    if [ $# -lt 4 ]; then
+        echo "Not enough arguments"
+        usage
+    else
+        prepare_line # prepare the line for the dnsmasq config file
+        shift 4 # 4 args: intent, type, name, value
+        # we need to get the --no-hosts flag if it's present
+        if [ "$1" = "--no-hosts" ]; then
+            NO_HOSTS=true
+            shift
+        fi
+    fi
+# if we're listing records, we need to have at least the TYPE arg
+elif [ "$INTENT" = "list" ] && [ $# -lt 2 ]; then
+    echo "No type provided"
+    usage
+fi
+
+# add a record to the dnsmasq config
+add_record() {
+  # check if the record already exists
+  if grep -Fxq "$LINE" "$DNSMASQ_CONF_PATH"; then
+    echo "Record already exists: $LINE"
+    exit 1
+  fi
+
+  # add the record to the dnsmasq config
+  echo "Adding record: $LINE"
+  printf "%s\n" "$LINE" >> "$DNSMASQ_CONF_PATH"
+
+  # if we're adding a CNAME record and --no-hosts is not used, we need to ask to add the record to the /etc/hosts file
+  if [ "$TYPE" = "cname" ] && [ "$NO_HOSTS" = false ]; then
+    # dnsmasq pamphlet 
+    printf "
+    While sndev will use dnsmasq DNS server, your system won't use it by default.
+    You can either manually point DNS to 127.0.0.1:5353 to access it system-wide,
+    or add this record to /etc/hosts to access it via browser.\n\n"
+    
+    # ask to add the record to the /etc/hosts file
+    printf "[sudo] Do you want to add '127.0.0.1 %s' to /etc/hosts? [y/N] " "$NAME"
+    read -r response
+    case "$response" in
+      [Yy]*)
+        # add the record to the /etc/hosts file
+        if ! add_record_to_hosts "$NAME"; then
+          echo "/etc/hosts hasn't been touched."
+        fi
+        ;;
+    esac
+  fi
+  echo "Done."
+  exit 0
+}
+
+# remove a record from the dnsmasq config
+remove_record() {
+  # check if the record exists
+  if ! grep -Fxq "$LINE" "$DNSMASQ_CONF_PATH"; then
+    echo "Can't find record: $LINE"
+    echo "The record may have been removed or the name/value is incorrect or incomplete."
+    echo "Use 'sndev domains dns list' to see all records."
+    exit 1
+  fi
+
+  # remove the record from the dnsmasq config
+  echo "Removing record: $LINE"
+  if [ "$IS_DARWIN" = true ]; then
+    sed -i '' "/^$LINE$/d" "$DNSMASQ_CONF_PATH"
+  else
+    sed -i "/^$LINE$/d" "$DNSMASQ_CONF_PATH"
+  fi
+
+  # if we're removing a CNAME record and --no-hosts is not used, we need to ask to remove the record from the /etc/hosts file
+  if [ "$TYPE" = "cname" ] && [ "$NO_HOSTS" = false ]; then
+    # ask to remove the record from the /etc/hosts file
+    printf "[sudo] Do you want to remove this record from /etc/hosts? [y/N] "
+    read -r response
+    case "$response" in
+      [Yy]*)
+        # remove the record from the /etc/hosts file
+        if ! remove_record_from_hosts "$NAME"; then
+          echo "/etc/hosts hasn't been touched."
+        fi
+        ;;
+    esac
+  fi
+  echo "Done."
+  exit 0
+}
+
+# list all records of a given type
+list_records() {
+  if [ "$TYPE" = "txt" ]; then
+    TYPE="txt-record"
+  fi
+  grep "^$TYPE=" "$DNSMASQ_CONF_PATH" || echo "No $TYPE records found."
+}
+
+# add a record to the /etc/hosts file
+add_record_to_hosts() {
+  domain="$1"
+  
+  # this script doesn't support Windows /etc/hosts editing
+  if [ "$IS_WINDOWS" = true ]; then
+    echo "Adding records to /etc/hosts via this script is not supported on Windows"
+    return 1
+  fi
+
+  # check if the record already exists in the /etc/hosts file
+  if check_record_hosts_exists "$domain"; then
+    echo "Record already exists in /etc/hosts: $domain"
+    return 1
+  fi
+
+  # add the record to the /etc/hosts file
+  echo "Adding record to /etc/hosts: $domain"
+  echo "This operation will require sudo privileges"
+  if ! echo "127.0.0.1 $domain" | sudo tee -a "/etc/hosts" > /dev/null; then
+    echo "Failed to add record to /etc/hosts"
+    return 1
+  fi
+  echo "$domain added to /etc/hosts."
+  echo "You can now access http://$domain:3000 via browser."
+  return 0
+}
+
+remove_record_from_hosts() {
+  domain="$1"
+  
+  # this script doesn't support Windows /etc/hosts editing
+  if [ "$IS_WINDOWS" = true ]; then
+    echo "Removing records from /etc/hosts via this script is not supported on Windows"
+    return 1
+  fi
+
+  # check if the record exists in the /etc/hosts file
+  if ! check_record_hosts_exists "$domain"; then
+    echo "Record not found in /etc/hosts: $domain"
+    return 1
+  fi
+
+  # remove the record from the /etc/hosts file
+  echo "Removing record from /etc/hosts: $domain"
+  echo "This operation will require sudo privileges"
+  if [ "$IS_DARWIN" = true ]; then
+    if ! sudo sed -i '' "/^127.0.0.1 $domain$/d" "/etc/hosts" 2>/dev/null; then
+      echo "Failed to remove record from /etc/hosts."
+      return 1
+    fi
+  else
+    if ! sudo sed -i "/^127.0.0.1 $domain$/d" "/etc/hosts" 2>/dev/null; then
+      echo "Failed to remove record from /etc/hosts."
+      return 1
+    fi
+  fi
+  echo "$domain removed from /etc/hosts."
+  return 0
+}
+
+# check if a record exists in the /etc/hosts file
+check_record_hosts_exists() {
+  domain="$1"
+  
+  # grep for the record
+  if grep -Fxq "127.0.0.1 $domain" "/etc/hosts"; then
+    return 0
+  fi
+  return 1
+}
+
+# switch intents
+case "$INTENT" in
+  add)    add_record ;;
+  remove) remove_record ;;
+  list)   list_records ;;
+  *)      usage ;;
+esac
+

sndev

@@ -322,6 +322,43 @@ COMMANDS"
   sndev__prisma --help | awk '/Commands/{y=1;next}y' | awk '!/^([\t ]+init)|([\t ]+studio)/' | sed -n '/Flags/q;p'
 }
 
+sndev__domains() {
+  shift
+  case $1 in
+  dns)
+    shift
+    if ./scripts/set-dnsmasq "$@" && { [ "$1" = "add" ] || [ "$1" = "remove" ]; }; then
+      echo "restarting dnsmasq to apply changes"
+      if docker ps | grep -q dnsmasq; then
+        docker__compose restart dnsmasq
+        exit 0
+      else
+        echo "dnsmasq is not running, you may need to start it manually"
+        exit 1
+      fi
+    fi
+    ;;
+  # PLACEHOLDER for domain verification management
+  *)
+    sndev__help_domains
+    exit 0
+  esac
+}
+
+sndev__help_domains() {
+  help="
+manage custom domains
+
+USAGE
+  $ sndev domains [COMMAND]
+
+COMMANDS
+  dns             [add|remove|list]
+"
+
+  echo "$help"
+}
+
 sndev__lint() {
   shift
   docker__exec -t -u apprunner app npm run lint
@@ -607,6 +644,9 @@ COMMANDS
     psql                   open psql on db
     prisma                 run prisma commands
 
+  domains:
+    domains                custom domains dev management
+
   dev:
     pr                     fetch and checkout a pr
     lint                   run linters