Merge branch 'tests'

.drone.yml

@@ -0,0 +1,235 @@
+kind: pipeline
+type: exec
+name: default
+
+clone:
+  disable: true
+
+steps:
+- name: deploy
+  environment:
+    PASSWORD:
+      from_secret: HETZNER_TOKEN
+    DOMAIN:
+      from_secret: DOMAIN
+    CLOUDFLARE_TOKEN:
+      from_secret: CLOUDFLARE_TOKEN
+    USER_PASS:
+      from_secret: USER_PASS
+    INFECT_COMMIT_SHA: ${DRONE_COMMIT_SHA}
+
+  commands:
+  - set -o nounset
+  - > # TODO pass Base64 encoded password from Drone instead of this
+    ENCODED_PASSWORD="$(base64 <<<"$USER_PASS")"
+  # Create infect user script and then push it to a remote machine on server creation.
+  - |
+    cat << EOF > infect.sh
+    #! /usr/bin/env bash
+
+    set -o allexport
+    set -o errtrace
+    set -o pipefail
+    shopt -s inherit_errexit
+    trap 'echo \$LINENO: "\$BASH_COMMAND"; exit 1' ERR
+
+    API_TOKEN="$USER_PASS"
+    CONFIG_URL=https://git.selfprivacy.org/api/v1/repos/SelfPrivacy/selfprivacy-nixos-template/archive/4d18c8dba9765bde9040ddf1c285bec96ba23f43.tar.gz
+    DNS_PROVIDER_TOKEN=$CLOUDFLARE_TOKEN
+    DNS_PROVIDER_TYPE=CLOUDFLARE
+    DOMAIN=$DOMAIN
+    ENCODED_PASSWORD="$ENCODED_PASSWORD"
+    HOSTNAME=selfprivacy-ci-test
+    LUSER=cicdcicd
+    NIXOS_CONFIG_ID=default
+    NIX_VERSION=2.18.1
+    PROVIDER=hetzner
+    SSH_AUTHORIZED_KEY="ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIMBb3yVhYF4slhf1iQCiGLOVcbGKP/MmkQiEMl2un+4K"
+    STAGING_ACME=true
+
+    curl --fail https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect/raw/commit/$INFECT_COMMIT_SHA/nixos-infect \
+    | bash 2>&1 | tee /root/infect.log
+    EOF
+  - cat infect.sh
+  - location=fsn1
+  - set -o pipefail
+  - >
+    volume_id="$(http -d --print=HBb --ignore-stdin POST https://api.hetzner.cloud/v1/volumes Authorization:"Bearer $PASSWORD" format=ext4 size:=10 location=$location name=10gb-by-drone-ci automount:=false | jq -re .volume.id)"
+  # TODO pass linux device volume_id name to infect.sh script
+  - >
+    http -d --print=HBb --ignore-stdin POST https://api.hetzner.cloud/v1/servers Authorization:"Bearer $PASSWORD" name=ci-sibling server_type=cx11 start_after_create:=true image=ubuntu-20.04 volumes:=[$volume_id] user_data=@infect.sh automount:=false location=$location
+
+- name: dns
+  environment:
+    PASSWORD:
+      from_secret: HETZNER_TOKEN
+    CLOUDFLARE_TOKEN:
+      from_secret: CLOUDFLARE_TOKEN
+    DOMAIN:
+      from_secret: DOMAIN
+    ZONE_ID:
+      from_secret: ZONE_ID
+
+  commands:
+  # Read all available machines and export IP of 'ci-sibling'
+  # Create DNS entires for main domain and subdomains
+  - >
+    curl -s -H "Authorization: Bearer $PASSWORD" 'https://api.hetzner.cloud/v1/servers' > .machine.json
+  - >
+    export machineip=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].public_net.ipv4.ip else null end' .machine.json; done | grep -v null | sed 's/"//' | sed 's/"//' )
+  - echo $machineip
+  - >
+    curl -s -X GET "https://api.cloudflare.com/client/v4/zones?name=$DOMAIN"
+    -H "Authorization: Bearer $CLOUDFLARE_TOKEN"
+    -H "Content-Type: application/json"
+  - >
+    curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records"
+    -H "Authorization: Bearer $CLOUDFLARE_TOKEN"
+    -H "Content-Type: application/json"
+    --data '{"type":"A","name":"@","content":"'"$$machineip"'","ttl":3600,"priority":10,"proxied":false}'
+  - >
+    http --check-status --ignore-stdin POST https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records Authorization:"Bearer $CLOUDFLARE_TOKEN" type=A name=api content=$machineip ttl:=3600 priority:=10 proxied:=false
+  - >
+    http --check-status --ignore-stdin POST https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records Authorization:"Bearer $CLOUDFLARE_TOKEN" type=A name=cloud content=$machineip ttl:=3600 priority:=10 proxied:=false
+  - >
+    http --check-status --ignore-stdin POST https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records Authorization:"Bearer $CLOUDFLARE_TOKEN" type=A name=password content=$machineip ttl:=3600 priority:=10 proxied:=false
+  - >
+    curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records"
+    -H "Authorization: Bearer $CLOUDFLARE_TOKEN"
+    -H "Content-Type: application/json"
+    --data '{"type":"MX","name":"@","content":"'"$$DOMAIN"'","ttl":3600,"priority":10,"proxied":false}'
+  - >
+    curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records"
+    -H "Authorization: Bearer $CLOUDFLARE_TOKEN"
+    -H "Content-Type: application/json"
+    --data '{"type":"TXT","name":"_dmarc","content":"v=DMARC1; p=none","ttl":18000,"priority":10,"proxied":false}'
+  - >
+    curl -s -X POST "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records"
+    -H "Authorization: Bearer $CLOUDFLARE_TOKEN"
+    -H "Content-Type: application/json"
+    --data "{\"type\":\"TXT\",\"name\":\"$DOMAIN\",\"content\":\"v=spf1 a mx ip4:$machineip -all\",\"ttl\":18000,\"priority\":10,\"proxied\":false}"
+
+- name: wait for NixOS boot
+  environment:
+    DOMAIN:
+      from_secret: DOMAIN
+    TIMEOUT: 160
+  commands:
+  - sleep 300
+  - t0="$(date '+%s')"; until nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w1 "$DOMAIN" 443; do sleep 10; cur_date="$(date '+%s')"; remaining=$((t0 + TIMEOUT - cur_date)); [ "$remaining" -gt 0 ]; echo "$remaining seconds remain before teardown"; done
+  - echo "give time for obtaining ACME certificate"
+  - sleep 60
+
+- name: basic test
+  environment:
+    DOMAIN:
+      from_secret: DOMAIN
+    USER_PASS:
+      from_secret: USER_PASS
+    TEST_EMAIL_PASS:
+      from_secret: TEST_EMAIL_PASS
+    API_TOKEN:
+      from_secret: USER_PASS
+    TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git
+    TIMEOUT_1: 100
+  commands:
+  # ideally it should depend on $STAGING_ACME
+  # CERT_VERIFY=$(if [[ $STAGING_ACME == true ]]; then echo "--cacert root.pem"; else echo ""; fi)
+  - CERT_VERIFY="--cacert root.pem"
+  - source $(nix build $TESTS_REPO#library --print-out-paths)
+  - curl https://letsencrypt.org/certs/staging/letsencrypt-stg-root-x1.pem -sS --fail -o root.pem
+  - dig api.$DOMAIN
+  - >
+    retry $TIMEOUT_1 10 curl https://api.$DOMAIN/services/status -sS --fail-with-body $CERT_VERIFY -H "Authorization: Bearer $USER_PASS"
+  - swaks --to ci@$DOMAIN -s selfprivacy.org -tls -au test@selfprivacy.org -ap $TEST_EMAIL_PASS -ahp
+  - >
+    curl https://cloud.$DOMAIN/ocs/v2.php/apps/serverinfo/api/v1/info?format=json -sS --fail-with-body $CERT_VERIFY -u "admin:$USER_PASS" | jq -re '.ocs.meta.status == "ok"'
+  - echo 'File contents to test nextcloud upload' > file
+  - >
+    curl -X PUT https://cloud.$DOMAIN/remote.php/dav/files/admin/file -sS --fail-with-body $CERT_VERIFY -d @file -u "admin:$USER_PASS"
+  - >
+    curl https://cloud.$DOMAIN/remote.php/dav/files/admin/file -sS --fail-with-body $CERT_VERIFY -u "admin:$USER_PASS"
+
+- name: API & nixos-rebuild test
+  environment:
+    DOMAIN:
+      from_secret: DOMAIN
+    API_TOKEN:
+      from_secret: USER_PASS
+    TESTS_REPO: git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git
+    TIMEOUT_1: 100
+  commands:
+  - source $(nix build $TESTS_REPO#library --print-out-paths)
+  - retry $TIMEOUT_1 10 query-minimum-services
+  - enable-service bitwarden
+  - enable-service gitea
+  - enable-service jitsi-meet
+  - enable-service ocserv
+  - enable-service pleroma
+  - nixos-rebuild Rebuild
+  - sleep 100
+  - retry $TIMEOUT_1 10 query-all-services
+  - nixos-rebuild Rollback
+  - sleep 30
+  - retry $TIMEOUT_1 10 query-minimum-services
+  - enable-service bitwarden
+  - enable-service gitea
+  - enable-service jitsi-meet
+  - enable-service ocserv
+  - enable-service pleroma
+  - nixos-rebuild Rebuild
+  - sleep 60
+  - retry $TIMEOUT_1 10 query-all-services
+
+- name: sleep after failure
+  environment:
+    DOMAIN:
+      from_secret: DOMAIN
+    TIMEOUT: 1500
+  commands:
+    - timer=0; while [ "$timer" -lt "$TIMEOUT" ]; do nix shell --inputs-from git+https://git.selfprivacy.org/alexoundos/selfprivacy-tests.git nixpkgs#netcat -c nc -zv -w4 "$DOMAIN" 22; echo "$((TIMEOUT - timer)) seconds remain before teardown - login via ssh to shutdown the machine"; ((timer+=10)); sleep 10; done
+  when:
+    status:
+      - failure
+
+- name: teardown
+  environment:
+    PASSWORD:
+      from_secret: HETZNER_TOKEN
+    CLOUDFLARE_TOKEN:
+      from_secret: CLOUDFLARE_TOKEN
+    DOMAIN:
+      from_secret: DOMAIN
+    ZONE_ID:
+      from_secret: ZONE_ID
+  commands:
+  # Delete our machine called 'ci-sibling' and all the found DNS records on it.
+  - >
+    curl -s "https://api.hetzner.cloud/v1/servers" -H "Authorization: Bearer $PASSWORD" > .hetzner_servers.json
+  - >
+    export machineid=$( for i in {0..24}; do jq 'if .servers['$i'].name == "ci-sibling" then .servers['$i'].id else null end' .hetzner_servers.json; done | grep -v null | sed 's/"//g')
+  - >
+    curl -s -X DELETE "https://api.hetzner.cloud/v1/servers/$machineid" -H "Authorization: Bearer $PASSWORD"
+  - >
+    json=$(curl -X GET "https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records" -H "Authorization: Bearer $CLOUDFLARE_TOKEN" -H "Content-Type: application/json")
+  - >
+    echo "$json" > .dns-records.json
+  - >
+    result=$(jq -r ".result" <<< "$json")
+  - >
+    echo "$ZONE_ID"
+  - >
+    echo "$result" | jq '.[] | "\(.id)"' | while read id ; do echo "Deleting $id"; http -v --check-status --ignore-stdin DELETE https://api.cloudflare.com/client/v4/zones/$ZONE_ID/dns_records/$(echo $id | cut -f 2 -d '"') Authorization:"Bearer $CLOUDFLARE_TOKEN"; done
+  - location="fsn1"
+  - volume_name="10gb-by-drone-ci"
+  - >
+    volumes_json="$(http -d --print=HBb --ignore-stdin GET https://api.hetzner.cloud/v1/volumes Authorization:"Bearer $PASSWORD" name=$volume_name)"
+    # as of 2023-12-27 Hetzner ignores "name" query parameter (and others too)!!
+  - >
+    volume_id="$(jq <<<"$volumes_json" -re "(.volumes[] | select(.name == \"$volume_name\")).id")"
+  - >
+    http -d --print=HBb --ignore-stdin DELETE https://api.hetzner.cloud/v1/volumes/$volume_id Authorization:"Bearer $PASSWORD"
+  when:
+    status:
+    - failure
+    - success

nixos-infect

@@ -1,124 +1,266 @@
 #! /usr/bin/env bash
 
-# More info at: https://github.com/elitak/nixos-infect
+# More info at:
+# - https://github.com/elitak/nixos-infect
+# - https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-infect
 
-set -e -o pipefail
+# mandatory variables
+: "${API_TOKEN:?API_TOKEN variable is not set}"
+: "${CONFIG_URL:?CONFIG_URL variable is not set}"
+: "${DNS_PROVIDER_TOKEN:?DNS_PROVIDER_TOKEN variable is not set}"
+: "${DNS_PROVIDER_TYPE:?DNS_PROVIDER_TYPE variable is not set}"
+: "${DOMAIN:?DOMAIN variable is not set}"
+: "${ENCODED_PASSWORD:?ENCODED_PASSWORD variable is not set}"
+: "${HOSTNAME:?HOSTNAME variable is not set}"
+: "${LUSER:?LUSER variable is not set}"
+: "${NIX_VERSION:?NIX_VERSION variable is not set}"
+: "${PROVIDER:?PROVIDER variable is not set}"
+: "${STAGING_ACME:?STAGING_ACME variable is not set}"
 
-makeConf() {
+# optional variables
-  # Skip everything if main config already present
+: "${NIXOS_CONFIG_ID:=default}"
-  [[ -e /etc/nixos/configuration.nix ]] && return 0
+: "${SSH_AUTHORIZED_KEY:=}"
-  if [[ $PASSWORD == null ]]; then
+: "${VOLUME_DEV_PATH:=}"
-      export PASSWORD=$(printf $ENCODED_PASSWORD | base64 --decode)
+
+readonly NL=$'\n'
+readonly LOCAL_FLAKE_DIR="/etc/nixos"
+readonly SECRETS_FILEPATH="/etc/selfprivacy/secrets.json"
+readonly NIX="/root/.nix-profile/bin/nix"
+readonly NIX_OPTS=(--extra-experimental-features "nix-command flakes")
+DoNetConf=
+
+genOptionalSsh() {
+  [ -n "${SSH_AUTHORIZED_KEY}" ] && cat << EOF
+"ssh": { "rootKeys": [ "${SSH_AUTHORIZED_KEY}" ] },
+EOF
+}
+
+# TODO receive disk device path from outside
+determine2ndDisk() {
+  local volDevPath rootFsDev
+  if [ -n "$VOLUME_DEV_PATH" ]; then
+      volDevPath="$VOLUME_DEV_PATH"
+  else
+      case "$PROVIDER" in
+          hetzner)
+              volDevPath="/dev/sdb"
+              ;;
+          digitalocean)
+              volDevPath="/dev/sda"
+              ;;
+          *)
+              return 1
+              ;;
+      esac
   fi
+  rootFsDev="$(findmnt -no source -T /)"
+  [[ -b "$volDevPath" && "$volDevPath" != "$rootFsDev" ]] && echo "$volDevPath"
+}
 
-  export ESCAPED_PASSWORD=$(printf $ENCODED_PASSWORD | base64 --decode | jq -Rs .)
+# Merge original userdata.json with deployment specific fields and print result.
-  export HASHED_PASSWORD=$( mkpasswd -m sha-512 "$PASSWORD" )
+genUserdata() {
+  local hashed_password diskDeviceName userdata_infect
+  hashed_password="$(mkpasswd -m sha-512 "$USER_PASS")"
+  diskDevice="$(determine2ndDisk)"
+  diskDeviceName="${diskDevice##/dev}"
 
-  # NB <<"EOF" quotes / $ ` in heredocs, <<EOF does not
+  userdata_infect=$(cat << EOF
-  mkdir -p /etc/nixos
-
-  git clone https://git.selfprivacy.org/SelfPrivacy/selfprivacy-nixos-config.git /etc/nixos
-
-  # Prevent grep for sending error code 1 (and halting execution) when no lines are selected : https://www.unix.com/man-page/posix/1P/grep
-  local IFS=$'\n'
-  for trypath in /root/.ssh/authorized_keys /home/$SUDO_USER/.ssh/authorized_keys $HOME/.ssh/authorized_keys; do
-      [[ -r "$trypath" ]] \
-      && keys=$(sed -E 's/^.*((ssh|ecdsa)-[^[:space:]]+)[[:space:]]+([^[:space:]]+)([[:space:]]*.*)$/\1 \3\4/' "$trypath") \
-      && break
-  done
-  local network_import=""
-
-  cat > /etc/nixos/userdata/userdata.json << EOF
 {
-    "api": {
+    $(genOptionalSsh)
-        "token": "$API_TOKEN"
+    "dns": {
+        "provider": "$DNS_PROVIDER_TYPE",
+        "useStagingACME": $STAGING_ACME
     },
-    "backblaze": {
+    "server": {
-        "accountId": "$BACKBLAZE_KEY_ID",
+        "provider": "$PROVIDER"
-        "accountKey": "$BACKBLAZE_ACCOUNT_KEY",
-        "bucket": "$BACKBLAZE_BUCKET_NAME"
     },
-    "bitwarden": {
-        "enable": true
-    },
-    "cloudflare": {
-        "apiKey": "$CF_TOKEN"
-    },
-    "databasePassword": "$DB_PASSWORD",
     "domain": "$DOMAIN",
-    "hashedMasterPassword": "$HASHED_PASSWORD",
+    "hashedMasterPassword": "$hashed_password",
     "hostname": "$HOSTNAME",
-    "nextcloud": {
+    "username": "$LUSER",
-        "enable": true,
+    "volumes": [
-        "adminPassword": $ESCAPED_PASSWORD,
+        {
-        "databasePassword": $ESCAPED_PASSWORD
+            "device": "$diskDevice",
-    },
+            "mountPoint": "/volumes/$diskDeviceName",
-    "gitea": {
+            "fsType": "ext4"
-        "enable": true
+        }
-    },
+    ],
-    "jitsi": {
+    "modules": {
-        "enable": true
+        "bitwarden": {
-    },
+            "location": "$diskDeviceName"
-    "ocserv": {
+        },
-        "enable": true
+        "gitea": {
-    },
+            "location": "$diskDeviceName"
-    "pleroma": {
+        },
-        "enable": true
+        "nextcloud": {
-    },
+            "location": "$diskDeviceName"
-    "timezone": "Europe/Uzhgorod",
+        },
-    "resticPassword": $ESCAPED_PASSWORD,
+        "pleroma": {
-    "username": "$LUSER"
+            "location": "$diskDeviceName"
+        },
+        "simple-nixos-mailserver": {
+            "location": "$diskDeviceName"
+        }
+    }
 }
 EOF
-  chmod 0600 /etc/nixos/userdata/userdata.json
+)
 
-  if isEFI; then
+  jq -s '.[0] * .[1]' \
+  "${1:?no userdata.json given to merge with}" <(printf "%s" "$userdata_infect")
+}
+
+genSecrets() {
+  local dbpass
+  dbpass="$(shuf --random-source=/dev/urandom -erz -n32 {A..Z} {a..z} {0..9} | tr -d '\n')"
+
+  cat << EOF
+{
+    "api": {
+        "token": "$API_TOKEN",
+        "skippedMigrations": ["migrate_to_selfprivacy_channel", "mount_volume"]
+    },
+    "databasePassword": "$dbpass",
+    "dns": {
+        "apiKey": "$DNS_PROVIDER_TOKEN"
+    },
+    "modules": {
+        "nextcloud": {
+            "adminPassword": "$USER_PASS",
+            "databasePassword": "$USER_PASS"
+        }
+    }
+}
+EOF
+}
+
+genHardwareConfiguration() {
+  local bootcfg
+  if ((isEFI)); then
     bootcfg=$(cat << EOF
   boot.loader.grub = {
     efiSupport = true;
     efiInstallAsRemovable = true;
     device = "nodev";
   };
-  fileSystems."/boot" = { device = "$esp"; fsType = "vfat"; };
+  fileSystems."/boot" = { device = "$ESP"; fsType = "vfat"; };
 EOF
 )
   else
     bootcfg=$(cat << EOF
-  boot.loader.grub.device = "$grubdev";
+  boot.loader.grub.device = "$GRUBDEV";
 EOF
 )
   fi
 
-  # If you rerun this later, be sure to prune the filesSystems attr
+  local currentSystem
-  cat > /etc/nixos/hardware-configuration.nix << EOF
+  if ! currentSystem="$($NIX "${NIX_OPTS[@]}" eval --impure --raw --expr builtins.currentSystem)"
+  then
+      echo "cannot determine Nix currentSystem identifier"
+      return 1
+  fi
+
+  # FIXME it's questionable whether these modules are needed at all...
+  declare -a availableKernelModules=()
+
+  [ "$PROVIDER" == "digitalocean" ] \
+  && availableKernelModules+=('"ata_piix"' '"uhci_hcd"' '"xen_blkfront"')
+
+  [ "$(uname -m)" == "x86_64" ] \
+  && availableKernelModules+=('"vmw_pvscsi"')
+
+  # TODO try nixos-generate-config first, resorting to the way below if failed
+  # FIXME "nvme" is not needed for hetzner?
+
+  cat << EOF
 { modulesPath, ... }:
 {
   imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
 $bootcfg
   boot.initrd.kernelModules = [ "nvme" ];
-  fileSystems."/" = { device = "$rootfsdev"; fsType = "$rootfstype"; };
+  boot.initrd.availableKernelModules = [ ${availableKernelModules[@]} ];
+  fileSystems."/" = { device = "$ROOTFSDEV"; fsType = "$ROOTFSTYPE"; };
+  nixpkgs.hostPlatform = "${currentSystem}";
 }
 EOF
-
-  [[ -n "$doNetConf" ]] && makeNetworkingConf || true
 }
 
-makeNetworkingConf() {
+genDeploymentConfiguration() {
-  # XXX It'd be better if we used procfs for all this...
+  local release
-  local IFS=$'\n'
-  eth0_name=$(ip address show | grep '^2:' | awk -F': ' '{print $2}')
-  eth0_ip4s=$(ip address show dev "$eth0_name" | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')
-  eth0_ip6s=$(ip address show dev "$eth0_name" | grep 'inet6 ' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|' || '')
-  gateway=$(ip route show dev "$eth0_name" | grep default | sed -r 's|default via ([0-9.]+).*|\1|')
-  gateway6=$(ip -6 route show dev "$eth0_name" | grep default | sed -r 's|default via ([0-9a-f:]+).*|\1|' || true)
-  ether0=$(ip address show dev "$eth0_name" | grep link/ether | sed -r 's|.*link/ether ([0-9a-f:]+) .*|\1|')
 
-  eth1_name=$(ip address show | grep '^3:' | awk -F': ' '{print $2}')||true
+  if ! release="$($NIX "${NIX_OPTS[@]}" eval --impure --raw --expr "(builtins.getFlake (builtins.toString ${LOCAL_FLAKE_DIR})).inputs.selfprivacy-nixos-config.inputs.nixpkgs.lib.trivial.release")"
-  if [ -n "$eth1_name" ];then
+  then
-    eth1_ip4s=$(ip address show dev "$eth1_name" | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')
+      echo "cannot determine NixOS release version"
-    eth1_ip6s=$(ip address show dev "$eth1_name" | grep 'inet6 ' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|' || '')
+      return 1
-    ether1=$(ip address show dev "$eth1_name" | grep link/ether | sed -r 's|.*link/ether ([0-9a-f:]+) .*|\1|')
+  fi
-    interfaces1=<< EOF
+
+  cat << EOF
+{ lib, ... }: {
+  # The content below is static and belongs to this deployment only!
+  # Do not copy this configuration file to another NixOS installation!
+
+  system.stateVersion = lib.mkDefault "$release";`
+`$(if [ "$DoNetConf" == "y" ]; then echo -e "$NL"; genNetworkingConf; fi)
+}
+EOF
+}
+
+setupConf() {
+  mkdir -p ${LOCAL_FLAKE_DIR}
+  if ! curl --fail "${CONFIG_URL}" \
+  | tar -xz -C ${LOCAL_FLAKE_DIR} --strip-components=1 --exclude=".*"
+  then
+      echo "Error downloading/extracting top level flake configuration!"
+      exit 1
+  fi
+
+  # generate and write hardware-configuration.nix
+  if ! genHardwareConfiguration > ${LOCAL_FLAKE_DIR}/hardware-configuration.nix
+  then
+      echo "error generating ${LOCAL_FLAKE_DIR}/hardware-configuration.nix"
+      exit 1
+  fi
+
+  # generate and write deployment.nix
+  if ! genDeploymentConfiguration > ${LOCAL_FLAKE_DIR}/deployment.nix
+  then
+      echo "error generating ${LOCAL_FLAKE_DIR}/deployment.nix"
+      exit 1
+  fi
+
+  # generate infected userdata based on original
+  local userdataInfected
+  userdataInfected="$(genUserdata ${LOCAL_FLAKE_DIR}/userdata.json)"
+  printf "%s" "$userdataInfected" > ${LOCAL_FLAKE_DIR}/userdata.json
+
+  # generate and write secrets
+  local secrets
+  secrets="$(genSecrets)"
+  install -m0600 <(printf "%s" "$secrets") -DT ${SECRETS_FILEPATH}
+}
+
+# shellcheck disable=SC2207
+genNetworkingConf() {
+  # XXX It'd be better if we used procfs for all this...
+
+  local IFS=$'\n'
+  local eth0_name eth0_ip4s eth0_ip6s gateway gateway6 ether0 eth1_name
+  local interfaces1 extraRules1 predictable_inames
+
+  eth0_name="$(ip address show | grep '^2:' | awk -F': ' '{print $2}')"
+  eth0_ip4s=($(ip address show dev "$eth0_name" | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|'))
+  eth0_ip6s=($(ip address show dev "$eth0_name" | grep 'inet6 ' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')) || true
+  gateway="$(ip route show dev "$eth0_name" | grep default | sed -r 's|default via ([0-9.]+).*|\1|')"
+  gateway6="$(ip -6 route show dev "$eth0_name" | grep default | sed -r 's|default via ([0-9a-f:]+).*|\1|')" || true
+  ether0="$(ip address show dev "$eth0_name" | grep link/ether | sed -r 's|.*link/ether ([0-9a-f:]+) .*|\1|')"
+
+  eth1_name="$(ip address show | grep '^3:' | awk -F': ' '{print $2}')" || true
+  if [ -n "$eth1_name" ]; then
+      local eth1_ip4s eth1_ip6s ether1
+      eth1_ip4s="$(ip address show dev "$eth1_name" | grep 'inet ' | sed -r 's|.*inet ([0-9.]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')"
+      eth1_ip6s="$(ip address show dev "$eth1_name" | grep 'inet6 ' | sed -r 's|.*inet6 ([0-9a-f:]+)/([0-9]+).*|{ address="\1"; prefixLength=\2; }|')" || true
+      ether1="$(ip address show dev "$eth1_name" | grep link/ether | sed -r 's|.*link/ether ([0-9a-f:]+) .*|\1|')"
+      interfaces1=$(cat << EOF
       $eth1_name = {
         ipv4.addresses = [$(for a in "${eth1_ip4s[@]}"; do echo -n "
           $a"; done)
@@ -126,31 +268,30 @@ makeNetworkingConf() {
         ipv6.addresses = [$(for a in "${eth1_ip6s[@]}"; do echo -n "
           $a"; done)
         ];
+      };
 EOF
-    extraRules1="ATTR{address}==\"${ether1}\", NAME=\"${eth1_name}\""
+)
+      extraRules1="ATTR{address}==\"${ether1}\", NAME=\"${eth1_name}\""
   else
-    interfaces1=""
+      interfaces1=""
-    extraRules1=""
+      extraRules1=""
   fi
 
-  readarray nameservers < <(grep ^nameserver /etc/resolv.conf | sed -r \
-    -e 's/^nameserver[[:space:]]+([0-9.a-fA-F:]+).*/"\1"/' \
-    -e 's/127[0-9.]+/9.9.9.9/' \
-    -e 's/::1/9.9.9.9/' )
-
   if [[ "$eth0_name" = eth* ]]; then
-    predictable_inames="usePredictableInterfaceNames = lib.mkForce false;"
+      predictable_inames="usePredictableInterfaceNames = lib.mkForce false;"
   else
-    predictable_inames="usePredictableInterfaceNames = lib.mkForce true;"
+      predictable_inames="usePredictableInterfaceNames = lib.mkForce true;"
   fi
-  cat > /etc/nixos/networking.nix << EOF
+
-{ lib, ... }: {
+  local defaultGateway6=${gateway6:+defaultGateway6 = \{ address = "${gateway6}"; interface = "${eth0_name}"; \};}
-  # This file was populated at runtime with the networking
+  local ipv6routes=${gateway6:+ipv6.routes = \[ \{ address = "${gateway6}"; prefixLength = 128; \} \];}
-  # details gathered from the active system.
+  cat << EOF
+  # Networking configuration was populated by nixos-infect with the networking
+  # details gathered from the running system.
   networking = {
-    nameservers = [ ${nameservers[@]} ];
+    defaultGateway = "${gateway}";`
-    defaultGateway = "${gateway}";
+    `${defaultGateway6:+
-    defaultGateway6 = "${gateway6}";
+    defaultGateway6}
     dhcpcd.enable = false;
     $predictable_inames
     interfaces = {
@@ -161,17 +302,19 @@ EOF
         ipv6.addresses = [$(for a in "${eth0_ip6s[@]}"; do echo -n "
           $a"; done)
         ];
-        ipv4.routes = [ { address = "${gateway}"; prefixLength = 32; } ];
+        ipv4.routes = [ { address = "${gateway}"; prefixLength = 32; } ];`
-        ipv6.routes = [ { address = "${gateway6}"; prefixLength = 128; } ];
+        `${ipv6routes:+
-      };
+        $ipv6routes}
-      $interfaces1
+      };`
+`${interfaces1:+
+$interfaces1}
     };
   };
   services.udev.extraRules = ''
-    ATTR{address}=="${ether0}", NAME="${eth0_name}"
+    ATTR{address}=="${ether0}", NAME="${eth0_name}"`
-    $extraRules1
+    `${extraRules1:+
+    $extraRules1}
   '';
-}
 EOF
 }
 
@@ -189,12 +332,8 @@ removeSwap() {
   rm -vf /tmp/nixos-infect.*.swp
 }
 
-isEFI() {
-  [ -d /sys/firmware/efi ]
-}
-
 findESP() {
-  esp=""
+  local esp
   for d in /boot/EFI /boot/efi /boot; do
     [[ ! -d "$d" ]] && continue
     [[ "$d" == "$(df "$d" --output=target | sed 1d)" ]] \
@@ -203,22 +342,32 @@ findESP() {
   done
   [[ -z "$esp" ]] && { echo "ERROR: No ESP mount point found"; return 1; }
   for uuid in /dev/disk/by-uuid/*; do
-    [[ $(readlink -f "$uuid") == "$esp" ]] && echo $uuid && return 0
+    [[ $(readlink -f "$uuid") == "$esp" ]] && echo "$uuid" && return 0
   done
 }
 
 prepareEnv() {
-  # $esp and $grubdev are used in makeConf()
+  if ! USER_PASS="$(base64 -d <<<"$ENCODED_PASSWORD")"; then
-  if isEFI; then
+      echo "Error decoding ENCODED_PASSWORD from Base64!"
-    esp="$(findESP)"
+      exit 1
+  fi
+  readonly USER_PASS
+
+  isEFI=0
+  [ -d /sys/firmware/efi ] && isEFI=1
+
+  if ((isEFI)); then
+    ESP="$(findESP)"
   else
-    for grubdev in /dev/vda /dev/sda /dev/nvme0n1 ; do [[ -e $grubdev ]] && break; done
+    for GRUBDEV in /dev/vda /dev/sda /dev/nvme0n1; do
+        [[ -e $GRUBDEV ]] && break;
+    done
   fi
 
   # Retrieve root fs block device
   #                   (get root mount)  (get partition or logical volume)
-  rootfsdev=$(mount | grep "on / type" | awk '{print $1;}')
+  ROOTFSDEV=$(mount | grep "on / type" | awk '{print $1;}')
-  rootfstype=$(df $rootfsdev --output=fstype | sed 1d)
+  ROOTFSTYPE=$(df "$ROOTFSDEV" --output=fstype | sed 1d)
 
   # DigitalOcean doesn't seem to set USER while running user data
   export USER="root"
@@ -268,9 +417,6 @@ checkEnv() {
   # Perform some easy fixups before checking
   # TODO prevent multiple calls to apt-get update
   (which dnf && dnf install -y perl-Digest-SHA) || true # Fedora 24
-  which bzcat || (which yum && yum install -y bzip2) \
-              || (which apt-get && apt-get update && apt-get install -y bzip2) \
-              || true
   which xzcat || (which yum && yum install -y xz-utils) \
               || (which apt-get && apt-get update && apt-get install -y xz-utils) \
               || true
@@ -279,78 +425,125 @@ checkEnv() {
               || true
 
   req curl || req wget || { echo "ERROR: Missing both curl and wget";  return 1; }
-  req bzcat            || { echo "ERROR: Missing bzcat";               return 1; }
   req xzcat            || { echo "ERROR: Missing xzcat";               return 1; }
-  req groupadd         || { echo "ERROR: Missing groupadd";            return 1; }
-  req useradd          || { echo "ERROR: Missing useradd";             return 1; }
-  req ip               || { echo "ERROR: Missing ip";                  return 1; }
   req awk              || { echo "ERROR: Missing awk";                 return 1; }
   req cut || req df    || { echo "ERROR: Missing coreutils (cut, df)"; return 1; }
+  req mkpasswd         || { echo "ERROR: Missing mkpasswd";            return 1; }
+  req shuf             || { echo "ERROR: Missing shuf";                return 1; }
+}
+
+# Download and execute the nix installer script.
+
+installNix() {
+  # install multiuser (system-wide with nix-daemon) Nix in the current system
+
+  local nixReleaseBase='https://releases.nixos.org'
+  local installURL="${nixReleaseBase}/nix/nix-${NIX_VERSION}/install"
+  local shaURL="${installURL}.sha256"
+  local sha tmpNixInstall
+
+  # temporary destination for install script
+  tmpNixInstall="$(mktemp -t nix-install-XXXXXXXXXX)"
+  if [[ ! -f "${tmpNixInstall}" ]]; then
+      echo "Failed creating a temporary file for Nix install script!"
+      return 1
+  fi
+
+  echo "Downloading install script from ${installURL}..."
+  if ! curl --fail "${installURL}" -o "${tmpNixInstall}" &>/dev/null; then
+      echo "Failure while downloading Nix install script!"
+      return 1
+  fi
+
+  if ! sha="$(curl --fail "${shaURL}")"; then
+      echo "Failure while downloading Nix install script sha!"
+      return 1
+  fi
+
+  echo "Validating Nix install script checksum..."
+  if ! echo "${sha}  ${tmpNixInstall}" | sha256sum -c; then
+      echo "Checksum validation failed!"
+      return 1
+  fi
+
+  echo "Running nix installer..."
+  if $SHELL "${tmpNixInstall}" \
+    --daemon --no-channel-add --daemon-user-count 4; then
+      echo "Nix is installed"
+      rm "${tmpNixInstall}"
+  else
+      echo "Nix installation script failed!"
+      return 1
+  fi
 }
 
 infect() {
-  # Add nix build users
+  # this is needed solely for accepting the sp-module sub-flake
-  # FIXME run only if necessary, rather than defaulting true
+  # see https://github.com/NixOS/nix/issues/3978#issuecomment-952418478
-  groupadd nixbld -g 30000 || true
+  cd /etc/nixos || return 1
-  for i in {1..10}; do
+  $NIX "${NIX_OPTS[@]}" flake lock ${LOCAL_FLAKE_DIR} \
-    useradd -c "Nix build user $i" -d /var/empty -g nixbld -G nixbld -M -N -r -s "$(which nologin)" "nixbld$i" || true
+  --override-input sp-modules path:./sp-modules
-  done
-  # TODO use addgroup and adduser as fallbacks
-  #addgroup nixbld -g 30000 || true
-  #for i in {1..10}; do adduser -DH -G nixbld nixbld$i || true; done
 
-  curl -L https://nixos.org/nix/install | $SHELL
+  echo "nix build the configuration flake..."
+  if ! $NIX "${NIX_OPTS[@]}" build \
+  --profile /nix/var/nix/profiles/system \
+  ${LOCAL_FLAKE_DIR}/#nixosConfigurations."$NIXOS_CONFIG_ID".config.system.build.toplevel
+  then
+      echo "Failed!"
+      exit 1
+  fi
 
-  # shellcheck disable=SC1090
+  # Reify resolv.conf (???)
-  source ~/.nix-profile/etc/profile.d/nix.sh
-
-  nix-channel --remove nixpkgs
-  nix-channel --add "https://channel.selfprivacy.org/nixos-selfpricacy" nixos
-  nix-channel --update
-
-  export NIXOS_CONFIG=/etc/nixos/configuration.nix
-
-  nix-env --set \
-    -I nixpkgs=$HOME/.nix-defexpr/channels/nixos \
-    -f '<nixpkgs/nixos>' \
-    -p /nix/var/nix/profiles/system \
-    -A system
-
-  # Remove nix installed with curl | bash
-  rm -fv /nix/var/nix/profiles/default*
-  /nix/var/nix/profiles/system/sw/bin/nix-collect-garbage
-
-  # Reify resolv.conf
   [[ -L /etc/resolv.conf ]] && mv -v /etc/resolv.conf /etc/resolv.conf.lnk && cat /etc/resolv.conf.lnk > /etc/resolv.conf
 
   # Stage the Nix coup d'état
   touch /etc/NIXOS
-  echo etc/nixos                   > /etc/NIXOS_LUSTRATE
+  echo etc/nixos            > /etc/NIXOS_LUSTRATE
-  echo etc/resolv.conf            >> /etc/NIXOS_LUSTRATE
+  echo etc/resolv.conf     >> /etc/NIXOS_LUSTRATE
-  echo root/.nix-defexpr/channels >> /etc/NIXOS_LUSTRATE
+  echo ${SECRETS_FILEPATH} >> /etc/NIXOS_LUSTRATE
 
   rm -rf /boot.bak
-  isEFI && umount "$esp"
+  ((isEFI)) && umount "$ESP"
   mv -v /boot /boot.bak
-  if isEFI; then
+  if ((isEFI)); then
     mkdir /boot
-    mount "$esp" /boot
+    mount "$ESP" /boot
     find /boot -depth ! -path /boot -exec rm -rf {} +
   fi
-  /nix/var/nix/profiles/system/bin/switch-to-configuration boot
+
+  echo "make configuration boot by default..."
+  if ! /nix/var/nix/profiles/system/bin/switch-to-configuration boot; then
+      echo "Failed!"; exit 1
+  fi
+
+  # Remove nix installed by the "install" script.
+  rm -fv /nix/var/nix/profiles/default*
+  /nix/var/nix/profiles/system/sw/bin/nix-collect-garbage
 }
 
-[ "$PROVIDER" = "digitalocean" ] && doNetConf=y # digitalocean requires detailed network config to be generated
+set -o errtrace
+set -o nounset
+set -o pipefail
+set -o xtrace
+shopt -s inherit_errexit
+trap 'echo ${LINENO}: "$BASH_COMMAND"; exit 1' ERR
+
+genNetworkingConf
+
+# digitalocean requires detailed network config to be generated
+[ "$PROVIDER" == "digitalocean" ] && DoNetConf="y"
 
 apt update
-apt install -y git tar wget curl whois jq
+apt install -y git tar curl whois jq
+
 checkEnv
 prepareEnv
 makeSwap # smallest (512MB) droplet needs extra memory!
-makeConf
+installNix
+setupConf
 infect
 removeSwap
 
-if [[ -z "$NO_REBOOT" ]]; then
+if [[ -z "${NO_REBOOT+x}" ]]; then
   reboot
 fi