Web edge kit (euler-caddy): Caddy, acme.sh, the Caddyfile router and the static web content.

1,147 lines of Bash Raw file Open on GitHub

#!/usr/bin/env bash
# Web edge kit (euler-caddy): Caddy, acme.sh, the Caddyfile router and the static web content.
# ============================================================================================
#
# Single orchestrator for the solver web edge: deploys / removes / upgrades
# Caddy (TLS + reverse-proxy edge) and the acme.sh cert client, creates the
# dedicated service identities, generates the Caddyfile router, deploys the static
# maintenance page, and installs the root-owned systemd unit. Supersedes the old
# scripts/setup/caddy.sh + scripts/setup/acme.sh. See docs/web-server-guide.md
# (Unix-socket upstreams, dedicated non-root service users, forward_auth routing).
#
# Topology (multi-tenant):
#   - Caddy terminates TLS on :443, serves a Caddy-native health endpoint
#     (`/healthz` -> 200) and the shared static assets (`/assets/*`), routes the
#     public auth surface (/login, /register*, /reset*, /forgot, /auth/*) to the
#     auth service's unix socket, and gates every other request through the auth
#     service's forward_auth endpoint (200 + X-User/X-Profile/X-User-Slug continues;
#     401 redirects to /login). Client-supplied copies of all three identity headers
#     are stripped. Authenticated requests are then routed **by X-User-Slug**
#     to that collaborator's own per-user instance: content pages AND /ws (the
#     terminal's WebSocket) both go to that collaborator's own /run/euler/user-<slug>.sock —
#     the slug in the dial address is a request-header placeholder, so one static
#     Caddyfile covers every provisioned user (verified on Caddy 2.11.4). A
#     missing slug falls through to the static "under maintenance" holding page
#     (HTTP 503), which a handle_errors block also serves when an upstream is down
#     (e.g. a deprovisioned or not-yet-provisioned user's socket).
#   - The cert is issued out-of-band by acme.sh via DNS-01 (Caddy performs no ACME).
#
# Service identity:
#   - group  `euler-web` — reaches auth.sock and msg.sock: the services, and the per-user
#     uids, whose shells need them.
#   - group  `euler-edge` — reaches the per-user instance sockets and site.sock, and owns
#     /run/euler: Caddy and the services that create or call those sockets, never a
#     per-user uid (docs/web-server-guide.md § 1).
#   - group  `euler-tls` — reads the TLS private key: Caddy alone.
#   - user   `euler-caddy` — runs the edge; primary group `euler-caddy` (which holds only
#     euler-acme, so the admin socket reaches no one else), plus euler-web, euler-edge and
#     euler-tls; binds :443 via CAP_NET_BIND_SERVICE. (euler-auth / the per-user uids /
#     euler-proxy are created by their own phases.)
#
# Config + secrets live under /etc/euler, NOT in the repo: the dedicated service users
# cannot traverse the repo owner's 0750 home dir, so the edge is decoupled from the
# checkout. No service runs as root: Caddy runs as euler-caddy, and acme.sh runs as
# euler-acme (home /usr/local/share/euler-acme), deploying the cert into a setgid
# /etc/euler/tls and reloading Caddy via its admin API (`caddy reload`) — a unix socket
# (/run/euler-caddy/admin.sock, 0660 euler-caddy:euler-caddy), never TCP. Renewal runs on
# euler-acme.timer. ~/.euler/env is the authoring source; the installer deploys scoped runtime
# config into /etc/euler.
#
#   /etc/euler/Caddyfile          root:euler-web       0644   (generated; non-secret)
#   /etc/euler/edge.env           root:euler-web       0644   (FQDN + email; runtime config)
#   /etc/euler/web-content/       root:euler-web       0755   (deployed static maintenance page)
#   /etc/euler/tls/               euler-acme:euler-tls 2750   (setgid)
#   /etc/euler/tls/server.crt     euler-acme:euler-tls 0644
#   /etc/euler/tls/server.key     euler-acme:euler-tls 0640   (readable by euler-caddy via group)
#   /run/euler-caddy/admin.sock   euler-caddy:euler-caddy 0660 (Caddy admin; euler-acme reloads)
#   euler-caddy.service · euler-acme.service + .timer   (root-owned, boot-enabled)
#
# The deployment FQDN is the single source of truth in the project env file ~/.euler/env
# (`EULER_TLS_DOMAIN=...`); install / upgrade / renew / status all read it from there
# and fail if it is unset. The Caddyfile is regenerated from it on install/upgrade.
#
# Because the unit lives in root's systemd and runs as a locked-down user, lifecycle
# (start/stop/restart) requires sudo.
#
# Actions: deploy | remove | upgrade | redeploy | status | renew | rekey | reload | help
#
# Author: Vikas Munshi <vikas.munshi@gmail.com>
# Copyright (c) 2026. All rights reserved.
# Licensed under the MIT License.

set -euo pipefail

# ── Paths, derived from this script's location so nothing is machine-specific ──────
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
PROJECT_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)"

# ── System locations (edge is decoupled from the repo — see header) ───────────────
SYS_DIR="/etc/euler"
CADDYFILE="${SYS_DIR}/Caddyfile"
TLS_DIR="${SYS_DIR}/tls"
CERT_FILE="${TLS_DIR}/server.crt"          # full chain
KEY_FILE="${TLS_DIR}/server.key"           # private key (readable by euler-tls: Caddy)

# ── Static web content (maintenance page + assets) ────────────────────────────────
# Authored in the repo, deployed under /etc/euler because euler-caddy cannot read the
# repo owner's 0750 home dir (the config-location rationale).
WEB_CONTENT_SRC="${PROJECT_ROOT}/packages/site/euler_site/content"   # repo source of truth
WEB_CONTENT_DIR="${SYS_DIR}/web-content"               # deployed, readable by euler-caddy
# The content paths the public site serves (docs/package-guide.md § 4) — signed-out reads,
# and stale-session reads, go there. One list for both Caddy matchers that name it.
PUBLIC_PATHS="/ /solutions /solutions/* /docs /docs/* /blogs /blogs/* /topics /topics/* /about/* /shell /story /story/*"

# ── Service identity ───────────────────────────────────────────────────────
WEB_GROUP="euler-web"
EDGE_GROUP="euler-edge"                    # per-user + site sockets, /run/euler (no tenant)
TLS_GROUP="euler-tls"                      # the private key (Caddy only)
CADDY_USER="euler-caddy"
CADDY_GROUP="euler-caddy"                  # Caddy's primary group: Caddy + euler-acme
# Caddy's admin API, on a unix socket in Caddy's own runtime dir. The |0660 sets the socket's
# mode; its group is Caddy's primary group, so only euler-acme (a member) can reload.
ADMIN_DIR="/run/euler-caddy"
ADMIN_ADDR="unix/${ADMIN_DIR}/admin.sock|0660"
SERVICE_NAME="euler-caddy.service"
SERVICE_DEST="/etc/systemd/system/${SERVICE_NAME}"

# ── Caddy apt repository (Cloudsmith), per https://caddyserver.com/docs/install ───
CADDY_GPG_URL="https://dl.cloudsmith.io/public/caddy/stable/gpg.key"
CADDY_REPO_URL="https://dl.cloudsmith.io/public/caddy/stable/debian.deb.txt"
CADDY_KEYRING="/usr/share/keyrings/caddy-stable-archive-keyring.gpg"
CADDY_SOURCES_LIST="/etc/apt/sources.list.d/caddy-stable.list"

# ── acme.sh, installed and run as the dedicated euler-acme user ─────────────
ACME_USER="euler-acme"
ACME_GROUP="euler-acme"
ACME_HOME="/usr/local/share/euler-acme"    # acme.sh working/config dir, owned by euler-acme
ACME_BIN="${ACME_HOME}/acme.sh"
ACME_INSTALL_URL="https://get.acme.sh"
ACME_SERVICE="euler-acme.service"          # oneshot: acme.sh --cron (renewal)
ACME_TIMER="euler-acme.timer"              # daily renewal trigger
ACME_SERVICE_DEST="/etc/systemd/system/${ACME_SERVICE}"
ACME_TIMER_DEST="/etc/systemd/system/${ACME_TIMER}"

# ── Config ────────────────────────────────────────────────────────────────────────
# The authoring env (~/.euler/env, possibly vault-encrypted): ENV_FILE +
# load_authoring_env, shared so every kit reads it one way.
# shellcheck source=scripts/setup/authoring_env.sh
. "${SCRIPT_DIR}/authoring_env.sh"
EDGE_ENV="${SYS_DIR}/edge.env"             # deployed runtime config (FQDN + email), non-secret
DOMAIN=""                                  # the deployment FQDN (load_fqdn)
ACME_EMAIL="${EULER_TLS_EMAIL:-vikas.munshi@gmail.com}"
DNS_PROVIDER="${EULER_TLS_DNS_PROVIDER:-namecom}"

usage() {
    cat <<USAGE
Usage: $0 [deploy|remove|upgrade|redeploy|status|renew|rekey|reload|help]

  deploy     Full edge setup: create euler-web group + euler-caddy user, install
             Caddy + acme.sh, generate /etc/euler/Caddyfile for the FQDN from ~/.euler/env
             (EULER_TLS_DOMAIN), issue+deploy the cert, and install the root-owned
             ${SERVICE_NAME} (boot-enabled). Idempotent.
  remove     Remove the unit and Caddy; prompt before deleting /etc/euler, acme.sh,
             and the service users/group.
  upgrade    Upgrade the Caddy + acme.sh packages themselves and regenerate the
             Caddyfile + unit, and re-register acme.sh's deploy/reload hook (no cert
             issuance) — unlike the other kits, this is NOT an alias of deploy.
  redeploy   Fast path: re-copy the static web-content (assets + vendor), regenerate
             the Caddyfile (picking up route changes), validate, and reload — no
             package upgrade, cert, or unit changes.
  status     Show deploy state, cert expiry, unit state, and a /healthz ping.
  renew      Force-renew the certificate now (as root; creds cached by acme.sh).
  rekey      Force-renew with a NEW private key, and keep rotating the key on every later
             renewal — for when the old key may have been read.
  reload     Reload the running edge (sudo systemctl reload; restarts it instead if
             the running unit's namespace can no longer be re-entered).

  The deployment FQDN is read from ~/.euler/env (EULER_TLS_DOMAIN); commands fail if unset.
  DNS provider (\$EULER_TLS_DNS_PROVIDER, default ${DNS_PROVIDER}): one of
  namecom, cloudflare, route53, godaddy, digitalocean, gandi.
USAGE
}

# ── Helpers ───────────────────────────────────────────────────────────────────────

# Verify sudo is available and the caller has privileges; else fail early.
check_can_sudo() {
    if ! command -v sudo &> /dev/null; then
        echo "Error: sudo is not installed or not found in PATH" >&2
        return 1
    fi
    if ! sudo -v 2>/dev/null; then
        echo "Error: current user does not have sudo privileges" >&2
        return 1
    fi
}

require_systemd() {
    if [ ! -d /run/systemd/system ] || ! command -v systemctl &> /dev/null; then
        echo "Error: systemd is required for the edge (root-owned units); it is not active here." >&2
        return 1
    fi
}

# Load the deployment FQDN (+ email) into $DOMAIN. Prefers the deployed runtime config
# /etc/euler/edge.env; falls back to the repo authoring source ~/.euler/env (for a first
# install). Fails if EULER_TLS_DOMAIN is set in neither.
load_fqdn() {
    local src=""
    if [ -r "${EDGE_ENV}" ]; then
        src="${EDGE_ENV}"
    elif [ -f "${ENV_FILE}" ]; then
        src="${ENV_FILE}"
    fi
    if [ -n "${src}" ]; then
        load_authoring_env "${src}" || return 1
    fi
    DOMAIN="${EULER_TLS_DOMAIN:-}"
    ACME_EMAIL="${EULER_TLS_EMAIL:-${ACME_EMAIL}}"
    if [ -z "${DOMAIN}" ]; then
        echo "Error: EULER_TLS_DOMAIN is not set in ${EDGE_ENV} or ${ENV_FILE}" >&2
        echo "       Add a line like:  EULER_TLS_DOMAIN=euler.example.com" >&2
        return 1
    fi
}

# What to write for PrivateTmp= in the unit.
#
# `PrivateTmp=true` binds a per-invocation /tmp/systemd-private-* directory into the
# service's namespace. The edge is long-lived, and `systemctl reload` re-enters that
# namespace to run ExecReload — so once the host has cleaned the directory away, every
# reload fails at 226/NAMESPACE on a service that is otherwise perfectly healthy, and a
# redeploy carrying a valid new Caddyfile dies with it. `disconnected` (systemd 257+)
# mounts an unshared tmpfs inside the namespace instead, with nothing outside to clean.
# Older systemd cannot parse the value, so fall back to the bind there; do_reload's
# restart fallback covers that case.
private_tmp_mode() {
    local ver
    ver="$(systemctl --version 2>/dev/null | awk 'NR==1 {print $2}')"
    ver="${ver%%[!0-9]*}"
    if [ -n "${ver}" ] && [ "${ver}" -ge 257 ]; then
        echo "disconnected"
    else
        echo "true"
    fi
}

caddy_is_installed() { command -v caddy &> /dev/null; }
caddy_version() { caddy version 2>/dev/null | head -n1 || echo "unknown"; }
caddy_bin() { command -v caddy; }

# ── Caddy (apt) ───────────────────────────────────────────────────────────────────

add_caddy_apt_repo() {
    if [ -f "${CADDY_KEYRING}" ] && [ -f "${CADDY_SOURCES_LIST}" ]; then
        echo "Caddy apt repository already configured"
        return 0
    fi
    echo "Configuring Caddy apt repository..."
    sudo apt-get install -y debian-keyring debian-archive-keyring apt-transport-https curl gpg
    curl -1sLf "${CADDY_GPG_URL}" | sudo gpg --dearmor -o "${CADDY_KEYRING}"
    curl -1sLf "${CADDY_REPO_URL}" | sudo tee "${CADDY_SOURCES_LIST}" > /dev/null
    sudo apt-get update
}

# Stop and disable any Caddy unit that would clash on :80/:443 with our own — the
# stock apt `caddy.service`, and the old repo-owner `caddy-euler.service` this script
# replaces. No-op when absent.
disable_conflicting_services() {
    local unit
    for unit in caddy.service caddy-euler.service; do
        if systemctl list-unit-files "${unit}" &> /dev/null; then
            echo "Disabling ${unit} (superseded by ${SERVICE_NAME})..."
            sudo systemctl disable --now "${unit}" 2>/dev/null || true
        fi
    done
    if [ -f "/etc/systemd/system/caddy-euler.service" ]; then
        sudo rm -f "/etc/systemd/system/caddy-euler.service"
        sudo systemctl daemon-reload
    fi
}

install_caddy_pkg() {
    if caddy_is_installed; then
        echo "Caddy already installed: $(caddy_version)"
    else
        add_caddy_apt_repo
        echo "Installing Caddy..."
        sudo apt-get install -y caddy
        echo "Caddy installed: $(caddy_version)"
    fi
    disable_conflicting_services
}

# ── Service identity + system dirs ─────────────────────────────────────────

ensure_group_and_users() {
    local group
    for group in "${WEB_GROUP}" "${EDGE_GROUP}" "${TLS_GROUP}" "${CADDY_GROUP}"; do
        if ! getent group "${group}" > /dev/null; then
            echo "Creating group ${group}..."
            sudo groupadd --system "${group}"
        fi
    done
    # euler-caddy — the edge. Its primary group is its own, so what it creates (the admin
    # socket) reaches only that group's members; euler-web / euler-edge / euler-tls are
    # supplementary. (euler-auth / the per-user uids / euler-proxy are created by their
    # own phases.)
    if ! getent passwd "${CADDY_USER}" > /dev/null; then
        echo "Creating system user ${CADDY_USER} (group ${CADDY_GROUP})..."
        sudo useradd --system --no-create-home --shell /usr/sbin/nologin \
            -g "${CADDY_GROUP}" -G "${WEB_GROUP},${EDGE_GROUP},${TLS_GROUP}" "${CADDY_USER}"
    else
        sudo usermod -g "${CADDY_GROUP}" -G "${WEB_GROUP},${EDGE_GROUP},${TLS_GROUP}" "${CADDY_USER}"
    fi
    # euler-acme — runs acme.sh, own group, home in /usr/local/share (not root).
    if ! getent group "${ACME_GROUP}" > /dev/null; then
        sudo groupadd --system "${ACME_GROUP}"
    fi
    if ! getent passwd "${ACME_USER}" > /dev/null; then
        echo "Creating system user ${ACME_USER}..."
        sudo useradd --system --home-dir "${ACME_HOME}" --shell /usr/sbin/nologin \
            -g "${ACME_GROUP}" "${ACME_USER}"
    fi
    # euler-acme reloads Caddy after a renewal, over the admin socket — Caddy's group.
    sudo usermod -aG "${CADDY_GROUP}" "${ACME_USER}"
}

ensure_sys_dirs() {
    # /etc/euler is 0755 so euler-acme (outside euler-web) can traverse to its tls dir;
    # the sensitive files inside carry their own perms. tls/ is setgid euler-tls (2750),
    # so certs euler-acme writes there inherit group euler-tls — Caddy's alone, never a
    # per-user uid's. The chgrp moves files a previous deploy left in the old group.
    sudo install -d -o root -g "${WEB_GROUP}" -m 0755 "${SYS_DIR}"
    sudo install -d -o "${ACME_USER}" -g "${TLS_GROUP}" -m 2750 "${TLS_DIR}"
    sudo find "${TLS_DIR}" -mindepth 1 -exec chgrp "${TLS_GROUP}" {} +
    sudo install -d -o "${ACME_USER}" -g "${ACME_GROUP}" -m 0750 "${ACME_HOME}"
}

# Deploy the non-secret runtime config the services read instead of the repo.
deploy_edge_env() {
    sudo tee "${EDGE_ENV}" > /dev/null <<EOF
# GENERATED by scripts/setup/frontend.sh — deployed runtime config (non-secret).
# Authoring source: ~/.euler/env. Read by frontend.sh (FQDN) and the euler services.
EULER_TLS_DOMAIN=${DOMAIN}
EULER_TLS_EMAIL=${ACME_EMAIL}
EOF
    sudo chown root:"${WEB_GROUP}" "${EDGE_ENV}"
    sudo chmod 0644 "${EDGE_ENV}"
}

# Deploy the static web content (maintenance page + assets) from the repo into
# /etc/euler/web-content, where euler-caddy can read it. Fully replaced each run so the
# deployed tree mirrors the repo source; non-secret, so root:euler-web 0644/0755.
deploy_web_content() {
    if [ ! -d "${WEB_CONTENT_SRC}" ]; then
        echo "Warning: ${WEB_CONTENT_SRC} not found; skipping static content deploy" >&2
        return 0
    fi
    echo "Deploying static web content to ${WEB_CONTENT_DIR}..."
    sudo rm -rf "${WEB_CONTENT_DIR}"
    sudo cp -r "${WEB_CONTENT_SRC}" "${WEB_CONTENT_DIR}"
    sudo chown -R root:"${WEB_GROUP}" "${WEB_CONTENT_DIR}"
    sudo find "${WEB_CONTENT_DIR}" -type d -exec chmod 0755 {} +
    sudo find "${WEB_CONTENT_DIR}" -type f -exec chmod 0644 {} +
}

# ── acme.sh (as root) ─────────────────────────────────────────────────────────────

acme_is_installed() { sudo test -x "${ACME_BIN}"; }

# Run acme.sh as the euler-acme user. acme.sh refuses to run under a bare `sudo`
# (it detects the SUDO_* env vars), so strip those markers and pin HOME to its working
# dir. For calls needing extra environment (the DNS credentials at issue time) use
# acme_run_with_creds, which prepends the $cred_env `VAR=val` pairs before the binary.
#
# LE_WORKING_DIR is pinned to ACME_HOME because we install with `--home ${ACME_HOME}`
# (everything directly in that dir, not the stock `$HOME/.acme.sh` subdir). Most
# subcommands re-derive it from the script's own location, but `--upgrade` falls back
# to `$HOME/.acme.sh` when it is unset — a dir that does not exist here, so the upgrade
# `cd`s nowhere, then tries to download the tarball into the launch CWD (which euler-acme
# cannot write) and fails with a "Permission denied". Setting it explicitly matches the
# install layout for every call.
ACME_CLEAN_ENV=(env -u SUDO_COMMAND -u SUDO_USER -u SUDO_UID -u SUDO_GID \
    "HOME=${ACME_HOME}" "LE_CONFIG_HOME=${ACME_HOME}" "LE_WORKING_DIR=${ACME_HOME}")
acme_run() {
    sudo -u "${ACME_USER}" "${ACME_CLEAN_ENV[@]}" "${ACME_BIN}" "$@"
}
acme_run_with_creds() {
    sudo -u "${ACME_USER}" "${ACME_CLEAN_ENV[@]}" "${cred_env[@]}" "${ACME_BIN}" "$@"
}

install_acme() {
    if ! command -v curl &> /dev/null; then
        echo "Error: curl is required to install acme.sh" >&2
        return 1
    fi
    if acme_is_installed; then
        echo "acme.sh already installed (${ACME_USER}): $(acme_run --version 2>/dev/null | tail -n1)"
    else
        echo "Installing acme.sh as ${ACME_USER} (into ${ACME_HOME})..."
        # Run the installer *from* ACME_HOME: its bootstrap downloads the tarball into the
        # CWD, and euler-acme cannot write to the repo the operator launched this from.
        # get.acme.sh treats its FIRST positional as an `email=` token (it prepends `--`),
        # so pass `email=...` first, then the install flags.
        sudo -u "${ACME_USER}" env "HOME=${ACME_HOME}" sh -c \
            "cd '${ACME_HOME}' && curl -fsSL '${ACME_INSTALL_URL}' | sh -s -- email='${ACME_EMAIL}' --home '${ACME_HOME}' --config-home '${ACME_HOME}' --nocron"
    fi
    acme_run --set-default-ca --server letsencrypt
    # Migrate off any prior root install so renewals don't run twice (clean-root env —
    # acme.sh refuses a bare sudo).
    if sudo test -x /root/.acme.sh/acme.sh; then
        echo "Removing the superseded root acme.sh cron..."
        sudo env -u SUDO_COMMAND -u SUDO_USER -u SUDO_UID -u SUDO_GID HOME=/root \
            /root/.acme.sh/acme.sh --uninstall 2>/dev/null || true
    fi
    echo "acme.sh ready (default CA: Let's Encrypt)"
}

# Resolve $DNS_PROVIDER to its acme.sh hook (into $DNS_HOOK) and the required credential
# variable names (into the $REQUIRED array), loading values from ~/.euler/env (read as the
# caller) or the environment. Errors on an unknown provider or missing credentials.
DNS_HOOK=""
REQUIRED=()
load_dns_creds() {
    if [ -f "${ENV_FILE}" ]; then
        load_authoring_env "${ENV_FILE}" || return 1
    fi
    case "${DNS_PROVIDER}" in
        namecom)
            DNS_HOOK="dns_namecom"
            : "${Namecom_Username:=${NAMEDOTCOM_USERNAME:-}}"
            : "${Namecom_Token:=${NAMEDOTCOM_TOKEN:-}}"
            REQUIRED=(Namecom_Username Namecom_Token) ;;
        cloudflare | cf)
            DNS_HOOK="dns_cf"; REQUIRED=(CF_Token CF_Account_ID) ;;
        route53 | aws)
            DNS_HOOK="dns_aws"; REQUIRED=(AWS_ACCESS_KEY_ID AWS_SECRET_ACCESS_KEY) ;;
        godaddy | gd)
            DNS_HOOK="dns_gd"; REQUIRED=(GD_Key GD_Secret) ;;
        digitalocean | do)
            DNS_HOOK="dns_dgon"; REQUIRED=(DO_API_KEY) ;;
        gandi)
            DNS_HOOK="dns_gandi_livedns"; REQUIRED=(GANDI_LIVEDNS_KEY) ;;
        *)
            echo "Error: unsupported DNS provider '${DNS_PROVIDER}'." >&2
            echo "Supported: namecom, cloudflare, route53, godaddy, digitalocean, gandi" >&2
            return 1 ;;
    esac
    local missing=() var
    for var in "${REQUIRED[@]}"; do
        [ -z "${!var:-}" ] && missing+=("${var}")
    done
    if [ "${#missing[@]}" -ne 0 ]; then
        echo "Error: missing ${DNS_PROVIDER} credentials: ${missing[*]}" >&2
        echo "Add them to ${ENV_FILE} (or export them)." >&2
        return 1
    fi
}

# Issue (DNS-01) + deploy the cert into /etc/euler/tls as root, registering the
# perms-fixing reload command. acme.sh caches the DNS creds for future renewals, so
# they are only supplied here at issue time.
issue_cert() {
    load_dns_creds || return 1
    local -a cred_env=() var
    for var in "${REQUIRED[@]}"; do cred_env+=("${var}=${!var}"); done

    echo "Issuing certificate for ${DOMAIN} via ${DNS_PROVIDER} DNS-01 (${DNS_HOOK})..."
    local rc=0
    acme_run_with_creds --issue --dns "${DNS_HOOK}" -d "${DOMAIN}" --server letsencrypt || rc=$?
    # exit code 2 = "skipped, still valid" — success for our purposes.
    if [ "${rc}" -ne 0 ] && [ "${rc}" -ne 2 ]; then
        echo "Error: certificate issuance failed (acme.sh exit ${rc})" >&2
        return "${rc}"
    fi

    # Clear any stale cert from a prior (root) deploy so euler-acme can write fresh files:
    # it owns the dir (can create/delete) but cannot overwrite root-owned files in place.
    sudo rm -f "${CERT_FILE}" "${KEY_FILE}"
    install_cert_hook
}

# Register (or re-register) how acme.sh deploys the cert into /etc/euler/tls and what it
# runs afterwards, and deploy the current cert now. acme.sh STORES this per domain, so it
# must be re-run whenever the reload path changes — `upgrade` does — or renewals keep
# running the old command.
#
# The reload command, re-run by euler-acme on every renewal: keep the key group-readable
# (the setgid dir gives the euler-tls group) and reload Caddy through its admin socket — no
# root, no systemctl. --force because a renewal changes only the files the Caddyfile names, never
# the Caddyfile itself, and Caddy skips a reload whose config text is unchanged — without it the
# new certificate is written but never served. Best effort: on first issue the edge is not up yet.
install_cert_hook() {
    local caddy reload_cmd
    caddy="$(caddy_bin)"
    reload_cmd="chmod 0640 ${KEY_FILE}; chmod 0644 ${CERT_FILE}; ${caddy} reload --config ${CADDYFILE} --address '${ADMIN_ADDR}' --force 2>/dev/null || true"
    echo "Deploying certificate to ${TLS_DIR}..."
    acme_run --install-cert -d "${DOMAIN}" \
        --fullchain-file "${CERT_FILE}" \
        --key-file "${KEY_FILE}" \
        --reloadcmd "${reload_cmd}"
    if ! sudo test -f "${CERT_FILE}" || ! sudo test -f "${KEY_FILE}"; then
        echo "Error: cert/key were not installed" >&2
        return 1
    fi
    # euler-acme owns them and the setgid dir gave them group euler-tls; just fix modes.
    sudo chmod 0644 "${CERT_FILE}"
    sudo chmod 0640 "${KEY_FILE}"
    echo "Deployed cert -> ${CERT_FILE} (${ACME_USER}:${TLS_GROUP} 0644)"
    echo "Deployed key  -> ${KEY_FILE} (${ACME_USER}:${TLS_GROUP} 0640)"
}

# ── Caddyfile router ──────────────────────────────────────────────────────────────

generate_caddyfile() {
    echo "Writing ${CADDYFILE} for ${DOMAIN}..."
    sudo tee "${CADDYFILE}" > /dev/null <<EOF
# Caddy configuration for the solver web edge (multi-tenant).
#
# GENERATED by scripts/setup/frontend.sh — edits are overwritten on install/upgrade.
# Caddy terminates TLS, serves the health endpoint + static assets, routes the
# public auth surface to the auth service (unix //run/euler/auth.sock), and gates
# every other request through the auth service's forward_auth endpoint:
# 200 + X-User/X-Profile/X-User-Slug continues downstream, 401 redirects to /login.
# Authenticated requests — content pages and the /ws terminal alike — are routed by
# X-User-Slug to that collaborator's own per-user instance socket
# /run/euler/user-<slug>.sock: the slug in the dial address is a request-header
# placeholder, so this one static file covers every provisioned user. A
# missing slug falls through to the static maintenance holding page (from
# ${WEB_CONTENT_DIR}), which also serves whenever an upstream is down. The cert is
# issued out-of-band by acme.sh (DNS-01) into ${TLS_DIR}; Caddy loads it via \`tls\`
# and performs no ACME itself.
{
	# DNS-01 needs no inbound :80, so keep Caddy off :80 (no HTTP->HTTPS redirect).
	auto_https disable_redirects
	# The admin API on a unix socket, never TCP: on loopback TCP any local process could
	# reconfigure the edge. Only Caddy's own group (euler-acme, for renewals) can connect.
	admin ${ADMIN_ADDR}
}

# Transport-level security headers + edge fallback CSP, shared by the live routes and
# the maintenance/error responses. The CSP uses the '?' (set-if-absent) operator so it
# is only a **fallback**: the app tier mints a per-response, nonce'd CSP with the
# exceptions its pages need (style-src 'unsafe-inline' for MathJax/CodeMirror injected
# styles; frame-ancestors 'self' for the terminal iframe — docs/web-server-guide.md
# § Content-Security-Policy), and Caddy leaves that untouched. This strict policy applies
# only to Caddy-native responses (maintenance / static / redirects) that carry no app CSP.
# (docs/web-server-guide.md § The edge.)
(security_headers) {
	header {
		Strict-Transport-Security "max-age=31536000; includeSubDomains"
		X-Content-Type-Options "nosniff"
		Referrer-Policy "no-referrer"
		?Content-Security-Policy "default-src 'self'; script-src 'self'; style-src 'self'; img-src 'self' data:; connect-src 'self'; frame-ancestors 'none'; base-uri 'none'; object-src 'none'"
		-Server
	}
}

# Cache-Control for the static trees: a request carrying ?v= names the bytes it gets
# (the templates hash each file into its URL), so it is kept for a year unasked; a bare
# path is revalidated before every use.
(asset_cache) {
	@versioned query v=*
	header @versioned Cache-Control "public, max-age=31536000, immutable"
	@unversioned not query v=*
	header @unversioned Cache-Control "no-cache"
}

# Reusable holding page: serve the static maintenance HTML with 503 Service
# Unavailable + Retry-After. Its stylesheet under /assets is served (200) by the
# route below, so the page renders styled even while the site is "down".
(maintenance) {
	root * ${WEB_CONTENT_DIR}
	rewrite * /maintenance.html
	file_server {
		status 503
	}
	header Content-Type "text/html; charset=utf-8"
	header Retry-After "3600"
	# A cached "we are down" outlives the outage: the next visit would render the
	# holding page from cache with the site already back. Never store it.
	header Cache-Control "no-store"
}

${DOMAIN} {
	tls ${CERT_FILE} ${KEY_FILE}
	import security_headers

	# Identity headers reach the app tier ONLY as copies of the forward_auth
	# response — anything a client sends is stripped before any routing happens.
	# X-User-Slug is the routing key: a forged copy would otherwise dial
	# another user's socket.
	request_header -X-User
	request_header -X-Profile
	request_header -X-User-Slug

	# Per-user routing: the matcher admits only a well-formed system
	# slug (packages/solver/solver/auth system_slug emits ^[a-z][a-z0-9-]*$) set by forward_auth —
	# defined at site scope, evaluated where referenced (after forward_auth has set
	# the header). Content and /ws share the one per-user instance, so no path split.
	@user header_regexp X-User-Slug ^[a-z][a-z0-9-]{1,30}$

	# The per-user instances' socket-peer plane (/internal/logout — the auth service's
	# teardown push; /internal/status — the operator's shell report). Answered here so
	# it is never proxied: those handlers trust their caller precisely because only
	# root and the euler-web tier can reach that user's socket, and the catch-all route
	# below would otherwise let any signed-in browser reach its own instance's copy.
	handle /internal/* {
		header Cache-Control "no-store"
		respond 404
	}

	# Health probe — Caddy-native, no upstream. no-store because a probe answers
	# "is the edge up *now*": a monitor that got its "ok" from a cache is not
	# monitoring anything.
	handle /healthz {
		header Cache-Control "no-store"
		respond "ok" 200
	}

	# Static assets (the auth pages' CSS/JS, srp.js, terms, maintenance styling),
	# served same-origin so the CSP 'self' policy covers them.
	#
	# Two rules, on whether the URL names its bytes. The templates write every asset
	# as /assets/site.css?v=<hash of its contents> (euler_site/assets.py): a redeploy
	# that changes the file changes the URL, so a versioned request is kept for a
	# year and never re-asked — immutable.
	#
	# A bare path (a script's own import, the maintenance page's stylesheet) names
	# different bytes after a redeploy: no-cache — "you may keep it, but ask before
	# you use it". Without it Caddy sends only ETag/Last-Modified, browsers fall back
	# to *heuristic* freshness, and a deploy leaves new HTML driven by old JS —
	# silently, and only for the users who happened to have it cached.
	handle /assets/* {
		root * ${WEB_CONTENT_DIR}
		import asset_cache
		file_server
	}

	# Vendored front-end libraries (htmx, MathJax + its fonts and the TeX extensions
	# its autoload fetches on first use) — Caddy-native,
	# same-origin, per the web-server-guide path-ownership table. Same two rules:
	# immutable when versioned, revalidated when bare (the editor's CodeMirror
	# modules, the fonts and extensions MathJax fetches). The SRI hashes in the
	# templates pin the *content*, so a stale vendor file is a blocked script rather
	# than a subtle bug.
	handle /vendor/* {
		root * ${WEB_CONTENT_DIR}
		import asset_cache
		file_server
	}

	# Public auth surface: the login/registration/reset/forgot/request-invite
	# pages and the JSON auth API, straight to the auth service. The shell-ticket
	# and admin endpoints are deliberately NOT routed (socket-peer / local-only).
	handle /login {
		reverse_proxy unix//run/euler/auth.sock
	}
	handle /register* {
		reverse_proxy unix//run/euler/auth.sock
	}
	handle /reset* {
		reverse_proxy unix//run/euler/auth.sock
	}
	handle /forgot {
		reverse_proxy unix//run/euler/auth.sock
	}
	handle /request-invite {
		reverse_proxy unix//run/euler/auth.sock
	}
	handle /password {
		reverse_proxy unix//run/euler/auth.sock
	}
	handle /terms {
		reverse_proxy unix//run/euler/auth.sock
	}
	handle /auth/* {
		reverse_proxy unix//run/euler/auth.sock
	}

	# Signed out, the published content is public (docs/package-guide.md § 4): a read
	# of the content paths from a browser carrying NO session cookie goes to the public
	# site (euler-site), which serves the packaged copy — no clone, no private tree, no
	# identity, read routes only. A browser with a session cookie skips this and goes
	# through forward_auth below, to its own instance — and if that cookie turns out to
	# be stale, the 401 there comes back here (see @stale_read). Nothing here is a trust
	# decision: the cookie only picks which copy of public content to show, and the
	# public site reads no identity header at all.
	# The code viewer (web-server-guide § The code viewer) is the public site's alone, for
	# every reader, signed in or not: it shows the deployed release's code, which is the
	# same for everyone, so there is one copy of it and no per-user instance serves it.
	@code_read {
		method GET HEAD
		path /code /code/*
	}
	handle @code_read {
		reverse_proxy unix//run/euler/site.sock
	}

	@public_read {
		method GET HEAD
		path ${PUBLIC_PATHS}
		not header_regexp Cookie (^|;\s*)solver_(session|remember)=
	}
	handle @public_read {
		reverse_proxy unix//run/euler/site.sock
	}

	# Everything else requires an authenticated session: the auth service
	# answers 200 + X-User/X-Profile/X-User-Slug (copied onto the proxied request)
	# or 401, which becomes a redirect to the login page. route{} preserves written
	# order — forward_auth runs first (setting the identity headers or 401→redirect),
	# then the session's own per-user instance proxies, else the holding page.
	handle {
		route {
			forward_auth unix//run/euler/auth.sock {
				uri /auth/check
				copy_headers X-User X-Profile X-User-Slug
				@unauthed status 401
				handle_response @unauthed {
					# A stale session cookie (expired, or dropped by an auth
					# restart) on a read of public content: the reader is signed
					# out, so they get what a signed-out reader gets — the public
					# page, not a login form in its place — and the dead cookie is
					# expired so the next request goes straight to the public site.
					# A remember-me cookie is the exception: it may still sign them
					# back in, which the login page tries first (login.js).
					@stale_read {
						method GET HEAD
						path ${PUBLIC_PATHS}
						not header_regexp Cookie (^|;\s*)solver_remember=
					}
					handle @stale_read {
						header +Set-Cookie "solver_session=; Max-Age=0; Path=/; Secure; HttpOnly; SameSite=Lax"
						reverse_proxy unix//run/euler/site.sock
					}
					handle {
						# "You are not signed in" is a statement about *this* moment.
						# A stored copy would keep sending a signed-in browser to the
						# login page; 302 is not heuristically cacheable, so this is
						# belt-and-braces — but the belt costs one header.
						header Cache-Control "no-store"
						# Back to this page once signed in (?next=; the auth service
						# validates it — euler_site/auth/returnto.py).
						redir * /login?next={http.request.uri.path} 302
					}
				}
			}

			# One per-user instance serves this collaborator's content pages
			# AND their /ws terminal — dial the socket named by the slug forward_auth
			# returned. Caddy proxies the WebSocket upgrade natively. The
			# instance re-checks the slug against X-User itself, so a routing bug
			# here is refused at the service too.
			reverse_proxy @user unix//run/euler/user-{http.request.header.X-User-Slug}.sock

			# No (well-formed) slug → holding page.
			import maintenance
		}
	}

	# When an upstream reverse_proxy fails — e.g. a not-yet-provisioned or
	# deprovisioned user's socket — serve the same holding page instead of a bare
	# 502/504.
	handle_errors {
		import maintenance
	}
}
EOF
    sudo chown root:"${WEB_GROUP}" "${CADDYFILE}"
    sudo chmod 0644 "${CADDYFILE}"          # non-secret; euler-acme reads it for `caddy reload`
}

# Validate the Caddyfile (as root, so it can read the 0640 key).
validate_caddyfile() {
    sudo "$(caddy_bin)" validate --config "${CADDYFILE}" &> /dev/null
}

# ── systemd unit (root-owned, boot-enabled) ─────────────────────────────────

install_service() {
    require_systemd || return 1
    if ! caddy_is_installed; then
        echo "Warning: caddy not installed; skipping ${SERVICE_NAME}" >&2
        return 0
    fi
    local caddy
    caddy="$(caddy_bin)"

    echo "Installing ${SERVICE_NAME} (User=${CADDY_USER}, Group=${CADDY_GROUP})..."
    sudo tee "${SERVICE_DEST}" > /dev/null <<EOF
[Unit]
Description=euler web edge (Caddy)
Documentation=https://github.com/vikasmunshi/euler/blob/master/docs/web-server-guide.md
After=network-online.target
Wants=network-online.target

[Service]
Type=notify
User=${CADDY_USER}
Group=${CADDY_GROUP}
SupplementaryGroups=${WEB_GROUP} ${EDGE_GROUP} ${TLS_GROUP}
ExecStart=${caddy} run --config ${CADDYFILE}
ExecReload=${caddy} reload --config ${CADDYFILE} --address '${ADMIN_ADDR}' --force
Restart=on-failure
RestartSec=5s
WorkingDirectory=${SYS_DIR}

# Bind :443 as a non-root user.
AmbientCapabilities=CAP_NET_BIND_SERVICE
CapabilityBoundingSet=CAP_NET_BIND_SERVICE

# The admin socket's dir (Caddy's own; euler-acme traverses it as a group member) + state
# (euler-caddy has no home). /run/euler itself is a tmpfiles.d dir (auth.sh), not Caddy's.
RuntimeDirectory=euler-caddy
RuntimeDirectoryMode=0750
StateDirectory=euler-caddy
Environment=XDG_DATA_HOME=/var/lib/euler-caddy
Environment=XDG_CONFIG_HOME=/var/lib/euler-caddy

# Hardening. Every write lands in the runtime and state dirs above, so the rest of the
# filesystem is read-only; it dials sockets (unix, and :443 over TCP/UDP) and nothing else.
NoNewPrivileges=true
ProtectHome=true
ProtectSystem=strict
PrivateTmp=$(private_tmp_mode)
PrivateDevices=true
ProtectKernelModules=true
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6 AF_NETLINK
ProtectKernelTunables=true
ProtectKernelLogs=true
ProtectControlGroups=true
ProtectClock=true
ProtectHostname=true
RestrictNamespaces=true
RestrictSUIDSGID=true
RestrictRealtime=true
LockPersonality=true
MemoryDenyWriteExecute=true
SystemCallArchitectures=native
SystemCallFilter=@system-service
SystemCallErrorNumber=EPERM

[Install]
WantedBy=multi-user.target
EOF
    sudo systemctl daemon-reload
    sudo systemctl enable "${SERVICE_NAME}" 2>/dev/null || true

    if sudo test -f "${CADDYFILE}" && sudo test -f "${CERT_FILE}" && validate_caddyfile; then
        if sudo systemctl restart "${SERVICE_NAME}"; then
            echo "${SERVICE_NAME} started"
        else
            echo "Warning: ${SERVICE_NAME} failed to start; check: systemctl status ${SERVICE_NAME}" >&2
        fi
    else
        echo "${SERVICE_NAME} enabled but not started: Caddyfile/cert missing or invalid."
        echo "  Fix the cert, then: $0 status / sudo systemctl start ${SERVICE_NAME}"
    fi
}

remove_service() {
    if [ -f "${SERVICE_DEST}" ]; then
        echo "Removing ${SERVICE_NAME}..."
        sudo systemctl disable --now "${SERVICE_NAME}" 2>/dev/null || true
        sudo rm -f "${SERVICE_DEST}"
        sudo systemctl daemon-reload
    fi
}

# Root-owned systemd timer that runs `acme.sh --cron` as euler-acme — replaces
# acme.sh's own user crontab (installed with --nocron).
install_acme_timer() {
    require_systemd || return 1
    echo "Installing ${ACME_TIMER} (daily renewal as ${ACME_USER})..."
    sudo tee "${ACME_SERVICE_DEST}" > /dev/null <<EOF
[Unit]
Description=euler certificate renewal (acme.sh --cron)
Documentation=https://github.com/vikasmunshi/euler/blob/master/docs/web-server-guide.md
After=network-online.target
Wants=network-online.target

[Service]
Type=oneshot
User=${ACME_USER}
Group=${ACME_GROUP}
Environment=HOME=${ACME_HOME}
Environment=LE_CONFIG_HOME=${ACME_HOME}
ExecStart=${ACME_BIN} --cron --home ${ACME_HOME} --config-home ${ACME_HOME}

# Hardening. acme.sh writes its own home and installs the renewed pair into the TLS dir;
# the reload hook only dials Caddy's admin socket. Everything else is read-only.
NoNewPrivileges=true
ProtectSystem=strict
ReadWritePaths=${ACME_HOME} ${TLS_DIR}
ProtectHome=true
PrivateTmp=true
PrivateDevices=true
ProtectKernelModules=true
CapabilityBoundingSet=
RestrictAddressFamilies=AF_UNIX AF_INET AF_INET6
ProtectKernelTunables=true
ProtectKernelLogs=true
ProtectControlGroups=true
ProtectClock=true
ProtectHostname=true
RestrictNamespaces=true
RestrictSUIDSGID=true
RestrictRealtime=true
LockPersonality=true
MemoryDenyWriteExecute=true
SystemCallArchitectures=native
SystemCallFilter=@system-service
SystemCallErrorNumber=EPERM
EOF
    sudo tee "${ACME_TIMER_DEST}" > /dev/null <<EOF
[Unit]
Description=Daily euler certificate renewal check

[Timer]
OnCalendar=daily
RandomizedDelaySec=1h
Persistent=true

[Install]
WantedBy=timers.target
EOF
    sudo systemctl daemon-reload
    sudo systemctl enable --now "${ACME_TIMER}"
}

remove_acme_units() {
    local u
    for u in "${ACME_TIMER}" "${ACME_SERVICE}"; do
        if [ -f "/etc/systemd/system/${u}" ]; then
            sudo systemctl disable --now "${u}" 2>/dev/null || true
            sudo rm -f "/etc/systemd/system/${u}"
        fi
    done
    sudo systemctl daemon-reload
}

# ── Actions ───────────────────────────────────────────────────────────────────────

do_deploy() {
    check_can_sudo || return 1
    require_systemd || return 1
    load_fqdn || return 1

    install_caddy_pkg
    ensure_group_and_users
    ensure_sys_dirs
    deploy_edge_env
    deploy_web_content
    generate_caddyfile
    install_acme
    issue_cert
    install_service
    install_acme_timer
    echo "Frontend edge setup complete for ${DOMAIN}."
    do_status || true
}

do_upgrade() {
    check_can_sudo || return 1
    require_systemd || return 1
    load_fqdn || return 1

    if caddy_is_installed; then
        echo "Upgrading Caddy..."
        sudo apt-get update
        sudo apt-get install -y --only-upgrade caddy
    else
        echo "Caddy not installed; installing..."
        install_caddy_pkg
    fi
    disable_conflicting_services
    if acme_is_installed; then
        echo "Upgrading acme.sh..."
        acme_run --upgrade || true
    fi
    ensure_group_and_users
    ensure_sys_dirs
    deploy_edge_env
    deploy_web_content                 # refresh the static maintenance page + assets
    generate_caddyfile                 # regenerate from the FQDN
    install_service                    # re-lay the unit and reload
    install_cert_hook                  # re-register the reload path acme.sh stores per domain
    install_acme_timer
    echo "Frontend upgrade complete: $(caddy_version)"
}

# Replace the TLS private key, not just the certificate: a forced renewal with a freshly
# generated domain key, deployed by the stored hook. acme.sh keeps the flag for this domain,
# so every later renewal rotates the key too — no key outlives its certificate.
do_rekey() {
    check_can_sudo || return 1
    load_fqdn || return 1
    if ! acme_is_installed; then
        echo "Error: acme.sh is not installed; run '$0 deploy' first." >&2
        return 1
    fi
    local before after
    before="$(sudo openssl pkey -in "${KEY_FILE}" -pubout 2>/dev/null | sha256sum | cut -c1-16)"
    echo "Renewing ${DOMAIN} with a new private key (force)..."
    acme_run --renew -d "${DOMAIN}" --force --always-force-new-domain-key
    after="$(sudo openssl pkey -in "${KEY_FILE}" -pubout 2>/dev/null | sha256sum | cut -c1-16)"
    echo "Public key fingerprint: ${before:-?} → ${after:-?}"
    if [ -z "${after}" ] || [ "${before}" = "${after}" ]; then
        echo "Error: the deployed key did not change" >&2
        return 1
    fi
    echo "Rekey complete for ${DOMAIN}"
}

do_renew() {
    check_can_sudo || return 1
    load_fqdn || return 1
    if ! acme_is_installed; then
        echo "Error: acme.sh is not installed (root); run '$0 deploy' first." >&2
        return 1
    fi
    echo "Renewing certificate for ${DOMAIN} (force)..."
    acme_run --renew -d "${DOMAIN}" --force
    echo "Renewal complete for ${DOMAIN}"
}

# Reload the running edge, falling back to a restart.
#
# `systemctl reload` runs ExecReload *inside the running main process's namespace*, so
# it fails with 226/NAMESPACE when the private /tmp that namespace was set up with has
# since been cleaned away from the host — nothing is wrong with the Caddyfile (it has
# already validated by the time we get here), the long-lived unit's namespace is simply
# no longer re-enterable. A restart lays a fresh namespace, so take one rather than
# leave a freshly generated Caddyfile unserved.
do_reload() {
    check_can_sudo || return 1
    if sudo systemctl reload "${SERVICE_NAME}" 2> /dev/null; then
        echo "Reloaded ${SERVICE_NAME}"
        return 0
    fi
    echo "Reload of ${SERVICE_NAME} failed (stale unit namespace); restarting instead..."
    sudo systemctl restart "${SERVICE_NAME}"
    echo "Restarted ${SERVICE_NAME}"
}

# Fast redeploy: push the repo's static web-content + regenerated Caddyfile to the
# running edge, then reload. No package upgrade, cert issuance, or unit re-lay — the
# common "I changed CSS/JS/vendor or a Caddy route" turnaround.
do_redeploy() {
    check_can_sudo || return 1
    load_fqdn || return 1
    if ! sudo test -f "${CERT_FILE}"; then
        echo "Error: edge not installed (no cert at ${CERT_FILE}); run '$0 deploy' first." >&2
        return 1
    fi
    deploy_web_content
    generate_caddyfile
    if ! validate_caddyfile; then
        echo "Error: generated Caddyfile failed validation; not reloading." >&2
        return 1
    fi
    do_reload
    echo "Frontend redeploy complete: web-content + Caddyfile refreshed and reloaded."
}

do_remove() {
    check_can_sudo || return 1
    remove_service
    remove_acme_units

    if dpkg-query -W -f='${Status}' caddy 2>/dev/null | grep -q "install ok installed"; then
        echo "Uninstalling Caddy..."
        sudo apt-get --purge remove -y caddy
        sudo apt-get autoremove -y
    fi
    [ -f "${CADDY_SOURCES_LIST}" ] && sudo rm -f "${CADDY_SOURCES_LIST}"
    [ -f "${CADDY_KEYRING}" ] && sudo rm -f "${CADDY_KEYRING}"

    local reply
    read -r -p "Remove ${SYS_DIR} (Caddyfile + deployed certs + edge.env)? [y/N] " reply
    [[ "${reply}" =~ ^[Yy]$ ]] && sudo rm -rf "${SYS_DIR}"

    read -r -p "Remove the acme.sh client (${ACME_HOME})? [y/N] " reply
    [[ "${reply}" =~ ^[Yy]$ ]] && sudo rm -rf "${ACME_HOME}"

    read -r -p "Remove service users (${CADDY_USER}, ${ACME_USER}) and their groups? [y/N] " reply
    if [[ "${reply}" =~ ^[Yy]$ ]]; then
        local u g
        for u in "${CADDY_USER}" "${ACME_USER}"; do
            if getent passwd "${u}" > /dev/null; then sudo userdel "${u}" 2>/dev/null || true; fi
        done
        for g in "${ACME_GROUP}" "${CADDY_GROUP}" "${TLS_GROUP}" "${EDGE_GROUP}" "${WEB_GROUP}"; do
            if getent group "${g}" > /dev/null; then sudo groupdel "${g}" 2>/dev/null || true; fi
        done
        sudo rm -rf /var/lib/euler-caddy
    fi
    echo "Frontend uninstall complete."
}

do_status() {
    load_fqdn || return 1
    if caddy_is_installed; then
        echo "Caddy:      ✓ installed ($(caddy_bin)) — $(caddy_version)"
    else
        echo "Caddy:      ✗ not installed"
    fi
    if acme_is_installed; then
        echo "acme.sh:    ✓ installed (${ACME_USER}) — $(acme_run --version 2>/dev/null | tail -n1)"
    else
        echo "acme.sh:    ✗ not installed"
    fi
    if sudo test -f "${CERT_FILE}"; then
        if command -v openssl &> /dev/null; then
            echo "Cert (${CERT_FILE}):"
            sudo openssl x509 -in "${CERT_FILE}" -noout -subject -enddate 2>/dev/null | sed 's/^/  /'
        else
            echo "Cert:       present (install openssl to show expiry)"
        fi
    else
        echo "Cert:       ✗ not deployed (run '$0 deploy')"
    fi
    # Auto-renewal: euler-acme.timer runs `acme.sh --cron` as euler-acme daily;
    # show its state and the next scheduled renewal (ARI window before expiry).
    if [ -f "${ACME_TIMER_DEST}" ] && command -v systemctl &> /dev/null; then
        local next
        next="$(acme_run --info -d "${DOMAIN}" 2>/dev/null | sed -n 's/^Le_NextRenewTimeStr=//p' | tr -d "\"'")"
        echo "Renewal:    ${ACME_TIMER} $(systemctl is-active "${ACME_TIMER}" 2>/dev/null)/$(systemctl is-enabled "${ACME_TIMER}" 2>/dev/null)${next:+ — next: ${next}}"
    else
        echo "Renewal:    ✗ ${ACME_TIMER} not installed"
    fi
    if command -v systemctl &> /dev/null && [ -f "${SERVICE_DEST}" ]; then
        echo "${SERVICE_NAME}: $(systemctl is-active "${SERVICE_NAME}" 2>/dev/null)/$(systemctl is-enabled "${SERVICE_NAME}" 2>/dev/null)"
    else
        echo "${SERVICE_NAME}: ✗ not installed"
    fi
    # The admin API must be the unix socket alone: a TCP listener on :2019 is reachable by
    # every local process, the per-user web shells included.
    if ss -Htln 'sport = :2019' 2>/dev/null | grep -q .; then
        echo "admin API:  ✗ listening on TCP :2019 — expected only ${ADMIN_DIR}/admin.sock" >&2
    elif sudo test -S "${ADMIN_DIR}/admin.sock"; then
        echo "admin API:  ✓ unix socket only ($(sudo stat -c '%a %U:%G' "${ADMIN_DIR}/admin.sock"))"
    else
        echo "admin API:  … no socket at ${ADMIN_DIR}/admin.sock (edge down?)"
    fi
    # Health ping (resolve the domain to loopback so it works without public DNS).
    # curl's -w prints 000 on failure; capture it (|| true) instead of appending another.
    local code
    code="$(curl -sS -o /dev/null -w '%{http_code}' --max-time 5 \
        --resolve "${DOMAIN}:443:127.0.0.1" "https://${DOMAIN}/healthz" 2>/dev/null || true)"
    echo "/healthz:   HTTP ${code:-000} (expect 200)"
}

# ── Dispatch ──────────────────────────────────────────────────────────────────────
ACTION="${1:-status}"
case "${ACTION}" in
    deploy)    do_deploy ;;
    remove)    do_remove ;;
    upgrade)   do_upgrade ;;
    redeploy)  do_redeploy ;;
    renew)     do_renew ;;
    rekey)     do_rekey ;;
    reload)    do_reload ;;
    status)    do_status ;;
    -h | --help | help) usage ;;
    *) echo "Unknown action: ${ACTION}"; usage; exit 1 ;;
esac