fix: roll out rotated traefik certificate secrets

nix/hosts/cc-ci-hetzner/configuration.nix

@@ -14,6 +14,7 @@
     ./networking.nix
     ../../modules/packages.nix
     ../../modules/secrets.nix
+    ../../modules/acme-dns.nix
     ../../modules/swarm.nix
     ../../modules/docker-prune.nix
     ../../modules/abra.nix

nix/modules/acme-dns.nix

@@ -0,0 +1,155 @@
+# Restricted DNS-01 certificate issuance for ci.commoninternet.net.
+#
+# This host is authoritative only for acme.commoninternet.net. Gandi continues
+# to own commoninternet.net; it delegates this narrow zone and one permanent
+# _acme-challenge CNAME manually. No Gandi credential is present here.
+{ pkgs, ... }:
+let
+  acmeDnsConfig = pkgs.writeText "cc-ci-acme-dns.conf" ''
+    [general]
+    listen = "91.98.47.73:53"
+    protocol = "both4"
+    domain = "acme.commoninternet.net"
+    nsname = "ns-acme.commoninternet.net"
+    nsadmin = "hostmaster.commoninternet.net"
+    records = [
+      "acme.commoninternet.net. NS ns-acme.commoninternet.net.",
+      "ns-acme.commoninternet.net. A 91.98.47.73",
+    ]
+    debug = false
+
+    [database]
+    # acme-dns 2.x registers the embedded driver under `sqlite` (not the
+    # legacy `sqlite3` identifier).
+    engine = "sqlite"
+    connection = "/var/lib/acme-dns/acme-dns.db"
+
+    [api]
+    ip = "127.0.0.1"
+    port = "8080"
+    tls = "none"
+    # The one Lego account was bootstrapped before this configuration was
+    # hardened. Updates authenticated by that account remain available.
+    disable_registration = true
+    corsorigins = []
+
+    [logconfig]
+    loglevel = "info"
+    logtype = "stdout"
+    logformat = "json"
+  '';
+
+  # These are wiring values only. The acme-dns account JSON is generated by
+  # Lego below /var/lib/acme and never enters Nix, git, or /etc.
+  legoEnvironment = pkgs.writeText "cc-ci-acme-dns-lego.env" ''
+    ACME_DNS_API_BASE=http://127.0.0.1:8080
+    ACME_DNS_STORAGE_PATH=/var/lib/acme/ci.commoninternet.net/acme-dns-accounts.json
+    ACME_DNS_ALLOWLIST=127.0.0.1/32
+  '';
+in
+{
+  users.groups.acme-dns = { };
+  users.users.acme-dns = {
+    isSystemUser = true;
+    group = "acme-dns";
+    home = "/var/lib/acme-dns";
+  };
+
+  environment.etc."acme-dns/lego.env".source = legoEnvironment;
+
+  # The staging order has completed successfully. This marker permits the
+  # production ACME post-run hook to hand a renewed certificate to Traefik.
+  systemd.tmpfiles.rules = [
+    "f /var/lib/ci-certs/acme-production-enabled 0600 root root -"
+  ];
+
+  networking.firewall = {
+    allowedTCPPorts = [ 53 ];
+    allowedUDPPorts = [ 53 ];
+  };
+
+  systemd.services.acme-dns = {
+    description = "Restricted authoritative DNS for cc-ci ACME DNS-01";
+    wantedBy = [ "multi-user.target" ];
+    after = [ "network-online.target" ];
+    wants = [ "network-online.target" ];
+    serviceConfig = {
+      User = "acme-dns";
+      Group = "acme-dns";
+      StateDirectory = "acme-dns";
+      StateDirectoryMode = "0700";
+      WorkingDirectory = "/var/lib/acme-dns";
+      ExecStart = "${pkgs.acme-dns}/bin/acme-dns -c ${acmeDnsConfig}";
+      Restart = "on-failure";
+      RestartSec = "5s";
+      AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
+      CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
+      NoNewPrivileges = true;
+      PrivateTmp = true;
+      PrivateDevices = true;
+      ProtectHome = true;
+      ProtectSystem = "strict";
+      ReadWritePaths = [ "/var/lib/acme-dns" ];
+      RestrictAddressFamilies = [ "AF_INET" "AF_UNIX" ];
+    };
+  };
+
+  # Traefik consumes its wildcard as immutable Swarm secrets, so a renewed
+  # host certificate must be copied and reconciled rather than merely reloaded.
+  # This service is started only by the production-mode ACME postRun hook.
+  systemd.services.cc-ci-acme-traefik-handoff = {
+    description = "Install renewed cc-ci wildcard into Traefik Swarm secrets";
+    after = [ "docker.service" "deploy-proxy.service" ];
+    requires = [ "docker.service" ];
+    path = [ pkgs.coreutils pkgs.docker pkgs.systemd pkgs.gnugrep ];
+    serviceConfig = {
+      Type = "oneshot";
+      UMask = "0077";
+    };
+    script = ''
+      src=/var/lib/acme/ci.commoninternet.net
+      dst=/var/lib/ci-certs/live
+      test -s "$src/fullchain.pem"
+      test -s "$src/key.pem"
+      install -d -m 0700 "$dst"
+      install -m 0444 "$src/fullchain.pem" "$dst/fullchain.pem.new"
+      install -m 0400 "$src/key.pem" "$dst/privkey.pem.new"
+      mv -f "$dst/fullchain.pem.new" "$dst/fullchain.pem"
+      mv -f "$dst/privkey.pem.new" "$dst/privkey.pem"
+
+      # deploy-proxy performs the health-gated Swarm rollout. Its reconciler
+      # derives a fresh version from the public certificate chain and inserts
+      # the matching ssl_cert/ssl_key secrets before deploying Traefik.
+      systemctl restart deploy-proxy.service
+
+      # A successful rollout no longer references old wildcard versions. Best
+      # effort removal retains any secret Docker still reports as in use.
+      keep="v$(sha256sum "$dst/fullchain.pem" | cut -c1-16)"
+      docker secret ls --format '{{.Name}}' | \
+        grep -E '^traefik_ci_commoninternet_net_ssl_(cert|key)_v' | \
+        grep -v -E "_(ssl_cert|ssl_key)_$keep\$" | \
+        while IFS= read -r stale; do docker secret rm "$stale" || true; done
+    '';
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    certs."ci.commoninternet.net" = {
+      domain = "ci.commoninternet.net";
+      extraDomainNames = [ "*.ci.commoninternet.net" ];
+      # Staging issuance succeeded using the permanent, narrowly delegated
+      # CNAME. Production uses the same restricted acme-dns account.
+      dnsProvider = "acmedns";
+      environmentFile = "/etc/acme-dns/lego.env";
+      dnsResolver = "1.1.1.1:53";
+      server = "https://acme-v02.api.letsencrypt.org/directory";
+      postRun = ''
+        # The production marker is deployed only after staging proves the
+        # permanent CNAME and restricted acme-dns account work end to end.
+        if [ -e /var/lib/ci-certs/acme-production-enabled ]; then
+          ${pkgs.systemd}/bin/systemctl --no-block start cc-ci-acme-traefik-handoff.service
+        fi
+      '';
+    };
+  };
+}

runner/warm_reconcile.py

@@ -24,6 +24,7 @@ Run as root on cc-ci (direct docker/volume access). CLI: `warm_reconcile.py <app
 
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 import re
@@ -37,12 +38,51 @@ from harness import abra, lifecycle, warmsnap  # noqa: E402
 # --------------------------------------------------------------------------- specs
 
 
-def _traefik_setup(recipe: str, domain: str, version: str) -> None:
+CERT_DIR = "/var/lib/ci-certs/live"
+
+
+def wildcard_secret_version(cert_dir: str = CERT_DIR) -> str:
+    """Stable Swarm-secret version for the public certificate chain.
+
+    The certificate chain is public material, so its digest is safe to use as a
+    version label. The key is deliberately never read or hashed for logging.
+    """
+    chain = os.path.join(cert_dir, "fullchain.pem")
+    if not os.path.isfile(chain):
+        raise RuntimeError(f"FATAL: wildcard certificate missing at {chain}")
+    with open(chain, "rb") as certificate:
+        digest = hashlib.sha256(certificate.read()).hexdigest()
+    return "v" + digest[:16]
+
+
+def _traefik_requires_certificate_rollout(domain: str, secret_version: str) -> bool:
+    """Whether Traefik's active service still references an older cert version."""
+    stack = lifecycle._stack_name(domain)  # noqa: SLF001
+    service = f"{stack}_app"
+    result = _run(
+        [
+            "docker",
+            "service",
+            "inspect",
+            service,
+            "--format",
+            "{{range .Spec.TaskTemplate.ContainerSpec.Secrets}}{{.SecretName}} {{end}}",
+        ],
+        timeout=30,
+    )
+    expected = {
+        f"{stack}_ssl_cert_{secret_version}",
+        f"{stack}_ssl_key_{secret_version}",
+    }
+    return not expected.issubset(set(result.stdout.split()))
+
+
+def _traefik_setup(recipe: str, domain: str, version: str) -> bool:
     """Per-app config for the traefik reverse-proxy reconcile — preserves EXACTLY what the prior
     proxy.nix bash reconcile did (wildcard/file-provider mode serving the pre-issued cert as
     ssl_cert/ssl_key swarm secrets; NO ACME). Uses the proven abra.env_set (newline-safe, unlike the
     bash set_env that bit keycloak)."""
-    cert_dir = "/var/lib/ci-certs/live"
+    cert_dir = CERT_DIR
     if not (
         os.path.isfile(f"{cert_dir}/fullchain.pem") and os.path.isfile(f"{cert_dir}/privkey.pem")
     ):
@@ -56,14 +96,15 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
     abra.env_set(domain, "DOMAIN", domain)
     abra.env_set(domain, "LETS_ENCRYPT_ENV", "")
     abra.env_set(domain, "WILDCARDS_ENABLED", "1")
-    abra.env_set(domain, "SECRET_WILDCARD_CERT_VERSION", "v1")
-    abra.env_set(domain, "SECRET_WILDCARD_KEY_VERSION", "v1")
+    secret_version = wildcard_secret_version(cert_dir)
+    abra.env_set(domain, "SECRET_WILDCARD_CERT_VERSION", secret_version)
+    abra.env_set(domain, "SECRET_WILDCARD_KEY_VERSION", secret_version)
     abra.env_set(domain, "COMPOSE_FILE", '"compose.yml:compose.wildcard.yml"')
     stack = lifecycle._stack_name(domain)  # noqa: SLF001
     have = set(lifecycle._docker_names("secret", stack))  # noqa: SLF001
 
     def _has(name):
-        return any(s.endswith(f"_{name}_v1") for s in have)
+        return any(s.endswith(f"_{name}_{secret_version}") for s in have)
 
     if not _has("ssl_cert"):
         _run(
@@ -74,7 +115,7 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
                 "insert",
                 domain,
                 "ssl_cert",
-                "v1",
+                secret_version,
                 f"{cert_dir}/fullchain.pem",
                 "-f",
                 "-n",
@@ -91,7 +132,7 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
                 "insert",
                 domain,
                 "ssl_key",
-                "v1",
+                secret_version,
                 f"{cert_dir}/privkey.pem",
                 "-f",
                 "-n",
@@ -99,6 +140,7 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
             timeout=120,
             check=True,
         )
+    return _traefik_requires_certificate_rollout(domain, secret_version)
 
 
 SPECS: dict[str, dict] = {
@@ -457,8 +499,9 @@ def reconcile(app: str) -> str:
     # Per-app config/secrets: a spec may provide its own `setup` (traefik's cert/file-provider wiring);
     # otherwise the default keycloak-shaped path (app new + DOMAIN/LETS_ENCRYPT + generate secrets).
     setup = spec.get("setup")
+    setup_needs_rollout = False
     if setup:
-        setup(recipe, domain, latest)
+        setup_needs_rollout = bool(setup(recipe, domain, latest))
     else:
         ensure_app_config(recipe, domain, latest)
         ensure_secrets(domain)
@@ -476,6 +519,20 @@ def reconcile(app: str) -> str:
         write_last_good(recipe, target)
         return f"deployed-fresh:{target}"
 
+    # A certificate rotation changes Traefik's immutable Swarm secrets but
+    # must not be held hostage by an unrelated recipe-major upgrade policy.
+    # Redeploy the current recipe version so its compose spec references the
+    # just-created cert/key secret pair, then apply the usual health gate.
+    if setup_needs_rollout:
+        if not current:
+            raise RuntimeError(f"{app} has services but no current version")
+        print(f"[{app}] certificate changed → redeploy {current}", flush=True)
+        deploy_version(recipe, domain, current, dt)
+        if not wait_healthy(spec):
+            raise RuntimeError(f"{app} certificate rollout {current} did not become healthy")
+        write_last_good(recipe, current)
+        return f"certificate-rolled-out:{current}"
+
     # Deployed & already on latest → converge to a no-op (commit last-good if healthy).
     if current == latest:
         if wait_healthy(spec, timeout=60):

tests/unit/test_acme_dns.py

@@ -0,0 +1,21 @@
+"""Unit coverage for the public wildcard-secret version label."""
+
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parents[2] / "runner"))
+import warm_reconcile as wr  # noqa: E402
+
+
+def test_wildcard_secret_version_is_stable_and_does_not_need_key(tmp_path):
+    (tmp_path / "fullchain.pem").write_text("public certificate chain\n")
+    assert wr.wildcard_secret_version(str(tmp_path)) == wr.wildcard_secret_version(str(tmp_path))
+    assert wr.wildcard_secret_version(str(tmp_path)).startswith("v")
+
+
+def test_wildcard_secret_version_changes_with_certificate_chain(tmp_path):
+    chain = tmp_path / "fullchain.pem"
+    chain.write_text("first public certificate chain\n")
+    first = wr.wildcard_secret_version(str(tmp_path))
+    chain.write_text("replacement public certificate chain\n")
+    assert wr.wildcard_secret_version(str(tmp_path)) != first

tests/unit/test_warm_reconcile.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import os
 import sys
+from types import SimpleNamespace
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
@@ -115,6 +116,20 @@ def test_traefik_spec_is_stateless_with_setup():
     assert "setup" not in wr.SPECS["keycloak"]
 
 
+def test_traefik_certificate_rollout_detects_active_secret_version(monkeypatch):
+    stack = "traefik_ci_commoninternet_net"
+    monkeypatch.setattr(wr.lifecycle, "_stack_name", lambda _domain: stack)
+    monkeypatch.setattr(
+        wr,
+        "_run",
+        lambda *_args, **_kwargs: SimpleNamespace(
+            stdout=f"{stack}_ssl_cert_vnew {stack}_ssl_key_vnew"
+        ),
+    )
+    assert not wr._traefik_requires_certificate_rollout("traefik.ci.commoninternet.net", "vnew")
+    assert wr._traefik_requires_certificate_rollout("traefik.ci.commoninternet.net", "vold")
+
+
 def test_manual_migration_markers():
     assert wr.notes_flag_manual_migration("This release requires a MANUAL MIGRATION of the DB.")
     assert wr.notes_flag_manual_migration("Breaking change: action required before upgrade.")