fix: roll out rotated traefik certificate secrets

nix/modules/acme-dns.nix

@@ -14,6 +14,7 @@ let
     nsadmin = "hostmaster.commoninternet.net"
     records = [
       "acme.commoninternet.net. NS ns-acme.commoninternet.net.",
+      "ns-acme.commoninternet.net. A 91.98.47.73",
     ]
     debug = false
 
@@ -56,6 +57,12 @@ in
 
   environment.etc."acme-dns/lego.env".source = legoEnvironment;
 
+  # The staging order has completed successfully. This marker permits the
+  # production ACME post-run hook to hand a renewed certificate to Traefik.
+  systemd.tmpfiles.rules = [
+    "f /var/lib/ci-certs/acme-production-enabled 0600 root root -"
+  ];
+
   networking.firewall = {
     allowedTCPPorts = [ 53 ];
     allowedUDPPorts = [ 53 ];
@@ -130,15 +137,15 @@ in
     certs."ci.commoninternet.net" = {
       domain = "ci.commoninternet.net";
       extraDomainNames = [ "*.ci.commoninternet.net" ];
-      # The pinned Lego provider spells this `acmedns`; keep the service on
-      # staging until the operator has installed the permanent CNAME.
+      # Staging issuance succeeded using the permanent, narrowly delegated
+      # CNAME. Production uses the same restricted acme-dns account.
       dnsProvider = "acmedns";
       environmentFile = "/etc/acme-dns/lego.env";
       dnsResolver = "1.1.1.1:53";
-      server = "https://acme-staging-v02.api.letsencrypt.org/directory";
+      server = "https://acme-v02.api.letsencrypt.org/directory";
       postRun = ''
-        # Production cutover creates this marker in a separate reviewed
-        # deployment. Staging issuance must never replace the live cert.
+        # The production marker is deployed only after staging proves the
+        # permanent CNAME and restricted acme-dns account work end to end.
         if [ -e /var/lib/ci-certs/acme-production-enabled ]; then
           ${pkgs.systemd}/bin/systemctl --no-block start cc-ci-acme-traefik-handoff.service
         fi

runner/warm_reconcile.py

@@ -55,7 +55,29 @@ def wildcard_secret_version(cert_dir: str = CERT_DIR) -> str:
     return "v" + digest[:16]
 
 
-def _traefik_setup(recipe: str, domain: str, version: str) -> None:
+def _traefik_requires_certificate_rollout(domain: str, secret_version: str) -> bool:
+    """Whether Traefik's active service still references an older cert version."""
+    stack = lifecycle._stack_name(domain)  # noqa: SLF001
+    service = f"{stack}_app"
+    result = _run(
+        [
+            "docker",
+            "service",
+            "inspect",
+            service,
+            "--format",
+            "{{range .Spec.TaskTemplate.ContainerSpec.Secrets}}{{.SecretName}} {{end}}",
+        ],
+        timeout=30,
+    )
+    expected = {
+        f"{stack}_ssl_cert_{secret_version}",
+        f"{stack}_ssl_key_{secret_version}",
+    }
+    return not expected.issubset(set(result.stdout.split()))
+
+
+def _traefik_setup(recipe: str, domain: str, version: str) -> bool:
     """Per-app config for the traefik reverse-proxy reconcile — preserves EXACTLY what the prior
     proxy.nix bash reconcile did (wildcard/file-provider mode serving the pre-issued cert as
     ssl_cert/ssl_key swarm secrets; NO ACME). Uses the proven abra.env_set (newline-safe, unlike the
@@ -118,6 +140,7 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
             timeout=120,
             check=True,
         )
+    return _traefik_requires_certificate_rollout(domain, secret_version)
 
 
 SPECS: dict[str, dict] = {
@@ -476,8 +499,9 @@ def reconcile(app: str) -> str:
     # Per-app config/secrets: a spec may provide its own `setup` (traefik's cert/file-provider wiring);
     # otherwise the default keycloak-shaped path (app new + DOMAIN/LETS_ENCRYPT + generate secrets).
     setup = spec.get("setup")
+    setup_needs_rollout = False
     if setup:
-        setup(recipe, domain, latest)
+        setup_needs_rollout = bool(setup(recipe, domain, latest))
     else:
         ensure_app_config(recipe, domain, latest)
         ensure_secrets(domain)
@@ -495,6 +519,20 @@ def reconcile(app: str) -> str:
         write_last_good(recipe, target)
         return f"deployed-fresh:{target}"
 
+    # A certificate rotation changes Traefik's immutable Swarm secrets but
+    # must not be held hostage by an unrelated recipe-major upgrade policy.
+    # Redeploy the current recipe version so its compose spec references the
+    # just-created cert/key secret pair, then apply the usual health gate.
+    if setup_needs_rollout:
+        if not current:
+            raise RuntimeError(f"{app} has services but no current version")
+        print(f"[{app}] certificate changed → redeploy {current}", flush=True)
+        deploy_version(recipe, domain, current, dt)
+        if not wait_healthy(spec):
+            raise RuntimeError(f"{app} certificate rollout {current} did not become healthy")
+        write_last_good(recipe, current)
+        return f"certificate-rolled-out:{current}"
+
     # Deployed & already on latest → converge to a no-op (commit last-good if healthy).
     if current == latest:
         if wait_healthy(spec, timeout=60):

tests/unit/test_warm_reconcile.py

@@ -9,6 +9,7 @@ from __future__ import annotations
 
 import os
 import sys
+from types import SimpleNamespace
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", ".."))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
@@ -115,6 +116,20 @@ def test_traefik_spec_is_stateless_with_setup():
     assert "setup" not in wr.SPECS["keycloak"]
 
 
+def test_traefik_certificate_rollout_detects_active_secret_version(monkeypatch):
+    stack = "traefik_ci_commoninternet_net"
+    monkeypatch.setattr(wr.lifecycle, "_stack_name", lambda _domain: stack)
+    monkeypatch.setattr(
+        wr,
+        "_run",
+        lambda *_args, **_kwargs: SimpleNamespace(
+            stdout=f"{stack}_ssl_cert_vnew {stack}_ssl_key_vnew"
+        ),
+    )
+    assert not wr._traefik_requires_certificate_rollout("traefik.ci.commoninternet.net", "vnew")
+    assert wr._traefik_requires_certificate_rollout("traefik.ci.commoninternet.net", "vold")
+
+
 def test_manual_migration_markers():
     assert wr.notes_flag_manual_migration("This release requires a MANUAL MIGRATION of the DB.")
     assert wr.notes_flag_manual_migration("Breaking change: action required before upgrade.")