runner: never embed a fossil screenshot from a collided run dir (#45)

nix/modules/acme-dns.nix

@@ -111,82 +111,89 @@ in
       "f /var/lib/ci-certs/acme-production-enabled 0600 root root -"
     ];
 
-    services.acme-dns = {
+    # Three systemd units (W201 statix): one attrset, distinct unit names — avoids
-      description = "Restricted authoritative DNS for cc-ci ACME DNS-01";
+    # `services = { ... }` attribute sets repeating the `services` key.
-      wantedBy = [ "multi-user.target" ];
+    services = {
-      after = [ "network-online.target" ];
+      acme-dns = {
-      wants = [ "network-online.target" ];
+        description = "Restricted authoritative DNS for cc-ci ACME DNS-01";
-      serviceConfig = {
+        wantedBy = [ "multi-user.target" ];
-        User = "acme-dns";
+        after = [ "network-online.target" ];
-        Group = "acme-dns";
+        wants = [ "network-online.target" ];
-        StateDirectory = "acme-dns";
+        serviceConfig = {
-        StateDirectoryMode = "0700";
+          User = "acme-dns";
-        WorkingDirectory = "/var/lib/acme-dns";
+          Group = "acme-dns";
-        ExecStart = "${pkgs.acme-dns}/bin/acme-dns -c ${acmeDnsConfig}";
+          StateDirectory = "acme-dns";
-        Restart = "on-failure";
+          StateDirectoryMode = "0700";
-        RestartSec = "5s";
+          WorkingDirectory = "/var/lib/acme-dns";
-        AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
+          ExecStart = "${pkgs.acme-dns}/bin/acme-dns -c ${acmeDnsConfig}";
-        CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
+          Restart = "on-failure";
-        NoNewPrivileges = true;
+          RestartSec = "5s";
-        PrivateTmp = true;
+          AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
-        PrivateDevices = true;
+          CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
-        ProtectHome = true;
+          NoNewPrivileges = true;
-        ProtectSystem = "strict";
+          PrivateTmp = true;
-        ReadWritePaths = [ "/var/lib/acme-dns" ];
+          PrivateDevices = true;
-        RestrictAddressFamilies = [ "AF_INET" "AF_UNIX" ];
+          ProtectHome = true;
+          ProtectSystem = "strict";
+          ReadWritePaths = [ "/var/lib/acme-dns" ];
+          RestrictAddressFamilies = [ "AF_INET" "AF_UNIX" ];
+        };
       };
-    };
 
-    # Seed the new cert's acmedns storage before the ACME unit first runs (see
+      # Seed the new cert's acmedns storage before the ACME unit first runs (see
-    # acmeStorageSeed above). Ordering via the generated acme unit name.
+      # acmeStorageSeed above). Ordering via the generated acme unit name.
-    services.cc-ci-acme-storage-seed = {
+      cc-ci-acme-storage-seed = {
-      description = "Seed ci.autonomic.zone acme-dns storage from the legacy account";
+        description = "Seed ci.autonomic.zone acme-dns storage from the legacy account";
-      after = [ "acme-dns.service" ];
+        after = [ "acme-dns.service" ];
-      wantedBy = [ "acme-ci.autonomic.zone.service" ];
+        wantedBy = [ "acme-ci.autonomic.zone.service" ];
-      before = [ "acme-ci.autonomic.zone.service" ];
+        before = [ "acme-ci.autonomic.zone.service" ];
-      serviceConfig = {
+        serviceConfig = {
-        Type = "oneshot";
+          Type = "oneshot";
-        UMask = "0077";
+          UMask = "0077";
+        };
+        script = "${acmeStorageSeed}/bin/cc-ci-acme-storage-seed";
       };
-      script = "${acmeStorageSeed}/bin/cc-ci-acme-storage-seed";
-    };
 
-    # Traefik consumes its wildcard as immutable Swarm secrets, so a renewed
+      # Traefik consumes its wildcard as immutable Swarm secrets, so a renewed
-    # host certificate must be copied and reconciled rather than merely reloaded.
+      # host certificate must be copied and reconciled rather than merely reloaded.
-    # This service is started only by the production-mode ACME postRun hook.
+      # This service is started only by the production-mode ACME postRun hook.
-    services.cc-ci-acme-traefik-handoff = {
+      # Traefik consumes its wildcard as immutable Swarm secrets, so a renewed
-      description = "Install renewed cc-ci wildcard into Traefik Swarm secrets";
+      # host certificate must be copied and reconciled rather than merely reloaded.
-      after = [ "docker.service" "deploy-proxy.service" ];
+      # This service is started only by the production-mode ACME postRun hook.
-      requires = [ "docker.service" ];
+      cc-ci-acme-traefik-handoff = {
-      path = [ pkgs.coreutils pkgs.docker pkgs.systemd pkgs.gnugrep ];
+        description = "Install renewed cc-ci wildcard into Traefik Swarm secrets";
-      serviceConfig = {
+        after = [ "docker.service" "deploy-proxy.service" ];
-        Type = "oneshot";
+        requires = [ "docker.service" ];
-        UMask = "0077";
+        path = [ pkgs.coreutils pkgs.docker pkgs.systemd pkgs.gnugrep ];
+        serviceConfig = {
+          Type = "oneshot";
+          UMask = "0077";
+        };
+        script = ''
+          src=/var/lib/acme/ci.autonomic.zone
+          dst=/var/lib/ci-certs/live
+          test -s "$src/fullchain.pem"
+          test -s "$src/key.pem"
+          install -d -m 0700 "$dst"
+          install -m 0444 "$src/fullchain.pem" "$dst/fullchain.pem.new"
+          install -m 0400 "$src/key.pem" "$dst/privkey.pem.new"
+          mv -f "$dst/fullchain.pem.new" "$dst/fullchain.pem"
+          mv -f "$dst/privkey.pem.new" "$dst/privkey.pem"
+
+          # deploy-proxy performs the health-gated Swarm rollout. Its reconciler
+          # derives a fresh version from the public certificate chain and inserts
+          # the matching ssl_cert/ssl_key secrets before deploying Traefik.
+          systemctl restart deploy-proxy.service
+
+          # A successful rollout no longer references old wildcard versions. Best
+          # effort removal retains any secret Docker still reports as in use.
+          keep="v$(sha256sum "$dst/fullchain.pem" | cut -c1-16)"
+          docker secret ls --format '{{.Name}}' | \
+            grep -E '^traefik_ci_commoninternet_net_ssl_(cert|key)_v' | \
+            grep -v -E "_(ssl_cert|ssl_key)_$keep\$" | \
+            while IFS= read -r stale; do docker secret rm "$stale" || true; done
+        '';
       };
-      script = ''
-        src=/var/lib/acme/ci.autonomic.zone
-        dst=/var/lib/ci-certs/live
-        test -s "$src/fullchain.pem"
-        test -s "$src/key.pem"
-        install -d -m 0700 "$dst"
-        install -m 0444 "$src/fullchain.pem" "$dst/fullchain.pem.new"
-        install -m 0400 "$src/key.pem" "$dst/privkey.pem.new"
-        mv -f "$dst/fullchain.pem.new" "$dst/fullchain.pem"
-        mv -f "$dst/privkey.pem.new" "$dst/privkey.pem"
-
-        # deploy-proxy performs the health-gated Swarm rollout. Its reconciler
-        # derives a fresh version from the public certificate chain and inserts
-        # the matching ssl_cert/ssl_key secrets before deploying Traefik.
-        systemctl restart deploy-proxy.service
-
-        # A successful rollout no longer references old wildcard versions. Best
-        # effort removal retains any secret Docker still reports as in use.
-        keep="v$(sha256sum "$dst/fullchain.pem" | cut -c1-16)"
-        docker secret ls --format '{{.Name}}' | \
-          grep -E '^traefik_ci_commoninternet_net_ssl_(cert|key)_v' | \
-          grep -v -E "_(ssl_cert|ssl_key)_$keep\$" | \
-          while IFS= read -r stale; do docker secret rm "$stale" || true; done
-      '';
     };
   };
 

runner/harness/results.py

@@ -27,6 +27,7 @@ from __future__ import annotations
 
 import json
 import os
+import shutil
 import xml.etree.ElementTree as ET
 
 from . import level as level_mod
@@ -35,6 +36,17 @@ from . import level as level_mod
 # The dashboard serves these read-only at /runs/<run_id>/... (U0.4). Overridable for tests.
 RUNS_DIR_DEFAULT = "/var/lib/cc-ci-runs"
 
+# The artifact files/dirs a run OWNS in its run dir (written fresh every run).
+_OWNED_ARTIFACTS = (
+    "results.json",
+    "summary.html",
+    "summary.png",
+    "badge.svg",
+    "lint.txt",
+    "screenshot.png",
+    "junit",
+)
+
 
 def runs_dir() -> str:
     return os.environ.get("CCCI_RUNS_DIR", RUNS_DIR_DEFAULT)
@@ -49,6 +61,31 @@ def run_id() -> str:
     return os.environ.get("CCCI_APP_DOMAIN") or os.environ.get("CCCI_RUN_ID") or "manual"
 
 
+def fresh_artifact_dir(path: str) -> str:
+    """Ensure the run artifact dir holds no stale artifacts from a PREVIOUS run with the same id.
+
+    Drone build numbers restarted when its DB was re-created (2026-09-27) while /var/lib/cc-ci-runs
+    still holds old-era dirs numbered into the thousands — a new build reusing an old number would
+    otherwise mix old artifacts into the new run's card (observed 2026-10-05: !testme gitea build 35
+    collided with a May-31 run 35 and embedded custom-html-tiny's fossil screenshot.png, because
+    screenshot.capture() skips its snap when out_path already exists). Removes ONLY the files/dirs
+    this run owns (see _OWNED_ARTIFACTS); anything else in the dir is left untouched. Best-effort
+    (R7): never raises into the run."""
+    os.makedirs(path, exist_ok=True)
+    for name in _OWNED_ARTIFACTS:
+        p = os.path.join(path, name)
+        if os.path.isdir(p):
+            shutil.rmtree(p, ignore_errors=True)
+        else:
+            try:
+                os.remove(p)
+            except FileNotFoundError:
+                pass
+            except OSError:
+                pass  # R7: a stuck file must never fail the run — the card is best-effort anyway
+    return path
+
+
 def junit_file(junit_dir: str, tier: str, source: str, path: str) -> str:
     """Deterministic per-(tier,source,file) JUnit XML path under junit_dir."""
     base = os.path.splitext(os.path.basename(path))[0]

runner/harness/screenshot.py

@@ -142,12 +142,13 @@ def capture(domain: str, out_path: str, *, recipe_meta: dict | None = None) -> s
                 page = context.new_page()
                 if hook is not None:
                     # Recipe-specific safe view (post-login etc.). The hook owns navigation +
-                    # the no-secret-page guarantee; it should call page.screenshot itself, but if
+                    # the no-secret-page guarantee; we ALWAYS snap the page after it returns.
-                    # it doesn't, we still snap the resulting page below. SCREENSHOT(page, ctx) —
+                    # Never trust a pre-existing out_path: a recycled run artifact dir (drone
-                    # the uniform ctx convention (rcust P3).
+                    # build numbers restarted 2026-09-27 while old-era dirs linger) must not get
+                    # its fossil screenshot embedded (observed 2026-10-05, !testme gitea #10
+                    # build 35). SCREENSHOT(page, ctx) — the uniform ctx convention (rcust P3).
                     hook(page, meta_mod.hook_ctx(domain, recipe_meta))
-                    if not os.path.exists(out_path):
+                    _snap_with_blank_retry(page, out_path)
-                        _snap_with_blank_retry(page, out_path)
                 else:
                     # Default: landing page. Accept any rendered status (200 or an auth redirect to a
                     # login form) — both are credential-free and representative of "the app is up".

runner/harness/warm.py

@@ -41,7 +41,9 @@ _CTX.check_hostname = False
 _CTX.verify_mode = ssl.CERT_NONE
 
 # A cold per-run stack name looks like "<tag>-<6hex>_ci_commoninternet_net_<svc>"; extract the hex.
-_STACK_HEX_RE = re.compile(r"^[a-z0-9]{1,4}-([0-9a-f]{6})_ci_(?:autonomic_zone|commoninternet_net)_")
+_STACK_HEX_RE = re.compile(
+    r"^[a-z0-9]{1,4}-([0-9a-f]{6})_ci_(?:autonomic_zone|commoninternet_net)_"
+)
 
 
 def stable_domain(recipe: str) -> str:

runner/run_recipe_ci.py

@@ -1184,6 +1184,10 @@ def main() -> int:
     # {tier,source,file,rc,junit} record; after the run we assemble results.json (per-stage/per-test +
     # level) into the artifact dir. Best-effort — never changes the verdict (R7).
     run_artifact_dir = os.path.join(results_mod.runs_dir(), results_mod.run_id())
+    # Collision guard (2026-10-05): drone build numbers restarted when its DB was re-created while
+    # old-era run dirs (same numbers, other recipes' artifacts — e.g. a fossil screenshot.png) are
+    # still on disk. Wipe the files THIS run owns so nothing stale survives into the card.
+    results_mod.fresh_artifact_dir(run_artifact_dir)
     junit_dir = os.path.join(run_artifact_dir, "junit")
     records: list[dict] = []
 

tests/gitea/custom/test_lfs_roundtrip.py

@@ -23,6 +23,7 @@ import shutil
 import subprocess
 import sys
 import tempfile
+import time
 import urllib.error
 import urllib.request
 
@@ -45,7 +46,15 @@ def _lfs_available() -> bool:
     return os.path.exists(os.path.join(abra_dir, "recipes", "gitea", _LFS_OVERLAY))
 
 
-def _api(domain, path, method="GET", body=None, user="", password=""):
+API_TIMEOUT = 20
+
+
+def _api(domain, path, method="GET", body=None, user="", password="", timeout=API_TIMEOUT):
+    """One API call → (status, JSON). Never raises: transient errors (connection refused /
+    dropped mid-restart, DNS, socket timeout) return (0, {}) so a caller's bounded poll can
+    simply retry, mirroring lifecycle.http_fetch. Only HTTPError was caught before; a socket
+    timeout on a slow post-restart boot (gitea 28.0.0 migration window, PR #10 run #31) escaped
+    as TimeoutError and aborted the restart poll itself."""
     data = json.dumps(body).encode() if body is not None else None
     auth = base64.b64encode(f"{user}:{password}".encode()).decode()
     headers = {"Authorization": f"Basic {auth}"}
@@ -55,7 +64,7 @@ def _api(domain, path, method="GET", body=None, user="", password=""):
         f"https://{domain}/api/v1{path}", data=data, headers=headers, method=method
     )
     try:
-        with urllib.request.urlopen(req, timeout=20, context=_CTX) as r:
+        with urllib.request.urlopen(req, timeout=timeout, context=_CTX) as r:
             raw = r.read()
             return r.status, (json.loads(raw) if raw else {})
     except urllib.error.HTTPError as e:
@@ -64,6 +73,8 @@ def _api(domain, path, method="GET", body=None, user="", password=""):
             return e.code, json.loads(raw)
         except (ValueError, json.JSONDecodeError):
             return e.code, {}
+    except (urllib.error.URLError, TimeoutError, OSError):  # TimeoutError ⊃ socket.timeout (UP041)
+        return 0, {}
 
 
 def _run_git(args, cwd, env=None):
@@ -164,6 +175,8 @@ def test_lfs_roundtrip(live_app):
         finally:
             shutil.rmtree(fresh_dir, ignore_errors=True)
 
+        import recipe_meta  # noqa: E402 — per-recipe declared timeouts (tests/STYLE.md §5)
+
         # 7. JWT-secret stability: restart gitea + assert LFS_JWT_SECRET unchanged.
         #    Read the current secret from inside the container's rendered app.ini.
         current_jwt = lifecycle.exec_in_app(
@@ -182,14 +195,15 @@ def test_lfs_roundtrip(live_app):
             capture_output=True,
             timeout=120,
         )
-        # Wait for gitea to come back up
+        # Wait for gitea to come back up. Window + probe are DERIVED from the recipe's declared
-
+        # readiness (recipe_meta.HTTP_TIMEOUT / READY_PROBE — the /api/v1/version readiness that
-        # Re-read meta from the live_app fixture (meta is not in scope here — use the stored meta)
+        # recipe_meta already declares for the harness, not a guess) — tests/STYLE.md §5.
-        import time
+        wait_timeout = int(getattr(recipe_meta, "HTTP_TIMEOUT", 300))
-
+        probe_path = "/version"  # READY_PROBE's readiness path (rules carry only ok + domain)
-        deadline = time.time() + 120
+        deadline = time.time() + wait_timeout
+        status2 = 0
         while time.time() < deadline:
-            status2, _ = _api(live_app, "/version", user=user, password=password)
+            status2, _ = _api(live_app, probe_path, user=user, password=password)
             if status2 == 200:
                 break
             time.sleep(5)

tests/unit/test_results.py

@@ -489,3 +489,50 @@ def test_write_results_roundtrip(tmp_path):
     assert path.endswith("/42/results.json")
     with open(path) as f:
         assert json.load(f)["level"] == 3
+
+
+def test_fresh_artifact_dir_removes_owned_stale_artifacts(tmp_path):
+    """Run-dir collision guard (2026-10-05): drone build numbers restarted while old-era run dirs
+    linger in /var/lib/cc-ci-runs — a new build reusing an old number must not mix the old run's
+    artifacts (fossil screenshot.png from ANOTHER recipe) into its card. Only OWNED artifacts go."""
+    d = tmp_path / "35"
+    d.mkdir()
+    (d / "results.json").write_text('{"old": true}')
+    (d / "screenshot.png").write_bytes(b"stale-fossil")
+    (d / "summary.png").write_bytes(b"stale")
+    (d / "badge.svg").write_text("<svg/>")
+    (d / "lint.txt").write_text("old")
+    (d / "summary.html").write_text("<html/>")
+    (d / "junit").mkdir()
+    (d / "junit" / "old.xml").write_text("<x/>")
+    (d / "untracked-thing.bin").write_bytes(b"keep me")  # NOT owned — must survive
+
+    R.fresh_artifact_dir(str(d))
+
+    for name in (
+        "results.json",
+        "screenshot.png",
+        "summary.png",
+        "badge.svg",
+        "lint.txt",
+        "summary.html",
+    ):
+        assert not (d / name).exists(), f"stale owned artifact {name} survived"
+    assert not (d / "junit").exists(), "stale junit dir survived"
+    assert (d / "untracked-thing.bin").read_bytes() == b"keep me", "unowned file was deleted"
+
+
+def test_fresh_artifact_dir_creates_missing_dir(tmp_path):
+    d = tmp_path / "36"
+    out = R.fresh_artifact_dir(str(d))
+    assert out == str(d) and d.is_dir()
+
+
+def test_fresh_artifact_dir_tolerant_of_odd_entries(tmp_path):
+    """R7: never raise into the run — a dir where an owned name is itself a dir must still work."""
+    d = tmp_path / "37"
+    d.mkdir()
+    (d / "screenshot.png").mkdir()  # perverse: a DIRECTORY named screenshot.png
+    (d / "junit").mkdir()
+    R.fresh_artifact_dir(str(d))
+    assert not (d / "screenshot.png").exists() and not (d / "junit").exists()

tests/unit/test_screenshot.py

@@ -184,3 +184,78 @@ def test_screenshot_reachable_through_real_load_path(tmp_path):
     hook = S._load_screenshot_hook(meta)
     assert callable(hook), "SCREENSHOT hook did not survive the orchestrator load path (R2)"
     assert S._load_screenshot_hook(meta_mod.load("no-such", tests_dir=str(tmp_path))) is None
+
+
+# ---- capture() fossil guard (2026-10-05) -----------------------------------------------------
+# Drone build numbers restarted when its DB was re-created while /var/lib/cc-ci-runs still holds
+# old-era dirs numbered into the thousands; run 35 (!testme gitea #10) collided with a May-31 run
+# 35 and embedded custom-html-tiny's fossil screenshot.png because the hook branch SKIPPED the
+# snap when out_path already existed. capture() must always snap fresh.
+
+
+class _FakeLaunchable:
+    def __init__(self, page):
+        self._page = page
+        self.chromium = self
+
+    def launch(self, args=None):
+        return self
+
+    def new_context(self, **kw):
+        return self
+
+    def new_page(self):
+        return self._page
+
+    def close(self):
+        pass
+
+
+class _FakeSyncPlaywright:
+    """sync_playwright() replacement: context-manager factory handing out one fake browser."""
+
+    def __init__(self, page):
+        self._page = page
+
+    def __call__(self):
+        return self
+
+    def __enter__(self):
+        return _FakeLaunchable(self._page)
+
+    def __exit__(self, *exc):
+        return False
+
+
+def _install_fake_playwright(monkeypatch, page):
+    import types
+
+    fake_api = types.ModuleType("playwright.sync_api")
+    fake_api.sync_playwright = _FakeSyncPlaywright(page)
+    fake = types.ModuleType("playwright")
+    fake.sync_api = fake_api
+    monkeypatch.setitem(sys.modules, "playwright", fake)
+    monkeypatch.setitem(sys.modules, "playwright.sync_api", fake_api)
+
+
+def test_capture_overwrites_fossil_screenshot(tmp_path, monkeypatch):
+    """A pre-existing out_path in a recycled run artifact dir must never be embedded: the hook
+    branch snaps fresh regardless (the old exists-guard returned the fossil as 'captured')."""
+    out_path = str(tmp_path / "screenshot.png")
+    with open(out_path, "wb") as f:
+        f.write(b"STALE-FOSSIL-FROM-ANOTHER-RECIPE" * 400)
+
+    fired = {}
+    page = _FakePage([35707])  # one real-sized frame, like a painted page
+
+    def hook(p, ctx):
+        fired["yes"] = True  # navigation only — hooks never save the PNG themselves
+
+    _install_fake_playwright(monkeypatch, page)
+    got = S.capture("gitea.example.org", out_path, recipe_meta={"SCREENSHOT": hook})
+
+    assert got == out_path, "capture must report success after snapping fresh"
+    assert fired.get("yes"), "SCREENSHOT hook must run"
+    assert page.shots == 1, "capture must snap exactly once — the fossil is not a captured frame"
+    with open(out_path, "rb") as f:
+        assert f.read(4) == b"\x89PNG", "out_path still holds the fossil — snap did not overwrite"