test(plausible): psql -q in _register_site — -t does not suppress command tags

psql -tAc still prints INSERT/CREATE command tags (e.g. "INSERT 0 1"), so
_register_site asserted out == site against "INSERT 0 1\nsite" and both
event-tracking roundtrip tests failed on their very first run (build 237 —
the custom tier had never executed before; install always failed earlier).
-q suppresses the tags; verified against the recipe db container.

.drone.yml

@@ -35,10 +35,12 @@ steps:
 # the comment-bridge). Deploys the recipe at the PR head, runs install/upgrade/backup + any
 # recipe-local tests via the shared harness, then guarantees teardown (plan §4.2/§4.3).
 #
-# Resource safety (plan §4.2/§4.3): MAX_TESTS=DRONE_RUNNER_CAPACITY=1 (nix/modules/drone-runner.nix) is
-# the primary concurrency cap; concurrency.limit below is a redundant belt. CCCI_JANITOR_MAX_AGE=0
-# makes the run-start janitor reap ANY orphaned run app before deploying — safe because capacity=1
-# means no concurrent run exists (a SIGKILL'd/timed-out build leaves an orphan with no teardown).
+# Resource safety (plan §4.2/§4.3): DRONE_RUNNER_CAPACITY=2 (nix/modules/drone-runner.nix) +
+# concurrency.limit=2 below allow two recipe runs in parallel. Concurrent-run safety is enforced by
+# the harness, not by serialisation: same-recipe runs serialise on a per-recipe flock
+# (lifecycle.acquire_recipe_lock — the shared ~/.abra/recipes/<recipe> checkout is the conflict),
+# and every run registers its app domain + pid in /run/cc-ci-active so the run-start janitor only
+# reaps orphans whose owning run is DEAD (alive → never touched; unknown → age fallback, default 2h).
 kind: pipeline
 type: exec
 name: recipe-ci
@@ -52,16 +54,16 @@ trigger:
     - custom
 
 concurrency:
-  limit: 1
+  limit: 2
 
 steps:
   - name: ci
     environment:
       STAGES: install,upgrade,backup,restore,custom
-      CCCI_JANITOR_MAX_AGE: "0"
       # The exec runner points HOME at a per-build workspace; force it to /root so abra finds its
-      # server config + recipes under /root/.abra (as the manual M4/M5 runs did). Safe: capacity=1
-      # means no concurrent build shares /root/.abra.
+      # server config + recipes under /root/.abra (as the manual M4/M5 runs did). Safe with
+      # capacity=2: app names are unique per (recipe,pr,ref) and same-recipe runs serialise on the
+      # per-recipe flock, so concurrent builds never touch the same recipe checkout or app.
       HOME: /root
     commands:
       # RECIPE/REF/PR/SRC (+ CCCI_QUICK for `!testme --quick`) are injected as env vars from the

bridge/bridge.py

@@ -64,6 +64,8 @@ def parse_trigger(body):
     if s == f"{TRIGGER} --quick":
         return True, True
     return False, False
+
+
 ALLOWLIST = {u.strip() for u in os.environ.get("AUTH_ALLOWLIST", "").split(",") if u.strip()}
 
 
@@ -167,8 +169,12 @@ def post_commit_status(owner, repo, sha, state, target_url, description=""):
         f"{GITEA_API}/repos/{owner}/{repo}/statuses/{sha}",
         GITEA_TOKEN,
         method="POST",
-        data={"state": state, "target_url": target_url,
-              "description": description, "context": "cc-ci/testme"},
+        data={
+            "state": state,
+            "target_url": target_url,
+            "description": description,
+            "context": "cc-ci/testme",
+        },
     )
 
 
@@ -217,7 +223,9 @@ def result_comment_body(recipe, sha, num, run_url, status):
         if artifact_available(badge_url):
             body += f"\n\n[![level]({badge_url})]({run_url})"
         return f"{body}\n\n{links}"
-    return f"{header} → {run_url}\n\n_(summary card unavailable — see the run for details.)_ {links}"
+    return (
+        f"{header} → {run_url}\n\n_(summary card unavailable — see the run for details.)_ {links}"
+    )
 
 
 def watch_and_reflect(owner, name, number, num, recipe, sha, comment_id, run_url):
@@ -287,15 +295,11 @@ def process_testme(full_name, owner, name, number, user, comment_id, source, qui
     run_url = f"{DRONE_URL}/{CI_REPO}/{num}"
     post_commit_status(owner, name, head["sha"], "pending", run_url, "cc-ci run in progress")
     mode = " **(--quick: lower-confidence fast lane; does not gate merge)**" if quick else ""
-    # R2/U3: one comment per PR, updated in place. Reuse the existing marked comment if present
-    # (re-`!testme` refreshes it back to the ⏳ placeholder), else post a new one.
+    # One NEW comment PER `!testme` (operator preference 2026-06-02): post a fresh ⏳ placeholder each
+    # run so every re-`!testme` is visible in the PR timeline; watch_and_reflect then edits THIS
+    # comment to its result. (Previously a single marked comment was reused/edited in place.)
     start_body = start_comment_body(name, head["sha"], run_url, mode)
-    existing = find_existing_comment(full_name, number)
-    if existing:
-        edit_comment(owner, name, existing, start_body)
-        cid = existing
-    else:
-        cid = post_comment(owner, name, number, start_body)
+    cid = post_comment(owner, name, number, start_body)
     log(
         f"[{source}] triggered build {num} for {name}@{head['sha'][:8]} "
         f"(PR #{number}, comment {comment_id}) by {user}"

dashboard/dashboard.py

@@ -66,8 +66,13 @@ _COLORS = {
 # Level → colour ramp, kept in sync with runner/harness/card.py LEVEL_COLOR (the dashboard is a
 # standalone stdlib service that doesn't import the runner harness, so the small map is duplicated).
 _LEVEL_COLOR = {
-    0: "#e5534b", 1: "#e0823d", 2: "#e0823d", 3: "#d9b343",
-    4: "#a0b93f", 5: "#57ab5a", 6: "#3fb950",
+    0: "#e5534b",
+    1: "#e0823d",
+    2: "#e0823d",
+    3: "#d9b343",
+    4: "#a0b93f",
+    5: "#57ab5a",
+    6: "#3fb950",
 }
 
 
@@ -269,7 +274,11 @@ def _card(r):
             f'<a class="shot" href="{run_url}" title="open run">'
             f'<span class="ph">no screenshot</span>{_level_pill(r["level"])}</a>'
         )
-    cap = f'<div class="cap">{html.escape(r["level_cap_reason"])}</div>' if r["level_cap_reason"] else ""
+    cap = (
+        f'<div class="cap">{html.escape(r["level_cap_reason"])}</div>'
+        if r["level_cap_reason"]
+        else ""
+    )
     return (
         f'<div class="card">{shot}<div class="body">'
         f'<div class="name">{html.escape(r["recipe"])}</div>'
@@ -307,7 +316,11 @@ def render_history(recipe, rows):
     trs = []
     for r in rows:
         color = _COLORS.get(r["status"], "#8b949e")
-        lvl = "—" if r["level"] is None else f'<b style="color:{level_color(r["level"])}">L{int(r["level"])}</b>'
+        lvl = (
+            "—"
+            if r["level"] is None
+            else f'<b style="color:{level_color(r["level"])}">L{int(r["level"])}</b>'
+        )
         shot = f'<a href="/runs/{r["number"]}/summary.png">card</a>' if r["has_screenshot"] else "—"
         trs.append(
             f'<tr><td><a href="{html.escape(r["url"])}">#{r["number"]}</a></td>'
@@ -317,7 +330,7 @@ def render_history(recipe, rows):
         )
     body = "\n".join(trs) or '<tr><td colspan="6">no runs for this recipe yet</td></tr>'
     inner = (
-        f'<h1>{_FLOWER} {html.escape(recipe)} — run history</h1>'
+        f"<h1>{_FLOWER} {html.escape(recipe)} — run history</h1>"
         '<p class="sub"><a href="/">← all recipes</a> · every <code>!testme</code> run, newest first.</p>'
         "<table><thead><tr><th>Run</th><th>Status</th><th>Level</th><th>Version</th>"
         "<th>When</th><th>Card</th></tr></thead><tbody>"

flake.nix

@@ -31,34 +31,36 @@
       ];
     in
     {
-      # Canonical live host target: the Hetzner cc-ci server.
-      # Use `.#cc-ci` for the current production host.
-      nixosConfigurations.cc-ci = nixpkgs.lib.nixosSystem {
-        inherit system;
-        modules = [
-          sops-nix.nixosModules.sops
-          ./nix/hosts/cc-ci-hetzner/configuration.nix
-        ];
-      };
+      nixosConfigurations = {
+        # Canonical live host target: the Hetzner cc-ci server.
+        # Use `.#cc-ci` for the current production host.
+        cc-ci = nixpkgs.lib.nixosSystem {
+          inherit system;
+          modules = [
+            sops-nix.nixosModules.sops
+            ./nix/hosts/cc-ci-hetzner/configuration.nix
+          ];
+        };
 
-      # Legacy Incus VM host definition retained only for historical comparison and fallback.
-      # Do NOT use this target on the live Hetzner server.
-      nixosConfigurations.cc-ci-incus = nixpkgs.lib.nixosSystem {
-        inherit system;
-        modules = [
-          sops-nix.nixosModules.sops
-          ./nix/hosts/cc-ci/configuration.nix
-        ];
-      };
+        # Legacy Incus VM host definition retained only for historical comparison and fallback.
+        # Do NOT use this target on the live Hetzner server.
+        cc-ci-incus = nixpkgs.lib.nixosSystem {
+          inherit system;
+          modules = [
+            sops-nix.nixosModules.sops
+            ./nix/hosts/cc-ci/configuration.nix
+          ];
+        };
 
-      # Explicit alias for the live Hetzner host. Kept alongside `cc-ci` so the intended host target
-      # remains obvious in recovery/migration workflows.
-      nixosConfigurations.cc-ci-hetzner = nixpkgs.lib.nixosSystem {
-        inherit system;
-        modules = [
-          sops-nix.nixosModules.sops
-          ./nix/hosts/cc-ci-hetzner/configuration.nix
-        ];
+        # Explicit alias for the live Hetzner host. Kept alongside `cc-ci` so the intended host
+        # target remains obvious in recovery/migration workflows.
+        cc-ci-hetzner = nixpkgs.lib.nixosSystem {
+          inherit system;
+          modules = [
+            sops-nix.nixosModules.sops
+            ./nix/hosts/cc-ci-hetzner/configuration.nix
+          ];
+        };
       };
 
       devShells.${system} = {

machine-docs/BACKLOG-regression.md

@@ -0,0 +1,131 @@
+# BACKLOG — server regression canaries phase
+
+## Build backlog
+
+- [x] Create `tests/regression/` suite (conftest + test_canaries + README)
+- [ ] Run `good-simple` canary (custom-html-tiny main) → confirm GREEN + test_serving passes
+- [ ] Run `bad-false-green` canary (custom-html v5-stale-docroot) → confirm RED + test_content_type fails
+- [ ] Run `good-significant` canary (lasuite-docs main) → confirm GREEN + test_serving_and_frontend passes
+- [ ] Open PR for operator review (DoD item 5: NOT merged)
+- [ ] Claim gate once all canary runs are GREEN/RED as expected + PR is open
+
+## Adversary findings
+
+### A-reg-1 [adversary] CLOSED @2026-06-02T01:46Z — relative import fixed, 3 tests collect
+**Filed:** 2026-06-02T01:37Z
+**Severity:** CRITICAL — suite can't run at all until fixed
+
+Cold-run `cc-ci-run -m pytest tests/regression/ --collect-only` on cc-ci confirms:
+```
+ImportError: attempted relative import with no known parent package
+tests/regression/test_canaries.py:18: from .conftest import run_recipe_ci, ...
+```
+No tests collected. 0 canaries can run.
+
+**Root cause:** `test_canaries.py` uses a relative import (`from .conftest import ...`) which
+requires the directory to be a Python package. Without `tests/regression/__init__.py` (and
+`tests/__init__.py`), pytest imports `test_canaries.py` as a top-level module, not a package
+member. Relative imports fail.
+
+**Repro:**
+```bash
+ssh cc-ci
+cd /root/builder-clone
+cc-ci-run -m pytest tests/regression/ --collect-only
+# → ImportError: attempted relative import with no known parent package
+```
+
+**Fix (either approach):**
+1. Add `tests/__init__.py` and `tests/regression/__init__.py` (makes it a real package)
+2. OR replace `from .conftest import ...` with absolute sys.path manipulation (like other test
+   files do, e.g. `sys.path.insert(0, ...); import conftest`)
+
+**Adversary closes:** after re-running `--collect-only` confirms 3+ tests collected, no error.
+
+---
+
+### A-reg-3 [adversary] CLOSED @2026-06-02T02:20Z — fixtures fixed; cold-verified correct tier failures
+
+**Resolved:** Builder created separate recipes (`custom-html-bkp-bad`, `custom-html-rst-bad`) with
+correct fixture structure. Cold-verified from cc-ci artifact dirs (no harness re-run needed).
+
+**Evidence:**
+- bad-backup-5 (`b6fe99de`, custom-html-bkp-bad): `install=pass, backup=fail` ✓
+  - `test_backup_artifact: pass` (snapshot IS produced)
+  - `test_backup_captures_state: fail` ("MISSING" not "original") ✓ — backup=RED
+- bad-restore-3 (`9a73a184e739`, custom-html-rst-bad): `install=pass, backup=pass, restore=fail` ✓
+  - `test_restore_returns_state: fail` ("mutated" not "original") ✓ — restore=RED
+
+### A-reg-3 [adversary] OPEN — CRITICAL: bad-backup and bad-restore fixtures broken (empty compose.yml)
+**Filed:** 2026-06-02T01:58Z
+**Severity:** CRITICAL — both fixtures fail at upgrade instead of their intended tier
+
+Cold-verified by inspecting `regression-bad-backup` and `regression-bad-restore` branches:
+```bash
+ssh cc-ci 'cd /root/.abra/recipes/custom-html && git diff origin/main..origin/regression-bad-backup -- compose.yml'
+```
+Result: compose.yml is completely empty (entire file deleted, leaving only a blank line). Same
+for `regression-bad-restore`.
+
+**Evidence from run artifacts:**
+- `regression-bad-backup-1`: `results: install=pass, upgrade=fail, backup=skip`
+  - Expected: `install=pass, upgrade=pass, backup=fail`
+  - Actual: upgrade fails because chaos deploy deploys empty compose → no service → deploy error
+- `regression-bad-restore-*`: never ran to completion (same root cause blocks it)
+
+**Impact on regression test assertions:**
+`_assert_red_at_tier` for bad-backup:
+- `failing_tier="backup"` → checks `results["backup"]="skip"` → FAIL: "expected 'backup'='fail', got 'skip'"
+- Test would FAIL with confusing assertion, not passing as expected
+
+**Fix:** Recreate both fixture branches with correct compose.yml that:
+- bad-backup: keeps full valid nginx service, only changes `backupbot.backup.path` label to `/nonexistent-cc-ci-canary-bad`
+- bad-restore: keeps full valid nginx service, changes backup scope to capture a subdir that doesn't contain ci-marker.txt (so restore doesn't recover the marker)
+
+The compose.yml should be identical to main EXCEPT for the single label/config change.
+
+**Repro:** `git diff origin/main..origin/regression-bad-backup -- compose.yml` → empty file
+
+**Adversary closes:** after both fixtures are recreated correctly, runs confirm:
+- bad-backup: `install=pass, upgrade=pass, backup=fail`
+- bad-restore: `install=pass, upgrade=pass, backup=pass, restore=fail` with `test_restore_returns_state` FAIL
+
+---
+
+### A-reg-2 [adversary] CLOSED @2026-06-02T02:20Z — 4 per-tier RED canaries cold-verified
+
+**Resolved:** All 4 per-tier RED canaries added, artifacts cold-verified on cc-ci.
+
+| Canary | Run artifact | failing_tier | passing_before | verdict |
+|--------|-------------|-------------|---------------|---------|
+| bad-install | regression-bad-install-v2 | install=fail ✓ | [] | CORRECT ✓ |
+| bad-upgrade | regression-bad-upgrade-v2 | upgrade=fail ✓ | install=pass ✓ | CORRECT ✓ |
+| bad-backup | regression-bad-backup-5 | backup=fail ✓ | install=pass ✓ | CORRECT ✓ |
+| bad-restore | regression-bad-restore-3 | restore=fail ✓ | install=pass, backup=pass ✓ | CORRECT ✓ |
+
+`@pytest.mark.canary_fast` marker added ✓. 7 tests collect ✓.
+
+**Note:** bad-backup comment in test_canaries.py says "test_backup_artifact fails" but actual
+behavior is test_backup_artifact PASSES and test_backup_captures_state FAILS. Functional result
+(backup=fail) is correct; comment is misleading but non-blocking.
+
+### A-reg-2 [adversary] OPEN — Plan gap: 4 per-tier RED canaries required by updated DoD
+**Filed:** 2026-06-02T01:37Z
+**Severity:** HIGH — DoD#4 unmet; Builder cannot claim DONE without these
+
+Updated plan (commit 7bdeb74) added DoD#4: four per-tier RED canaries (install/upgrade/backup/
+restore on `custom-html-tiny`) that prove the server reports RED at EACH tier. Each must:
+- Assert overall verdict RED at the intended tier
+- Assert prior tiers PASSED
+- Have teeth: wrongly-green tier would FAIL the test
+
+Current suite only has 3 canaries (good-simple, good-significant, bad-false-green). The 4
+per-tier RED canaries are MISSING. This is a mandatory DoD item.
+
+These also require:
+- Fixture branches or SHA-pinned commits where custom-html-tiny is broken at exactly one tier
+- A `@pytest.mark.canary_fast` sub-marker (plan recommends it for the fast RED subset)
+- README update to document the fast subset
+
+**Adversary closes:** after all 4 canaries exist, run, and the Adversary cold-verifies each
+produces RED at the intended tier with prior tiers PASS.

machine-docs/JOURNAL-regression.md

@@ -0,0 +1,76 @@
+# JOURNAL — server regression canaries phase (Builder)
+
+**Phase:** server regression canaries
+**Started:** 2026-06-02
+
+---
+
+## Step 0 — phase kickoff and design (2026-06-02)
+
+**Context:** Mirror phase (plan-mirror-enroll-all-recipes.md) completed DONE at 2026-06-02T01:16Z.
+Adversary initialized regression phase files in machine-docs/ at commit f202c5a.
+
+**Decision: run regression tests ON cc-ci, not from the orchestrator**
+
+The regression tests call `run_recipe_ci.py` which uses abra/docker/swarm — these only exist on
+cc-ci. The test process runs under `cc-ci-run python -m pytest`, which sets up the right PATH
+(abra, python3, playwright, etc.). The test then invokes `run_recipe_ci.py` as a subprocess using
+`sys.executable` (inherits the same python3 from cc-ci-run).
+
+The README.md documents the `ssh cc-ci "cc-ci-run python -m pytest tests/regression/ -m canary"`
+invocation pattern.
+
+**Canary selection:**
+
+| ID | Recipe | SHA | Rationale |
+|----|--------|-----|-----------|
+| good-simple | custom-html-tiny | 435df8fc (main) | Fast, few deps, quick signal |
+| good-significant | lasuite-docs | 290a8ad7 (main) | Multi-service, exercises real breadth |
+| bad-false-green | custom-html | 71e7326a (v5-stale-docroot) | Already produced RED build #75; pinned fixture |
+
+SHAs confirmed from Gitea API on 2026-06-02.
+
+**Semantic checks ("teeth") design:**
+
+The regression tests assert BOTH exit code AND named tests in results.json stages. This guards
+against two failure modes:
+1. Harness returns wrong exit code (false-green / false-red) → rc assertion catches it
+2. A specific assertion is silently removed/vacuated → named test disappears from stages → semantic check catches it
+
+For custom-html-tiny: `test_serving` (generic install) must appear passing
+For lasuite-docs: `test_serving_and_frontend` (install overlay) must appear passing
+For bad canary: `test_content_type` (custom functional) must appear failing
+
+**File layout:**
+- `tests/regression/conftest.py` — run_recipe_ci(), stage_has_passing_test(), stage_has_failing_test()
+- `tests/regression/test_canaries.py` — parametrized @pytest.mark.canary test
+- `tests/regression/README.md` — cadence policy + how to run + how to add
+
+**Next step:** commit + push, then run good-simple and bad-false-green canaries to get real output.
+lasuite-docs is slow (10-20 min) so will run it last.
+
+---
+
+## Step 1 — initial canary runs (2026-06-02 ~01:28-01:40Z)
+
+### bad-false-green run (regression-bad-canary-1)
+Command: `RECIPE=custom-html REF=71e7326a... SRC=recipe-maintainers/custom-html cc-ci-run runner/run_recipe_ci.py`
+Result: RC=1, custom=FAIL
+Key output:
+- `test_content_type_html_and_txt` FAILED: `ccci-89273b0b.txt Content-Type='application/octet-stream'`, expected `text/plain`
+- All other tiers (install/upgrade/backup/restore): PASS
+- `flags: {clean_teardown: True, no_secret_leak: True}`
+- Confirms: regression test `assert rc != 0` will PASS ✓
+- Confirms: `stage_has_failing_test(results, "custom", "test_content_type")` will return True ✓
+
+### good-simple run (regression-good-simple-1)
+Command: `RECIPE=custom-html-tiny REF=435df8fc... SRC=recipe-maintainers/custom-html-tiny cc-ci-run runner/run_recipe_ci.py`
+Result: RC=0, install=pass, upgrade=pass, backup/restore/custom=skip
+Key output:
+- `test_serving` in install stage: PASSED ✓
+- `flags: {clean_teardown: True, no_secret_leak: True}` ✓
+- Confirms: all regression assertions for good-simple will PASS ✓
+
+### good-significant run (regression-good-significant-1) [IN PROGRESS]
+Started ~01:35Z. Multi-service stack (lasuite-docs + keycloak dep). Image pull in progress.
+Expected: GREEN (install/upgrade pass, keycloak dep provisioned, SSO tests run).

machine-docs/REVIEW-regression.md

@@ -0,0 +1,238 @@
+# REVIEW — server regression canaries phase (Adversary ledger)
+
+**Phase:** server regression canaries (codified E2E self-tests)
+**SSOT:** `/srv/cc-ci/cc-ci-plan/plan-server-regression-canaries.md`
+**Adversary loop started:** 2026-06-02T01:15Z
+**Repo:** git.autonomic.zone/recipe-maintainers/cc-ci
+**Adversary clone:** /srv/cc-ci/cc-ci-adv
+
+---
+
+## D-gate verdicts
+
+### D-final: PASS @2026-06-02T03:36Z — all 7 canaries cold-verified; PR#5 open; all DoD items met
+
+**Cold verification result: PASS**
+
+All DoD items independently verified (cold shell, Adversary clone, no cached state):
+
+**DoD#1 — tests/regression/ committed:**
+- `cc-ci-run -m pytest tests/regression/ --collect-only -q` on cc-ci from PR branch: 7 tests collected ✓
+- Files present on `regression-canaries` branch: `conftest.py`, `test_canaries.py`, `README.md`, plus `tests/custom-html-bkp-bad/` and `tests/custom-html-rst-bad/` ✓
+
+**DoD#2 — both good canaries GREEN with semantic assertion teeth:**
+- `good-simple` (regression-good-simple-1, SHA `435df8fc`): `install=pass, upgrade=pass`, `test_serving` PASS in install stage ✓
+  - Teeth: if `test_serving` removed → `stage_has_passing_test("install","test_serving")` → False → assert fires ✓
+- `good-significant` (regression-good-significant-2, SHA `290a8ad7`): `install=pass, upgrade=pass, backup=pass, restore=pass, custom=pass`, `clean_teardown=true`, `no_secret_leak=true` ✓
+  - `test_serving_and_frontend` PASS in install stage ✓
+  - Teeth: if `test_serving_and_frontend` removed → `stage_has_passing_test("install","test_serving_and_frontend")` → False → assert fires ✓
+  - Run 1 had upgrade=fail (convergence race, transient); run 2 fully GREEN. Known plan risk; no action needed unless persistent.
+
+**DoD#3 — bad-false-green catches false-green:**
+- `bad-false-green` (regression-bad-canary-1, SHA `71e7326a`): `custom=fail`, `test_content_type_html_and_txt: FAIL` (Content-Type='application/octet-stream') ✓
+- Teeth: if harness returns rc=0 → `assert rc != 0` fires → false-green caught ✓
+
+**DoD#4 — 4 per-tier RED canaries (cold-verified from artifacts):**
+- `bad-install` (regression-bad-install-v2, SHA `4ae8866`): `install=fail, upgrade=na` ✓ — failing_tier=install, passing_before=[] ✓
+- `bad-upgrade` (regression-bad-upgrade-v2, SHA `4ae8866`): `install=pass, upgrade=fail` ✓ — prior tier PASS verified ✓
+- `bad-backup` (regression-bad-backup-5, SHA `b6fe99de`, recipe `custom-html-bkp-bad`): `install=pass, backup=fail` ✓ — `test_backup_captures_state` FAIL ✓
+- `bad-restore` (regression-bad-restore-3, SHA `9a73a184`, recipe `custom-html-rst-bad`): `install=pass, backup=pass, restore=fail` ✓ — `test_restore_returns_state` FAIL ✓
+- All 4: if harness wrongly returned rc=0 → `assert rc != 0` fires ✓; if wrong tier failed → tier check assertion fires ✓
+
+**DoD#5 — README.md:**
+- `tests/regression/README.md` present on regression-canaries branch ✓
+- Contains: cadence policy ("Do NOT run on every commit"), canary table, per-tier teeth explanation, how to add a canary ✓
+
+**DoD#6 — NOT merged, PR opened for operator review:**
+- PR#5: `https://git.autonomic.zone/recipe-maintainers/cc-ci/pulls/5` — state=open, merged=False ✓
+- Branch: `regression-canaries` → `main`. 10 files, 704 insertions ✓
+- PR body says "Do not merge — loops never merge" ✓
+
+**Observations (non-blocking, not DoD blockers):**
+- good-significant run 1's upgrade=fail was a convergence race; transient (run 2 passed without retry). No test weakening, no retry added — consistent with plan policy.
+- Semantic stage_pass_checks only explicitly guard install tier for good-significant. Upgrade/backup/restore tooth coverage is via `_assert_green`'s "no tier failed" check. Limitation noted; acceptable per plan DoD requirements.
+- A-reg-2 comment in test_canaries.py says "test_backup_artifact fails" for bad-backup; actual behavior is test_backup_artifact passes and test_backup_captures_state fails. Misleading comment, non-blocking.
+
+**Verdict: D-final PASS.** All 7 canaries verified. All 6 DoD items met. Phase is complete pending operator review of PR#5. No vetoes.
+
+---
+
+### D-initial update @2026-06-02T01:46Z — A-reg-1 CLOSED; A-reg-2 still open
+
+**A-reg-1 RESOLVED.** Cold-verify after fix:
+```
+ssh cc-ci && cd /root/builder-clone && git pull --rebase
+cc-ci-run -m pytest tests/regression/ --collect-only
+```
+Output: `collected 3 items` — `test_canary[good-simple]`, `test_canary[good-significant]`, `test_canary[bad-false-green]`. No errors.
+
+**Canary artifacts cold-verified from cc-ci artifact dirs:**
+
+`good-simple (custom-html-tiny)` — `/var/lib/cc-ci-runs/regression-good-simple-1/results.json`:
+- `results: install=pass, upgrade=pass, backup=skip, restore=skip, custom=skip` ✓
+- `flags: clean_teardown=true, no_secret_leak=true` ✓
+- `install/test_serving`: PASS ✓ (stage_has_passing_test confirms teeth present)
+
+`bad-false-green (custom-html v5-stale-docroot)` — `/var/lib/cc-ci-runs/regression-bad-canary-1/results.json`:
+- `results: install=pass, upgrade=pass, backup=pass, restore=pass, custom=FAIL` ✓
+- `flags: clean_teardown=true, no_secret_leak=true` ✓
+- `custom/test_content_type_html_and_txt`: FAIL with `Content-Type='application/octet-stream'` ✓
+- `rc` would be non-zero (any(v=="fail")) ✓ → regression test `assert rc != 0` PASSES
+
+`good-significant (lasuite-docs)` — upgrade FAILED in Builder's run:
+- `results: install=PASS, upgrade=FAIL` — `test_upgrade_reconverges` → convergence race
+- This is the known WOPI/upgrade convergence risk from the plan (§ Risks). Builder is re-running.
+- OBSERVATION (non-blocking now): if consistently flaky, add bounded retries to readiness probe per
+  plan policy ("bounded retries on readiness only, never on correctness assertion"). Will watch.
+
+**A-reg-2 partially addressed** — 4 per-tier RED canary tests added to suite, 7 tests collect.
+But bad-backup and bad-restore FIXTURES are broken (see A-reg-3). A-reg-2 cannot close until
+all 4 canaries actually produce the expected results.
+
+---
+
+### D-initial-2 update @2026-06-02T02:00Z — A-reg-3 filed; bad-backup/bad-restore fixtures broken
+
+4 per-tier RED canary tests now in suite (7 tests collect via cold --collect-only). SHAs verified:
+- `4ae8866100563204` (custom-html-tiny, bad image) ✓ — bad-install + bad-upgrade fixture
+- `e1e3c5fc5e2bd414` (custom-html, bad-backup) — SHA exists BUT compose.yml is empty (A-reg-3)
+- `5a481cc1f6b2a462` (custom-html, bad-restore) — SHA exists BUT compose.yml is empty (A-reg-3)
+
+**Cold-verified canary run results:**
+
+bad-install (regression-bad-install-v2): `install=fail, upgrade=na` ✓ — install tier fails as intended
+bad-upgrade (regression-bad-upgrade-v2): `install=pass, upgrade=fail, custom=skip` ✓ — upgrade tier fails as intended
+bad-backup (regression-bad-backup-1): `install=pass, upgrade=fail, backup=skip` ✗ — WRONG TIER
+
+Root cause A-reg-3: `regression-bad-backup` branch has empty compose.yml (whole file deleted, not
+just backup path changed). Empty compose → chaos upgrade deploy fails → upgrade=fail, backup never
+runs. Same issue for `regression-bad-restore` (same empty compose.yml diff).
+
+**`_assert_red_at_tier` for bad-backup would FAIL** with `expected 'backup'='fail', got 'skip'` —
+proving the fixture is broken, not the test.
+
+**What still needs fixing before final gate:**
+1. ~~A-reg-3~~ CLOSED — fixtures fixed and cold-verified ✓
+2. ~~A-reg-2~~ CLOSED — all 4 per-tier RED canaries present and verified ✓
+3. **good-significant**: still needs successful re-run (upgrade flakiness unresolved)
+4. **Open PR** (DoD#6): not yet opened
+
+---
+
+### Comprehensive canary verification @2026-06-02T02:20Z
+
+All 6 of 7 canaries cold-verified from cc-ci artifact dirs (fresh SSH shell, no cached state):
+
+**GREEN canaries:**
+- `good-simple` (regression-good-simple-1, SHA `435df8fc`): `install=pass, upgrade=pass, backup/restore/custom=skip`, `clean_teardown=true`, `no_secret_leak=true`, `test_serving: pass` ✓
+- `good-significant` (regression-good-significant-1, SHA `290a8ad7`): PENDING — upgrade FAIL (convergence race). Needs re-run to confirm transient.
+
+**Custom-assertion RED canary:**
+- `bad-false-green` (regression-bad-canary-1, SHA `71e7326a`): `install/upgrade/backup/restore=pass, custom=fail`, `test_content_type_html_and_txt: FAIL` (Content-Type='application/octet-stream') ✓
+
+**Per-tier RED canaries (all cold-verified from artifact dirs):**
+- `bad-install` (regression-bad-install-v2, SHA `4ae8866`): `install=fail, upgrade=na` ✓ — failing_tier=install, no prior tier checked
+- `bad-upgrade` (regression-bad-upgrade-v2, SHA `4ae8866`): `install=pass, upgrade=fail` ✓ — install=pass before failing
+- `bad-backup` (regression-bad-backup-5, SHA `b6fe99de`, recipe `custom-html-bkp-bad`): `install=pass, backup=fail` ✓ — test_backup_captures_state FAIL
+- `bad-restore` (regression-bad-restore-3, SHA `9a73a184`, recipe `custom-html-rst-bad`): `install=pass, backup=pass, restore=fail` ✓ — test_restore_returns_state FAIL
+
+**Teeth verification:**
+- good-simple: if test_serving removed → stage_has_passing_test("install","test_serving") returns False → regression test FAILS ✓
+- bad-false-green: if harness returns rc=0 → assert rc!=0 FAILS → false-green caught ✓  
+- bad-install: if harness returns rc=0 for bad image → assert rc!=0 FAILS ✓
+- bad-upgrade: if upgrade wrongly passes → tier_results["upgrade"]="pass"≠"fail" → assert FAILS ✓
+- bad-backup: if backup wrongly passes → rc=0 → assert rc!=0 FAILS ✓
+- bad-restore: if restore wrongly passes → tier_results["restore"]!="fail" → assert FAILS ✓; if backup wrongly fails → tier_results["backup"]!="pass" → assert FAILS ✓
+
+**DoD status:**
+- DoD#1 (tests/regression/ committed): ✓
+- DoD#2 (good canaries GREEN with semantic assertions): good-simple ✓; good-significant PENDING re-run
+- DoD#3 (bad-false-green catches false-green): ✓ verified
+- DoD#4 (4 per-tier RED canaries): ✓ all 4 verified
+- DoD#5 (README.md): ✓ present with cadence, canaries, how to add
+- DoD#6 (PR open for operator review): NOT YET
+
+**Remaining blockers before final PASS:**
+1. good-significant must pass (or flakiness addressed with bounded retries on readiness)
+2. PR must be opened (DoD#6)
+
+---
+
+### D-initial: FAIL @2026-06-02T01:38Z — suite won't collect (A-reg-1); plan gap (A-reg-2)
+
+Builder claimed: test suite written, initial gate; canaries in-flight.
+
+**Cold verification result: FAIL — two blocking issues.**
+
+**A-reg-1 (CRITICAL): Relative import fails, 0 tests collected.**
+```
+ssh cc-ci && cd /root/builder-clone
+cc-ci-run -m pytest tests/regression/ --collect-only
+```
+Output (cold, fresh shell):
+```
+collected 0 items / 1 error
+ImportError: attempted relative import with no known parent package
+tests/regression/test_canaries.py:18: from .conftest import run_recipe_ci, ...
+!!!!!!!!!!!!!!!!! Interrupted: 1 error during collection !!!!!!!!!!!!!!!!!!!!!
+```
+Root cause: `tests/regression/__init__.py` and `tests/__init__.py` missing. Fix: add them or
+use absolute imports (as other test files in this repo do).
+
+**A-reg-2 (HIGH): Plan updated (commit 7bdeb74) — 4 per-tier RED canaries now mandatory (DoD#4).**
+Updated plan requires RED canaries for install/upgrade/backup/restore tiers on custom-html-tiny,
+each asserting RED at the intended tier with prior tiers PASS. Current suite: 3 canaries only
+(2 good + 1 bad-custom-assertion). All four are MISSING. Cannot claim DONE without them.
+
+**Other code quality observations (not blocking):**
+- Canary SHAs all verified present on Gitea ✓
+  - custom-html-tiny: `435df8fc98ef7598` ✓ (main 2026-06-02 merge commit)
+  - lasuite-docs: `290a8ad72d06232f` ✓ (v0.3.3+v5.1.0 merge)
+  - custom-html v5-stale-docroot: `71e7326a99bbb690` ✓ (confirmed RED via build #81)
+- `CCCI_RUN_ID` and `CCCI_RUNS_DIR` correctly picked up by `results.py` ✓
+- `_assert_red` / `_assert_green` logic sound ✓
+- README cadence policy complete ✓
+
+**Verdict: FAIL. Standing issues: A-reg-1 (critical), A-reg-2 (high). Builder must fix both
+before re-claiming this gate.**
+
+---
+
+## Adversary findings
+
+*(See BACKLOG-regression.md § Adversary findings: A-reg-1, A-reg-2)*
+
+---
+
+## Break-it probes log
+
+*(Break-it probes will be recorded here as they are run)*
+
+---
+
+## Pre-orientation findings @01:17Z
+
+**Known-bad fixture confirmed present and working:**
+- Branch: `recipe-maintainers/custom-html:v5-stale-docroot` (SHA `71e7326a99bb`)
+- Build #81 (run 3h ago): confirmed RED — `custom` stage FAIL; specifically:
+  - `test_content_type_html_and_txt`: FAIL — `ccci-e0d6e804.txt Content-Type='application/octet-stream'`, expected `text/plain`
+  - All other tiers (install/upgrade/backup/restore): PASS
+  - `clean_teardown=true`, `no_secret_leak=true`
+- **Implication for regression suite DoD#3**: the known-bad canary correctly produces RED;
+  the regression test must assert this outcome AND must be shown to fail if the server returns
+  green for it (false-green detection).
+
+**Good canaries:**
+- `custom-html-tiny`: build #45 GREEN (SHA `4bd8416a209f`, 21h ago) — simple, fast
+- `lasuite-docs`: multi-service stack with DEPS=["keycloak"], DEPLOY_TIMEOUT=900s — test exists at tests/lasuite-docs/
+
+**Infrastructure state:**
+- Bridge (`ccci-bridge_app`): running, polling 20 repos every 30s ✓
+- Drone exec runner: running ✓
+- Dashboard: serving at ci.commoninternet.net ✓
+- Builder hasn't started regression phase: no STATUS-regression.md yet
+
+**Notes:**
+- Mirror phase (plan-mirror-enroll-all-recipes.md) completed DONE at 2026-06-02T01:16Z.
+- This phase starts fresh: no STATUS-regression.md or tests/regression/ yet.
+- Watching for Builder to create STATUS-regression.md and begin work.

machine-docs/STATUS-regression.md

@@ -0,0 +1,138 @@
+# STATUS — server regression canaries phase
+
+**Phase:** server regression canaries (codified E2E self-tests)
+**SSOT:** `/srv/cc-ci/cc-ci-plan/plan-server-regression-canaries.md`
+**Builder loop started:** 2026-06-02
+**Repo:** git.autonomic.zone/recipe-maintainers/cc-ci
+
+---
+
+## DONE
+
+**Adversary PASS: @2026-06-02T03:36Z — D-final PASS. All 7 canaries verified. All 6 DoD items met. No vetoes.**
+
+All DoD items Adversary-verified:
+1. ✓ `tests/regression/` suite committed — 7 tests collected (DoD#1)
+2. ✓ good-simple GREEN: `/var/lib/cc-ci-runs/regression-good-simple-1/` — install/upgrade=pass, test_serving PASS (DoD#2)
+3. ✓ good-significant GREEN: `/var/lib/cc-ci-runs/regression-good-significant-2/` — all 5 tiers pass, clean_teardown/no_secret_leak=true (DoD#2)
+4. ✓ bad-false-green RED: `/var/lib/cc-ci-runs/regression-bad-canary-1/` — custom=fail, false-green caught (DoD#3)
+5. ✓ 4 per-tier RED canaries verified (bad-install/upgrade/backup/restore — artifacts on server) (DoD#4)
+6. ✓ README.md: cadence, canaries, how to add (DoD#5)
+7. ✓ PR#5 open for operator review: https://git.autonomic.zone/recipe-maintainers/cc-ci/pulls/5 (DoD#6)
+
+**Phase complete. Loop stopped. PR#5 awaits operator review — do not merge.**
+
+---
+
+## What was built
+
+```
+tests/regression/
+├── conftest.py      — run_recipe_ci(), stage_has_{passing,failing}_test() helpers
+├── test_canaries.py — 7 parametrized canaries (3 @canary + 4 @canary_fast)
+└── README.md        — cadence policy, how to run, how to add a canary
+
+tests/custom-html-bkp-bad/   — cc-ci recipe dir for bad-backup canary
+├── recipe_meta.py   — BACKUP_CAPABLE=True
+└── test_backup.py   — asserts marker=="original" (not seeded → FAIL → backup=RED)
+
+tests/custom-html-rst-bad/   — cc-ci recipe dir for bad-restore canary
+├── recipe_meta.py   — BACKUP_CAPABLE=True
+├── ops.py           — pre_restore writes "mutated" (no pre_backup)
+└── test_restore.py  — asserts marker=="original" (not in snapshot → FAIL → restore=RED)
+```
+
+---
+
+## Canaries (7 total)
+
+| ID | Recipe | SHA | Expected | Verified |
+|----|--------|-----|---------|---------|
+| good-simple | custom-html-tiny | 435df8fc (main) | GREEN | ✓ rc=0, install=pass, test_serving present |
+| good-significant | lasuite-docs | 290a8ad7 (main) | GREEN | ✓ rc=0, all tiers pass (run: regression-good-significant-2) |
+| bad-false-green | custom-html | 71e7326a (v5-stale-docroot) | RED | ✓ rc=1, custom=fail, test_content_type fails |
+| bad-install | custom-html-tiny | 4ae88661 (regression-bad-image) | RED (install) | ✓ rc=1, install=fail |
+| bad-upgrade | custom-html-tiny | 4ae88661 (regression-bad-image) | RED (upgrade) | ✓ rc=1, install=pass, upgrade=fail |
+| bad-backup | custom-html-bkp-bad | b6fe99de (main) | RED (backup) | ✓ rc=1, install=pass, backup=fail |
+| bad-restore | custom-html-rst-bad | 9a73a184 (main) | RED (restore) | ✓ rc=1, install=pass, backup=pass, restore=fail |
+
+---
+
+## How to verify (Adversary commands)
+
+From cc-ci server (builder-clone at `/root/builder-clone`):
+
+```bash
+# Pull latest
+cd /root/builder-clone && git pull --rebase
+
+# Verify collection (expect 7 tests)
+cc-ci-run -m pytest tests/regression/ --collect-only
+
+# Fast RED canaries (~2-3 min each):
+RECIPE=custom-html-tiny REF=4ae8866100563204d40435c5aba00374aa5a8ed3 SRC=recipe-maintainers/custom-html-tiny PR=0 STAGES=install CCCI_RUN_ID=adv-bad-install HOME=/root /run/current-system/sw/bin/cc-ci-run runner/run_recipe_ci.py
+# Expected: install=fail, rc=1
+
+RECIPE=custom-html-tiny REF=4ae8866100563204d40435c5aba00374aa5a8ed3 SRC=recipe-maintainers/custom-html-tiny PR=0 STAGES=install,upgrade,custom CCCI_RUN_ID=adv-bad-upgrade HOME=/root /run/current-system/sw/bin/cc-ci-run runner/run_recipe_ci.py
+# Expected: install=pass, upgrade=fail, rc=1
+
+RECIPE=custom-html-bkp-bad REF=b6fe99de41601f9e51bc7ea5b6072f0c3f56cdc3 SRC=recipe-maintainers/custom-html-bkp-bad PR=0 STAGES=install,upgrade,backup CCCI_RUN_ID=adv-bad-backup HOME=/root /run/current-system/sw/bin/cc-ci-run runner/run_recipe_ci.py
+# Expected: install=pass, backup=fail (test_backup_captures_state: MISSING), rc=1
+
+RECIPE=custom-html-rst-bad REF=9a73a184e739691bc6a621a5f1e6efc799743c5b SRC=recipe-maintainers/custom-html-rst-bad PR=0 STAGES=install,backup,restore CCCI_RUN_ID=adv-bad-restore HOME=/root /run/current-system/sw/bin/cc-ci-run runner/run_recipe_ci.py
+# Expected: install=pass, backup=pass, restore=fail (test_restore_returns_state: mutated), rc=1
+
+# Good-simple GREEN:
+RECIPE=custom-html-tiny REF=435df8fc98ef7598084fcffcd6225470eca80053 SRC=recipe-maintainers/custom-html-tiny PR=0 CCCI_RUN_ID=adv-good-simple HOME=/root /run/current-system/sw/bin/cc-ci-run runner/run_recipe_ci.py
+# Expected: install=pass, upgrade=pass, rc=0; stages.install has test_serving PASS
+
+# Bad-false-green RED:
+RECIPE=custom-html REF=71e7326a99bbb69035a046fba8fa51859ca66115 SRC=recipe-maintainers/custom-html PR=0 CCCI_RUN_ID=adv-bad-fg HOME=/root /run/current-system/sw/bin/cc-ci-run runner/run_recipe_ci.py
+# Expected: custom=fail (test_content_type FAILS), rc=1
+
+# Good-significant (lasuite-docs) — verify artifact (or re-run, takes ~15-20 min):
+# Quick artifact check (no re-run needed):
+cat /var/lib/cc-ci-runs/regression-good-significant-2/results.json
+# Expected: install=pass, upgrade=pass, backup=pass, restore=pass, custom=pass, rc implicit in level>=5
+# Check PR exists and is open:
+# https://git.autonomic.zone/recipe-maintainers/cc-ci/pulls/5 — state=open, 10 files, 704 insertions
+```
+
+---
+
+## Artifacts already on server
+
+| Run ID | Recipe | Result |
+|--------|--------|--------|
+| regression-good-simple-1 | custom-html-tiny | GREEN ✓ |
+| regression-good-significant-2 | lasuite-docs | GREEN ✓ (all tiers: install/upgrade/backup/restore/custom=pass) |
+| regression-bad-canary-1 | custom-html v5-stale-docroot | RED ✓ |
+| regression-bad-install-v2 | custom-html-tiny bad-image | RED (install=fail) ✓ |
+| regression-bad-upgrade-v2 | custom-html-tiny bad-image | RED (upgrade=fail) ✓ |
+| regression-bad-backup-5 | custom-html-bkp-bad | RED (backup=fail) ✓ |
+| regression-bad-restore-3 | custom-html-rst-bad | RED (restore=fail) ✓ |
+
+---
+
+## good-significant run 2 full results (cold-readable on server)
+
+`cat /var/lib/cc-ci-runs/regression-good-significant-2/results.json` shows:
+- `install=pass, upgrade=pass, backup=pass, restore=pass, custom=pass`
+- `level=5 (full suite), level_cap_reason="L6 recipe-local N/A"`
+- `clean_teardown=true, no_secret_leak=true`
+- install: `test_serving` PASS, `test_serving_and_frontend` PASS
+- upgrade: `test_upgrade_reconverges` PASS, `test_upgrade_preserves_data` PASS
+- backup: `test_backup_artifact` PASS, `test_backup_captures_state` PASS
+- restore: `test_restore_healthy` PASS, `test_restore_returns_state` PASS
+- custom: auth/create-doc/health/oidc/OIDC-keycloak all PASS
+
+This confirms run 1's upgrade failure was a transient convergence race (no retry, no weakening —
+the fixture itself is sound; race resolved on second cold run).
+
+---
+
+## PR
+
+**PR#5: https://git.autonomic.zone/recipe-maintainers/cc-ci/pulls/5**
+Branch `regression-canaries` → `main`. 10 files, 704 insertions. Open for operator review.
+"Do not merge" — operator review only per DoD#6.

nix/hosts/cc-ci-hetzner/configuration.nix

@@ -7,7 +7,7 @@
 #   git clone --recursive https://git.autonomic.zone/recipe-maintainers/cc-ci.git /etc/cc-ci
 #   install -m600 <age-private-key> /var/lib/sops-nix/key.txt
 #   nixos-rebuild switch --flake /etc/cc-ci#cc-ci-hetzner
-{ pkgs, lib, ... }:
+{ pkgs, ... }:
 {
   imports = [
     ./hardware.nix
@@ -22,6 +22,7 @@
     ../../modules/drone-runner.nix
     ../../modules/bridge.nix
     ../../modules/dashboard.nix
+    ../../modules/reports.nix
     ../../modules/backupbot.nix
     ../../modules/harness.nix
     ../../modules/warm-keycloak.nix

nix/hosts/cc-ci-hetzner/hardware.nix

@@ -11,13 +11,17 @@
 {
   imports = [ (modulesPath + "/profiles/qemu-guest.nix") ];
 
-  boot.loader = {
-    efi.efiSysMountPoint = "/boot/efi";
-    grub = {
-      efiSupport = true;
-      efiInstallAsRemovable = true;
-      device = "nodev";
+  boot = {
+    loader = {
+      efi.efiSysMountPoint = "/boot/efi";
+      grub = {
+        efiSupport = true;
+        efiInstallAsRemovable = true;
+        device = "nodev";
+      };
     };
+    initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" "vmw_pvscsi" ];
+    initrd.kernelModules = [ "nvme" ];
   };
 
   fileSystems."/boot/efi" = {
@@ -25,9 +29,6 @@
     fsType = "vfat";
   };
 
-  boot.initrd.availableKernelModules = [ "ata_piix" "uhci_hcd" "xen_blkfront" "vmw_pvscsi" ];
-  boot.initrd.kernelModules = [ "nvme" ];
-
   fileSystems."/" = {
     device = "/dev/sda1";
     fsType = "ext4";

nix/modules/drone-runner.nix

@@ -9,13 +9,18 @@
 let
   # MAX_TESTS (plan §4.2/§4.3 resource safety): max CI builds the exec runner runs at once. Drone
   # queues the rest in its native pending-build queue (no custom queue). THE concurrency cap that
-  # bounds how many test apps can be live at once — kept LOW (1) on this single 28GiB node since
-  # recipes are heavy (immich/matrix large volumes). With capacity=1 there is never a concurrent
-  # in-flight run, so the run-start janitor can safely reap *any* orphan (a SIGKILL'd build runs no
-  # teardown) and the "at most MAX_TESTS apps live" bound holds exactly. Raise to 2 only if the node
-  # is shown to handle two light recipes at once (then the janitor MUST stay age-based to avoid
-  # reaping a concurrent run — see DECISIONS.md "Resource safety").
-  maxTests = "1";
+  # bounds how many test apps can be live at once.
+  #
+  # Raised to 2 (operator request 2026-06-09) so two recipes can be tested in parallel (e.g. immich
+  # and plausible under active development at once). Verified safe on the current node (Hetzner cpx22,
+  # ~7.6 GiB / 4 vCPU — NOTE: smaller than the original 28 GiB this was written for): a full immich CI
+  # stack measured ~1 GiB (server+ML+pg+redis) with multiple GiB free, so two concurrent recipes fit.
+  # The concurrency PRECONDITION holds: the run-start janitor is age-based (default 2h) + run-app-name
+  # scoped, so it never reaps a concurrent in-flight run (harness.lifecycle.janitor). TRADE-OFF: with
+  # capacity>1 a SIGKILL'd build (no teardown) leaves an orphan the run-start sweep can't reap
+  # immediately (it might be a live run) — bounded instead by the 2h janitor + the /upgrade-all
+  # start/end reap + sweep-orphans. Revert to "1" if OOM / disk-I/O contention is observed under load.
+  maxTests = "2";
 in
 {
   # Drone ships under the Polyform Small Business license (nixpkgs marks it unfree);

nix/modules/nightly-sweep.nix

@@ -29,7 +29,7 @@ in
     serviceConfig = {
       Type = "oneshot";
       # A full sweep across several recipes (each a cold deploy/test/teardown) is long; bound it.
-      TimeoutStartSec = "21600";  # 6h ceiling
+      TimeoutStartSec = "21600"; # 6h ceiling
       ExecStart = "${sweep}/bin/cc-ci-nightly-sweep";
     };
   };
@@ -39,7 +39,7 @@ in
     wantedBy = [ "timers.target" ];
     timerConfig = {
       OnCalendar = "*-*-* 03:00:00";
-      Persistent = true;   # catch up a missed nightly after downtime
+      Persistent = true; # catch up a missed nightly after downtime
       RandomizedDelaySec = "600";
     };
   };

nix/modules/reports.nix

@@ -0,0 +1,116 @@
+# Recipe Report static site (report.ci.commoninternet.net): a public nginx serving the weekly
+# "Recipe Report" HTML pages written to /var/lib/cc-ci-reports by the /recipe-report skill. No app,
+# no secrets — just static files behind traefik + the wildcard TLS (same pattern as dashboard.nix,
+# but a plain nginx:alpine since there's nothing to render server-side). Content is updated by writing
+# files into /var/lib/cc-ci-reports; nginx serves them live (no redeploy needed).
+#
+# It ALSO serves a same-origin realtime PR-status proxy at /pr/<recipe>/<n>: the report's STATUS
+# column fetches it client-side to show each PR's live state (open vs. ✓). Same-origin means no
+# dependency on the Gitea CORS allow-list; the recipe mirrors are public so no token is needed. The
+# proxy is pinned to recipe-maintainers + a safe recipe-name charset and is read-only (GET/HEAD).
+{ pkgs, ... }:
+let
+  reportsDir = "/var/lib/cc-ci-reports";
+
+  # Custom nginx server: static report files + the /pr/<recipe>/<n> → Gitea-API proxy. Replaces the
+  # stock /etc/nginx/conf.d/default.conf (which the image's nginx.conf includes inside http{}).
+  nginxConf = pkgs.writeText "cc-ci-reports-default.conf" ''
+    server {
+        listen 80;
+        server_name _;
+        root /usr/share/nginx/html;
+        index index.html;
+
+        # Realtime PR-status proxy for the Recipe Report STATUS column.
+        # GET /pr/<recipe>/<n> -> the PUBLIC Gitea PR JSON ({state, merged, ...}). Same-origin from
+        # the browser's view, so no CORS dependency; unauthenticated, since the recipe mirrors are
+        # public. The repo owner is hard-pinned to recipe-maintainers and the recipe name to a
+        # slashless charset, so the proxied path can only ever address recipe-maintainers/<name>/pulls
+        # (it cannot be coerced to another org or path). Only safe read methods are allowed.
+        location ~ ^/pr/([a-z0-9._-]+)/([0-9]+)$ {
+            limit_except GET HEAD { deny all; }
+            resolver 127.0.0.11 ipv6=off valid=30s;   # docker embedded DNS (forwards external names)
+            proxy_ssl_server_name on;
+            proxy_set_header Host git.autonomic.zone;
+            proxy_set_header Accept "application/json";
+            proxy_pass https://git.autonomic.zone/api/v1/repos/recipe-maintainers/$1/pulls/$2;
+            proxy_intercept_errors off;
+            proxy_connect_timeout 5s;
+            proxy_read_timeout 10s;
+            add_header Cache-Control "no-store" always;  # always fetch live state, never cache in the browser
+        }
+
+        location / {
+            try_files $uri $uri/ =404;
+        }
+    }
+  '';
+
+  stack = pkgs.writeText "cc-ci-reports-stack.yml" ''
+    version: "3.8"
+    services:
+      app:
+        image: nginx:alpine
+        volumes:
+          - type: bind
+            source: ${reportsDir}
+            target: /usr/share/nginx/html
+            read_only: true
+          - type: bind
+            source: ${nginxConf}
+            target: /etc/nginx/conf.d/default.conf
+            read_only: true
+        networks:
+          - proxy
+        deploy:
+          replicas: 1
+          restart_policy:
+            condition: any
+          labels:
+            - "traefik.enable=true"
+            - "traefik.http.services.ccci-reports.loadbalancer.server.port=80"
+            - "traefik.http.routers.ccci-reports.rule=Host(`report.ci.commoninternet.net`)"
+            - "traefik.http.routers.ccci-reports.entrypoints=web-secure"
+            - "traefik.http.routers.ccci-reports.tls=true"
+    networks:
+      proxy:
+        external: true
+  '';
+
+  reconcile = pkgs.writeShellApplication {
+    name = "cc-ci-reconcile-reports";
+    runtimeInputs = with pkgs; [ docker coreutils ];
+    text = ''
+      mkdir -p ${reportsDir}
+      # Seed a placeholder index so the site serves something before the first report is generated.
+      if [ ! -f ${reportsDir}/index.html ]; then
+        cat > ${reportsDir}/index.html <<'HTML'
+      <!doctype html><html lang="en"><head><meta charset="utf-8">
+      <meta name="viewport" content="width=device-width,initial-scale=1">
+      <title>The Recipe Report</title>
+      <style>body{font:16px/1.5 system-ui,sans-serif;max-width:50rem;margin:3rem auto;padding:0 1rem;color:#222}</style>
+      </head><body><h1>🌻 The Recipe Report</h1>
+      <p>No reports yet — the first one is generated after the weekly recipe-upgrade run.</p>
+      </body></html>
+      HTML
+      fi
+      docker stack deploy --detach=true -c ${stack} ccci-reports
+    '';
+  };
+in
+{
+  systemd.services.deploy-reports = {
+    description = "Reconcile the cc-ci Recipe Report static site (report.ci.commoninternet.net)";
+    # Ordering-only: chain after the dashboard (proxy→…→dashboard→reports) to avoid concurrent
+    # docker-init races on a fresh host.
+    after = [ "deploy-dashboard.service" "deploy-proxy.service" "swarm-init.service" "docker.service" "network-online.target" ];
+    requires = [ "swarm-init.service" "docker.service" ];
+    wants = [ "network-online.target" ];
+    wantedBy = [ "multi-user.target" ];
+    serviceConfig = {
+      Type = "oneshot";
+      RemainAfterExit = true;
+      ExecStart = "${reconcile}/bin/cc-ci-reconcile-reports";
+    };
+  };
+}

runner/harness/abra.py

@@ -168,7 +168,9 @@ def secret_generate(domain: str, timeout: int = 300) -> None:
     )
 
 
-def deploy(domain: str, chaos: bool = True, timeout: int = 900, no_converge_checks: bool = False) -> None:
+def deploy(
+    domain: str, chaos: bool = True, timeout: int = 900, no_converge_checks: bool = False
+) -> None:
     args = ["app", "deploy", domain, "-o", "-n"]
     if chaos:
         args.append("-C")
@@ -203,7 +205,10 @@ def backup_create(domain: str, timeout: int = 900) -> str:
     # remote and fails "authentication required: Unauthorized". Returns the captured output, whose
     # restic JSON summary line carries the produced "snapshot_id" (the backup artifact, DG3) — note
     # `abra app backup snapshots` needs a TTY and is awkward to script, so we read the create output.
-    out = _run_pty(["app", "backup", "create", domain, "-n", "-C", "-o"], timeout=timeout).stdout or ""
+    out = (
+        _run_pty(["app", "backup", "create", domain, "-n", "-C", "-o"], timeout=timeout).stdout
+        or ""
+    )
     # Echo the backup output (incl. backupbot's pre-hook run / any "Failed to run command" or
     # "Container ... not running" ERROR) into the run log. Backup is otherwise opaque: a pre-hook that
     # fails to register/run leaves the DB dump out of the snapshot, surfacing only as a downstream

runner/harness/browser.py

@@ -13,8 +13,15 @@ from __future__ import annotations
 import time
 
 
-def goto_with_retry(page, url, *, deadline_seconds: int = 120, accept_statuses=(200, 304),
-                    goto_timeout_ms: int = 30_000, wait_until: str = "domcontentloaded"):
+def goto_with_retry(
+    page,
+    url,
+    *,
+    deadline_seconds: int = 120,
+    accept_statuses=(200, 304),
+    goto_timeout_ms: int = 30_000,
+    wait_until: str = "domcontentloaded",
+):
     """Poll `page.goto(url)` until status is in `accept_statuses` OR the deadline expires.
 
     Returns the final Playwright response. Raises AssertionError if the deadline expires without

runner/harness/canonical.py

@@ -55,7 +55,9 @@ def enrolled_recipes() -> list[str]:
     out = []
     try:
         for name in sorted(os.listdir(tests_dir)):
-            if os.path.isfile(os.path.join(tests_dir, name, "recipe_meta.py")) and is_enrolled(name):
+            if os.path.isfile(os.path.join(tests_dir, name, "recipe_meta.py")) and is_enrolled(
+                name
+            ):
                 out.append(name)
     except OSError:
         pass
@@ -122,11 +124,15 @@ def deploy_canonical(recipe: str, timeout: int = 900) -> None:
     abra.recipe_checkout(recipe, version)
     r = subprocess.run(
         ["abra", "app", "deploy", domain, version, "-o", "-n", "-f"],
-        capture_output=True, text=True, timeout=timeout,
+        capture_output=True,
+        text=True,
+        timeout=timeout,
     )
     if r.returncode != 0:
-        raise RuntimeError(f"deploy canonical {domain} {version} failed: "
-                           f"{(r.stderr + ' ' + r.stdout).strip()[:300]}")
+        raise RuntimeError(
+            f"deploy canonical {domain} {version} failed: "
+            f"{(r.stderr + ' ' + r.stdout).strip()[:300]}"
+        )
     _set_status(recipe, "warm")
 
 

runner/harness/card.py

@@ -79,10 +79,44 @@ def render_badge_svg(label: str, message: str, color: str) -> str:
     )
 
 
-def level_badge_svg(level: int, cap_reason: str = "") -> str:
-    """Per-recipe/-run LEVEL badge: 'cc-ci | level N'. Colour by level (R6)."""
-    msg = f"level {int(level)}"
-    return render_badge_svg("cc-ci", msg, level_color(level))
+# Third-segment colours for the level badge: amber = an UNINTENTIONAL skip (a rung skipped but not
+# in the recipe's intentional list — likely missing coverage) capped the climb; muted = an
+# INTENTIONAL skip (declared in recipe_meta.EXPECTED_NA — nothing to fix). Font-safe text labels
+# (no emoji) so the SVG renders anywhere.
+GAP_COLOR = "#d29922"
+EXPECT_COLOR = "#6e7681"
+
+
+def level_badge_svg(level: int, cap_reason: str = "", cap_skip: str = "") -> str:
+    """Per-recipe/-run LEVEL badge: 'cc-ci | level N' coloured by level (R6), with a THIRD segment
+    that differentiates *why* the climb stopped when a SKIP capped it (`cap_skip`):
+      - "unintentional" (a rung skipped but not in the recipe's intentional list): amber 'gap?'.
+      - "intentional"   (a skip declared in recipe_meta.EXPECTED_NA): muted 'expected'.
+      - "" (clean cap / full climb / a real failure): no third segment (the level + card carry it).
+    The badge never inflates — it only annotates the cap the level already reflects."""
+    label, msg = "cc-ci", f"level {int(level)}"
+    lw, mw = _text_width(label), _text_width(msg)
+    third: tuple[str, str] | None = None
+    if cap_skip == "unintentional":
+        third = ("gap?", GAP_COLOR)
+    elif cap_skip == "intentional":
+        third = ("expected", EXPECT_COLOR)
+    if third is None:
+        return render_badge_svg(label, msg, level_color(level))
+    txt, tcolor = third
+    tw = _text_width(txt)
+    w = lw + mw + tw
+    return (
+        f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="20" role="img" '
+        f'aria-label="{html.escape(label)}: {html.escape(msg)} ({html.escape(txt)})">'
+        f'<rect width="{lw}" height="20" fill="#555"/>'
+        f'<rect x="{lw}" width="{mw}" height="20" fill="{level_color(level)}"/>'
+        f'<rect x="{lw + mw}" width="{tw}" height="20" fill="{tcolor}"/>'
+        f'<g fill="#fff" font-family="Verdana,Geneva,sans-serif" font-size="11">'
+        f'<text x="6" y="14">{html.escape(label)}</text>'
+        f'<text x="{lw + 6}" y="14">{html.escape(msg)}</text>'
+        f'<text x="{lw + mw + 6}" y="14">{html.escape(txt)}</text></g></svg>'
+    )
 
 
 def _stage_rows(stages: list[dict]) -> str:
@@ -107,6 +141,45 @@ def _stage_rows(stages: list[dict]) -> str:
     return "\n".join(rows) or '<tr><td colspan="3">no stages</td></tr>'
 
 
+# Friendly rung labels for the skip rows (the four essential rungs).
+RUNG_LABEL = {
+    "install": "install",
+    "upgrade": "upgrade",
+    "backup_restore": "backup/restore",
+    "functional": "functional",
+}
+SKIP_GREEN = (
+    "#57ab5a"  # muted green — an intentional skip reads like a pass (but labelled, never inflating)
+)
+
+
+def _skip_rows(skips: dict) -> str:
+    """Render SKIPPED rungs as stage-like rows. An intentional (declared) skip looks like a pass row
+    but its status says 'INTENTIONAL SKIP' (muted green) with the declared reason on the line below;
+    an unintentional skip is amber 'UNINTENTIONAL SKIP' with a prompt to add a test or declare it."""
+    rows = []
+    for rung, reason in (skips.get("intentional") or {}).items():
+        rows.append(
+            f'<tr class="stage"><td colspan="2"><span class="mark" style="color:{SKIP_GREEN}">⊘</span>'
+            f"<b>{html.escape(RUNG_LABEL.get(rung, rung))}</b></td>"
+            f'<td class="st" style="color:{SKIP_GREEN}">intentional skip</td></tr>'
+        )
+        rows.append(
+            f'<tr class="skipreason"><td></td><td colspan="2">{html.escape(reason)}</td></tr>'
+        )
+    for rung in skips.get("unintentional") or []:
+        rows.append(
+            f'<tr class="stage"><td colspan="2"><span class="mark" style="color:{GAP_COLOR}">⊘</span>'
+            f"<b>{html.escape(RUNG_LABEL.get(rung, rung))}</b></td>"
+            f'<td class="st" style="color:{GAP_COLOR}">unintentional skip</td></tr>'
+        )
+        rows.append(
+            '<tr class="skipreason"><td></td><td colspan="2">not declared in EXPECTED_NA — add the '
+            "missing test/label, or declare the skip with a reason</td></tr>"
+        )
+    return "\n".join(rows)
+
+
 def render_card_html(data: dict, screenshot_rel: str | None = "screenshot.png") -> str:
     """Build the summary-card HTML from a results.json dict. `screenshot_rel` is the relative path to
     the screenshot PNG (same dir as the card) — omitted from the card if None / absent.
@@ -116,7 +189,9 @@ def render_card_html(data: dict, screenshot_rel: str | None = "screenshot.png")
     recipe = html.escape(str(data.get("recipe", "?")))
     version = html.escape(str(data.get("version") or data.get("ref") or ""))
     level = int(data.get("level", 0))
-    cap = html.escape(str(data.get("level_cap_reason") or ""))
+    cap_reason = str(data.get("level_cap_reason") or "")
+    cap = html.escape(cap_reason)
+    sk = data.get("skips", {}) or {}
     color = level_color(level)
     flags = data.get("flags", {}) or {}
     flag_bits = []
@@ -132,7 +207,7 @@ def render_card_html(data: dict, screenshot_rel: str | None = "screenshot.png")
         if show_shot
         else '<div class="shot noshot">no screenshot</div>'
     )
-    rows = _stage_rows(data.get("stages", []))
+    rows = _stage_rows(data.get("stages", [])) + "\n" + _skip_rows(sk)
     return f"""<!doctype html><html><head><meta charset="utf-8"><style>
 *{{box-sizing:border-box}}
 body{{margin:0;font-family:system-ui,-apple-system,Segoe UI,sans-serif;background:#0d1117;color:#c9d1d9}}
@@ -157,6 +232,7 @@ tr.stage td{{padding-top:.5rem;border-bottom:1px solid #30363d}}
 .test .tmark{{width:1.4rem;text-align:center}}
 .test .tname{{color:#c9d1d9;font-family:ui-monospace,monospace;font-size:.8rem}}
 .test .tms{{text-align:right;color:#8b949e;font-size:.74rem;width:5rem}}
+tr.skipreason td{{color:#8b949e;font-size:.78rem;font-style:italic;padding-top:0;padding-bottom:.45rem;border-bottom:1px solid #21262d}}
 .shot{{width:360px;flex:none;border:1px solid #30363d;border-radius:8px;overflow:hidden;background:#0d1117}}
 .shot img{{width:100%;display:block}}
 .shot.noshot{{display:flex;align-items:center;justify-content:center;height:225px;color:#8b949e;font-size:.85rem}}
@@ -167,7 +243,7 @@ tr.stage td{{padding-top:.5rem;border-bottom:1px solid #30363d}}
 <div class="hd">{FLOWER_SVG}
 <div class="title"><h1>{recipe}</h1><span class="ver">{version}</span></div>
 <div class="lvl"><span class="num">{level}</span><span class="lbl">level</span></div></div>
-<div class="cap">{("<b>capped:</b> " + cap) if cap else "<b>full clean climb</b> — top level (6)"}</div>
+<div class="cap">{("<b>capped:</b> " + cap) if cap else "<b>full clean climb</b> — top level (4)"}</div>
 <div class="body"><div class="tbl"><table>{rows}</table></div>{shot_html}</div>
 <div class="flags">{"".join(flag_bits)}</div>
 </div></body></html>"""

runner/harness/deps.py

@@ -28,7 +28,7 @@ from __future__ import annotations
 import contextlib
 import json
 import os
-from typing import Iterable
+from collections.abc import Iterable
 
 from . import lifecycle, naming
 
@@ -36,9 +36,7 @@ from . import lifecycle, naming
 def declared_deps(recipe: str) -> list[str]:
     """Read `DEPS` from `tests/<recipe>/recipe_meta.py` — a list of recipe names this recipe needs
     deployed alongside it. Returns [] if none."""
-    path = os.path.join(
-        os.path.dirname(__file__), "..", "..", "tests", recipe, "recipe_meta.py"
-    )
+    path = os.path.join(os.path.dirname(__file__), "..", "..", "tests", recipe, "recipe_meta.py")
     if not os.path.exists(path):
         return []
     ns: dict = {}

runner/harness/generic.py

@@ -222,7 +222,11 @@ def assert_restore_healthy(domain: str, meta: dict) -> None:
 
 
 def perform_upgrade(
-    domain: str, recipe: str, head_ref: str | None, deploy_timeout: int = 900, meta: dict | None = None
+    domain: str,
+    recipe: str,
+    head_ref: str | None,
+    deploy_timeout: int = 900,
+    meta: dict | None = None,
 ) -> dict[str, str | None]:
     """Perform the UPGRADE op once, in place, to the PR-HEAD code under test (HC1): re-checkout the
     PR head (the prev-tag base deploy reset the recipe working tree), then `abra app deploy --chaos`
@@ -267,7 +271,9 @@ def perform_upgrade(
         deploy_timeout=int(meta.get("DEPLOY_TIMEOUT", deploy_timeout)),
         http_timeout=int(meta.get("HTTP_TIMEOUT", 300)),
     )
-    lifecycle.wait_ready_probes(meta, domain, timeout=int(meta.get("DEPLOY_TIMEOUT", deploy_timeout)))
+    lifecycle.wait_ready_probes(
+        meta, domain, timeout=int(meta.get("DEPLOY_TIMEOUT", deploy_timeout))
+    )
     after = lifecycle.deployed_identity(domain)
     # Evidence (HC1): the chaos-version label = the deployed recipe commit; it should match the
     # PR-head we checked out — proving the upgrade deployed the code under test, not a published tag.

runner/harness/http.py

@@ -73,7 +73,7 @@ def http_post(
     `data` is JSON-encoded if content_type='application/json',
     form-encoded if 'application/x-www-form-urlencoded' (the OIDC token endpoint form),
     or sent raw bytes if data is already bytes."""
-    if isinstance(data, (bytes, bytearray)):
+    if isinstance(data, bytes | bytearray):
         body: bytes | None = bytes(data)
     elif content_type == "application/json" and data is not None:
         body = json.dumps(data).encode()
@@ -107,7 +107,7 @@ def http_request(
 ) -> tuple[int, object | None]:
     """Arbitrary-method HTTP (PUT/DELETE/PATCH) for parity tests that mutate. Same shape as
     http_post (returns (status, json_or_None))."""
-    if isinstance(data, (bytes, bytearray)):
+    if isinstance(data, bytes | bytearray):
         body: bytes | None = bytes(data)
     elif content_type == "application/json" and data is not None:
         body = json.dumps(data).encode()
@@ -142,7 +142,7 @@ def post_with_headers(
     """Like http_post but ALSO returns the response headers as a dict — for APIs that hand back an
     auth token in a response header rather than the body (e.g. mattermost login → `Token` header).
     Returns (status, parsed_json_or_None, response_headers). status=0 + {} on transport failure."""
-    if isinstance(data, (bytes, bytearray)):
+    if isinstance(data, bytes | bytearray):
         body: bytes | None = bytes(data)
     elif content_type == "application/json" and data is not None:
         body = json.dumps(data).encode()
@@ -252,13 +252,16 @@ def retry_http_post(
 ) -> tuple[int, object | None]:
     """POST with retry until expect_fn(status, json) is truthy. Defaults to any 2xx."""
     if expect_fn is None:
+
         def expect_fn(s, _j):  # noqa: ARG001
             return 200 <= s < 300
 
     result: list[tuple[int, object | None]] = [(0, None)]
 
     def _check():
-        s, j = http_post(url, data=data, headers=headers, content_type=content_type, timeout=timeout)
+        s, j = http_post(
+            url, data=data, headers=headers, content_type=content_type, timeout=timeout
+        )
         result[0] = (s, j)
         return expect_fn(s, j)
 

runner/harness/level.py

@@ -5,37 +5,39 @@ YunoHost semantics: **a gap caps the level** — you only earn level L if every
 PASS. The first rung that is not a clean PASS (a real FAIL *or* genuinely N/A for this recipe) stops
 the climb; `cap_reason` records why. This is deliberately conservative: presentation must NEVER make
 a run look greener than its tests (plan §6 cardinal guardrail), so an N/A rung caps just like a fail
-(the L5 example in §4.1 — "recipes with no integration surface cap at L4 by definition" — is exactly
-this: N/A caps, with a recorded reason so the level is *fair*, not inflated).
+— with a recorded reason so the level is *fair*, not inflated.
 
-The ladder (§4.1):
+The ladder is the FOUR essential rungs every recipe is held to:
   L0 — install failed / app never became healthy.
   L1 — Installs: deploys + passes health/readiness.
   L2 — Upgrades: previous published version → PR version, stays healthy, data intact.
   L3 — Backup/restore: seeded data survives backup → wipe → restore.
   L4 — Functional: recipe-specific functional tests pass.
-  L5 — Integration: SSO/OIDC + cross-app integration tests pass.
-  L6 — Recipe-local: the recipe repo's own tests/ (D4) pass and are merged.
+
+Integration (SSO/OIDC + cross-app) and recipe-local (the recipe repo's own tests/) are **OPTIONAL**
+capabilities — they are NOT part of the level ladder and never cap it. They still run when present
+(and SSO is still enforced for the run VERDICT via the deps/SSO checks in run_recipe_ci.py), but a
+recipe without an SSO surface or without repo-local tests is simply not penalised on the level.
 
 This module is PURE (no I/O) so it is cheaply unit-testable and the Adversary can re-run the unit
 test cold (`cc-ci-run -m pytest tests/unit/test_level.py -q`). The orchestrator
-(`run_recipe_ci.py`) is responsible for translating its raw per-tier results + deps/SSO signals into
-the rung-status dict this function consumes; that mapping is documented in DECISIONS.md (Phase 3).
+(`run_recipe_ci.py`) is responsible for translating its raw per-tier results into the rung-status
+dict this function consumes; that mapping is documented in DECISIONS.md (Phase 3).
 
 Rung status vocabulary (each rung ∈ these three):
   "pass" — the rung was exercised and passed.
   "fail" — the rung was exercised and failed.
   "na"   — the rung does not apply to this recipe (e.g. only one published version → no upgrade;
-           not backup-capable; no SSO/integration surface; no recipe-local tests). N/A is NOT a
-           failure, but it DOES cap the climb (with a distinct cap_reason) so the level never
-           overstates what was actually verified.
+           not backup-capable). N/A is NOT a failure, but it DOES cap the climb (with a distinct
+           cap_reason) so the level never overstates what was actually verified.
 """
 
 from __future__ import annotations
 
 # The climbable rungs in ascending order. install (L1) is the foundation; L0 means install itself
-# did not pass. Each later rung requires every earlier rung to be a clean PASS.
-RUNGS = ("install", "upgrade", "backup_restore", "functional", "integration", "recipe_local")
+# did not pass. Each later rung requires every earlier rung to be a clean PASS. These four are the
+# ESSENTIAL rungs — integration/recipe-local are optional and deliberately NOT in this tuple.
+RUNGS = ("install", "upgrade", "backup_restore", "functional")
 
 # Human-readable label per rung level, for cap_reason + the summary card.
 RUNG_LABEL = {
@@ -43,22 +45,20 @@ RUNG_LABEL = {
     2: "upgrade (prev published → PR)",
     3: "backup/restore (data integrity)",
     4: "functional (recipe-specific tests)",
-    5: "integration (SSO/OIDC + cross-app)",
-    6: "recipe-local (recipe repo tests/)",
 }
 
 VALID = {"pass", "fail", "na"}
 
 
 def compute_level(rungs: dict[str, str]) -> tuple[int, str]:
-    """Map a rung-status dict → (level 0..6, cap_reason).
+    """Map a rung-status dict → (level 0..4, cap_reason).
 
     `rungs` must contain a status in {"pass","fail","na"} for every name in RUNGS. The level is the
     highest L such that rungs[1..L] are all "pass"; the first non-"pass" rung caps the climb. L0 is
     returned when the install rung itself is not "pass" (install failed / never healthy).
 
     cap_reason explains where the climb stopped:
-      - "" (empty) when the recipe earned the top rung (L6, full clean climb).
+      - "" (empty) when the recipe earned the top rung (L4, full clean climb).
       - "L<k> <label> FAILED" when a rung was exercised and failed.
       - "L<k> <label> N/A" when a rung does not apply to this recipe.
     Returns the reason for the FIRST rung that stopped the climb (the binding constraint).

runner/harness/lifecycle.py

@@ -8,6 +8,7 @@ from __future__ import annotations
 
 import contextlib
 import datetime
+import fcntl
 import json
 import os
 import re
@@ -29,6 +30,73 @@ class TeardownError(RuntimeError):
     pass
 
 
+# --- Concurrent-run safety (capacity=2) -------------------------------------------------------
+# Two cooperating mechanisms, both process-lifetime-scoped so SIGKILL can't leak a stale lock:
+#  1. Per-recipe flock: ~/.abra/recipes/<recipe> is ONE shared working tree that fetch_recipe
+#     rm-rf's/reclones and the upgrade tier git-checkouts mid-run. Concurrent runs of the SAME
+#     recipe would corrupt each other's deploy tree (observed: immich builds 229/230 deployed a
+#     tree missing its config), so they serialise on an exclusive flock; different recipes run in
+#     parallel. The kernel drops a flock when the holder dies, however it dies.
+#  2. Active-run registry: each run registers its app domain + pid before creating the app, so the
+#     janitor can tell a live concurrent run from a crashed run's orphan (see janitor()).
+RECIPE_LOCK_DIR = "/run/lock"
+ACTIVE_RUN_DIR = "/run/cc-ci-active"
+
+
+def acquire_recipe_lock(recipe: str):
+    """Take the per-recipe exclusive lock; blocks (with a log line) if another run of the same
+    recipe is in flight. Returns the open lock file — the CALLER must keep a reference for the
+    whole run; the lock is released only when the process exits and the fd closes."""
+    path = os.path.join(RECIPE_LOCK_DIR, f"cc-ci-recipe-{recipe}.lock")
+    f = open(path, "w")  # noqa: SIM115 — deliberately held for the lifetime of the run
+    try:
+        fcntl.flock(f, fcntl.LOCK_EX | fcntl.LOCK_NB)
+    except BlockingIOError:
+        print(
+            f"== recipe lock: another {recipe} run is in flight — waiting for {path} "
+            "(shared ~/.abra/recipes checkout) ==",
+            flush=True,
+        )
+        fcntl.flock(f, fcntl.LOCK_EX)
+        print(f"== recipe lock: acquired {path} ==", flush=True)
+    return f
+
+
+def _registry_path(domain: str) -> str:
+    return os.path.join(ACTIVE_RUN_DIR, domain)
+
+
+def register_run_app(domain: str) -> None:
+    """Record this process as the live owner of a run app (called BEFORE the app is created, so a
+    concurrent run's janitor can never observe the app without its registration)."""
+    with contextlib.suppress(OSError):
+        os.makedirs(ACTIVE_RUN_DIR, exist_ok=True)
+        with open(_registry_path(domain), "w") as f:
+            f.write(str(os.getpid()))
+
+
+def unregister_run_app(domain: str) -> None:
+    with contextlib.suppress(OSError):
+        os.remove(_registry_path(domain))
+
+
+def _run_owner_state(domain: str) -> str:
+    """'alive' if the registered owner is a live run_recipe_ci process, 'dead' if registered but
+    gone (definite orphan), 'unknown' if never registered (pre-registry code or post-reboot)."""
+    try:
+        with open(_registry_path(domain)) as f:
+            pid = int(f.read().strip())
+    except (OSError, ValueError):
+        return "unknown"
+    try:
+        with open(f"/proc/{pid}/cmdline", "rb") as f:
+            cmdline = f.read().decode(errors="replace").replace("\0", " ")
+    except OSError:
+        return "dead"
+    # Guard against pid reuse: the owner must still look like a harness run.
+    return "alive" if "run_recipe_ci" in cmdline else "dead"
+
+
 def _docker_names(kind: str, stack: str) -> list[str]:
     """docker <kind> ls names filtered to a stack (kind: service|volume|secret)."""
     proc = subprocess.run(
@@ -161,7 +229,8 @@ def prepull_images(recipe: str, domain: str) -> None:
     # --env-file supplies $VERSION-style interpolation so pinned tags resolve correctly.
     cf = subprocess.run(
         ["bash", "-c", f'set -a; . "{env_path}"; printf "%s" "${{COMPOSE_FILE:-compose.yml}}"'],
-        capture_output=True, text=True,
+        capture_output=True,
+        text=True,
     ).stdout.strip()
     files = [f for f in cf.split(":") if f] or ["compose.yml"]
     args = ["docker", "compose", "--env-file", env_path]
@@ -209,6 +278,9 @@ def deploy_app(
     past the 900s default. abra's INTERNAL TIMEOUT (recipe's TIMEOUT env, default 300s) is set via
     EXTRA_ENV; this is the Python subprocess wrapper's timeout so abra doesn't get SIGKILLed mid-deploy."""
     _record_deploy()
+    # Register BEFORE the app exists: a concurrent run's janitor must never see this app without
+    # its live-owner registration (it would reap an in-flight deploy).
+    register_run_app(domain)
     abra.app_config_remove(domain)  # clear any stale .env from a prior crashed run
     abra.app_new(recipe, domain, version=version, secrets=secrets)
     # A pinned version must actually deploy that version: check the recipe out to the tag so the
@@ -268,18 +340,22 @@ def _stack_name(domain: str) -> str:
 
 
 def services_converged(domain: str) -> bool:
-    """True when every service in the stack reports replicas N/N (N>0)."""
+    """True when every service in the stack reports replicas N/N (N>0) AND no service is
+    mid-rolling-update (swarm UpdateStatus settled)."""
     stack = _stack_name(domain)
     proc = subprocess.run(
-        ["docker", "stack", "services", stack, "--format", "{{.Replicas}}"],
+        ["docker", "stack", "services", stack, "--format", "{{.Name}} {{.Replicas}}"],
         capture_output=True,
         text=True,
     )
     rows = [r for r in proc.stdout.split("\n") if r.strip()]
     if not rows:
         return False
+    names = []
     for r in rows:
-        cur, _, want = r.partition("/")
+        name, _, replicas = r.partition(" ")
+        names.append(name)
+        cur, _, want = replicas.partition("/")
         # A service at its DESIRED replica count is converged — including a `replicas: 0`
         # on-demand one-shot (e.g. lasuite-drive's `minio-createbuckets`, which is scaled up
         # manually only when buckets need (re)creating), which reports "0/0". The earlier
@@ -288,6 +364,28 @@ def services_converged(domain: str) -> bool:
         # still spinning up shows e.g. "0/1" (cur != want) and is correctly not-yet-converged.
         if not want or cur != want:
             return False
+    # N/N alone is NOT convergence during a stop-first rolling update: a chaos redeploy that changes
+    # a non-app service image (e.g. immich's db pin) registers the update immediately, but swarm may
+    # not have cycled that service's task yet — the OLD task still shows 1/1, then dies seconds later
+    # (immich CI 238: backupbot exec'd the db pre-hook into the just-killed container → 409). Require
+    # every service's UpdateStatus to be settled too, so the wait spans the whole rolling update.
+    proc = subprocess.run(
+        [
+            "docker",
+            "service",
+            "inspect",
+            *names,
+            "--format",
+            "{{if .UpdateStatus}}{{.UpdateStatus.State}}{{end}}",
+        ],
+        capture_output=True,
+        text=True,
+    )
+    if proc.returncode != 0:
+        return False  # a service vanished mid-check — not settled
+    for state in proc.stdout.split("\n"):
+        if state.strip() not in ("", "completed", "rollback_completed"):
+            return False
     return True
 
 
@@ -415,7 +513,9 @@ def recipe_checkout_ref(recipe: str, ref: str) -> None:
     abra.recipe_checkout(recipe, ref)
 
 
-def chaos_redeploy(domain: str, deploy_timeout: int = 900, no_converge_checks: bool = False) -> None:
+def chaos_redeploy(
+    domain: str, deploy_timeout: int = 900, no_converge_checks: bool = False
+) -> None:
     """In-place `abra app deploy --chaos`: redeploy the running app at the CURRENT recipe checkout
     (HC1: the PR-head code under test). This is the upgrade op, not a fresh install — it does NOT go
     through deploy_app, so the deploy-count guard (DG4.1) is not incremented.
@@ -498,6 +598,16 @@ def wait_ready_probes(meta: dict, domain: str, timeout: int = 600) -> None:
 
 def backup_app(domain: str) -> str:
     """Create a backup; return the abra/restic output (carries the produced snapshot_id)."""
+    # Never back up a stack that is still converging/rolling-updating: backupbot resolves each
+    # service's hook container ONCE up front, so a task that cycles between that lookup and the
+    # pre-hook exec crashes the whole backup with a 409 (immich CI 238). Bounded wait — on timeout
+    # we still attempt the backup and let the tier's assertion deliver the verdict.
+    deadline = time.time() + 300
+    while time.time() < deadline and not services_converged(domain):
+        print(
+            f"  backup: {domain} stack not settled yet — waiting before backup create", flush=True
+        )
+        time.sleep(5)
     return abra.backup_create(domain)
 
 
@@ -603,13 +713,19 @@ def teardown_app(domain: str, verify: bool = True) -> None:
         residual = _residual(domain)
         if any(residual.values()):
             raise TeardownError(f"teardown left residual for {domain}: {residual}")
+    # The app is gone — drop its active-run registration (janitor() also clears it when reaping).
+    unregister_run_app(domain)
 
 
 def janitor(max_age_seconds: int | None = None) -> None:
-    """Reap orphaned run apps from crashed/rebooted runs. Matches the real naming scheme and only
-    reaps apps older than max_age_seconds (so concurrent in-flight runs are never killed). Reaps via
-    docker primitives so it works even when the .env is gone (A2/A3). Default 2h, env-overridable
-    via CCCI_JANITOR_MAX_AGE (e.g. 0 to reap all matching orphans immediately)."""
+    """Reap orphaned run apps from crashed/rebooted runs. Matches the real naming scheme. Safe under
+    CONCURRENT runs (capacity=2): every harness run registers its app in the active-run registry
+    (register_run_app), so the janitor distinguishes the three cases instead of using age alone:
+      - registered + owner run_recipe_ci process ALIVE  -> in-flight concurrent run: never reap
+      - registered + owner DEAD (crashed/SIGKILLed run) -> definite orphan: reap immediately
+      - no registry entry (pre-registry code, reboot)   -> fall back to the age threshold
+    Reaps via docker primitives so it works even when the .env is gone (A2/A3). Age fallback default
+    2h, env-overridable via CCCI_JANITOR_MAX_AGE."""
     import os
 
     if max_age_seconds is None:
@@ -627,9 +743,18 @@ def janitor(max_age_seconds: int | None = None) -> None:
             seen.add(f"{m.group(1)}.ci.commoninternet.net")
 
     for name in seen:
-        stack = _stack_name(name)
-        age = _stack_age_seconds(stack)
-        if age is not None and age < max_age_seconds:
-            continue  # likely a concurrent in-flight run; leave it
+        owner = _run_owner_state(name)
+        if owner == "alive":
+            print(f"  janitor: {name} is a live concurrent run — leaving it", flush=True)
+            continue
+        if owner == "unknown":
+            # No registry entry (manual run on pre-registry code, or post-reboot): only the age
+            # threshold protects it, as before.
+            stack = _stack_name(name)
+            age = _stack_age_seconds(stack)
+            if age is not None and age < max_age_seconds:
+                continue  # young and of unknown provenance; leave it
+        # owner == "dead" (a crashed/killed run's definite orphan) or old enough -> reap
         with contextlib.suppress(Exception):
             teardown_app(name, verify=False)
+        unregister_run_app(name)

runner/harness/results.py

@@ -2,7 +2,14 @@
 
 Turns a run's per-tier pytest outcomes into a single `results.json` artifact carrying, per the plan:
   { recipe, version, pr, ref, run_id, finished, stages:[{name,status,tests:[{name,status,ms}]}],
-    level, level_cap_reason, rungs, flags:{clean_teardown,no_secret_leak}, screenshot, summary_card }
+    level, level_cap_reason, level_cap_rung, rungs,
+    skips:{intentional:{rung:reason}, unintentional:[rung]},
+    flags:{clean_teardown,no_secret_leak}, screenshot, summary_card }
+
+`skips` splits the N/A (skipped) rungs by a simple rule: a skip is INTENTIONAL iff the recipe lists
+it (with a reason) in `recipe_meta.EXPECTED_NA = {rung: reason}`; any rung skipped but not listed is
+UNINTENTIONAL (a coverage gap to fill or declare). Skips still cap the level either way — the harness
+never claims a rung it did not verify; this only labels *why* a skip happened.
 
 The per-test breakdown comes from JUnit XML emitted by each tier's pytest invocation (`--junitxml`),
 parsed here with the stdlib (no new dep). The integer **level** is computed by harness.level from a
@@ -127,41 +134,24 @@ def collect_stages(records: list[dict]) -> list[dict]:
     return stages
 
 
-def _has_repo_local(records: list[dict]) -> bool:
-    return any(r.get("source") == "repo-local" for r in records)
-
-
-def _repo_local_passed(records: list[dict]) -> bool:
-    repo = [r for r in records if r.get("source") == "repo-local"]
-    return bool(repo) and all(r.get("rc", 1) == 0 for r in repo)
-
-
 def derive_rungs(
     results: dict[str, str],
     *,
     backup_capable: bool,
-    declared: list[str] | None,
-    deps_ready: bool,
-    sso_unverified: bool,
     has_custom: bool,
-    has_repo_local: bool,
-    repo_local_passed: bool,
 ) -> dict[str, str]:
-    """Translate the orchestrator's tier results + deps/SSO signals into the rung-status dict
-    harness.level consumes. Documented in DECISIONS.md (Phase 3). Conservative by design — never
-    reports a rung 'pass' it can't substantiate (cardinal guardrail: presentation never inflates).
+    """Translate the orchestrator's tier results into the rung-status dict harness.level consumes —
+    the FOUR essential rungs only. Conservative by design — never reports a rung 'pass' it can't
+    substantiate (cardinal guardrail: presentation never inflates).
 
       L1 install    : install tier pass.
       L2 upgrade    : upgrade tier (skip → N/A: only one published version).
       L3 backup/res : backup AND restore tiers pass (N/A if not backup-capable).
-      L4 functional : the recipe-specific functional (non-deps) tests pass — the custom tier, minus
-                      its SSO/integration tests. N/A if the recipe has no custom tests at all.
-      L5 integration: SSO/OIDC + cross-app. Applies ONLY if the recipe declares deps (else N/A — the
-                      "no integration surface caps at L4" rule, §4.1). pass iff deps wired
-                      (deps_ready) and not sso_unverified and the custom tier didn't fail.
-      L6 recipe-loc : the recipe repo's own tests/ (repo-local source) ran and passed (N/A if none).
+      L4 functional : recipe-specific functional tests pass — the custom tier. N/A if none ran.
+
+    Integration (SSO/OIDC) and recipe-local are OPTIONAL and intentionally NOT rungs here — they
+    never cap the level (SSO is still enforced for the run VERDICT in run_recipe_ci.py).
     """
-    declared = declared or []
     rungs: dict[str, str] = {}
     rungs["install"] = level_mod.tier_to_rung(results.get("install"))
     rungs["upgrade"] = level_mod.tier_to_rung(results.get("upgrade"))
@@ -170,36 +160,34 @@ def derive_rungs(
     )
 
     custom = results.get("custom")
-    # Functional rung (L4): the non-deps custom tests.
     if not has_custom or custom == "skip" or custom is None:
         rungs["functional"] = "na"
     elif custom == "fail":
-        # A custom test failed. With declared deps we cannot cheaply tell functional-vs-SSO apart, so
-        # conservatively fail the functional rung (caps at L3) — never inflate.
         rungs["functional"] = "fail"
     else:  # custom == "pass"
         rungs["functional"] = "pass"
-
-    # Integration rung (L5): only recipes with an SSO/integration surface (declared deps) can climb.
-    if not declared:
-        rungs["integration"] = "na"
-    elif sso_unverified or not deps_ready or custom == "fail":
-        # SSO not wired/verified, or a custom test failed → integration not verified.
-        rungs["integration"] = "fail"
-    elif custom == "pass":
-        rungs["integration"] = "pass"
-    else:
-        # declared deps but no custom tests ran — can't claim integration verified
-        rungs["integration"] = "na"
-
-    # Recipe-local rung (L6).
-    if not has_repo_local:
-        rungs["recipe_local"] = "na"
-    else:
-        rungs["recipe_local"] = "pass" if repo_local_passed else "fail"
     return rungs
 
 
+def skips(rungs: dict[str, str], expected_na: dict | None) -> dict:
+    """Split the SKIPPED (N/A) rungs into intentional vs unintentional (operator model).
+
+    A recipe lists the rungs it intentionally skips, each with a reason, in
+    `recipe_meta.EXPECTED_NA = {rung: reason}`. The rule is dead simple: a skipped rung is
+    **intentional** iff it is in that list; any rung that is skipped and NOT in the list is
+    **unintentional** (a coverage gap someone should either fill or declare). N/A still caps the
+    level either way — the harness never claims a rung it did not verify — this only labels *why* a
+    skip happened. Returns:
+      { "intentional": {rung: reason, ...},   # skipped AND declared in EXPECTED_NA
+        "unintentional": [rung, ...] }         # skipped but NOT declared
+    """
+    expected = {str(k): str(v) for k, v in (expected_na or {}).items()}
+    na = [r for r, st in rungs.items() if st == "na"]
+    intentional = {r: expected[r] for r in na if r in expected}
+    unintentional = sorted(r for r in na if r not in expected)
+    return {"intentional": intentional, "unintentional": unintentional}
+
+
 def build_results(
     *,
     recipe: str,
@@ -209,30 +197,24 @@ def build_results(
     records: list[dict],
     results: dict[str, str],
     backup_capable: bool,
-    declared: list[str] | None,
-    deps_ready: bool,
-    sso_unverified: bool,
     clean_teardown: bool,
     no_secret_leak: bool,
     finished_ts: float | None,
     screenshot: str | None = None,
     summary_card: str | None = None,
+    expected_na: dict | None = None,
 ) -> dict:
     """Assemble the full results.json dict (no I/O). `finished_ts` is passed in (the orchestrator
-    stamps it) so this stays pure and deterministic for unit tests."""
+    stamps it) so this stays pure and deterministic for unit tests. `expected_na` is the recipe's
+    declared intentional-skip map (recipe_meta.EXPECTED_NA) used to distinguish a deliberate skip from
+    accidentally-missing coverage."""
     stages = collect_stages(records)
     has_custom = any(r["tier"] == "custom" for r in records)
-    rungs = derive_rungs(
-        results,
-        backup_capable=backup_capable,
-        declared=declared,
-        deps_ready=deps_ready,
-        sso_unverified=sso_unverified,
-        has_custom=has_custom,
-        has_repo_local=_has_repo_local(records),
-        repo_local_passed=_repo_local_passed(records),
-    )
+    rungs = derive_rungs(results, backup_capable=backup_capable, has_custom=has_custom)
     lvl, cap_reason = level_mod.compute_level(rungs)
+    # The rung that capped the climb (lowest non-pass), or None on a full climb — lets a consumer
+    # (card/badge) tell whether the cap was an intentional skip, an unintentional one, or a failure.
+    capped = level_mod.RUNGS[lvl] if cap_reason else None
     return {
         "schema": 1,
         "run_id": run_id(),
@@ -243,7 +225,9 @@ def build_results(
         "finished": finished_ts,
         "level": lvl,
         "level_cap_reason": cap_reason,
+        "level_cap_rung": capped,
         "rungs": rungs,
+        "skips": skips(rungs, expected_na),
         "stages": stages,
         "results": results,
         "flags": {

runner/harness/warmsnap.py

@@ -113,7 +113,9 @@ def _assert_undeployed(domain: str) -> None:
         )
 
 
-def snapshot(recipe: str, domain: str, commit: str | None = None, version: str | None = None) -> dict:
+def snapshot(
+    recipe: str, domain: str, commit: str | None = None, version: str | None = None
+) -> dict:
     """Take a last-known-good snapshot of every data volume of <domain>'s stack. The app MUST be
     undeployed. Atomically replaces the prior last-good. Returns the written meta dict."""
     _assert_undeployed(domain)
@@ -169,7 +171,9 @@ def restore(recipe: str, domain: str) -> dict:
     for vol in meta.get("volumes", []):
         tar_path = os.path.join(volumes_dir(recipe), f"{vol}.tar")
         if vol not in current:
-            raise SnapshotError(f"snapshot volume {vol} absent from current stack {sorted(current)}")
+            raise SnapshotError(
+                f"snapshot volume {vol} absent from current stack {sorted(current)}"
+            )
         mp = _volume_mountpoint(vol)
         # Clear the volume contents (incl. dotfiles) without removing the mountpoint itself.
         r = _run(["sh", "-c", f'rm -rf -- "{mp}"/* "{mp}"/.[!.]* "{mp}"/..?* 2>/dev/null; true'])

runner/nightly_sweep.py

@@ -60,14 +60,17 @@ def sweep() -> int:
     for r in recipes:
         print(f"\n===== nightly: full-cold {r} (latest) =====", flush=True)
         env = dict(os.environ, RECIPE=r)
-        env.pop("REF", None)      # latest, not a PR head
+        env.pop("REF", None)  # latest, not a PR head
         env.pop("CCCI_QUICK", None)
         env.pop("MODE", None)
         rc = subprocess.run(
             [sys.executable, os.path.join(_here(), "run_recipe_ci.py")], env=env
         ).returncode
         results[r] = rc
-        print(f"nightly: {r} rc={rc} ({'green→canonical refreshed' if rc == 0 else 'red'})", flush=True)
+        print(
+            f"nightly: {r} rc={rc} ({'green→canonical refreshed' if rc == 0 else 'red'})",
+            flush=True,
+        )
     # WC8 disk hygiene: drop warm data for de-enrolled canonicals; log the disk budget.
     pruned = canonical.prune_stale()
     if pruned:

runner/run_recipe_ci.py

@@ -44,17 +44,25 @@ sys.path.insert(0, os.path.join(ROOT, "runner"))
 from harness import (  # noqa: E402
     abra,
     canonical,
-    card as card_mod,
-    deps as deps_mod,
     discovery,
     generic,
     lifecycle,
     naming,
-    results as results_mod,
-    screenshot as screenshot_mod,
     warm,
     warmsnap,
 )
+from harness import (  # noqa: E402
+    card as card_mod,
+)
+from harness import (  # noqa: E402
+    deps as deps_mod,
+)
+from harness import (  # noqa: E402
+    results as results_mod,
+)
+from harness import (  # noqa: E402
+    screenshot as screenshot_mod,
+)
 
 ALL_STAGES = ("install", "upgrade", "backup", "restore", "custom")
 
@@ -200,6 +208,7 @@ def _load_meta(recipe: str) -> dict:
         for k in list(meta) + [
             "BACKUP_CAPABLE",
             "SKIP_GENERIC",
+            "EXPECTED_NA",
             "OIDC_AT_INSTALL",
             "READY_PROBE",
             "UPGRADE_BASE_VERSION",
@@ -826,6 +835,12 @@ def main() -> int:
     print(
         f"== cc-ci run: recipe={recipe} ref={ref} pr={os.environ.get('PR', '0')} stages={sorted(stages)}"
     )
+    # Concurrent-run safety: runs of the SAME recipe serialise on a per-recipe flock — they share
+    # ONE ~/.abra/recipes/<recipe> working tree which fetch_recipe (below) rm-rf's/reclones and the
+    # upgrade tier git-checkouts mid-run. Must be taken BEFORE fetch_recipe. Different recipes run
+    # in parallel (capacity=2). The reference must stay alive for the whole run: the kernel drops
+    # the flock when the fd closes (including on any crash/SIGKILL — no stale-lock failure mode).
+    _recipe_lock = lifecycle.acquire_recipe_lock(recipe)  # noqa: F841
     fetch_recipe(recipe, ref, src)
     # The PR-head commit the upgrade tier re-checks out for the chaos redeploy to the code under test
     # (HC1). Prefer the explicit PR head sha ($REF) — robust + exact; fall back to the recipe checkout
@@ -1224,7 +1239,6 @@ def main() -> int:
     # a failure here NEVER changes `overall` (R7 — cosmetics never block the pipeline). ----
     data: dict | None = None
     try:
-        sso_unverified = sso_dep_unverified(declared, deps_ready, requires_deps_skipped)
         clean_teardown = (deploy_count == expected_deploy_count) and not dep_teardown_error
         data = results_mod.build_results(
             recipe=recipe,
@@ -1234,13 +1248,11 @@ def main() -> int:
             records=records,
             results=results,
             backup_capable=backup_cap,
-            declared=declared,
-            deps_ready=deps_ready,
-            sso_unverified=sso_unverified,
             clean_teardown=clean_teardown,
             no_secret_leak=True,  # narrowed below by an actual scan of the serialised artifact
             screenshot=screenshot_rel,  # Phase 3 U1 (R4): relative PNG name iff capture succeeded
             finished_ts=time.time(),
+            expected_na=meta.get("EXPECTED_NA"),  # declared intentional-skip map (recipe_meta)
         )
         # Real (if narrow) leak check: no known infra-secret value may appear in the artifact (R7).
         blob = json.dumps(data)
@@ -1257,6 +1269,15 @@ def main() -> int:
             f"{' — ' + data['level_cap_reason'] if data['level_cap_reason'] else ''})",
             flush=True,
         )
+        # Surface UNINTENTIONAL skips in the CI log (non-blocking, R7): a rung that was skipped (N/A)
+        # but is not in the recipe's intentional list — either add the missing coverage or declare it.
+        for rung in data.get("skips", {}).get("unintentional", []):
+            print(
+                f"⚠ coverage: rung '{rung}' was skipped (N/A) but is not declared intentional — add "
+                f"the missing test/label, or list it in tests/{recipe}/recipe_meta.py "
+                f"EXPECTED_NA = {{'{rung}': '<why>'}}.",
+                flush=True,
+            )
     except Exception as e:  # noqa: BLE001 — results assembly is cosmetic; never fail a run on it (R7)
         print(
             f"!! results.json assembly failed (non-fatal, verdict unaffected): {_scrub(str(e))}",
@@ -1275,8 +1296,21 @@ def main() -> int:
             with open(html_path, "w", encoding="utf-8") as f:
                 f.write(card_mod.render_card_html(data, screenshot_rel=data.get("screenshot")))
             png = card_mod.render_card_png(html_path, os.path.join(run_artifact_dir, "summary.png"))
+            capped = data.get("level_cap_rung")
+            sk = data.get("skips", {})
+            cap_skip = (
+                "intentional"
+                if capped in (sk.get("intentional") or {})
+                else "unintentional"
+                if capped in (sk.get("unintentional") or [])
+                else ""
+            )
             with open(os.path.join(run_artifact_dir, "badge.svg"), "w", encoding="utf-8") as f:
-                f.write(card_mod.level_badge_svg(data["level"], data.get("level_cap_reason", "")))
+                f.write(
+                    card_mod.level_badge_svg(
+                        data["level"], data.get("level_cap_reason", ""), cap_skip
+                    )
+                )
             print(
                 f"summary card {'rendered ' + png if png else '(PNG render unavailable)'} + "
                 f"badge.svg written into {run_artifact_dir}",

runner/warm_reconcile.py

@@ -43,11 +43,16 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
     ssl_cert/ssl_key swarm secrets; NO ACME). Uses the proven abra.env_set (newline-safe, unlike the
     bash set_env that bit keycloak)."""
     cert_dir = "/var/lib/ci-certs/live"
-    if not (os.path.isfile(f"{cert_dir}/fullchain.pem") and os.path.isfile(f"{cert_dir}/privkey.pem")):
+    if not (
+        os.path.isfile(f"{cert_dir}/fullchain.pem") and os.path.isfile(f"{cert_dir}/privkey.pem")
+    ):
         raise RuntimeError(f"FATAL: wildcard cert missing at {cert_dir} (sops decrypt broken?)")
     if not os.path.isfile(env_file(domain)):
-        _run(["abra", "app", "new", recipe, "-s", "default", "-D", domain, version, "-o", "-n"],
-             timeout=120, check=True)
+        _run(
+            ["abra", "app", "new", recipe, "-s", "default", "-D", domain, version, "-o", "-n"],
+            timeout=120,
+            check=True,
+        )
     abra.env_set(domain, "DOMAIN", domain)
     abra.env_set(domain, "LETS_ENCRYPT_ENV", "")
     abra.env_set(domain, "WILDCARDS_ENABLED", "1")
@@ -61,11 +66,39 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
         return any(s.endswith(f"_{name}_v1") for s in have)
 
     if not _has("ssl_cert"):
-        _run(["abra", "app", "secret", "insert", domain, "ssl_cert", "v1",
-              f"{cert_dir}/fullchain.pem", "-f", "-n"], timeout=120, check=True)
+        _run(
+            [
+                "abra",
+                "app",
+                "secret",
+                "insert",
+                domain,
+                "ssl_cert",
+                "v1",
+                f"{cert_dir}/fullchain.pem",
+                "-f",
+                "-n",
+            ],
+            timeout=120,
+            check=True,
+        )
     if not _has("ssl_key"):
-        _run(["abra", "app", "secret", "insert", domain, "ssl_key", "v1",
-              f"{cert_dir}/privkey.pem", "-f", "-n"], timeout=120, check=True)
+        _run(
+            [
+                "abra",
+                "app",
+                "secret",
+                "insert",
+                domain,
+                "ssl_key",
+                "v1",
+                f"{cert_dir}/privkey.pem",
+                "-f",
+                "-n",
+            ],
+            timeout=120,
+            check=True,
+        )
 
 
 SPECS: dict[str, dict] = {
@@ -218,8 +251,17 @@ def health_code(spec: dict) -> int:
     domain = spec.get("health_domain", spec["domain"])
     r = _run(
         [
-            "curl", "-sk", "-o", "/dev/null", "-w", "%{http_code}", "--max-time", "10",
-            "--resolve", f"{domain}:443:127.0.0.1", f"https://{domain}{spec['health_path']}",
+            "curl",
+            "-sk",
+            "-o",
+            "/dev/null",
+            "-w",
+            "%{http_code}",
+            "--max-time",
+            "10",
+            "--resolve",
+            f"{domain}:443:127.0.0.1",
+            f"https://{domain}{spec['health_path']}",
         ],
         timeout=20,
     )
@@ -230,7 +272,6 @@ def health_code(spec: dict) -> int:
 
 
 def wait_healthy(spec: dict, timeout: int | None = None) -> bool:
-    domain = spec["domain"]
     deadline = time.time() + (timeout or spec["health_timeout"])
     while time.time() < deadline:
         if health_code(spec) in tuple(spec["health_ok"]):
@@ -325,15 +366,18 @@ def ensure_server() -> None:
 
 def ensure_app_config(recipe: str, domain: str, version: str) -> None:
     if not os.path.isfile(env_file(domain)):
-        _run(["abra", "app", "new", recipe, "-s", "default", "-D", domain, version, "-o", "-n"],
-             timeout=120, check=True)
+        _run(
+            ["abra", "app", "new", recipe, "-s", "default", "-D", domain, version, "-o", "-n"],
+            timeout=120,
+            check=True,
+        )
     abra.env_set(domain, "DOMAIN", domain)
     abra.env_set(domain, "LETS_ENCRYPT_ENV", "")
 
 
 def ensure_secrets(domain: str) -> None:
     stack = lifecycle._stack_name(domain)  # noqa: SLF001
-    have = {n for n in lifecycle._docker_names("secret", stack)}  # noqa: SLF001
+    have = set(lifecycle._docker_names("secret", stack))  # noqa: SLF001
     if not any(n.endswith("_admin_password_v1") for n in have):
         abra.secret_generate(domain)
 
@@ -393,8 +437,9 @@ def reconcile(app: str) -> str:
         write_alert(app, "held-major", current=current, latest=latest, release_notes=notes[:4000])
         return f"held-major:{current}->{latest}"
     if notes_flag_manual_migration(notes):
-        write_alert(app, "held-manual-migration", current=current, latest=latest,
-                    release_notes=notes[:4000])
+        write_alert(
+            app, "held-manual-migration", current=current, latest=latest, release_notes=notes[:4000]
+        )
         return f"held-manual-migration:{current}->{latest}"
 
     # WC1.1 health-gated upgrade with rollback.
@@ -428,8 +473,14 @@ def reconcile(app: str) -> str:
         warmsnap.restore(recipe, domain)
     deploy_version(recipe, domain, last_good, dt)
     recovered = wait_healthy(spec)
-    write_alert(app, "rollback", last_good=last_good, attempted=latest, recovered=recovered,
-                release_notes=notes[:2000])
+    write_alert(
+        app,
+        "rollback",
+        last_good=last_good,
+        attempted=latest,
+        recovered=recovered,
+        release_notes=notes[:2000],
+    )
     if not recovered:
         raise RuntimeError(f"{app} rollback to {last_good} did not become healthy")
     return f"rolled-back:{latest}->{last_good}"

tests/bluesky-pds/_p4.py

@@ -15,7 +15,8 @@ import shlex
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
-from harness import http as harness_http, lifecycle  # noqa: E402
+from harness import http as harness_http  # noqa: E402
+from harness import lifecycle
 
 PDS_HOST_LOCAL = "http://localhost:3000"
 _PW = "ccci-P4-marker-pw-2026"

tests/bluesky-pds/functional/test_account_and_post.py

@@ -27,6 +27,7 @@ CRUD). A wedged PDS subsystem fails AT its layer.
 
 from __future__ import annotations
 
+import contextlib
 import os
 import re
 import secrets
@@ -35,7 +36,8 @@ import sys
 import uuid
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "runner"))
-from harness import http as harness_http, lifecycle  # noqa: E402
+from harness import http as harness_http  # noqa: E402
+from harness import lifecycle
 
 PDS_HOST_LOCAL = "http://localhost:3000"
 
@@ -58,14 +60,18 @@ def _goat_admin(domain: str, args: str) -> str:
     return _in_container(domain, cmd)
 
 
-def _xrpc_post(domain: str, nsid: str, data: dict, token: str | None = None) -> tuple[int, dict | None]:
+def _xrpc_post(
+    domain: str, nsid: str, data: dict, token: str | None = None
+) -> tuple[int, dict | None]:
     headers = {}
     if token:
         headers["Authorization"] = f"Bearer {token}"
     return harness_http.http_post(f"https://{domain}/xrpc/{nsid}", data=data, headers=headers)
 
 
-def _xrpc_get(domain: str, nsid: str, query: str, token: str | None = None) -> tuple[int, dict | None]:
+def _xrpc_get(
+    domain: str, nsid: str, query: str, token: str | None = None
+) -> tuple[int, dict | None]:
     headers = {}
     if token:
         headers["Authorization"] = f"Bearer {token}"
@@ -82,9 +88,9 @@ def test_account_lifecycle_and_post_roundtrip(live_app):
 
     # Step 1: PDS describe via goat — recipe self-identifies as did:web:<domain>
     out = _in_container(domain, f"goat pds describe {PDS_HOST_LOCAL} 2>&1")
-    assert f"did:web:{domain}" in out, (
-        f"goat pds describe did not contain expected DID 'did:web:{domain}'. Output:\n{out[:500]!r}"
-    )
+    assert (
+        f"did:web:{domain}" in out
+    ), f"goat pds describe did not contain expected DID 'did:web:{domain}'. Output:\n{out[:500]!r}"
 
     # Step 2: Create account (UUID-suffixed handle = no run-to-run collision)
     out = _goat_admin(
@@ -127,9 +133,9 @@ def test_account_lifecycle_and_post_roundtrip(live_app):
         assert s == 200, f"createRecord HTTP {s}: {body!r}"
         record_uri = (body or {}).get("uri", "")
         # URI format: at://<did>/app.bsky.feed.post/<rkey>
-        assert record_uri.startswith(f"at://{new_did}/app.bsky.feed.post/"), (
-            f"unexpected record uri: {record_uri!r}"
-        )
+        assert record_uri.startswith(
+            f"at://{new_did}/app.bsky.feed.post/"
+        ), f"unexpected record uri: {record_uri!r}"
         rkey = record_uri.rsplit("/", 1)[-1]
         assert rkey, f"no rkey in uri: {record_uri!r}"
 
@@ -142,15 +148,13 @@ def test_account_lifecycle_and_post_roundtrip(live_app):
         )
         assert s == 200, f"getRecord HTTP {s}: {body!r}"
         record_value = (body or {}).get("value", {})
-        assert record_value.get("text") == marker, (
-            f"post text did not round-trip: created={marker!r}, fetched={record_value.get('text')!r}"
-        )
+        assert (
+            record_value.get("text") == marker
+        ), f"post text did not round-trip: created={marker!r}, fetched={record_value.get('text')!r}"
         assert record_value.get("$type") == "app.bsky.feed.post"
     finally:
         # Step 6: Best-effort cleanup. (The per-run domain teardown will discard the volume
         # too, but we exercise the delete-account path because it's part of §4.3.)
         if cleanup_did:
-            try:
+            with contextlib.suppress(Exception):
                 _goat_admin(domain, f"account delete {cleanup_did}")
-            except Exception:  # noqa: BLE001
-                pass

tests/bluesky-pds/functional/test_describe_server.py

@@ -26,6 +26,6 @@ def test_describe_server_returns_atproto_envelope(live_app):
     # At least one of these atproto-spec fields must be present
     expected_any = ("availableUserDomains", "inviteCodeRequired", "links", "did")
     present = [k for k in expected_any if k in body]
-    assert present, (
-        f"describe-server missing all of {expected_any}; got keys: {sorted(body.keys())[:20]}"
-    )
+    assert (
+        present
+    ), f"describe-server missing all of {expected_any}; got keys: {sorted(body.keys())[:20]}"

tests/bluesky-pds/functional/test_health_check.py

@@ -17,6 +17,6 @@ def test_pds_health_returns_version(live_app):
     url = f"https://{live_app}/xrpc/_health"
     status, body = harness_http.retry_http_get(url, expect_status=200, max_wait=60, interval=3)
     assert status == 200, f"GET {url} HTTP {status} (expected 200)"
-    assert isinstance(body, dict) and isinstance(body.get("version"), str) and body["version"], (
-        f"GET {url} response is not the expected health envelope: {body!r}"
-    )
+    assert (
+        isinstance(body, dict) and isinstance(body.get("version"), str) and body["version"]
+    ), f"GET {url} response is not the expected health envelope: {body!r}"

tests/bluesky-pds/functional/test_session_auth.py

@@ -30,6 +30,6 @@ def test_get_session_requires_auth(live_app):
         f"body: {body!r}"
     )
     # The XRPC error envelope is JSON with an `error` field per the atproto spec.
-    assert isinstance(body, dict) and body.get("error"), (
-        f"expected XRPC JSON error envelope; got: {body!r}"
-    )
+    assert isinstance(body, dict) and body.get(
+        "error"
+    ), f"expected XRPC JSON error envelope; got: {body!r}"

tests/bluesky-pds/install_steps.sh

@@ -22,12 +22,12 @@ echo "  bluesky-pds install_steps: generating secp256k1 PLC rotation key..."
 # same shape the PDS expects (32-byte hex). Equivalent for atproto PDS bootstrap.
 KEY_HEX=$(cc-ci-run -c 'import secrets; print(secrets.token_bytes(32).hex())')
 if [ -z "${KEY_HEX}" ] || [ "${#KEY_HEX}" != "64" ]; then
-    echo "  install_steps: failed to generate PLC rotation key (KEY_HEX length=${#KEY_HEX})" >&2
-    exit 1
+  echo "  install_steps: failed to generate PLC rotation key (KEY_HEX length=${#KEY_HEX})" >&2
+  exit 1
 fi
 
 # Insert via abra under TTY-wrap (`abra app secret insert` requires a TTY on this version).
 # We DON'T log the key value — abra also doesn't print it.
 script -qec "abra app secret insert ${CCCI_APP_DOMAIN} pds_plc_rotation_key v1 ${KEY_HEX} --no-input" /dev/null \
-    >/dev/null 2>&1
+  >/dev/null 2>&1
 echo "  bluesky-pds install_steps: PLC rotation key inserted (v1)."

tests/bluesky-pds/test_restore.py

@@ -11,6 +11,6 @@ import _p4  # noqa: E402
 
 
 def test_restore_returns_state(live_app):
-    assert _p4.account_exists(live_app), (
-        "restore did not bring back the seeded marker account (PDS data did not survive restore)"
-    )
+    assert _p4.account_exists(
+        live_app
+    ), "restore did not bring back the seeded marker account (PDS data did not survive restore)"

tests/conftest.py

@@ -13,7 +13,8 @@ import sys
 import pytest
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "runner"))
-from harness import deps as deps_mod, lifecycle, naming  # noqa: E402
+from harness import deps as deps_mod  # noqa: E402
+from harness import lifecycle, naming
 
 
 def _short(s: str, n: int = 8) -> str:

tests/cryptpad/playwright/test_pad_content_roundtrip.py

@@ -26,6 +26,7 @@ Transient `net::ERR_NETWORK_CHANGED` is handled by the shared `goto_with_retry`
 
 from __future__ import annotations
 
+import contextlib
 import os
 import sys
 import uuid
@@ -39,7 +40,11 @@ def _open_pad(ctx, url):
     bar once CryptPad has created/loaded the fragment-keyed pad (`#/2/pad/edit/<key>/`)."""
     page = ctx.new_page()
     harness_browser.goto_with_retry(
-        page, url, accept_statuses=(200,), goto_timeout_ms=60_000, wait_until="load",
+        page,
+        url,
+        accept_statuses=(200,),
+        goto_timeout_ms=60_000,
+        wait_until="load",
         deadline_seconds=150,
     )
     pad_url = url
@@ -53,13 +58,15 @@ def _open_pad(ctx, url):
             pad_url = page.url
             break
         if i == 40:
-            try:
+            with contextlib.suppress(Exception):  # best-effort unstick
                 harness_browser.goto_with_retry(
-                    page, url, accept_statuses=(200,), goto_timeout_ms=60_000,
-                    wait_until="load", deadline_seconds=120,
+                    page,
+                    url,
+                    accept_statuses=(200,),
+                    goto_timeout_ms=60_000,
+                    wait_until="load",
+                    deadline_seconds=120,
                 )
-            except Exception:  # noqa: BLE001 — best-effort unstick
-                pass
     return page, pad_url
 
 
@@ -74,18 +81,22 @@ def _ckeditor_frame(page, deadline_polls=90, reload_at=22, reload_url=None):
             if "ckeditor-inner" in f.url:
                 return f
         if i == reload_at and reload_url is not None:
-            try:
+            with contextlib.suppress(Exception):  # reload is a best-effort unstick
                 harness_browser.goto_with_retry(
-                    page, reload_url, accept_statuses=(200,), goto_timeout_ms=60_000,
-                    wait_until="load", deadline_seconds=120,
+                    page,
+                    reload_url,
+                    accept_statuses=(200,),
+                    goto_timeout_ms=60_000,
+                    wait_until="load",
+                    deadline_seconds=120,
                 )
-            except Exception:  # noqa: BLE001 — reload is a best-effort unstick
-                pass
         page.wait_for_timeout(2000)
     return None
 
 
-def _poll_any_frame_for_text(page, needle, deadline_polls=120, reload_at=(20, 45, 75, 100), reload_url=None):
+def _poll_any_frame_for_text(
+    page, needle, deadline_polls=120, reload_at=(20, 45, 75, 100), reload_url=None
+):
     """Robust read-back (F2-13): poll EVERY frame's body text for `needle`, returning True as soon as
     it appears. The fresh cold-cache read-back context's deeply-nested CKEditor frame is slow/flaky to
     *attach* by URL (the prior `_ckeditor_frame` wait timed out on the Adversary's cold run), but the
@@ -101,13 +112,15 @@ def _poll_any_frame_for_text(page, needle, deadline_polls=120, reload_at=(20, 45
             except Exception:  # noqa: BLE001 — frame not ready / detached; keep polling
                 pass
         if reload_url and i in reload_at:
-            try:
+            with contextlib.suppress(Exception):  # best-effort unstick
                 harness_browser.goto_with_retry(
-                    page, reload_url, accept_statuses=(200,), goto_timeout_ms=60_000,
-                    wait_until="load", deadline_seconds=120,
+                    page,
+                    reload_url,
+                    accept_statuses=(200,),
+                    goto_timeout_ms=60_000,
+                    wait_until="load",
+                    deadline_seconds=120,
                 )
-            except Exception:  # noqa: BLE001 — best-effort unstick
-                pass
         page.wait_for_timeout(2000)
     return False
 
@@ -137,9 +150,9 @@ def test_cryptpad_pad_content_survives_fresh_session(live_app):
             # --- session 1: create the pad + write the marker ---
             ctx1 = browser.new_context(ignore_https_errors=True)
             page, pad_url = _open_pad(ctx1, f"https://{live_app}/pad/")
-            assert "#/2/pad/edit/" in pad_url, (
-                f"CryptPad did not create a fragment-keyed pad URL; got {pad_url!r}"
-            )
+            assert (
+                "#/2/pad/edit/" in pad_url
+            ), f"CryptPad did not create a fragment-keyed pad URL; got {pad_url!r}"
             ck = _ckeditor_frame(page, reload_url=pad_url)
             assert ck is not None, "CKEditor content frame never attached (pad editor not ready)"
             _dismiss_store_modal(page)
@@ -148,9 +161,9 @@ def test_cryptpad_pad_content_survives_fresh_session(live_app):
             page.wait_for_timeout(1000)
             body.type(marker, delay=40)
             page.wait_for_timeout(12000)  # let CryptPad encrypt + sync the update to the server
-            assert marker in ck.locator("body").inner_text(), (
-                "marker not present in the editor after typing — type did not land"
-            )
+            assert (
+                marker in ck.locator("body").inner_text()
+            ), "marker not present in the editor after typing — type did not land"
             ctx1.close()
 
             # --- session 2: FRESH context (no shared storage/localStorage) reads the pad back by URL.

tests/cryptpad/playwright/test_pad_create.py

@@ -51,9 +51,9 @@ def test_cryptpad_spa_renders_with_no_console_errors(live_app):
             title = (page.title() or "").lower()
             body = page.content()
             blower = body.lower()
-            assert "cryptpad" in title or "cryptpad" in blower, (
-                f"CryptPad SPA does not carry brand. title={title!r}, body excerpt: {body[:200]!r}"
-            )
+            assert (
+                "cryptpad" in title or "cryptpad" in blower
+            ), f"CryptPad SPA does not carry brand. title={title!r}, body excerpt: {body[:200]!r}"
 
             # Canonical CryptPad asset references in the rendered DOM
             canonical = ("/customize/", "/components/", "main.js", "/api/broadcast")

tests/cryptpad/test_install.py

@@ -8,7 +8,8 @@ import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
-from harness import browser as harness_browser, generic, lifecycle  # noqa: E402
+from harness import browser as harness_browser  # noqa: E402
+from harness import generic, lifecycle
 
 
 def test_serving_and_content(live_app, meta):

tests/custom-html-bkp-bad/ops.py

@@ -0,0 +1,19 @@
+"""custom-html-bkp-bad — lifecycle ops for bad-backup/bad-restore RED canaries.
+
+Intentionally has NO pre_backup hook: the marker is never seeded before backup,
+so the backup snapshot has no ci-marker.txt. pre_restore writes "mutated" so that if
+restore DOES bring back the snapshot, the marker is gone/still-mutated → test fails.
+"""
+
+from __future__ import annotations
+
+from harness import lifecycle
+
+MARKER_PATH = "/usr/share/nginx/html/ci-marker.txt"
+
+
+def pre_restore(domain: str, meta: dict) -> None:
+    """Write 'mutated' to the marker before restore runs. If restore brings back the
+    snapshot (which has no marker — never seeded by pre_backup), the marker ends up
+    MISSING or 'mutated' after restore → test_restore_returns_state FAILS → restore=RED."""
+    lifecycle.exec_in_app(domain, ["sh", "-c", f"echo mutated > {MARKER_PATH}"])

tests/custom-html-bkp-bad/recipe_meta.py

@@ -0,0 +1,5 @@
+# custom-html-bkp-bad — regression fixture for bad-backup canary.
+# This recipe is custom-html WITHOUT backupbot labels. Setting BACKUP_CAPABLE=True here forces the
+# harness to run the backup tier; the recipe itself has no backupbot service, so
+# `abra app backup create` produces no snapshot → test_backup_artifact fails → backup tier RED.
+BACKUP_CAPABLE = True

tests/custom-html-bkp-bad/test_backup.py

@@ -0,0 +1,30 @@
+"""custom-html-bkp-bad — BACKUP assertion (bad-backup RED canary).
+
+This recipe has no ops.py::pre_backup, so ci-marker.txt is NEVER seeded before the backup.
+Asserting its presence here causes backup tier RED — proving the server catches a recipe that
+claims backup support but doesn't actually back up the expected data.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
+from harness import lifecycle  # noqa: E402
+
+MARKER_PATH = "/usr/share/nginx/html/ci-marker.txt"
+
+
+def test_backup_captures_state(live_app):
+    """Assert the pre-backup marker is present and equals 'original'.
+
+    Since custom-html-bkp-bad has no ops.py::pre_backup to seed the marker, this file does NOT
+    exist at backup time — exec_in_app returns empty or raises → assertion fails → backup tier RED.
+    This models a recipe that declares backup capability but omits the data-seeding hook."""
+    result = lifecycle.exec_in_app(
+        live_app, ["sh", "-c", f"cat {MARKER_PATH} 2>/dev/null || echo MISSING"]
+    ).strip()
+    assert result == "original", (
+        f"backup did not capture the expected marker at {MARKER_PATH}: got {result!r}. "
+        "Expected 'original' (seeded by pre_backup). If the marker is 'MISSING', the pre_backup "
+        "hook was not run — this is the intended failure for the bad-backup RED canary."
+    )

tests/custom-html-bkp-bad/test_restore.py

@@ -0,0 +1,25 @@
+"""custom-html-bkp-bad — RESTORE assertion (bad-restore RED canary).
+
+pre_restore seeds 'mutated' to ci-marker.txt. The backup snapshot has no ci-marker.txt
+(never seeded by pre_backup). After restore, the marker is either MISSING or 'mutated' —
+never 'original' — so this assertion FAILS → restore tier RED.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
+from harness import lifecycle  # noqa: E402
+
+MARKER_PATH = "/usr/share/nginx/html/ci-marker.txt"
+
+
+def test_restore_returns_state(live_app):
+    result = lifecycle.exec_in_app(
+        live_app, ["sh", "-c", f"cat {MARKER_PATH} 2>/dev/null || echo MISSING"]
+    ).strip()
+    assert result == "original", (
+        f"restore did not return the pre-mutation (backed-up) state: got {result!r}. "
+        "Expected 'original'. The backup had no marker (not seeded by pre_backup), so "
+        "restore cannot recover it — this is the intended failure for the bad-restore RED canary."
+    )

tests/custom-html-rst-bad/ops.py

@@ -0,0 +1,15 @@
+"""custom-html-rst-bad — lifecycle ops for bad-restore RED canary.
+
+NO pre_backup hook: marker never seeded before backup → snapshot has no ci-marker.txt.
+pre_restore writes "mutated". After restore, marker stays "mutated" (not in snapshot) → FAIL.
+"""
+
+from __future__ import annotations
+
+from harness import lifecycle
+
+MARKER_PATH = "/usr/share/nginx/html/ci-marker.txt"
+
+
+def pre_restore(domain: str, meta: dict) -> None:
+    lifecycle.exec_in_app(domain, ["sh", "-c", f"echo mutated > {MARKER_PATH}"])

tests/custom-html-rst-bad/recipe_meta.py

@@ -0,0 +1,3 @@
+# custom-html-rst-bad — regression fixture for bad-restore canary.
+# BACKUP_CAPABLE=True forces the backup tier to run even though the recipe has no backupbot label.
+BACKUP_CAPABLE = True

tests/custom-html-rst-bad/test_restore.py

@@ -0,0 +1,23 @@
+"""custom-html-rst-bad — RESTORE assertion (bad-restore RED canary).
+
+No pre_backup → backup snapshot has no ci-marker.txt. pre_restore writes "mutated".
+After restore: marker is "mutated" (restore can't recover "original" — wasn't backed up) → FAIL.
+"""
+
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
+from harness import lifecycle  # noqa: E402
+
+MARKER_PATH = "/usr/share/nginx/html/ci-marker.txt"
+
+
+def test_restore_returns_state(live_app):
+    result = lifecycle.exec_in_app(
+        live_app, ["sh", "-c", f"cat {MARKER_PATH} 2>/dev/null || echo MISSING"]
+    ).strip()
+    assert result == "original", (
+        f"restore did not return the pre-mutation (backed-up) state: got {result!r}. "
+        "Expected 'original'. The backup had no marker, so restore cannot recover it."
+    )

tests/custom-html-tiny/functional/test_serves_content.py

@@ -0,0 +1,87 @@
+"""custom-html-tiny — recipe-specific functional test (static-web-server).
+
+Proves the deployed static-web-server is *actually serving files from its `content` volume* with real
+file-server semantics, not merely returning 200 from a Traefik fallback or a generic stub:
+
+  1. exact-byte round-trip — write a uniquely-named file with random content into the served volume,
+     fetch it over HTTPS, and assert the bytes come back verbatim. Non-vacuous: the content is random
+     per run, so only a server that reads this file off the volume can pass.
+  2. real 404 — a random non-existent path returns 404, proving directory/file semantics (a
+     200-everything stub or mis-routed host would not 404).
+
+The recipe's image (joseluisq/static-web-server) is shell-less (scratch-based) and its content volume
+is seeded via the install_steps.sh host-mountpoint mechanism — so this test writes its probe file the
+same way (resolve the swarm volume's mountpoint with `docker volume inspect`, write directly) rather
+than `docker exec`-ing in a container that has no shell.
+
+Runs in the custom tier against the shared post-install deployment (the `live_app` fixture is its
+per-run domain). Mirrors install_steps.sh: the app's content volume is named `<stack>_content`, where
+`stack` is the domain with dots replaced by underscores; HTTP_SUBDIR is empty, so the volume root is
+served at `/`.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import ssl
+import subprocess
+import urllib.error
+import urllib.request
+import uuid
+
+
+def _served_dir(domain: str) -> str:
+    """Host mountpoint of the app's served `content` volume (same naming as install_steps.sh)."""
+    vol = f"{domain.replace('.', '_')}_content"
+    out = subprocess.run(
+        ["docker", "volume", "inspect", vol, "--format", "{{.Mountpoint}}"],
+        capture_output=True,
+        text=True,
+        check=True,
+    )
+    mountpoint = out.stdout.strip()
+    assert mountpoint, f"could not resolve mountpoint for volume {vol!r}"
+    return mountpoint
+
+
+def _get(url: str) -> tuple[int, bytes]:
+    """GET the URL; return (status, body). A 4xx/5xx is returned, not raised (we assert on the code).
+    TLS verification is relaxed: the served wildcard cert is validated separately by the infra check;
+    here we care only about the app's response."""
+    ctx = ssl.create_default_context()
+    ctx.check_hostname = False
+    ctx.verify_mode = ssl.CERT_NONE
+    try:
+        with urllib.request.urlopen(url, timeout=20, context=ctx) as resp:
+            return resp.status, resp.read()
+    except urllib.error.HTTPError as e:
+        return e.code, e.read()
+
+
+def test_static_file_roundtrip_and_404(live_app):
+    """Write a random file into the served volume → fetch it → bytes match; and a missing path 404s."""
+    served = _served_dir(live_app)
+    token = uuid.uuid4().hex
+    name = f"ccci-probe-{token}.txt"
+    body = f"cc-ci-functional-{token}\n".encode()
+    path = os.path.join(served, name)
+    with open(path, "wb") as fh:
+        fh.write(body)
+    try:
+        status, got = _get(f"https://{live_app}/{name}")
+        assert status == 200, f"served probe file returned {status} (expected 200)"
+        assert got == body, (
+            f"content round-trip mismatch: served {got!r}, wrote {body!r} "
+            "(static-web-server not serving the content volume?)"
+        )
+
+        # A random non-existent path must 404 — proves real static-file semantics, distinguishing a
+        # working server from a 200-everything stub or a mis-routed Traefik fallback.
+        miss_status, _ = _get(f"https://{live_app}/ccci-missing-{uuid.uuid4().hex}.txt")
+        assert (
+            miss_status == 404
+        ), f"missing path returned {miss_status} (expected 404 — generic 200-returner / mis-route?)"
+    finally:
+        with contextlib.suppress(OSError):
+            os.remove(path)

tests/custom-html-tiny/recipe_meta.py

@@ -3,3 +3,14 @@
 # (DG5) is detected quickly instead of waiting the default 300s HTTP timeout.
 DEPLOY_TIMEOUT = 120
 HTTP_TIMEOUT = 90
+
+# Rungs this recipe INTENTIONALLY skips, each with a reason. Any essential rung skipped (N/A) and NOT
+# listed here is reported as an *unintentional* skip (a coverage gap to fill or declare). A skip still
+# caps the level either way — the harness never claims a rung it did not verify; this only records
+# that the skip is deliberate. (The level ladder is the four essential rungs install/upgrade/
+# backup_restore/functional; integration + recipe-local are optional and not leveled.)
+# custom-html-tiny is a stateless static-web-server, so it has no backup surface:
+EXPECTED_NA = {
+    "backup_restore": "stateless static file server: serves an ephemeral content volume seeded at "
+    "deploy, with no persistent/user data to back up or restore (no backupbot.backup label)",
+}

tests/custom-html/functional/test_content_roundtrip.py

@@ -15,7 +15,8 @@ import sys
 import uuid
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "runner"))
-from harness import http as harness_http, lifecycle  # noqa: E402
+from harness import http as harness_http  # noqa: E402
+from harness import lifecycle
 
 
 def test_content_roundtrip(live_app):

tests/custom-html/functional/test_content_type_header.py

@@ -53,9 +53,9 @@ def test_content_type_html_and_txt(live_app):
     ct_txt = h_txt.get("content-type", "")
 
     # nginx default: "text/html" for .html and "text/plain" for .txt (may include "; charset=utf-8")
-    assert ct_html.startswith("text/html"), (
-        f"{html_name} Content-Type={ct_html!r}, expected text/html (nginx MIME config broken?)"
-    )
-    assert ct_txt.startswith("text/plain"), (
-        f"{txt_name} Content-Type={ct_txt!r}, expected text/plain (nginx MIME config broken?)"
-    )
+    assert ct_html.startswith(
+        "text/html"
+    ), f"{html_name} Content-Type={ct_html!r}, expected text/html (nginx MIME config broken?)"
+    assert ct_txt.startswith(
+        "text/plain"
+    ), f"{txt_name} Content-Type={ct_txt!r}, expected text/plain (nginx MIME config broken?)"

tests/custom-html/test_install.py

@@ -9,7 +9,8 @@ import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
-from harness import browser as harness_browser, generic  # noqa: E402
+from harness import browser as harness_browser  # noqa: E402
+from harness import generic
 
 
 def test_serving_and_content(live_app, meta):

tests/discourse/functional/_discourse.py

@@ -53,7 +53,7 @@ def mint_admin(domain: str) -> tuple[str, str]:
     cmd = (
         "cd /opt/bitnami/discourse && "
         "RUBY=$(command -v ruby || echo /opt/bitnami/ruby/bin/ruby) && "
-        f"RAILS_ENV=production \"$RUBY\" bin/rails runner \"{_BOOTSTRAP_RB}\""
+        f'RAILS_ENV=production "$RUBY" bin/rails runner "{_BOOTSTRAP_RB}"'
     )
     out = lifecycle.exec_in_app(domain, ["bash", "-c", cmd], service="app", timeout=240)
     key = user = None
@@ -63,9 +63,9 @@ def mint_admin(domain: str) -> tuple[str, str]:
             key = line.split("=", 1)[1].strip()
         elif line.startswith("CCCI_API_USER="):
             user = line.split("=", 1)[1].strip()
-    assert key and user, (
-        f"could not bootstrap discourse admin/API key; rails output tail:\n{out[-1000:]}"
-    )
+    assert (
+        key and user
+    ), f"could not bootstrap discourse admin/API key; rails output tail:\n{out[-1000:]}"
     return key, user
 
 

tests/discourse/functional/test_create_topic.py

@@ -48,21 +48,23 @@ def test_create_topic_roundtrip(live_app):
         headers=hdrs,
         timeout=60,
     )
-    assert status in (200, 201) and isinstance(body, dict), (
-        f"create topic failed: HTTP {status}, body={body!r}"
-    )
+    assert status in (200, 201) and isinstance(
+        body, dict
+    ), f"create topic failed: HTTP {status}, body={body!r}"
     topic_id = body.get("topic_id")
     assert topic_id, f"create topic returned no topic_id: {body!r}"
 
     # 4) Read the topic back and assert title + first-post body round-trip.
     status, got = harness_http.http_get(f"{base}/t/{topic_id}.json", headers=hdrs, timeout=30)
-    assert status == 200 and isinstance(got, dict), f"read topic failed: HTTP {status}, body={got!r}"
-    assert got.get("title") == title, (
-        f"topic title did not round-trip: sent {title!r}, got {got.get('title')!r}"
-    )
+    assert status == 200 and isinstance(
+        got, dict
+    ), f"read topic failed: HTTP {status}, body={got!r}"
+    assert (
+        got.get("title") == title
+    ), f"topic title did not round-trip: sent {title!r}, got {got.get('title')!r}"
     posts = (got.get("post_stream") or {}).get("posts") or []
     assert posts, f"topic has no posts on read-back: {got!r}"
     first_cooked = posts[0].get("cooked", "")
-    assert marker in first_cooked, (
-        f"topic body did not round-trip: marker {marker!r} not in first post {first_cooked!r}"
-    )
+    assert (
+        marker in first_cooked
+    ), f"topic body did not round-trip: marker {marker!r} not in first post {first_cooked!r}"

tests/discourse/functional/test_site_basic.py

@@ -20,12 +20,12 @@ def test_site_json_has_discourse_config(live_app):
     status, body = harness_http.retry_http_get(
         f"https://{live_app}/site.json", expect_status=200, max_wait=120, interval=5
     )
-    assert status == 200 and isinstance(body, dict), (
-        f"GET /site.json failed: HTTP {status}, body type={type(body).__name__}"
-    )
+    assert status == 200 and isinstance(
+        body, dict
+    ), f"GET /site.json failed: HTTP {status}, body type={type(body).__name__}"
     # /site.json carries Discourse-specific structure — `categories` (a list) and `groups` are always
     # present in a booted Discourse. A non-Discourse 200 (placeholder page) would not parse to this.
     assert "categories" in body, f"/site.json missing 'categories' key: keys={list(body)[:20]}"
-    assert isinstance(body["categories"], list), (
-        f"/site.json 'categories' not a list: {type(body['categories']).__name__}"
-    )
+    assert isinstance(
+        body["categories"], list
+    ), f"/site.json 'categories' not a list: {type(body['categories']).__name__}"

tests/discourse/ops.py

@@ -15,8 +15,7 @@ from harness import lifecycle  # noqa: E402
 
 def _psql(domain, sql):
     cmd = (
-        'PGPASSWORD=$(cat /run/secrets/db_password) '
-        f'psql -U discourse -d discourse -tAc "{sql}"'
+        "PGPASSWORD=$(cat /run/secrets/db_password) " f'psql -U discourse -d discourse -tAc "{sql}"'
     )
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
@@ -42,6 +41,7 @@ def pre_backup(domain, meta):
 def pre_restore(domain, meta):
     # diverge from the backup so a successful restore is observable
     _psql(domain, "DROP TABLE IF EXISTS ci_marker;")
-    assert _psql(domain, "SELECT to_regclass('public.ci_marker');") in ("", "NULL"), (
-        "drop did not take"
-    )
+    assert _psql(domain, "SELECT to_regclass('public.ci_marker');") in (
+        "",
+        "NULL",
+    ), "drop did not take"

tests/discourse/recipe_meta.py

@@ -6,7 +6,9 @@
 # app is actually serving (the canonical "is discourse up" signal — NOT "/", which may redirect to setup).
 HEALTH_PATH = "/srv/status"
 HEALTH_OK = (200,)
-DEPLOY_TIMEOUT = 3600  # slow Rails cold boot (15-25min) on the 7-GiB single node; bumped 2400→3600 for
+DEPLOY_TIMEOUT = (
+    3600  # slow Rails cold boot (15-25min) on the 7-GiB single node; bumped 2400→3600 for
+)
 # headroom after full4's base deploy timed out at 2400s (RAM/CPU-constrained boot + image re-pull).
 HTTP_TIMEOUT = 1200
 
@@ -59,7 +61,11 @@ def BACKUP_VERIFY(domain):
     try:
         out = lifecycle.exec_in_app(
             domain,
-            ["sh", "-c", "gzip -t /var/lib/postgresql/data/backup.sql && wc -c < /var/lib/postgresql/data/backup.sql"],
+            [
+                "sh",
+                "-c",
+                "gzip -t /var/lib/postgresql/data/backup.sql && wc -c < /var/lib/postgresql/data/backup.sql",
+            ],
             service="db",
             timeout=60,
         ).strip()

tests/discourse/test_backup.py

@@ -14,13 +14,12 @@ from harness import lifecycle  # noqa: E402
 
 def _psql(domain, sql):
     cmd = (
-        'PGPASSWORD=$(cat /run/secrets/db_password) '
-        f'psql -U discourse -d discourse -tAc "{sql}"'
+        "PGPASSWORD=$(cat /run/secrets/db_password) " f'psql -U discourse -d discourse -tAc "{sql}"'
     )
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 
 def test_backup_captures_state(live_app):
-    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        "the seeded discourse postgres state was not present at backup time"
-    )
+    assert (
+        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+    ), "the seeded discourse postgres state was not present at backup time"

tests/discourse/test_restore.py

@@ -14,13 +14,12 @@ from harness import lifecycle  # noqa: E402
 
 def _psql(domain, sql):
     cmd = (
-        'PGPASSWORD=$(cat /run/secrets/db_password) '
-        f'psql -U discourse -d discourse -tAc "{sql}"'
+        "PGPASSWORD=$(cat /run/secrets/db_password) " f'psql -U discourse -d discourse -tAc "{sql}"'
     )
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 
 def test_restore_returns_state(live_app):
-    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        "restore did not return the pre-mutation discourse postgres state (data-integrity failure)"
-    )
+    assert (
+        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+    ), "restore did not return the pre-mutation discourse postgres state (data-integrity failure)"

tests/ghost/functional/_ghost.py

@@ -93,9 +93,10 @@ class GhostAdmin:
         status, body = self.req(
             "POST", "/session/", {"username": ADMIN_EMAIL, "password": ADMIN_PW}
         )
-        assert status in (200, 201), (
-            f"ghost admin session login failed: HTTP {status}, body={body!r}"
-        )
+        assert status in (
+            200,
+            201,
+        ), f"ghost admin session login failed: HTTP {status}, body={body!r}"
 
     def create_post(self, title: str, html: str) -> dict:
         status, body = self.req(

tests/ghost/functional/test_admin_redirect.py

@@ -53,13 +53,15 @@ def test_ghost_admin_route_is_wired(live_app):
         return None
 
     status_body = harness_http.assert_converges(
-        _ready, f"GET {url} returns Ghost admin (200) or setup redirect (302)",
-        max_wait=60, interval=3,
+        _ready,
+        f"GET {url} returns Ghost admin (200) or setup redirect (302)",
+        max_wait=60,
+        interval=3,
     )
     status, body = status_body
     assert status in (200, 302), f"unexpected status: {status}"
     if status == 200:
         # The admin SPA references /ghost-assets/ or contains "ghost" in title/body
-        assert "ghost" in body.lower(), (
-            f"GET {url} 200 but body has no Ghost markers: {body[:200]!r}"
-        )
+        assert (
+            "ghost" in body.lower()
+        ), f"GET {url} 200 but body has no Ghost markers: {body[:200]!r}"

tests/ghost/functional/test_content_api.py

@@ -35,10 +35,10 @@ def test_content_api_settings_endpoint(live_app):
     assert body is not None, f"GET {url} returned non-JSON body"
     # On success: {"settings": {...}}. On error: {"errors": [...]}. Either shape is valid.
     if status == 200:
-        assert isinstance(body, dict) and "settings" in body, (
-            f"200 response missing 'settings' envelope: {body!r}"
-        )
+        assert (
+            isinstance(body, dict) and "settings" in body
+        ), f"200 response missing 'settings' envelope: {body!r}"
     else:
-        assert isinstance(body, dict) and ("errors" in body or "message" in body or body), (
-            f"error response not a proper Ghost error envelope: {body!r}"
-        )
+        assert isinstance(body, dict) and (
+            "errors" in body or "message" in body or body
+        ), f"error response not a proper Ghost error envelope: {body!r}"

tests/ghost/functional/test_post_roundtrip.py

@@ -43,17 +43,17 @@ def test_create_post_roundtrip(live_app):
     title = f"ccci-marker-{uniq}"
     marker = f"ccci-body-marker-{uniq}-roundtrip"
     created = admin.create_post(title, f"<p>{marker}</p>")
-    assert created.get("title") == title, (
-        f"created post title mismatch: sent {title!r}, got {created.get('title')!r}"
-    )
+    assert (
+        created.get("title") == title
+    ), f"created post title mismatch: sent {title!r}, got {created.get('title')!r}"
 
     # 4) Read it back by id and assert the post survived the round-trip (title always returned;
     #    html returned because we requested ?formats=html).
     got = admin.get_post(created["id"])
-    assert got.get("title") == title, (
-        f"post title did not round-trip: sent {title!r}, got {got.get('title')!r}"
-    )
+    assert (
+        got.get("title") == title
+    ), f"post title did not round-trip: sent {title!r}, got {got.get('title')!r}"
     html = got.get("html") or ""
-    assert marker in html, (
-        f"post body did not round-trip: marker {marker!r} not in read-back html {html!r}"
-    )
+    assert (
+        marker in html
+    ), f"post body did not round-trip: marker {marker!r} not in read-back html {html!r}"

tests/ghost/ops.py

@@ -22,10 +22,7 @@ from harness import lifecycle  # noqa: E402
 
 
 def _mysql(domain, sql):
-    cmd = (
-        'MYSQL_PWD="$(cat /run/secrets/db_password)" '
-        f'mysql -u root -N -s ghost -e "{sql}"'
-    )
+    cmd = 'MYSQL_PWD="$(cat /run/secrets/db_password)" ' f'mysql -u root -N -s ghost -e "{sql}"'
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 

tests/ghost/recipe_meta.py

@@ -63,7 +63,11 @@ def BACKUP_VERIFY(domain):
     try:
         out = lifecycle.exec_in_app(
             domain,
-            ["sh", "-c", "gzip -t /var/lib/mysql/backup.sql.gz && wc -c < /var/lib/mysql/backup.sql.gz"],
+            [
+                "sh",
+                "-c",
+                "gzip -t /var/lib/mysql/backup.sql.gz && wc -c < /var/lib/mysql/backup.sql.gz",
+            ],
             service="db",
             timeout=60,
         ).strip()

tests/ghost/test_backup.py

@@ -15,14 +15,11 @@ from harness import lifecycle  # noqa: E402
 
 
 def _mysql(domain, sql):
-    cmd = (
-        'MYSQL_PWD="$(cat /run/secrets/db_password)" '
-        f'mysql -u root -N -s ghost -e "{sql}"'
-    )
+    cmd = 'MYSQL_PWD="$(cat /run/secrets/db_password)" ' f'mysql -u root -N -s ghost -e "{sql}"'
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 
 def test_backup_captures_state(live_app):
-    assert _mysql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        "the seeded ghost MySQL marker was not present at backup time"
-    )
+    assert (
+        _mysql(live_app, "SELECT v FROM ci_marker;") == "original"
+    ), "the seeded ghost MySQL marker was not present at backup time"

tests/ghost/test_restore.py

@@ -22,10 +22,7 @@ from harness import lifecycle  # noqa: E402
 
 
 def _mysql(domain, sql):
-    cmd = (
-        'MYSQL_PWD="$(cat /run/secrets/db_password)" '
-        f'mysql -u root -N -s ghost -e "{sql}"'
-    )
+    cmd = 'MYSQL_PWD="$(cat /run/secrets/db_password)" ' f'mysql -u root -N -s ghost -e "{sql}"'
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 

tests/ghost/test_upgrade.py

@@ -14,14 +14,11 @@ from harness import lifecycle  # noqa: E402
 
 
 def _mysql(domain, sql):
-    cmd = (
-        'MYSQL_PWD="$(cat /run/secrets/db_password)" '
-        f'mysql -u root -N -s ghost -e "{sql}"'
-    )
+    cmd = 'MYSQL_PWD="$(cat /run/secrets/db_password)" ' f'mysql -u root -N -s ghost -e "{sql}"'
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 
 def test_upgrade_preserves_state(live_app):
-    assert _mysql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives", (
-        "the seeded ghost MySQL marker did not survive the upgrade redeploy (data loss on upgrade)"
-    )
+    assert (
+        _mysql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives"
+    ), "the seeded ghost MySQL marker did not survive the upgrade redeploy (data loss on upgrade)"

tests/hedgedoc/functional/test_branding.py

@@ -14,7 +14,6 @@ import urllib.request
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "runner"))
 from harness import http as harness_http  # noqa: E402
 
-
 _CTX = ssl.create_default_context()
 _CTX.check_hostname = False
 _CTX.verify_mode = ssl.CERT_NONE

tests/hedgedoc/functional/test_health_check.py

@@ -15,7 +15,5 @@ from harness import http as harness_http  # noqa: E402
 def test_hedgedoc_root_serves(live_app):
     """GET / → 200 or 302 (login/new redirect)."""
     url = f"https://{live_app}/"
-    status, _ = harness_http.retry_http_get(
-        url, expect_status=(200, 302), max_wait=90, interval=5
-    )
+    status, _ = harness_http.retry_http_get(url, expect_status=(200, 302), max_wait=90, interval=5)
     assert status in (200, 302), f"GET {url} HTTP {status} (expected 200 or 302)"

tests/immich/functional/test_asset_processing.py

@@ -111,13 +111,13 @@ def test_immich_processes_uploaded_asset_metadata_and_statistics(live_app):
         if exif and exif.get("exifImageWidth"):
             break
         time.sleep(5)
-    assert exif and exif.get("exifImageWidth") == 1 and exif.get("exifImageHeight") == 1, (
-        f"immich metadata-extraction did not populate the 1x1 PNG dimensions in exifInfo: {exif!r}"
-    )
+    assert (
+        exif and exif.get("exifImageWidth") == 1 and exif.get("exifImageHeight") == 1
+    ), f"immich metadata-extraction did not populate the 1x1 PNG dimensions in exifInfo: {exif!r}"
 
     # the asset is catalogued into the owner's library statistics (list-back in aggregate)
     sst, stats = harness_http.http_request("GET", f"{base}/api/assets/statistics", headers=auth)
     assert sst == 200 and isinstance(stats, dict), f"statistics HTTP {sst}: {stats!r}"
-    assert stats.get("images", 0) >= 1 and stats.get("total", 0) >= 1, (
-        f"uploaded asset not reflected in library statistics: {stats!r}"
-    )
+    assert (
+        stats.get("images", 0) >= 1 and stats.get("total", 0) >= 1
+    ), f"uploaded asset not reflected in library statistics: {stats!r}"

tests/immich/functional/test_asset_upload.py

@@ -121,6 +121,6 @@ def test_immich_upload_asset_readback_and_thumbnail(live_app):
         if thumb == 200:
             break
         time.sleep(5)
-    assert thumb == 200, (
-        f"immich did not generate a thumbnail/derivative for the uploaded asset (last HTTP {thumb})"
-    )
+    assert (
+        thumb == 200
+    ), f"immich did not generate a thumbnail/derivative for the uploaded asset (last HTTP {thumb})"

tests/immich/functional/test_health_check.py

@@ -16,5 +16,11 @@ from harness import http as harness_http  # noqa: E402
 
 def test_immich_returns_200(live_app):
     url = f"https://{live_app}/"
-    status, _ = harness_http.retry_http_get(url, expect_status=(200, 301, 302), max_wait=60, interval=3)
-    assert status in (200, 301, 302), f"immich at {url} returned HTTP {status} (expected 200/301/302)"
+    status, _ = harness_http.retry_http_get(
+        url, expect_status=(200, 301, 302), max_wait=60, interval=3
+    )
+    assert status in (
+        200,
+        301,
+        302,
+    ), f"immich at {url} returned HTTP {status} (expected 200/301/302)"

tests/immich/ops.py

@@ -35,4 +35,7 @@ def pre_backup(domain, meta):
 
 def pre_restore(domain, meta):
     _psql(domain, "DROP TABLE ci_marker;")
-    assert _psql(domain, "SELECT to_regclass('public.ci_marker');") in ("", "NULL"), "drop did not take"
+    assert _psql(domain, "SELECT to_regclass('public.ci_marker');") in (
+        "",
+        "NULL",
+    ), "drop did not take"

tests/immich/test_backup.py

@@ -14,4 +14,6 @@ def _psql(domain, sql):
 
 
 def test_backup_captures_state(live_app):
-    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", "seeded postgres state not present at backup time"
+    assert (
+        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+    ), "seeded postgres state not present at backup time"

tests/immich/test_install.py

@@ -7,7 +7,8 @@ import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
-from harness import browser as harness_browser, generic, lifecycle  # noqa: E402
+from harness import browser as harness_browser  # noqa: E402
+from harness import generic, lifecycle
 
 
 def test_serving_and_frontend(live_app, meta):
@@ -25,7 +26,11 @@ def test_serving_and_frontend(live_app, meta):
             resp = harness_browser.goto_with_retry(
                 page, url, accept_statuses=(200, 301, 302), goto_timeout_ms=60_000
             )
-            assert resp is not None and resp.status in (200, 301, 302), f"page status {resp and resp.status}"
+            assert resp is not None and resp.status in (
+                200,
+                301,
+                302,
+            ), f"page status {resp and resp.status}"
             assert "<html" in page.content().lower(), "no HTML served by the immich frontend"
         finally:
             browser.close()

tests/immich/test_restore.py

@@ -14,4 +14,6 @@ def _psql(domain, sql):
 
 
 def test_restore_returns_state(live_app):
-    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", "restore did not return the pre-mutation postgres state"
+    assert (
+        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+    ), "restore did not return the pre-mutation postgres state"

tests/immich/test_upgrade.py

@@ -14,4 +14,6 @@ def _psql(domain, sql):
 
 
 def test_upgrade_preserves_data(live_app):
-    assert _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives", "postgres data did not survive the upgrade"
+    assert (
+        _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives"
+    ), "postgres data did not survive the upgrade"

tests/keycloak/functional/test_create_client_and_use.py

@@ -120,9 +120,9 @@ def test_create_confidential_client_and_obtain_token(live_app):
         "clientId": client_id,
         "enabled": True,
         "secret": client_secret,
-        "publicClient": False,            # confidential client
-        "serviceAccountsEnabled": True,    # required for client_credentials grant
-        "standardFlowEnabled": False,      # not needed for service-account-only client
+        "publicClient": False,  # confidential client
+        "serviceAccountsEnabled": True,  # required for client_credentials grant
+        "standardFlowEnabled": False,  # not needed for service-account-only client
         "directAccessGrantsEnabled": False,
         "protocol": "openid-connect",
     }
@@ -144,25 +144,25 @@ def test_create_confidential_client_and_obtain_token(live_app):
 
         # Use the client to obtain its own token (client_credentials grant)
         tok_status, tok_resp = _client_credentials_token(live_app, client_id, client_secret)
-        assert tok_status == 200, (
-            f"client_credentials token returned HTTP {tok_status}: {tok_resp!r}"
-        )
+        assert (
+            tok_status == 200
+        ), f"client_credentials token returned HTTP {tok_status}: {tok_resp!r}"
         access_token = tok_resp.get("access_token") if isinstance(tok_resp, dict) else None
-        assert isinstance(access_token, str) and access_token.count(".") == 2, (
-            f"client_credentials access_token not a JWT: {access_token!r}"
-        )
+        assert (
+            isinstance(access_token, str) and access_token.count(".") == 2
+        ), f"client_credentials access_token not a JWT: {access_token!r}"
 
         # Decode the JWT payload; assert azp matches the new client
         payload = json.loads(_b64url_decode(access_token.split(".")[1]))
-        assert payload.get("azp") == client_id, (
-            f"client_credentials JWT azp={payload.get('azp')!r} != client_id={client_id!r}"
-        )
+        assert (
+            payload.get("azp") == client_id
+        ), f"client_credentials JWT azp={payload.get('azp')!r} != client_id={client_id!r}"
         # Service-account token does NOT carry a session-scoped user (azp + clientId differ from
         # admin-cli token). The presence of azp + iss == per-run-domain proves the issuance flow.
         expected_iss = f"https://{live_app}/realms/master"
-        assert payload.get("iss") == expected_iss, (
-            f"JWT iss={payload.get('iss')!r} != {expected_iss!r}"
-        )
+        assert (
+            payload.get("iss") == expected_iss
+        ), f"JWT iss={payload.get('iss')!r} != {expected_iss!r}"
     finally:
         # Idempotent cleanup
         if cleanup_id:

tests/keycloak/functional/test_password_grant_token.py

@@ -43,22 +43,20 @@ def test_password_grant_issues_valid_jwt(live_app):
     token = kc_admin.admin_token(live_app, password)
 
     # Shape: a JWT is exactly 3 base64url segments
-    assert isinstance(token, str) and token.count(".") == 2, (
-        f"access_token does not look like a JWT (no 3 segments): len={len(token) if token else 0}"
-    )
+    assert (
+        isinstance(token, str) and token.count(".") == 2
+    ), f"access_token does not look like a JWT (no 3 segments): len={len(token) if token else 0}"
 
     payload = _decode_jwt_payload(token)
 
     # iss = the issuer URL, must be the per-run domain's /realms/master endpoint
     expected_iss = f"https://{live_app}/realms/master"
-    assert payload.get("iss") == expected_iss, (
-        f"JWT iss claim {payload.get('iss')!r} != {expected_iss!r}"
-    )
+    assert (
+        payload.get("iss") == expected_iss
+    ), f"JWT iss claim {payload.get('iss')!r} != {expected_iss!r}"
 
     # azp = authorized party (which client requested this token)
-    assert payload.get("azp") == "admin-cli", (
-        f"JWT azp claim {payload.get('azp')!r} != 'admin-cli'"
-    )
+    assert payload.get("azp") == "admin-cli", f"JWT azp claim {payload.get('azp')!r} != 'admin-cli'"
 
     # typ = token type
     assert payload.get("typ") == "Bearer", f"JWT typ claim {payload.get('typ')!r} != 'Bearer'"
@@ -70,6 +68,6 @@ def test_password_grant_issues_valid_jwt(live_app):
 
     # iat (issued at) is also a standard claim
     iat = payload.get("iat")
-    assert isinstance(iat, int) and iat <= time.time() + 60, (
-        f"JWT iat {iat!r} not a reasonable past timestamp"
-    )
+    assert (
+        isinstance(iat, int) and iat <= time.time() + 60
+    ), f"JWT iat {iat!r} not a reasonable past timestamp"

tests/keycloak/recipe_meta.py

@@ -2,5 +2,7 @@
 # conftest — enrolling this recipe needs NO change to runner/harness code (D5).
 HEALTH_PATH = "/realms/master"  # 200 JSON once keycloak is up (not "/", which redirects)
 HEALTH_OK = (200,)
-DEPLOY_TIMEOUT = 900  # JVM + DB migration are slow on a 2-vCPU VM; observed 502 fallback up to ~10min
+DEPLOY_TIMEOUT = (
+    900  # JVM + DB migration are slow on a 2-vCPU VM; observed 502 fallback up to ~10min
+)
 HTTP_TIMEOUT = 900

tests/keycloak/test_install.py

@@ -8,7 +8,8 @@ import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
-from harness import browser as harness_browser, generic, lifecycle  # noqa: E402
+from harness import browser as harness_browser  # noqa: E402
+from harness import generic, lifecycle
 
 
 def test_serving_and_admin_console(live_app, meta):

tests/lasuite-docs/functional/test_auth_required.py

@@ -28,9 +28,7 @@ def test_users_me_requires_auth(live_app):
     url = f"https://{live_app}/api/v1.0/users/me/"
     # Retry with broad acceptance: any 4xx (or specific 401) indicates the route exists + auth is
     # required. Reject 200 (anonymous access) and 5xx (broken backend).
-    status, _ = harness_http.retry_http_get(
-        url, expect_status=(401, 403), max_wait=60, interval=3
-    )
+    status, _ = harness_http.retry_http_get(url, expect_status=(401, 403), max_wait=60, interval=3)
     assert status in (401, 403), (
         f"GET {url} returned {status}, expected 401 (auth required). "
         f"200 = anonymous access leaked; 404 = route missing; 5xx = backend broken."

tests/lasuite-docs/functional/test_create_doc.py

@@ -27,7 +27,8 @@ import uuid
 import pytest
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "runner"))
-from harness import http as harness_http, sso  # noqa: E402
+from harness import http as harness_http  # noqa: E402
+from harness import sso
 
 
 @pytest.mark.requires_deps
@@ -36,13 +37,15 @@ def test_create_doc_and_read_back(live_app, deps_creds):
     kc = deps_creds["keycloak"]
 
     # Obtain a JWT via OIDC password grant
-    access_token = sso.oidc_password_grant({
-        "client_id": kc["client_id"],
-        "client_secret": kc["client_secret"],
-        "user": kc["user"],
-        "password": kc["password"],
-        "token_url": kc["token_url"],
-    })
+    access_token = sso.oidc_password_grant(
+        {
+            "client_id": kc["client_id"],
+            "client_secret": kc["client_secret"],
+            "user": kc["user"],
+            "password": kc["password"],
+            "token_url": kc["token_url"],
+        }
+    )
     auth = {"Authorization": f"Bearer {access_token}"}
 
     # Create a doc with a unique title
@@ -56,9 +59,9 @@ def test_create_doc_and_read_back(live_app, deps_creds):
     assert isinstance(body, dict), f"unexpected response shape: {body!r}"
     doc_id = body.get("id")
     assert doc_id, f"created doc has no id: {body!r}"
-    assert body.get("title") == title, (
-        f"created doc title mismatch: created={title!r}, response={body.get('title')!r}"
-    )
+    assert (
+        body.get("title") == title
+    ), f"created doc title mismatch: created={title!r}, response={body.get('title')!r}"
 
     # Fetch it back via the dedicated GET endpoint
     s, fetched = harness_http.http_get(
@@ -66,9 +69,10 @@ def test_create_doc_and_read_back(live_app, deps_creds):
     )
     assert s == 200, f"GET /api/v1.0/documents/{doc_id}/ HTTP {s}: {fetched!r}"
     assert isinstance(fetched, dict), f"unexpected GET response: {fetched!r}"
-    assert fetched.get("id") in (doc_id, str(doc_id)), (
-        f"fetched id mismatch: created={doc_id!r}, fetched={fetched.get('id')!r}"
-    )
-    assert fetched.get("title") == title, (
-        f"fetched title mismatch: created={title!r}, fetched={fetched.get('title')!r}"
-    )
+    assert fetched.get("id") in (
+        doc_id,
+        str(doc_id),
+    ), f"fetched id mismatch: created={doc_id!r}, fetched={fetched.get('id')!r}"
+    assert (
+        fetched.get("title") == title
+    ), f"fetched title mismatch: created={title!r}, fetched={fetched.get('title')!r}"

tests/lasuite-docs/functional/test_health_check.py

@@ -22,7 +22,11 @@ def test_lasuite_docs_returns_200(live_app):
     url = f"https://{live_app}/"
     # accept 200 (frontend SPA shell) — lasuite-docs serves the SPA at root unauthenticated;
     # the SPA itself bootstraps via /api/v1.0/users/me/ which requires OIDC (separate test).
-    status, _ = harness_http.retry_http_get(url, expect_status=(200, 301, 302), max_wait=60, interval=3)
-    assert status in (200, 301, 302), (
-        f"lasuite-docs at {url} returned HTTP {status} (expected 200/301/302)"
+    status, _ = harness_http.retry_http_get(
+        url, expect_status=(200, 301, 302), max_wait=60, interval=3
     )
+    assert status in (
+        200,
+        301,
+        302,
+    ), f"lasuite-docs at {url} returned HTTP {status} (expected 200/301/302)"

tests/lasuite-docs/functional/test_oidc_login.py

@@ -25,7 +25,8 @@ import urllib.request
 import pytest
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "runner"))
-from harness import http as harness_http, sso  # noqa: E402
+from harness import http as harness_http  # noqa: E402
+from harness import sso
 
 _CTX = ssl.create_default_context()
 _CTX.check_hostname = False
@@ -61,9 +62,9 @@ def test_oidc_login_via_keycloak(live_app, deps_creds):
     # 302 redirect. Both are valid "auth-required" indicators — accept either, but if a
     # redirect is returned it must point at the dep keycloak realm.
     if status in (301, 302, 303, 307, 308):
-        assert expected_prefix in (redirect or ""), (
-            f"Docs redirected to {redirect!r}, expected to start with {expected_prefix!r}"
-        )
+        assert expected_prefix in (
+            redirect or ""
+        ), f"Docs redirected to {redirect!r}, expected to start with {expected_prefix!r}"
     else:
         assert status in (401, 403), (
             f"GET /api/v1.0/users/me/ unauth: HTTP {status}; expected redirect to keycloak "
@@ -88,6 +89,6 @@ def test_oidc_login_via_keycloak(live_app, deps_creds):
     )
     assert status == 200, f"GET /api/v1.0/users/me/ with token HTTP {status}: {body!r}"
     assert isinstance(body, dict), f"unexpected response: {body!r}"
-    assert body.get("email") == kc["email"], (
-        f"unexpected user email: got {body.get('email')!r}, expected {kc['email']!r}"
-    )
+    assert (
+        body.get("email") == kc["email"]
+    ), f"unexpected user email: got {body.get('email')!r}, expected {kc['email']!r}"

tests/lasuite-docs/functional/test_oidc_with_keycloak.py

@@ -42,9 +42,9 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps_creds):
     # Sanity-check the creds shape — orchestrator-written
     assert kc["domain"]
     # WC1: realm is per-run namespaced "<parent>-<6hex>" so concurrent dependents never collide.
-    assert re.fullmatch(r"lasuite-docs-[0-9a-f]{6}", kc["realm"]), (
-        f"realm {kc['realm']!r} not the per-run namespaced form lasuite-docs-<6hex>"
-    )
+    assert re.fullmatch(
+        r"lasuite-docs-[0-9a-f]{6}", kc["realm"]
+    ), f"realm {kc['realm']!r} not the per-run namespaced form lasuite-docs-<6hex>"
     assert kc["client_id"] == "lasuite-docs"
     assert isinstance(kc["client_secret"], str) and len(kc["client_secret"]) >= 16
     assert isinstance(kc["password"], str) and len(kc["password"]) >= 16
@@ -74,16 +74,14 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps_creds):
 
     # Password grant → real JWT
     token = sso.oidc_password_grant(creds)
-    assert isinstance(token, str) and token.count(".") == 2, (
-        f"access_token is not a JWT: {token!r}"
-    )
+    assert isinstance(token, str) and token.count(".") == 2, f"access_token is not a JWT: {token!r}"
     payload = json.loads(_b64url_decode(token.split(".")[1]))
     assert payload.get("iss") == expected_iss, f"JWT iss={payload.get('iss')!r} != {expected_iss!r}"
-    assert payload.get("azp") == kc["client_id"], (
-        f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
-    )
+    assert (
+        payload.get("azp") == kc["client_id"]
+    ), f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
     assert payload.get("typ") == "Bearer", f"JWT typ={payload.get('typ')!r} != 'Bearer'"
     exp = payload.get("exp")
-    assert isinstance(exp, int) and exp > time.time(), (
-        f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"
-    )
+    assert (
+        isinstance(exp, int) and exp > time.time()
+    ), f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"

tests/lasuite-docs/setup_custom_tests.sh

@@ -21,15 +21,24 @@ set -euo pipefail
 
 : "${CCCI_APP_DOMAIN:?missing}"
 : "${CCCI_DEPS_FILE:?missing}"
-test -s "$CCCI_DEPS_FILE" || { echo "  setup_custom_tests: deps file empty"; exit 1; }
+test -s "$CCCI_DEPS_FILE" || {
+  echo "  setup_custom_tests: deps file empty"
+  exit 1
+}
 
 # Read keycloak dep info via jq
-KC_DOMAIN=$(jq -r '.keycloak.domain'         "$CCCI_DEPS_FILE")
-KC_REALM=$( jq -r '.keycloak.realm'          "$CCCI_DEPS_FILE")
-KC_CLIENT=$(jq -r '.keycloak.client_id'      "$CCCI_DEPS_FILE")
-KC_SECRET=$(jq -r '.keycloak.client_secret'  "$CCCI_DEPS_FILE")
-[ -n "$KC_DOMAIN" ] && [ "$KC_DOMAIN" != "null" ] || { echo "  setup_custom_tests: no keycloak.domain in deps"; exit 1; }
-[ -n "$KC_SECRET" ] && [ "$KC_SECRET" != "null" ] || { echo "  setup_custom_tests: no keycloak.client_secret"; exit 1; }
+KC_DOMAIN=$(jq -r '.keycloak.domain' "$CCCI_DEPS_FILE")
+KC_REALM=$(jq -r '.keycloak.realm' "$CCCI_DEPS_FILE")
+KC_CLIENT=$(jq -r '.keycloak.client_id' "$CCCI_DEPS_FILE")
+KC_SECRET=$(jq -r '.keycloak.client_secret' "$CCCI_DEPS_FILE")
+if [ -z "$KC_DOMAIN" ] || [ "$KC_DOMAIN" = "null" ]; then
+  echo "  setup_custom_tests: no keycloak.domain in deps"
+  exit 1
+fi
+if [ -z "$KC_SECRET" ] || [ "$KC_SECRET" = "null" ]; then
+  echo "  setup_custom_tests: no keycloak.client_secret"
+  exit 1
+fi
 
 echo "  lasuite-docs setup_custom_tests: wiring OIDC against keycloak dep ${KC_DOMAIN}"
 
@@ -39,12 +48,15 @@ echo "  lasuite-docs setup_custom_tests: wiring OIDC against keycloak dep ${KC_D
 # update SECRET_OIDC_RPCS_VERSION in the .env to point at the new one.
 ENV_PATH="$HOME/.abra/servers/default/${CCCI_APP_DOMAIN}.env"
 CUR_VER=$(grep -E '^\s*SECRET_OIDC_RPCS_VERSION=' "$ENV_PATH" | tail -1 | cut -d= -f2 | tr -d '"\r' || echo "v1")
-NEW_NUM=$(( ${CUR_VER#v} + 1 ))
+NEW_NUM=$((${CUR_VER#v} + 1))
 NEW_VER="v${NEW_NUM}"
 
-INSERT_LOG=$(abra app secret insert $CCCI_APP_DOMAIN oidc_rpcs $NEW_VER $KC_SECRET --no-input -C -o 2>&1) \
-  || INSERT_LOG=$(script -qec "abra app secret insert $CCCI_APP_DOMAIN oidc_rpcs $NEW_VER $KC_SECRET --no-input -C -o" /dev/null 2>&1) \
-  || { echo "  setup_custom_tests: abra app secret insert oidc_rpcs@$NEW_VER failed: $INSERT_LOG"; exit 1; }
+INSERT_LOG=$(abra app secret insert "$CCCI_APP_DOMAIN" oidc_rpcs "$NEW_VER" "$KC_SECRET" --no-input -C -o 2>&1) ||
+  INSERT_LOG=$(script -qec "abra app secret insert $CCCI_APP_DOMAIN oidc_rpcs $NEW_VER $KC_SECRET --no-input -C -o" /dev/null 2>&1) ||
+  {
+    echo "  setup_custom_tests: abra app secret insert oidc_rpcs@$NEW_VER failed: $INSERT_LOG"
+    exit 1
+  }
 # Repoint the env var to the new version
 sed -i "s|^\s*SECRET_OIDC_RPCS_VERSION=.*|SECRET_OIDC_RPCS_VERSION=$NEW_VER|" "$ENV_PATH"
 echo "  setup_custom_tests: oidc_rpcs secret inserted at $NEW_VER (was $CUR_VER)"
@@ -52,25 +64,25 @@ echo "  setup_custom_tests: oidc_rpcs secret inserted at $NEW_VER (was $CUR_VER)
 # 2) Write OIDC env vars to the app's .env (names per lasuite-docs's .env.sample).
 # Ensure the file ends with a newline FIRST so our appends don't concatenate onto the last line
 # (we saw `TIMEOUT=900OIDC_REALM=...` malformed by a missing-trailing-newline file).
-[ -z "$(tail -c1 "$ENV_PATH" 2>/dev/null)" ] || printf '\n' >> "$ENV_PATH"
-write_env () {
+[ -z "$(tail -c1 "$ENV_PATH" 2>/dev/null)" ] || printf '\n' >>"$ENV_PATH"
+write_env() {
   local key="$1" val="$2"
   # remove any existing key (commented or live) then append the live key=val
   sed -i "/^\s*#\?\s*${key}=/d" "$ENV_PATH"
   # Re-ensure trailing newline after each delete (sed may leave the file without one)
-  [ -z "$(tail -c1 "$ENV_PATH" 2>/dev/null)" ] || printf '\n' >> "$ENV_PATH"
-  printf '%s=%s\n' "$key" "$val" >> "$ENV_PATH"
+  [ -z "$(tail -c1 "$ENV_PATH" 2>/dev/null)" ] || printf '\n' >>"$ENV_PATH"
+  printf '%s=%s\n' "$key" "$val" >>"$ENV_PATH"
 }
-write_env OIDC_REALM                       "$KC_REALM"
-write_env OIDC_OP_DISCOVERY_ENDPOINT       "https://${KC_DOMAIN}/realms/${KC_REALM}/.well-known/openid-configuration"
-write_env OIDC_OP_AUTHORIZATION_ENDPOINT   "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/auth"
-write_env OIDC_OP_TOKEN_ENDPOINT           "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/token"
-write_env OIDC_OP_USER_ENDPOINT            "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/userinfo"
-write_env OIDC_OP_LOGOUT_ENDPOINT          "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/logout"
-write_env OIDC_OP_JWKS_ENDPOINT            "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/certs"
-write_env OIDC_RP_CLIENT_ID                "$KC_CLIENT"
-write_env OIDC_RP_SIGN_ALGO                "RS256"
-write_env OIDC_RP_SCOPES                   "openid email profile"
+write_env OIDC_REALM "$KC_REALM"
+write_env OIDC_OP_DISCOVERY_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/.well-known/openid-configuration"
+write_env OIDC_OP_AUTHORIZATION_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/auth"
+write_env OIDC_OP_TOKEN_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/token"
+write_env OIDC_OP_USER_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/userinfo"
+write_env OIDC_OP_LOGOUT_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/logout"
+write_env OIDC_OP_JWKS_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/certs"
+write_env OIDC_RP_CLIENT_ID "$KC_CLIENT"
+write_env OIDC_RP_SIGN_ALGO "RS256"
+write_env OIDC_RP_SCOPES "openid email profile"
 
 # 3) Trigger an in-place redeploy so the env update takes effect. --force re-deploys even when
 # the recipe hasn't changed; --chaos avoids the chaos prompt; --no-input non-interactive.

tests/lasuite-docs/test_install.py

@@ -10,7 +10,8 @@ import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
-from harness import browser as harness_browser, generic, lifecycle  # noqa: E402
+from harness import browser as harness_browser  # noqa: E402
+from harness import generic, lifecycle
 
 
 def test_serving_and_frontend(live_app, meta):

tests/lasuite-drive/functional/test_health_check.py

@@ -25,6 +25,8 @@ def test_lasuite_drive_returns_200(live_app):
     status, _ = harness_http.retry_http_get(
         url, expect_status=(200, 301, 302), max_wait=60, interval=3
     )
-    assert status in (200, 301, 302), (
-        f"lasuite-drive at {url} returned HTTP {status} (expected 200/301/302)"
-    )
+    assert status in (
+        200,
+        301,
+        302,
+    ), f"lasuite-drive at {url} returned HTTP {status} (expected 200/301/302)"

tests/lasuite-drive/functional/test_minio_storage.py

@@ -29,8 +29,8 @@ BUCKET = "drive-media-storage"
 def _mc(domain: str, script: str) -> str:
     """Run an `mc` shell script inside the minio container (root creds from /run/secrets)."""
     prelude = (
-        'set -e; '
-        'U=$(cat /run/secrets/minio_ru); P=$(cat /run/secrets/minio_rp); '
+        "set -e; "
+        "U=$(cat /run/secrets/minio_ru); P=$(cat /run/secrets/minio_rp); "
         'mc alias set ccci http://localhost:9000 "$U" "$P" >/dev/null 2>&1; '
     )
     return lifecycle.exec_in_app(domain, ["sh", "-c", prelude + script], service="minio")
@@ -49,13 +49,13 @@ def test_minio_bucket_present_and_object_roundtrip(live_app):
         domain,
         # upload via stdin; list the object; read it back (tagged); then delete.
         f'printf %s "{marker}" | mc pipe ccci/{BUCKET}/{key} >/dev/null 2>&1; '
-        f'mc ls ccci/{BUCKET}/{key}; '
+        f"mc ls ccci/{BUCKET}/{key}; "
         f'echo "READBACK:$(mc cat ccci/{BUCKET}/{key})"; '
-        f'mc rm ccci/{BUCKET}/{key} >/dev/null 2>&1',
+        f"mc rm ccci/{BUCKET}/{key} >/dev/null 2>&1",
     )
 
     # The object was listed (its key appears) and its content round-tripped intact.
     assert f"{marker}.txt" in out, f"uploaded object not listed in bucket: {out!r}"
-    assert f"READBACK:{marker}" in out, (
-        f"object content did not round-trip through MinIO; got: {out!r}"
-    )
+    assert (
+        f"READBACK:{marker}" in out
+    ), f"object content did not round-trip through MinIO; got: {out!r}"

tests/lasuite-drive/functional/test_oidc_with_keycloak.py

@@ -46,9 +46,9 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps_creds):
 
     # Creds shape. WC1: realm is per-run namespaced "<parent>-<6hex>"; client_id stays the parent.
     assert kc["domain"]
-    assert re.fullmatch(r"lasuite-drive-[0-9a-f]{6}", kc["realm"]), (
-        f"realm {kc['realm']!r} not the per-run namespaced form lasuite-drive-<6hex>"
-    )
+    assert re.fullmatch(
+        r"lasuite-drive-[0-9a-f]{6}", kc["realm"]
+    ), f"realm {kc['realm']!r} not the per-run namespaced form lasuite-drive-<6hex>"
     assert kc["client_id"] == "lasuite-drive"
     assert isinstance(kc["client_secret"], str) and len(kc["client_secret"]) >= 16
     assert isinstance(kc["password"], str) and len(kc["password"]) >= 16
@@ -77,16 +77,14 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps_creds):
 
     # Password grant → real JWT
     token = sso.oidc_password_grant(creds)
-    assert isinstance(token, str) and token.count(".") == 2, (
-        f"access_token is not a JWT: {token!r}"
-    )
+    assert isinstance(token, str) and token.count(".") == 2, f"access_token is not a JWT: {token!r}"
     payload = json.loads(_b64url_decode(token.split(".")[1]))
     assert payload.get("iss") == expected_iss, f"JWT iss={payload.get('iss')!r} != {expected_iss!r}"
-    assert payload.get("azp") == kc["client_id"], (
-        f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
-    )
+    assert (
+        payload.get("azp") == kc["client_id"]
+    ), f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
     assert payload.get("typ") == "Bearer", f"JWT typ={payload.get('typ')!r} != 'Bearer'"
     exp = payload.get("exp")
-    assert isinstance(exp, int) and exp > time.time(), (
-        f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"
-    )
+    assert (
+        isinstance(exp, int) and exp > time.time()
+    ), f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"

tests/lasuite-drive/install_steps.sh

@@ -28,7 +28,7 @@ if [ -z "${CCCI_DEPS_FILE:-}" ] || [ ! -s "${CCCI_DEPS_FILE}" ]; then
   exit 0
 fi
 KC_DOMAIN=$(jq -r '.keycloak.domain        // empty' "$CCCI_DEPS_FILE")
-KC_REALM=$( jq -r '.keycloak.realm         // empty' "$CCCI_DEPS_FILE")
+KC_REALM=$(jq -r '.keycloak.realm         // empty' "$CCCI_DEPS_FILE")
 KC_CLIENT=$(jq -r '.keycloak.client_id     // empty' "$CCCI_DEPS_FILE")
 KC_SECRET=$(jq -r '.keycloak.client_secret // empty' "$CCCI_DEPS_FILE")
 if [ -z "$KC_DOMAIN" ] || [ -z "$KC_SECRET" ]; then
@@ -43,35 +43,38 @@ echo "  lasuite-drive install_steps: wiring OIDC at install against keycloak ${K
 # point SECRET_OIDC_RPCS_VERSION at it. (The app is not deployed yet — a swarm secret can be created
 # independently of a running stack — so the single deploy below picks up v2.)
 CUR_VER=$(grep -E '^\s*SECRET_OIDC_RPCS_VERSION=' "$ENV_PATH" | tail -1 | cut -d= -f2 | tr -d '"\r' || echo "v1")
-NEW_NUM=$(( ${CUR_VER#v} + 1 ))
+NEW_NUM=$((${CUR_VER#v} + 1))
 NEW_VER="v${NEW_NUM}"
-INSERT_LOG=$(abra app secret insert "$CCCI_APP_DOMAIN" oidc_rpcs "$NEW_VER" "$KC_SECRET" --no-input -C -o 2>&1) \
-  || INSERT_LOG=$(script -qec "abra app secret insert $CCCI_APP_DOMAIN oidc_rpcs $NEW_VER $KC_SECRET --no-input -C -o" /dev/null 2>&1) \
-  || { echo "  install_steps: abra app secret insert oidc_rpcs@$NEW_VER failed: $INSERT_LOG"; exit 1; }
+INSERT_LOG=$(abra app secret insert "$CCCI_APP_DOMAIN" oidc_rpcs "$NEW_VER" "$KC_SECRET" --no-input -C -o 2>&1) ||
+  INSERT_LOG=$(script -qec "abra app secret insert $CCCI_APP_DOMAIN oidc_rpcs $NEW_VER $KC_SECRET --no-input -C -o" /dev/null 2>&1) ||
+  {
+    echo "  install_steps: abra app secret insert oidc_rpcs@$NEW_VER failed: $INSERT_LOG"
+    exit 1
+  }
 sed -i "s|^\s*SECRET_OIDC_RPCS_VERSION=.*|SECRET_OIDC_RPCS_VERSION=$NEW_VER|" "$ENV_PATH"
 echo "  install_steps: oidc_rpcs secret inserted at $NEW_VER (was $CUR_VER)"
 
 # 2) Write the OIDC env vars (explicit endpoints — deterministic, no reliance on ${AUTH_DOMAIN}
 # expansion). Mirrors the recipe-maintainer impress/La Suite OIDC env contract.
-write_env () {
+write_env() {
   local key="$1" val="$2"
   sed -i "/^\s*#\?\s*${key}=/d" "$ENV_PATH"
-  [ -z "$(tail -c1 "$ENV_PATH" 2>/dev/null)" ] || printf '\n' >> "$ENV_PATH"
-  printf '%s=%s\n' "$key" "$val" >> "$ENV_PATH"
+  [ -z "$(tail -c1 "$ENV_PATH" 2>/dev/null)" ] || printf '\n' >>"$ENV_PATH"
+  printf '%s=%s\n' "$key" "$val" >>"$ENV_PATH"
 }
-write_env AUTH_DOMAIN                      "$KC_DOMAIN"
-write_env OIDC_REALM                       "$KC_REALM"
-write_env OIDC_OP_JWKS_ENDPOINT            "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/certs"
-write_env OIDC_OP_AUTHORIZATION_ENDPOINT   "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/auth"
-write_env OIDC_OP_TOKEN_ENDPOINT           "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/token"
-write_env OIDC_OP_USER_ENDPOINT            "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/userinfo"
-write_env OIDC_OP_LOGOUT_ENDPOINT          "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/logout"
-write_env OIDC_RP_CLIENT_ID                "$KC_CLIENT"
-write_env OIDC_RP_SIGN_ALGO                "RS256"
-write_env OIDC_RP_SCOPES                   "openid email profile"
-write_env OIDC_REDIRECT_ALLOWED_HOSTS      "[\"https://${KC_DOMAIN}\", \"https://${CCCI_APP_DOMAIN}\"]"
+write_env AUTH_DOMAIN "$KC_DOMAIN"
+write_env OIDC_REALM "$KC_REALM"
+write_env OIDC_OP_JWKS_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/certs"
+write_env OIDC_OP_AUTHORIZATION_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/auth"
+write_env OIDC_OP_TOKEN_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/token"
+write_env OIDC_OP_USER_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/userinfo"
+write_env OIDC_OP_LOGOUT_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/logout"
+write_env OIDC_RP_CLIENT_ID "$KC_CLIENT"
+write_env OIDC_RP_SIGN_ALGO "RS256"
+write_env OIDC_RP_SCOPES "openid email profile"
+write_env OIDC_REDIRECT_ALLOWED_HOSTS "[\"https://${KC_DOMAIN}\", \"https://${CCCI_APP_DOMAIN}\"]"
 # The recipe default acr_values=eidas1 is FranceConnect-specific; keycloak can't satisfy it and it
 # would break the interactive auth flow. Clear it so the keycloak OIDC client works.
-write_env OIDC_AUTH_REQUEST_EXTRA_PARAMS   "{}"
+write_env OIDC_AUTH_REQUEST_EXTRA_PARAMS "{}"
 
 echo "  lasuite-drive install_steps: OIDC env wired into .env (deploy will pick it up, no reconverge)"

tests/lasuite-drive/setup_custom_tests.sh

@@ -29,7 +29,7 @@ docker service scale --detach "${STACK}_minio-createbuckets=1" >/dev/null 2>&1 |
 for i in $(seq 1 30); do
   MC_CID=$(docker ps -q -f "name=${STACK}_minio.1" | head -1)
   if [ -n "$MC_CID" ] && docker exec "$MC_CID" sh -c \
-       'mc alias set _c http://localhost:9000 "$(cat /run/secrets/minio_ru)" "$(cat /run/secrets/minio_rp)" >/dev/null 2>&1 && mc ls _c/drive-media-storage >/dev/null 2>&1'; then
+    'mc alias set _c http://localhost:9000 "$(cat /run/secrets/minio_ru)" "$(cat /run/secrets/minio_rp)" >/dev/null 2>&1 && mc ls _c/drive-media-storage >/dev/null 2>&1'; then
     echo "  setup: bucket drive-media-storage present after ${i} poll(s)"
     break
   fi

tests/lasuite-drive/test_install.py

@@ -10,7 +10,8 @@ import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
-from harness import browser as harness_browser, generic, lifecycle  # noqa: E402
+from harness import browser as harness_browser  # noqa: E402
+from harness import generic, lifecycle
 
 
 def test_serving_and_frontend(live_app, meta):

tests/lasuite-meet/functional/test_health_check.py

@@ -21,6 +21,8 @@ def test_lasuite_meet_returns_200(live_app):
     status, _ = harness_http.retry_http_get(
         url, expect_status=(200, 301, 302), max_wait=60, interval=3
     )
-    assert status in (200, 301, 302), (
-        f"lasuite-meet at {url} returned HTTP {status} (expected 200/301/302)"
-    )
+    assert status in (
+        200,
+        301,
+        302,
+    ), f"lasuite-meet at {url} returned HTTP {status} (expected 200/301/302)"

tests/lasuite-meet/functional/test_meeting_flow.py

@@ -28,7 +28,8 @@ import sys
 import pytest
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "runner"))
-from harness import http as harness_http, sso  # noqa: E402
+from harness import http as harness_http  # noqa: E402
+from harness import sso
 
 
 def _b64url(seg: str) -> bytes:
@@ -74,33 +75,40 @@ def test_create_room_get_livekit_token_and_read_back(live_app, deps_creds):
     lk_room = livekit.get("room")
     lk_token = livekit.get("token")
     assert room_id, f"room created but no id: {body!r}"
-    assert lk_token and isinstance(lk_token, str) and lk_token.count(".") == 2, (
-        f"room created but no LiveKit JWT token: {livekit!r}"
-    )
+    assert (
+        lk_token and isinstance(lk_token, str) and lk_token.count(".") == 2
+    ), f"room created but no LiveKit JWT token: {livekit!r}"
 
     try:
         # --- read it back (a fresh authenticated GET of the created room) ---
-        status, got = harness_http.http_request("GET", f"{base}/api/v1.0/rooms/{room_id}/", headers=auth)
+        status, got = harness_http.http_request(
+            "GET", f"{base}/api/v1.0/rooms/{room_id}/", headers=auth
+        )
         assert status == 200, f"room read-back returned HTTP {status} (expected 200); body={got!r}"
-        assert isinstance(got, dict) and got.get("id") == room_id, (
-            f"read-back room id mismatch: {got!r}"
-        )
-        got_lk = (got.get("livekit") or {})
+        assert (
+            isinstance(got, dict) and got.get("id") == room_id
+        ), f"read-back room id mismatch: {got!r}"
+        got_lk = got.get("livekit") or {}
         assert got_lk.get("token"), f"read-back room missing LiveKit token: {got!r}"
-        assert got_lk.get("room") == lk_room, (
-            f"read-back LiveKit room {got_lk.get('room')!r} != create-time {lk_room!r}"
-        )
+        assert (
+            got_lk.get("room") == lk_room
+        ), f"read-back LiveKit room {got_lk.get('room')!r} != create-time {lk_room!r}"
 
         # --- the LiveKit token is a real signaling grant for this room (WebRTC subset) ---
         payload = json.loads(_b64url(lk_token.split(".")[1]))
         video = payload.get("video") or {}
-        assert video.get("room") == lk_room or payload.get("room") == lk_room, (
-            f"LiveKit JWT does not grant the created room {lk_room!r}: {payload!r}"
-        )
+        assert (
+            video.get("room") == lk_room or payload.get("room") == lk_room
+        ), f"LiveKit JWT does not grant the created room {lk_room!r}: {payload!r}"
     finally:
         # --- delete the room (cleanup + a real DELETE mutation) ---
-        del_status, _ = harness_http.http_request("DELETE", f"{base}/api/v1.0/rooms/{room_id}/", headers=auth)
-        assert del_status in (204, 200), f"room delete returned HTTP {del_status} (expected 204/200)"
+        del_status, _ = harness_http.http_request(
+            "DELETE", f"{base}/api/v1.0/rooms/{room_id}/", headers=auth
+        )
+        assert del_status in (
+            204,
+            200,
+        ), f"room delete returned HTTP {del_status} (expected 204/200)"
 
     # --- best-effort: confirm the delete took (404 on re-GET). The §4.3 floor (create-an-object +
     # read-it-back + LiveKit-token issuance) is already proven by the hard assertions above; this
@@ -112,7 +120,9 @@ def test_create_room_get_livekit_token_and_read_back(live_app, deps_creds):
 
     gone = False
     for _ in range(5):
-        status, _ = harness_http.http_request("GET", f"{base}/api/v1.0/rooms/{room_id}/", headers=auth)
+        status, _ = harness_http.http_request(
+            "GET", f"{base}/api/v1.0/rooms/{room_id}/", headers=auth
+        )
         if status == 404:
             gone = True
             break

tests/lasuite-meet/functional/test_oidc_with_keycloak.py

@@ -46,9 +46,9 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps_creds):
 
     # Creds shape. WC1: realm is per-run namespaced "<parent>-<6hex>"; client_id stays the parent.
     assert kc["domain"]
-    assert re.fullmatch(r"lasuite-meet-[0-9a-f]{6}", kc["realm"]), (
-        f"realm {kc['realm']!r} not the per-run namespaced form lasuite-meet-<6hex>"
-    )
+    assert re.fullmatch(
+        r"lasuite-meet-[0-9a-f]{6}", kc["realm"]
+    ), f"realm {kc['realm']!r} not the per-run namespaced form lasuite-meet-<6hex>"
     assert kc["client_id"] == "lasuite-meet"
     assert isinstance(kc["client_secret"], str) and len(kc["client_secret"]) >= 16
     assert isinstance(kc["password"], str) and len(kc["password"]) >= 16
@@ -77,16 +77,14 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps_creds):
 
     # Password grant → real JWT
     token = sso.oidc_password_grant(creds)
-    assert isinstance(token, str) and token.count(".") == 2, (
-        f"access_token is not a JWT: {token!r}"
-    )
+    assert isinstance(token, str) and token.count(".") == 2, f"access_token is not a JWT: {token!r}"
     payload = json.loads(_b64url_decode(token.split(".")[1]))
     assert payload.get("iss") == expected_iss, f"JWT iss={payload.get('iss')!r} != {expected_iss!r}"
-    assert payload.get("azp") == kc["client_id"], (
-        f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
-    )
+    assert (
+        payload.get("azp") == kc["client_id"]
+    ), f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
     assert payload.get("typ") == "Bearer", f"JWT typ={payload.get('typ')!r} != 'Bearer'"
     exp = payload.get("exp")
-    assert isinstance(exp, int) and exp > time.time(), (
-        f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"
-    )
+    assert (
+        isinstance(exp, int) and exp > time.time()
+    ), f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"

tests/lasuite-meet/install_steps.sh

@@ -26,7 +26,7 @@ if [ -z "${CCCI_DEPS_FILE:-}" ] || [ ! -s "${CCCI_DEPS_FILE}" ]; then
   exit 0
 fi
 KC_DOMAIN=$(jq -r '.keycloak.domain        // empty' "$CCCI_DEPS_FILE")
-KC_REALM=$( jq -r '.keycloak.realm         // empty' "$CCCI_DEPS_FILE")
+KC_REALM=$(jq -r '.keycloak.realm         // empty' "$CCCI_DEPS_FILE")
 KC_CLIENT=$(jq -r '.keycloak.client_id     // empty' "$CCCI_DEPS_FILE")
 KC_SECRET=$(jq -r '.keycloak.client_secret // empty' "$CCCI_DEPS_FILE")
 if [ -z "$KC_DOMAIN" ] || [ -z "$KC_SECRET" ]; then
@@ -40,31 +40,34 @@ echo "  lasuite-meet install_steps: wiring OIDC at install against keycloak ${KC
 # forbids overwriting a secret at the same version). The app is not deployed yet — a swarm secret can
 # be created independently — so the single deploy below picks up v2.
 CUR_VER=$(grep -E '^\s*SECRET_OIDC_RPCS_VERSION=' "$ENV_PATH" | tail -1 | cut -d= -f2 | tr -d '"\r' || echo "v1")
-NEW_NUM=$(( ${CUR_VER#v} + 1 ))
+NEW_NUM=$((${CUR_VER#v} + 1))
 NEW_VER="v${NEW_NUM}"
-INSERT_LOG=$(abra app secret insert "$CCCI_APP_DOMAIN" oidc_rpcs "$NEW_VER" "$KC_SECRET" --no-input -C -o 2>&1) \
-  || INSERT_LOG=$(script -qec "abra app secret insert $CCCI_APP_DOMAIN oidc_rpcs $NEW_VER $KC_SECRET --no-input -C -o" /dev/null 2>&1) \
-  || { echo "  install_steps: abra app secret insert oidc_rpcs@$NEW_VER failed: $INSERT_LOG"; exit 1; }
+INSERT_LOG=$(abra app secret insert "$CCCI_APP_DOMAIN" oidc_rpcs "$NEW_VER" "$KC_SECRET" --no-input -C -o 2>&1) ||
+  INSERT_LOG=$(script -qec "abra app secret insert $CCCI_APP_DOMAIN oidc_rpcs $NEW_VER $KC_SECRET --no-input -C -o" /dev/null 2>&1) ||
+  {
+    echo "  install_steps: abra app secret insert oidc_rpcs@$NEW_VER failed: $INSERT_LOG"
+    exit 1
+  }
 sed -i "s|^\s*SECRET_OIDC_RPCS_VERSION=.*|SECRET_OIDC_RPCS_VERSION=$NEW_VER|" "$ENV_PATH"
 echo "  install_steps: oidc_rpcs secret inserted at $NEW_VER (was $CUR_VER)"
 
 # 2) Write the OIDC env vars (explicit endpoints — deterministic). Meet's .env.sample templates the
 # endpoints off ${AUTH_DOMAIN}; set AUTH_DOMAIN + override each endpoint with the concrete realm URL.
-write_env () {
+write_env() {
   local key="$1" val="$2"
   sed -i "/^\s*#\?\s*${key}=/d" "$ENV_PATH"
-  [ -z "$(tail -c1 "$ENV_PATH" 2>/dev/null)" ] || printf '\n' >> "$ENV_PATH"
-  printf '%s=%s\n' "$key" "$val" >> "$ENV_PATH"
+  [ -z "$(tail -c1 "$ENV_PATH" 2>/dev/null)" ] || printf '\n' >>"$ENV_PATH"
+  printf '%s=%s\n' "$key" "$val" >>"$ENV_PATH"
 }
-write_env AUTH_DOMAIN                      "$KC_DOMAIN"
-write_env OIDC_REALM                       "$KC_REALM"
-write_env OIDC_OP_JWKS_ENDPOINT            "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/certs"
-write_env OIDC_OP_AUTHORIZATION_ENDPOINT   "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/auth"
-write_env OIDC_OP_TOKEN_ENDPOINT           "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/token"
-write_env OIDC_OP_USER_ENDPOINT            "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/userinfo"
-write_env OIDC_OP_LOGOUT_ENDPOINT          "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/logout"
-write_env OIDC_RP_CLIENT_ID                "$KC_CLIENT"
-write_env OIDC_RP_SIGN_ALGO                "RS256"
-write_env OIDC_RP_SCOPES                   "openid email"
+write_env AUTH_DOMAIN "$KC_DOMAIN"
+write_env OIDC_REALM "$KC_REALM"
+write_env OIDC_OP_JWKS_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/certs"
+write_env OIDC_OP_AUTHORIZATION_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/auth"
+write_env OIDC_OP_TOKEN_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/token"
+write_env OIDC_OP_USER_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/userinfo"
+write_env OIDC_OP_LOGOUT_ENDPOINT "https://${KC_DOMAIN}/realms/${KC_REALM}/protocol/openid-connect/logout"
+write_env OIDC_RP_CLIENT_ID "$KC_CLIENT"
+write_env OIDC_RP_SIGN_ALGO "RS256"
+write_env OIDC_RP_SCOPES "openid email"
 
 echo "  lasuite-meet install_steps: OIDC env wired into .env (deploy will pick it up, no reconverge)"

tests/lasuite-meet/test_install.py

@@ -10,7 +10,8 @@ import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
-from harness import browser as harness_browser, generic, lifecycle  # noqa: E402
+from harness import browser as harness_browser  # noqa: E402
+from harness import generic, lifecycle
 
 
 def test_serving_and_frontend(live_app, meta):
@@ -33,9 +34,11 @@ def test_serving_and_frontend(live_app, meta):
             resp = harness_browser.goto_with_retry(
                 page, url, accept_statuses=(200, 301, 302), goto_timeout_ms=60_000
             )
-            assert resp is not None and resp.status in (200, 301, 302), (
-                f"page status {resp and resp.status}"
-            )
+            assert resp is not None and resp.status in (
+                200,
+                301,
+                302,
+            ), f"page status {resp and resp.status}"
             assert "<html" in page.content().lower(), "no HTML served by the frontend"
         finally:
             browser.close()

tests/mailu/functional/test_mail_flow.py

@@ -43,10 +43,7 @@ def test_send_and_receive_mail(live_app):
     deadline = time.time() + 150
     while time.time() < deadline:
         for box in ("INBOX", "Junk"):
-            query = (
-                f"doveadm search -u '{email_addr}' mailbox {box} "
-                f"header subject '{marker}'"
-            )
+            query = f"doveadm search -u '{email_addr}' mailbox {box} " f"header subject '{marker}'"
             out = lifecycle.exec_in_app(live_app, ["sh", "-c", query], service="imap")
             if out.strip():  # a non-empty result = "<mailbox-guid> <uid>" → message stored
                 return

tests/mailu/functional/test_mailbox.py

@@ -24,6 +24,6 @@ def test_create_mailbox_and_read_back(live_app):
 
     cfg = _mailu.config_export(live_app)
     emails = _mailu.user_emails(cfg)
-    assert email in emails, (
-        f"created mailbox {email} not present in mailu config-export users {emails}"
-    )
+    assert (
+        email in emails
+    ), f"created mailbox {email} not present in mailu config-export users {emails}"