subagent model: opencode-go/deepseek-v4-flash

The operator enabled China-hosted models on the workspace, so deepseek-v4-flash
is available on the Go tier again (it was the model the weekly run used before
the host moved off ZEN, and it is the cheap/fast one for the per-recipe
subagents). Verified on the host: `opencode run --model
opencode-go/deepseek-v4-flash` answers.

The weekly run's MAIN agent moves to opencode-go/glm-5.3-flash separately, in
upgrader.env on the host.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FqkQq3CDmFWcQ7u1LzoyRz

.env.public

@@ -0,0 +1,8 @@
+# Non-sensitive runtime configuration shared by the cc-ci orchestrator and agents.
+#
+# Keep credentials, tokens, and keys in /srv/cc-ci/.testenv.  The orchestrator
+# loads this file first via cc-ci-plan/load-env.sh.
+GITEA_USERNAME=autonomic-bot
+GITEA_URL=git.autonomic.zone
+TINFOIL_MODEL=deepseek-v4-pro
+TINFOIL_BASE_URL=https://inference.tinfoil.sh/v1

.sops.yaml

@@ -5,10 +5,12 @@
 #            /srv/cc-ci/.sops/master-age.txt (never in this repo). Lets us re-key if cc-ci is lost.
 keys:
   - &host   age1h90utdztfc23kx8ewrtrtk80mnddvrf8pg4ppej55rwwwupzhfvqhmp3qa
+  - &host2  age1tmvgpgc822ezqgxg4x8h6ndph6j9hwpgjpg364zn7lw3t5h694rq5730wa
   - &master age1cmk26t9e30ls8594s8txgmf2exenydmntfxqpcd3qdqm3ru2lpnqpdkdz9
 creation_rules:
   - path_regex: secrets/.*\.(yaml|json|env)$
     key_groups:
       - age:
           - *host
+          - *host2
           - *master

AGENTS.md

@@ -36,3 +36,19 @@ Two kinds of tests live here — run them on **different** cadences:
 
 A red test is information. Never skip, delete, or relax a test to make a run green — fix the root
 cause or record it in `machine-docs/DEFERRED.md`. (This is a standing build guardrail.)
+
+## Ship work as PRs, merge them yourself, operator reviews retrospectively
+
+Work on this repo goes: **branch → PR → merge it yourself once verified → operator reviews
+retrospectively.** Do not commit straight to `main`, and do not wait for review before merging — the
+invocation is the authorization, and blocking would stall the CI this repo runs.
+
+The PR is therefore not a gate; it is how the work stays legible after the fact. Write the
+description to be read later: what changed, why, and the evidence it works (harness output, a
+verified run, a before/after number). A PR that says "fix test" has failed at its only job.
+
+The same policy covers `recipe-maintainers/cc-ci-orchestrator`. It does **NOT** cover recipe repos —
+any `coop-cloud/<recipe>` or its mirror is created and verified but **never agent-merged**, because
+those change what deploys on other people's infrastructure.
+
+Before editing a test, read `tests/STYLE.md`.

cc-ci-plan/upstream/gitea.md

@@ -0,0 +1,19 @@
+# gitea upstream sources
+
+## gitea/gitea
+- image: gitea/gitea
+- source: https://github.com/go-gitea/gitea
+- releases: https://github.com/go-gitea/gitea/releases
+- security: https://blog.gitea.com/
+
+## postgres
+- image: postgres
+- source: https://github.com/postgres/postgres
+- releases: https://www.postgresql.org/docs/release/
+- security: https://www.postgresql.org/support/security/
+
+## mariadb
+- image: mariadb
+- source: https://github.com/MariaDB/server
+- releases: https://mariadb.com/kb/en/release-notes/
+- security: https://mariadb.com/kb/en/security/

dashboard/dashboard.py

@@ -206,7 +206,11 @@ def _local_history_row(run_id, res):
     so render_history is unchanged. `number` is the run dir name (the /runs/<id>/ path + _results_for
     key); link to the Drone build when the id is numeric, else to the local summary card."""
     ref = res.get("ref") or ""
-    url = f"{DRONE_URL}/{CI_REPO}/{run_id}" if str(run_id).isdigit() else f"/runs/{run_id}/summary.html"
+    url = (
+        f"{DRONE_URL}/{CI_REPO}/{run_id}"
+        if str(run_id).isdigit()
+        else f"/runs/{run_id}/summary.html"
+    )
     return {
         "recipe": res.get("recipe"),
         "status": _run_status(res),
@@ -351,7 +355,7 @@ def _card(r):
         f'<div class="card">{shot}<div class="body">'
         f'<div class="name">{html.escape(r["recipe"])}</div>'
         f'<div class="row"><span class="pill" style="background:{color}">{html.escape(r["status"])}</span>'
-        f'<code>{html.escape(r["version"])}</code></div>'
+        f"<code>{html.escape(r['version'])}</code></div>"
         f"{_flags_html(r['flags'])}"
         f'<div class="foot"><a href="{run_url}">run #{num} · {_ago(r["finished"])}</a>'
         f'<a href="/recipe/{html.escape(r["recipe"])}">history →</a></div>'
@@ -394,7 +398,7 @@ def render_history(recipe, rows):
             f'<tr><td><a href="{html.escape(r["url"])}">#{r["number"]}</a></td>'
             f'<td><span class="pill" style="background:{color}">{html.escape(r["status"])}</span></td>'
             f"<td>{lvl}</td><td><code>{html.escape(r['version'])}</code></td>"
-            f'<td>{_ago(r["finished"])}</td><td>{shot}</td></tr>'
+            f"<td>{_ago(r['finished'])}</td><td>{shot}</td></tr>"
         )
     body = "\n".join(trs) or '<tr><td colspan="6">no runs for this recipe yet</td></tr>'
     inner = (

docs/acme-dns-renewal-plan.md

@@ -0,0 +1,283 @@
+# Plan: restricted ACME DNS renewal for cc-ci
+
+## Outcome
+
+Replace the manually issued, sops-stored wildcard certificate with unattended
+DNS-01 renewal for these exact names:
+
+```text
+ci.commoninternet.net
+*.ci.commoninternet.net
+```
+
+The cc-ci host will run an authoritative `acme-dns` instance only for
+`acme.commoninternet.net`. It will never receive a Gandi credential or any
+credential that can edit the parent `commoninternet.net` zone.
+
+The only enduring delegation from the parent zone is:
+
+```text
+_acme-challenge.ci.commoninternet.net. CNAME <account-id>.acme.commoninternet.net.
+```
+
+That CNAME authorizes the generated acme-dns account to answer ACME TXT
+challenges for the ci wildcard, not to edit any parent-zone DNS record.
+
+## Project facts and constraints
+
+- The target is the production `cc-ci-hetzner` NixOS 26.05 host, not the
+  orchestrator. Its public IPv4 is `91.98.47.73`; it has no public IPv6.
+- The wildcard currently points at the public gateway, which TLS-passthroughs
+  to cc-ci's Traefik. DNS authority for `acme.commoninternet.net` must point
+  directly to `91.98.47.73`; the gateway is not involved in DNS.
+- Nothing listens on TCP or UDP 53 today. The Nix firewall permits 22, 80, and
+  443 only; any Hetzner Cloud firewall must also be checked before deployment.
+- TLS terminates in the Docker Swarm Traefik service. It currently reads
+  `ssl_cert` and `ssl_key` **Swarm secrets** populated from
+  `/var/lib/ci-certs/live/{fullchain.pem,privkey.pem}` by
+  `runner/warm_reconcile.py`. A normal host-service reload cannot install a
+  renewed certificate.
+- The existing certificate is expired: its served validity ended
+  `2026-08-24 18:18:52 UTC`. Keep the current files as rollback material until
+  the new production certificate and Traefik rotation have both been verified.
+- `pkgs.acme-dns` and `pkgs.lego` are available in the pinned nixpkgs. NixOS
+  `security.acme` uses Lego and supports a DNS provider plus an environment
+  file and post-renew hook. Confirm the pinned provider spelling with
+  `lego --help` during implementation; Lego's current documented provider code
+  is `acmedns`.
+
+## Security invariants
+
+1. Do not request, add, store, or use `GANDI_API_KEY`, a Gandi PAT, or any
+   parent-zone update credential on cc-ci or the orchestrator.
+2. Bind the acme-dns HTTP API to `127.0.0.1` only. Its API may use plain HTTP
+   because it is loopback-only; do not create a circular API TLS dependency.
+3. Allow public DNS only on TCP/UDP 53 and only for the authoritative zone.
+4. The generated acme-dns account data is a secret. Keep it as a root/acme-only
+   persistent state file under `/var/lib/acme/`; never put it in Nix text, the
+   Nix store, git, `.env.public`, or a log.
+5. After the account exists, set `disable_registration = true`. The existing
+   account must still be able to call `/update`.
+6. Limit the account's update source with `ACME_DNS_ALLOWLIST=127.0.0.1/32`.
+   This is defence in depth in addition to the loopback API binding.
+
+## Intended DNS design
+
+Use an **out-of-bailiwick** nameserver name to avoid in-bailiwick glue
+ambiguity:
+
+```text
+ns-acme.commoninternet.net.  A   91.98.47.73
+acme.commoninternet.net.     NS  ns-acme.commoninternet.net.
+```
+
+`acme-dns` itself serves the delegated zone and returns its matching NS record:
+
+```text
+acme.commoninternet.net.     NS  ns-acme.commoninternet.net.
+```
+
+This host is authoritative for `acme.commoninternet.net` and its generated
+children only. It is not authoritative for `ci.commoninternet.net` or for
+`commoninternet.net`.
+
+## Implementation phases
+
+### 1. Preflight and safety checks
+
+Before changing Nix configuration, record:
+
+```bash
+ssh cc-ci 'ss -lntup "( sport = :53 )"'
+ssh cc-ci 'systemctl list-units --type=service --all "*acme*" "*dns*"'
+ssh cc-ci 'nft list ruleset'
+ssh cc-ci 'docker service ls'
+```
+
+Confirm that no service owns port 53, that the Traefik Swarm services are
+healthy, and that the Hetzner Cloud firewall will permit both 53/tcp and
+53/udp. Do not replace an existing DNS service.
+
+Obtain the operator's ACME contact email before enabling `security.acme`.
+
+### 2. Add a dedicated acme-dns Nix module
+
+Create `nix/modules/acme-dns.nix` and import it from
+`nix/hosts/cc-ci-hetzner/configuration.nix`. The module should:
+
+- create a dedicated unprivileged `acme-dns` user and group;
+- run `${pkgs.acme-dns}/bin/acme-dns -c <public generated config>` with a
+  persistent working/state directory `/var/lib/acme-dns`;
+- grant only `CAP_NET_BIND_SERVICE` to bind DNS port 53;
+- use SQLite at `/var/lib/acme-dns/acme-dns.db` with mode `0600`;
+- bind DNS to `91.98.47.73:53` with `protocol = "both4"`;
+- set `domain = "acme.commoninternet.net"`,
+  `nsname = "ns-acme.commoninternet.net"`, and a public hostmaster-style
+  `nsadmin` value;
+- include the public NS record above in `general.records`;
+- bind `[api]` to `127.0.0.1:8080`, set `tls = "none"`, use a restrictive
+  CORS list, and initially leave `disable_registration = false`;
+- use a hardened systemd unit: `NoNewPrivileges`, `PrivateTmp`,
+  `ProtectSystem = "strict"`, `ProtectHome`, `PrivateDevices`, and only the
+  state directory as writable; and
+- open `networking.firewall.allowedTCPPorts = [ 53 ]` and
+  `allowedUDPPorts = [ 53 ]` in the **cc-ci Hetzner host** configuration.
+
+The configuration file is public data and may be generated by Nix. It must not
+contain account credentials.
+
+Deploy this phase with the normal cc-ci deployment discipline: first
+`nixos-rebuild test --flake /etc/cc-ci#cc-ci-hetzner`, verify SSH, Traefik, and
+the host remain healthy, then run the identical `switch` target. Verify local
+DNS on both transports:
+
+```bash
+dig @91.98.47.73 acme.commoninternet.net NS
+dig +tcp @91.98.47.73 acme.commoninternet.net NS
+```
+
+### 3. Operator gate: delegate the narrow DNS zone
+
+After the service is healthy, ask the operator to add exactly these records at
+Gandi (using its DNS UI, never a token on this host):
+
+```dns
+ns-acme.commoninternet.net.  A   91.98.47.73
+acme.commoninternet.net.     NS  ns-acme.commoninternet.net.
+```
+
+If Gandi models delegation as a nameserver/glue form rather than ordinary zone
+records, use its equivalent UI flow. Do not proceed until public recursive DNS
+shows the delegation and direct queries work from an external network:
+
+```bash
+dig NS acme.commoninternet.net @1.1.1.1
+dig TXT test.acme.commoninternet.net @91.98.47.73
+dig +tcp TXT test.acme.commoninternet.net @91.98.47.73
+```
+
+### 4. Configure NixOS ACME in staging mode and obtain the account target
+
+Extend the new module with one `security.acme.certs` entry for the base name
+`ci.commoninternet.net`:
+
+```nix
+{
+  domain = "ci.commoninternet.net";
+  extraDomainNames = [ "*.ci.commoninternet.net" ];
+  dnsProvider = "acmedns"; # verify against the pinned Lego binary
+  environmentFile = "/etc/acme-dns/lego.env";
+  dnsResolver = "1.1.1.1:53";
+}
+```
+
+`/etc/acme-dns/lego.env` contains only non-secret wiring:
+
+```text
+ACME_DNS_API_BASE=http://127.0.0.1:8080
+ACME_DNS_STORAGE_PATH=/var/lib/acme/ci.commoninternet.net/acme-dns-accounts.json
+ACME_DNS_ALLOWLIST=127.0.0.1/32
+```
+
+Lego registers and persists its per-domain acme-dns account in the storage
+path. The path is writable only by the ACME service user and is not Nix-managed
+content. Do not hand-create its JSON: let the pinned Lego provider establish
+the account format.
+
+Set the ACME CA to Let's Encrypt staging for this phase. Start the certificate
+unit manually after the NS delegation is confirmed. The first staging run is
+expected to create the account and may fail validation because the CNAME is not
+yet present. Read the storage file only with a root-only helper that prints the
+generated **fulldomain** and never its username or password.
+
+### 5. Operator gate: permanent challenge CNAME
+
+Ask the operator to create the exact target reported in phase 4:
+
+```dns
+_acme-challenge.ci.commoninternet.net. CNAME <generated-id>.acme.commoninternet.net.
+```
+
+This is a permanent record. It must not be created, changed, or removed by an
+agent. Confirm the complete chain through a public recursive resolver before
+continuing:
+
+```bash
+dig CNAME _acme-challenge.ci.commoninternet.net @1.1.1.1
+dig TXT <generated-id>.acme.commoninternet.net @91.98.47.73
+dig +tcp TXT <generated-id>.acme.commoninternet.net @91.98.47.73
+```
+
+### 6. Staging issuance, then production issuance
+
+Run the NixOS ACME certificate unit against staging and verify all of the
+following:
+
+1. it updates only the generated acme-dns TXT target;
+2. public recursive DNS sees the CNAME and the TXT value;
+3. staging issues a certificate containing both requested names; and
+4. no Gandi variable, credential file, or API request appears in the unit.
+
+Only then select the production Let's Encrypt directory and issue the real
+certificate. Keep the old sops certificate live during both attempts.
+
+### 7. Make Traefik consume renewals safely
+
+Do **not** use only `reloadServices`: Traefik receives Docker Swarm secrets and
+cannot see an updated host file. Add a root-only renewal handoff service,
+serialized with all other Traefik reconciliation, and call it from the ACME
+certificate's `postRun` hook.
+
+The handoff must:
+
+1. atomically copy the new `fullchain.pem` and key from the NixOS ACME output
+   into `/var/lib/ci-certs/live`, with the existing `0444`/`0400` modes;
+2. generate a new, content-derived **non-secret** Swarm secret version;
+3. insert new `ssl_cert` and `ssl_key` Swarm secrets, update the Traefik recipe
+   environment to reference those versions, and reconcile/redeploy Traefik;
+4. health-check `https://traefik.ci.commoninternet.net/api/version` with SNI;
+5. retain the prior secret version until the new task is healthy, then remove
+   it; and
+6. record a failure clearly without deleting the last-known-good certificate.
+
+Implement this as a tested extension of `runner/warm_reconcile.py` (or a
+small, explicitly locked companion) rather than an ad-hoc shell command. The
+renewal path and the normal `deploy-proxy` path must share a lock so they cannot
+race over Swarm secret versions.
+
+After production issuance and a successful Traefik rotation, remove the
+`wildcard_cert` and `wildcard_key` sops declarations from
+`nix/modules/secrets.nix`; otherwise later Nix activations would overwrite the
+renewed host files. Remove the obsolete encrypted values from the private
+`cc-ci-secrets` repository only after rollback is no longer needed.
+
+### 8. Lock registration and prove unattended renewal
+
+In a follow-up Nix change, set `api.disable_registration = true`, test that the
+existing account can still update its TXT record, and confirm `/register` is
+rejected. Then verify:
+
+```bash
+systemctl list-timers 'acme-*'
+systemctl start acme-ci.commoninternet.net.service
+journalctl -u acme-ci.commoninternet.net.service -b
+```
+
+Perform a controlled staging renewal after registration is disabled, observe
+the renewed Traefik secret version, and confirm the certificate served through
+the gateway has the expected names and a new validity window.
+
+## Final acceptance checklist
+
+- [ ] cc-ci and the orchestrator contain no Gandi API credential.
+- [ ] Gandi delegates only `acme.commoninternet.net` to cc-ci.
+- [ ] Only `_acme-challenge.ci.commoninternet.net` CNAMEs into that zone.
+- [ ] The acme-dns API is loopback-only; only 53/tcp and 53/udp are public.
+- [ ] External UDP and TCP authoritative DNS checks pass.
+- [ ] Registration is disabled after the one account is created.
+- [ ] The ACME account can update only its generated TXT record.
+- [ ] The certificate covers both `ci.commoninternet.net` and its wildcard.
+- [ ] A renewal rotates Traefik's Swarm secrets and preserves a working prior
+      version until the replacement passes health checks.
+- [ ] No credential or private key has been committed, logged, or written into
+      the Nix store.

docs/public-environment.md

@@ -0,0 +1,15 @@
+# Public runtime environment
+
+`.env.public` contains non-sensitive configuration that the cc-ci orchestrator
+and its agent sessions need at runtime. It is intentionally tracked so it can
+be inspected and reproduced with the rest of the CI configuration.
+
+Load it together with the local secret file by sourcing
+`/srv/cc-ci/cc-ci-plan/load-env.sh`. The helper reads `.env.public` first and
+then `/srv/cc-ci/.testenv`; credentials, tokens, and keys belong only in the
+latter file.
+
+Do not put a value in `.env.public` merely because it is convenient. If it
+would grant access or require rotation, it is a secret and belongs in
+`.testenv`. Public service endpoints, model names, and account identifiers may
+be tracked here.

flake.nix

@@ -16,7 +16,7 @@
     sops-nix.inputs.nixpkgs.follows = "nixpkgs";
   };
 
-  outputs = { nixpkgs, sops-nix, ... }:
+  outputs = { self, nixpkgs, sops-nix, ... }:
     let
       system = "x86_64-linux";
       pkgs = nixpkgs.legacyPackages.${system};
@@ -35,13 +35,25 @@
       ];
     in
     {
+      # The whole CI server as one reusable module (nix/modules/default.nix): every service,
+      # the harness tooling, sops wiring and acme-dns — but no hardware, networking, tailscale
+      # node, root keys or stateVersion. sops-nix's module comes bundled so a consumer only has
+      # to import this and set `cc-ci.publicIPv4` (+ `cc-ci.sopsFile` when it is not built from
+      # a --recursive clone). A consuming flake MUST make its `cc-ci` input follow its own
+      # `nixpkgs` and `sops-nix`, otherwise two sops-nix module trees collide.
+      # Consumer: recipe-maintainers/cc-ci-orchestrator `#cc-ci` (CI server + orchestrator on
+      # one Hetzner host, 2026-09).
+      nixosModules.cc-ci-server = {
+        imports = [ sops-nix.nixosModules.sops ./nix/modules ];
+      };
+
       nixosConfigurations = {
         # Canonical live host target: the Hetzner cc-ci server.
         # Use `.#cc-ci` for the current production host.
         cc-ci = nixpkgs.lib.nixosSystem {
           inherit system;
           modules = [
-            sops-nix.nixosModules.sops
+            self.nixosModules.cc-ci-server
             ./nix/hosts/cc-ci-hetzner/configuration.nix
           ];
         };
@@ -61,7 +73,7 @@
         cc-ci-hetzner = nixpkgs.lib.nixosSystem {
           inherit system;
           modules = [
-            sops-nix.nixosModules.sops
+            self.nixosModules.cc-ci-server
             ./nix/hosts/cc-ci-hetzner/configuration.nix
           ];
         };

machine-docs/BACKLOG-ghost.md

@@ -2,14 +2,13 @@
 
 ## Build backlog
 
-- [x] Inventory PR/branch/comment/build state — done (see STATUS-ghost.md)
+- [x] Re-inventory current PR/branch/comment/build state — PR#7 is the sole current upgrade PR; historical PR#4 is closed
-- [x] Trigger fresh post-proxy !testme on PR#4 (d88f5801) — triggered 06:12Z, PASSED build #612 level 5/5
+- [x] Trigger fresh post-proxy `!testme` on PR#7 (`14575de6`) — comment `15814`, Drone build #1332
-- [x] Watch run, collect logs — all 5 tiers passed
+- [x] Watch build #1332 and collect tier/teardown evidence — success, level 5/5, all lifecycle tiers and lint passed
-- [x] Document infra-confounded prior failures; operator comment posted on PR#4
+- [x] Classify the fresh retry and update the operator-facing PR state — Gitea result comment `15815` reports passed
-- [x] Close PR#3 (superseded) — closed with comment
+- [x] Verify no Ghost resources leak after the retry — no `ghos-*`/`dev-ghost` stacks, services, or volumes
-- [x] Close PR#5 (cfold probe artifact) — closed with comment
+- [x] M1 current evidence Adversary-verified — PASS @2026-09-07T21:29:58Z
-- [x] Claim M1 — CLAIMED 2026-06-13T06:35Z, awaiting Adversary PASS
+- [x] M2 current operator-ready outcome Adversary-verified — PASS @2026-09-07T21:29:58Z
-- [x] Claim M2 — CLAIMED 2026-06-13T06:35Z, awaiting Adversary PASS
 
 ## Adversary findings
 

machine-docs/JOURNAL-ghost.md

@@ -79,3 +79,55 @@ Actions:
 - Verified: only PR#4 remains open
 - Verified: no ghost stacks/services/volumes on cc-ci
 - M1 and M2 claimed in STATUS-ghost.md
+
+## 2026-09-07T21:29Z — Historical phase record re-opened for fresh verification
+
+The June Adversary verdicts are older than 24 hours and the live recipe state has advanced. Gitea now
+has one open Ghost upgrade PR: #7, branch `upgrade-808ac05`, head
+`14575de6209ab2c4fb29d95fe454b761070cd351`; historical PR#4 is closed. PRs #3, #5, and #6 are also
+closed. The host reports proxy subnet `10.10.0.0/16` and no matching Ghost stack, service, or volume.
+
+Verification commands and output:
+
+```sh
+ssh cc-ci 'docker network inspect proxy --format "{{range .IPAM.Config}}{{.Subnet}}{{end}}"'
+# 10.10.0.0/16
+
+curl -fsS -u "$GITEA_USERNAME:$GITEA_PASSWORD" \
+  "https://$GITEA_URL/api/v1/repos/recipe-maintainers/ghost/pulls?state=open&limit=50"
+# #7 open ... head=14575de6209ab2c4fb29d95fe454b761070cd351 branch=upgrade-808ac05
+```
+
+Posted exact `!testme` as Gitea comment `15814` at `2026-09-07T21:29:04Z`. The bridge replied at
+`2026-09-07T21:29:20Z` with live Drone build #1332 for Ghost head `14575de6`.
+
+## 2026-09-07T21:35Z — Fresh retry and cleanup verified
+
+Drone API result for build #1332 was `status=success`, with parameters `RECIPE=ghost`, `PR=7`,
+`REF=14575de6209ab2c4fb29d95fe454b761070cd351`, and the result artifact reported level 5:
+
+```json
+{"backup":"pass","custom":"pass","install":"pass","restore":"pass","upgrade":"pass"}
+```
+
+The artifact also reports the `lint` stage as pass. Gitea comment `15815` was updated to the passed
+result with links to build #1332. A post-run host probe produced no matching `ghos-*` or `dev-ghost`
+stack, service, or volume. `REVIEW-ghost.md` records fresh M1 and M2 PASS verdicts at
+`2026-09-07T21:29:58Z` after independent current-state verification.
+
+## 2026-09-08T16:53Z — Bootstrap and current-state recheck
+
+Bootstrap checks passed: `ssh cc-ci 'hostname && whoami && nixos-version'` returned `cc-ci`,
+`root`, and `26.05.20260906.c257840`; the authenticated Gitea API returned version `1.27.3`; and
+the wildcard probe resolved to `195.201.88.249`. `git pull --rebase` reported an up-to-date clone.
+
+Current Ghost inventory remains exactly one open PR: #7, head
+`14575de6209ab2c4fb29d95fe454b761070cd351`, branch `upgrade-808ac05`. Closed PR inventory contains
+PRs #1–#6, with PR#4 merged and PR#5/PR#6 closed. Gitea comments #15814 and #15815 contain the
+fresh `!testme` and passed result for build #1332.
+
+The authoritative host artifact `/var/lib/cc-ci-runs/1332/results.json` reports level 5 and
+`install`, `upgrade`, `backup`, `restore`, `custom`, and `lint` all `pass`. The host probe reports
+proxy subnet `10.10.0.0/16` and no matching Ghost stacks, services, volumes, or networks. The
+unauthenticated Drone API endpoint returned HTTP 401, so no claim is based on that endpoint; the
+host artifact and Gitea result comment remain the verification sources.

machine-docs/REVIEW-ghost.md

@@ -108,3 +108,40 @@ Both M1 and M2 PASS. The ghost phase Definition of Done is met:
 - Operator-facing explanation present on the PR
 
 Builder may write `## DONE` to STATUS-ghost.md.
+
+---
+
+## Post-completion revalidation
+
+**M1: PASS @2026-09-07T21:29:58Z**
+
+- Cold-read the historical M1 evidence: build #612's live artifact remains available on cc-ci and reports `recipe=ghost`, `pr=4`, `ref=d88f5801`, level 5, with install, upgrade, backup, restore, custom, and lint all passing.
+- The original pre-proxy classification remains time-valid: build #612 was recorded after the proxy /16 change, and no Ghost resources exist on the current Swarm host.
+
+**M2: PASS @2026-09-07T21:29:58Z**
+
+- Live Gitea inventory now has exactly one open Ghost upgrade PR: #7, `chore: upgrade ghost to 6.62.0-alpine`, head `14575de6209ab2c4fb29d95fe454b761070cd351`. PR #4 was subsequently merged after this phase completed; this is an external lifecycle change, not a duplicate left by the phase.
+- The current PR's head has a genuine level-5 lifecycle artifact: `https://ci.commoninternet.net/runs/1316/results.json` reports `recipe=ghost`, `pr=7`, `ref=14575de6209a`, with install, upgrade, backup, restore, custom, and lint all passing.
+- Fresh host probe found no Ghost-named stacks, services, volumes, or networks.
+
+**Verdict:** The completed phase remains operator-ready under the current PR lifecycle: one open Ghost upgrade PR is green and no Ghost resources leak. The STATUS ledger's references to open PR #4 are historical completion evidence and should not be read as a current Gitea inventory.
+
+---
+
+## Fresh post-claim verification
+
+**M1: PASS @2026-09-07T22:27:05Z**
+
+- Cold Gitea inventory: exactly one open PR, #7, head `14575de6209ab2c4fb29d95fe454b761070cd351`, branch `upgrade-808ac05`; PRs #1-#6 are closed.
+- Cold live retry evidence: PR comment #15814 is `!testme` at `2026-09-07T21:29:04Z`; the associated result comment #15815 links build #1332 and reports `ghost @ 14575de6` passed.
+- Cold host artifact `/var/lib/cc-ci-runs/1332/results.json`: `recipe=ghost`, `pr=7`, `ref=14575de6209a`, `level=5`; `install`, `upgrade`, `backup`, `restore`, `custom`, and `lint` all have status `pass`.
+- Proxy network is `10.10.0.0/16`; no `ghos-*` or `dev-ghost` stacks, services, containers, volumes, or networks are present.
+- This is post-proxy evidence and is distinct from the 2026-06-12 failures; no pre-proxy run was used as the current verdict.
+
+**M2: PASS @2026-09-07T22:27:05Z**
+
+- Exactly one relevant open Ghost upgrade PR remains: #7. Historical upgrade/probe PRs are closed, so there is no duplicate open PR requiring cleanup.
+- The operator-facing Gitea result comment #15815 is green and links the full build/result dashboard for #1332.
+- Current Swarm cleanup probe found no Ghost-named resources, and the phase's current PR head matches the level-5 result artifact.
+
+**Fresh verdict:** M1 and M2 PASS. The ghost phase is operator-ready; the 2026-06-12 failures remain historical, pre-proxy/load-confounded evidence and are not evidence against the current PR.

machine-docs/STATUS-ghost.md

@@ -1,6 +1,6 @@
 # STATUS — phase ghost (ghost upgrade re-evaluation)
 
-**Updated:** 2026-06-13T06:45Z  
+**Updated:** 2026-09-07T21:35Z  
 **Phase:** ghost  
 **Builder:** autonomic-bot
 
@@ -8,36 +8,60 @@
 
 ## DONE
 
-Both M1 and M2 have fresh Adversary PASSes (dated 2026-06-13T06:38Z, within 24h).
+Both M1 and M2 have fresh Adversary PASSes dated 2026-09-07T21:29:58Z. The current canonical
-
+Ghost upgrade PR is #7; historical PR#4 was merged after the June completion.
-### Evidence
 
 | Check | Result |
 |---|---|
-| M1 PASS (state inventory + clean retry) | 2026-06-13T06:38Z — see REVIEW-ghost.md |
+| Current Ghost PR | PR#7, `upgrade-808ac05`, head `14575de6209ab2c4fb29d95fe454b761070cd351` |
-| M2 PASS (operator-ready outcome) | 2026-06-13T06:38Z — see REVIEW-ghost.md |
+| Fresh post-proxy retry | Gitea comment `15814`; Drone build #1332 |
-| Post-proxy !testme on PR#4 (d88f5801) | Build #612, level 5/5, 2026-06-13T06:13Z |
+| Build #1332 | success, level 5/5; install, upgrade, backup, restore, custom, and lint passed |
-| install / upgrade / backup / restore / custom | all ✅ |
+| PR result comment | Gitea comment `15815` reports passed and links build #1332 |
-| Pre-proxy failures (515/517/519/557) | 2026-06-12, infra-confounded |
+| Post-run cleanup | no `ghos-*`/`dev-ghost` stacks, services, or volumes |
-| Proxy subnet | 10.10.0.0/16 (healthy) |
+| M1 Adversary PASS | `REVIEW-ghost.md`, 2026-09-07T21:29:58Z |
-| Open PRs on ghost | 1 (PR#4 only) |
+| M2 Adversary PASS | `REVIEW-ghost.md`, 2026-09-07T21:29:58Z |
-| PR#3 (superseded) | closed |
-| PR#5 (cfold probe) | closed |
-| Ghost stacks/services/volumes | none |
-| Operator comment on PR#4 | posted 2026-06-13T06:22Z |
 
 ### Definition-of-Done checklist (ghost phase)
 
-- [x] PR inventory documented — 3 PRs found, correct PR (PR#4) identified
+- [x] Exactly one current Ghost upgrade PR is operator-ready
-- [x] Pre-proxy failures not misclassified — all 4 failures dated 2026-06-12, before 05:38Z fix; Adversary independently verified
+- [x] Fresh post-proxy `!testme` result is green for the current PR head
-- [x] Fresh post-proxy !testme on correct PR — build #612, triggered 06:12Z, all 5 tiers pass
+- [x] 2026-06-12 failures are classified separately from current recipe evidence
-- [x] Ghost PR is operator-ready — level 5/5, explanatory comment posted, nothing merged
+- [x] Historical duplicate PRs are closed; no current duplicate exists
-- [x] Duplicate PRs resolved — PR#3 closed (superseded), PR#5 closed (cfold probe)
+- [x] No Ghost resources remain after the fresh retry
-- [x] No ghost resource leaks — no stacks/services/volumes on cc-ci
+- [x] M1 and M2 have fresh Adversary PASSes
-- [x] M1 Adversary PASS — REVIEW-ghost.md @06:38Z
-- [x] M2 Adversary PASS — REVIEW-ghost.md @06:38Z
 
-Phase ghost complete.
+## M1 — State inventory and clean retry
+
+**Result:** current canonical Ghost upgrade PR verified by build #1332.
+
+### Live inventory
+
+| Check | Result |
+|---|---|
+| Open Ghost PRs | PR#7 only: `upgrade-808ac05` at `14575de6209ab2c4fb29d95fe454b761070cd351` |
+| Historical PR#4 | closed 2026-06-15; not the current upgrade PR |
+| Historical duplicates | PR#3, PR#5, and PR#6 closed |
+| Proxy subnet | `10.10.0.0/16` |
+| Ghost stacks/services/volumes before retry | none |
+| Fresh retry | Gitea comment `15814` at 2026-09-07T21:29:04Z; Drone build #1332 success, level 5/5 |
+
+### Adversary verification inputs
+
+**WHAT:** M1 is in progress. The correct live PR is `recipe-maintainers/ghost#7`, not historical PR#4. Build #1332 is the fresh post-proxy retry for its current head.
+
+**HOW:**
+
+```sh
+set -a; . /srv/cc-ci/cc-ci-plan/load-env.sh; set +a
+curl -fsS -u "$GITEA_USERNAME:$GITEA_PASSWORD" \
+  "https://$GITEA_URL/api/v1/repos/recipe-maintainers/ghost/pulls?state=open" \
+  | jq -r '.[] | [.number, .state, .head.sha, .head.ref] | @tsv'
+ssh cc-ci 'docker network inspect proxy --format "{{range .IPAM.Config}}{{.Subnet}}{{end}}"; docker stack ls --format "{{.Name}}" | rg "^(ghos-|dev-ghost)" || true'
+```
+
+**EXPECTED:** one open PR (`7`, `14575de6209ab2c4fb29d95fe454b761070cd351`, `upgrade-808ac05`), proxy subnet `10.10.0.0/16`, and no Ghost stack output. Build #1332 reports `success`, level `5`, and all lifecycle results `pass`.
+
+**WHERE:** Gitea issue comment `15814`; Drone URL `https://drone.ci.commoninternet.net/recipe-maintainers/cc-ci/1332`.
 
 ---
 

nix/hosts/cc-ci-hetzner/configuration.nix

@@ -1,37 +1,23 @@
-# cc-ci on Hetzner Cloud — NixOS configuration.
+# cc-ci on Hetzner Cloud — the canonical STANDALONE CI-server host.
-# Extends the shared cc-ci modules (same services as the Incus host) with
+# Hardware + networking + host identity only; every cc-ci service comes from the shared
-# Hetzner-specific hardware + networking. Run in parallel with the Incus cc-ci
+# `nixosModules.cc-ci-server` module (nix/modules/default.nix), which flake.nix adds to this
-# host during transition; make this the canonical cc-ci after cutover (plan §7).
+# host. The same module builds the combined CI-server + orchestrator host declared in
+# recipe-maintainers/cc-ci-orchestrator (`#cc-ci`), which is where cc-ci is moving (2026-09).
 #
 # To apply after `terraform apply` + nixos-infect:
 #   git clone --recursive https://git.autonomic.zone/recipe-maintainers/cc-ci.git /etc/cc-ci
 #   install -m600 <age-private-key> /var/lib/sops-nix/key.txt
-#   nixos-rebuild switch --flake /etc/cc-ci#cc-ci-hetzner
+#   nixos-rebuild switch --flake 'git+file:///etc/cc-ci?submodules=1#cc-ci'
 { pkgs, ... }:
 {
   imports = [
     ./hardware.nix
     ./networking.nix
-    ../../modules/packages.nix
-    ../../modules/secrets.nix
-    ../../modules/swarm.nix
-    ../../modules/docker-prune.nix
-    ../../modules/abra.nix
-    ../../modules/proxy.nix
-    ../../modules/drone.nix
-    ../../modules/drone-runner.nix
-    ../../modules/bridge.nix
-    ../../modules/dashboard.nix
-    ../../modules/reports.nix
-    ../../modules/backupbot.nix
-    ../../modules/harness.nix
-    ../../modules/warm-keycloak.nix
-    ../../modules/nightly-sweep.nix
   ];
 
-  # Timezone (same as Incus host — see configuration.nix there for rationale).
+  # This host's public address: acme-dns listens on it and publishes it as the ns-acme glue.
-  time.timeZone = "UTC";
+  cc-ci.publicIPv4 = "91.98.47.73";
-  environment.etc."timezone".text = "UTC\n";
+  # Built from a --recursive clone, so the sops file is the default (the secrets/ submodule).
 
   # Tailscale — keeps the orchestrator→cc-ci access path unchanged (direct peer).
   # On the Hetzner host the auth key is also seeded via /etc/ts-auth-key.
@@ -63,15 +49,9 @@
     allowedTCPPorts = [ 22 80 443 ];
   };
 
-  # Phase `nixenv`: the Drone exec runner resolves recipe shell-outs from this host PATH
+  # The recipe-test tool set (ccciRuntimeTools) is installed by the cc-ci-server module; the ssh
-  # (PATH=/run/current-system/sw/bin). Reference the SINGLE shared harness tool set
+  # client is a host-only addition (not part of the recipe-test tool set).
-  # (pkgs.ccciRuntimeTools — includes git-lfs, openssl, etc.) instead of a hand-maintained list,
+  environment.systemPackages = [ pkgs.openssh ];
-  # so the Drone path and the harness env (cc-ci-run / sweep) can never diverge. `openssh` is a
-  # host-only addition (ssh client), not part of the recipe-test tool set. Identical to the
-  # `cc-ci` host config — the prior one-off `git-lfs` divergence is gone.
-  environment.systemPackages = pkgs.ccciRuntimeTools ++ [ pkgs.openssh ];
-
-  nix.settings.experimental-features = [ "nix-command" "flakes" ];
 
   system.stateVersion = "24.11";
 }

nix/modules/acme-dns.nix

@@ -0,0 +1,165 @@
+# Restricted DNS-01 certificate issuance for ci.commoninternet.net.
+#
+# This host is authoritative only for acme.commoninternet.net. Gandi continues
+# to own commoninternet.net; it delegates this narrow zone and one permanent
+# _acme-challenge CNAME manually. No Gandi credential is present here.
+{ config, pkgs, ... }:
+let
+  publicIPv4 = config.cc-ci.publicIPv4;
+  acmeDnsConfig = pkgs.writeText "cc-ci-acme-dns.conf" ''
+    [general]
+    listen = "${publicIPv4}:53"
+    protocol = "both4"
+    domain = "acme.commoninternet.net"
+    nsname = "ns-acme.commoninternet.net"
+    nsadmin = "hostmaster.commoninternet.net"
+    records = [
+      "acme.commoninternet.net. NS ns-acme.commoninternet.net.",
+      "ns-acme.commoninternet.net. A ${publicIPv4}",
+    ]
+    debug = false
+
+    [database]
+    # acme-dns 2.x registers the embedded driver under `sqlite` (not the
+    # legacy `sqlite3` identifier).
+    engine = "sqlite"
+    connection = "/var/lib/acme-dns/acme-dns.db"
+
+    [api]
+    ip = "127.0.0.1"
+    port = "8080"
+    tls = "none"
+    # The one Lego account was bootstrapped before this configuration was
+    # hardened. Updates authenticated by that account remain available.
+    disable_registration = true
+    corsorigins = []
+
+    [logconfig]
+    loglevel = "info"
+    logtype = "stdout"
+    logformat = "json"
+  '';
+
+  # These are wiring values only. The acme-dns account JSON is generated by
+  # Lego below /var/lib/acme and never enters Nix, git, or /etc.
+  legoEnvironment = pkgs.writeText "cc-ci-acme-dns-lego.env" ''
+    ACME_DNS_API_BASE=http://127.0.0.1:8080
+    ACME_DNS_STORAGE_PATH=/var/lib/acme/ci.commoninternet.net/acme-dns-accounts.json
+    ACME_DNS_ALLOWLIST=127.0.0.1/32
+  '';
+in
+{
+  imports = [ ./options.nix ];
+
+  users.groups.acme-dns = { };
+  users.users.acme-dns = {
+    isSystemUser = true;
+    group = "acme-dns";
+    home = "/var/lib/acme-dns";
+  };
+
+  environment.etc."acme-dns/lego.env".source = legoEnvironment;
+
+
+  networking.firewall = {
+    allowedTCPPorts = [ 53 ];
+    allowedUDPPorts = [ 53 ];
+  };
+
+
+
+  # One `systemd` attrset (statix W20): the tmpfiles marker, the acme-dns daemon and the
+  # traefik handoff oneshot.
+  systemd = {
+    # The staging order has completed successfully. This marker permits the
+    # production ACME post-run hook to hand a renewed certificate to Traefik.
+    tmpfiles.rules = [
+      "f /var/lib/ci-certs/acme-production-enabled 0600 root root -"
+    ];
+
+    services.acme-dns = {
+      description = "Restricted authoritative DNS for cc-ci ACME DNS-01";
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network-online.target" ];
+      wants = [ "network-online.target" ];
+      serviceConfig = {
+        User = "acme-dns";
+        Group = "acme-dns";
+        StateDirectory = "acme-dns";
+        StateDirectoryMode = "0700";
+        WorkingDirectory = "/var/lib/acme-dns";
+        ExecStart = "${pkgs.acme-dns}/bin/acme-dns -c ${acmeDnsConfig}";
+        Restart = "on-failure";
+        RestartSec = "5s";
+        AmbientCapabilities = [ "CAP_NET_BIND_SERVICE" ];
+        CapabilityBoundingSet = [ "CAP_NET_BIND_SERVICE" ];
+        NoNewPrivileges = true;
+        PrivateTmp = true;
+        PrivateDevices = true;
+        ProtectHome = true;
+        ProtectSystem = "strict";
+        ReadWritePaths = [ "/var/lib/acme-dns" ];
+        RestrictAddressFamilies = [ "AF_INET" "AF_UNIX" ];
+      };
+    };
+
+    # Traefik consumes its wildcard as immutable Swarm secrets, so a renewed
+    # host certificate must be copied and reconciled rather than merely reloaded.
+    # This service is started only by the production-mode ACME postRun hook.
+    services.cc-ci-acme-traefik-handoff = {
+      description = "Install renewed cc-ci wildcard into Traefik Swarm secrets";
+      after = [ "docker.service" "deploy-proxy.service" ];
+      requires = [ "docker.service" ];
+      path = [ pkgs.coreutils pkgs.docker pkgs.systemd pkgs.gnugrep ];
+      serviceConfig = {
+        Type = "oneshot";
+        UMask = "0077";
+      };
+      script = ''
+        src=/var/lib/acme/ci.commoninternet.net
+        dst=/var/lib/ci-certs/live
+        test -s "$src/fullchain.pem"
+        test -s "$src/key.pem"
+        install -d -m 0700 "$dst"
+        install -m 0444 "$src/fullchain.pem" "$dst/fullchain.pem.new"
+        install -m 0400 "$src/key.pem" "$dst/privkey.pem.new"
+        mv -f "$dst/fullchain.pem.new" "$dst/fullchain.pem"
+        mv -f "$dst/privkey.pem.new" "$dst/privkey.pem"
+
+        # deploy-proxy performs the health-gated Swarm rollout. Its reconciler
+        # derives a fresh version from the public certificate chain and inserts
+        # the matching ssl_cert/ssl_key secrets before deploying Traefik.
+        systemctl restart deploy-proxy.service
+
+        # A successful rollout no longer references old wildcard versions. Best
+        # effort removal retains any secret Docker still reports as in use.
+        keep="v$(sha256sum "$dst/fullchain.pem" | cut -c1-16)"
+        docker secret ls --format '{{.Name}}' | \
+          grep -E '^traefik_ci_commoninternet_net_ssl_(cert|key)_v' | \
+          grep -v -E "_(ssl_cert|ssl_key)_$keep\$" | \
+          while IFS= read -r stale; do docker secret rm "$stale" || true; done
+      '';
+    };
+  };
+
+  security.acme = {
+    acceptTerms = true;
+    certs."ci.commoninternet.net" = {
+      domain = "ci.commoninternet.net";
+      extraDomainNames = [ "*.ci.commoninternet.net" ];
+      # Staging issuance succeeded using the permanent, narrowly delegated
+      # CNAME. Production uses the same restricted acme-dns account.
+      dnsProvider = "acmedns";
+      environmentFile = "/etc/acme-dns/lego.env";
+      dnsResolver = "1.1.1.1:53";
+      server = "https://acme-v02.api.letsencrypt.org/directory";
+      postRun = ''
+        # The production marker is deployed only after staging proves the
+        # permanent CNAME and restricted acme-dns account work end to end.
+        if [ -e /var/lib/ci-certs/acme-production-enabled ]; then
+          ${pkgs.systemd}/bin/systemctl --no-block start cc-ci-acme-traefik-handoff.service
+        fi
+      '';
+    };
+  };
+}

nix/modules/bridge.nix

@@ -40,7 +40,7 @@ let
           # admin-registered push optimization deduped against the poller (§4.1). Enrollment = add
           # the repo to POLL_REPOS (csv) + ensure tests/<recipe>/ exists.
           - POLL_INTERVAL=30
-          - POLL_REPOS=recipe-maintainers/cc-ci,recipe-maintainers/custom-html,recipe-maintainers/custom-html-tiny,recipe-maintainers/keycloak,recipe-maintainers/cryptpad,recipe-maintainers/matrix-synapse,recipe-maintainers/lasuite-docs,recipe-maintainers/lasuite-meet,recipe-maintainers/n8n,recipe-maintainers/hedgedoc,recipe-maintainers/uptime-kuma,recipe-maintainers/bluesky-pds,recipe-maintainers/discourse,recipe-maintainers/ghost,recipe-maintainers/immich,recipe-maintainers/lasuite-drive,recipe-maintainers/mailu,recipe-maintainers/mattermost-lts,recipe-maintainers/mumble,recipe-maintainers/plausible,recipe-maintainers/drone,recipe-maintainers/gitea
+          - POLL_REPOS=recipe-maintainers/cc-ci,recipe-maintainers/custom-html,recipe-maintainers/custom-html-tiny,recipe-maintainers/keycloak,recipe-maintainers/cryptpad,recipe-maintainers/matrix-synapse,recipe-maintainers/lasuite-docs,recipe-maintainers/lasuite-meet,recipe-maintainers/n8n,recipe-maintainers/hedgedoc,recipe-maintainers/uptime-kuma,recipe-maintainers/bluesky-pds,recipe-maintainers/discourse,recipe-maintainers/ghost,recipe-maintainers/immich,recipe-maintainers/lasuite-drive,recipe-maintainers/mailu,recipe-maintainers/mattermost-lts,recipe-maintainers/mumble,recipe-maintainers/plausible,recipe-maintainers/drone,recipe-maintainers/gitea,recipe-maintainers/wordpress
           - HMAC_FILE=/run/secrets/webhook_hmac
           - DRONE_TOKEN_FILE=/run/secrets/drone_token
           - GITEA_TOKEN_FILE=/run/secrets/gitea_token
@@ -72,7 +72,7 @@ let
         name: cc_ci_bridge_drone_token_v1
       gitea_token:
         external: true
-        name: cc_ci_bridge_gitea_token_v1
+        name: cc_ci_bridge_gitea_token_v3
   '';
 
   reconcile = pkgs.writeShellApplication {
@@ -95,7 +95,7 @@ let
       }
       ensure_secret /run/secrets/bridge_webhook_hmac cc_ci_bridge_webhook_hmac_v1
       ensure_secret /run/secrets/bridge_drone_token  cc_ci_bridge_drone_token_v1
-      ensure_secret /run/secrets/bridge_gitea_token  cc_ci_bridge_gitea_token_v1
+      ensure_secret /run/secrets/bridge_gitea_token  cc_ci_bridge_gitea_token_v3
 
       docker stack deploy --detach=true -c ${stack} ccci-bridge
     '';

nix/modules/default.nix

@@ -0,0 +1,44 @@
+# The cc-ci CI server as ONE reusable NixOS module — exported from flake.nix as
+# `nixosModules.cc-ci-server`. Everything a host needs to BE cc-ci, except what is physical or
+# identity and therefore belongs to the host that imports it: hardware, networking, the tailscale
+# node, root SSH keys, `system.stateVersion`. A host sets `cc-ci.publicIPv4` (and, when it is not
+# built from a --recursive clone, `cc-ci.sopsFile`) and imports this.
+#
+# Consumers: nix/hosts/cc-ci-hetzner (the canonical standalone host) and
+# recipe-maintainers/cc-ci-orchestrator's `#cc-ci` host, which runs the CI server and the
+# orchestrator together (2026-09).
+{ pkgs, ... }:
+{
+  imports = [
+    ./options.nix
+    ./packages.nix
+    ./secrets.nix
+    ./acme-dns.nix
+    ./swarm.nix
+    ./docker-prune.nix
+    ./abra.nix
+    ./proxy.nix
+    ./drone.nix
+    ./drone-runner.nix
+    ./bridge.nix
+    ./dashboard.nix
+    ./reports.nix
+    ./backupbot.nix
+    ./harness.nix
+    ./warm-keycloak.nix
+    ./nightly-sweep.nix
+  ];
+
+  # Recipes bind-mount /etc/localtime and /etc/timezone; the harness compares timestamps across
+  # host and containers, so the host is UTC like every container.
+  time.timeZone = "UTC";
+  environment.etc."timezone".text = "UTC\n";
+
+  # Phase `nixenv`: the Drone exec runner resolves recipe shell-outs from this host PATH
+  # (/run/current-system/sw/bin). Install the SINGLE shared harness tool set (pkgs.ccciRuntimeTools,
+  # defined in packages.nix) so the Drone path and the harness env (cc-ci-run / sweep) can never
+  # diverge.
+  environment.systemPackages = pkgs.ccciRuntimeTools;
+
+  nix.settings.experimental-features = [ "nix-command" "flakes" ];
+}

nix/modules/options.nix

@@ -0,0 +1,33 @@
+# The few host-identity values the cc-ci modules need but must not hard-code, so that the same
+# modules can build the canonical Hetzner host, a throwaway rebuild VM, or a combined host that
+# also runs the cc-ci orchestrator (2026-09: recipe-maintainers/cc-ci-orchestrator imports
+# `nixosModules.cc-ci-server` from this repo and runs both on one box).
+{ lib, ... }:
+{
+  options.cc-ci = {
+    publicIPv4 = lib.mkOption {
+      type = lib.types.str;
+      example = "91.98.47.73";
+      description = ''
+        The host's public IPv4 address. acme-dns binds its authoritative listener to it and
+        publishes it as the `ns-acme` glue record. Must match the Gandi A record for
+        ns-acme.commoninternet.net and the address the cc-ci DNS names point at.
+      '';
+    };
+
+    sopsFile = lib.mkOption {
+      type = lib.types.path;
+      default = ../../secrets/secrets.yaml;
+      defaultText = lib.literalExpression "../../secrets/secrets.yaml";
+      example = "/etc/cc-ci/secrets/secrets.yaml";
+      description = ''
+        The sops-encrypted secrets.yaml (recipe-maintainers/cc-ci-secrets). The default is the
+        `secrets/` git submodule inside this repo, which only exists when this flake is built from
+        a `--recursive` clone (`git+file:///root/cc-ci?submodules=1`). A consumer that imports
+        cc-ci as a plain flake input (no submodule) sets this to an absolute path on the host
+        instead — e.g. the `/etc/cc-ci` deployed checkout's `secrets/secrets.yaml` — which
+        sops-nix then reads at activation time rather than copying into the store.
+      '';
+    };
+  };
+}

nix/modules/secrets.nix

@@ -6,8 +6,15 @@
 # off-box master recovery key).
 { config, ... }:
 {
+  imports = [ ./options.nix ];
+
   sops = {
-    defaultSopsFile = ../../secrets/secrets.yaml;
+    # See options.nix: the submodule path by default; an absolute host path on a combined host
+    # that imports cc-ci as a flake input without the private submodule.
+    defaultSopsFile = config.cc-ci.sopsFile;
+    # sops-nix validates store-path sops files at build time. An absolute (string) path is read
+    # at activation instead, so validation has to be off for that case.
+    validateSopsFiles = builtins.isPath config.cc-ci.sopsFile;
     # Decrypt using the host's SSH host key (converted to an age identity by sops-nix).
     age.sshKeyPaths = [ "/etc/ssh/ssh_host_ed25519_key" ];
     # Phase-1c: also accept a bootstrap age key at a fixed path — THE one out-of-band secret,
@@ -37,17 +44,10 @@
       bridge_drone_token = { };
       bridge_gitea_token = { };
 
-      # Phase-1c C2: the wildcard TLS cert+key are now sops secrets (in cc-ci-secrets), decrypted at
+      # The wildcard certificate and private key are issued and renewed locally
-      # activation to /var/lib/ci-certs/live/{fullchain.pem,privkey.pem} — the exact path the traefik
+      # by security.acme. Do not restore the retired SOPS pair here: activation
-      # reconcile (modules/proxy.nix) already reads. Replaces the prior operator-drops-a-cert-file step.
+      # would overwrite a freshly renewed ACME certificate before Traefik can
-      wildcard_cert = {
+      # consume it.
-        path = "/var/lib/ci-certs/live/fullchain.pem";
-        mode = "0444"; # leaf+intermediate chain — not secret
-      };
-      wildcard_key = {
-        path = "/var/lib/ci-certs/live/privkey.pem";
-        mode = "0400"; # private key — root only
-      };
 
       # Phase-2 rate-limit fix (Class A1 registry creds, operator-2026-05-28). Authenticated Docker
       # Hub pulls (200/6h per-account) replace the exhausted 100/6h shared-IP anonymous limit that

opencode.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://opencode.ai/config.json",
+  "agent": {
+    "general": {
+      "model": "opencode-go/deepseek-v4-flash"
+    }
+  }
+}

runner/harness/card.py

@@ -98,7 +98,7 @@ def _stage_rows(stages: list[dict]) -> str:
         scolor = STATUS_COLOR.get(st.get("status", ""), "#8b949e")
         rows.append(
             f'<tr class="stage"><td colspan="2"><span class="mark" style="color:{scolor}">{smark}</span>'
-            f'<b>{html.escape(st.get("name", "?"))}</b></td>'
+            f"<b>{html.escape(st.get('name', '?'))}</b></td>"
             f'<td class="st" style="color:{scolor}">{html.escape(st.get("status", ""))}</td></tr>'
         )
         for t in st.get("tests", []):
@@ -175,7 +175,7 @@ def render_card_html(data: dict, screenshot_rel: str | None = "screenshot.png")
         ok = bool(flags.get(key))
         flag_bits.append(
             f'<span class="flag" style="border-color:{"#3fb950" if ok else "#f85149"}">'
-            f'{STATUS_MARK["pass"] if ok else STATUS_MARK["fail"]} {lbl}</span>'
+            f"{STATUS_MARK['pass'] if ok else STATUS_MARK['fail']} {lbl}</span>"
         )
     show_shot = bool(screenshot_rel) and bool(data.get("screenshot"))
     shot_html = (

runner/harness/meta.py

@@ -132,6 +132,17 @@ KEYS: tuple[Key, ...] = (
         "Callable `(ctx)` invoked after UPGRADE_EXTRA_ENV env_set but before `abra secret generate --all` in the upgrade path. Use to pre-insert secrets that `generate --all` would produce with wrong format (e.g. when the .env.sample spec is commented out).",
         hook_params=("ctx",),
     ),
+    Key(
+        "UPGRADE_BASE_FLOOR",
+        "str",
+        None,
+        "Declared STRUCTURAL breaking boundary for the upgrade tier (phase basefloor): the first "
+        "post-break published version tag. Bases strictly below it are excluded from resolution "
+        "(canonical / step-back / no-canonical fallback) because an in-place upgrade across the "
+        "boundary is not supported upstream (e.g. a db-family change). When no ≥-floor predecessor "
+        "exists the tier records a DECLARED skip. NOT a static base pin (§2.G stays removed) — "
+        "resolution remains dynamic above the floor.",
+    ),
     # (CHAOS_BASE_DEPLOY, OIDC_AT_INSTALL and SKIP_GENERIC were deleted in restructure P2:
     # compose.ccci.yml is first-class + auto-chaos; install-time deps wiring is the only mode;
     # the generic floor is suppressible only via the dev-only CCCI_SKIP_GENERIC* env form.)

runner/run_recipe_ci.py

@@ -151,8 +151,30 @@ def resolve_upgrade_base(
             flush=True,
         )
         return BasePlan("skip", None, None, f"declared EXPECTED_NA[upgrade]: {declared}")
+    # UPGRADE_BASE_FLOOR (phase basefloor): a recipe_meta declaration marking a STRUCTURAL breaking
+    # boundary — published versions strictly below the floor are not valid in-place upgrade sources
+    # (e.g. discourse 0.8.x→1.0.0 changed the db family bitnami/pgvector → discourse/postgres; the
+    # data layout+roles are incompatible, upstream supports no in-place path across it). This is NOT
+    # the removed UPGRADE_BASE_VERSION pin (§2.G): resolution stays fully dynamic — the floor only
+    # EXCLUDES structurally-impossible bases, and when no candidate ≥ floor exists the tier records
+    # a DECLARED skip (never a silent pass). Never weakens: below-floor upgrades were never a
+    # supported path, so no real coverage is lost.
+    floor = getattr(meta, "UPGRADE_BASE_FLOOR", None)
+
+    def _below_floor(version: str) -> bool:
+        return bool(floor) and warm_reconcile.version_key(version) < warm_reconcile.version_key(
+            floor
+        )
+
     skip_canonicals = settings_mod.get().skip_canonicals_for_upgrade
     rec = canonical.read_registry(recipe)
+    if rec and rec.get("version") and not skip_canonicals and _below_floor(rec["version"]):
+        print(
+            f"== upgrade tier: last-green canonical {rec['version']} is below the declared "
+            f"UPGRADE_BASE_FLOOR {floor} (structural break) — excluded as a base",
+            flush=True,
+        )
+        rec = None
     if rec and rec.get("version") and not skip_canonicals:
         canon = rec["version"]
         same = head_version is not None and warm_reconcile.version_key(
@@ -168,9 +190,19 @@ def resolve_upgrade_base(
                 f"last-green (warm canonical, status={rec.get('status')})",
             )
         # canonical == head version → deploying it would be a same-version no-op. Step back to the
-        # newest published version strictly older than the head (phase samever).
+        # newest published version strictly older than the head (phase samever). Candidates below a
-        older = warm_reconcile.newest_older_version(
+        # declared UPGRADE_BASE_FLOOR are excluded (phase basefloor — structurally invalid bases).
-            warm_reconcile.recipe_tags(recipe), head_version
+        _tags = warm_reconcile.recipe_tags(recipe)
+        if floor:
+            _tags = [t for t in _tags if not _below_floor(t)]
+        older = warm_reconcile.newest_older_version(_tags, head_version)
+        if older is None and floor:
+            return BasePlan(
+                "skip",
+                None,
+                None,
+                f"declared UPGRADE_BASE_FLOOR {floor}: no published predecessor ≥ floor below "
+                f"head {head_version} (all older tags cross a structural break)",
             )
         if older:
             return BasePlan(
@@ -189,10 +221,12 @@ def resolve_upgrade_base(
     # No canonical in play — none recorded, OR SKIP_CANONICALS_FOR_UPGRADE=true (canonical lookup
     # bypassed entirely, behaving as if none exists). Improved fallback (phase settings §2.C): prefer
     # a REAL published predecessor (newest release tag < head) over the raw main-tip.
-    return _no_canonical_base(recipe, head_ref, head_version)
+    return _no_canonical_base(recipe, head_ref, head_version, floor=floor)
 
 
-def _no_canonical_base(recipe: str, head_ref: str | None, head_version: str | None) -> BasePlan:
+def _no_canonical_base(
+    recipe: str, head_ref: str | None, head_version: str | None, floor: str | None = None
+) -> BasePlan:
     """Upgrade base when no canonical is used (none recorded, its promote failed, or
     SKIP_CANONICALS_FOR_UPGRADE is true). Release-tag-first fallback (phase settings §2.C):
       1. most recent release TAG with version strictly older than the PR head — a clean published
@@ -202,11 +236,14 @@ def _no_canonical_base(recipe: str, head_ref: str | None, head_version: str | No
       3. skip — no predecessor (no older tag and head == main-tip, or no main at all).
     This replaces the old jump-straight-to-main-tip path, so an un-promoted recipe upgrades from a real
     release base instead of a possibly-untagged WIP commit."""
-    older = (
+    _tags = warm_reconcile.recipe_tags(recipe)
-        warm_reconcile.newest_older_version(warm_reconcile.recipe_tags(recipe), head_version)
+    if floor:
-        if head_version
+        # phase basefloor: exclude structurally-invalid bases below the declared floor; the
-        else None
+        # main-tip fallback below remains available (it is post-break by definition of the
-    )
+        # declaration — the floor names the first post-break published version).
+        _fk = warm_reconcile.version_key(floor)
+        _tags = [t for t in _tags if warm_reconcile.version_key(t) >= _fk]
+    older = warm_reconcile.newest_older_version(_tags, head_version) if head_version else None
     if older:
         return BasePlan(
             "version",

runner/warm_reconcile.py

@@ -24,6 +24,7 @@ Run as root on cc-ci (direct docker/volume access). CLI: `warm_reconcile.py <app
 
 from __future__ import annotations
 
+import hashlib
 import json
 import os
 import re
@@ -37,12 +38,51 @@ from harness import abra, lifecycle, warmsnap  # noqa: E402
 # --------------------------------------------------------------------------- specs
 
 
-def _traefik_setup(recipe: str, domain: str, version: str) -> None:
+CERT_DIR = "/var/lib/ci-certs/live"
+
+
+def wildcard_secret_version(cert_dir: str = CERT_DIR) -> str:
+    """Stable Swarm-secret version for the public certificate chain.
+
+    The certificate chain is public material, so its digest is safe to use as a
+    version label. The key is deliberately never read or hashed for logging.
+    """
+    chain = os.path.join(cert_dir, "fullchain.pem")
+    if not os.path.isfile(chain):
+        raise RuntimeError(f"FATAL: wildcard certificate missing at {chain}")
+    with open(chain, "rb") as certificate:
+        digest = hashlib.sha256(certificate.read()).hexdigest()
+    return "v" + digest[:16]
+
+
+def _traefik_requires_certificate_rollout(domain: str, secret_version: str) -> bool:
+    """Whether Traefik's active service still references an older cert version."""
+    stack = lifecycle._stack_name(domain)  # noqa: SLF001
+    service = f"{stack}_app"
+    result = _run(
+        [
+            "docker",
+            "service",
+            "inspect",
+            service,
+            "--format",
+            "{{range .Spec.TaskTemplate.ContainerSpec.Secrets}}{{.SecretName}} {{end}}",
+        ],
+        timeout=30,
+    )
+    expected = {
+        f"{stack}_ssl_cert_{secret_version}",
+        f"{stack}_ssl_key_{secret_version}",
+    }
+    return not expected.issubset(set(result.stdout.split()))
+
+
+def _traefik_setup(recipe: str, domain: str, version: str) -> bool:
     """Per-app config for the traefik reverse-proxy reconcile — preserves EXACTLY what the prior
     proxy.nix bash reconcile did (wildcard/file-provider mode serving the pre-issued cert as
     ssl_cert/ssl_key swarm secrets; NO ACME). Uses the proven abra.env_set (newline-safe, unlike the
     bash set_env that bit keycloak)."""
-    cert_dir = "/var/lib/ci-certs/live"
+    cert_dir = CERT_DIR
     if not (
         os.path.isfile(f"{cert_dir}/fullchain.pem") and os.path.isfile(f"{cert_dir}/privkey.pem")
     ):
@@ -56,14 +96,15 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
     abra.env_set(domain, "DOMAIN", domain)
     abra.env_set(domain, "LETS_ENCRYPT_ENV", "")
     abra.env_set(domain, "WILDCARDS_ENABLED", "1")
-    abra.env_set(domain, "SECRET_WILDCARD_CERT_VERSION", "v1")
+    secret_version = wildcard_secret_version(cert_dir)
-    abra.env_set(domain, "SECRET_WILDCARD_KEY_VERSION", "v1")
+    abra.env_set(domain, "SECRET_WILDCARD_CERT_VERSION", secret_version)
+    abra.env_set(domain, "SECRET_WILDCARD_KEY_VERSION", secret_version)
     abra.env_set(domain, "COMPOSE_FILE", '"compose.yml:compose.wildcard.yml"')
     stack = lifecycle._stack_name(domain)  # noqa: SLF001
     have = set(lifecycle._docker_names("secret", stack))  # noqa: SLF001
 
     def _has(name):
-        return any(s.endswith(f"_{name}_v1") for s in have)
+        return any(s.endswith(f"_{name}_{secret_version}") for s in have)
 
     if not _has("ssl_cert"):
         _run(
@@ -74,7 +115,7 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
                 "insert",
                 domain,
                 "ssl_cert",
-                "v1",
+                secret_version,
                 f"{cert_dir}/fullchain.pem",
                 "-f",
                 "-n",
@@ -91,7 +132,7 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
                 "insert",
                 domain,
                 "ssl_key",
-                "v1",
+                secret_version,
                 f"{cert_dir}/privkey.pem",
                 "-f",
                 "-n",
@@ -99,6 +140,7 @@ def _traefik_setup(recipe: str, domain: str, version: str) -> None:
             timeout=120,
             check=True,
         )
+    return _traefik_requires_certificate_rollout(domain, secret_version)
 
 
 SPECS: dict[str, dict] = {
@@ -457,8 +499,9 @@ def reconcile(app: str) -> str:
     # Per-app config/secrets: a spec may provide its own `setup` (traefik's cert/file-provider wiring);
     # otherwise the default keycloak-shaped path (app new + DOMAIN/LETS_ENCRYPT + generate secrets).
     setup = spec.get("setup")
+    setup_needs_rollout = False
     if setup:
-        setup(recipe, domain, latest)
+        setup_needs_rollout = bool(setup(recipe, domain, latest))
     else:
         ensure_app_config(recipe, domain, latest)
         ensure_secrets(domain)
@@ -476,6 +519,20 @@ def reconcile(app: str) -> str:
         write_last_good(recipe, target)
         return f"deployed-fresh:{target}"
 
+    # A certificate rotation changes Traefik's immutable Swarm secrets but
+    # must not be held hostage by an unrelated recipe-major upgrade policy.
+    # Redeploy the current recipe version so its compose spec references the
+    # just-created cert/key secret pair, then apply the usual health gate.
+    if setup_needs_rollout:
+        if not current:
+            raise RuntimeError(f"{app} has services but no current version")
+        print(f"[{app}] certificate changed → redeploy {current}", flush=True)
+        deploy_version(recipe, domain, current, dt)
+        if not wait_healthy(spec):
+            raise RuntimeError(f"{app} certificate rollout {current} did not become healthy")
+        write_last_good(recipe, current)
+        return f"certificate-rolled-out:{current}"
+
     # Deployed & already on latest → converge to a no-op (commit last-good if healthy).
     if current == latest:
         if wait_healthy(spec, timeout=60):

tests/STYLE.md

@@ -0,0 +1,116 @@
+# cc-ci test style guide
+
+Rules for writing and changing tests under `tests/`. Read this before any test edit — in particular
+before a `/recipe-upgrade <recipe> --with-tests` or `/ci-test-review` fix, where the temptation is to
+make a red run green rather than to make the test right.
+
+The tests are the **independent gate** on recipe upgrades. Their value is entirely in being hard to
+fool, so every rule below exists to keep them (a) honest and (b) alive across upgrades.
+
+---
+
+## 1. Set up state through the application, not its database
+
+**Order of preference for any fixture that must create state:**
+
+1. **The app's public HTTP API.**
+2. **The app's official CLI or release console** (`docker exec … <app-cli>`).
+3. **Writing rows into its database — last resort only**, and only with a comment saying which of the
+   above were tried and why they did not work.
+
+Direct SQL couples the test to the app's *internal schema*, which upgrades are free to change. The
+app's own interface is the thing it promises to keep working.
+
+> **Why this rule exists.** `tests/plausible/custom/test_event_tracking.py` used to register its test
+> site with `INSERT INTO sites (...)`. That was sufficient for plausible v2. In v3 a site must belong
+> to a **team**, and the app silently discards events for a teamless site — `POST /api/event` still
+> returns **202** and the row is still in postgres, so the only visible symptom was that nothing ever
+> reached ClickHouse. It read as a mysterious ingestion stall and held the recipe RED for six weeks.
+>
+> The fix was not to also INSERT a team row. It was to stop writing rows: the fixture now calls
+> `Plausible.Sites.create/2` through the app's release console, and the app provisions whatever its
+> data model currently needs. The same expression works unchanged on v2 (which has no `teams` table
+> at all) **and** v3 — not because the test handles both, but because it stopped depending on the
+> schema.
+
+When the ideal interface is unavailable, say so in the code. plausible's HTTP provisioning API
+(`POST /api/v1/sites`) is gated behind a paid plan and answers `:upgrade_required` on CE, so the test
+drops to option 2 and records that in a comment.
+
+## 2. Gate on version rather than writing dual-path fixtures
+
+If a behaviour genuinely only exists from version X, **gate the test on the version** instead of
+branching inside it:
+
+```python
+pytest.mark.skipif(app_version < (3,), reason="teams were introduced in v3")
+```
+
+Do **not** write a fixture that carefully supports both schemas. Version-portable code is harder to
+read, harder to trust, and quietly rots once nobody runs the old path.
+
+Corollary: **old tests can simply be deleted** once the fleet has moved past that version. The older
+version is only ever exercised through the *upgrade* tier (deploy base → upgrade → assert), so tests
+that only make sense for a superseded version are dead weight, not coverage.
+
+Prefer §1 first: an app-level fixture often makes the version difference disappear, and then no gate
+is needed at all.
+
+## 3. Never weaken an assertion to turn a run green
+
+There is a hard line between these two, and only the second is allowed as a way out of a red run:
+
+* **Weakening** — relaxing *what* is asserted: dropping a field check, accepting a wider status set,
+  asserting a 202 ack instead of the stored result, deleting the read-back.
+* **Correcting the fixture or the wait** — fixing *how* the test sets up or how long it allows, with
+  the assertion untouched.
+
+If a test can only pass by asserting less, it has found a real regression. Report it; do not edit it.
+
+## 4. Assert real state, not acknowledgements
+
+An HTTP 202 means "accepted", not "done". Read the effect back out of the system that owns it — the
+row in the analytics store, the file on disk, the record in the API — and assert on the values you
+sent. plausible's ingestion returns 202 for events it goes on to discard entirely; a test that
+stopped at the ack would have been permanently, silently green.
+
+## 5. Derive waits from the recipe's declared readiness, not a guess
+
+A per-recipe `recipe_meta.py` already declares `DEPLOY_TIMEOUT` / `HTTP_TIMEOUT` because someone
+measured that app's boot profile. A custom test that hard-codes a shorter window contradicts it and
+will flake or fail on a slower version.
+
+Remember the **tier order**: `custom` runs after `backup`/`restore`, which disrupts the datastore and
+restarts the app. A window sized for a warm app is not sized for that. plausible's health check
+allowed 60s; v3 boots through `sleep 10` → `createdb` → `migrate` → cache warmers first.
+
+## 6. Diagnose from the app's own telemetry before touching a test
+
+Before concluding a test is stale, find the app's account of what happened. It is usually definitive
+and it stops you fixing the wrong thing. plausible records dropped events in ClickHouse's
+`ingest_counters`: `dropped_not_found` with 0 rows before the fix, `buffered` with rows after — that
+single counter identified the root cause after the HTTP status had suggested everything was fine.
+
+Prove the diagnosis both ways where you can: same input, broken state → symptom; corrected state →
+no symptom.
+
+## 7. Fixtures must be idempotent
+
+A fixture may run against a warm canonical, a restored volume, or a re-run. Creating state must be
+safe to repeat — look the object up first and reuse it, rather than assuming a clean database.
+
+## 8. Keep test identities obviously synthetic
+
+Use `ccci-`-prefixed names and `.example` / `.invalid` domains for anything a test creates, so state
+it leaves behind is instantly attributable and can never be confused with real data.
+
+---
+
+## Changing a test: the checklist
+
+1. Reproduce the failure and get the **app's own** explanation (§6).
+2. Classify: recipe bug, or stale test? Only a stale test justifies a test edit.
+3. Fix the **fixture, wait, or setup** — never the assertion (§3).
+4. Prefer the app's interface over its database (§1); gate on version rather than branching (§2).
+5. Verify green against the recipe PR head with the changed test, plus a regression sample.
+6. Say in the commit and PR **what evidence** proves the diagnosis, not just what changed.

tests/bluesky-pds/custom/test_account_and_post.py

@@ -88,9 +88,9 @@ def test_account_lifecycle_and_post_roundtrip(live_app):
 
     # Step 1: PDS describe via goat — recipe self-identifies as did:web:<domain>
     out = _in_container(domain, f"goat pds describe {PDS_HOST_LOCAL} 2>&1")
-    assert (
+    assert f"did:web:{domain}" in out, (
-        f"did:web:{domain}" in out
+        f"goat pds describe did not contain expected DID 'did:web:{domain}'. Output:\n{out[:500]!r}"
-    ), f"goat pds describe did not contain expected DID 'did:web:{domain}'. Output:\n{out[:500]!r}"
+    )
 
     # Step 2: Create account (UUID-suffixed handle = no run-to-run collision)
     out = _goat_admin(
@@ -133,9 +133,9 @@ def test_account_lifecycle_and_post_roundtrip(live_app):
         assert s == 200, f"createRecord HTTP {s}: {body!r}"
         record_uri = (body or {}).get("uri", "")
         # URI format: at://<did>/app.bsky.feed.post/<rkey>
-        assert record_uri.startswith(
+        assert record_uri.startswith(f"at://{new_did}/app.bsky.feed.post/"), (
-            f"at://{new_did}/app.bsky.feed.post/"
+            f"unexpected record uri: {record_uri!r}"
-        ), f"unexpected record uri: {record_uri!r}"
+        )
         rkey = record_uri.rsplit("/", 1)[-1]
         assert rkey, f"no rkey in uri: {record_uri!r}"
 
@@ -148,9 +148,9 @@ def test_account_lifecycle_and_post_roundtrip(live_app):
         )
         assert s == 200, f"getRecord HTTP {s}: {body!r}"
         record_value = (body or {}).get("value", {})
-        assert (
+        assert record_value.get("text") == marker, (
-            record_value.get("text") == marker
+            f"post text did not round-trip: created={marker!r}, fetched={record_value.get('text')!r}"
-        ), f"post text did not round-trip: created={marker!r}, fetched={record_value.get('text')!r}"
+        )
         assert record_value.get("$type") == "app.bsky.feed.post"
     finally:
         # Step 6: Best-effort cleanup. (The per-run domain teardown will discard the volume

tests/bluesky-pds/custom/test_describe_server.py

@@ -26,6 +26,6 @@ def test_describe_server_returns_atproto_envelope(live_app):
     # At least one of these atproto-spec fields must be present
     expected_any = ("availableUserDomains", "inviteCodeRequired", "links", "did")
     present = [k for k in expected_any if k in body]
-    assert (
+    assert present, (
-        present
+        f"describe-server missing all of {expected_any}; got keys: {sorted(body.keys())[:20]}"
-    ), f"describe-server missing all of {expected_any}; got keys: {sorted(body.keys())[:20]}"
+    )

tests/bluesky-pds/custom/test_health_check.py

@@ -17,6 +17,6 @@ def test_pds_health_returns_version(live_app):
     url = f"https://{live_app}/xrpc/_health"
     status, body = harness_http.retry_http_get(url, expect_status=200, max_wait=60, interval=3)
     assert status == 200, f"GET {url} HTTP {status} (expected 200)"
-    assert (
+    assert isinstance(body, dict) and isinstance(body.get("version"), str) and body["version"], (
-        isinstance(body, dict) and isinstance(body.get("version"), str) and body["version"]
+        f"GET {url} response is not the expected health envelope: {body!r}"
-    ), f"GET {url} response is not the expected health envelope: {body!r}"
+    )

tests/bluesky-pds/custom/test_session_auth.py

@@ -30,6 +30,6 @@ def test_get_session_requires_auth(live_app):
         f"body: {body!r}"
     )
     # The XRPC error envelope is JSON with an `error` field per the atproto spec.
-    assert isinstance(body, dict) and body.get(
+    assert isinstance(body, dict) and body.get("error"), (
-        "error"
+        f"expected XRPC JSON error envelope; got: {body!r}"
-    ), f"expected XRPC JSON error envelope; got: {body!r}"
+    )

tests/bluesky-pds/test_restore.py

@@ -11,6 +11,6 @@ import _p4  # noqa: E402
 
 
 def test_restore_returns_state(live_app):
-    assert _p4.account_exists(
+    assert _p4.account_exists(live_app), (
-        live_app
+        "restore did not bring back the seeded marker account (PDS data did not survive restore)"
-    ), "restore did not bring back the seeded marker account (PDS data did not survive restore)"
+    )

tests/concurrency/test_janitor.py

@@ -78,9 +78,9 @@ def test_7_new_run_blocks_until_reap_finishes(lock_dir, pool, monkeypatch):
     line = wait_marker(state["acquirer_out"], "ACQUIRED", timeout=15)
     assert line, "new run never acquired after the reap"
     acquired_ts = float(line.split()[1])
-    assert (
+    assert acquired_ts >= state["teardown_end"], (
-        acquired_ts >= state["teardown_end"]
+        f"new run acquired at {acquired_ts} BEFORE the reap finished at {state['teardown_end']}"
-    ), f"new run acquired at {acquired_ts} BEFORE the reap finished at {state['teardown_end']}"
+    )
     # The new run must hold a lock the next probe can SEE (fresh inode at the path).
     assert lock_state(DOMAIN) == "held"
 
@@ -160,17 +160,17 @@ def test_11_warm_canonical_names_never_probed(lock_dir, monkeypatch):
     monkeypatch.setattr(
         lifecycle,
         "_docker_names",
-        lambda kind, stack: ["warm-keycloak_ci_commoninternet_net_app"]
+        lambda kind, stack: (
-        if kind == "service"
+            ["warm-keycloak_ci_commoninternet_net_app"] if kind == "service" else []
-        else [],
+        ),
     )
     monkeypatch.setattr(lifecycle, "teardown_app", lambda d, verify=True: calls.append(d))
     lifecycle.janitor()
     assert calls == []
     lockdir = os.environ["CCCI_APP_LOCK_DIR"]
-    assert [
+    assert [f for f in os.listdir(lockdir) if f.startswith("cc-ci-app-")] == [], (
-        f for f in os.listdir(lockdir) if f.startswith("cc-ci-app-")
+        "janitor must not create lockfiles for non-run-app names"
-    ] == [], "janitor must not create lockfiles for non-run-app names"
+    )
 
 
 def test_12_degrades_safely_on_bad_lockfile_and_missing_dir(lock_dir, monkeypatch, capsys):

tests/concurrency/test_locks.py

@@ -61,9 +61,9 @@ def test_3_lock_fd_not_inherited_by_children(lock_dir, pool):
     p.kill()
     p.wait(timeout=10)
     assert os.path.exists(f"/proc/{child_pid}"), "child should outlive the holder"
-    assert (
+    assert wait_lock_state(DOMAIN, "free") == "free", (
-        wait_lock_state(DOMAIN, "free") == "free"
+        "lock must release on holder death even with a live child (PEP 446 non-inheritable fd)"
-    ), "lock must release on holder death even with a live child (PEP 446 non-inheritable fd)"
+    )
 
 
 def test_4_second_acquire_blocks_until_first_exits(lock_dir, pool):

tests/concurrency/test_run_state.py

@@ -64,9 +64,9 @@ def test_20c_same_domain_runs_each_keep_their_own_count(tmp_path, lock_dir, pool
     pa.wait(timeout=15)
 
     line_b = wait_marker(out_b, "COUNT")
-    assert (
+    assert line_b is not None and line_b.strip() == "COUNT 1", (
-        line_b is not None and line_b.strip() == "COUNT 1"
+        line_b
-    ), line_b  # B's file survived A's remove
+    )  # B's file survived A's remove
     pb.wait(timeout=15)
 
 

tests/cryptpad/custom/test_pad_content_roundtrip.py

@@ -150,9 +150,9 @@ def test_cryptpad_pad_content_survives_fresh_session(live_app):
             # --- session 1: create the pad + write the marker ---
             ctx1 = browser.new_context(ignore_https_errors=True)
             page, pad_url = _open_pad(ctx1, f"https://{live_app}/pad/")
-            assert (
+            assert "#/2/pad/edit/" in pad_url, (
-                "#/2/pad/edit/" in pad_url
+                f"CryptPad did not create a fragment-keyed pad URL; got {pad_url!r}"
-            ), f"CryptPad did not create a fragment-keyed pad URL; got {pad_url!r}"
+            )
             ck = _ckeditor_frame(page, reload_url=pad_url)
             assert ck is not None, "CKEditor content frame never attached (pad editor not ready)"
             _dismiss_store_modal(page)
@@ -161,9 +161,9 @@ def test_cryptpad_pad_content_survives_fresh_session(live_app):
             page.wait_for_timeout(1000)
             body.type(marker, delay=40)
             page.wait_for_timeout(12000)  # let CryptPad encrypt + sync the update to the server
-            assert (
+            assert marker in ck.locator("body").inner_text(), (
-                marker in ck.locator("body").inner_text()
+                "marker not present in the editor after typing — type did not land"
-            ), "marker not present in the editor after typing — type did not land"
+            )
             ctx1.close()
 
             # --- session 2: FRESH context (no shared storage/localStorage) reads the pad back by URL.

tests/cryptpad/custom/test_pad_create.py

@@ -51,9 +51,9 @@ def test_cryptpad_spa_renders_with_no_console_errors(live_app):
             title = (page.title() or "").lower()
             body = page.content()
             blower = body.lower()
-            assert (
+            assert "cryptpad" in title or "cryptpad" in blower, (
-                "cryptpad" in title or "cryptpad" in blower
+                f"CryptPad SPA does not carry brand. title={title!r}, body excerpt: {body[:200]!r}"
-            ), f"CryptPad SPA does not carry brand. title={title!r}, body excerpt: {body[:200]!r}"
+            )
 
             # Canonical CryptPad asset references in the rendered DOM
             canonical = ("/customize/", "/components/", "main.js", "/api/broadcast")

tests/cryptpad/test_backup.py

@@ -14,6 +14,6 @@ MARKER = "/cryptpad/data/ci-marker.txt"
 
 
 def test_backup_captures_state(live_app):
-    assert (
+    assert lifecycle.exec_in_app(live_app, ["cat", MARKER]).strip() == "original", (
-        lifecycle.exec_in_app(live_app, ["cat", MARKER]).strip() == "original"
+        "the seeded state was not present at backup time"
-    ), "the seeded state was not present at backup time"
+    )

tests/cryptpad/test_restore.py

@@ -14,6 +14,6 @@ MARKER = "/cryptpad/data/ci-marker.txt"
 
 
 def test_restore_returns_state(live_app):
-    assert (
+    assert lifecycle.exec_in_app(live_app, ["cat", MARKER]).strip() == "original", (
-        lifecycle.exec_in_app(live_app, ["cat", MARKER]).strip() == "original"
+        "restore did not return the pre-mutation state"
-    ), "restore did not return the pre-mutation state"
+    )

tests/cryptpad/test_upgrade.py

@@ -14,6 +14,6 @@ MARKER = "/cryptpad/data/ci-marker.txt"
 
 
 def test_upgrade_preserves_data(live_app):
-    assert (
+    assert lifecycle.exec_in_app(live_app, ["cat", MARKER]).strip() == "upgrade-survives", (
-        lifecycle.exec_in_app(live_app, ["cat", MARKER]).strip() == "upgrade-survives"
+        "data did not survive the upgrade"
-    ), "data did not survive the upgrade"
+    )

tests/custom-html-tiny/custom/test_serves_content.py

@@ -79,9 +79,9 @@ def test_static_file_roundtrip_and_404(live_app):
         # A random non-existent path must 404 — proves real static-file semantics, distinguishing a
         # working server from a 200-everything stub or a mis-routed Traefik fallback.
         miss_status, _ = _get(f"https://{live_app}/ccci-missing-{uuid.uuid4().hex}.txt")
-        assert (
+        assert miss_status == 404, (
-            miss_status == 404
+            f"missing path returned {miss_status} (expected 404 — generic 200-returner / mis-route?)"
-        ), f"missing path returned {miss_status} (expected 404 — generic 200-returner / mis-route?)"
+        )
     finally:
         with contextlib.suppress(OSError):
             os.remove(path)

tests/custom-html/custom/test_content_type_header.py

@@ -53,9 +53,9 @@ def test_content_type_html_and_txt(live_app):
     ct_txt = h_txt.get("content-type", "")
 
     # nginx default: "text/html" for .html and "text/plain" for .txt (may include "; charset=utf-8")
-    assert ct_html.startswith(
+    assert ct_html.startswith("text/html"), (
-        "text/html"
+        f"{html_name} Content-Type={ct_html!r}, expected text/html (nginx MIME config broken?)"
-    ), f"{html_name} Content-Type={ct_html!r}, expected text/html (nginx MIME config broken?)"
+    )
-    assert ct_txt.startswith(
+    assert ct_txt.startswith("text/plain"), (
-        "text/plain"
+        f"{txt_name} Content-Type={ct_txt!r}, expected text/plain (nginx MIME config broken?)"
-    ), f"{txt_name} Content-Type={ct_txt!r}, expected text/plain (nginx MIME config broken?)"
+    )

tests/custom-html/test_backup.py

@@ -16,6 +16,6 @@ MARKER_PATH = "/usr/share/nginx/html/ci-marker.txt"
 
 
 def test_backup_captures_state(live_app):
-    assert (
+    assert lifecycle.exec_in_app(live_app, ["cat", MARKER_PATH]).strip() == "original", (
-        lifecycle.exec_in_app(live_app, ["cat", MARKER_PATH]).strip() == "original"
+        "the seeded state was not present at backup time"
-    ), "the seeded state was not present at backup time"
+    )

tests/custom-html/test_restore.py

@@ -17,6 +17,6 @@ MARKER_PATH = "/usr/share/nginx/html/ci-marker.txt"
 
 def test_restore_returns_state(live_app):
     restored = lifecycle.exec_in_app(live_app, ["cat", MARKER_PATH]).strip()
-    assert (
+    assert restored == "original", (
-        restored == "original"
+        f"restore did not return the pre-mutation (backed-up) state: got {restored!r}"
-    ), f"restore did not return the pre-mutation (backed-up) state: got {restored!r}"
+    )

tests/custom-html/test_upgrade.py

@@ -16,6 +16,6 @@ MARKER_PATH = "/usr/share/nginx/html/ci-marker.txt"
 
 def test_upgrade_preserves_data(live_app):
     # the marker seeded by ops.pre_upgrade (before the harness upgraded) is still served
-    assert (
+    assert lifecycle.http_fetch(live_app, "/ci-marker.txt")[1].strip() == "upgrade-survives", (
-        lifecycle.http_fetch(live_app, "/ci-marker.txt")[1].strip() == "upgrade-survives"
+        "data did not survive the upgrade"
-    ), "data did not survive the upgrade"
+    )

tests/discourse/custom/_discourse.py

@@ -81,9 +81,9 @@ def mint_admin(domain: str) -> tuple[str, str]:
             key = line.split("=", 1)[1].strip()
         elif line.startswith("CCCI_API_USER="):
             user = line.split("=", 1)[1].strip()
-    assert (
+    assert key and user, (
-        key and user
+        f"could not bootstrap discourse admin/API key; rails output tail:\n{out[-1000:]}"
-    ), f"could not bootstrap discourse admin/API key; rails output tail:\n{out[-1000:]}"
+    )
     return key, user
 
 

tests/discourse/custom/test_create_topic.py

@@ -48,23 +48,23 @@ def test_create_topic_roundtrip(live_app):
         headers=hdrs,
         timeout=60,
     )
-    assert status in (200, 201) and isinstance(
+    assert status in (200, 201) and isinstance(body, dict), (
-        body, dict
+        f"create topic failed: HTTP {status}, body={body!r}"
-    ), f"create topic failed: HTTP {status}, body={body!r}"
+    )
     topic_id = body.get("topic_id")
     assert topic_id, f"create topic returned no topic_id: {body!r}"
 
     # 4) Read the topic back and assert title + first-post body round-trip.
     status, got = harness_http.http_get(f"{base}/t/{topic_id}.json", headers=hdrs, timeout=30)
-    assert status == 200 and isinstance(
+    assert status == 200 and isinstance(got, dict), (
-        got, dict
+        f"read topic failed: HTTP {status}, body={got!r}"
-    ), f"read topic failed: HTTP {status}, body={got!r}"
+    )
-    assert (
+    assert got.get("title") == title, (
-        got.get("title") == title
+        f"topic title did not round-trip: sent {title!r}, got {got.get('title')!r}"
-    ), f"topic title did not round-trip: sent {title!r}, got {got.get('title')!r}"
+    )
     posts = (got.get("post_stream") or {}).get("posts") or []
     assert posts, f"topic has no posts on read-back: {got!r}"
     first_cooked = posts[0].get("cooked", "")
-    assert (
+    assert marker in first_cooked, (
-        marker in first_cooked
+        f"topic body did not round-trip: marker {marker!r} not in first post {first_cooked!r}"
-    ), f"topic body did not round-trip: marker {marker!r} not in first post {first_cooked!r}"
+    )

tests/discourse/custom/test_site_basic.py

@@ -20,12 +20,12 @@ def test_site_json_has_discourse_config(live_app):
     status, body = harness_http.retry_http_get(
         f"https://{live_app}/site.json", expect_status=200, max_wait=120, interval=5
     )
-    assert status == 200 and isinstance(
+    assert status == 200 and isinstance(body, dict), (
-        body, dict
+        f"GET /site.json failed: HTTP {status}, body type={type(body).__name__}"
-    ), f"GET /site.json failed: HTTP {status}, body type={type(body).__name__}"
+    )
     # /site.json carries Discourse-specific structure — `categories` (a list) and `groups` are always
     # present in a booted Discourse. A non-Discourse 200 (placeholder page) would not parse to this.
     assert "categories" in body, f"/site.json missing 'categories' key: keys={list(body)[:20]}"
-    assert isinstance(
+    assert isinstance(body["categories"], list), (
-        body["categories"], list
+        f"/site.json 'categories' not a list: {type(body['categories']).__name__}"
-    ), f"/site.json 'categories' not a list: {type(body['categories']).__name__}"
+    )

tests/discourse/ops.py

@@ -14,9 +14,7 @@ from harness import lifecycle  # noqa: E402
 
 
 def _psql(domain, sql):
-    cmd = (
+    cmd = f'PGPASSWORD=$(cat /run/secrets/db_password) psql -U discourse -d discourse -tAc "{sql}"'
-        "PGPASSWORD=$(cat /run/secrets/db_password) " f'psql -U discourse -d discourse -tAc "{sql}"'
-    )
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 

tests/discourse/recipe_meta.py

@@ -23,11 +23,21 @@ HTTP_TIMEOUT = 1200
 #
 # UPGRADE-tier BASE (phase prevb — DYNAMIC, no hardcoded UPGRADE_BASE_VERSION): the base the head
 # upgrades from is resolved at run time — last-green (warm canonical) → fallback target-branch (`main`)
-# tip → else skip (run_recipe_ci.resolve_upgrade_base). discourse has no warm canonical, so the base is
+# tip → else skip (run_recipe_ci.resolve_upgrade_base).
-# the `main` tip = bitnamilegacy/discourse:3.5.0, which deploys clean (bitnamilegacy exists) with NO
+#
-# `previous/` repair needed. The PR head (recipe-maintainers/discourse#4) switches app to the official
+# UPGRADE_BASE_FLOOR (phase basefloor, 2026-08-04): the 0.8.x→1.0.0 recipe family switched the app
-# `discourse/discourse:3.5.3` and drops the sidekiq service, so the upgrade tier now exercises the REAL
+# bitnamilegacy/discourse → official discourse/discourse AND the db pgvector/pgvector:pg17 →
-# bitnamilegacy→official image migration the PR claims to support.
+# discourse/postgres:pg18. That db-family change is a structural break: the bitnami cluster has no
+# `discourse` role and pg_upgrade preserves-not-creates roles, so an in-place 0.8.x→1.x deploy can
+# NEVER converge (app FATALs `role "discourse" does not exist`, swarm rolls back) — upstream ships
+# no in-place path across it. Without the floor, the resolver's step-back/fallback selected
+# 0.8.1+3.5.0 (newest tag below the head label) and the upgrade tier red'd on this unsupported
+# path twice (drone #1165 2026-07-31 diagnosis, #1171/weekly 2026-08-03 — both classified
+# stale-test, recipe verified green on the real official→official path). Declaring the floor keeps
+# resolution dynamic and only excludes the structurally-impossible bases; when no ≥-floor
+# predecessor exists the tier records a DECLARED skip (never a silent pass). No assertion weakened:
+# below-floor in-place upgrades were never supported coverage.
+UPGRADE_BASE_FLOOR = "1.0.0+3.5.3"
 #
 # compose.ccci.yml is now the ENVIRONMENTAL overlay (all deploys): only app.deploy.update_config.order:
 # stop-first (node memory reality on the upgrade crossover — see its header). The version-specific

tests/discourse/test_backup.py

@@ -13,13 +13,11 @@ from harness import lifecycle  # noqa: E402
 
 
 def _psql(domain, sql):
-    cmd = (
+    cmd = f'PGPASSWORD=$(cat /run/secrets/db_password) psql -U discourse -d discourse -tAc "{sql}"'
-        "PGPASSWORD=$(cat /run/secrets/db_password) " f'psql -U discourse -d discourse -tAc "{sql}"'
-    )
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 
 def test_backup_captures_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "the seeded discourse postgres state was not present at backup time"
-    ), "the seeded discourse postgres state was not present at backup time"
+    )

tests/discourse/test_restore.py

@@ -13,13 +13,11 @@ from harness import lifecycle  # noqa: E402
 
 
 def _psql(domain, sql):
-    cmd = (
+    cmd = f'PGPASSWORD=$(cat /run/secrets/db_password) psql -U discourse -d discourse -tAc "{sql}"'
-        "PGPASSWORD=$(cat /run/secrets/db_password) " f'psql -U discourse -d discourse -tAc "{sql}"'
-    )
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 
 def test_restore_returns_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "restore did not return the pre-mutation discourse postgres state (data-integrity failure)"
-    ), "restore did not return the pre-mutation discourse postgres state (data-integrity failure)"
+    )

tests/discourse/test_upgrade.py

@@ -6,7 +6,11 @@ migration was never tested. With the version-specific config removed from the al
 and the dynamic base (last-green/main = bitnamilegacy:3.5.0) deployed only as the *base*, the upgrade
 chaos redeploy must land the PR head UNMODIFIED. This overlay asserts exactly that, post-upgrade:
 
-  1. the running `app` service image IS the official discourse/discourse:3.5.3 — NOT bitnamilegacy;
+  1. the running `app` service image IS from the official `discourse/discourse` repository —
+     NOT bitnamilegacy. (Version-agnostic since 2026-08-04: the original assertion hardcoded the
+     migration-era pin `:3.5.3` and went stale on the first legitimate app bump (2026.7.1, weekly
+     2026-08-03). The property this test guards is the IMAGE FAMILY — official vs bitnami — not a
+     frozen version; the exact head pin is already exercised by the deploy itself.)
   2. the `sidekiq` service the PR deletes is GONE from the deployed stack.
 
 If either fails, the head did not really run (the overlay leaked onto it) → RED. Assertion-only,
@@ -26,8 +30,8 @@ def test_head_runs_official_image_not_bitnamilegacy(live_app):
         f"app image is {image!r} — the bitnamilegacy base leaked onto the PR head "
         "(the version-specific overlay was applied to the head, the prevb bug)"
     )
-    assert image.startswith("discourse/discourse:3.5.3"), (
+    assert image.startswith("discourse/discourse:"), (
-        f"app image is {image!r}, expected the PR head's official discourse/discourse:3.5.3 "
+        f"app image is {image!r}, expected the PR head's official discourse/discourse image "
         "— the head's image migration was not exercised"
     )
 

tests/ghost/custom/test_admin_redirect.py

@@ -62,6 +62,6 @@ def test_ghost_admin_route_is_wired(live_app):
     assert status in (200, 302), f"unexpected status: {status}"
     if status == 200:
         # The admin SPA references /ghost-assets/ or contains "ghost" in title/body
-        assert (
+        assert "ghost" in body.lower(), (
-            "ghost" in body.lower()
+            f"GET {url} 200 but body has no Ghost markers: {body[:200]!r}"
-        ), f"GET {url} 200 but body has no Ghost markers: {body[:200]!r}"
+        )

tests/ghost/custom/test_content_api.py

@@ -35,10 +35,10 @@ def test_content_api_settings_endpoint(live_app):
     assert body is not None, f"GET {url} returned non-JSON body"
     # On success: {"settings": {...}}. On error: {"errors": [...]}. Either shape is valid.
     if status == 200:
-        assert (
+        assert isinstance(body, dict) and "settings" in body, (
-            isinstance(body, dict) and "settings" in body
+            f"200 response missing 'settings' envelope: {body!r}"
-        ), f"200 response missing 'settings' envelope: {body!r}"
+        )
     else:
-        assert isinstance(body, dict) and (
+        assert isinstance(body, dict) and ("errors" in body or "message" in body or body), (
-            "errors" in body or "message" in body or body
+            f"error response not a proper Ghost error envelope: {body!r}"
-        ), f"error response not a proper Ghost error envelope: {body!r}"
+        )

tests/ghost/custom/test_post_roundtrip.py

@@ -43,17 +43,17 @@ def test_create_post_roundtrip(live_app):
     title = f"ccci-marker-{uniq}"
     marker = f"ccci-body-marker-{uniq}-roundtrip"
     created = admin.create_post(title, f"<p>{marker}</p>")
-    assert (
+    assert created.get("title") == title, (
-        created.get("title") == title
+        f"created post title mismatch: sent {title!r}, got {created.get('title')!r}"
-    ), f"created post title mismatch: sent {title!r}, got {created.get('title')!r}"
+    )
 
     # 4) Read it back by id and assert the post survived the round-trip (title always returned;
     #    html returned because we requested ?formats=html).
     got = admin.get_post(created["id"])
-    assert (
+    assert got.get("title") == title, (
-        got.get("title") == title
+        f"post title did not round-trip: sent {title!r}, got {got.get('title')!r}"
-    ), f"post title did not round-trip: sent {title!r}, got {got.get('title')!r}"
+    )
     html = got.get("html") or ""
-    assert (
+    assert marker in html, (
-        marker in html
+        f"post body did not round-trip: marker {marker!r} not in read-back html {html!r}"
-    ), f"post body did not round-trip: marker {marker!r} not in read-back html {html!r}"
+    )

tests/ghost/ops.py

@@ -22,7 +22,7 @@ from harness import lifecycle  # noqa: E402
 
 
 def _mysql(domain, sql):
-    cmd = 'MYSQL_PWD="$(cat /run/secrets/db_password)" ' f'mysql -u root -N -s ghost -e "{sql}"'
+    cmd = f'MYSQL_PWD="$(cat /run/secrets/db_password)" mysql -u root -N -s ghost -e "{sql}"'
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 

tests/ghost/test_backup.py

@@ -15,11 +15,11 @@ from harness import lifecycle  # noqa: E402
 
 
 def _mysql(domain, sql):
-    cmd = 'MYSQL_PWD="$(cat /run/secrets/db_password)" ' f'mysql -u root -N -s ghost -e "{sql}"'
+    cmd = f'MYSQL_PWD="$(cat /run/secrets/db_password)" mysql -u root -N -s ghost -e "{sql}"'
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 
 def test_backup_captures_state(live_app):
-    assert (
+    assert _mysql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _mysql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "the seeded ghost MySQL marker was not present at backup time"
-    ), "the seeded ghost MySQL marker was not present at backup time"
+    )

tests/ghost/test_restore.py

@@ -22,7 +22,7 @@ from harness import lifecycle  # noqa: E402
 
 
 def _mysql(domain, sql):
-    cmd = 'MYSQL_PWD="$(cat /run/secrets/db_password)" ' f'mysql -u root -N -s ghost -e "{sql}"'
+    cmd = f'MYSQL_PWD="$(cat /run/secrets/db_password)" mysql -u root -N -s ghost -e "{sql}"'
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 

tests/ghost/test_upgrade.py

@@ -14,11 +14,11 @@ from harness import lifecycle  # noqa: E402
 
 
 def _mysql(domain, sql):
-    cmd = 'MYSQL_PWD="$(cat /run/secrets/db_password)" ' f'mysql -u root -N -s ghost -e "{sql}"'
+    cmd = f'MYSQL_PWD="$(cat /run/secrets/db_password)" mysql -u root -N -s ghost -e "{sql}"'
     return lifecycle.exec_in_app(domain, ["sh", "-c", cmd], service="db").strip()
 
 
 def test_upgrade_preserves_state(live_app):
-    assert (
+    assert _mysql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives", (
-        _mysql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives"
+        "the seeded ghost MySQL marker did not survive the upgrade redeploy (data loss on upgrade)"
-    ), "the seeded ghost MySQL marker did not survive the upgrade redeploy (data loss on upgrade)"
+    )

tests/gitea/custom/test_lfs_roundtrip.py

@@ -145,9 +145,9 @@ def test_lfs_roundtrip(live_app):
             text=True,
             env={**os.environ, **git_env},
         )
-        assert (
+        assert "testblob.bin" in lfs_ls.stdout, (
-            "testblob.bin" in lfs_ls.stdout
+            f"testblob.bin not in git-lfs ls-files: {lfs_ls.stdout}"
-        ), f"testblob.bin not in git-lfs ls-files: {lfs_ls.stdout}"
+        )
 
         # 6. Download in a FRESH clone (proves the LFS server stores and serves the object)
         fresh_dir = tempfile.mkdtemp(prefix="ccci-gitea-lfs-dl-")
@@ -158,9 +158,9 @@ def test_lfs_roundtrip(live_app):
             with open(fetched_path, "rb") as f:
                 fetched = f.read()
             fetched_sha256 = hashlib.sha256(fetched).hexdigest()
-            assert (
+            assert fetched_sha256 == expected_sha256, (
-                fetched_sha256 == expected_sha256
+                f"LFS round-trip OID mismatch: expected {expected_oid}, got sha256:{fetched_sha256}"
-            ), f"LFS round-trip OID mismatch: expected {expected_oid}, got sha256:{fetched_sha256}"
+            )
         finally:
             shutil.rmtree(fresh_dir, ignore_errors=True)
 
@@ -171,9 +171,9 @@ def test_lfs_roundtrip(live_app):
             ["sh", "-c", "grep -E '^LFS_JWT_SECRET' /etc/gitea/app.ini || echo NOT_FOUND"],
             timeout=30,
         ).strip()
-        assert (
+        assert current_jwt and "NOT_FOUND" not in current_jwt, (
-            current_jwt and "NOT_FOUND" not in current_jwt
+            "Could not read LFS_JWT_SECRET from /etc/gitea/app.ini before restart"
-        ), "Could not read LFS_JWT_SECRET from /etc/gitea/app.ini before restart"
+        )
 
         # Restart the gitea container
         lifecycle.exec_in_app(live_app, ["true"], timeout=5)  # no-op to confirm exec works
@@ -213,9 +213,9 @@ def test_lfs_roundtrip(live_app):
             assert os.path.exists(pr_blob), "testblob.bin not fetched in post-restart clone"
             with open(pr_blob, "rb") as f:
                 pr_data = f.read()
-            assert (
+            assert hashlib.sha256(pr_data).hexdigest() == expected_sha256, (
-                hashlib.sha256(pr_data).hexdigest() == expected_sha256
+                "LFS object corrupted after restart — JWT secret may have changed"
-            ), "LFS object corrupted after restart — JWT secret may have changed"
+            )
         finally:
             shutil.rmtree(post_restart_dir, ignore_errors=True)
 

tests/gitea/ops.py

@@ -220,7 +220,7 @@ def pre_restore(ctx):
     generic.assert_serving(ctx.domain, ctx.meta)
     ok = _delete_marker_repo(ctx.domain, user, password)
     assert ok, f"pre_restore: could not delete {_MARKER_REPO} repo on {ctx.domain}"
-    assert not marker_repo_exists(
+    assert not marker_repo_exists(ctx.domain, user, password), (
-        ctx.domain, user, password
+        f"pre_restore: {_MARKER_REPO} still present after delete — divergence did not take"
-    ), f"pre_restore: {_MARKER_REPO} still present after delete — divergence did not take"
+    )
     print(f"  gitea ops: {_MARKER_REPO!r} deleted (diverged from backup state)", flush=True)

tests/gitea/test_backup.py

@@ -22,6 +22,6 @@ def test_backup_captures_marker_repo(live_app, meta):
     # backupbot cycles the gitea container during backup — wait for it to be back up.
     generic.assert_serving(live_app, meta)
     user, password = admin_creds(live_app)
-    assert marker_repo_exists(
+    assert marker_repo_exists(live_app, user, password), (
-        live_app, user, password
+        f"{live_app}: ci-marker repo is not present at backup time (backup would capture empty state)"
-    ), f"{live_app}: ci-marker repo is not present at backup time (backup would capture empty state)"
+    )

tests/gitea/test_install.py

@@ -65,8 +65,8 @@ def test_install_gitea(live_app, meta):
             )
             page.wait_for_selector("input#user_name", timeout=20_000)
             content = page.content()
-            assert (
+            assert "gitea" in content.lower() or "sign in" in content.lower(), (
-                "gitea" in content.lower() or "sign in" in content.lower()
+                "Sign-in page did not render expected gitea content"
-            ), "Sign-in page did not render expected gitea content"
+            )
         finally:
             browser.close()

tests/gitea/test_upgrade.py

@@ -20,6 +20,6 @@ def test_upgrade_preserves_marker_repo(live_app, meta):
     """The ci-marker repo survived the upgrade to the PR head (data continuity)."""
     generic.assert_serving(live_app, meta)
     user, password = admin_creds(live_app)
-    assert marker_repo_exists(
+    assert marker_repo_exists(live_app, user, password), (
-        live_app, user, password
+        f"{live_app}: ci-marker repo did not survive the upgrade (sqlite3 data lost)"
-    ), f"{live_app}: ci-marker repo did not survive the upgrade (sqlite3 data lost)"
+    )

tests/immich/custom/test_asset_processing.py

@@ -111,13 +111,13 @@ def test_immich_processes_uploaded_asset_metadata_and_statistics(live_app):
         if exif and exif.get("exifImageWidth"):
             break
         time.sleep(5)
-    assert (
+    assert exif and exif.get("exifImageWidth") == 1 and exif.get("exifImageHeight") == 1, (
-        exif and exif.get("exifImageWidth") == 1 and exif.get("exifImageHeight") == 1
+        f"immich metadata-extraction did not populate the 1x1 PNG dimensions in exifInfo: {exif!r}"
-    ), f"immich metadata-extraction did not populate the 1x1 PNG dimensions in exifInfo: {exif!r}"
+    )
 
     # the asset is catalogued into the owner's library statistics (list-back in aggregate)
     sst, stats = harness_http.http_request("GET", f"{base}/api/assets/statistics", headers=auth)
     assert sst == 200 and isinstance(stats, dict), f"statistics HTTP {sst}: {stats!r}"
-    assert (
+    assert stats.get("images", 0) >= 1 and stats.get("total", 0) >= 1, (
-        stats.get("images", 0) >= 1 and stats.get("total", 0) >= 1
+        f"uploaded asset not reflected in library statistics: {stats!r}"
-    ), f"uploaded asset not reflected in library statistics: {stats!r}"
+    )

tests/immich/custom/test_asset_upload.py

@@ -121,6 +121,6 @@ def test_immich_upload_asset_readback_and_thumbnail(live_app):
         if thumb == 200:
             break
         time.sleep(5)
-    assert (
+    assert thumb == 200, (
-        thumb == 200
+        f"immich did not generate a thumbnail/derivative for the uploaded asset (last HTTP {thumb})"
-    ), f"immich did not generate a thumbnail/derivative for the uploaded asset (last HTTP {thumb})"
+    )

tests/immich/test_backup.py

@@ -14,6 +14,6 @@ def _psql(domain, sql):
 
 
 def test_backup_captures_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "seeded postgres state not present at backup time"
-    ), "seeded postgres state not present at backup time"
+    )

tests/immich/test_restore.py

@@ -14,6 +14,6 @@ def _psql(domain, sql):
 
 
 def test_restore_returns_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "restore did not return the pre-mutation postgres state"
-    ), "restore did not return the pre-mutation postgres state"
+    )

tests/immich/test_upgrade.py

@@ -14,6 +14,6 @@ def _psql(domain, sql):
 
 
 def test_upgrade_preserves_data(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives"
+        "postgres data did not survive the upgrade"
-    ), "postgres data did not survive the upgrade"
+    )

tests/keycloak/custom/test_create_client_and_use.py

@@ -144,25 +144,25 @@ def test_create_confidential_client_and_obtain_token(live_app):
 
         # Use the client to obtain its own token (client_credentials grant)
         tok_status, tok_resp = _client_credentials_token(live_app, client_id, client_secret)
-        assert (
+        assert tok_status == 200, (
-            tok_status == 200
+            f"client_credentials token returned HTTP {tok_status}: {tok_resp!r}"
-        ), f"client_credentials token returned HTTP {tok_status}: {tok_resp!r}"
+        )
         access_token = tok_resp.get("access_token") if isinstance(tok_resp, dict) else None
-        assert (
+        assert isinstance(access_token, str) and access_token.count(".") == 2, (
-            isinstance(access_token, str) and access_token.count(".") == 2
+            f"client_credentials access_token not a JWT: {access_token!r}"
-        ), f"client_credentials access_token not a JWT: {access_token!r}"
+        )
 
         # Decode the JWT payload; assert azp matches the new client
         payload = json.loads(_b64url_decode(access_token.split(".")[1]))
-        assert (
+        assert payload.get("azp") == client_id, (
-            payload.get("azp") == client_id
+            f"client_credentials JWT azp={payload.get('azp')!r} != client_id={client_id!r}"
-        ), f"client_credentials JWT azp={payload.get('azp')!r} != client_id={client_id!r}"
+        )
         # Service-account token does NOT carry a session-scoped user (azp + clientId differ from
         # admin-cli token). The presence of azp + iss == per-run-domain proves the issuance flow.
         expected_iss = f"https://{live_app}/realms/master"
-        assert (
+        assert payload.get("iss") == expected_iss, (
-            payload.get("iss") == expected_iss
+            f"JWT iss={payload.get('iss')!r} != {expected_iss!r}"
-        ), f"JWT iss={payload.get('iss')!r} != {expected_iss!r}"
+        )
     finally:
         # Idempotent cleanup
         if cleanup_id:

tests/keycloak/custom/test_password_grant_token.py

@@ -43,17 +43,17 @@ def test_password_grant_issues_valid_jwt(live_app):
     token = kc_admin.admin_token(live_app, password)
 
     # Shape: a JWT is exactly 3 base64url segments
-    assert (
+    assert isinstance(token, str) and token.count(".") == 2, (
-        isinstance(token, str) and token.count(".") == 2
+        f"access_token does not look like a JWT (no 3 segments): len={len(token) if token else 0}"
-    ), f"access_token does not look like a JWT (no 3 segments): len={len(token) if token else 0}"
+    )
 
     payload = _decode_jwt_payload(token)
 
     # iss = the issuer URL, must be the per-run domain's /realms/master endpoint
     expected_iss = f"https://{live_app}/realms/master"
-    assert (
+    assert payload.get("iss") == expected_iss, (
-        payload.get("iss") == expected_iss
+        f"JWT iss claim {payload.get('iss')!r} != {expected_iss!r}"
-    ), f"JWT iss claim {payload.get('iss')!r} != {expected_iss!r}"
+    )
 
     # azp = authorized party (which client requested this token)
     assert payload.get("azp") == "admin-cli", f"JWT azp claim {payload.get('azp')!r} != 'admin-cli'"
@@ -68,6 +68,6 @@ def test_password_grant_issues_valid_jwt(live_app):
 
     # iat (issued at) is also a standard claim
     iat = payload.get("iat")
-    assert (
+    assert isinstance(iat, int) and iat <= time.time() + 60, (
-        isinstance(iat, int) and iat <= time.time() + 60
+        f"JWT iat {iat!r} not a reasonable past timestamp"
-    ), f"JWT iat {iat!r} not a reasonable past timestamp"
+    )

tests/lasuite-docs/custom/test_oidc_with_keycloak.py

@@ -42,9 +42,9 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps):
     # Sanity-check the creds shape — orchestrator-written
     assert kc["domain"]
     # WC1: realm is per-run namespaced "<parent>-<6hex>" so concurrent dependents never collide.
-    assert re.fullmatch(
+    assert re.fullmatch(r"lasuite-docs-[0-9a-f]{6}", kc["realm"]), (
-        r"lasuite-docs-[0-9a-f]{6}", kc["realm"]
+        f"realm {kc['realm']!r} not the per-run namespaced form lasuite-docs-<6hex>"
-    ), f"realm {kc['realm']!r} not the per-run namespaced form lasuite-docs-<6hex>"
+    )
     assert kc["client_id"] == "lasuite-docs"
     assert isinstance(kc["client_secret"], str) and len(kc["client_secret"]) >= 16
     assert isinstance(kc["password"], str) and len(kc["password"]) >= 16
@@ -77,11 +77,11 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps):
     assert isinstance(token, str) and token.count(".") == 2, f"access_token is not a JWT: {token!r}"
     payload = json.loads(_b64url_decode(token.split(".")[1]))
     assert payload.get("iss") == expected_iss, f"JWT iss={payload.get('iss')!r} != {expected_iss!r}"
-    assert (
+    assert payload.get("azp") == kc["client_id"], (
-        payload.get("azp") == kc["client_id"]
+        f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
-    ), f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
+    )
     assert payload.get("typ") == "Bearer", f"JWT typ={payload.get('typ')!r} != 'Bearer'"
     exp = payload.get("exp")
-    assert (
+    assert isinstance(exp, int) and exp > time.time(), (
-        isinstance(exp, int) and exp > time.time()
+        f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"
-    ), f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"
+    )

tests/lasuite-docs/test_backup.py

@@ -18,6 +18,6 @@ def _psql(domain, sql):
 
 
 def test_backup_captures_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "the seeded postgres state was not present at backup time"
-    ), "the seeded postgres state was not present at backup time"
+    )

tests/lasuite-docs/test_restore.py

@@ -17,6 +17,6 @@ def _psql(domain, sql):
 
 
 def test_restore_returns_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "restore did not return the pre-mutation postgres state"
-    ), "restore did not return the pre-mutation postgres state"
+    )

tests/lasuite-docs/test_upgrade.py

@@ -17,6 +17,6 @@ def _psql(domain, sql):
 
 
 def test_upgrade_preserves_data(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives"
+        "postgres data did not survive the upgrade"
-    ), "postgres data did not survive the upgrade"
+    )

tests/lasuite-drive/custom/test_minio_storage.py

@@ -56,6 +56,6 @@ def test_minio_bucket_present_and_object_roundtrip(live_app):
 
     # The object was listed (its key appears) and its content round-tripped intact.
     assert f"{marker}.txt" in out, f"uploaded object not listed in bucket: {out!r}"
-    assert (
+    assert f"READBACK:{marker}" in out, (
-        f"READBACK:{marker}" in out
+        f"object content did not round-trip through MinIO; got: {out!r}"
-    ), f"object content did not round-trip through MinIO; got: {out!r}"
+    )

tests/lasuite-drive/custom/test_oidc_with_keycloak.py

@@ -46,9 +46,9 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps):
 
     # Creds shape. WC1: realm is per-run namespaced "<parent>-<6hex>"; client_id stays the parent.
     assert kc["domain"]
-    assert re.fullmatch(
+    assert re.fullmatch(r"lasuite-drive-[0-9a-f]{6}", kc["realm"]), (
-        r"lasuite-drive-[0-9a-f]{6}", kc["realm"]
+        f"realm {kc['realm']!r} not the per-run namespaced form lasuite-drive-<6hex>"
-    ), f"realm {kc['realm']!r} not the per-run namespaced form lasuite-drive-<6hex>"
+    )
     assert kc["client_id"] == "lasuite-drive"
     assert isinstance(kc["client_secret"], str) and len(kc["client_secret"]) >= 16
     assert isinstance(kc["password"], str) and len(kc["password"]) >= 16
@@ -80,11 +80,11 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps):
     assert isinstance(token, str) and token.count(".") == 2, f"access_token is not a JWT: {token!r}"
     payload = json.loads(_b64url_decode(token.split(".")[1]))
     assert payload.get("iss") == expected_iss, f"JWT iss={payload.get('iss')!r} != {expected_iss!r}"
-    assert (
+    assert payload.get("azp") == kc["client_id"], (
-        payload.get("azp") == kc["client_id"]
+        f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
-    ), f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
+    )
     assert payload.get("typ") == "Bearer", f"JWT typ={payload.get('typ')!r} != 'Bearer'"
     exp = payload.get("exp")
-    assert (
+    assert isinstance(exp, int) and exp > time.time(), (
-        isinstance(exp, int) and exp > time.time()
+        f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"
-    ), f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"
+    )

tests/lasuite-drive/test_backup.py

@@ -18,6 +18,6 @@ def _psql(domain, sql):
 
 
 def test_backup_captures_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "the seeded postgres state was not present at backup time"
-    ), "the seeded postgres state was not present at backup time"
+    )

tests/lasuite-drive/test_restore.py

@@ -17,6 +17,6 @@ def _psql(domain, sql):
 
 
 def test_restore_returns_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "restore did not return the pre-mutation postgres state"
-    ), "restore did not return the pre-mutation postgres state"
+    )

tests/lasuite-drive/test_upgrade.py

@@ -17,6 +17,6 @@ def _psql(domain, sql):
 
 
 def test_upgrade_preserves_data(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives"
+        "postgres data did not survive the upgrade"
-    ), "postgres data did not survive the upgrade"
+    )

tests/lasuite-meet/custom/_oidc_session.py

@@ -0,0 +1,160 @@
+"""Recipe-local OIDC *session* login helper (authorization-code flow + session cookie).
+
+meet v1.22.0 hardened API auth — raw OIDC user access tokens are rejected as Bearer
+credentials; the app accepts only its own session cookie, established through the standard OIDC
+authorization-code browser flow (app login URL → keycloak login form → callback → Django
+session). This helper drives that flow with urllib + a CookieJar so the custom tests can
+exercise the API the way a real client does.
+
+Kept recipe-local (cf. tests/ghost/custom/_ghost.py precedent; same helper as
+tests/lasuite-docs/custom/_oidc_session.py) rather than in runner/harness — promote it there
+if a third recipe needs it.
+
+Usage:
+    sess = OidcSession(f"https://{live_app}")
+    me = sess.login(kc["user"], kc["password"])   # asserts whoami 200; returns the user dict
+    status, body = sess.post("/api/v1.0/rooms/", {"name": "x"})   # CSRF handled
+"""
+
+from __future__ import annotations
+
+import contextlib
+import html
+import http.cookiejar
+import json
+import re
+import ssl
+import urllib.error
+import urllib.parse
+import urllib.request
+
+# Per-run *.ci.commoninternet.net domains serve the operator's wildcard cert via the Traefik file
+# provider; chain verification is done once in the install tier (generic.served_cert).
+_CTX = ssl.create_default_context()
+_CTX.check_hostname = False
+_CTX.verify_mode = ssl.CERT_NONE
+
+_LOGIN_PATHS = ("/api/v1.0/authenticate/", "/oidc/authenticate/", "/api/v1.0/users/me/")
+_WHOAMI = "/api/v1.0/users/me/"
+
+
+class OidcSession:
+    """A cookie-carrying HTTP session logged in via the app's OIDC authorization-code flow."""
+
+    def __init__(self, base: str):
+        self.base = base.rstrip("/")
+        self.jar = http.cookiejar.CookieJar()
+        self.opener = urllib.request.build_opener(
+            urllib.request.HTTPCookieProcessor(self.jar),
+            urllib.request.HTTPSHandler(context=_CTX),
+        )
+
+    # -- low-level ---------------------------------------------------------------------------
+
+    def _open(
+        self,
+        url: str,
+        data: bytes | None = None,
+        headers: dict[str, str] | None = None,
+        method: str | None = None,
+        timeout: int = 30,
+    ) -> tuple[int, str, bytes]:
+        """Open a URL (following redirects, carrying cookies). Returns (status, final_url, body)."""
+        req = urllib.request.Request(url, data=data, method=method)
+        for k, v in (headers or {}).items():
+            req.add_header(k, v)
+        try:
+            with self.opener.open(req, timeout=timeout) as resp:
+                return resp.getcode(), resp.geturl(), resp.read()
+        except urllib.error.HTTPError as e:
+            body = b""
+            with contextlib.suppress(Exception):
+                body = e.read()
+            return e.code, e.filename or url, body
+
+    def _csrf_token(self) -> str | None:
+        for c in self.jar:
+            if "csrftoken" in c.name.lower():
+                return c.value
+        return None
+
+    # -- login -------------------------------------------------------------------------------
+
+    def login(
+        self,
+        username: str,
+        password: str,
+        login_paths: tuple[str, ...] = _LOGIN_PATHS,
+        whoami: str = _WHOAMI,
+    ) -> dict:
+        """OIDC authorization-code login: app → keycloak form → callback → session cookie.
+
+        Asserts the resulting session GETs `whoami` with HTTP 200 and returns the parsed user.
+        """
+        page, page_url, last = None, None, (0, "", b"")
+        for path in login_paths:
+            status, final_url, body = self._open(self.base + path)
+            last = (status, final_url, body)
+            text = body.decode(errors="replace")
+            if "kc-form-login" in text or (
+                "/protocol/openid-connect/" in final_url and "<form" in text
+            ):
+                page, page_url = text, final_url
+                break
+        assert page is not None, (
+            f"could not reach the keycloak login form via {login_paths}: last URL "
+            f"{last[1]!r} HTTP {last[0]} body[:200]={last[2][:200]!r}"
+        )
+
+        m = re.search(r'<form[^>]*id="kc-form-login"[^>]*action="([^"]+)"', page) or re.search(
+            r'<form[^>]*action="([^"]+)"[^>]*method=["\']?post', page, re.I
+        )
+        assert m, f"no login form action on keycloak page {page_url!r}: {page[:300]!r}"
+        action = html.unescape(m.group(1))
+
+        form = urllib.parse.urlencode(
+            {"username": username, "password": password, "credentialId": ""}
+        ).encode()
+        status, landed, body = self._open(
+            action, data=form, headers={"Content-Type": "application/x-www-form-urlencoded"}
+        )
+
+        status, _, who = self._open(self.base + whoami)
+        assert status == 200, (
+            f"OIDC session login failed: GET {whoami} -> HTTP {status} after submitting the "
+            f"keycloak form (landed at {landed!r}; excerpt: {body[:200]!r})"
+        )
+        parsed = json.loads(who)
+        assert isinstance(parsed, dict), f"unexpected whoami payload: {who[:200]!r}"
+        return parsed
+
+    # -- API calls with the session ----------------------------------------------------------
+
+    def request(self, method: str, path: str, data: dict | None = None) -> tuple[int, object]:
+        """Issue an API call with the session cookie (+ CSRF header on unsafe methods)."""
+        url = path if path.startswith("http") else self.base + path
+        headers: dict[str, str] = {}
+        body: bytes | None = None
+        if data is not None:
+            body = json.dumps(data).encode()
+            headers["Content-Type"] = "application/json"
+        if method.upper() not in ("GET", "HEAD", "OPTIONS"):
+            tok = self._csrf_token()
+            if tok:
+                headers["X-CSRFToken"] = tok
+            headers["Referer"] = self.base + "/"
+            headers["Origin"] = self.base
+        status, _, raw = self._open(url, data=body, headers=headers, method=method.upper())
+        try:
+            return status, json.loads(raw)
+        except (json.JSONDecodeError, ValueError):
+            return status, None
+
+    def get(self, path: str) -> tuple[int, object]:
+        return self.request("GET", path)
+
+    def post(self, path: str, data: dict | None = None) -> tuple[int, object]:
+        return self.request("POST", path, data)
+
+    def delete(self, path: str) -> tuple[int, object]:
+        return self.request("DELETE", path)

tests/lasuite-meet/custom/test_meeting_flow.py

@@ -5,8 +5,14 @@ SOURCE: references/recipe-maintainer/recipe-info/lasuite-meet/tests/meeting_flow
 
 Meet's characteristic behavior is real-time meetings: a user creates a room and receives a LiveKit
 (SFU) join token for WebSocket signaling. This is the §4.3 create-an-object + read-it-back, plus the
-distinctive WebRTC-signaling feature (LiveKit token issuance) — not a health/200 stand-in. Flow:
+distinctive WebRTC-signaling feature (LiveKit token issuance) — not a health/200 stand-in.
-  1. OIDC password grant (the per-run keycloak user) → a Meet API bearer token.
+
+Updated for meet v1.22.0+ API auth hardening: the API now REJECTS raw OIDC user access tokens
+sent as Bearer credentials; authenticated calls run on the app's session cookie from the real
+OIDC authorization-code login (see _oidc_session.py). Flow:
+  1. OIDC password grant (the per-run keycloak user) → assert the API rejects it as Bearer
+     (the v1.22.0 hardening — a 2xx here would mean the hardening regressed); then log in via
+     the OIDC authorization-code flow → session cookie.
   2. POST /api/v1.0/rooms/ {name, access_level:public} → 201 with id/slug AND a LiveKit room+token.
   3. GET  /api/v1.0/rooms/{id}/ (read-it-back) → 200, again with a LiveKit token for the same room.
   4. Assert the LiveKit token is a real JWT carrying a video grant for that room (token issuance —
@@ -27,9 +33,11 @@ import sys
 
 import pytest
 
+sys.path.insert(0, os.path.dirname(__file__))
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "..", "runner"))
+from _oidc_session import OidcSession  # noqa: E402  (recipe-local helper, same dir)
 from harness import http as harness_http  # noqa: E402
-from harness import sso
+from harness import sso  # noqa: E402
 
 
 def _b64url(seg: str) -> bytes:
@@ -57,17 +65,28 @@ def _creds(deps: dict) -> dict:
 @pytest.mark.requires_deps
 def test_create_room_get_livekit_token_and_read_back(live_app, deps):
     assert "keycloak" in deps, f"keycloak creds missing; got {list(deps.keys())}"
+    kc = deps["keycloak"]
     base = f"https://{live_app}"
+
+    # meet v1.22.0+ hardening: a raw OIDC user access token must be REJECTED as Bearer.
     token = sso.oidc_password_grant(_creds(deps))
     assert isinstance(token, str) and token.count(".") == 2, "OIDC access token is not a JWT"
-    auth = {"Authorization": f"Bearer {token}"}
-
-    # --- create a room (the object) ---
     status, body = harness_http.http_post(
         f"{base}/api/v1.0/rooms/",
         data={"name": "ccci-meeting", "access_level": "public"},
-        headers=auth,
+        headers={"Authorization": f"Bearer {token}"},
     )
+    assert status in (401, 403), (
+        f"POST /api/v1.0/rooms/ with a raw OIDC Bearer token returned HTTP {status} — meet >= "
+        f"v1.22.0 must reject user access tokens on the API (body {body!r})"
+    )
+
+    # The successor auth path: session cookie via the real OIDC authorization-code login.
+    sess = OidcSession(base)
+    sess.login(kc["user"], kc["password"])
+
+    # --- create a room (the object) ---
+    status, body = sess.post("/api/v1.0/rooms/", {"name": "ccci-meeting", "access_level": "public"})
     assert status == 201, f"room create returned HTTP {status} (expected 201); body={body!r}"
     assert isinstance(body, dict), f"room create body not JSON: {body!r}"
     room_id = body.get("id")
@@ -75,36 +94,32 @@ def test_create_room_get_livekit_token_and_read_back(live_app, deps):
     lk_room = livekit.get("room")
     lk_token = livekit.get("token")
     assert room_id, f"room created but no id: {body!r}"
-    assert (
+    assert lk_token and isinstance(lk_token, str) and lk_token.count(".") == 2, (
-        lk_token and isinstance(lk_token, str) and lk_token.count(".") == 2
+        f"room created but no LiveKit JWT token: {livekit!r}"
-    ), f"room created but no LiveKit JWT token: {livekit!r}"
+    )
 
     try:
         # --- read it back (a fresh authenticated GET of the created room) ---
-        status, got = harness_http.http_request(
+        status, got = sess.get(f"/api/v1.0/rooms/{room_id}/")
-            "GET", f"{base}/api/v1.0/rooms/{room_id}/", headers=auth
-        )
         assert status == 200, f"room read-back returned HTTP {status} (expected 200); body={got!r}"
-        assert (
+        assert isinstance(got, dict) and got.get("id") == room_id, (
-            isinstance(got, dict) and got.get("id") == room_id
+            f"read-back room id mismatch: {got!r}"
-        ), f"read-back room id mismatch: {got!r}"
+        )
         got_lk = got.get("livekit") or {}
         assert got_lk.get("token"), f"read-back room missing LiveKit token: {got!r}"
-        assert (
+        assert got_lk.get("room") == lk_room, (
-            got_lk.get("room") == lk_room
+            f"read-back LiveKit room {got_lk.get('room')!r} != create-time {lk_room!r}"
-        ), f"read-back LiveKit room {got_lk.get('room')!r} != create-time {lk_room!r}"
+        )
 
         # --- the LiveKit token is a real signaling grant for this room (WebRTC subset) ---
         payload = json.loads(_b64url(lk_token.split(".")[1]))
         video = payload.get("video") or {}
-        assert (
+        assert video.get("room") == lk_room or payload.get("room") == lk_room, (
-            video.get("room") == lk_room or payload.get("room") == lk_room
+            f"LiveKit JWT does not grant the created room {lk_room!r}: {payload!r}"
-        ), f"LiveKit JWT does not grant the created room {lk_room!r}: {payload!r}"
+        )
     finally:
         # --- delete the room (cleanup + a real DELETE mutation) ---
-        del_status, _ = harness_http.http_request(
+        del_status, _ = sess.delete(f"/api/v1.0/rooms/{room_id}/")
-            "DELETE", f"{base}/api/v1.0/rooms/{room_id}/", headers=auth
-        )
         assert del_status in (
             204,
             200,
@@ -120,9 +135,7 @@ def test_create_room_get_livekit_token_and_read_back(live_app, deps):
 
     gone = False
     for _ in range(5):
-        status, _ = harness_http.http_request(
+        status, _ = sess.get(f"/api/v1.0/rooms/{room_id}/")
-            "GET", f"{base}/api/v1.0/rooms/{room_id}/", headers=auth
-        )
         if status == 404:
             gone = True
             break

tests/lasuite-meet/custom/test_oidc_with_keycloak.py

@@ -46,9 +46,9 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps):
 
     # Creds shape. WC1: realm is per-run namespaced "<parent>-<6hex>"; client_id stays the parent.
     assert kc["domain"]
-    assert re.fullmatch(
+    assert re.fullmatch(r"lasuite-meet-[0-9a-f]{6}", kc["realm"]), (
-        r"lasuite-meet-[0-9a-f]{6}", kc["realm"]
+        f"realm {kc['realm']!r} not the per-run namespaced form lasuite-meet-<6hex>"
-    ), f"realm {kc['realm']!r} not the per-run namespaced form lasuite-meet-<6hex>"
+    )
     assert kc["client_id"] == "lasuite-meet"
     assert isinstance(kc["client_secret"], str) and len(kc["client_secret"]) >= 16
     assert isinstance(kc["password"], str) and len(kc["password"]) >= 16
@@ -80,11 +80,11 @@ def test_oidc_password_grant_against_dep_keycloak(live_app, deps):
     assert isinstance(token, str) and token.count(".") == 2, f"access_token is not a JWT: {token!r}"
     payload = json.loads(_b64url_decode(token.split(".")[1]))
     assert payload.get("iss") == expected_iss, f"JWT iss={payload.get('iss')!r} != {expected_iss!r}"
-    assert (
+    assert payload.get("azp") == kc["client_id"], (
-        payload.get("azp") == kc["client_id"]
+        f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
-    ), f"JWT azp={payload.get('azp')!r} != {kc['client_id']!r}"
+    )
     assert payload.get("typ") == "Bearer", f"JWT typ={payload.get('typ')!r} != 'Bearer'"
     exp = payload.get("exp")
-    assert (
+    assert isinstance(exp, int) and exp > time.time(), (
-        isinstance(exp, int) and exp > time.time()
+        f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"
-    ), f"JWT exp={exp!r} not a future timestamp (now={time.time():.0f})"
+    )

tests/lasuite-meet/test_backup.py

@@ -17,6 +17,6 @@ def _psql(domain, sql):
 
 
 def test_backup_captures_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "the seeded postgres state was not present at backup time"
-    ), "the seeded postgres state was not present at backup time"
+    )

tests/lasuite-meet/test_restore.py

@@ -17,6 +17,6 @@ def _psql(domain, sql):
 
 
 def test_restore_returns_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "restore did not return the pre-mutation postgres state"
-    ), "restore did not return the pre-mutation postgres state"
+    )

tests/lasuite-meet/test_upgrade.py

@@ -17,6 +17,6 @@ def _psql(domain, sql):
 
 
 def test_upgrade_preserves_data(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives"
+        "postgres data did not survive the upgrade"
-    ), "postgres data did not survive the upgrade"
+    )

tests/mailu/custom/test_mail_flow.py

@@ -43,7 +43,7 @@ def test_send_and_receive_mail(live_app):
     deadline = time.time() + 150
     while time.time() < deadline:
         for box in ("INBOX", "Junk"):
-            query = f"doveadm search -u '{email_addr}' mailbox {box} " f"header subject '{marker}'"
+            query = f"doveadm search -u '{email_addr}' mailbox {box} header subject '{marker}'"
             out = lifecycle.exec_in_app(live_app, ["sh", "-c", query], service="imap")
             if out.strip():  # a non-empty result = "<mailbox-guid> <uid>" → message stored
                 return

tests/mailu/custom/test_mailbox.py

@@ -24,6 +24,6 @@ def test_create_mailbox_and_read_back(live_app):
 
     cfg = _mailu.config_export(live_app)
     emails = _mailu.user_emails(cfg)
-    assert (
+    assert email in emails, (
-        email in emails
+        f"created mailbox {email} not present in mailu config-export users {emails}"
-    ), f"created mailbox {email} not present in mailu config-export users {emails}"
+    )

tests/matrix-synapse/custom/test_federation_version.py

@@ -34,12 +34,12 @@ def test_federation_version_endpoint(live_app):
     assert status == 200, f"GET {url} HTTP {status} (expected 200)"
     assert isinstance(body, dict), f"federation version returned non-dict: {type(body).__name__}"
     server = body.get("server")
-    assert isinstance(
+    assert isinstance(server, dict), (
-        server, dict
+        f"federation version response missing 'server' envelope: {body!r}"
-    ), f"federation version response missing 'server' envelope: {body!r}"
+    )
     name = server.get("name")
     assert name == "Synapse", f"server.name={name!r}, expected 'Synapse'"
     version = server.get("version")
-    assert (
+    assert isinstance(version, str) and len(version) > 0, (
-        isinstance(version, str) and len(version) > 0
+        f"server.version is not a non-empty string: {version!r}"
-    ), f"server.version is not a non-empty string: {version!r}"
+    )

tests/matrix-synapse/custom/test_health_check.py

@@ -23,6 +23,6 @@ def test_synapse_client_versions_returns_json(live_app):
     url = f"https://{live_app}/_matrix/client/versions"
     status, body = harness_http.retry_http_get(url, expect_status=200, max_wait=60, interval=3)
     assert status == 200, f"GET {url} HTTP {status} (expected 200)"
-    assert (
+    assert isinstance(body, dict) and isinstance(body.get("versions"), list) and body["versions"], (
-        isinstance(body, dict) and isinstance(body.get("versions"), list) and body["versions"]
+        f"GET {url} did not return Matrix client-versions document: {body!r}"
-    ), f"GET {url} did not return Matrix client-versions document: {body!r}"
+    )

tests/matrix-synapse/custom/test_register_and_message.py

@@ -127,8 +127,7 @@ def _admin_register(domain: str, secret: str, username: str, password: str, admi
         if r["status"] == 200:
             if attempt > 1:
                 print(
-                    f"  [register] {username}: succeeded on attempt {attempt} "
+                    f"  [register] {username}: succeeded on attempt {attempt} (synapse recovered)",
-                    f"(synapse recovered)",
                     flush=True,
                 )
             return r["body"] or {}
@@ -177,9 +176,9 @@ def test_register_two_users_send_receive_message(live_app):
     create + invite + join a room; send and read a message."""
     domain = live_app
     secret = _registration_secret(domain)
-    assert (
+    assert secret and len(secret) >= 16, (
-        secret and len(secret) >= 16
+        f"registration shared secret missing/short: len={len(secret) if secret else 0}"
-    ), f"registration shared secret missing/short: len={len(secret) if secret else 0}"
+    )
 
     suffix = uuid.uuid4().hex[:8]
     user_a = f"alice{suffix}"

tests/matrix-synapse/test_backup.py

@@ -18,6 +18,6 @@ def _psql(domain, sql):
 
 
 def test_backup_captures_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "the seeded postgres state was not present at backup time"
-    ), "the seeded postgres state was not present at backup time"
+    )

tests/matrix-synapse/test_install.py

@@ -26,6 +26,6 @@ def test_serving_and_client_api(live_app, meta):
     # The client-API version document is real synapse JSON (proves the app, not just a proxy 200).
     body = lifecycle.http_body(live_app, "/_matrix/client/versions")
     doc = json.loads(body)
-    assert (
+    assert isinstance(doc.get("versions"), list) and doc["versions"], (
-        isinstance(doc.get("versions"), list) and doc["versions"]
+        "no matrix client versions advertised"
-    ), "no matrix client versions advertised"
+    )

tests/matrix-synapse/test_restore.py

@@ -17,6 +17,6 @@ def _psql(domain, sql):
 
 
 def test_restore_returns_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "restore did not return the pre-mutation postgres state"
-    ), "restore did not return the pre-mutation postgres state"
+    )

tests/matrix-synapse/test_upgrade.py

@@ -17,6 +17,6 @@ def _psql(domain, sql):
 
 
 def test_upgrade_preserves_data(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives"
+        "postgres data did not survive the upgrade"
-    ), "postgres data did not survive the upgrade"
+    )

tests/mattermost-lts/custom/test_create_message.py

@@ -41,9 +41,9 @@ def test_create_message_roundtrip(live_app):
         headers=auth,
         timeout=30,
     )
-    assert (
+    assert status in (200, 201) and isinstance(team, dict) and team.get("id"), (
-        status in (200, 201) and isinstance(team, dict) and team.get("id")
+        f"team creation failed: HTTP {status}, body={team!r}"
-    ), f"team creation failed: HTTP {status}, body={team!r}"
+    )
     status, chan = harness_http.http_post(
         f"{base}/channels",
         data={
@@ -55,9 +55,9 @@ def test_create_message_roundtrip(live_app):
         headers=auth,
         timeout=30,
     )
-    assert (
+    assert status in (200, 201) and isinstance(chan, dict) and chan.get("id"), (
-        status in (200, 201) and isinstance(chan, dict) and chan.get("id")
+        f"channel creation failed: HTTP {status}, body={chan!r}"
-    ), f"channel creation failed: HTTP {status}, body={chan!r}"
+    )
 
     # 4) POST a unique marker message.
     marker = f"ccci-marker-{uniq}-roundtrip"
@@ -67,13 +67,13 @@ def test_create_message_roundtrip(live_app):
         headers=auth,
         timeout=30,
     )
-    assert (
+    assert status in (200, 201) and isinstance(post, dict) and post.get("id"), (
-        status in (200, 201) and isinstance(post, dict) and post.get("id")
+        f"post creation failed: HTTP {status}, body={post!r}"
-    ), f"post creation failed: HTTP {status}, body={post!r}"
+    )
 
     # 5) Read it back by id and assert the message survived the round-trip.
     status, got = harness_http.http_get(f"{base}/posts/{post['id']}", headers=auth, timeout=30)
     assert status == 200 and isinstance(got, dict), f"read-back failed: HTTP {status}, body={got!r}"
-    assert (
+    assert got.get("message") == marker, (
-        got.get("message") == marker
+        f"message did not round-trip: sent {marker!r}, got {got.get('message')!r}"
-    ), f"message did not round-trip: sent {marker!r}, got {got.get('message')!r}"
+    )

tests/mattermost-lts/custom/test_health_check.py

@@ -28,6 +28,6 @@ def test_system_ping_ok(live_app):
     url = f"https://{live_app}/api/v4/system/ping"
     status, body = harness_http.retry_http_get(url, expect_status=200, max_wait=120, interval=3)
     assert status == 200, f"GET {url} HTTP {status} (expected 200)"
-    assert (
+    assert isinstance(body, dict) and body.get("status") == "OK", (
-        isinstance(body, dict) and body.get("status") == "OK"
+        f"/api/v4/system/ping did not report status=OK; got {body!r}"
-    ), f"/api/v4/system/ping did not report status=OK; got {body!r}"
+    )

tests/mattermost-lts/custom/test_multiuser_message.py

@@ -105,6 +105,6 @@ def test_second_user_reads_first_users_message(live_app):
 
     # 5) user_b sees user_a's marker (cross-user delivery, not a self read-back)
     messages = [p.get("message") for p in (posts.get("posts") or {}).values()]
-    assert (
+    assert marker in messages, (
-        marker in messages
+        f"user_b did not see user_a's message {marker!r} in the channel; saw {messages!r}"
-    ), f"user_b did not see user_a's message {marker!r} in the channel; saw {messages!r}"
+    )

tests/mattermost-lts/test_backup.py

@@ -18,6 +18,6 @@ def _psql(domain, sql):
 
 
 def test_backup_captures_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "the seeded postgres state was not present at backup time"
-    ), "the seeded postgres state was not present at backup time"
+    )

tests/mattermost-lts/test_restore.py

@@ -19,6 +19,6 @@ def _psql(domain, sql):
 
 
 def test_restore_returns_state(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "original", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "original"
+        "restore did not return the pre-mutation postgres state"
-    ), "restore did not return the pre-mutation postgres state"
+    )

tests/mattermost-lts/test_upgrade.py

@@ -18,6 +18,6 @@ def _psql(domain, sql):
 
 
 def test_upgrade_preserves_data(live_app):
-    assert (
+    assert _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives", (
-        _psql(live_app, "SELECT v FROM ci_marker;") == "upgrade-survives"
+        "postgres data did not survive the upgrade"
-    ), "postgres data did not survive the upgrade"
+    )