AGENTS.md: ship work as PRs, self-merge, operator reviews retrospectively

Operator policy (2026-08-11), matching cc-ci-orchestrator. Branch, PR, merge once
verified, operator reviews after. The PR is not a gate - it is how the work stays
legible - so the description carries what changed, why, and the evidence.

Recipe repos are explicitly excluded: created and verified, never agent-merged.

AGENTS.md

@@ -36,3 +36,19 @@ Two kinds of tests live here — run them on **different** cadences:
 
 A red test is information. Never skip, delete, or relax a test to make a run green — fix the root
 cause or record it in `machine-docs/DEFERRED.md`. (This is a standing build guardrail.)
+
+## Ship work as PRs, merge them yourself, operator reviews retrospectively
+
+Work on this repo goes: **branch → PR → merge it yourself once verified → operator reviews
+retrospectively.** Do not commit straight to `main`, and do not wait for review before merging — the
+invocation is the authorization, and blocking would stall the CI this repo runs.
+
+The PR is therefore not a gate; it is how the work stays legible after the fact. Write the
+description to be read later: what changed, why, and the evidence it works (harness output, a
+verified run, a before/after number). A PR that says "fix test" has failed at its only job.
+
+The same policy covers `recipe-maintainers/cc-ci-orchestrator`. It does **NOT** cover recipe repos —
+any `coop-cloud/<recipe>` or its mirror is created and verified but **never agent-merged**, because
+those change what deploys on other people's infrastructure.
+
+Before editing a test, read `tests/STYLE.md`.

opencode.json

@@ -0,0 +1,8 @@
+{
+  "$schema": "https://opencode.ai/config.json",
+  "agent": {
+    "general": {
+      "model": "opencode/deepseek-v4-pro"
+    }
+  }
+}

runner/harness/meta.py

@@ -132,6 +132,17 @@ KEYS: tuple[Key, ...] = (
         "Callable `(ctx)` invoked after UPGRADE_EXTRA_ENV env_set but before `abra secret generate --all` in the upgrade path. Use to pre-insert secrets that `generate --all` would produce with wrong format (e.g. when the .env.sample spec is commented out).",
         hook_params=("ctx",),
     ),
+    Key(
+        "UPGRADE_BASE_FLOOR",
+        "str",
+        None,
+        "Declared STRUCTURAL breaking boundary for the upgrade tier (phase basefloor): the first "
+        "post-break published version tag. Bases strictly below it are excluded from resolution "
+        "(canonical / step-back / no-canonical fallback) because an in-place upgrade across the "
+        "boundary is not supported upstream (e.g. a db-family change). When no ≥-floor predecessor "
+        "exists the tier records a DECLARED skip. NOT a static base pin (§2.G stays removed) — "
+        "resolution remains dynamic above the floor.",
+    ),
     # (CHAOS_BASE_DEPLOY, OIDC_AT_INSTALL and SKIP_GENERIC were deleted in restructure P2:
     # compose.ccci.yml is first-class + auto-chaos; install-time deps wiring is the only mode;
     # the generic floor is suppressible only via the dev-only CCCI_SKIP_GENERIC* env form.)

runner/run_recipe_ci.py

@@ -151,8 +151,30 @@ def resolve_upgrade_base(
             flush=True,
         )
         return BasePlan("skip", None, None, f"declared EXPECTED_NA[upgrade]: {declared}")
+    # UPGRADE_BASE_FLOOR (phase basefloor): a recipe_meta declaration marking a STRUCTURAL breaking
+    # boundary — published versions strictly below the floor are not valid in-place upgrade sources
+    # (e.g. discourse 0.8.x→1.0.0 changed the db family bitnami/pgvector → discourse/postgres; the
+    # data layout+roles are incompatible, upstream supports no in-place path across it). This is NOT
+    # the removed UPGRADE_BASE_VERSION pin (§2.G): resolution stays fully dynamic — the floor only
+    # EXCLUDES structurally-impossible bases, and when no candidate ≥ floor exists the tier records
+    # a DECLARED skip (never a silent pass). Never weakens: below-floor upgrades were never a
+    # supported path, so no real coverage is lost.
+    floor = getattr(meta, "UPGRADE_BASE_FLOOR", None)
+
+    def _below_floor(version: str) -> bool:
+        return bool(floor) and warm_reconcile.version_key(version) < warm_reconcile.version_key(
+            floor
+        )
+
     skip_canonicals = settings_mod.get().skip_canonicals_for_upgrade
     rec = canonical.read_registry(recipe)
+    if rec and rec.get("version") and not skip_canonicals and _below_floor(rec["version"]):
+        print(
+            f"== upgrade tier: last-green canonical {rec['version']} is below the declared "
+            f"UPGRADE_BASE_FLOOR {floor} (structural break) — excluded as a base",
+            flush=True,
+        )
+        rec = None
     if rec and rec.get("version") and not skip_canonicals:
         canon = rec["version"]
         same = head_version is not None and warm_reconcile.version_key(
@@ -168,10 +190,20 @@ def resolve_upgrade_base(
                 f"last-green (warm canonical, status={rec.get('status')})",
             )
         # canonical == head version → deploying it would be a same-version no-op. Step back to the
-        # newest published version strictly older than the head (phase samever).
-        older = warm_reconcile.newest_older_version(
-            warm_reconcile.recipe_tags(recipe), head_version
-        )
+        # newest published version strictly older than the head (phase samever). Candidates below a
+        # declared UPGRADE_BASE_FLOOR are excluded (phase basefloor — structurally invalid bases).
+        _tags = warm_reconcile.recipe_tags(recipe)
+        if floor:
+            _tags = [t for t in _tags if not _below_floor(t)]
+        older = warm_reconcile.newest_older_version(_tags, head_version)
+        if older is None and floor:
+            return BasePlan(
+                "skip",
+                None,
+                None,
+                f"declared UPGRADE_BASE_FLOOR {floor}: no published predecessor ≥ floor below "
+                f"head {head_version} (all older tags cross a structural break)",
+            )
         if older:
             return BasePlan(
                 "version",
@@ -189,10 +221,12 @@ def resolve_upgrade_base(
     # No canonical in play — none recorded, OR SKIP_CANONICALS_FOR_UPGRADE=true (canonical lookup
     # bypassed entirely, behaving as if none exists). Improved fallback (phase settings §2.C): prefer
     # a REAL published predecessor (newest release tag < head) over the raw main-tip.
-    return _no_canonical_base(recipe, head_ref, head_version)
+    return _no_canonical_base(recipe, head_ref, head_version, floor=floor)
 
 
-def _no_canonical_base(recipe: str, head_ref: str | None, head_version: str | None) -> BasePlan:
+def _no_canonical_base(
+    recipe: str, head_ref: str | None, head_version: str | None, floor: str | None = None
+) -> BasePlan:
     """Upgrade base when no canonical is used (none recorded, its promote failed, or
     SKIP_CANONICALS_FOR_UPGRADE is true). Release-tag-first fallback (phase settings §2.C):
       1. most recent release TAG with version strictly older than the PR head — a clean published
@@ -202,11 +236,14 @@ def _no_canonical_base(recipe: str, head_ref: str | None, head_version: str | No
       3. skip — no predecessor (no older tag and head == main-tip, or no main at all).
     This replaces the old jump-straight-to-main-tip path, so an un-promoted recipe upgrades from a real
     release base instead of a possibly-untagged WIP commit."""
-    older = (
-        warm_reconcile.newest_older_version(warm_reconcile.recipe_tags(recipe), head_version)
-        if head_version
-        else None
-    )
+    _tags = warm_reconcile.recipe_tags(recipe)
+    if floor:
+        # phase basefloor: exclude structurally-invalid bases below the declared floor; the
+        # main-tip fallback below remains available (it is post-break by definition of the
+        # declaration — the floor names the first post-break published version).
+        _fk = warm_reconcile.version_key(floor)
+        _tags = [t for t in _tags if warm_reconcile.version_key(t) >= _fk]
+    older = warm_reconcile.newest_older_version(_tags, head_version) if head_version else None
     if older:
         return BasePlan(
             "version",

tests/STYLE.md

@@ -0,0 +1,116 @@
+# cc-ci test style guide
+
+Rules for writing and changing tests under `tests/`. Read this before any test edit — in particular
+before a `/recipe-upgrade <recipe> --with-tests` or `/ci-test-review` fix, where the temptation is to
+make a red run green rather than to make the test right.
+
+The tests are the **independent gate** on recipe upgrades. Their value is entirely in being hard to
+fool, so every rule below exists to keep them (a) honest and (b) alive across upgrades.
+
+---
+
+## 1. Set up state through the application, not its database
+
+**Order of preference for any fixture that must create state:**
+
+1. **The app's public HTTP API.**
+2. **The app's official CLI or release console** (`docker exec … <app-cli>`).
+3. **Writing rows into its database — last resort only**, and only with a comment saying which of the
+   above were tried and why they did not work.
+
+Direct SQL couples the test to the app's *internal schema*, which upgrades are free to change. The
+app's own interface is the thing it promises to keep working.
+
+> **Why this rule exists.** `tests/plausible/custom/test_event_tracking.py` used to register its test
+> site with `INSERT INTO sites (...)`. That was sufficient for plausible v2. In v3 a site must belong
+> to a **team**, and the app silently discards events for a teamless site — `POST /api/event` still
+> returns **202** and the row is still in postgres, so the only visible symptom was that nothing ever
+> reached ClickHouse. It read as a mysterious ingestion stall and held the recipe RED for six weeks.
+>
+> The fix was not to also INSERT a team row. It was to stop writing rows: the fixture now calls
+> `Plausible.Sites.create/2` through the app's release console, and the app provisions whatever its
+> data model currently needs. The same expression works unchanged on v2 (which has no `teams` table
+> at all) **and** v3 — not because the test handles both, but because it stopped depending on the
+> schema.
+
+When the ideal interface is unavailable, say so in the code. plausible's HTTP provisioning API
+(`POST /api/v1/sites`) is gated behind a paid plan and answers `:upgrade_required` on CE, so the test
+drops to option 2 and records that in a comment.
+
+## 2. Gate on version rather than writing dual-path fixtures
+
+If a behaviour genuinely only exists from version X, **gate the test on the version** instead of
+branching inside it:
+
+```python
+pytest.mark.skipif(app_version < (3,), reason="teams were introduced in v3")
+```
+
+Do **not** write a fixture that carefully supports both schemas. Version-portable code is harder to
+read, harder to trust, and quietly rots once nobody runs the old path.
+
+Corollary: **old tests can simply be deleted** once the fleet has moved past that version. The older
+version is only ever exercised through the *upgrade* tier (deploy base → upgrade → assert), so tests
+that only make sense for a superseded version are dead weight, not coverage.
+
+Prefer §1 first: an app-level fixture often makes the version difference disappear, and then no gate
+is needed at all.
+
+## 3. Never weaken an assertion to turn a run green
+
+There is a hard line between these two, and only the second is allowed as a way out of a red run:
+
+* **Weakening** — relaxing *what* is asserted: dropping a field check, accepting a wider status set,
+  asserting a 202 ack instead of the stored result, deleting the read-back.
+* **Correcting the fixture or the wait** — fixing *how* the test sets up or how long it allows, with
+  the assertion untouched.
+
+If a test can only pass by asserting less, it has found a real regression. Report it; do not edit it.
+
+## 4. Assert real state, not acknowledgements
+
+An HTTP 202 means "accepted", not "done". Read the effect back out of the system that owns it — the
+row in the analytics store, the file on disk, the record in the API — and assert on the values you
+sent. plausible's ingestion returns 202 for events it goes on to discard entirely; a test that
+stopped at the ack would have been permanently, silently green.
+
+## 5. Derive waits from the recipe's declared readiness, not a guess
+
+A per-recipe `recipe_meta.py` already declares `DEPLOY_TIMEOUT` / `HTTP_TIMEOUT` because someone
+measured that app's boot profile. A custom test that hard-codes a shorter window contradicts it and
+will flake or fail on a slower version.
+
+Remember the **tier order**: `custom` runs after `backup`/`restore`, which disrupts the datastore and
+restarts the app. A window sized for a warm app is not sized for that. plausible's health check
+allowed 60s; v3 boots through `sleep 10` → `createdb` → `migrate` → cache warmers first.
+
+## 6. Diagnose from the app's own telemetry before touching a test
+
+Before concluding a test is stale, find the app's account of what happened. It is usually definitive
+and it stops you fixing the wrong thing. plausible records dropped events in ClickHouse's
+`ingest_counters`: `dropped_not_found` with 0 rows before the fix, `buffered` with rows after — that
+single counter identified the root cause after the HTTP status had suggested everything was fine.
+
+Prove the diagnosis both ways where you can: same input, broken state → symptom; corrected state →
+no symptom.
+
+## 7. Fixtures must be idempotent
+
+A fixture may run against a warm canonical, a restored volume, or a re-run. Creating state must be
+safe to repeat — look the object up first and reuse it, rather than assuming a clean database.
+
+## 8. Keep test identities obviously synthetic
+
+Use `ccci-`-prefixed names and `.example` / `.invalid` domains for anything a test creates, so state
+it leaves behind is instantly attributable and can never be confused with real data.
+
+---
+
+## Changing a test: the checklist
+
+1. Reproduce the failure and get the **app's own** explanation (§6).
+2. Classify: recipe bug, or stale test? Only a stale test justifies a test edit.
+3. Fix the **fixture, wait, or setup** — never the assertion (§3).
+4. Prefer the app's interface over its database (§1); gate on version rather than branching (§2).
+5. Verify green against the recipe PR head with the changed test, plus a regression sample.
+6. Say in the commit and PR **what evidence** proves the diagnosis, not just what changed.

tests/discourse/recipe_meta.py

@@ -23,11 +23,21 @@ HTTP_TIMEOUT = 1200
 #
 # UPGRADE-tier BASE (phase prevb — DYNAMIC, no hardcoded UPGRADE_BASE_VERSION): the base the head
 # upgrades from is resolved at run time — last-green (warm canonical) → fallback target-branch (`main`)
-# tip → else skip (run_recipe_ci.resolve_upgrade_base). discourse has no warm canonical, so the base is
-# the `main` tip = bitnamilegacy/discourse:3.5.0, which deploys clean (bitnamilegacy exists) with NO
-# `previous/` repair needed. The PR head (recipe-maintainers/discourse#4) switches app to the official
-# `discourse/discourse:3.5.3` and drops the sidekiq service, so the upgrade tier now exercises the REAL
-# bitnamilegacy→official image migration the PR claims to support.
+# tip → else skip (run_recipe_ci.resolve_upgrade_base).
+#
+# UPGRADE_BASE_FLOOR (phase basefloor, 2026-08-04): the 0.8.x→1.0.0 recipe family switched the app
+# bitnamilegacy/discourse → official discourse/discourse AND the db pgvector/pgvector:pg17 →
+# discourse/postgres:pg18. That db-family change is a structural break: the bitnami cluster has no
+# `discourse` role and pg_upgrade preserves-not-creates roles, so an in-place 0.8.x→1.x deploy can
+# NEVER converge (app FATALs `role "discourse" does not exist`, swarm rolls back) — upstream ships
+# no in-place path across it. Without the floor, the resolver's step-back/fallback selected
+# 0.8.1+3.5.0 (newest tag below the head label) and the upgrade tier red'd on this unsupported
+# path twice (drone #1165 2026-07-31 diagnosis, #1171/weekly 2026-08-03 — both classified
+# stale-test, recipe verified green on the real official→official path). Declaring the floor keeps
+# resolution dynamic and only excludes the structurally-impossible bases; when no ≥-floor
+# predecessor exists the tier records a DECLARED skip (never a silent pass). No assertion weakened:
+# below-floor in-place upgrades were never supported coverage.
+UPGRADE_BASE_FLOOR = "1.0.0+3.5.3"
 #
 # compose.ccci.yml is now the ENVIRONMENTAL overlay (all deploys): only app.deploy.update_config.order:
 # stop-first (node memory reality on the upgrade crossover — see its header). The version-specific

tests/discourse/test_upgrade.py

@@ -6,7 +6,11 @@ migration was never tested. With the version-specific config removed from the al
 and the dynamic base (last-green/main = bitnamilegacy:3.5.0) deployed only as the *base*, the upgrade
 chaos redeploy must land the PR head UNMODIFIED. This overlay asserts exactly that, post-upgrade:
 
-  1. the running `app` service image IS the official discourse/discourse:3.5.3 — NOT bitnamilegacy;
+  1. the running `app` service image IS from the official `discourse/discourse` repository —
+     NOT bitnamilegacy. (Version-agnostic since 2026-08-04: the original assertion hardcoded the
+     migration-era pin `:3.5.3` and went stale on the first legitimate app bump (2026.7.1, weekly
+     2026-08-03). The property this test guards is the IMAGE FAMILY — official vs bitnami — not a
+     frozen version; the exact head pin is already exercised by the deploy itself.)
   2. the `sidekiq` service the PR deletes is GONE from the deployed stack.
 
 If either fails, the head did not really run (the overlay leaked onto it) → RED. Assertion-only,
@@ -26,8 +30,8 @@ def test_head_runs_official_image_not_bitnamilegacy(live_app):
         f"app image is {image!r} — the bitnamilegacy base leaked onto the PR head "
         "(the version-specific overlay was applied to the head, the prevb bug)"
     )
-    assert image.startswith("discourse/discourse:3.5.3"), (
-        f"app image is {image!r}, expected the PR head's official discourse/discourse:3.5.3 "
+    assert image.startswith("discourse/discourse:"), (
+        f"app image is {image!r}, expected the PR head's official discourse/discourse image "
         "— the head's image migration was not exercised"
     )
 

tests/plausible/custom/test_event_tracking.py

@@ -14,7 +14,11 @@ Both assert real app state (the event reached the analytics store), not just the
 
 plausible only ingests events for *known* sites — the in-memory `sites_cache` gates ingestion and
 drops events for unregistered domains (empirically confirmed: an event for an unregistered domain
-never appears in events_v2). So each test first registers a site row in the metadata postgres, then
+never appears in events_v2). Sites are therefore provisioned through plausible's OWN creation path
+rather than by writing rows — under v3 a site must belong to a TEAM, and a teamless site is dropped as
+`dropped_not_found` while the POST still acks 202, which reads as a silent ingestion stall. Letting the
+app create the site sidesteps that entirely, and works unchanged on v2. So each test first provisions
+the site, then
 POSTs repeatedly while polling ClickHouse: the sites_cache must refresh to admit the new site and the
 event write-buffer must flush to ClickHouse, so the first landing is not instantaneous. Re-POSTing the
 same event is safe — we assert the row count is >= 1.
@@ -40,6 +44,10 @@ _UA = (
     "(KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
 )
 
+# Identity the harness provisions sites under. Ephemeral per-run deploy, never a real account.
+_HARNESS_EMAIL = "cc-ci@ci.invalid"
+_HARNESS_PW = "ccci-harness-passphrase-2026"
+
 
 def _ch(domain: str, sql: str) -> str:
     """Run a ClickHouse query against the `plausible_events_db` service; return stdout (stripped)."""
@@ -50,18 +58,61 @@ def _ch(domain: str, sql: str) -> str:
     ).strip()
 
 
+# plausible's own provisioning path. Creating a site through the app (rather than INSERTing rows)
+# means the app applies whatever its current data model requires — which is what makes this work
+# unchanged across the v2→v3 jump, where sites gained a mandatory owning TEAM. Verified on cc-ci
+# against BOTH v2.0.0 (no `teams` table at all) and v3.2.1: identical expression, site usable, events
+# ingested. See tests/STYLE.md.
+#
+# The HTTP provisioning API (`POST /api/v1/sites`) would be the first choice, but it is gated behind
+# a paid plan — on CE it answers `:upgrade_required` — so the app's release console is the closest
+# public interface available here.
+_PROVISION_SITE_EXS = """
+pw = "__PW__"
+email = "__EMAIL__"
+user =
+  case Plausible.Auth.find_user_by(email: email) do
+    nil ->
+      {:ok, u} =
+        Plausible.Auth.User.new(%{name: "cc-ci", email: email, password: pw, password_confirmation: pw})
+        |> Plausible.Repo.insert()
+      u
+    u -> u
+  end
+site = "__SITE__"
+result =
+  case Plausible.Sites.get_by_domain(site) do
+    nil -> Plausible.Sites.create(user, %{"domain" => site, "timezone" => "UTC"})
+    s -> {:ok, s}
+  end
+case result do
+  {:ok, _} -> IO.puts("CCCI_SITE_OK " <> site)
+  other -> IO.puts("CCCI_SITE_ERR " <> inspect(other))
+end
+"""
+
+
 def _register_site(domain: str, site: str) -> None:
-    """Insert a site row into the metadata postgres (`db` service) so plausible will ingest events for
-    it. Idempotent (ON CONFLICT DO NOTHING)."""
-    sql = (
-        "INSERT INTO sites (domain, timezone, inserted_at, updated_at, native_stats_start_at) "
-        f"VALUES ('{site}','UTC', now(), now(), now()) ON CONFLICT (domain) DO NOTHING; "
-        f"SELECT domain FROM sites WHERE domain = '{site}';"
+    """Provision `site` via plausible's own site-creation path, so it is a site the app will ingest for.
+
+    Idempotent: an existing domain is reused rather than re-created.
+
+    Do NOT reach into postgres to do this. A `sites` INSERT was enough under v2, but v3 requires the
+    site to belong to a TEAM and silently discards events for a teamless site — `POST /api/event`
+    still acks 202 and the row still exists, so the only symptom is that nothing reaches ClickHouse
+    (ClickHouse's own `ingest_counters` records it as `dropped_not_found`). That is what put this
+    recipe RED on build 1224. Going through the app removes the whole class of problem: it provisions
+    the team itself.
+    """
+    exs = (
+        _PROVISION_SITE_EXS.replace("__PW__", _HARNESS_PW)
+        .replace("__EMAIL__", _HARNESS_EMAIL)
+        .replace("__SITE__", site)
+    )
+    out = lifecycle.exec_in_app(domain, ["/app/bin/plausible", "rpc", exs], service="app")
+    assert f"CCCI_SITE_OK {site}" in out, (
+        f"could not provision site {site!r} via the app: {out.strip()[-400:]}"
     )
-    out = lifecycle.exec_in_app(
-        domain, ["psql", "-q", "-U", "plausible", "-d", "plausible", "-tAc", sql], service="db"
-    ).strip()
-    assert out == site, f"site {site!r} not registered in postgres (got {out!r})"
 
 
 def _post_event(base_domain: str, site: str, name: str, pathname: str) -> int:
@@ -93,9 +144,9 @@ def _ingest_and_count(
     last_status = None
     while True:
         last_status = _post_event(base_domain, site, name, pathname)
-        assert (
-            last_status == 202
-        ), f"POST /api/event for {name!r} → HTTP {last_status} (expected 202)"
+        assert last_status == 202, (
+            f"POST /api/event for {name!r} → HTTP {last_status} (expected 202)"
+        )
         time.sleep(interval)
         raw = _ch(base_domain, count_sql)
         count = int(raw) if raw.isdigit() else 0
@@ -143,6 +194,6 @@ def test_custom_event_roundtrip(live_app):
         live_app,
         f"SELECT name FROM events_v2 WHERE pathname = '{pathname}' LIMIT 1",
     )
-    assert (
-        stored_name == event_name
-    ), f"custom event stored as {stored_name!r}, expected {event_name!r}"
+    assert stored_name == event_name, (
+        f"custom event stored as {stored_name!r}, expected {event_name!r}"
+    )

tests/plausible/custom/test_health_check.py

@@ -17,6 +17,12 @@ def test_plausible_root_serves(live_app):
     62-char SECRET_KEY_BASE, see recipe_meta.EXTRA_ENV); the dedicated
     /api/health endpoint is.
     """
+    # The custom tier runs AFTER the backup/restore tier, which disrupts postgres under the app and
+    # restarts it. v3 (community-edition) then boots through `sleep 10` + `db createdb` + `db migrate`
+    # + cache warmers before /api/health flips to 200, which does not fit in 60s — that is what put
+    # this recipe RED on build 1224 while install/upgrade/backup/restore all passed. The assertion is
+    # unchanged (still a hard 200 from the real readiness endpoint); only the wait matches the boot
+    # profile the recipe already declares via recipe_meta.HTTP_TIMEOUT (1200).
     url = f"https://{live_app}/api/health"
-    status, _ = harness_http.retry_http_get(url, expect_status=(200,), max_wait=60, interval=3)
+    status, _ = harness_http.retry_http_get(url, expect_status=(200,), max_wait=300, interval=5)
     assert status == 200, f"GET {url} HTTP {status}"