fix(keycloak): key warm state by stack namespace, not bare recipe (F-redfix-4)

The M2 keycloak enrollment made the canonical collision-free at the DOMAIN layer
(warm-canon-keycloak vs warm-keycloak) but warm STATE stayed keyed by bare recipe:
warmsnap.app_dir("keycloak") resolved both the live-warm reconciler's last_good and
the data-warm canonical's canonical.json + snapshot/ into /var/lib/ci-warm/keycloak/.
snapshot() atomically REPLACES that slot, so the two deployments destroyed each
other's known-good; restore() then raised SnapshotError (fails closed, no cross-stack
data write). Worst case: a sweep promote landing inside the reconciler's
snapshot->wait_healthy window makes its rollback restore() raise after
abra.undeploy(live), leaving the shared OIDC provider undeployed.

Fix: canonical.canonical_ns() is now the single namespace from which BOTH the
canonical's domain and its warm-state slot derive, so they cannot drift apart. A
live-warm provider gets ns "canon-<recipe>": domain warm-canon-keycloak (unchanged)
and slot /var/lib/ci-warm/canon-keycloak/. Every other recipe keeps ns "<recipe>" —
zero on-disk change for the 15 existing canonicals, and no migration on cc-ci
(keycloak's canonical was never seeded: its dir holds only last_good).

- warmsnap: functions take a SLOT, not a recipe; add live_slot(); meta records "slot".
- warmsnap: _assert_slot_not_foreign() refuses to snapshot/restore a slot recorded
  against a different domain -- defence in depth, naming-scheme-independent, fails
  before the destructive swap rather than at the next restore.
- canonical: registry_path/seed_canonical/prune_stale go through canonical_slot().
- prune_stale: the "reconciler dirs are never pruned" invariant is now STRUCTURAL --
  <recipe>/ never gains a canonical.json, so de-enrolling keycloak can no longer
  rmtree the reconciler's last_good (consequence 4).
- warm_reconcile: last_good + snapshot/restore go through warmsnap.live_slot().
- run_recipe_ci: canonical rollback restores from canonical_slot(recipe).
- Correct the two comments that claimed the deployments "can never touch each other".

Tests: 10 new (slot disjointness for every WARM_DOMAINS recipe, slot<->stack 1:1,
registry not in the reconciler dir, prune spares last_good, foreign-slot refusal in
both snapshot and restore). Unit suite 315 -> 325, no regressions.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FS8p1esg57UAC69riNvuBX

runner/harness/canonical.py

@@ -39,19 +39,35 @@ def is_enrolled(recipe: str) -> bool:
     return bool(meta_mod.load(recipe).WARM_CANONICAL)
 
 
-def canonical_domain(recipe: str) -> str:
-    """Stable data-warm domain for the recipe's canonical.
+def canonical_ns(recipe: str) -> str:
+    """The data-warm canonical's NAMESPACE for a recipe — the single source from which BOTH its
+    domain and its warm-state slot are derived, so the two can never drift apart (F-redfix-4).
 
     For a recipe that is ALSO a live-warm provider (in `warm.WARM_DOMAINS` — e.g. keycloak, whose
-    always-on shared OIDC instance lives at `warm-keycloak…`), the data-warm canonical MUST use a
-    DISTINCT domain: otherwise the sweep's promote deploy/teardown at `warm-<recipe>` collides with —
-    and could disrupt — the live shared service that other recipes (lasuite-*/drone) depend on. Give
-    those recipes a collision-free `warm-canon-<recipe>` namespace (a separate stack/domain that can
-    never touch the live provider); every other recipe keeps the plain `warm-<recipe>` scheme
-    (zero blast radius on the 15 existing canonicals)."""
+    always-on shared OIDC instance lives at `warm-keycloak…`), the canonical MUST be namespaced apart
+    from the live provider: otherwise the sweep's promote deploy/teardown at `warm-<recipe>` collides
+    with — and could disrupt — the live shared service that other recipes (lasuite-*/drone) depend on,
+    and (F-redfix-4) the two deployments share one `/var/lib/ci-warm/<recipe>/` snapshot slot, so each
+    destroys the other's known-good. Every other recipe keeps the plain `<recipe>` namespace (zero
+    blast radius on the 15 existing canonicals, whose on-disk layout is unchanged)."""
     if recipe in warm.WARM_DOMAINS:
-        return f"warm-canon-{recipe}.ci.commoninternet.net"
-    return warm.stable_domain(recipe)
+        return f"canon-{recipe}"
+    return recipe
+
+
+def canonical_domain(recipe: str) -> str:
+    """Stable data-warm domain for the recipe's canonical (`warm-<ns>`)."""
+    return warm.stable_domain(canonical_ns(recipe))
+
+
+def canonical_slot(recipe: str) -> str:
+    """The canonical's warm-state slot: `$CCCI_WARM_ROOT/<ns>/` — holding `canonical.json` and the
+    known-good `snapshot/`.
+
+    DISJOINT from `warmsnap.live_slot(recipe)` (the live-warm reconciler's `<recipe>/`, holding its
+    `last_good`) exactly when the recipe is a live-warm provider. Asserted in
+    `tests/unit/test_canonical.py::test_live_and_canonical_slots_are_disjoint`."""
+    return canonical_ns(recipe)
 
 
 def enrolled_recipes() -> list[str]:
@@ -71,7 +87,7 @@ def enrolled_recipes() -> list[str]:
 
 
 def registry_path(recipe: str) -> str:
-    return os.path.join(warmsnap.app_dir(recipe), "canonical.json")
+    return os.path.join(warmsnap.app_dir(canonical_slot(recipe)), "canonical.json")
 
 
 def read_registry(recipe: str) -> dict | None:
@@ -84,7 +100,7 @@ def read_registry(recipe: str) -> dict | None:
 
 def write_registry(recipe: str, *, version: str, commit: str | None, status: str) -> dict:
     """Atomically write the canonical registry record for a recipe."""
-    os.makedirs(warmsnap.app_dir(recipe), exist_ok=True)
+    os.makedirs(warmsnap.app_dir(canonical_slot(recipe)), exist_ok=True)
     rec = {
         "recipe": recipe,
         "domain": canonical_domain(recipe),
@@ -151,16 +167,22 @@ def undeploy_keep_volume(recipe: str) -> None:
 
 
 def prune_stale() -> list[str]:
-    """WC8 disk hygiene: remove warm data for DE-ENROLLED canonicals — a `/var/lib/ci-warm/<recipe>/`
-    that carries a `canonical.json` but whose recipe is no longer enrolled (WARM_CANONICAL dropped).
-    Drops the dir (snapshot + registry) AND the retained `warm-<recipe>` data volumes. Leaves the
-    live-warm reconciler dirs (keycloak/traefik — they have a `last_good`, no `canonical.json`),
-    `alerts/`, and currently-enrolled canonicals untouched. Returns the recipes pruned."""
+    """WC8 disk hygiene: remove warm data for DE-ENROLLED canonicals — a `/var/lib/ci-warm/<ns>/`
+    that carries a `canonical.json` but whose namespace is no longer an enrolled canonical's slot
+    (WARM_CANONICAL dropped). Drops the dir (snapshot + registry) AND the retained `warm-<ns>` data
+    volumes. Leaves the live-warm reconciler dirs (keycloak/traefik — they have a `last_good`, no
+    `canonical.json`), `alerts/`, and currently-enrolled canonicals untouched. Returns the namespaces
+    pruned.
+
+    The reconciler-dir invariant is STRUCTURAL, not incidental (F-redfix-4): a live-warm provider's
+    canonical registers under `canon-<recipe>/`, so `<recipe>/` never gains a `canonical.json` and is
+    never a prune candidate — de-enrolling keycloak can no longer `rmtree` the reconciler's
+    `last_good`. Dir name == namespace, so the stale stack's domain is `warm.stable_domain(name)`."""
     import shutil
     import subprocess
 
     root = warmsnap.warm_root()
-    keep = set(enrolled_recipes())
+    keep = {canonical_slot(r) for r in enrolled_recipes()}
     pruned: list[str] = []
     try:
         entries = sorted(os.listdir(root))
@@ -172,8 +194,8 @@ def prune_stale() -> list[str]:
             continue
         if not os.path.isfile(os.path.join(d, "canonical.json")):
             continue  # not a data-warm canonical (e.g. keycloak/traefik reconciler dir, alerts/)
-        # drop the retained warm-<recipe> volumes, then the snapshot/registry dir
-        for vol in warmsnap.stack_volumes(canonical_domain(name)):
+        # drop the retained warm-<ns> volumes, then the snapshot/registry dir
+        for vol in warmsnap.stack_volumes(warm.stable_domain(name)):
             subprocess.run(["docker", "volume", "rm", vol], capture_output=True, text=True)
         shutil.rmtree(d, ignore_errors=True)
         pruned.append(name)
@@ -186,5 +208,7 @@ def seed_canonical(recipe: str, version: str, commit: str | None = None) -> dict
     healthy first, then undeploys before calling this (WC3: snapshot while undeployed). The retained
     volume IS the canonical. Returns the registry record."""
     rec = write_registry(recipe, version=version, commit=commit, status="idle")
-    warmsnap.snapshot(recipe, canonical_domain(recipe), commit=commit, version=version)
+    warmsnap.snapshot(
+        canonical_slot(recipe), canonical_domain(recipe), commit=commit, version=version
+    )
     return rec

runner/harness/warmsnap.py

@@ -1,8 +1,8 @@
 """Known-good snapshot/restore of an app's data volumes (Phase 2w / WC3).
 
 A snapshot is a **raw copy of every docker volume belonging to an app's stack, taken while the app is
-UNDEPLOYED** (nothing is writing → consistent). Stored under `/var/lib/ci-warm/<recipe>/` as one
-last-known-good per app, replaced atomically. Restore clears each volume and untars it back.
+UNDEPLOYED** (nothing is writing → consistent). Stored under `/var/lib/ci-warm/<slot>/` as one
+last-known-good per slot, replaced atomically. Restore clears each volume and untars it back.
 
 Used by:
 - WC1.1 — snapshot keycloak's data volume BEFORE an auto-upgrade; restore on health-gate rollback
@@ -12,12 +12,25 @@ Used by:
 Warm snapshots are **cache, excluded from the D8 reproducibility closure** (WC8) — re-seeded by cold
 runs, not restored on a VM rebuild.
 
-Layout (atomic dir swap of the `snapshot/` subdir; one last-good per app). Sibling per-app state
-(e.g. the reconciler's `last_good`) lives in `<recipe>/` and is NOT clobbered by the swap:
-    $CCCI_WARM_ROOT/<recipe>/
+SLOT, not recipe (F-redfix-4). Warm state belongs to a **deployed stack**, not to a recipe name. A
+recipe can own TWO warm stacks at once — keycloak is both the live-warm shared OIDC provider
+(`warm-keycloak…`) and a data-warm canonical (`warm-canon-keycloak…`) — and they must never share a
+snapshot slot: `snapshot()` atomically REPLACES the slot, so a shared slot means each deployment
+destroys the other's known-good, and `restore()` (which requires every recorded volume to exist in
+the target stack) then raises. Callers therefore pass a **slot key**, not a bare recipe:
+- live-warm reconciler → `live_slot(recipe)`            (`<recipe>/`)
+- data-warm canonical  → `canonical.canonical_slot(r)`  (`<recipe>/`, or `canon-<recipe>/` when the
+                                                         recipe is also a live-warm provider)
+Every slot key `<ns>` maps 1:1 to the stack at `warm.stable_domain(<ns>)`, so distinct slot ⇔
+distinct stack. `canonical.py` derives its domain and its slot from ONE namespace so the two cannot
+drift apart.
+
+Layout (atomic dir swap of the `snapshot/` subdir; one last-good per slot). Sibling per-slot state
+(e.g. the reconciler's `last_good`) lives in `<slot>/` and is NOT clobbered by the swap:
+    $CCCI_WARM_ROOT/<slot>/
         last_good                 # (reconciler) the version known healthy — survives snapshot swaps
         snapshot/
-            meta.json             # {recipe, domain, commit, version, ts, volumes:[...]}
+            meta.json             # {slot, domain, commit, version, ts, volumes:[...]}
             volumes/<volname>.tar # raw tar of the volume root, one per stack volume
 
 Implementation note: volumes are tarred from their host mountpoint
@@ -46,38 +59,46 @@ def warm_root() -> str:
     return os.environ.get("CCCI_WARM_ROOT", DEFAULT_WARM_ROOT)
 
 
-def app_dir(recipe: str) -> str:
-    return os.path.join(warm_root(), recipe)
+def live_slot(recipe: str) -> str:
+    """Warm-state slot of the LIVE-warm reconciler's deployment (the stack at `warm-<recipe>`).
+
+    The canonical's slot is `canonical.canonical_slot()`, which diverges from this one exactly when
+    the recipe is also a live-warm provider (F-redfix-4)."""
+    return recipe
 
 
-def snap_dir(recipe: str) -> str:
+def app_dir(slot: str) -> str:
+    return os.path.join(warm_root(), slot)
+
+
+def snap_dir(slot: str) -> str:
     """The snapshot subdir — atomically swapped on update. Kept SEPARATE from app_dir so sibling
-    per-app state (the reconciler's last_good) survives a snapshot swap."""
-    return os.path.join(app_dir(recipe), "snapshot")
+    per-slot state (the reconciler's last_good) survives a snapshot swap."""
+    return os.path.join(app_dir(slot), "snapshot")
 
 
-def meta_path(recipe: str) -> str:
-    return os.path.join(snap_dir(recipe), "meta.json")
+def meta_path(slot: str) -> str:
+    return os.path.join(snap_dir(slot), "meta.json")
 
 
-def volumes_dir(recipe: str) -> str:
-    return os.path.join(snap_dir(recipe), "volumes")
+def volumes_dir(slot: str) -> str:
+    return os.path.join(snap_dir(slot), "volumes")
 
 
-def has_snapshot(recipe: str) -> bool:
+def has_snapshot(slot: str) -> bool:
     """True iff a complete last-good snapshot (meta + at least its declared volume tars) exists."""
-    meta = read_meta(recipe)
+    meta = read_meta(slot)
     if not meta:
         return False
     for v in meta.get("volumes", []):
-        if not os.path.isfile(os.path.join(volumes_dir(recipe), f"{v}.tar")):
+        if not os.path.isfile(os.path.join(volumes_dir(slot), f"{v}.tar")):
             return False
     return True
 
 
-def read_meta(recipe: str) -> dict | None:
+def read_meta(slot: str) -> dict | None:
     try:
-        with open(meta_path(recipe)) as f:
+        with open(meta_path(slot)) as f:
             return json.load(f)
     except (OSError, ValueError):
         return None
@@ -113,18 +134,35 @@ def _assert_undeployed(domain: str) -> None:
         )
 
 
-def snapshot(
-    recipe: str, domain: str, commit: str | None = None, version: str | None = None
-) -> dict:
-    """Take a last-known-good snapshot of every data volume of <domain>'s stack. The app MUST be
-    undeployed. Atomically replaces the prior last-good. Returns the written meta dict."""
+def _assert_slot_not_foreign(slot: str, domain: str) -> None:
+    """Refuse to touch a slot that already holds another DOMAIN's known-good (F-redfix-4).
+
+    Defence in depth behind the slot separation itself: naming-scheme-independent, it catches any
+    future caller that pairs a slot with the wrong stack. Fails BEFORE the destructive swap rather
+    than at the next restore. A slot with no snapshot yet is free to claim."""
+    meta = read_meta(slot)
+    if meta and meta.get("domain") and meta["domain"] != domain:
+        raise SnapshotError(
+            f"warm slot {slot!r} holds the known-good of {meta['domain']!r}, not {domain!r}: "
+            "refusing to clobber another deployment's snapshot (F-redfix-4). "
+            "Live-warm and data-warm deployments of one recipe need distinct slots."
+        )
+
+
+def snapshot(slot: str, domain: str, commit: str | None = None, version: str | None = None) -> dict:
+    """Take a last-known-good snapshot of every data volume of <domain>'s stack into <slot>. The app
+    MUST be undeployed. Atomically replaces the prior last-good in that slot. Returns the meta dict.
+
+    `slot` must be the warm-state slot OWNING `domain` (see module docstring) — passing another
+    deployment's slot would destroy its known-good, so the pairing is asserted below."""
+    _assert_slot_not_foreign(slot, domain)
     _assert_undeployed(domain)
     volumes = stack_volumes(domain)
     if not volumes:
         raise SnapshotError(f"no volumes found for {domain} — nothing to snapshot")
 
-    os.makedirs(app_dir(recipe), exist_ok=True)
-    staging = os.path.join(app_dir(recipe), ".snapshot.staging")
+    os.makedirs(app_dir(slot), exist_ok=True)
+    staging = os.path.join(app_dir(slot), ".snapshot.staging")
     shutil.rmtree(staging, ignore_errors=True)
     os.makedirs(os.path.join(staging, "volumes"), exist_ok=True)
 
@@ -138,7 +176,7 @@ def snapshot(
             raise SnapshotError(f"tar of volume {vol} failed: {r.stderr.strip()}")
 
     meta = {
-        "recipe": recipe,
+        "slot": slot,
         "domain": domain,
         "commit": commit,
         "version": version,
@@ -149,8 +187,8 @@ def snapshot(
         json.dump(meta, f)
 
     # Atomic-ish swap of the snapshot subdir only (sibling state like last_good is untouched).
-    target = snap_dir(recipe)
-    old = os.path.join(app_dir(recipe), ".snapshot.old")
+    target = snap_dir(slot)
+    old = os.path.join(app_dir(slot), ".snapshot.old")
     shutil.rmtree(old, ignore_errors=True)
     if os.path.exists(target):
         os.rename(target, old)
@@ -159,17 +197,19 @@ def snapshot(
     return meta
 
 
-def restore(recipe: str, domain: str) -> dict:
-    """Restore the last-known-good snapshot into <domain>'s stack volumes. The app MUST be undeployed.
-    Clears each volume then untars the snapshot back. Returns the snapshot meta. Raises if no
-    snapshot exists or a snapshot volume is missing from the current stack."""
+def restore(slot: str, domain: str) -> dict:
+    """Restore <slot>'s last-known-good snapshot into <domain>'s stack volumes. The app MUST be
+    undeployed. Clears each volume then untars the snapshot back. Returns the snapshot meta. Raises
+    if no snapshot exists, the slot belongs to another domain, or a snapshot volume is missing from
+    the current stack."""
     _assert_undeployed(domain)
-    meta = read_meta(recipe)
-    if not meta or not has_snapshot(recipe):
-        raise SnapshotError(f"no complete snapshot for {recipe} to restore")
+    meta = read_meta(slot)
+    if not meta or not has_snapshot(slot):
+        raise SnapshotError(f"no complete snapshot in slot {slot!r} to restore")
+    _assert_slot_not_foreign(slot, domain)
     current = set(stack_volumes(domain))
     for vol in meta.get("volumes", []):
-        tar_path = os.path.join(volumes_dir(recipe), f"{vol}.tar")
+        tar_path = os.path.join(volumes_dir(slot), f"{vol}.tar")
         if vol not in current:
             raise SnapshotError(
                 f"snapshot volume {vol} absent from current stack {sorted(current)}"

runner/run_recipe_ci.py

@@ -893,7 +893,7 @@ def run_quick(
                 )
                 abra.undeploy(domain)
                 _wait_undeployed(domain)
-                warmsnap.restore(recipe, domain)
+                warmsnap.restore(canonical.canonical_slot(recipe), domain)
                 # reset recorded version to the known-good (the failed upgrade set TYPE to the broken
                 # PR commit) so the idle canonical's .env agrees with the registry + re-warms cleanly.
                 if reg.get("version"):

runner/warm_reconcile.py

@@ -384,7 +384,9 @@ def wait_undeployed(domain: str, timeout: int = 120) -> None:
 
 
 def last_good_path(recipe: str) -> str:
-    return os.path.join(warmsnap.app_dir(recipe), "last_good")
+    # live_slot, not the bare recipe: a recipe that is also a data-warm canonical keeps its canonical
+    # state in a DIFFERENT slot, so the reconciler's last_good is never clobbered (F-redfix-4).
+    return os.path.join(warmsnap.app_dir(warmsnap.live_slot(recipe)), "last_good")
 
 
 def read_last_good(recipe: str) -> str | None:
@@ -396,7 +398,7 @@ def read_last_good(recipe: str) -> str | None:
 
 
 def write_last_good(recipe: str, version: str) -> None:
-    os.makedirs(warmsnap.app_dir(recipe), exist_ok=True)
+    os.makedirs(warmsnap.app_dir(warmsnap.live_slot(recipe)), exist_ok=True)
     tmp = last_good_path(recipe) + ".tmp"
     with open(tmp, "w") as f:
         f.write(version)
@@ -509,7 +511,7 @@ def reconcile(app: str) -> str:
     if stateful:
         abra.undeploy(domain)
         wait_undeployed(domain)
-        warmsnap.snapshot(recipe, domain, version=last_good)
+        warmsnap.snapshot(warmsnap.live_slot(recipe), domain, version=last_good)
         # snapshot requires undeployed; now bring up latest.
     # A broken "latest" can fail in two ways: deploy_version raises (abra converge times out on a
     # crash-looping task) OR it deploys but never becomes healthy. BOTH must roll back, so treat a
@@ -531,7 +533,7 @@ def reconcile(app: str) -> str:
     if stateful:
         abra.undeploy(domain)
         wait_undeployed(domain)
-        warmsnap.restore(recipe, domain)
+        warmsnap.restore(warmsnap.live_slot(recipe), domain)
     deploy_version(recipe, domain, last_good, dt)
     recovered = wait_healthy(spec)
     write_alert(