fix(keycloak): key warm state by stack namespace, not bare recipe (F-redfix-4)

The M2 keycloak enrollment made the canonical collision-free at the DOMAIN layer
(warm-canon-keycloak vs warm-keycloak) but warm STATE stayed keyed by bare recipe:
warmsnap.app_dir("keycloak") resolved both the live-warm reconciler's last_good and
the data-warm canonical's canonical.json + snapshot/ into /var/lib/ci-warm/keycloak/.
snapshot() atomically REPLACES that slot, so the two deployments destroyed each
other's known-good; restore() then raised SnapshotError (fails closed, no cross-stack
data write). Worst case: a sweep promote landing inside the reconciler's
snapshot->wait_healthy window makes its rollback restore() raise after
abra.undeploy(live), leaving the shared OIDC provider undeployed.

Fix: canonical.canonical_ns() is now the single namespace from which BOTH the
canonical's domain and its warm-state slot derive, so they cannot drift apart. A
live-warm provider gets ns "canon-<recipe>": domain warm-canon-keycloak (unchanged)
and slot /var/lib/ci-warm/canon-keycloak/. Every other recipe keeps ns "<recipe>" —
zero on-disk change for the 15 existing canonicals, and no migration on cc-ci
(keycloak's canonical was never seeded: its dir holds only last_good).

- warmsnap: functions take a SLOT, not a recipe; add live_slot(); meta records "slot".
- warmsnap: _assert_slot_not_foreign() refuses to snapshot/restore a slot recorded
  against a different domain -- defence in depth, naming-scheme-independent, fails
  before the destructive swap rather than at the next restore.
- canonical: registry_path/seed_canonical/prune_stale go through canonical_slot().
- prune_stale: the "reconciler dirs are never pruned" invariant is now STRUCTURAL --
  <recipe>/ never gains a canonical.json, so de-enrolling keycloak can no longer
  rmtree the reconciler's last_good (consequence 4).
- warm_reconcile: last_good + snapshot/restore go through warmsnap.live_slot().
- run_recipe_ci: canonical rollback restores from canonical_slot(recipe).
- Correct the two comments that claimed the deployments "can never touch each other".

Tests: 10 new (slot disjointness for every WARM_DOMAINS recipe, slot<->stack 1:1,
registry not in the reconciler dir, prune spares last_good, foreign-slot refusal in
both snapshot and restore). Unit suite 315 -> 325, no regressions.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01FS8p1esg57UAC69riNvuBX

tests/unit/test_canonical.py

@@ -12,7 +12,7 @@ import os
 import sys
 
 sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
-from harness import canonical, warm  # noqa: E402
+from harness import canonical, warm, warmsnap  # noqa: E402
 from harness import meta as harness_meta  # noqa: E402
 
 
@@ -96,3 +96,78 @@ def test_prune_stale_drops_deenrolled_only(tmp_path, monkeypatch):
     assert (tmp_path / "keepme").exists()
     assert (tmp_path / "keycloak").exists()  # no canonical.json → not a canonical → kept
     assert (tmp_path / "alerts").exists()
+
+
+# ------------------------------------------- F-redfix-4: canonical ns/slot vs live-warm provider
+
+
+def test_canonical_ns_and_domain_for_live_warm_provider():
+    # keycloak is a live-warm provider → its canonical is namespaced apart on BOTH axes.
+    assert canonical.canonical_ns("keycloak") == "canon-keycloak"
+    assert canonical.canonical_domain("keycloak") == "warm-canon-keycloak.ci.commoninternet.net"
+    assert canonical.canonical_slot("keycloak") == "canon-keycloak"
+    # ...and an ordinary recipe is untouched (zero blast radius on existing canonicals).
+    assert canonical.canonical_ns("cryptpad") == "cryptpad"
+    assert canonical.canonical_domain("cryptpad") == "warm-cryptpad.ci.commoninternet.net"
+    assert canonical.canonical_slot("cryptpad") == "cryptpad"
+
+
+def test_live_and_canonical_slots_are_disjoint():
+    """F-redfix-4: for EVERY live-warm provider, the reconciler's slot and the canonical's slot must
+    differ — they are two deployments and each `snapshot()` atomically replaces its slot."""
+    for recipe in warm.WARM_DOMAINS:
+        assert canonical.canonical_slot(recipe) != warmsnap.live_slot(recipe)
+    # every non-provider recipe keeps one slot (there is only one deployment)
+    assert canonical.canonical_slot("cryptpad") == warmsnap.live_slot("cryptpad")
+
+
+def test_slot_maps_1to1_to_its_stack():
+    # The invariant warmsnap relies on: distinct slot ⇔ distinct stack (domain = warm-<ns>).
+    for recipe in warm.WARM_DOMAINS:
+        assert warm.stable_domain(canonical.canonical_slot(recipe)) == canonical.canonical_domain(
+            recipe
+        )
+        assert warm.stable_domain(warmsnap.live_slot(recipe)) == warm.WARM_DOMAINS[recipe]
+
+
+def test_registry_path_of_live_warm_provider_is_not_the_reconciler_dir(tmp_path, monkeypatch):
+    """The canonical registry must NOT land in `<recipe>/`, where the reconciler keeps last_good."""
+    monkeypatch.setenv("CCCI_WARM_ROOT", str(tmp_path))
+    assert canonical.registry_path("keycloak") == str(
+        tmp_path / "canon-keycloak" / "canonical.json"
+    )
+    canonical.write_registry("keycloak", version="10.7.1+26.6.2", commit=None, status="idle")
+    assert (tmp_path / "canon-keycloak" / "canonical.json").is_file()
+    assert not (tmp_path / "keycloak").exists()  # reconciler dir untouched
+
+
+def test_prune_stale_keeps_enrolled_provider_canonical_and_reconciler(tmp_path, monkeypatch):
+    """Enrolled keycloak: `canon-keycloak/` is its canonical slot (keep), `keycloak/` is the
+    reconciler's (keep — no canonical.json)."""
+    monkeypatch.setenv("CCCI_WARM_ROOT", str(tmp_path))
+    monkeypatch.setattr(canonical, "enrolled_recipes", lambda: ["keycloak"])
+    monkeypatch.setattr(canonical.warmsnap, "stack_volumes", lambda d: [])
+    (tmp_path / "canon-keycloak").mkdir()
+    (tmp_path / "canon-keycloak" / "canonical.json").write_text('{"recipe":"keycloak"}')
+    (tmp_path / "keycloak").mkdir()
+    (tmp_path / "keycloak" / "last_good").write_text("10.7.1+26.6.2")
+    assert canonical.prune_stale() == []
+    assert (tmp_path / "canon-keycloak").exists()
+    assert (tmp_path / "keycloak" / "last_good").read_text() == "10.7.1+26.6.2"
+
+
+def test_prune_stale_deenrolled_provider_spares_reconciler_last_good(tmp_path, monkeypatch):
+    """F-redfix-4 consequence 4: de-enrolling keycloak must drop `canon-keycloak/` ONLY — never the
+    reconciler's `keycloak/last_good`. It also targets the canon stack's volumes, not the live ones."""
+    monkeypatch.setenv("CCCI_WARM_ROOT", str(tmp_path))
+    monkeypatch.setattr(canonical, "enrolled_recipes", lambda: [])  # de-enrolled
+    seen = []
+    monkeypatch.setattr(canonical.warmsnap, "stack_volumes", lambda d: seen.append(d) or [])
+    (tmp_path / "canon-keycloak").mkdir()
+    (tmp_path / "canon-keycloak" / "canonical.json").write_text('{"recipe":"keycloak"}')
+    (tmp_path / "keycloak").mkdir()
+    (tmp_path / "keycloak" / "last_good").write_text("10.7.1+26.6.2")
+    assert canonical.prune_stale() == ["canon-keycloak"]
+    assert not (tmp_path / "canon-keycloak").exists()
+    assert (tmp_path / "keycloak" / "last_good").read_text() == "10.7.1+26.6.2"
+    assert seen == ["warm-canon-keycloak.ci.commoninternet.net"]  # never the LIVE provider's stack

tests/unit/test_warmsnap.py

@@ -68,3 +68,62 @@ def test_has_snapshot_incomplete_missing_tar(monkeypatch, tmp_path):
     (snapdir / "meta.json").write_text(json.dumps({"recipe": "keycloak", "volumes": ["a", "b"]}))
     (snapdir / "volumes" / "a.tar").write_bytes(b"fake")
     assert warmsnap.has_snapshot("keycloak") is False
+
+
+# --------------------------------------------------------------- F-redfix-4: slot ≠ recipe
+
+
+def test_live_slot_is_the_recipe():
+    assert warmsnap.live_slot("keycloak") == "keycloak"
+
+
+def test_snapshot_refuses_to_clobber_another_domains_slot(monkeypatch, tmp_path):
+    """F-redfix-4 regression: a slot holding domain A's known-good must not be overwritten by a
+    snapshot of domain B. Before the fix this silently destroyed the other deployment's snapshot."""
+    monkeypatch.setenv("CCCI_WARM_ROOT", str(tmp_path))
+    snapdir = tmp_path / "keycloak" / "snapshot"
+    (snapdir / "volumes").mkdir(parents=True)
+    (snapdir / "meta.json").write_text(
+        json.dumps({"slot": "keycloak", "domain": "warm-keycloak.ci.x", "volumes": []})
+    )
+    # Never reaches docker: the foreign-slot guard fires before _assert_undeployed.
+    try:
+        warmsnap.snapshot("keycloak", "warm-canon-keycloak.ci.x")
+    except warmsnap.SnapshotError as e:
+        assert "warm-keycloak.ci.x" in str(e) and "refusing to clobber" in str(e)
+    else:
+        raise AssertionError("snapshot() overwrote a foreign slot")
+    # the original known-good is intact
+    assert warmsnap.read_meta("keycloak")["domain"] == "warm-keycloak.ci.x"
+
+
+def test_snapshot_may_reclaim_its_own_slot(monkeypatch, tmp_path):
+    # Same domain → not foreign → the guard must not fire (it would break every re-snapshot).
+    monkeypatch.setenv("CCCI_WARM_ROOT", str(tmp_path))
+    snapdir = tmp_path / "keycloak" / "snapshot"
+    (snapdir / "volumes").mkdir(parents=True)
+    (snapdir / "meta.json").write_text(
+        json.dumps({"slot": "keycloak", "domain": "warm-keycloak.ci.x", "volumes": []})
+    )
+    warmsnap._assert_slot_not_foreign("keycloak", "warm-keycloak.ci.x")  # no raise
+    warmsnap._assert_slot_not_foreign("fresh-slot", "anything.ci.x")  # empty slot is free to claim
+
+
+def test_restore_refuses_a_foreign_slot(monkeypatch, tmp_path):
+    """restore() must reject a slot recorded against another domain BEFORE touching volumes."""
+    monkeypatch.setenv("CCCI_WARM_ROOT", str(tmp_path))
+    monkeypatch.setattr(warmsnap, "_assert_undeployed", lambda d: None)
+    _write_snapshot(tmp_path, "keycloak", ["warm-keycloak_ci_x_mariadb"])
+    snapdir = tmp_path / "keycloak" / "snapshot"
+    meta = json.loads((snapdir / "meta.json").read_text())
+    meta["domain"] = "warm-keycloak.ci.x"
+    (snapdir / "meta.json").write_text(json.dumps(meta))
+    called = []
+    monkeypatch.setattr(warmsnap, "stack_volumes", lambda d: called.append(d) or [])
+    try:
+        warmsnap.restore("keycloak", "warm-canon-keycloak.ci.x")
+    except warmsnap.SnapshotError as e:
+        assert "refusing to clobber" in str(e)
+    else:
+        raise AssertionError("restore() accepted a foreign slot")
+    assert called == [], "restore() must fail before inspecting the target stack's volumes"