diff --git a/runner/harness/canonical.py b/runner/harness/canonical.py new file mode 100644 index 0000000..308cb0b --- /dev/null +++ b/runner/harness/canonical.py @@ -0,0 +1,130 @@ +"""Data-warm canonical registry + lifecycle (Phase 2w / WC2, with WC3 snapshots). + +A **canonical** is a per-recipe known-good deployment kept at the STABLE domain +`warm-.ci.commoninternet.net`, **data-warm**: deployed while in use, **undeployed-when-idle +with its data volume retained**, so a later `--quick` run (W2) reattaches the volume and boots warm +(skipping fresh DB-init/first-boot). A small declarative registry tracks which recipes are canonical +and **at which known-good commit/version**. + +Distinct from W0's *live-warm* keycloak (always running, shared SSO dep). Both use the +`warm-` scheme + warmsnap snapshots; the difference is the idle lifecycle (live = up, +data = undeployed-keep-volume). + +- **Enrollment (declarative):** `tests//recipe_meta.py` sets `WARM_CANONICAL = True` + (consistent with DEPS/EXTRA_ENV — enrolling stays a tests// change, D5). +- **Registry state (per recipe), under `/var/lib/ci-warm//canonical.json`:** + `{recipe, domain, version, commit, status, ts}`. The retained data volume + the warmsnap + `snapshot/` live alongside. All of this is **cache, excluded from the D8 closure** (WC8) — + re-seeded by cold runs (WC5), not restored on a VM rebuild. + +W1 builds the registry + the data-warm lifecycle and proves it (seed → undeploy-keep-volume → +redeploy-reattach → data survives). The automatic **promote-on-green-cold** seeding/advancement (WC5) ++ nightly refresh (WC6) are W3; here `seed_canonical` is the primitive they will call. +""" + +from __future__ import annotations + +import json +import os +import subprocess +import time + +from . import abra, warm, warmsnap + + +def is_enrolled(recipe: str) -> bool: + """True if `tests//recipe_meta.py` sets `WARM_CANONICAL = True`. Missing meta → False.""" + path = os.path.join(os.path.dirname(__file__), "..", "..", "tests", recipe, "recipe_meta.py") + if not os.path.exists(path): + return False + ns: dict = {} + with open(path) as fh: + exec(compile(fh.read(), path, "exec"), ns) # noqa: S102 (trusted, in-repo) + return bool(ns.get("WARM_CANONICAL")) + + +def canonical_domain(recipe: str) -> str: + """Stable data-warm domain for the recipe's canonical.""" + return warm.stable_domain(recipe) + + +def registry_path(recipe: str) -> str: + return os.path.join(warmsnap.app_dir(recipe), "canonical.json") + + +def read_registry(recipe: str) -> dict | None: + try: + with open(registry_path(recipe)) as f: + return json.load(f) + except (OSError, ValueError): + return None + + +def write_registry(recipe: str, *, version: str, commit: str | None, status: str) -> dict: + """Atomically write the canonical registry record for a recipe.""" + os.makedirs(warmsnap.app_dir(recipe), exist_ok=True) + rec = { + "recipe": recipe, + "domain": canonical_domain(recipe), + "version": version, + "commit": commit, + "status": status, # "warm" (deployed/in-use) | "idle" (undeployed, volume retained) + "ts": time.strftime("%Y%m%dT%H%M%SZ", time.gmtime()), + } + tmp = registry_path(recipe) + ".tmp" + with open(tmp, "w") as f: + json.dump(rec, f, indent=2) + os.replace(tmp, registry_path(recipe)) + return rec + + +def has_canonical(recipe: str) -> bool: + """True iff a registry record exists AND the data volume(s) are retained on the host (so a + redeploy can reattach them). Mirrors WC2's 'data-warm: volume retained'.""" + rec = read_registry(recipe) + if not rec: + return False + return bool(warmsnap.stack_volumes(canonical_domain(recipe))) + + +def _set_status(recipe: str, status: str) -> None: + rec = read_registry(recipe) + if rec: + write_registry(recipe, version=rec.get("version"), commit=rec.get("commit"), status=status) + + +def deploy_canonical(recipe: str, timeout: int = 900) -> None: + """Bring a data-warm canonical UP at its known-good version, reattaching the retained data + volume (warm boot). Requires an existing registry record (seeded by a cold run / W1 proof).""" + rec = read_registry(recipe) + if not rec: + raise RuntimeError(f"no canonical registry for {recipe} — seed one first (cold run)") + domain, version = rec["domain"], rec["version"] + # The .env + retained volume already exist; redeploy the recorded version (idempotent with -f). + abra.recipe_checkout(recipe, version) + r = subprocess.run( + ["abra", "app", "deploy", domain, version, "-o", "-n", "-f"], + capture_output=True, text=True, timeout=timeout, + ) + if r.returncode != 0: + raise RuntimeError(f"deploy canonical {domain} {version} failed: " + f"{(r.stderr + ' ' + r.stdout).strip()[:300]}") + _set_status(recipe, "warm") + + +def undeploy_keep_volume(recipe: str) -> None: + """Make the canonical idle: undeploy (free RAM) but RETAIN the data volume (data-warm). Does NOT + remove volumes/secrets/.env — only `abra app undeploy`.""" + domain = canonical_domain(recipe) + abra.undeploy(domain) + _set_status(recipe, "idle") + + +def seed_canonical(recipe: str, version: str, commit: str | None = None) -> dict: + """Record (already deployed at `version`) as the recipe's canonical: write the + registry, then (app must be UNDEPLOYED) take the known-good snapshot. Caller deploys + verifies + healthy first, then undeploys before calling this (WC3: snapshot while undeployed). The retained + volume IS the canonical. Returns the registry record.""" + rec = write_registry(recipe, version=version, commit=commit, status="idle") + warmsnap.snapshot(recipe, canonical_domain(recipe), commit=commit, version=version) + return rec diff --git a/runner/harness/warm.py b/runner/harness/warm.py index ef9935b..9de854b 100644 --- a/runner/harness/warm.py +++ b/runner/harness/warm.py @@ -41,6 +41,13 @@ _CTX.verify_mode = ssl.CERT_NONE _STACK_HEX_RE = re.compile(r"^[a-z0-9]{1,4}-([0-9a-f]{6})_ci_commoninternet_net_") +def stable_domain(recipe: str) -> str: + """The stable warm domain for a recipe: `warm-.ci.commoninternet.net` — the canonical + scheme for BOTH the live-warm keycloak and the data-warm canonicals (WC2), distinct from cold + per-run `-<6hex>`. (WARM_DOMAINS['keycloak'] equals stable_domain('keycloak').)""" + return f"warm-{recipe}.ci.commoninternet.net" + + def warm_domain(recipe: str) -> str | None: """The stable warm domain for a dep recipe, or None if this recipe is not served warm.""" return WARM_DOMAINS.get(recipe) diff --git a/tests/unit/test_canonical.py b/tests/unit/test_canonical.py new file mode 100644 index 0000000..60c2dcb --- /dev/null +++ b/tests/unit/test_canonical.py @@ -0,0 +1,61 @@ +"""Unit tests for the WC2 canonical registry (runner/harness/canonical.py). + +Pure parts: enrollment (recipe_meta.WARM_CANONICAL), stable domain, registry read/write. The +data-warm lifecycle (deploy/undeploy-keep-volume/seed) is integration, proven live on a real recipe +canonical (W1). +""" + +from __future__ import annotations + +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner")) +from harness import canonical, warm # noqa: E402 + + +def test_canonical_domain(): + assert canonical.canonical_domain("custom-html") == "warm-custom-html.ci.commoninternet.net" + assert warm.stable_domain("keycloak") == "warm-keycloak.ci.commoninternet.net" + # stable_domain matches the live-warm keycloak's mapping (no divergence) + assert warm.stable_domain("keycloak") == warm.WARM_DOMAINS["keycloak"] + + +def test_is_enrolled_missing_meta_false(tmp_path, monkeypatch): + # A recipe with no recipe_meta.py is not enrolled. + assert canonical.is_enrolled("definitely-not-a-recipe-xyz") is False + + +def test_is_enrolled_reads_flag(tmp_path, monkeypatch): + # Point the module's tests// lookup at a temp recipe by monkeypatching __file__ dir. + recipe = "tmpwarm" + tests_dir = tmp_path / "tests" / recipe + tests_dir.mkdir(parents=True) + (tests_dir / "recipe_meta.py").write_text("WARM_CANONICAL = True\n") + # canonical.is_enrolled builds the path from canonical.__file__/../../tests/; emulate by + # creating the layout under a fake harness dir and pointing __file__ there. + fake_harness = tmp_path / "runner" / "harness" + fake_harness.mkdir(parents=True) + monkeypatch.setattr(canonical, "__file__", str(fake_harness / "canonical.py")) + assert canonical.is_enrolled(recipe) is True + (tests_dir / "recipe_meta.py").write_text("WARM_CANONICAL = False\n") + assert canonical.is_enrolled(recipe) is False + (tests_dir / "recipe_meta.py").write_text("DEPS = ['keycloak']\n") # flag absent + assert canonical.is_enrolled(recipe) is False + + +def test_registry_roundtrip(tmp_path, monkeypatch): + monkeypatch.setenv("CCCI_WARM_ROOT", str(tmp_path)) + assert canonical.read_registry("custom-html") is None + rec = canonical.write_registry("custom-html", version="1.10.0+x", commit="abc123", status="idle") + assert rec["domain"] == "warm-custom-html.ci.commoninternet.net" + assert rec["version"] == "1.10.0+x" and rec["commit"] == "abc123" and rec["status"] == "idle" + back = canonical.read_registry("custom-html") + assert back == rec + # atomic overwrite to a new status + canonical.write_registry("custom-html", version="1.10.0+x", commit="abc123", status="warm") + assert canonical.read_registry("custom-html")["status"] == "warm" + # the file is valid JSON on disk + with open(canonical.registry_path("custom-html")) as f: + assert json.load(f)["status"] == "warm"