From ef44d4658bf2ebca3092077ec6a2723df9ee83f8 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Wed, 27 May 2026 23:27:55 +0100 Subject: [PATCH] =?UTF-8?q?feat(1d):=20G0=20=E2=80=94=20generic=20install?= =?UTF-8?q?=20+=20deploy-once=20orchestrator=20(DG1=20green=20on=20hedgedo?= =?UTF-8?q?c)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - harness/generic.py: recipe-agnostic assert_serving (converged + real HTTP, 404-excluded + not Traefik 404 body + CA-verified trusted wildcard cert), op helpers, backup_capable detect - harness/discovery.py: per-op overlay resolution (repo-local > cc-ci > generic), custom + hook - tests/_generic/: assertion-only tiers (install/upgrade/backup/restore) on the shared deployment - run_recipe_ci.py: deploy-ONCE orchestrator, per-op summary, deploy-count guard (DG4.1) - conftest live_app fixture; lifecycle deploy-count + install-steps hook + pin DOMAIN to run domain DG1 cold-verified green on hedgedoc (pure generic, deploy-count=1, clean teardown). G0 CLAIMED. Co-Authored-By: Claude Opus 4.7 (1M context) --- machine-docs/BACKLOG-1d.md | 21 +-- machine-docs/JOURNAL-1d.md | 39 +++++ machine-docs/STATUS-1d.md | 20 ++- runner/harness/discovery.py | 71 +++++++++ runner/harness/generic.py | 145 +++++++++++++++++ runner/harness/lifecycle.py | 50 +++++- runner/run_recipe_ci.py | 283 ++++++++++++++++++++++----------- tests/_generic/test_backup.py | 17 ++ tests/_generic/test_install.py | 16 ++ tests/_generic/test_restore.py | 15 ++ tests/_generic/test_upgrade.py | 17 ++ tests/conftest.py | 11 ++ 12 files changed, 599 insertions(+), 106 deletions(-) create mode 100644 runner/harness/discovery.py create mode 100644 runner/harness/generic.py create mode 100644 tests/_generic/test_backup.py create mode 100644 tests/_generic/test_install.py create mode 100644 tests/_generic/test_restore.py create mode 100644 tests/_generic/test_upgrade.py diff --git a/machine-docs/BACKLOG-1d.md b/machine-docs/BACKLOG-1d.md index bbe82ed..e12a174 100644 --- a/machine-docs/BACKLOG-1d.md +++ b/machine-docs/BACKLOG-1d.md @@ -2,18 +2,19 @@ ## Build backlog (Builder-only) -### G0 — Generic install + deploy-once orchestrator (DG1) -- [ ] `runner/harness/generic.py`: generic assertion helpers (`assert_serving` — real HTTP, not - Traefik fallback/default cert) + op helpers (`do_upgrade`, `do_backup`, `do_restore`) + +### G0 — Generic install + deploy-once orchestrator (DG1) — CLAIMED, awaiting Adversary +- [x] `runner/harness/generic.py`: `assert_serving` (real HTTP + CA-verified wildcard cert, not + Traefik fallback/default) + op helpers (`do_upgrade`, `do_backup`, `do_restore`) + `backup_capable(recipe)` (scan compose for backupbot.backup). -- [ ] `runner/harness/discovery.py`: per-op overlay resolution (repo-local > cc-ci > generic), +- [x] `runner/harness/discovery.py`: per-op overlay resolution (repo-local > cc-ci > generic), custom-test discovery (both locations, additive), install-steps hook discovery. -- [ ] `tests/_generic/`: assertion-only generic tier files (test_install/upgrade/backup/restore.py). -- [ ] Refactor `run_recipe_ci.py` → deploy-once: deploy base version once, run tiers in order against - the shared deployment, one teardown in finally; per-op result summary. -- [ ] Refactor `tests/conftest.py` fixtures to expose the shared live deployment (no per-tier deploy). -- [ ] Deploy-count guard (`CCCI_DEPLOY_COUNT`) in `lifecycle.deploy_app`; assert ==1 per run. -- [ ] Prove generic install green on custom-html-tiny (no cc-ci/repo-local tests). → claim G0. +- [x] `tests/_generic/`: assertion-only generic tier files (test_install/upgrade/backup/restore.py). +- [x] Refactor `run_recipe_ci.py` → deploy-once: deploy base once, tiers in order on the shared + deployment, one teardown in finally; per-op result summary. +- [x] `tests/conftest.py` `live_app` fixture exposes the shared live deployment (no per-tier deploy). +- [x] Deploy-count guard (`CCCI_DEPLOY_COUNT_FILE`) in `lifecycle.deploy_app`; orchestrator asserts ==1. +- [x] Generic install green on **hedgedoc** (no cc-ci/repo-local tests, deploy-count=1, clean + teardown). custom-html-tiny rejected (empty static volume → 404 zero-config). → G0 CLAIMED. ### G1 — Generic upgrade + backup/restore (DG2, DG3) - [ ] Generic upgrade tier: previous→target in place; reconverge + serving. diff --git a/machine-docs/JOURNAL-1d.md b/machine-docs/JOURNAL-1d.md index 73f15cb..2637de1 100644 --- a/machine-docs/JOURNAL-1d.md +++ b/machine-docs/JOURNAL-1d.md @@ -27,3 +27,42 @@ generic with extend-by-composition; deploy-ONCE with a deploy-count guard; base Seeded STATUS-1d / BACKLOG-1d / JOURNAL-1d. Next: implement G0 (generic.py + discovery.py + tests/_generic/ + deploy-once orchestrator), then verify generic install green on custom-html-tiny. + +## 2026-05-27 — G0 generic install + deploy-once orchestrator: DG1 GREEN + +Built the G0 machinery and proved DG1 end-to-end on the real server: +- `runner/harness/generic.py` — `assert_serving` (services converged + real HTTP in HEALTH_OK [excludes + 404] + not Traefik's 404 body + **CA-verified TLS cert is the trusted wildcard**), op helpers + (`do_upgrade`/`do_backup`/`do_restore`), `backup_capable` (scan compose for backupbot.backup). +- `runner/harness/discovery.py` — per-op overlay resolution (repo-local > cc-ci > generic), custom + test discovery (both locations, additive), install-steps hook discovery. +- `tests/_generic/test_{install,upgrade,backup,restore}.py` — assertion-only tiers using `live_app`. +- `runner/run_recipe_ci.py` — deploy-ONCE orchestrator: base version (prev if upgrade+exists else + target), tiers run against the shared deployment, one teardown in finally, deploy-count guard + + per-op summary. +- `tests/conftest.py` — `live_app` fixture (reads CCCI_APP_DOMAIN; tiers never deploy). +- `lifecycle.deploy_app` — deploy-count recorder + install-steps hook + **pin DOMAIN to the run + domain** (fixes recipes whose .env.sample uses `{{ .Domain }}`, which this abra leaves unexpanded). + +**Two real generic bugs found+fixed via live runs (not "should work"):** +1. custom-html-tiny deploy failed: `DOMAIN={{ .Domain }}` not auto-filled by `abra app new -D` on + 0.13.0-beta → `can't evaluate field Domain`. Fix: `env_set(domain,"DOMAIN",domain)` in deploy_app. +2. `served_cert_subject` used `openssl s_client`, but **openssl is not on the host** (`cc-ci-run` + runtimeInputs has no openssl) → it silently returned None → the "not default cert" check was a + no-op (a DG7 can't-fail smell). Replaced with a pure-Python **CA-verified handshake** (`ssl`): + a publicly-trusted LE wildcard verifies + matches hostname; Traefik's self-signed default fails + verification → a genuine assertion. Verified the verify path on the host: + `ssl.create_default_context()` against ci.commoninternet.net → VERIFIED, CN=*.ci.commoninternet.net, + SAN=[*.ci.commoninternet.net, ci.commoninternet.net]. + +**DG1 evidence (cc-ci, final code):** custom-html-tiny is a static-web-server with an empty content +volume → genuinely serves 404 zero-config (not a serving demo), so picked **hedgedoc** (simple +category, NO cc-ci/repo-local tests → pure generic; backup-capable bonus): +``` +$ RECIPE=hedgedoc STAGES=install cc-ci-run runner/run_recipe_ci.py +===== TIER: install (generic: tests/_generic/test_install.py) ===== +tests/_generic/test_install.py::test_serving PASSED +===== RUN SUMMARY ===== deploy-count = 1 (expect 1) install : pass +$ docker stack ls | grep hedg -> (none — clean teardown) +``` +Lint+format clean (`ruff check`/`ruff format --check` via `nix develop .#lint`). Claiming the G0 gate. diff --git a/machine-docs/STATUS-1d.md b/machine-docs/STATUS-1d.md index 0206dd0..af5fb76 100644 --- a/machine-docs/STATUS-1d.md +++ b/machine-docs/STATUS-1d.md @@ -34,12 +34,24 @@ per-recipe overlay authoring is Phase 2. - **G4** — `!testme` e2e + per-op reporting + docs + cold verify. *Accept: DG6, DG7, DG8 → DONE.* ## In flight -**G0 — generic install + deploy-once orchestrator.** Design recorded in DECISIONS.md (tier model, -override precedence, deploy-once, backup-capability auto-detect, install-steps shell hook). Building -`harness/generic.py` + `harness/discovery.py` + new deploy-once `run_recipe_ci.py` + `tests/_generic/`. +**G1 — generic upgrade + backup/restore (next).** G0 code is in place and DG1 is green; while the +Adversary verifies G0, I'll build/prove the generic upgrade tier (previous→target in place) and the +backup/restore tiers gated on backup-capability (hedgedoc & custom-html are both backup-capable). ## Gate -(none yet — will claim G0 when generic install is green on custom-html-tiny) +**Gate: G0 CLAIMED, awaiting Adversary (DG1).** Generic INSTALL tier is green on **hedgedoc** — +a simple recipe with NO cc-ci/repo-local tests (pure generic), asserting it ACTUALLY serves (services +converged + real HTTP in HEALTH_OK [404 excluded] + not Traefik's 404 body + a CA-verified trusted +wildcard cert, not the default), with **deploy-count = 1** (DG4.1 one-deploy) and clean teardown +(no residual stack). Evidence in JOURNAL-1d (commands + output). custom-html-tiny was rejected as the +demo recipe: it's a static-web-server with an empty content volume → genuinely 404 zero-config. + +To reproduce (cold): on cc-ci, `cd /root/cc-ci && RECIPE=hedgedoc STAGES=install HOME=/root \ +CCCI_JANITOR_MAX_AGE=0 cc-ci-run runner/run_recipe_ci.py` → install: pass, deploy-count=1. + +Design (DECISIONS.md Phase 1d): tier model with the lifecycle OP owned by the shared harness (test +files = assertions only); override precedence repo-local > cc-ci > generic + extend-by-composition; +deploy-once with a deploy-count guard; backup-capability auto-detect; install-steps shell hook. ## Blocked (none) — bootstrap access re-verified @2026-05-27: ssh cc-ci ok (root, NixOS 24.11), abra 0.13.0-beta, diff --git a/runner/harness/discovery.py b/runner/harness/discovery.py new file mode 100644 index 0000000..9bfe68f --- /dev/null +++ b/runner/harness/discovery.py @@ -0,0 +1,71 @@ +"""Overlay / custom-test / install-steps discovery + precedence (Phase 1d, plan §2.5, DG4/DG5). + +The generic is the default for each lifecycle op; a recipe's `test_.py` OVERRIDES it. Sources, +in precedence order (machine-docs/DECISIONS.md): + + lifecycle op (install/upgrade/backup/restore) — exactly ONE assertion file runs: + repo-local tests/test_.py (upstream-authoritative, wins same-name collisions) + > cc-ci tests//test_.py + > generic tests/_generic/test_.py <- always present; the floor + + custom (non-lifecycle) test_*.py — ALL run, additively, from BOTH locations (opt-in). + + install-steps hook — install_steps.sh: repo-local > cc-ci, or none. + +Repo-local = the recipe repo's own tests/ dir, snapshotted after fetch (it survives abra +re-checking-out the recipe to a version tag — see the run orchestrator). +""" + +from __future__ import annotations + +import glob +import os + +LIFECYCLE_OPS = ("install", "upgrade", "backup", "restore") + +ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) +GENERIC_DIR = os.path.join(ROOT, "tests", "_generic") + + +def cc_ci_dir(recipe: str) -> str: + return os.path.join(ROOT, "tests", recipe) + + +def resolve_op(recipe: str, op: str, repo_local_dir: str | None) -> tuple[str, str]: + """Return (source, path) for the single assertion file to run for `op`: + source in {"repo-local","cc-ci","generic"}. The generic file is the floor and always exists.""" + fname = f"test_{op}.py" + if repo_local_dir: + p = os.path.join(repo_local_dir, fname) + if os.path.isfile(p): + return ("repo-local", p) + p = os.path.join(cc_ci_dir(recipe), fname) + if os.path.isfile(p): + return ("cc-ci", p) + return ("generic", os.path.join(GENERIC_DIR, fname)) + + +def custom_tests(recipe: str, repo_local_dir: str | None) -> list[tuple[str, str]]: + """All non-lifecycle test_*.py from cc-ci's tests// and the recipe's repo-local tests/. + These have no generic equivalent and run only when present (opt-in), additively from both.""" + lifecycle_names = {f"test_{op}.py" for op in LIFECYCLE_OPS} + found: list[tuple[str, str]] = [] + for source, d in (("cc-ci", cc_ci_dir(recipe)), ("repo-local", repo_local_dir)): + if not d or not os.path.isdir(d): + continue + for p in sorted(glob.glob(os.path.join(d, "test_*.py"))): + if os.path.basename(p) not in lifecycle_names: + found.append((source, p)) + return found + + +def install_steps(recipe: str, repo_local_dir: str | None) -> tuple[str, str] | None: + """The custom install-steps hook (install_steps.sh) for a recipe, or None. repo-local > cc-ci.""" + if repo_local_dir: + p = os.path.join(repo_local_dir, "install_steps.sh") + if os.path.isfile(p): + return ("repo-local", p) + p = os.path.join(cc_ci_dir(recipe), "install_steps.sh") + if os.path.isfile(p): + return ("cc-ci", p) + return None diff --git a/runner/harness/generic.py b/runner/harness/generic.py new file mode 100644 index 0000000..19551a4 --- /dev/null +++ b/runner/harness/generic.py @@ -0,0 +1,145 @@ +"""Generic, recipe-agnostic lifecycle assertions + op helpers (Phase 1d, plan §2.1). + +These are THE default for each lifecycle op: when a recipe ships no `test_.py` overlay, the +generic tier (tests/_generic/test_.py) runs these against the single shared deployment the +orchestrator brought up. The lifecycle OPERATIONS (upgrade/backup/restore) live here too — owned by +the shared harness, not copy-pasted per recipe (DG7 DRY) — so overlays are assertions-only and may +reuse these by composition (`from harness import generic; generic.assert_serving(...)`). + +Design + precedence: machine-docs/DECISIONS.md (Phase 1d). +""" + +from __future__ import annotations + +import glob +import os +import re +import socket +import ssl + +from . import abra, lifecycle + +# A recipe is backup-capable iff a compose file carries a truthy backupbot.backup label. +_BACKUPBOT_RE = re.compile(r"backupbot\.backup\b[^\n]*\btrue\b", re.IGNORECASE) + + +def _recipe_dir(recipe: str) -> str: + return os.path.expanduser(f"~/.abra/recipes/{recipe}") + + +def backup_capable(recipe: str, meta: dict | None = None) -> bool: + """Whether the harness should run the backup/restore tiers (else they are a clean N/A skip, DG3). + + `recipe_meta.BACKUP_CAPABLE` (bool) overrides; otherwise auto-detect by scanning the recipe's + compose*.yml for a truthy `backupbot.backup` label (the Co-op Cloud backup convention).""" + if meta and "BACKUP_CAPABLE" in meta: + return bool(meta["BACKUP_CAPABLE"]) + for path in glob.glob(os.path.join(_recipe_dir(recipe), "compose*.yml")): + try: + with open(path) as fh: + if _BACKUPBOT_RE.search(fh.read()): + return True + except OSError: + continue + return False + + +def served_cert(domain: str, port: int = 443) -> tuple[bool, str]: + """CA-verified TLS handshake to `domain` (via the gateway passthrough to cc-ci's Traefik). + Returns (verified, detail). The pre-issued wildcard is a publicly-trusted Let's Encrypt cert, so + a real serve VERIFIES against the system CA bundle and matches the hostname; Traefik's self-signed + DEFAULT cert (served only when no router/cert matches the SNI) FAILS verification — so this is a + genuine 'not the default cert' assertion with no openssl dependency. detail carries CN+SAN on + success, or the failure reason.""" + ctx = ssl.create_default_context() # verifies chain against system CAs + checks hostname + try: + with ( + socket.create_connection((domain, port), timeout=20) as sock, + ctx.wrap_socket(sock, server_hostname=domain) as ssock, + ): + cert = ssock.getpeercert() + except ssl.SSLCertVerificationError as e: + return (False, f"cert did not verify (Traefik default/self-signed?): {e}") + except (OSError, ssl.SSLError) as e: + return (False, f"TLS handshake error: {e}") + cn = next( + (v for rdn in cert.get("subject", ()) for k, v in rdn if k == "commonName"), + "", + ) + sans = [v for typ, v in cert.get("subjectAltName", ()) if typ == "DNS"] + return (True, f"CN={cn} SAN={sans}") + + +def assert_serving(domain: str, meta: dict) -> None: + """The single generic "is the app really serving?" assertion (DG1). Proves, end-to-end: + 1. every service in the stack converged (the app's own containers, not just Traefik); + 2. a real HTTP(S) response over the run domain with a status in HEALTH_OK — which EXCLUDES + 404, so a Traefik unmatched-router fallback fails here; + 3. the body is not Traefik's default 404 page; + 4. the served TLS cert is the wildcard, not Traefik's default cert. + No bare sleeps, no health-only shortcut.""" + assert lifecycle.services_converged(domain), f"{domain}: not all services converged" + + path = meta["HEALTH_PATH"] + ok = tuple(meta["HEALTH_OK"]) + status = lifecycle.http_get(domain, path) + assert status in ok, ( + f"{domain}{path}: HTTP {status} not in {ok} — app not serving " + "(a Traefik 404 fallback or an unhealthy backend)" + ) + + if status == 200: + body = lifecycle.http_body(domain, path) + assert ( + "404 page not found" not in body + ), f"{domain}{path}: served Traefik's default 404 page, not the app" + + verified, detail = served_cert(domain) + assert verified, f"{domain}: TLS cert is not the trusted wildcard — {detail}" + assert "commoninternet.net" in detail.lower(), f"{domain}: served cert unexpected — {detail}" + + +def wait_serving(domain: str, meta: dict) -> None: + """Wait for converged + healthy (per recipe_meta timeouts), then run the full serving assertion.""" + lifecycle.wait_healthy( + domain, + ok_codes=tuple(meta["HEALTH_OK"]), + path=meta["HEALTH_PATH"], + deploy_timeout=meta["DEPLOY_TIMEOUT"], + http_timeout=meta["HTTP_TIMEOUT"], + ) + assert_serving(domain, meta) + + +def do_upgrade(domain: str, target: str | None, meta: dict) -> None: + """UPGRADE op (in place on the shared deployment): abra app upgrade -> target, then wait serving.""" + lifecycle.upgrade_app(domain, version=target) + wait_serving(domain, meta) + + +def snapshots(domain: str) -> list[str]: + """Snapshot ids backup-bot-two holds for this app (the backup 'artifact', DG3).""" + proc = abra._run(["app", "backup", "snapshots", domain, "-n", "-o"], check=False) + ids = [] + for ln in proc.stdout.splitlines(): + # restic snapshot rows start with an 8-hex short id + m = re.match(r"^([0-9a-f]{8})\b", ln.strip()) + if m: + ids.append(m.group(1)) + return ids + + +def do_backup(domain: str) -> list[str]: + """BACKUP op: create a snapshot, then assert an artifact now exists (returns snapshot ids).""" + lifecycle.backup_app(domain) + snaps = snapshots(domain) + assert ( + snaps + ), f"{domain}: backup produced no snapshot artifact (abra app backup snapshots empty)" + return snaps + + +def do_restore(domain: str, meta: dict) -> None: + """RESTORE op: restore the latest snapshot, then assert the app is healthy + serving again.""" + lifecycle.restore_app(domain) + wait_serving(domain, meta) diff --git a/runner/harness/lifecycle.py b/runner/harness/lifecycle.py index 7bbd64a..bf1c4d2 100644 --- a/runner/harness/lifecycle.py +++ b/runner/harness/lifecycle.py @@ -89,17 +89,63 @@ def _recipe_extra_env(recipe: str, domain: str) -> dict[str, str]: return {str(k): str(v) for k, v in (ee or {}).items()} -def deploy_app(recipe: str, domain: str, version: str | None = None, secrets: bool = True) -> None: +def _record_deploy() -> None: + """Increment the per-run deploy counter (DG4.1: one deploy per run). No-op unless the + orchestrator set CCCI_DEPLOY_COUNT_FILE — so it never affects standalone/manual use.""" + path = os.environ.get("CCCI_DEPLOY_COUNT_FILE") + if not path: + return + n = 0 + with contextlib.suppress(OSError, ValueError), open(path) as f: + n = int(f.read().strip() or "0") + with contextlib.suppress(OSError), open(path, "w") as f: + f.write(str(n + 1)) + + +def _run_install_steps(hook: tuple[str, str], recipe: str, domain: str) -> None: + """Run a recipe's custom install-steps hook (install_steps.sh) during the install tier — after + `abra app new` + env defaults + secret generate, before deploy (Phase 1d DG5). The hook gets the + app .env path + domain so it can insert secrets / set env / seed before the app comes up.""" + source, path = hook + env_path = os.path.expanduser(f"~/.abra/servers/default/{domain}.env") + print(f" install-steps hook ({source}): {path}", flush=True) + subprocess.run( + ["bash", path], + check=True, + env=dict( + os.environ, + CCCI_APP_DOMAIN=domain, + CCCI_RECIPE=recipe, + CCCI_APP_ENV=env_path, + ), + ) + + +def deploy_app( + recipe: str, + domain: str, + version: str | None = None, + secrets: bool = True, + install_steps_hook: tuple[str, str] | None = None, +) -> None: """Create + configure + deploy an app. Forces LETS_ENCRYPT_ENV='' so traefik serves the wildcard cert via the file provider and NEVER attempts ACME (adversary finding A1). Applies any - per-recipe EXTRA_ENV (recipe_meta.py) before deploy.""" + per-recipe EXTRA_ENV (recipe_meta.py) and the custom install-steps hook (Phase 1d) before deploy.""" + _record_deploy() abra.app_config_remove(domain) # clear any stale .env from a prior crashed run abra.app_new(recipe, domain, version=version, secrets=secrets) + # Pin DOMAIN to the run domain explicitly. `abra app new -D` fills it for recipes whose + # .env.sample uses a literal placeholder, but NOT for ones using a `{{ .Domain }}` Go-template + # (this abra version leaves it unexpanded → deploy fails "can't evaluate field Domain"). Setting + # it ourselves is recipe-agnostic and canonical (the run domain IS the app's domain). + abra.env_set(domain, "DOMAIN", domain) abra.env_set(domain, "LETS_ENCRYPT_ENV", "") for k, v in _recipe_extra_env(recipe, domain).items(): abra.env_set(domain, k, v) if secrets: abra.secret_generate(domain) + if install_steps_hook: + _run_install_steps(install_steps_hook, recipe, domain) abra.deploy(domain) diff --git a/runner/run_recipe_ci.py b/runner/run_recipe_ci.py index e772179..466abc3 100644 --- a/runner/run_recipe_ci.py +++ b/runner/run_recipe_ci.py @@ -1,18 +1,24 @@ #!/usr/bin/env python3 -"""Top-level CI orchestrator (plan §4.3), invoked by the Drone pipeline (or by hand). +"""Top-level CI orchestrator (plan §4.3 + Phase 1d), invoked by the Drone pipeline (or by hand). -Reads the run parameters from env (set by the comment-bridge via Drone build params): +Phase 1d model: deploy the app ONCE, then run lifecycle TIERS against that single shared deployment +(install asserts; upgrade does `abra app upgrade` in place; backup/restore mutate in place; custom +asserts), then ONE teardown in `finally`. Each tier's assertions come from exactly one file — a +recipe overlay if present, else the generic default — discovered by `harness.discovery` +(precedence repo-local > cc-ci > generic). The generic is the default for every op, so ANY recipe is +testable with zero config (DG1–DG4). The lifecycle OPS live in the shared harness (harness.generic), +not per-recipe (DG7 DRY). + +Run parameters from env (set by the comment-bridge via Drone build params): RECIPE recipe name (e.g. custom-html) [required] - REF PR head commit sha [optional; recorded, used for fetch] + REF PR head commit sha [optional; used for fetch + run-domain hash] PR PR number [optional, default 0] SRC head repo full_name on the mirror [optional] - STAGES comma list: install,upgrade,backup [optional, default install] + VERSION upgrade target tag (else newest published) [optional] + STAGES comma filter of tiers to run [optional, default install,upgrade,backup,restore,custom] -It fetches the recipe at REF, then runs the requested per-stage pytest files under -tests//. Teardown is guaranteed by the conftest fixture finalizer. - -Run env (python with pytest+playwright, PLAYWRIGHT_BROWSERS_PATH) is provided by `cc-ci-run` -(modules/harness.nix); invoke as: cc-ci-run runner/run_recipe_ci.py +Run env (python + pytest + playwright) is provided by `cc-ci-run` (nix/modules/harness.nix); +invoke as: cc-ci-run runner/run_recipe_ci.py """ from __future__ import annotations @@ -26,13 +32,9 @@ import tempfile ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(ROOT, "runner")) -from harness import lifecycle, naming # noqa: E402 +from harness import discovery, generic, lifecycle, naming # noqa: E402 -STAGE_FILES = { - "install": "test_install.py", - "upgrade": "test_upgrade.py", - "backup": "test_backup.py", -} +ALL_STAGES = ("install", "upgrade", "backup", "restore", "custom") def _redact_values() -> list[str]: @@ -54,10 +56,10 @@ def _redact_values() -> list[str]: _REDACT = _redact_values() -def run_stage_redacted(cmd: list[str], env: dict | None = None) -> int: - """Run a stage subprocess, streaming its output live (so Drone logs stay tail-able) but masking - any known infra-secret value first. Belt-and-suspenders: the harness already never prints - secrets and abra doesn't echo generated ones.""" +def run_redacted(cmd: list[str], env: dict | None = None) -> int: + """Run a subprocess, streaming output live (so Drone logs stay tail-able) but masking any known + infra-secret value first. Belt-and-suspenders: the harness never prints secrets and abra doesn't + echo generated ones.""" proc = subprocess.Popen( cmd, cwd=ROOT, @@ -101,26 +103,92 @@ def fetch_recipe(recipe: str, ref: str | None, src: str | None) -> None: subprocess.run(["rm", "-rf", dest], check=False) subprocess.run([*git, "clone", "--quiet", url, dest], check=True) subprocess.run([*git, "-C", dest, "checkout", "--quiet", ref], check=True) - # Bring in the published version TAGS from the public upstream so the upgrade stage can deploy - # a previous published version — mirror PR branches carry no release tags (D10: all 3 stages - # must run on a real !testme PR, not skip upgrade). Read-only + guardrail-safe: we only FETCH - # tags from the public upstream, never push to the recipe repo. Plain git (no bot token sent - # to a foreign host). Non-fatal: if upstream is unreachable, upgrade degrades to a skip. + # Bring in published version TAGS from the public upstream so the upgrade tier can deploy a + # previous published version (mirror PR branches carry no release tags). Read-only + plain git + # (no bot token to a foreign host). Non-fatal: if unreachable, upgrade degrades to a skip. upstream = f"https://git.coopcloud.tech/coop-cloud/{recipe}.git" - # Explicit tags refspec — a bare `fetch --tags ` errors "couldn't find remote ref HEAD". subprocess.run( ["git", "-C", dest, "fetch", "--quiet", upstream, "refs/tags/*:refs/tags/*"], check=False, ) else: # Clean re-fetch from the catalogue. rm first so a leftover dir from a prior SRC+REF run - # (which points origin at the private mirror and may lack version tags) can't poison the - # catalogue fetch — that contamination makes `recipe versions`/backup hit the private remote - # and fail "authentication required". + # (origin → private mirror, maybe lacking tags) can't poison the catalogue fetch. subprocess.run(["rm", "-rf", dest], check=False) subprocess.run(["abra", "recipe", "fetch", recipe, "-n"], check=True) +def snapshot_recipe_tests(recipe: str) -> str | None: + """Copy the recipe-shipped tests/ to a stable temp dir, immune to abra re-checking-out the + recipe to a version tag during the run. Returns the snapshot path, or None if no tests/.""" + src = os.path.expanduser(f"~/.abra/recipes/{recipe}/tests") + if not os.path.isdir(src): + return None + has_overlay = glob.glob(os.path.join(src, "test_*.py")) or os.path.isfile( + os.path.join(src, "install_steps.sh") + ) + if not has_overlay: + return None + dst = os.path.join(tempfile.gettempdir(), f"ccci-recipe-tests-{recipe}") + shutil.rmtree(dst, ignore_errors=True) + shutil.copytree(src, dst) + return dst + + +def _load_meta(recipe: str) -> dict: + """Mirror tests/conftest._recipe_meta so the orchestrator's deploy/wait uses the same per-recipe + config the tiers see (timeouts, health path/codes).""" + meta = { + "HEALTH_PATH": "/", + "HEALTH_OK": (200, 301, 302), + "DEPLOY_TIMEOUT": 600, + "HTTP_TIMEOUT": 300, + } + path = os.path.join(ROOT, "tests", recipe, "recipe_meta.py") + if os.path.exists(path): + ns: dict = {} + with open(path) as fh: + exec(compile(fh.read(), path, "exec"), ns) # noqa: S102 (trusted, in-repo) + for k in list(meta) + ["BACKUP_CAPABLE"]: + if k in ns: + meta[k] = ns[k] + return meta + + +def _tier_env(domain: str) -> dict: + return dict(os.environ, CCCI_APP_DOMAIN=domain, CCCI_BASE_URL=f"https://{domain}") + + +def run_op_tier(recipe: str, op: str, repo_local: str | None, domain: str) -> str: + """Run the single assertion file for a lifecycle op (overlay or generic) against the shared + deployment. The file performs the op (upgrade/backup/restore) + asserts; install asserts only + (already deployed). Returns 'pass' | 'fail'.""" + source, path = discovery.resolve_op(recipe, op, repo_local) + rel = os.path.relpath(path, ROOT) + print(f"\n===== TIER: {op} ({source}: {rel}) =====", flush=True) + rc = run_redacted([sys.executable, "-m", "pytest", "-v", "-rA", path], env=_tier_env(domain)) + return "pass" if rc == 0 else "fail" + + +def run_custom(recipe: str, repo_local: str | None, domain: str) -> str: + """Run all discovered non-lifecycle custom test_*.py (both locations, additive). Returns + 'skip' if none defined, else 'pass'/'fail'.""" + customs = discovery.custom_tests(recipe, repo_local) + if not customs: + return "skip" + print("\n===== TIER: custom =====", flush=True) + rc_all = 0 + for source, path in customs: + rel = os.path.relpath(path, ROOT) + print(f" custom ({source}): {rel}", flush=True) + rc = run_redacted( + [sys.executable, "-m", "pytest", "-v", "-rA", path], env=_tier_env(domain) + ) + if rc != 0: + rc_all = rc + return "pass" if rc_all == 0 else "fail" + + def main() -> int: recipe = os.environ.get("RECIPE") if not recipe: @@ -128,75 +196,110 @@ def main() -> int: return 2 ref = os.environ.get("REF") or None src = os.environ.get("SRC") or None - stages = [s.strip() for s in os.environ.get("STAGES", "install").split(",") if s.strip()] + target = os.environ.get("VERSION") or None + stages = { + s.strip() for s in os.environ.get("STAGES", ",".join(ALL_STAGES)).split(",") if s.strip() + } - print(f"== cc-ci run: recipe={recipe} ref={ref} pr={os.environ.get('PR', '0')} stages={stages}") + print( + f"== cc-ci run: recipe={recipe} ref={ref} pr={os.environ.get('PR', '0')} stages={sorted(stages)}" + ) fetch_recipe(recipe, ref, src) - # Snapshot any recipe-shipped tests/ NOW — later abra commands (app ls, deploy, …) re-checkout - # the recipe to a version tag, which would drop the PR's tests/. (D4) - local_tests = snapshot_recipe_tests(recipe) + repo_local = snapshot_recipe_tests(recipe) + meta = _load_meta(recipe) + domain = naming.app_domain(recipe, os.environ.get("PR", "0"), ref) + + # Deploy-once base version: previous published version when the upgrade tier will run and one + # exists (so upgrade goes previous→target in place), else the target (current/$REF). (DECISIONS.) + want_upgrade = "upgrade" in stages + prev = lifecycle.previous_version(recipe) if want_upgrade else None + base = prev or target + backup_cap = generic.backup_capable(recipe, meta) + hook = discovery.install_steps(recipe, repo_local) + + # Deploy-count guard (DG4.1): exactly one deploy_app() per run. + countfile = os.path.join(tempfile.gettempdir(), f"ccci-deploys-{domain}") + with open(countfile, "w") as f: + f.write("0") + os.environ["CCCI_DEPLOY_COUNT_FILE"] = countfile + + results: dict[str, str] = {} + lifecycle.janitor() + try: + # ---- deploy ONCE + wait ready (the single deployment all tiers share) ---- + try: + lifecycle.deploy_app( + recipe, domain, version=base, secrets=True, install_steps_hook=hook + ) + lifecycle.wait_healthy( + domain, + ok_codes=tuple(meta["HEALTH_OK"]), + path=meta["HEALTH_PATH"], + deploy_timeout=meta["DEPLOY_TIMEOUT"], + http_timeout=meta["HTTP_TIMEOUT"], + ) + deploy_ok = True + except Exception as e: # noqa: BLE001 — a failed deploy is a reported INSTALL failure, not a crash + print(f"!! deploy/readiness failed: {e}", flush=True) + deploy_ok = False + + # ---- INSTALL tier (always) ---- + if "install" in stages: + results["install"] = ( + run_op_tier(recipe, "install", repo_local, domain) if deploy_ok else "fail" + ) + + if deploy_ok: + # ---- UPGRADE tier ---- + if "upgrade" in stages: + results["upgrade"] = ( + run_op_tier(recipe, "upgrade", repo_local, domain) + if prev + else "skip" # only one published version → nothing to upgrade from + ) + # ---- BACKUP + RESTORE tiers (backup-capable only; else clean N/A) ---- + if "backup" in stages: + results["backup"] = ( + run_op_tier(recipe, "backup", repo_local, domain) if backup_cap else "skip" + ) + if "restore" in stages: + results["restore"] = ( + run_op_tier(recipe, "restore", repo_local, domain) if backup_cap else "skip" + ) + # ---- CUSTOM tier ---- + if "custom" in stages: + results["custom"] = run_custom(recipe, repo_local, domain) + else: + # install failed → the shared deployment is dead; remaining tiers cannot run on it. + for op in ("upgrade", "backup", "restore", "custom"): + if op in stages: + results[op] = "skip" + finally: + lifecycle.teardown_app(domain, verify=False) + + # ---- deploy-count assertion (DG4.1) ---- + with open(countfile) as f: + deploy_count = int(f.read().strip() or "0") + os.remove(countfile) + + # ---- per-op summary (DG6 feed) ---- + print("\n===== RUN SUMMARY =====", flush=True) + print(f"deploy-count = {deploy_count} (expect 1)") + order = [s for s in ALL_STAGES if s in results] + for op in order: + print(f" {op:8s}: {results[op]}") - test_dir = os.path.join(ROOT, "tests", recipe) overall = 0 - ran = 0 - for stage in stages: - fname = STAGE_FILES.get(stage) - if not fname: - print(f"unknown stage {stage}", file=sys.stderr) - return 2 - path = os.path.join(test_dir, fname) - if not os.path.exists(path): - print(f" (skip {stage}: {path} not present)") - continue - print(f"\n===== STAGE: {stage} =====", flush=True) - # each stage is its own pytest invocation => its own reported result (D2 separate stages) - rc = run_stage_redacted([sys.executable, "-m", "pytest", "-v", "-rA", path]) - ran += 1 - if rc != 0: - overall = rc - # D4: recipe-local tests. If the recipe repo ships a tests/ dir, deploy the app and run those - # tests against the LIVE deployment (contract: CCCI_BASE_URL + CCCI_APP_DOMAIN env), merging the - # result into this run as another reported stage. Teardown is guaranteed. - rc = run_recipe_local(recipe, local_tests) - if rc is not None: - ran += 1 - if rc != 0: - overall = rc - - if ran == 0: - print("no stage test files found", file=sys.stderr) + if deploy_count != 1: + print(f"!! deploy-count {deploy_count} != 1 (DG4.1 violation)", file=sys.stderr) + overall = 1 + if any(v == "fail" for v in results.values()): + overall = 1 + if not results: + print("no tiers ran", file=sys.stderr) return 1 return overall -def snapshot_recipe_tests(recipe: str) -> str | None: - """Copy the recipe-shipped tests/ to a stable temp dir, immune to abra re-checking-out the - recipe to a version tag during the run. Returns the snapshot path, or None if no tests/.""" - src = os.path.expanduser(f"~/.abra/recipes/{recipe}/tests") - if not os.path.isdir(src) or not glob.glob(os.path.join(src, "test_*.py")): - return None - dst = os.path.join(tempfile.gettempdir(), f"ccci-recipe-tests-{recipe}") - shutil.rmtree(dst, ignore_errors=True) - shutil.copytree(src, dst) - return dst - - -def run_recipe_local(recipe: str, local_tests: str | None) -> int | None: - if not local_tests: - return None # recipe ships no tests/ — D4 is a no-op for it - print("\n===== STAGE: recipe-local (D4) =====", flush=True) - domain = naming.app_domain(recipe, os.environ.get("PR", "0"), os.environ.get("REF")) - lifecycle.janitor() - try: - lifecycle.deploy_app(recipe, domain, version=os.environ.get("VERSION") or None) - lifecycle.wait_healthy(domain) - env = dict(os.environ, CCCI_APP_DOMAIN=domain, CCCI_BASE_URL=f"https://{domain}") - return run_stage_redacted( - [sys.executable, "-m", "pytest", "-v", "-rA", local_tests], env=env - ) - finally: - lifecycle.teardown_app(domain, verify=False) - - if __name__ == "__main__": raise SystemExit(main()) diff --git a/tests/_generic/test_backup.py b/tests/_generic/test_backup.py new file mode 100644 index 0000000..74cf15a --- /dev/null +++ b/tests/_generic/test_backup.py @@ -0,0 +1,17 @@ +"""Generic BACKUP tier (Phase 1d DG3) — recipe-agnostic, backup-capable recipes only. + +Runs `abra app backup create` against the shared live deployment and asserts a snapshot artifact is +produced (abra app backup snapshots is non-empty). Honest limit: the generic verifies the backup +MECHANISM, not app-specific data integrity — that's a recipe overlay (test_backup.py seeds a marker). +For recipes that declare no backup config the orchestrator skips this tier as N/A (not a failure).""" + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner")) +from harness import generic # noqa: E402 + + +def test_backup_artifact(live_app, meta): + snaps = generic.do_backup(live_app) + assert snaps, "backup produced no snapshot artifact" diff --git a/tests/_generic/test_install.py b/tests/_generic/test_install.py new file mode 100644 index 0000000..3c9abc7 --- /dev/null +++ b/tests/_generic/test_install.py @@ -0,0 +1,16 @@ +"""Generic INSTALL tier (Phase 1d DG1) — recipe-agnostic. + +The orchestrator has already deployed the app ONCE (deploy-once, DG4.1) and waited for it to +converge. This tier asserts the app is ACTUALLY SERVING over real HTTPS through Traefik — not a +404 fallback, not the default cert, not health-only. Runs for ANY recipe that ships no +test_install.py overlay (the invariant: no overlay ⇒ generic runs).""" + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner")) +from harness import generic # noqa: E402 + + +def test_serving(live_app, meta): + generic.assert_serving(live_app, meta) diff --git a/tests/_generic/test_restore.py b/tests/_generic/test_restore.py new file mode 100644 index 0000000..a0c1f1f --- /dev/null +++ b/tests/_generic/test_restore.py @@ -0,0 +1,15 @@ +"""Generic RESTORE tier (Phase 1d DG3) — recipe-agnostic, backup-capable recipes only. + +Restores the latest snapshot (produced by the backup tier on the same shared deployment) and asserts +the restore completes and the app is healthy + serving afterwards. App-specific data-integrity +(marker survives) is a recipe overlay (test_restore.py); the generic verifies the restore mechanism.""" + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner")) +from harness import generic # noqa: E402 + + +def test_restore_healthy(live_app, meta): + generic.do_restore(live_app, meta) diff --git a/tests/_generic/test_upgrade.py b/tests/_generic/test_upgrade.py new file mode 100644 index 0000000..3ca3f9e --- /dev/null +++ b/tests/_generic/test_upgrade.py @@ -0,0 +1,17 @@ +"""Generic UPGRADE tier (Phase 1d DG2) — recipe-agnostic. + +The orchestrator deployed the PREVIOUS published version once; this tier upgrades it IN PLACE +(abra app upgrade) to the target (VERSION env, else newest published) on the same live deployment, +then asserts it reconverges and still serves. Data-continuity is a recipe overlay (test_upgrade.py), +not the generic — the generic verifies the upgrade mechanism + still-serving.""" + +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner")) +from harness import generic # noqa: E402 + + +def test_upgrade_reconverges(live_app, meta): + target = os.environ.get("VERSION") or None + generic.do_upgrade(live_app, target, meta) diff --git a/tests/conftest.py b/tests/conftest.py index cb69506..bbfe287 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -61,6 +61,17 @@ def meta(recipe) -> dict: return _recipe_meta(recipe) +@pytest.fixture(scope="session") +def live_app() -> str: + """Phase 1d shared-deployment contract: the orchestrator deploys ONCE and runs each tier + (generic or overlay) as its own pytest invocation against that single live deployment, passing + its domain in CCCI_APP_DOMAIN. Tiers are assertion-only (and lifecycle ops mutate in place) — + they NEVER deploy or tear down. This guarantees one deploy + one teardown per run (DG4.1).""" + domain = os.environ.get("CCCI_APP_DOMAIN") + assert domain, "CCCI_APP_DOMAIN not set — a tier must run under the deploy-once orchestrator" + return domain + + def _wait_healthy(domain, meta): lifecycle.wait_healthy( domain,