#!/usr/bin/env python3 """Top-level CI orchestrator (plan §4.3), invoked by the Drone pipeline (or by hand). Reads the run parameters from env (set by the comment-bridge via Drone build params): RECIPE recipe name (e.g. custom-html) [required] REF PR head commit sha [optional; recorded, used for fetch] PR PR number [optional, default 0] SRC head repo full_name on the mirror [optional] STAGES comma list: install,upgrade,backup [optional, default install] It fetches the recipe at REF, then runs the requested per-stage pytest files under tests//. Teardown is guaranteed by the conftest fixture finalizer. Run env (python with pytest+playwright, PLAYWRIGHT_BROWSERS_PATH) is provided by `cc-ci-run` (modules/harness.nix); invoke as: cc-ci-run runner/run_recipe_ci.py """ from __future__ import annotations import glob import os import shutil import subprocess import sys import tempfile ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.insert(0, os.path.join(ROOT, "runner")) from harness import lifecycle, naming # noqa: E402 STAGE_FILES = { "install": "test_install.py", "upgrade": "test_upgrade.py", "backup": "test_backup.py", } def _redact_values() -> list[str]: """Values to scrub from published logs (D6 redaction filter, plan §4.4). The infra secrets materialised at /run/secrets/* — if any subprocess ever echoes one, mask it. Only >=8-char values, so it never false-positives on short strings / SHAs.""" vals = set() for p in glob.glob("/run/secrets/*"): try: v = open(p).read().strip() except OSError: continue if len(v) >= 8: vals.add(v) return sorted(vals, key=len, reverse=True) _REDACT = _redact_values() def run_stage_redacted(cmd: list[str], env: dict | None = None) -> int: """Run a stage subprocess, streaming its output live (so Drone logs stay tail-able) but masking any known infra-secret value first. Belt-and-suspenders: the harness already never prints secrets and abra doesn't echo generated ones.""" proc = subprocess.Popen(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True, bufsize=1) assert proc.stdout is not None for line in proc.stdout: for v in _REDACT: if v in line: line = line.replace(v, "***REDACTED***") sys.stdout.write(line) sys.stdout.flush() return proc.wait() def _gitea_token() -> str | None: tok = os.environ.get("GITEA_TOKEN") if not tok and os.path.exists("/run/secrets/bridge_gitea_token"): tok = open("/run/secrets/bridge_gitea_token").read().strip() return tok or None def fetch_recipe(recipe: str, ref: str | None, src: str | None) -> None: """Make the recipe available at the code under test. If SRC+REF point at the mirror PR, clone it at that ref; otherwise fetch the catalogue copy. Private mirror repos need the bot token — passed via a per-command http.extraHeader (not persisted in .git/config, not printed).""" recipes_dir = os.path.expanduser("~/.abra/recipes") os.makedirs(recipes_dir, exist_ok=True) dest = os.path.join(recipes_dir, recipe) if src and ref: url = f"https://git.autonomic.zone/{src}.git" git = ["git"] tok = _gitea_token() if tok: git += ["-c", f"http.extraHeader=Authorization: token {tok}"] subprocess.run(["rm", "-rf", dest], check=False) subprocess.run([*git, "clone", "--quiet", url, dest], check=True) subprocess.run([*git, "-C", dest, "checkout", "--quiet", ref], check=True) # Bring in the published version TAGS from the public upstream so the upgrade stage can deploy # a previous published version — mirror PR branches carry no release tags (D10: all 3 stages # must run on a real !testme PR, not skip upgrade). Read-only + guardrail-safe: we only FETCH # tags from the public upstream, never push to the recipe repo. Plain git (no bot token sent # to a foreign host). Non-fatal: if upstream is unreachable, upgrade degrades to a skip. upstream = f"https://git.coopcloud.tech/coop-cloud/{recipe}.git" # Explicit tags refspec — a bare `fetch --tags ` errors "couldn't find remote ref HEAD". subprocess.run(["git", "-C", dest, "fetch", "--quiet", upstream, "refs/tags/*:refs/tags/*"], check=False) else: # Clean re-fetch from the catalogue. rm first so a leftover dir from a prior SRC+REF run # (which points origin at the private mirror and may lack version tags) can't poison the # catalogue fetch — that contamination makes `recipe versions`/backup hit the private remote # and fail "authentication required". subprocess.run(["rm", "-rf", dest], check=False) subprocess.run(["abra", "recipe", "fetch", recipe, "-n"], check=True) def main() -> int: recipe = os.environ.get("RECIPE") if not recipe: print("RECIPE env is required", file=sys.stderr) return 2 ref = os.environ.get("REF") or None src = os.environ.get("SRC") or None stages = [s.strip() for s in os.environ.get("STAGES", "install").split(",") if s.strip()] print(f"== cc-ci run: recipe={recipe} ref={ref} pr={os.environ.get('PR', '0')} stages={stages}") fetch_recipe(recipe, ref, src) # Snapshot any recipe-shipped tests/ NOW — later abra commands (app ls, deploy, …) re-checkout # the recipe to a version tag, which would drop the PR's tests/. (D4) local_tests = snapshot_recipe_tests(recipe) test_dir = os.path.join(ROOT, "tests", recipe) overall = 0 ran = 0 for stage in stages: fname = STAGE_FILES.get(stage) if not fname: print(f"unknown stage {stage}", file=sys.stderr) return 2 path = os.path.join(test_dir, fname) if not os.path.exists(path): print(f" (skip {stage}: {path} not present)") continue print(f"\n===== STAGE: {stage} =====", flush=True) # each stage is its own pytest invocation => its own reported result (D2 separate stages) rc = run_stage_redacted([sys.executable, "-m", "pytest", "-v", "-rA", path]) ran += 1 if rc != 0: overall = rc # D4: recipe-local tests. If the recipe repo ships a tests/ dir, deploy the app and run those # tests against the LIVE deployment (contract: CCCI_BASE_URL + CCCI_APP_DOMAIN env), merging the # result into this run as another reported stage. Teardown is guaranteed. rc = run_recipe_local(recipe, local_tests) if rc is not None: ran += 1 if rc != 0: overall = rc if ran == 0: print("no stage test files found", file=sys.stderr) return 1 return overall def snapshot_recipe_tests(recipe: str) -> str | None: """Copy the recipe-shipped tests/ to a stable temp dir, immune to abra re-checking-out the recipe to a version tag during the run. Returns the snapshot path, or None if no tests/.""" src = os.path.expanduser(f"~/.abra/recipes/{recipe}/tests") if not os.path.isdir(src) or not glob.glob(os.path.join(src, "test_*.py")): return None dst = os.path.join(tempfile.gettempdir(), f"ccci-recipe-tests-{recipe}") shutil.rmtree(dst, ignore_errors=True) shutil.copytree(src, dst) return dst def run_recipe_local(recipe: str, local_tests: str | None) -> int | None: if not local_tests: return None # recipe ships no tests/ — D4 is a no-op for it print("\n===== STAGE: recipe-local (D4) =====", flush=True) domain = naming.app_domain(recipe, os.environ.get("PR", "0"), os.environ.get("REF")) lifecycle.janitor() try: lifecycle.deploy_app(recipe, domain, version=os.environ.get("VERSION") or None) lifecycle.wait_healthy(domain) env = dict(os.environ, CCCI_APP_DOMAIN=domain, CCCI_BASE_URL=f"https://{domain}") return run_stage_redacted([sys.executable, "-m", "pytest", "-v", "-rA", local_tests], env=env) finally: lifecycle.teardown_app(domain, verify=False) if __name__ == "__main__": raise SystemExit(main())