cc-ci/runner/run_recipe_ci.py

#!/usr/bin/env python3
"""Top-level CI orchestrator (plan §4.3), invoked by the Drone pipeline (or by hand).

Reads the run parameters from env (set by the comment-bridge via Drone build params):
  RECIPE   recipe name (e.g. custom-html)        [required]
  REF      PR head commit sha                     [optional; recorded, used for fetch]
  PR       PR number                              [optional, default 0]
  SRC      head repo full_name on the mirror      [optional]
  STAGES   comma list: install,upgrade,backup     [optional, default install]

It fetches the recipe at REF, then runs the requested per-stage pytest files under
tests/<recipe>/. Teardown is guaranteed by the conftest fixture finalizer.

Run env (python with pytest+playwright, PLAYWRIGHT_BROWSERS_PATH) is provided by `cc-ci-run`
(modules/harness.nix); invoke as:  cc-ci-run runner/run_recipe_ci.py
"""
from __future__ import annotations

import glob
import os
import shutil
import subprocess
import sys
import tempfile

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
sys.path.insert(0, os.path.join(ROOT, "runner"))
from harness import lifecycle, naming  # noqa: E402
STAGE_FILES = {
    "install": "test_install.py",
    "upgrade": "test_upgrade.py",
    "backup": "test_backup.py",
}


def _redact_values() -> list[str]:
    """Values to scrub from published logs (D6 redaction filter, plan §4.4). The infra secrets
    materialised at /run/secrets/* — if any subprocess ever echoes one, mask it. Only >=8-char
    values, so it never false-positives on short strings / SHAs."""
    vals = set()
    for p in glob.glob("/run/secrets/*"):
        try:
            v = open(p).read().strip()
        except OSError:
            continue
        if len(v) >= 8:
            vals.add(v)
    return sorted(vals, key=len, reverse=True)


_REDACT = _redact_values()


def run_stage_redacted(cmd: list[str], env: dict | None = None) -> int:
    """Run a stage subprocess, streaming its output live (so Drone logs stay tail-able) but masking
    any known infra-secret value first. Belt-and-suspenders: the harness already never prints
    secrets and abra doesn't echo generated ones."""
    proc = subprocess.Popen(cmd, cwd=ROOT, env=env, stdout=subprocess.PIPE,
                            stderr=subprocess.STDOUT, text=True, bufsize=1)
    assert proc.stdout is not None
    for line in proc.stdout:
        for v in _REDACT:
            if v in line:
                line = line.replace(v, "***REDACTED***")
        sys.stdout.write(line)
        sys.stdout.flush()
    return proc.wait()


def _gitea_token() -> str | None:
    tok = os.environ.get("GITEA_TOKEN")
    if not tok and os.path.exists("/run/secrets/bridge_gitea_token"):
        tok = open("/run/secrets/bridge_gitea_token").read().strip()
    return tok or None


def fetch_recipe(recipe: str, ref: str | None, src: str | None) -> None:
    """Make the recipe available at the code under test. If SRC+REF point at the mirror PR,
    clone it at that ref; otherwise fetch the catalogue copy. Private mirror repos need the bot
    token — passed via a per-command http.extraHeader (not persisted in .git/config, not printed)."""
    recipes_dir = os.path.expanduser("~/.abra/recipes")
    os.makedirs(recipes_dir, exist_ok=True)
    dest = os.path.join(recipes_dir, recipe)
    if src and ref:
        url = f"https://git.autonomic.zone/{src}.git"
        git = ["git"]
        tok = _gitea_token()
        if tok:
            git += ["-c", f"http.extraHeader=Authorization: token {tok}"]
        subprocess.run(["rm", "-rf", dest], check=False)
        subprocess.run([*git, "clone", "--quiet", url, dest], check=True)
        subprocess.run([*git, "-C", dest, "checkout", "--quiet", ref], check=True)
        # Bring in the published version TAGS from the public upstream so the upgrade stage can deploy
        # a previous published version — mirror PR branches carry no release tags (D10: all 3 stages
        # must run on a real !testme PR, not skip upgrade). Read-only + guardrail-safe: we only FETCH
        # tags from the public upstream, never push to the recipe repo. Plain git (no bot token sent
        # to a foreign host). Non-fatal: if upstream is unreachable, upgrade degrades to a skip.
        upstream = f"https://git.coopcloud.tech/coop-cloud/{recipe}.git"
        # Explicit tags refspec — a bare `fetch --tags <url>` errors "couldn't find remote ref HEAD".
        subprocess.run(["git", "-C", dest, "fetch", "--quiet", upstream,
                        "refs/tags/*:refs/tags/*"], check=False)
    else:
        # Clean re-fetch from the catalogue. rm first so a leftover dir from a prior SRC+REF run
        # (which points origin at the private mirror and may lack version tags) can't poison the
        # catalogue fetch — that contamination makes `recipe versions`/backup hit the private remote
        # and fail "authentication required".
        subprocess.run(["rm", "-rf", dest], check=False)
        subprocess.run(["abra", "recipe", "fetch", recipe, "-n"], check=True)


def main() -> int:
    recipe = os.environ.get("RECIPE")
    if not recipe:
        print("RECIPE env is required", file=sys.stderr)
        return 2
    ref = os.environ.get("REF") or None
    src = os.environ.get("SRC") or None
    stages = [s.strip() for s in os.environ.get("STAGES", "install").split(",") if s.strip()]

    print(f"== cc-ci run: recipe={recipe} ref={ref} pr={os.environ.get('PR', '0')} stages={stages}")
    fetch_recipe(recipe, ref, src)
    # Snapshot any recipe-shipped tests/ NOW — later abra commands (app ls, deploy, …) re-checkout
    # the recipe to a version tag, which would drop the PR's tests/. (D4)
    local_tests = snapshot_recipe_tests(recipe)

    test_dir = os.path.join(ROOT, "tests", recipe)
    overall = 0
    ran = 0
    for stage in stages:
        fname = STAGE_FILES.get(stage)
        if not fname:
            print(f"unknown stage {stage}", file=sys.stderr)
            return 2
        path = os.path.join(test_dir, fname)
        if not os.path.exists(path):
            print(f"  (skip {stage}: {path} not present)")
            continue
        print(f"\n===== STAGE: {stage} =====", flush=True)
        # each stage is its own pytest invocation => its own reported result (D2 separate stages)
        rc = run_stage_redacted([sys.executable, "-m", "pytest", "-v", "-rA", path])
        ran += 1
        if rc != 0:
            overall = rc
    # D4: recipe-local tests. If the recipe repo ships a tests/ dir, deploy the app and run those
    # tests against the LIVE deployment (contract: CCCI_BASE_URL + CCCI_APP_DOMAIN env), merging the
    # result into this run as another reported stage. Teardown is guaranteed.
    rc = run_recipe_local(recipe, local_tests)
    if rc is not None:
        ran += 1
        if rc != 0:
            overall = rc

    if ran == 0:
        print("no stage test files found", file=sys.stderr)
        return 1
    return overall


def snapshot_recipe_tests(recipe: str) -> str | None:
    """Copy the recipe-shipped tests/ to a stable temp dir, immune to abra re-checking-out the
    recipe to a version tag during the run. Returns the snapshot path, or None if no tests/."""
    src = os.path.expanduser(f"~/.abra/recipes/{recipe}/tests")
    if not os.path.isdir(src) or not glob.glob(os.path.join(src, "test_*.py")):
        return None
    dst = os.path.join(tempfile.gettempdir(), f"ccci-recipe-tests-{recipe}")
    shutil.rmtree(dst, ignore_errors=True)
    shutil.copytree(src, dst)
    return dst


def run_recipe_local(recipe: str, local_tests: str | None) -> int | None:
    if not local_tests:
        return None  # recipe ships no tests/ — D4 is a no-op for it
    print("\n===== STAGE: recipe-local (D4) =====", flush=True)
    domain = naming.app_domain(recipe, os.environ.get("PR", "0"), os.environ.get("REF"))
    lifecycle.janitor()
    try:
        lifecycle.deploy_app(recipe, domain, version=os.environ.get("VERSION") or None)
        lifecycle.wait_healthy(domain)
        env = dict(os.environ, CCCI_APP_DOMAIN=domain, CCCI_BASE_URL=f"https://{domain}")
        return run_stage_redacted([sys.executable, "-m", "pytest", "-v", "-rA", local_tests], env=env)
    finally:
        lifecycle.teardown_app(domain, verify=False)


if __name__ == "__main__":
    raise SystemExit(main())