From 68954be53e8b91354a5617918d41ef142ecbce1e Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Wed, 10 Jun 2026 18:57:26 +0000 Subject: [PATCH] =?UTF-8?q?feat(harness):=20P5=20=E2=80=94=20customization?= =?UTF-8?q?=20manifest=20(rcust)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit One block at run start answering "what does this recipe customize?" across every surface (non-default recipe_meta keys, ops.py pre-ops, install_steps.sh, compose.ccci.yml, lifecycle overlays by source, custom-test counts, active CCCI_SKIP_GENERIC* env overrides — !!-flagged when riding a CI run, P2c), printed to the run log and embedded verbatim in results.json under "customization". Pure presentation — building/printing it never influences a verdict; the manifest honors the HC2 repo-local gate so it never advertises code the run will not execute. Unit tests: synthetic recipe exercising every surface -> complete + deterministic + JSON-clean; HC2 invisibility; env-override flagging; render golden lines; build_results threads the dict verbatim (key always present, None when absent). Co-Authored-By: Claude Fable 5 --- runner/harness/manifest.py | 141 ++++++++++++++++++++++++++++++++++ runner/harness/results.py | 4 + runner/run_recipe_ci.py | 10 +++ tests/unit/test_manifest.py | 147 ++++++++++++++++++++++++++++++++++++ tests/unit/test_results.py | 35 +++++++++ 5 files changed, 337 insertions(+) create mode 100644 runner/harness/manifest.py create mode 100644 tests/unit/test_manifest.py diff --git a/runner/harness/manifest.py b/runner/harness/manifest.py new file mode 100644 index 0000000..d60e4f6 --- /dev/null +++ b/runner/harness/manifest.py @@ -0,0 +1,141 @@ +"""Customization manifest (rcust P5; spec §8 R4 mitigation). + +One block at run start answering "what does this recipe customize?" across ALL the surfaces +(recipe_meta keys, hook files, file-presence, run-time env overrides) — printed to the run log and +embedded verbatim in results.json under "customization". PURE PRESENTATION: building or printing +the manifest must never influence any verdict (R7-class invariant). +""" + +from __future__ import annotations + +import os +import re + +from . import discovery, lifecycle +from . import meta as meta_mod + +_PRE_OP_RE = re.compile(r"^def (pre_[a-z]+)\(", re.MULTILINE) + + +def _jsonable(v): + """Manifest values must be JSON-serializable + deterministic: hooks render as '', + tuples become lists.""" + if callable(v): + return "" + if isinstance(v, tuple): + return list(v) + return v + + +def _pre_ops(path: str) -> list[str]: + """The pre_ hook names an ops.py defines (cheap source scan, same approach as + discovery._module_defines — no import).""" + try: + with open(path) as fh: + return sorted(set(_PRE_OP_RE.findall(fh.read()))) + except OSError: + return [] + + +def _custom_counts(recipe: str, repo_local: str | None) -> dict[str, dict[str, int]]: + out: dict[str, dict[str, int]] = {} + for source, path in discovery.custom_tests(recipe, repo_local): + sub = os.path.basename(os.path.dirname(path)) # functional | playwright + out.setdefault(source, {}).setdefault(sub, 0) + out[source][sub] += 1 + return out + + +def build(recipe: str, meta, repo_local: str | None) -> dict: + """Collect the run's resolved customization into one deterministic, JSON-serializable dict. + + Keys: meta_non_default (explicitly-customized recipe_meta keys), hooks (ops.py pre-ops + + install_steps.sh + compose.ccci.yml with their source), overlays (lifecycle overlay files by + op + source), custom_tests (counts per source/subdir), env_overrides (active + CCCI_SKIP_GENERIC* — the dev-only escape hatch, flagged when riding a CI run).""" + hooks: dict = {} + pre_ops: dict[str, list[str]] = {} + for source, d in ( + ("cc-ci", discovery.cc_ci_dir(recipe)), + ("repo-local", discovery._gated(recipe, repo_local)), # noqa: SLF001 — same HC2 gate + ): + if not d: + continue + p = os.path.join(d, "ops.py") + if os.path.isfile(p): + ops = _pre_ops(p) + if ops: + pre_ops[source] = ops + if pre_ops: + hooks["ops.py"] = pre_ops + ist = discovery.install_steps(recipe, repo_local) + if ist: + hooks["install_steps.sh"] = ist[0] + if lifecycle.has_ccci_overlay(recipe): + hooks["compose.ccci.yml"] = "cc-ci" + + overlays = {} + for op in discovery.LIFECYCLE_OPS: + ov = discovery.resolve_overlay_op(recipe, op, repo_local) + if ov: + overlays[op] = ov[0] + + env_overrides = sorted( + k + for k in os.environ + if k.startswith("CCCI_SKIP_GENERIC") + and str(os.environ.get(k) or "").strip().lower() in ("1", "true", "yes", "on") + ) + + return { + "meta_non_default": { + k: _jsonable(v) for k, v in sorted(meta_mod.non_default(meta).items()) + }, + "hooks": hooks, + "overlays": overlays, + "custom_tests": _custom_counts(recipe, repo_local), + "env_overrides": env_overrides, + } + + +def render(recipe: str, manifest: dict) -> str: + """The human block printed at run start (same content as the results.json key).""" + lines = [f"===== customization manifest: {recipe} ====="] + nd = manifest["meta_non_default"] + lines.append( + "meta (non-default): " + + (" ".join(f"{k}={v!r}" for k, v in nd.items()) if nd else "(none — zero-config floor)") + ) + hk = manifest["hooks"] + parts = [] + for source, ops in hk.get("ops.py", {}).items(): + parts.append(f"ops.py[{','.join(ops)}]({source})") + if "install_steps.sh" in hk: + parts.append(f"install_steps.sh({hk['install_steps.sh']})") + if "compose.ccci.yml" in hk: + parts.append(f"compose.ccci.yml({hk['compose.ccci.yml']})") + lines.append("hooks: " + (" ".join(parts) if parts else "(none)")) + ov = manifest["overlays"] + lines.append( + "overlays: " + + (" ".join(f"test_{op}.py({src})" for op, src in ov.items()) if ov else "(none)") + ) + ct = manifest["custom_tests"] + lines.append( + "custom tests: " + + ( + " ".join( + " ".join(f"{sub}/={n}" for sub, n in sorted(counts.items())) + f" ({source})" + for source, counts in sorted(ct.items()) + ) + if ct + else "(none)" + ) + ) + eo = manifest["env_overrides"] + if eo: + suffix = " !! dev-only override active in CI" if os.environ.get("DRONE") else "" + lines.append("env overrides: " + " ".join(f"{k}=1" for k in eo) + suffix) + else: + lines.append("env overrides: (none)") + return "\n".join(lines) diff --git a/runner/harness/results.py b/runner/harness/results.py index 910e827..72a40cf 100644 --- a/runner/harness/results.py +++ b/runner/harness/results.py @@ -203,6 +203,7 @@ def build_results( screenshot: str | None = None, summary_card: str | None = None, expected_na: dict | None = None, + customization: dict | None = None, ) -> dict: """Assemble the full results.json dict (no I/O). `finished_ts` is passed in (the orchestrator stamps it) so this stays pure and deterministic for unit tests. `expected_na` is the recipe's @@ -236,6 +237,9 @@ def build_results( }, "screenshot": screenshot, "summary_card": summary_card, + # rcust P5: the run's resolved customization manifest (pure presentation — consumers must + # never derive a verdict from it). + "customization": customization, } diff --git a/runner/run_recipe_ci.py b/runner/run_recipe_ci.py index aeef548..980b4f8 100644 --- a/runner/run_recipe_ci.py +++ b/runner/run_recipe_ci.py @@ -58,6 +58,9 @@ from harness import ( # noqa: E402 from harness import ( # noqa: E402 deps as deps_mod, ) +from harness import ( # noqa: E402 + manifest as manifest_mod, +) from harness import ( # noqa: E402 meta as meta_mod, ) @@ -880,6 +883,12 @@ def main() -> int: repo_local = snapshot_recipe_tests(recipe) meta = meta_mod.load(recipe) + # Customization manifest (rcust P5, R4): ONE block answering "what does this recipe + # customize?" across all surfaces — printed here and embedded verbatim in results.json under + # "customization". Pure presentation; never influences a verdict. + customization = manifest_mod.build(recipe, meta, repo_local) + print("\n" + manifest_mod.render(recipe, customization) + "\n", flush=True) + # WC4/WC7: opt-in `--quick` fast lane. Requires an existing data-warm canonical; if none, fall # back cleanly to the full COLD run below so the PR is still tested (DECISIONS Phase-2w). if os.environ.get("CCCI_QUICK") == "1" or os.environ.get("MODE") == "quick": @@ -1249,6 +1258,7 @@ def main() -> int: screenshot=screenshot_rel, # Phase 3 U1 (R4): relative PNG name iff capture succeeded finished_ts=time.time(), expected_na=meta.EXPECTED_NA, # declared intentional-skip map (recipe_meta) + customization=customization, # rcust P5: the run-start manifest, verbatim ) # Real (if narrow) leak check: no known infra-secret value may appear in the artifact (R7). blob = json.dumps(data) diff --git a/tests/unit/test_manifest.py b/tests/unit/test_manifest.py new file mode 100644 index 0000000..42c8830 --- /dev/null +++ b/tests/unit/test_manifest.py @@ -0,0 +1,147 @@ +"""Unit tests for the customization manifest (rcust P5; spec §8 R4 mitigation). + +The manifest is PURE PRESENTATION (must never influence a verdict); these tests pin that it is +COMPLETE (every customization surface a synthetic recipe exercises shows up), DETERMINISTIC +(same inputs -> byte-identical JSON), serializable, and HC2-honoring (unapproved repo-local +contributions are invisible). Pure / tmp-file only. Run cold: + cc-ci-run -m pytest tests/unit/test_manifest.py -q +""" + +from __future__ import annotations + +import json +import os +import sys + +sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner")) +from harness import discovery, manifest # noqa: E402 +from harness import meta as meta_mod # noqa: E402 + +RECIPE = "ccci-manifest-fixture" + + +def _mk_synthetic(tmp_path, monkeypatch, approved=True): + """A synthetic recipe dir exercising EVERY manifest surface, plus a repo-local tests dir. + + cc-ci side: meta (2 data keys + 1 hook key non-default), ops.py (2 pre-ops), install_steps.sh, + compose.ccci.yml, test_backup.py overlay, 2 functional + 1 playwright custom tests. + repo-local side: test_restore.py overlay + 1 functional custom test (visible iff approved, HC2). + """ + ccci_root = tmp_path / "cc-ci-tests" + d = ccci_root / RECIPE + (d / "functional").mkdir(parents=True) + (d / "playwright").mkdir() + (d / "recipe_meta.py").write_text( + "HTTP_TIMEOUT = 600\n" + "DEPS = ['keycloak']\n" + "def EXTRA_ENV(ctx):\n return {}\n" + "_PRIVATE = 'exempt'\n" + ) + (d / "ops.py").write_text("def pre_upgrade(ctx):\n pass\n\ndef pre_backup(ctx):\n pass\n") + (d / "install_steps.sh").write_text("#!/usr/bin/env bash\n") + (d / "compose.ccci.yml").write_text("version: '3.8'\n") + (d / "test_backup.py").write_text("# lifecycle overlay\n") + (d / "functional" / "test_a.py").write_text("# custom\n") + (d / "functional" / "test_b.py").write_text("# custom\n") + (d / "playwright" / "test_ui.py").write_text("# custom\n") + + rl = tmp_path / "repo-local" + (rl / "functional").mkdir(parents=True) + (rl / "functional" / "test_c.py").write_text("# repo-local custom\n") + (rl / "test_restore.py").write_text("# repo-local lifecycle overlay\n") + + monkeypatch.setattr(discovery, "cc_ci_dir", lambda r: str(ccci_root / r)) + monkeypatch.setattr(meta_mod, "TESTS_DIR", str(ccci_root)) # compose.ccci.yml discovery + approved_file = tmp_path / "approved.txt" + approved_file.write_text(f"{RECIPE}\n" if approved else "") + monkeypatch.setenv("CCCI_REPO_LOCAL_APPROVED_FILE", str(approved_file)) + + meta = meta_mod.load(RECIPE, tests_dir=str(ccci_root)) + return meta, str(rl) + + +def test_manifest_complete(tmp_path, monkeypatch): + # Every surface the synthetic recipe customizes appears — nothing silently dropped (R4). + meta, rl = _mk_synthetic(tmp_path, monkeypatch) + m = manifest.build(RECIPE, meta, rl) + assert m["meta_non_default"] == { + "DEPS": ["keycloak"], + "EXTRA_ENV": "", + "HTTP_TIMEOUT": 600, + } + assert m["hooks"] == { + "ops.py": {"cc-ci": ["pre_backup", "pre_upgrade"]}, + "install_steps.sh": "cc-ci", + "compose.ccci.yml": "cc-ci", + } + assert m["overlays"] == {"backup": "cc-ci", "restore": "repo-local"} + assert m["custom_tests"] == { + "cc-ci": {"functional": 2, "playwright": 1}, + "repo-local": {"functional": 1}, + } + assert m["env_overrides"] == [] + + +def test_manifest_deterministic_and_serializable(tmp_path, monkeypatch): + meta, rl = _mk_synthetic(tmp_path, monkeypatch) + a = manifest.build(RECIPE, meta, rl) + b = manifest.build(RECIPE, meta, rl) + assert json.dumps(a, sort_keys=True) == json.dumps(b, sort_keys=True) + assert json.loads(json.dumps(a)) == a # round-trips: no callables/tuples leak through + + +def test_manifest_zero_config_floor(tmp_path, monkeypatch): + # A recipe with NO customization at all -> every section empty, render says so explicitly. + ccci_root = tmp_path / "cc-ci-tests" + (ccci_root / RECIPE).mkdir(parents=True) + monkeypatch.setattr(discovery, "cc_ci_dir", lambda r: str(ccci_root / r)) + monkeypatch.setattr(meta_mod, "TESTS_DIR", str(ccci_root)) + monkeypatch.setenv("CCCI_REPO_LOCAL_APPROVED_FILE", str(tmp_path / "missing.txt")) + meta = meta_mod.load(RECIPE, tests_dir=str(ccci_root)) + m = manifest.build(RECIPE, meta, None) + assert m == { + "meta_non_default": {}, + "hooks": {}, + "overlays": {}, + "custom_tests": {}, + "env_overrides": [], + } + out = manifest.render(RECIPE, m) + assert f"===== customization manifest: {RECIPE} =====" in out + assert "(none — zero-config floor)" in out + + +def test_manifest_repo_local_hc2_gate(tmp_path, monkeypatch): + # Unapproved recipe -> repo-local overlay + custom tests INVISIBLE (same default-deny as the + # discovery they ride on; the manifest must not advertise code the run will not execute). + meta, rl = _mk_synthetic(tmp_path, monkeypatch, approved=False) + m = manifest.build(RECIPE, meta, rl) + assert m["overlays"] == {"backup": "cc-ci"} # repo-local test_restore.py gone + assert "repo-local" not in m["custom_tests"] + + +def test_manifest_env_overrides_and_ci_flag(tmp_path, monkeypatch): + meta, rl = _mk_synthetic(tmp_path, monkeypatch) + monkeypatch.setenv("CCCI_SKIP_GENERIC_BACKUP", "1") + monkeypatch.setenv("CCCI_SKIP_GENERIC_UPGRADE", "0") # falsy -> not an active override + m = manifest.build(RECIPE, meta, rl) + assert m["env_overrides"] == ["CCCI_SKIP_GENERIC_BACKUP"] + monkeypatch.delenv("DRONE", raising=False) + assert "!!" not in manifest.render(RECIPE, m) # local dev: no CI warning + monkeypatch.setenv("DRONE", "true") # riding a CI run -> loud flag (P2c) + assert "!! dev-only override active in CI" in manifest.render(RECIPE, m) + + +def test_render_lists_every_surface(tmp_path, monkeypatch): + meta, rl = _mk_synthetic(tmp_path, monkeypatch) + out = manifest.render(RECIPE, manifest.build(RECIPE, meta, rl)) + lines = out.splitlines() + assert lines[0] == f"===== customization manifest: {RECIPE} =====" + assert "meta (non-default): DEPS=['keycloak'] EXTRA_ENV='' HTTP_TIMEOUT=600" in lines + assert ( + "hooks: ops.py[pre_backup,pre_upgrade](cc-ci) install_steps.sh(cc-ci) compose.ccci.yml(cc-ci)" + in lines + ) + assert "overlays: test_backup.py(cc-ci) test_restore.py(repo-local)" in lines + assert "custom tests: functional/=2 playwright/=1 (cc-ci) functional/=1 (repo-local)" in lines + assert "env overrides: (none)" in lines diff --git a/tests/unit/test_results.py b/tests/unit/test_results.py index e8cc91d..d251ef0 100644 --- a/tests/unit/test_results.py +++ b/tests/unit/test_results.py @@ -280,6 +280,41 @@ def test_build_results_threads_expected_na(tmp_path): ) # backup_restore declared; functional passed → clean +def test_build_results_threads_customization(tmp_path): + # rcust P5: the run-start customization manifest lands verbatim under "customization"; + # omitted -> explicit None (key always present in the schema). + recs = [ + { + "tier": "install", + "source": "generic", + "file": "g/test_install.py", + "rc": 0, + "junit": _write(tmp_path, "i.xml", JUNIT_PASS), + }, + ] + cust = { + "meta_non_default": {"HTTP_TIMEOUT": 600}, + "hooks": {"install_steps.sh": "cc-ci"}, + "overlays": {}, + "custom_tests": {"cc-ci": {"functional": 2}}, + "env_overrides": [], + } + kwargs = { + "recipe": "hedgedoc", + "version": "1.2.3", + "pr": "7", + "ref": None, + "records": recs, + "results": _results(), + "backup_capable": True, + "clean_teardown": True, + "no_secret_leak": True, + "finished_ts": 0.0, + } + assert R.build_results(**kwargs, customization=cust)["customization"] == cust + assert R.build_results(**kwargs)["customization"] is None + + def test_write_results_roundtrip(tmp_path): data = {"run_id": "42", "level": 3, "stages": []} path = R.write_results(data, runs_dir_override=str(tmp_path))