diff --git a/runner/harness/discovery.py b/runner/harness/discovery.py index 9bfe68f..00b0c7d 100644 --- a/runner/harness/discovery.py +++ b/runner/harness/discovery.py @@ -1,19 +1,25 @@ -"""Overlay / custom-test / install-steps discovery + precedence (Phase 1d, plan §2.5, DG4/DG5). +"""Overlay / custom-test / install-steps discovery + precedence (Phase 1d/1e, DG4/DG5 + HC2/HC3). -The generic is the default for each lifecycle op; a recipe's `test_.py` OVERRIDES it. Sources, -in precedence order (machine-docs/DECISIONS.md): +The generic is the default floor for each lifecycle op and, per Phase 1e HC3, runs ADDITIVELY +alongside a recipe overlay by default (the orchestrator owns the op; both assertion sets evaluate the +shared post-op state). Discovery here only locates the candidate assertion files + the install-steps +hook; the orchestrator decides additive-vs-skip. Sources, in precedence order +(machine-docs/DECISIONS.md): - lifecycle op (install/upgrade/backup/restore) — exactly ONE assertion file runs: + lifecycle op (install/upgrade/backup/restore) — the OVERLAY assertion file, if any: repo-local tests/test_.py (upstream-authoritative, wins same-name collisions) > cc-ci tests//test_.py - > generic tests/_generic/test_.py <- always present; the floor + (the generic tests/_generic/test_.py is the always-present floor, run separately by default) custom (non-lifecycle) test_*.py — ALL run, additively, from BOTH locations (opt-in). install-steps hook — install_steps.sh: repo-local > cc-ci, or none. Repo-local = the recipe repo's own tests/ dir, snapshotted after fetch (it survives abra -re-checking-out the recipe to a version tag — see the run orchestrator). +re-checking-out the recipe to a version tag — see the run orchestrator). It is PR-author-controlled +code that runs on the CI host with /run/secrets/* present, so per Phase 1e HC2 it is **default-deny**: +the repo-local source is consulted ONLY when the recipe is on the cc-ci approval allowlist +(`tests/repo-local-approved.txt`). Otherwise precedence is cc-ci > generic only. """ from __future__ import annotations @@ -25,32 +31,81 @@ LIFECYCLE_OPS = ("install", "upgrade", "backup", "restore") ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) GENERIC_DIR = os.path.join(ROOT, "tests", "_generic") +APPROVED_FILE = os.path.join(ROOT, "tests", "repo-local-approved.txt") def cc_ci_dir(recipe: str) -> str: return os.path.join(ROOT, "tests", recipe) -def resolve_op(recipe: str, op: str, repo_local_dir: str | None) -> tuple[str, str]: - """Return (source, path) for the single assertion file to run for `op`: - source in {"repo-local","cc-ci","generic"}. The generic file is the floor and always exists.""" +def approved_file_path() -> str: + """Location of the repo-local approval allowlist. Overridable via CCCI_REPO_LOCAL_APPROVED_FILE + (used by tests + by the Adversary to demonstrate approved-vs-not without editing the checked-in + file). Default: the git-tracked tests/repo-local-approved.txt.""" + return os.environ.get("CCCI_REPO_LOCAL_APPROVED_FILE", APPROVED_FILE) + + +def approved_recipes() -> set[str]: + """Recipes whose repo-local (PR-authored) code is trusted to execute (HC2). One name per line in + the allowlist; `#` comments + blank lines ignored. Missing file ⇒ empty set ⇒ default-deny.""" + names: set[str] = set() + try: + with open(approved_file_path()) as fh: + for raw in fh: + line = raw.split("#", 1)[0].strip() + if line: + names.add(line) + except OSError: + pass + return names + + +def repo_local_approved(recipe: str) -> bool: + """True iff `recipe` is on the cc-ci repo-local approval allowlist (default-deny, HC2).""" + return recipe in approved_recipes() + + +def _gated(recipe: str, repo_local_dir: str | None) -> str | None: + """The repo-local dir to actually consult: the given dir if the recipe is approved, else None + (default-deny). Centralizes the HC2 gate so every discovery function honors it identically.""" + return repo_local_dir if (repo_local_dir and repo_local_approved(recipe)) else None + + +def resolve_overlay_op(recipe: str, op: str, repo_local_dir: str | None) -> tuple[str, str] | None: + """Return (source, path) for the OVERLAY assertion file for `op` (repo-local > cc-ci), or None if + the recipe ships no overlay for it. The generic floor is handled separately by the orchestrator + (HC3 additive). Repo-local is consulted only for allowlist-approved recipes (HC2).""" fname = f"test_{op}.py" - if repo_local_dir: - p = os.path.join(repo_local_dir, fname) + rl = _gated(recipe, repo_local_dir) + if rl: + p = os.path.join(rl, fname) if os.path.isfile(p): return ("repo-local", p) p = os.path.join(cc_ci_dir(recipe), fname) if os.path.isfile(p): return ("cc-ci", p) - return ("generic", os.path.join(GENERIC_DIR, fname)) + return None + + +def generic_op(op: str) -> tuple[str, str]: + """The always-present generic assertion file for `op` (the floor, HC3).""" + return ("generic", os.path.join(GENERIC_DIR, f"test_{op}.py")) + + +def resolve_op(recipe: str, op: str, repo_local_dir: str | None) -> tuple[str, str]: + """Back-compat single-file resolver (override semantics): overlay if present, else generic. + Phase-1e orchestration uses resolve_overlay_op + generic_op (additive); this remains for unit + tests and any caller wanting the legacy "one file wins" view. HC2 gate still applies.""" + return resolve_overlay_op(recipe, op, repo_local_dir) or generic_op(op) def custom_tests(recipe: str, repo_local_dir: str | None) -> list[tuple[str, str]]: - """All non-lifecycle test_*.py from cc-ci's tests// and the recipe's repo-local tests/. - These have no generic equivalent and run only when present (opt-in), additively from both.""" + """All non-lifecycle test_*.py from cc-ci's tests// and (if approved) the recipe's + repo-local tests/. These have no generic equivalent and run only when present (opt-in), additively + from both. Repo-local is consulted only for allowlist-approved recipes (HC2).""" lifecycle_names = {f"test_{op}.py" for op in LIFECYCLE_OPS} found: list[tuple[str, str]] = [] - for source, d in (("cc-ci", cc_ci_dir(recipe)), ("repo-local", repo_local_dir)): + for source, d in (("cc-ci", cc_ci_dir(recipe)), ("repo-local", _gated(recipe, repo_local_dir))): if not d or not os.path.isdir(d): continue for p in sorted(glob.glob(os.path.join(d, "test_*.py"))): @@ -60,12 +115,40 @@ def custom_tests(recipe: str, repo_local_dir: str | None) -> list[tuple[str, str def install_steps(recipe: str, repo_local_dir: str | None) -> tuple[str, str] | None: - """The custom install-steps hook (install_steps.sh) for a recipe, or None. repo-local > cc-ci.""" - if repo_local_dir: - p = os.path.join(repo_local_dir, "install_steps.sh") + """The custom install-steps hook (install_steps.sh) for a recipe, or None. repo-local > cc-ci. + Repo-local is consulted only for allowlist-approved recipes (HC2).""" + rl = _gated(recipe, repo_local_dir) + if rl: + p = os.path.join(rl, "install_steps.sh") if os.path.isfile(p): return ("repo-local", p) p = os.path.join(cc_ci_dir(recipe), "install_steps.sh") if os.path.isfile(p): return ("cc-ci", p) return None + + +def pre_op_hook(recipe: str, op: str, repo_local_dir: str | None) -> tuple[str, str] | None: + """The pre-op seed hook for `op`: the path to a recipe `ops.py` module that defines a + `pre_(domain, meta)` callable, or None. cc-ci's tests//ops.py wins; the repo-local + ops.py is consulted only for allowlist-approved recipes (HC2). The orchestrator imports the + module and calls pre_ BEFORE performing the op (HC3 op/assertion split — overlays seed + pre-op state here, then assert post-op in test_.py).""" + fn = f"pre_{op}" + for source, d in (("cc-ci", cc_ci_dir(recipe)), ("repo-local", _gated(recipe, repo_local_dir))): + if not d: + continue + p = os.path.join(d, "ops.py") + if os.path.isfile(p) and _module_defines(p, fn): + return (source, p) + return None + + +def _module_defines(path: str, name: str) -> bool: + """Cheap source scan for a top-level `def (` — avoids importing the module just to check.""" + try: + with open(path) as fh: + src = fh.read() + except OSError: + return False + return f"def {name}(" in src diff --git a/tests/repo-local-approved.txt b/tests/repo-local-approved.txt new file mode 100644 index 0000000..0d7cb09 --- /dev/null +++ b/tests/repo-local-approved.txt @@ -0,0 +1,13 @@ +# cc-ci repo-local approval allowlist (Phase 1e HC2) — DEFAULT-DENY. +# +# PR-author-controlled code in a recipe repo's own tests/ dir (repo-local `test_*.py`, +# `install_steps.sh`, `ops.py`) runs on the CI host with /run/secrets/* present, so it is an +# untrusted-code risk. By default the harness runs ONLY cc-ci-authored overlays (tests//...) +# + the generic floor. Repo-local code is discovered-but-NOT-executed unless its recipe is listed +# below. +# +# To approve a recipe: a cc-ci maintainer reviews that recipe's repo-local tests, then adds the +# recipe name here in a cc-ci PR (one name per line; `#` comments + blank lines ignored). A lone `*` +# is NOT a wildcard — every recipe must be listed explicitly. +# +# (default: empty — no recipe trusts repo-local code) diff --git a/tests/unit/test_discovery.py b/tests/unit/test_discovery.py index 6be845e..8c6e45b 100644 --- a/tests/unit/test_discovery.py +++ b/tests/unit/test_discovery.py @@ -1,9 +1,10 @@ -"""Unit tests for overlay/custom/hook discovery + precedence (Phase 1d, DG4). +"""Unit tests for overlay/custom/hook discovery + precedence (Phase 1d DG4 + Phase 1e HC2/HC3). -Deterministic, no deployment — proves the resolution rule (repo-local > cc-ci > generic) and the -invariant "no overlay for an op ⇒ generic runs". Run with: `cc-ci-run -m pytest tests/unit`. -These live under tests/unit/ (NOT a recipe name, NOT tests/_generic/) so the run orchestrator never -picks them up as overlays/custom tests.""" +Deterministic, no deployment. Proves: + - the overlay resolution rule (repo-local > cc-ci) + the generic floor (HC3: additive, separate); + - the HC2 repo-local approval gate (default-deny; repo-local consulted only for approved recipes). +Run with: `cc-ci-run -m pytest tests/unit`. These live under tests/unit/ (NOT a recipe name, NOT +tests/_generic/) so the run orchestrator never picks them up as overlays/custom tests.""" import os import sys @@ -12,42 +13,105 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner") from harness import discovery # noqa: E402 -def test_no_overlay_falls_back_to_generic(): - # hedgedoc has no tests/hedgedoc/ in cc-ci and no repo-local dir -> generic is the floor (DG4 invariant) - source, path = discovery.resolve_op("hedgedoc", "install", None) - assert source == "generic" +def _approve(tmp_path, *recipes): + """Point the allowlist at a temp file approving the given recipes (HC2), and return a cleanup.""" + f = tmp_path / "approved.txt" + f.write_text("".join(f"{r}\n" for r in recipes)) + os.environ["CCCI_REPO_LOCAL_APPROVED_FILE"] = str(f) + + +def teardown_function(): + os.environ.pop("CCCI_REPO_LOCAL_APPROVED_FILE", None) + + +# ---- HC3: generic is the floor; overlay resolution is separate + additive -------------------- + +def test_no_overlay_means_generic_floor(): + # hedgedoc ships no tests/hedgedoc/ overlay and no repo-local -> no overlay; generic floor exists + assert discovery.resolve_overlay_op("hedgedoc", "install", None) is None + src, path = discovery.generic_op("install") + assert src == "generic" assert path == os.path.join(discovery.GENERIC_DIR, "test_install.py") + # back-compat resolver still yields generic when there is no overlay + assert discovery.resolve_op("hedgedoc", "install", None) == ("generic", path) -def test_cc_ci_overlay_overrides_generic(): - # custom-html ships cc-ci overlays for all four ops -> cc-ci wins over generic when no repo-local +def test_cc_ci_overlay_found_for_each_op(): + # custom-html ships cc-ci overlays for all four ops -> resolve_overlay_op returns the cc-ci file for op in discovery.LIFECYCLE_OPS: - source, path = discovery.resolve_op("custom-html", op, None) - assert source == "cc-ci", op - assert path == os.path.join(discovery.cc_ci_dir("custom-html"), f"test_{op}.py") + res = discovery.resolve_overlay_op("custom-html", op, None) + assert res == ("cc-ci", os.path.join(discovery.cc_ci_dir("custom-html"), f"test_{op}.py")), op -def test_repo_local_wins_same_name_collision(tmp_path): - # repo-local is upstream-authoritative: a repo-local test_install.py beats cc-ci's for that op +# ---- HC2: repo-local approval gate (default-deny) -------------------------------------------- + +def test_repo_local_ignored_when_not_approved(tmp_path): + # default-deny: a repo-local overlay is NOT consulted for an unapproved recipe -> cc-ci wins + _approve(tmp_path) # empty allowlist (tmp_path / "test_install.py").write_text("# repo-local overlay\n") - source, path = discovery.resolve_op("custom-html", "install", str(tmp_path)) - assert source == "repo-local" - assert path == str(tmp_path / "test_install.py") + res = discovery.resolve_overlay_op("custom-html", "install", str(tmp_path)) + assert res == ("cc-ci", os.path.join(discovery.cc_ci_dir("custom-html"), "test_install.py")) + assert discovery.repo_local_approved("custom-html") is False -def test_custom_tests_additive_from_both_locations(tmp_path): - # non-lifecycle test_*.py are opt-in and additive; lifecycle names are excluded - (tmp_path / "test_sso.py").write_text("# repo-local custom\n") - (tmp_path / "test_install.py").write_text("# lifecycle name -> excluded from custom\n") - customs = discovery.custom_tests("custom-html", str(tmp_path)) +def test_repo_local_wins_when_approved(tmp_path): + # approved: repo-local is upstream-authoritative and beats cc-ci for the same op + rl = tmp_path / "repo" + rl.mkdir() + (rl / "test_install.py").write_text("# repo-local overlay\n") + _approve(tmp_path, "custom-html") + assert discovery.repo_local_approved("custom-html") is True + res = discovery.resolve_overlay_op("custom-html", "install", str(rl)) + assert res == ("repo-local", str(rl / "test_install.py")) + + +def test_custom_tests_repo_local_gated(tmp_path): + # non-lifecycle test_*.py from repo-local only count for approved recipes; lifecycle names excluded + rl = tmp_path / "repo" + rl.mkdir() + (rl / "test_sso.py").write_text("# repo-local custom\n") + (rl / "test_install.py").write_text("# lifecycle name -> excluded from custom\n") + + _approve(tmp_path) # not approved -> repo-local custom ignored + assert discovery.custom_tests("custom-html", str(rl)) == [] + + _approve(tmp_path, "custom-html") # approved -> repo-local custom honored + customs = discovery.custom_tests("custom-html", str(rl)) names = {(src, os.path.basename(p)) for src, p in customs} assert ("repo-local", "test_sso.py") in names assert all(os.path.basename(p) != "test_install.py" for _, p in customs) -def test_install_steps_repo_local_over_cc_ci(tmp_path): - (tmp_path / "install_steps.sh").write_text("#!/usr/bin/env bash\n") - hook = discovery.install_steps("custom-html", str(tmp_path)) - assert hook is not None - assert hook[0] == "repo-local" +def test_install_steps_repo_local_gated(tmp_path): + rl = tmp_path / "repo" + rl.mkdir() + (rl / "install_steps.sh").write_text("#!/usr/bin/env bash\n") + + _approve(tmp_path) # not approved -> repo-local hook ignored (custom-html has no cc-ci hook) + assert discovery.install_steps("custom-html", str(rl)) is None + + _approve(tmp_path, "custom-html") # approved -> repo-local hook honored + hook = discovery.install_steps("custom-html", str(rl)) + assert hook == ("repo-local", str(rl / "install_steps.sh")) assert discovery.install_steps("hedgedoc", None) is None + + +def test_pre_op_hook_repo_local_gated(tmp_path): + rl = tmp_path / "repo" + rl.mkdir() + (rl / "ops.py").write_text("def pre_upgrade(domain, meta):\n pass\n") + + _approve(tmp_path) # not approved -> repo-local ops.py ignored + assert discovery.pre_op_hook("custom-html", "upgrade", str(rl)) is None + + _approve(tmp_path, "custom-html") # approved -> repo-local pre-op hook honored + hook = discovery.pre_op_hook("custom-html", "upgrade", str(rl)) + assert hook == ("repo-local", str(rl / "ops.py")) + # an ops.py that does NOT define pre_ is not a hook for that op + assert discovery.pre_op_hook("custom-html", "backup", str(rl)) is None + + +def test_default_allowlist_is_empty(): + # the checked-in tests/repo-local-approved.txt approves nothing (default-deny invariant) + os.environ.pop("CCCI_REPO_LOCAL_APPROVED_FILE", None) + assert discovery.approved_recipes() == set()