feat(harness): P5 — customization manifest (rcust)
All checks were successful
continuous-integration/drone/push Build is passing
All checks were successful
continuous-integration/drone/push Build is passing
One block at run start answering "what does this recipe customize?" across every surface (non-default recipe_meta keys, ops.py pre-ops, install_steps.sh, compose.ccci.yml, lifecycle overlays by source, custom-test counts, active CCCI_SKIP_GENERIC* env overrides — !!-flagged when riding a CI run, P2c), printed to the run log and embedded verbatim in results.json under "customization". Pure presentation — building/printing it never influences a verdict; the manifest honors the HC2 repo-local gate so it never advertises code the run will not execute. Unit tests: synthetic recipe exercising every surface -> complete + deterministic + JSON-clean; HC2 invisibility; env-override flagging; render golden lines; build_results threads the dict verbatim (key always present, None when absent). Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
141
runner/harness/manifest.py
Normal file
141
runner/harness/manifest.py
Normal file
@ -0,0 +1,141 @@
|
||||
"""Customization manifest (rcust P5; spec §8 R4 mitigation).
|
||||
|
||||
One block at run start answering "what does this recipe customize?" across ALL the surfaces
|
||||
(recipe_meta keys, hook files, file-presence, run-time env overrides) — printed to the run log and
|
||||
embedded verbatim in results.json under "customization". PURE PRESENTATION: building or printing
|
||||
the manifest must never influence any verdict (R7-class invariant).
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
|
||||
from . import discovery, lifecycle
|
||||
from . import meta as meta_mod
|
||||
|
||||
_PRE_OP_RE = re.compile(r"^def (pre_[a-z]+)\(", re.MULTILINE)
|
||||
|
||||
|
||||
def _jsonable(v):
|
||||
"""Manifest values must be JSON-serializable + deterministic: hooks render as '<hook>',
|
||||
tuples become lists."""
|
||||
if callable(v):
|
||||
return "<hook>"
|
||||
if isinstance(v, tuple):
|
||||
return list(v)
|
||||
return v
|
||||
|
||||
|
||||
def _pre_ops(path: str) -> list[str]:
|
||||
"""The pre_<op> hook names an ops.py defines (cheap source scan, same approach as
|
||||
discovery._module_defines — no import)."""
|
||||
try:
|
||||
with open(path) as fh:
|
||||
return sorted(set(_PRE_OP_RE.findall(fh.read())))
|
||||
except OSError:
|
||||
return []
|
||||
|
||||
|
||||
def _custom_counts(recipe: str, repo_local: str | None) -> dict[str, dict[str, int]]:
|
||||
out: dict[str, dict[str, int]] = {}
|
||||
for source, path in discovery.custom_tests(recipe, repo_local):
|
||||
sub = os.path.basename(os.path.dirname(path)) # functional | playwright
|
||||
out.setdefault(source, {}).setdefault(sub, 0)
|
||||
out[source][sub] += 1
|
||||
return out
|
||||
|
||||
|
||||
def build(recipe: str, meta, repo_local: str | None) -> dict:
|
||||
"""Collect the run's resolved customization into one deterministic, JSON-serializable dict.
|
||||
|
||||
Keys: meta_non_default (explicitly-customized recipe_meta keys), hooks (ops.py pre-ops +
|
||||
install_steps.sh + compose.ccci.yml with their source), overlays (lifecycle overlay files by
|
||||
op + source), custom_tests (counts per source/subdir), env_overrides (active
|
||||
CCCI_SKIP_GENERIC* — the dev-only escape hatch, flagged when riding a CI run)."""
|
||||
hooks: dict = {}
|
||||
pre_ops: dict[str, list[str]] = {}
|
||||
for source, d in (
|
||||
("cc-ci", discovery.cc_ci_dir(recipe)),
|
||||
("repo-local", discovery._gated(recipe, repo_local)), # noqa: SLF001 — same HC2 gate
|
||||
):
|
||||
if not d:
|
||||
continue
|
||||
p = os.path.join(d, "ops.py")
|
||||
if os.path.isfile(p):
|
||||
ops = _pre_ops(p)
|
||||
if ops:
|
||||
pre_ops[source] = ops
|
||||
if pre_ops:
|
||||
hooks["ops.py"] = pre_ops
|
||||
ist = discovery.install_steps(recipe, repo_local)
|
||||
if ist:
|
||||
hooks["install_steps.sh"] = ist[0]
|
||||
if lifecycle.has_ccci_overlay(recipe):
|
||||
hooks["compose.ccci.yml"] = "cc-ci"
|
||||
|
||||
overlays = {}
|
||||
for op in discovery.LIFECYCLE_OPS:
|
||||
ov = discovery.resolve_overlay_op(recipe, op, repo_local)
|
||||
if ov:
|
||||
overlays[op] = ov[0]
|
||||
|
||||
env_overrides = sorted(
|
||||
k
|
||||
for k in os.environ
|
||||
if k.startswith("CCCI_SKIP_GENERIC")
|
||||
and str(os.environ.get(k) or "").strip().lower() in ("1", "true", "yes", "on")
|
||||
)
|
||||
|
||||
return {
|
||||
"meta_non_default": {
|
||||
k: _jsonable(v) for k, v in sorted(meta_mod.non_default(meta).items())
|
||||
},
|
||||
"hooks": hooks,
|
||||
"overlays": overlays,
|
||||
"custom_tests": _custom_counts(recipe, repo_local),
|
||||
"env_overrides": env_overrides,
|
||||
}
|
||||
|
||||
|
||||
def render(recipe: str, manifest: dict) -> str:
|
||||
"""The human block printed at run start (same content as the results.json key)."""
|
||||
lines = [f"===== customization manifest: {recipe} ====="]
|
||||
nd = manifest["meta_non_default"]
|
||||
lines.append(
|
||||
"meta (non-default): "
|
||||
+ (" ".join(f"{k}={v!r}" for k, v in nd.items()) if nd else "(none — zero-config floor)")
|
||||
)
|
||||
hk = manifest["hooks"]
|
||||
parts = []
|
||||
for source, ops in hk.get("ops.py", {}).items():
|
||||
parts.append(f"ops.py[{','.join(ops)}]({source})")
|
||||
if "install_steps.sh" in hk:
|
||||
parts.append(f"install_steps.sh({hk['install_steps.sh']})")
|
||||
if "compose.ccci.yml" in hk:
|
||||
parts.append(f"compose.ccci.yml({hk['compose.ccci.yml']})")
|
||||
lines.append("hooks: " + (" ".join(parts) if parts else "(none)"))
|
||||
ov = manifest["overlays"]
|
||||
lines.append(
|
||||
"overlays: "
|
||||
+ (" ".join(f"test_{op}.py({src})" for op, src in ov.items()) if ov else "(none)")
|
||||
)
|
||||
ct = manifest["custom_tests"]
|
||||
lines.append(
|
||||
"custom tests: "
|
||||
+ (
|
||||
" ".join(
|
||||
" ".join(f"{sub}/={n}" for sub, n in sorted(counts.items())) + f" ({source})"
|
||||
for source, counts in sorted(ct.items())
|
||||
)
|
||||
if ct
|
||||
else "(none)"
|
||||
)
|
||||
)
|
||||
eo = manifest["env_overrides"]
|
||||
if eo:
|
||||
suffix = " !! dev-only override active in CI" if os.environ.get("DRONE") else ""
|
||||
lines.append("env overrides: " + " ".join(f"{k}=1" for k in eo) + suffix)
|
||||
else:
|
||||
lines.append("env overrides: (none)")
|
||||
return "\n".join(lines)
|
||||
@ -203,6 +203,7 @@ def build_results(
|
||||
screenshot: str | None = None,
|
||||
summary_card: str | None = None,
|
||||
expected_na: dict | None = None,
|
||||
customization: dict | None = None,
|
||||
) -> dict:
|
||||
"""Assemble the full results.json dict (no I/O). `finished_ts` is passed in (the orchestrator
|
||||
stamps it) so this stays pure and deterministic for unit tests. `expected_na` is the recipe's
|
||||
@ -236,6 +237,9 @@ def build_results(
|
||||
},
|
||||
"screenshot": screenshot,
|
||||
"summary_card": summary_card,
|
||||
# rcust P5: the run's resolved customization manifest (pure presentation — consumers must
|
||||
# never derive a verdict from it).
|
||||
"customization": customization,
|
||||
}
|
||||
|
||||
|
||||
|
||||
@ -58,6 +58,9 @@ from harness import ( # noqa: E402
|
||||
from harness import ( # noqa: E402
|
||||
deps as deps_mod,
|
||||
)
|
||||
from harness import ( # noqa: E402
|
||||
manifest as manifest_mod,
|
||||
)
|
||||
from harness import ( # noqa: E402
|
||||
meta as meta_mod,
|
||||
)
|
||||
@ -880,6 +883,12 @@ def main() -> int:
|
||||
repo_local = snapshot_recipe_tests(recipe)
|
||||
meta = meta_mod.load(recipe)
|
||||
|
||||
# Customization manifest (rcust P5, R4): ONE block answering "what does this recipe
|
||||
# customize?" across all surfaces — printed here and embedded verbatim in results.json under
|
||||
# "customization". Pure presentation; never influences a verdict.
|
||||
customization = manifest_mod.build(recipe, meta, repo_local)
|
||||
print("\n" + manifest_mod.render(recipe, customization) + "\n", flush=True)
|
||||
|
||||
# WC4/WC7: opt-in `--quick` fast lane. Requires an existing data-warm canonical; if none, fall
|
||||
# back cleanly to the full COLD run below so the PR is still tested (DECISIONS Phase-2w).
|
||||
if os.environ.get("CCCI_QUICK") == "1" or os.environ.get("MODE") == "quick":
|
||||
@ -1249,6 +1258,7 @@ def main() -> int:
|
||||
screenshot=screenshot_rel, # Phase 3 U1 (R4): relative PNG name iff capture succeeded
|
||||
finished_ts=time.time(),
|
||||
expected_na=meta.EXPECTED_NA, # declared intentional-skip map (recipe_meta)
|
||||
customization=customization, # rcust P5: the run-start manifest, verbatim
|
||||
)
|
||||
# Real (if narrow) leak check: no known infra-secret value may appear in the artifact (R7).
|
||||
blob = json.dumps(data)
|
||||
|
||||
Reference in New Issue
Block a user