feat(harness): intentional skips + custom-html-tiny functional test; 4-rung ladder (#6)
Some checks failed
continuous-integration/drone/push Build is failing
Some checks failed
continuous-integration/drone/push Build is failing
Declare intentional skips + custom-html-tiny functional test; 4-rung level ladder
- recipe_meta.EXPECTED_NA = {rung: reason} lists intentionally-skipped rungs; any
essential rung skipped and not listed is unintentional. Skips still cap the level
(never inflate). results.json: skips:{intentional,unintentional} + level_cap_rung.
- Level ladder = the four essential rungs (install, upgrade, backup/restore,
functional; top = L4). integration & recipe-local are optional, not leveled
(SSO still enforced for the run verdict, unchanged).
- Card shows skipped rungs as INTENTIONAL SKIP (green, reason below) / UNINTENTIONAL
SKIP (amber); level badge gains an expected/gap? third segment.
- custom-html-tiny: functional serve test (exact-byte round-trip + 404); declares
backup_restore intentionally skipped (stateless static server).
Independently verified by the adversary: 138 unit tests pass cold; live full-stage
run on custom-html-tiny green (upgrade tier ran; level 2; correct skips/badge);
clean teardown.
This commit is contained in:
@ -200,6 +200,7 @@ def _load_meta(recipe: str) -> dict:
|
||||
for k in list(meta) + [
|
||||
"BACKUP_CAPABLE",
|
||||
"SKIP_GENERIC",
|
||||
"EXPECTED_NA",
|
||||
"OIDC_AT_INSTALL",
|
||||
"READY_PROBE",
|
||||
"UPGRADE_BASE_VERSION",
|
||||
@ -1224,7 +1225,6 @@ def main() -> int:
|
||||
# a failure here NEVER changes `overall` (R7 — cosmetics never block the pipeline). ----
|
||||
data: dict | None = None
|
||||
try:
|
||||
sso_unverified = sso_dep_unverified(declared, deps_ready, requires_deps_skipped)
|
||||
clean_teardown = (deploy_count == expected_deploy_count) and not dep_teardown_error
|
||||
data = results_mod.build_results(
|
||||
recipe=recipe,
|
||||
@ -1234,13 +1234,11 @@ def main() -> int:
|
||||
records=records,
|
||||
results=results,
|
||||
backup_capable=backup_cap,
|
||||
declared=declared,
|
||||
deps_ready=deps_ready,
|
||||
sso_unverified=sso_unverified,
|
||||
clean_teardown=clean_teardown,
|
||||
no_secret_leak=True, # narrowed below by an actual scan of the serialised artifact
|
||||
screenshot=screenshot_rel, # Phase 3 U1 (R4): relative PNG name iff capture succeeded
|
||||
finished_ts=time.time(),
|
||||
expected_na=meta.get("EXPECTED_NA"), # declared intentional-skip map (recipe_meta)
|
||||
)
|
||||
# Real (if narrow) leak check: no known infra-secret value may appear in the artifact (R7).
|
||||
blob = json.dumps(data)
|
||||
@ -1257,6 +1255,15 @@ def main() -> int:
|
||||
f"{' — ' + data['level_cap_reason'] if data['level_cap_reason'] else ''})",
|
||||
flush=True,
|
||||
)
|
||||
# Surface UNINTENTIONAL skips in the CI log (non-blocking, R7): a rung that was skipped (N/A)
|
||||
# but is not in the recipe's intentional list — either add the missing coverage or declare it.
|
||||
for rung in data.get("skips", {}).get("unintentional", []):
|
||||
print(
|
||||
f"⚠ coverage: rung '{rung}' was skipped (N/A) but is not declared intentional — add "
|
||||
f"the missing test/label, or list it in tests/{recipe}/recipe_meta.py "
|
||||
f"EXPECTED_NA = {{'{rung}': '<why>'}}.",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 — results assembly is cosmetic; never fail a run on it (R7)
|
||||
print(
|
||||
f"!! results.json assembly failed (non-fatal, verdict unaffected): {_scrub(str(e))}",
|
||||
@ -1275,8 +1282,19 @@ def main() -> int:
|
||||
with open(html_path, "w", encoding="utf-8") as f:
|
||||
f.write(card_mod.render_card_html(data, screenshot_rel=data.get("screenshot")))
|
||||
png = card_mod.render_card_png(html_path, os.path.join(run_artifact_dir, "summary.png"))
|
||||
capped = data.get("level_cap_rung")
|
||||
sk = data.get("skips", {})
|
||||
cap_skip = (
|
||||
"intentional" if capped in (sk.get("intentional") or {})
|
||||
else "unintentional" if capped in (sk.get("unintentional") or [])
|
||||
else ""
|
||||
)
|
||||
with open(os.path.join(run_artifact_dir, "badge.svg"), "w", encoding="utf-8") as f:
|
||||
f.write(card_mod.level_badge_svg(data["level"], data.get("level_cap_reason", "")))
|
||||
f.write(
|
||||
card_mod.level_badge_svg(
|
||||
data["level"], data.get("level_cap_reason", ""), cap_skip
|
||||
)
|
||||
)
|
||||
print(
|
||||
f"summary card {'rendered ' + png if png else '(PNG render unavailable)'} + "
|
||||
f"badge.svg written into {run_artifact_dir}",
|
||||
|
||||
Reference in New Issue
Block a user