feat(harness): intentional skips + custom-html-tiny functional test; 4-rung ladder (#6)
Some checks failed
continuous-integration/drone/push Build is failing
Some checks failed
continuous-integration/drone/push Build is failing
Declare intentional skips + custom-html-tiny functional test; 4-rung level ladder
- recipe_meta.EXPECTED_NA = {rung: reason} lists intentionally-skipped rungs; any
essential rung skipped and not listed is unintentional. Skips still cap the level
(never inflate). results.json: skips:{intentional,unintentional} + level_cap_rung.
- Level ladder = the four essential rungs (install, upgrade, backup/restore,
functional; top = L4). integration & recipe-local are optional, not leveled
(SSO still enforced for the run verdict, unchanged).
- Card shows skipped rungs as INTENTIONAL SKIP (green, reason below) / UNINTENTIONAL
SKIP (amber); level badge gains an expected/gap? third segment.
- custom-html-tiny: functional serve test (exact-byte round-trip + 404); declares
backup_restore intentionally skipped (stateless static server).
Independently verified by the adversary: 138 unit tests pass cold; live full-stage
run on custom-html-tiny green (upgrade tier ran; level 2; correct skips/badge);
clean teardown.
This commit is contained in:
@ -79,10 +79,44 @@ def render_badge_svg(label: str, message: str, color: str) -> str:
|
||||
)
|
||||
|
||||
|
||||
def level_badge_svg(level: int, cap_reason: str = "") -> str:
|
||||
"""Per-recipe/-run LEVEL badge: 'cc-ci | level N'. Colour by level (R6)."""
|
||||
msg = f"level {int(level)}"
|
||||
return render_badge_svg("cc-ci", msg, level_color(level))
|
||||
# Third-segment colours for the level badge: amber = an UNINTENTIONAL skip (a rung skipped but not
|
||||
# in the recipe's intentional list — likely missing coverage) capped the climb; muted = an
|
||||
# INTENTIONAL skip (declared in recipe_meta.EXPECTED_NA — nothing to fix). Font-safe text labels
|
||||
# (no emoji) so the SVG renders anywhere.
|
||||
GAP_COLOR = "#d29922"
|
||||
EXPECT_COLOR = "#6e7681"
|
||||
|
||||
|
||||
def level_badge_svg(level: int, cap_reason: str = "", cap_skip: str = "") -> str:
|
||||
"""Per-recipe/-run LEVEL badge: 'cc-ci | level N' coloured by level (R6), with a THIRD segment
|
||||
that differentiates *why* the climb stopped when a SKIP capped it (`cap_skip`):
|
||||
- "unintentional" (a rung skipped but not in the recipe's intentional list): amber 'gap?'.
|
||||
- "intentional" (a skip declared in recipe_meta.EXPECTED_NA): muted 'expected'.
|
||||
- "" (clean cap / full climb / a real failure): no third segment (the level + card carry it).
|
||||
The badge never inflates — it only annotates the cap the level already reflects."""
|
||||
label, msg = "cc-ci", f"level {int(level)}"
|
||||
lw, mw = _text_width(label), _text_width(msg)
|
||||
third: tuple[str, str] | None = None
|
||||
if cap_skip == "unintentional":
|
||||
third = ("gap?", GAP_COLOR)
|
||||
elif cap_skip == "intentional":
|
||||
third = ("expected", EXPECT_COLOR)
|
||||
if third is None:
|
||||
return render_badge_svg(label, msg, level_color(level))
|
||||
txt, tcolor = third
|
||||
tw = _text_width(txt)
|
||||
w = lw + mw + tw
|
||||
return (
|
||||
f'<svg xmlns="http://www.w3.org/2000/svg" width="{w}" height="20" role="img" '
|
||||
f'aria-label="{html.escape(label)}: {html.escape(msg)} ({html.escape(txt)})">'
|
||||
f'<rect width="{lw}" height="20" fill="#555"/>'
|
||||
f'<rect x="{lw}" width="{mw}" height="20" fill="{level_color(level)}"/>'
|
||||
f'<rect x="{lw + mw}" width="{tw}" height="20" fill="{tcolor}"/>'
|
||||
f'<g fill="#fff" font-family="Verdana,Geneva,sans-serif" font-size="11">'
|
||||
f'<text x="6" y="14">{html.escape(label)}</text>'
|
||||
f'<text x="{lw + 6}" y="14">{html.escape(msg)}</text>'
|
||||
f'<text x="{lw + mw + 6}" y="14">{html.escape(txt)}</text></g></svg>'
|
||||
)
|
||||
|
||||
|
||||
def _stage_rows(stages: list[dict]) -> str:
|
||||
@ -107,6 +141,41 @@ def _stage_rows(stages: list[dict]) -> str:
|
||||
return "\n".join(rows) or '<tr><td colspan="3">no stages</td></tr>'
|
||||
|
||||
|
||||
# Friendly rung labels for the skip rows (the four essential rungs).
|
||||
RUNG_LABEL = {
|
||||
"install": "install",
|
||||
"upgrade": "upgrade",
|
||||
"backup_restore": "backup/restore",
|
||||
"functional": "functional",
|
||||
}
|
||||
SKIP_GREEN = "#57ab5a" # muted green — an intentional skip reads like a pass (but labelled, never inflating)
|
||||
|
||||
|
||||
def _skip_rows(skips: dict) -> str:
|
||||
"""Render SKIPPED rungs as stage-like rows. An intentional (declared) skip looks like a pass row
|
||||
but its status says 'INTENTIONAL SKIP' (muted green) with the declared reason on the line below;
|
||||
an unintentional skip is amber 'UNINTENTIONAL SKIP' with a prompt to add a test or declare it."""
|
||||
rows = []
|
||||
for rung, reason in (skips.get("intentional") or {}).items():
|
||||
rows.append(
|
||||
f'<tr class="stage"><td colspan="2"><span class="mark" style="color:{SKIP_GREEN}">⊘</span>'
|
||||
f'<b>{html.escape(RUNG_LABEL.get(rung, rung))}</b></td>'
|
||||
f'<td class="st" style="color:{SKIP_GREEN}">intentional skip</td></tr>'
|
||||
)
|
||||
rows.append(f'<tr class="skipreason"><td></td><td colspan="2">{html.escape(reason)}</td></tr>')
|
||||
for rung in skips.get("unintentional") or []:
|
||||
rows.append(
|
||||
f'<tr class="stage"><td colspan="2"><span class="mark" style="color:{GAP_COLOR}">⊘</span>'
|
||||
f'<b>{html.escape(RUNG_LABEL.get(rung, rung))}</b></td>'
|
||||
f'<td class="st" style="color:{GAP_COLOR}">unintentional skip</td></tr>'
|
||||
)
|
||||
rows.append(
|
||||
'<tr class="skipreason"><td></td><td colspan="2">not declared in EXPECTED_NA — add the '
|
||||
"missing test/label, or declare the skip with a reason</td></tr>"
|
||||
)
|
||||
return "\n".join(rows)
|
||||
|
||||
|
||||
def render_card_html(data: dict, screenshot_rel: str | None = "screenshot.png") -> str:
|
||||
"""Build the summary-card HTML from a results.json dict. `screenshot_rel` is the relative path to
|
||||
the screenshot PNG (same dir as the card) — omitted from the card if None / absent.
|
||||
@ -116,7 +185,9 @@ def render_card_html(data: dict, screenshot_rel: str | None = "screenshot.png")
|
||||
recipe = html.escape(str(data.get("recipe", "?")))
|
||||
version = html.escape(str(data.get("version") or data.get("ref") or ""))
|
||||
level = int(data.get("level", 0))
|
||||
cap = html.escape(str(data.get("level_cap_reason") or ""))
|
||||
cap_reason = str(data.get("level_cap_reason") or "")
|
||||
cap = html.escape(cap_reason)
|
||||
sk = data.get("skips", {}) or {}
|
||||
color = level_color(level)
|
||||
flags = data.get("flags", {}) or {}
|
||||
flag_bits = []
|
||||
@ -132,7 +203,7 @@ def render_card_html(data: dict, screenshot_rel: str | None = "screenshot.png")
|
||||
if show_shot
|
||||
else '<div class="shot noshot">no screenshot</div>'
|
||||
)
|
||||
rows = _stage_rows(data.get("stages", []))
|
||||
rows = _stage_rows(data.get("stages", [])) + "\n" + _skip_rows(sk)
|
||||
return f"""<!doctype html><html><head><meta charset="utf-8"><style>
|
||||
*{{box-sizing:border-box}}
|
||||
body{{margin:0;font-family:system-ui,-apple-system,Segoe UI,sans-serif;background:#0d1117;color:#c9d1d9}}
|
||||
@ -157,6 +228,7 @@ tr.stage td{{padding-top:.5rem;border-bottom:1px solid #30363d}}
|
||||
.test .tmark{{width:1.4rem;text-align:center}}
|
||||
.test .tname{{color:#c9d1d9;font-family:ui-monospace,monospace;font-size:.8rem}}
|
||||
.test .tms{{text-align:right;color:#8b949e;font-size:.74rem;width:5rem}}
|
||||
tr.skipreason td{{color:#8b949e;font-size:.78rem;font-style:italic;padding-top:0;padding-bottom:.45rem;border-bottom:1px solid #21262d}}
|
||||
.shot{{width:360px;flex:none;border:1px solid #30363d;border-radius:8px;overflow:hidden;background:#0d1117}}
|
||||
.shot img{{width:100%;display:block}}
|
||||
.shot.noshot{{display:flex;align-items:center;justify-content:center;height:225px;color:#8b949e;font-size:.85rem}}
|
||||
@ -167,7 +239,7 @@ tr.stage td{{padding-top:.5rem;border-bottom:1px solid #30363d}}
|
||||
<div class="hd">{FLOWER_SVG}
|
||||
<div class="title"><h1>{recipe}</h1><span class="ver">{version}</span></div>
|
||||
<div class="lvl"><span class="num">{level}</span><span class="lbl">level</span></div></div>
|
||||
<div class="cap">{("<b>capped:</b> " + cap) if cap else "<b>full clean climb</b> — top level (6)"}</div>
|
||||
<div class="cap">{("<b>capped:</b> " + cap) if cap else "<b>full clean climb</b> — top level (4)"}</div>
|
||||
<div class="body"><div class="tbl"><table>{rows}</table></div>{shot_html}</div>
|
||||
<div class="flags">{"".join(flag_bits)}</div>
|
||||
</div></body></html>"""
|
||||
|
||||
@ -5,37 +5,39 @@ YunoHost semantics: **a gap caps the level** — you only earn level L if every
|
||||
PASS. The first rung that is not a clean PASS (a real FAIL *or* genuinely N/A for this recipe) stops
|
||||
the climb; `cap_reason` records why. This is deliberately conservative: presentation must NEVER make
|
||||
a run look greener than its tests (plan §6 cardinal guardrail), so an N/A rung caps just like a fail
|
||||
(the L5 example in §4.1 — "recipes with no integration surface cap at L4 by definition" — is exactly
|
||||
this: N/A caps, with a recorded reason so the level is *fair*, not inflated).
|
||||
— with a recorded reason so the level is *fair*, not inflated.
|
||||
|
||||
The ladder (§4.1):
|
||||
The ladder is the FOUR essential rungs every recipe is held to:
|
||||
L0 — install failed / app never became healthy.
|
||||
L1 — Installs: deploys + passes health/readiness.
|
||||
L2 — Upgrades: previous published version → PR version, stays healthy, data intact.
|
||||
L3 — Backup/restore: seeded data survives backup → wipe → restore.
|
||||
L4 — Functional: recipe-specific functional tests pass.
|
||||
L5 — Integration: SSO/OIDC + cross-app integration tests pass.
|
||||
L6 — Recipe-local: the recipe repo's own tests/ (D4) pass and are merged.
|
||||
|
||||
Integration (SSO/OIDC + cross-app) and recipe-local (the recipe repo's own tests/) are **OPTIONAL**
|
||||
capabilities — they are NOT part of the level ladder and never cap it. They still run when present
|
||||
(and SSO is still enforced for the run VERDICT via the deps/SSO checks in run_recipe_ci.py), but a
|
||||
recipe without an SSO surface or without repo-local tests is simply not penalised on the level.
|
||||
|
||||
This module is PURE (no I/O) so it is cheaply unit-testable and the Adversary can re-run the unit
|
||||
test cold (`cc-ci-run -m pytest tests/unit/test_level.py -q`). The orchestrator
|
||||
(`run_recipe_ci.py`) is responsible for translating its raw per-tier results + deps/SSO signals into
|
||||
the rung-status dict this function consumes; that mapping is documented in DECISIONS.md (Phase 3).
|
||||
(`run_recipe_ci.py`) is responsible for translating its raw per-tier results into the rung-status
|
||||
dict this function consumes; that mapping is documented in DECISIONS.md (Phase 3).
|
||||
|
||||
Rung status vocabulary (each rung ∈ these three):
|
||||
"pass" — the rung was exercised and passed.
|
||||
"fail" — the rung was exercised and failed.
|
||||
"na" — the rung does not apply to this recipe (e.g. only one published version → no upgrade;
|
||||
not backup-capable; no SSO/integration surface; no recipe-local tests). N/A is NOT a
|
||||
failure, but it DOES cap the climb (with a distinct cap_reason) so the level never
|
||||
overstates what was actually verified.
|
||||
not backup-capable). N/A is NOT a failure, but it DOES cap the climb (with a distinct
|
||||
cap_reason) so the level never overstates what was actually verified.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
# The climbable rungs in ascending order. install (L1) is the foundation; L0 means install itself
|
||||
# did not pass. Each later rung requires every earlier rung to be a clean PASS.
|
||||
RUNGS = ("install", "upgrade", "backup_restore", "functional", "integration", "recipe_local")
|
||||
# did not pass. Each later rung requires every earlier rung to be a clean PASS. These four are the
|
||||
# ESSENTIAL rungs — integration/recipe-local are optional and deliberately NOT in this tuple.
|
||||
RUNGS = ("install", "upgrade", "backup_restore", "functional")
|
||||
|
||||
# Human-readable label per rung level, for cap_reason + the summary card.
|
||||
RUNG_LABEL = {
|
||||
@ -43,22 +45,20 @@ RUNG_LABEL = {
|
||||
2: "upgrade (prev published → PR)",
|
||||
3: "backup/restore (data integrity)",
|
||||
4: "functional (recipe-specific tests)",
|
||||
5: "integration (SSO/OIDC + cross-app)",
|
||||
6: "recipe-local (recipe repo tests/)",
|
||||
}
|
||||
|
||||
VALID = {"pass", "fail", "na"}
|
||||
|
||||
|
||||
def compute_level(rungs: dict[str, str]) -> tuple[int, str]:
|
||||
"""Map a rung-status dict → (level 0..6, cap_reason).
|
||||
"""Map a rung-status dict → (level 0..4, cap_reason).
|
||||
|
||||
`rungs` must contain a status in {"pass","fail","na"} for every name in RUNGS. The level is the
|
||||
highest L such that rungs[1..L] are all "pass"; the first non-"pass" rung caps the climb. L0 is
|
||||
returned when the install rung itself is not "pass" (install failed / never healthy).
|
||||
|
||||
cap_reason explains where the climb stopped:
|
||||
- "" (empty) when the recipe earned the top rung (L6, full clean climb).
|
||||
- "" (empty) when the recipe earned the top rung (L4, full clean climb).
|
||||
- "L<k> <label> FAILED" when a rung was exercised and failed.
|
||||
- "L<k> <label> N/A" when a rung does not apply to this recipe.
|
||||
Returns the reason for the FIRST rung that stopped the climb (the binding constraint).
|
||||
|
||||
@ -2,7 +2,14 @@
|
||||
|
||||
Turns a run's per-tier pytest outcomes into a single `results.json` artifact carrying, per the plan:
|
||||
{ recipe, version, pr, ref, run_id, finished, stages:[{name,status,tests:[{name,status,ms}]}],
|
||||
level, level_cap_reason, rungs, flags:{clean_teardown,no_secret_leak}, screenshot, summary_card }
|
||||
level, level_cap_reason, level_cap_rung, rungs,
|
||||
skips:{intentional:{rung:reason}, unintentional:[rung]},
|
||||
flags:{clean_teardown,no_secret_leak}, screenshot, summary_card }
|
||||
|
||||
`skips` splits the N/A (skipped) rungs by a simple rule: a skip is INTENTIONAL iff the recipe lists
|
||||
it (with a reason) in `recipe_meta.EXPECTED_NA = {rung: reason}`; any rung skipped but not listed is
|
||||
UNINTENTIONAL (a coverage gap to fill or declare). Skips still cap the level either way — the harness
|
||||
never claims a rung it did not verify; this only labels *why* a skip happened.
|
||||
|
||||
The per-test breakdown comes from JUnit XML emitted by each tier's pytest invocation (`--junitxml`),
|
||||
parsed here with the stdlib (no new dep). The integer **level** is computed by harness.level from a
|
||||
@ -127,41 +134,24 @@ def collect_stages(records: list[dict]) -> list[dict]:
|
||||
return stages
|
||||
|
||||
|
||||
def _has_repo_local(records: list[dict]) -> bool:
|
||||
return any(r.get("source") == "repo-local" for r in records)
|
||||
|
||||
|
||||
def _repo_local_passed(records: list[dict]) -> bool:
|
||||
repo = [r for r in records if r.get("source") == "repo-local"]
|
||||
return bool(repo) and all(r.get("rc", 1) == 0 for r in repo)
|
||||
|
||||
|
||||
def derive_rungs(
|
||||
results: dict[str, str],
|
||||
*,
|
||||
backup_capable: bool,
|
||||
declared: list[str] | None,
|
||||
deps_ready: bool,
|
||||
sso_unverified: bool,
|
||||
has_custom: bool,
|
||||
has_repo_local: bool,
|
||||
repo_local_passed: bool,
|
||||
) -> dict[str, str]:
|
||||
"""Translate the orchestrator's tier results + deps/SSO signals into the rung-status dict
|
||||
harness.level consumes. Documented in DECISIONS.md (Phase 3). Conservative by design — never
|
||||
reports a rung 'pass' it can't substantiate (cardinal guardrail: presentation never inflates).
|
||||
"""Translate the orchestrator's tier results into the rung-status dict harness.level consumes —
|
||||
the FOUR essential rungs only. Conservative by design — never reports a rung 'pass' it can't
|
||||
substantiate (cardinal guardrail: presentation never inflates).
|
||||
|
||||
L1 install : install tier pass.
|
||||
L2 upgrade : upgrade tier (skip → N/A: only one published version).
|
||||
L3 backup/res : backup AND restore tiers pass (N/A if not backup-capable).
|
||||
L4 functional : the recipe-specific functional (non-deps) tests pass — the custom tier, minus
|
||||
its SSO/integration tests. N/A if the recipe has no custom tests at all.
|
||||
L5 integration: SSO/OIDC + cross-app. Applies ONLY if the recipe declares deps (else N/A — the
|
||||
"no integration surface caps at L4" rule, §4.1). pass iff deps wired
|
||||
(deps_ready) and not sso_unverified and the custom tier didn't fail.
|
||||
L6 recipe-loc : the recipe repo's own tests/ (repo-local source) ran and passed (N/A if none).
|
||||
L4 functional : recipe-specific functional tests pass — the custom tier. N/A if none ran.
|
||||
|
||||
Integration (SSO/OIDC) and recipe-local are OPTIONAL and intentionally NOT rungs here — they
|
||||
never cap the level (SSO is still enforced for the run VERDICT in run_recipe_ci.py).
|
||||
"""
|
||||
declared = declared or []
|
||||
rungs: dict[str, str] = {}
|
||||
rungs["install"] = level_mod.tier_to_rung(results.get("install"))
|
||||
rungs["upgrade"] = level_mod.tier_to_rung(results.get("upgrade"))
|
||||
@ -170,36 +160,34 @@ def derive_rungs(
|
||||
)
|
||||
|
||||
custom = results.get("custom")
|
||||
# Functional rung (L4): the non-deps custom tests.
|
||||
if not has_custom or custom == "skip" or custom is None:
|
||||
rungs["functional"] = "na"
|
||||
elif custom == "fail":
|
||||
# A custom test failed. With declared deps we cannot cheaply tell functional-vs-SSO apart, so
|
||||
# conservatively fail the functional rung (caps at L3) — never inflate.
|
||||
rungs["functional"] = "fail"
|
||||
else: # custom == "pass"
|
||||
rungs["functional"] = "pass"
|
||||
|
||||
# Integration rung (L5): only recipes with an SSO/integration surface (declared deps) can climb.
|
||||
if not declared:
|
||||
rungs["integration"] = "na"
|
||||
elif sso_unverified or not deps_ready or custom == "fail":
|
||||
# SSO not wired/verified, or a custom test failed → integration not verified.
|
||||
rungs["integration"] = "fail"
|
||||
elif custom == "pass":
|
||||
rungs["integration"] = "pass"
|
||||
else:
|
||||
# declared deps but no custom tests ran — can't claim integration verified
|
||||
rungs["integration"] = "na"
|
||||
|
||||
# Recipe-local rung (L6).
|
||||
if not has_repo_local:
|
||||
rungs["recipe_local"] = "na"
|
||||
else:
|
||||
rungs["recipe_local"] = "pass" if repo_local_passed else "fail"
|
||||
return rungs
|
||||
|
||||
|
||||
def skips(rungs: dict[str, str], expected_na: dict | None) -> dict:
|
||||
"""Split the SKIPPED (N/A) rungs into intentional vs unintentional (operator model).
|
||||
|
||||
A recipe lists the rungs it intentionally skips, each with a reason, in
|
||||
`recipe_meta.EXPECTED_NA = {rung: reason}`. The rule is dead simple: a skipped rung is
|
||||
**intentional** iff it is in that list; any rung that is skipped and NOT in the list is
|
||||
**unintentional** (a coverage gap someone should either fill or declare). N/A still caps the
|
||||
level either way — the harness never claims a rung it did not verify — this only labels *why* a
|
||||
skip happened. Returns:
|
||||
{ "intentional": {rung: reason, ...}, # skipped AND declared in EXPECTED_NA
|
||||
"unintentional": [rung, ...] } # skipped but NOT declared
|
||||
"""
|
||||
expected = {str(k): str(v) for k, v in (expected_na or {}).items()}
|
||||
na = [r for r, st in rungs.items() if st == "na"]
|
||||
intentional = {r: expected[r] for r in na if r in expected}
|
||||
unintentional = sorted(r for r in na if r not in expected)
|
||||
return {"intentional": intentional, "unintentional": unintentional}
|
||||
|
||||
|
||||
def build_results(
|
||||
*,
|
||||
recipe: str,
|
||||
@ -209,30 +197,24 @@ def build_results(
|
||||
records: list[dict],
|
||||
results: dict[str, str],
|
||||
backup_capable: bool,
|
||||
declared: list[str] | None,
|
||||
deps_ready: bool,
|
||||
sso_unverified: bool,
|
||||
clean_teardown: bool,
|
||||
no_secret_leak: bool,
|
||||
finished_ts: float | None,
|
||||
screenshot: str | None = None,
|
||||
summary_card: str | None = None,
|
||||
expected_na: dict | None = None,
|
||||
) -> dict:
|
||||
"""Assemble the full results.json dict (no I/O). `finished_ts` is passed in (the orchestrator
|
||||
stamps it) so this stays pure and deterministic for unit tests."""
|
||||
stamps it) so this stays pure and deterministic for unit tests. `expected_na` is the recipe's
|
||||
declared intentional-skip map (recipe_meta.EXPECTED_NA) used to distinguish a deliberate skip from
|
||||
accidentally-missing coverage."""
|
||||
stages = collect_stages(records)
|
||||
has_custom = any(r["tier"] == "custom" for r in records)
|
||||
rungs = derive_rungs(
|
||||
results,
|
||||
backup_capable=backup_capable,
|
||||
declared=declared,
|
||||
deps_ready=deps_ready,
|
||||
sso_unverified=sso_unverified,
|
||||
has_custom=has_custom,
|
||||
has_repo_local=_has_repo_local(records),
|
||||
repo_local_passed=_repo_local_passed(records),
|
||||
)
|
||||
rungs = derive_rungs(results, backup_capable=backup_capable, has_custom=has_custom)
|
||||
lvl, cap_reason = level_mod.compute_level(rungs)
|
||||
# The rung that capped the climb (lowest non-pass), or None on a full climb — lets a consumer
|
||||
# (card/badge) tell whether the cap was an intentional skip, an unintentional one, or a failure.
|
||||
capped = level_mod.RUNGS[lvl] if cap_reason else None
|
||||
return {
|
||||
"schema": 1,
|
||||
"run_id": run_id(),
|
||||
@ -243,7 +225,9 @@ def build_results(
|
||||
"finished": finished_ts,
|
||||
"level": lvl,
|
||||
"level_cap_reason": cap_reason,
|
||||
"level_cap_rung": capped,
|
||||
"rungs": rungs,
|
||||
"skips": skips(rungs, expected_na),
|
||||
"stages": stages,
|
||||
"results": results,
|
||||
"flags": {
|
||||
|
||||
@ -200,6 +200,7 @@ def _load_meta(recipe: str) -> dict:
|
||||
for k in list(meta) + [
|
||||
"BACKUP_CAPABLE",
|
||||
"SKIP_GENERIC",
|
||||
"EXPECTED_NA",
|
||||
"OIDC_AT_INSTALL",
|
||||
"READY_PROBE",
|
||||
"UPGRADE_BASE_VERSION",
|
||||
@ -1224,7 +1225,6 @@ def main() -> int:
|
||||
# a failure here NEVER changes `overall` (R7 — cosmetics never block the pipeline). ----
|
||||
data: dict | None = None
|
||||
try:
|
||||
sso_unverified = sso_dep_unverified(declared, deps_ready, requires_deps_skipped)
|
||||
clean_teardown = (deploy_count == expected_deploy_count) and not dep_teardown_error
|
||||
data = results_mod.build_results(
|
||||
recipe=recipe,
|
||||
@ -1234,13 +1234,11 @@ def main() -> int:
|
||||
records=records,
|
||||
results=results,
|
||||
backup_capable=backup_cap,
|
||||
declared=declared,
|
||||
deps_ready=deps_ready,
|
||||
sso_unverified=sso_unverified,
|
||||
clean_teardown=clean_teardown,
|
||||
no_secret_leak=True, # narrowed below by an actual scan of the serialised artifact
|
||||
screenshot=screenshot_rel, # Phase 3 U1 (R4): relative PNG name iff capture succeeded
|
||||
finished_ts=time.time(),
|
||||
expected_na=meta.get("EXPECTED_NA"), # declared intentional-skip map (recipe_meta)
|
||||
)
|
||||
# Real (if narrow) leak check: no known infra-secret value may appear in the artifact (R7).
|
||||
blob = json.dumps(data)
|
||||
@ -1257,6 +1255,15 @@ def main() -> int:
|
||||
f"{' — ' + data['level_cap_reason'] if data['level_cap_reason'] else ''})",
|
||||
flush=True,
|
||||
)
|
||||
# Surface UNINTENTIONAL skips in the CI log (non-blocking, R7): a rung that was skipped (N/A)
|
||||
# but is not in the recipe's intentional list — either add the missing coverage or declare it.
|
||||
for rung in data.get("skips", {}).get("unintentional", []):
|
||||
print(
|
||||
f"⚠ coverage: rung '{rung}' was skipped (N/A) but is not declared intentional — add "
|
||||
f"the missing test/label, or list it in tests/{recipe}/recipe_meta.py "
|
||||
f"EXPECTED_NA = {{'{rung}': '<why>'}}.",
|
||||
flush=True,
|
||||
)
|
||||
except Exception as e: # noqa: BLE001 — results assembly is cosmetic; never fail a run on it (R7)
|
||||
print(
|
||||
f"!! results.json assembly failed (non-fatal, verdict unaffected): {_scrub(str(e))}",
|
||||
@ -1275,8 +1282,19 @@ def main() -> int:
|
||||
with open(html_path, "w", encoding="utf-8") as f:
|
||||
f.write(card_mod.render_card_html(data, screenshot_rel=data.get("screenshot")))
|
||||
png = card_mod.render_card_png(html_path, os.path.join(run_artifact_dir, "summary.png"))
|
||||
capped = data.get("level_cap_rung")
|
||||
sk = data.get("skips", {})
|
||||
cap_skip = (
|
||||
"intentional" if capped in (sk.get("intentional") or {})
|
||||
else "unintentional" if capped in (sk.get("unintentional") or [])
|
||||
else ""
|
||||
)
|
||||
with open(os.path.join(run_artifact_dir, "badge.svg"), "w", encoding="utf-8") as f:
|
||||
f.write(card_mod.level_badge_svg(data["level"], data.get("level_cap_reason", "")))
|
||||
f.write(
|
||||
card_mod.level_badge_svg(
|
||||
data["level"], data.get("level_cap_reason", ""), cap_skip
|
||||
)
|
||||
)
|
||||
print(
|
||||
f"summary card {'rendered ' + png if png else '(PNG render unavailable)'} + "
|
||||
f"badge.svg written into {run_artifact_dir}",
|
||||
|
||||
Reference in New Issue
Block a user