feat(harness): intentional skips + custom-html-tiny functional test; 4-rung ladder (#6)
Some checks failed
continuous-integration/drone/push Build is failing
Some checks failed
continuous-integration/drone/push Build is failing
Declare intentional skips + custom-html-tiny functional test; 4-rung level ladder
- recipe_meta.EXPECTED_NA = {rung: reason} lists intentionally-skipped rungs; any
essential rung skipped and not listed is unintentional. Skips still cap the level
(never inflate). results.json: skips:{intentional,unintentional} + level_cap_rung.
- Level ladder = the four essential rungs (install, upgrade, backup/restore,
functional; top = L4). integration & recipe-local are optional, not leveled
(SSO still enforced for the run verdict, unchanged).
- Card shows skipped rungs as INTENTIONAL SKIP (green, reason below) / UNINTENTIONAL
SKIP (amber); level badge gains an expected/gap? third segment.
- custom-html-tiny: functional serve test (exact-byte round-trip + 404); declares
backup_restore intentionally skipped (stateless static server).
Independently verified by the adversary: 138 unit tests pass cold; live full-stage
run on custom-html-tiny green (upgrade tier ran; level 2; correct skips/badge);
clean teardown.
This commit is contained in:
@ -14,7 +14,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner")
|
||||
from harness import card as C # noqa: E402
|
||||
|
||||
|
||||
def _data(level=4, cap="L5 integration (SSO/OIDC + cross-app) N/A"):
|
||||
def _data(level=3, cap="L4 functional (recipe-specific tests) N/A"):
|
||||
return {
|
||||
"recipe": "uptime-kuma",
|
||||
"version": "1.23.0",
|
||||
@ -51,6 +51,35 @@ def test_badge_svg_wellformed():
|
||||
assert svg.startswith("<svg") and svg.endswith("</svg>")
|
||||
assert "level 4" in svg
|
||||
assert C.level_color(4) in svg
|
||||
# plain cap (no intent) → two-box badge, no third segment
|
||||
assert "expected" not in svg and "gap?" not in svg
|
||||
|
||||
|
||||
def test_badge_svg_differentiates_intentional_vs_unintentional_skip():
|
||||
# an intentional (declared) skip capped the climb → muted "expected" third segment
|
||||
exp = C.level_badge_svg(2, "L3 backup/restore N/A", "intentional")
|
||||
assert "level 2" in exp and "expected" in exp and C.EXPECT_COLOR in exp
|
||||
assert "gap?" not in exp
|
||||
# an unintentional skip (not declared) → amber "gap?" third segment
|
||||
gap = C.level_badge_svg(2, "L3 backup/restore N/A", "unintentional")
|
||||
assert "level 2" in gap and "gap?" in gap and C.GAP_COLOR in gap
|
||||
assert "expected" not in gap
|
||||
|
||||
|
||||
def test_skip_rows_intentional_and_unintentional():
|
||||
html_out = C._skip_rows(
|
||||
{"intentional": {"backup_restore": "no persistent data"}, "unintentional": ["functional"]}
|
||||
)
|
||||
# intentional skip: labelled row (muted green) + the reason on its own line
|
||||
assert "intentional skip" in html_out and C.SKIP_GREEN in html_out
|
||||
assert "backup/restore" in html_out and "no persistent data" in html_out
|
||||
# unintentional skip: amber row + prompt to declare/add coverage
|
||||
assert "unintentional skip" in html_out and C.GAP_COLOR in html_out
|
||||
assert "functional" in html_out and "EXPECTED_NA" in html_out
|
||||
|
||||
|
||||
def test_skip_rows_empty_when_no_skips():
|
||||
assert C._skip_rows({"intentional": {}, "unintentional": []}) == ""
|
||||
|
||||
|
||||
def test_card_html_reports_level_verbatim():
|
||||
|
||||
Reference in New Issue
Block a user