feat(harness): intentional skips + custom-html-tiny functional test; 4-rung ladder (#6)
Some checks failed
continuous-integration/drone/push Build is failing
Some checks failed
continuous-integration/drone/push Build is failing
Declare intentional skips + custom-html-tiny functional test; 4-rung level ladder
- recipe_meta.EXPECTED_NA = {rung: reason} lists intentionally-skipped rungs; any
essential rung skipped and not listed is unintentional. Skips still cap the level
(never inflate). results.json: skips:{intentional,unintentional} + level_cap_rung.
- Level ladder = the four essential rungs (install, upgrade, backup/restore,
functional; top = L4). integration & recipe-local are optional, not leveled
(SSO still enforced for the run verdict, unchanged).
- Card shows skipped rungs as INTENTIONAL SKIP (green, reason below) / UNINTENTIONAL
SKIP (amber); level badge gains an expected/gap? third segment.
- custom-html-tiny: functional serve test (exact-byte round-trip + 404); declares
backup_restore intentionally skipped (stateless static server).
Independently verified by the adversary: 138 unit tests pass cold; live full-stage
run on custom-html-tiny green (upgrade tier ran; level 2; correct skips/badge);
clean teardown.
This commit is contained in:
@ -19,33 +19,23 @@ def _rungs(
|
||||
upgrade="pass",
|
||||
backup_restore="pass",
|
||||
functional="pass",
|
||||
integration="pass",
|
||||
recipe_local="pass",
|
||||
):
|
||||
return {
|
||||
"install": install,
|
||||
"upgrade": upgrade,
|
||||
"backup_restore": backup_restore,
|
||||
"functional": functional,
|
||||
"integration": integration,
|
||||
"recipe_local": recipe_local,
|
||||
}
|
||||
|
||||
|
||||
# ---- the U0 gate: L4-pass and L2-cap ----
|
||||
# ---- the ladder: four essential rungs, top is L4 (functional) ----
|
||||
|
||||
|
||||
def test_full_clean_climb_to_L6():
|
||||
def test_full_clean_climb_to_L4():
|
||||
# All four essential rungs pass → L4 (the top; integration/recipe-local are optional, not leveled).
|
||||
lvl, reason = L.compute_level(_rungs())
|
||||
assert lvl == 6
|
||||
assert reason == ""
|
||||
|
||||
|
||||
def test_climbs_through_L4_then_no_integration_surface_caps_at_L4():
|
||||
# GATE: a recipe whose functional tests pass but has no SSO/integration surface caps at L4.
|
||||
lvl, reason = L.compute_level(_rungs(integration="na", recipe_local="na"))
|
||||
assert lvl == 4
|
||||
assert "L5" in reason and "N/A" in reason
|
||||
assert reason == ""
|
||||
|
||||
|
||||
def test_fails_at_L2_capped_at_L1():
|
||||
@ -69,34 +59,27 @@ def test_install_fail_is_L0():
|
||||
|
||||
def test_higher_pass_does_not_rescue_lower_na():
|
||||
# backup/restore N/A (stateless app) caps at L2 even though functional would pass.
|
||||
lvl, reason = L.compute_level(_rungs(backup_restore="na", functional="pass", integration="na"))
|
||||
lvl, reason = L.compute_level(_rungs(backup_restore="na", functional="pass"))
|
||||
assert lvl == 2
|
||||
assert "L3" in reason and "N/A" in reason
|
||||
|
||||
|
||||
def test_upgrade_na_caps_at_L1():
|
||||
# only one published version → no upgrade possible → N/A caps at L1.
|
||||
# only one published version → no upgrade possible → N/A caps at L1 (upgrade is essential).
|
||||
lvl, reason = L.compute_level(_rungs(upgrade="na"))
|
||||
assert lvl == 1
|
||||
assert "L2" in reason and "N/A" in reason
|
||||
|
||||
|
||||
def test_integration_fail_caps_at_L4():
|
||||
# SSO declared but unverified (failed) → integration rung fails → cap at L4.
|
||||
lvl, reason = L.compute_level(_rungs(integration="fail", recipe_local="na"))
|
||||
assert lvl == 4
|
||||
assert "L5" in reason and "FAILED" in reason
|
||||
|
||||
|
||||
def test_recipe_local_na_caps_at_L5():
|
||||
# SSO passes but no recipe-local tests → cap at L5 (L6 N/A).
|
||||
lvl, reason = L.compute_level(_rungs(recipe_local="na"))
|
||||
assert lvl == 5
|
||||
assert "L6" in reason and "N/A" in reason
|
||||
def test_functional_na_caps_at_L3():
|
||||
# no recipe-specific functional tests → functional N/A caps at L3.
|
||||
lvl, reason = L.compute_level(_rungs(functional="na"))
|
||||
assert lvl == 3
|
||||
assert "L4" in reason and "N/A" in reason
|
||||
|
||||
|
||||
def test_functional_fail_caps_at_L3():
|
||||
lvl, reason = L.compute_level(_rungs(functional="fail", integration="na"))
|
||||
lvl, reason = L.compute_level(_rungs(functional="fail"))
|
||||
assert lvl == 3
|
||||
assert "L4" in reason and "FAILED" in reason
|
||||
|
||||
|
||||
Reference in New Issue
Block a user