feat(harness): intentional skips + custom-html-tiny functional test; 4-rung ladder (#6)
Some checks failed
continuous-integration/drone/push Build is failing
Some checks failed
continuous-integration/drone/push Build is failing
Declare intentional skips + custom-html-tiny functional test; 4-rung level ladder
- recipe_meta.EXPECTED_NA = {rung: reason} lists intentionally-skipped rungs; any
essential rung skipped and not listed is unintentional. Skips still cap the level
(never inflate). results.json: skips:{intentional,unintentional} + level_cap_rung.
- Level ladder = the four essential rungs (install, upgrade, backup/restore,
functional; top = L4). integration & recipe-local are optional, not leveled
(SSO still enforced for the run verdict, unchanged).
- Card shows skipped rungs as INTENTIONAL SKIP (green, reason below) / UNINTENTIONAL
SKIP (amber); level badge gains an expected/gap? third segment.
- custom-html-tiny: functional serve test (exact-byte round-trip + 404); declares
backup_restore intentionally skipped (stateless static server).
Independently verified by the adversary: 138 unit tests pass cold; live full-stage
run on custom-html-tiny green (upgrade tier ran; level 2; correct skips/badge);
clean teardown.
This commit is contained in:
@ -105,83 +105,31 @@ def _results(**kw):
|
||||
return base
|
||||
|
||||
|
||||
def test_derive_rungs_full_stateful_sso():
|
||||
rungs = R.derive_rungs(
|
||||
_results(),
|
||||
backup_capable=True,
|
||||
declared=["keycloak"],
|
||||
deps_ready=True,
|
||||
sso_unverified=False,
|
||||
has_custom=True,
|
||||
has_repo_local=False,
|
||||
repo_local_passed=False,
|
||||
)
|
||||
def test_derive_rungs_full_climb_four_essential():
|
||||
rungs = R.derive_rungs(_results(), backup_capable=True, has_custom=True)
|
||||
# only the four essential rungs — integration/recipe-local are optional, not produced here.
|
||||
assert rungs == {
|
||||
"install": "pass",
|
||||
"upgrade": "pass",
|
||||
"backup_restore": "pass",
|
||||
"functional": "pass",
|
||||
"integration": "pass",
|
||||
"recipe_local": "na",
|
||||
}
|
||||
|
||||
|
||||
def test_derive_rungs_no_sso_surface_is_integration_na():
|
||||
rungs = R.derive_rungs(
|
||||
_results(),
|
||||
backup_capable=True,
|
||||
declared=[],
|
||||
deps_ready=True,
|
||||
sso_unverified=False,
|
||||
has_custom=True,
|
||||
has_repo_local=False,
|
||||
repo_local_passed=False,
|
||||
)
|
||||
assert rungs["integration"] == "na"
|
||||
assert rungs["functional"] == "pass"
|
||||
|
||||
|
||||
def test_derive_rungs_stateless_backup_na():
|
||||
def test_derive_rungs_stateless_backup_and_functional_na():
|
||||
rungs = R.derive_rungs(
|
||||
_results(backup="skip", restore="skip", custom="skip"),
|
||||
backup_capable=False,
|
||||
declared=[],
|
||||
deps_ready=True,
|
||||
sso_unverified=False,
|
||||
has_custom=False,
|
||||
has_repo_local=False,
|
||||
repo_local_passed=False,
|
||||
)
|
||||
assert rungs["backup_restore"] == "na"
|
||||
assert rungs["functional"] == "na"
|
||||
assert "integration" not in rungs and "recipe_local" not in rungs
|
||||
|
||||
|
||||
def test_derive_rungs_sso_unverified_is_integration_fail():
|
||||
rungs = R.derive_rungs(
|
||||
_results(),
|
||||
backup_capable=True,
|
||||
declared=["keycloak"],
|
||||
deps_ready=False,
|
||||
sso_unverified=True,
|
||||
has_custom=True,
|
||||
has_repo_local=False,
|
||||
repo_local_passed=False,
|
||||
)
|
||||
assert rungs["integration"] == "fail"
|
||||
|
||||
|
||||
def test_derive_rungs_repo_local_pass():
|
||||
rungs = R.derive_rungs(
|
||||
_results(),
|
||||
backup_capable=True,
|
||||
declared=[],
|
||||
deps_ready=True,
|
||||
sso_unverified=False,
|
||||
has_custom=True,
|
||||
has_repo_local=True,
|
||||
repo_local_passed=True,
|
||||
)
|
||||
assert rungs["recipe_local"] == "pass"
|
||||
def test_derive_rungs_functional_fail():
|
||||
rungs = R.derive_rungs(_results(custom="fail"), backup_capable=True, has_custom=True)
|
||||
assert rungs["functional"] == "fail"
|
||||
|
||||
|
||||
# ---- build_results: end-to-end incl level + flags ----
|
||||
@ -212,16 +160,13 @@ def test_build_results_level_and_flags(tmp_path):
|
||||
records=recs,
|
||||
results=_results(),
|
||||
backup_capable=True,
|
||||
declared=[],
|
||||
deps_ready=True,
|
||||
sso_unverified=False,
|
||||
clean_teardown=True,
|
||||
no_secret_leak=True,
|
||||
finished_ts=1234.0,
|
||||
)
|
||||
# stateful, functional pass, no SSO surface, no repo-local → caps at L4
|
||||
# all four essential rungs pass → full climb to L4 (the top), no cap
|
||||
assert data["level"] == 4
|
||||
assert "L5" in data["level_cap_reason"]
|
||||
assert data["level_cap_reason"] == ""
|
||||
assert data["recipe"] == "hedgedoc"
|
||||
assert data["ref"] == "deadbeefcafe"
|
||||
assert data["flags"] == {"clean_teardown": True, "no_secret_leak": True}
|
||||
@ -246,9 +191,6 @@ def test_build_results_capped_at_L1_on_upgrade_fail(tmp_path):
|
||||
records=recs,
|
||||
results=_results(upgrade="fail"),
|
||||
backup_capable=True,
|
||||
declared=[],
|
||||
deps_ready=True,
|
||||
sso_unverified=False,
|
||||
clean_teardown=True,
|
||||
no_secret_leak=True,
|
||||
finished_ts=0.0,
|
||||
@ -257,6 +199,85 @@ def test_build_results_capped_at_L1_on_upgrade_fail(tmp_path):
|
||||
assert "L2" in data["level_cap_reason"]
|
||||
|
||||
|
||||
# ---- skips: intentional (declared) vs unintentional (everything else skipped) ----
|
||||
|
||||
|
||||
def _rungs(**kw):
|
||||
base = {
|
||||
"install": "pass",
|
||||
"upgrade": "pass",
|
||||
"backup_restore": "pass",
|
||||
"functional": "pass",
|
||||
}
|
||||
base.update(kw)
|
||||
return base
|
||||
|
||||
|
||||
def test_skips_intentional_vs_unintentional():
|
||||
rungs = _rungs(backup_restore="na", functional="na")
|
||||
sk = R.skips(rungs, {"backup_restore": "stateless static server"})
|
||||
# backup_restore is declared (intentional, with reason); functional skipped but not declared.
|
||||
assert sk["intentional"] == {"backup_restore": "stateless static server"}
|
||||
assert sk["unintentional"] == ["functional"]
|
||||
|
||||
|
||||
def test_skips_none_declared_all_unintentional():
|
||||
rungs = _rungs(backup_restore="na")
|
||||
sk = R.skips(rungs, None)
|
||||
assert sk["intentional"] == {}
|
||||
assert sk["unintentional"] == ["backup_restore"]
|
||||
|
||||
|
||||
def test_skips_declaration_only_counts_when_actually_skipped():
|
||||
# backup_restore actually ran (pass) → not a skip, so a declaration for it is simply inert.
|
||||
rungs = _rungs(backup_restore="pass")
|
||||
sk = R.skips(rungs, {"backup_restore": "reason"})
|
||||
assert "backup_restore" not in sk["intentional"]
|
||||
assert "backup_restore" not in sk["unintentional"]
|
||||
|
||||
|
||||
def test_build_results_threads_expected_na(tmp_path):
|
||||
# Mirrors custom-html-tiny post-change: install + a passing functional (custom) test, but no
|
||||
# backup surface (backup_restore declared intentionally skipped).
|
||||
recs = [
|
||||
{
|
||||
"tier": "install",
|
||||
"source": "generic",
|
||||
"file": "g/test_install.py",
|
||||
"rc": 0,
|
||||
"junit": _write(tmp_path, "i.xml", JUNIT_PASS),
|
||||
},
|
||||
{
|
||||
"tier": "custom",
|
||||
"source": "cc-ci",
|
||||
"file": "c/test_serves_content.py",
|
||||
"rc": 0,
|
||||
"junit": _write(tmp_path, "c.xml", JUNIT_PASS),
|
||||
},
|
||||
]
|
||||
data = R.build_results(
|
||||
recipe="custom-html-tiny",
|
||||
version="1.1.0",
|
||||
pr="0",
|
||||
ref=None,
|
||||
records=recs,
|
||||
results=_results(backup="skip", restore="skip"), # custom=pass (default) → functional pass
|
||||
backup_capable=False, # no backupbot label → backup_restore skipped (N/A)
|
||||
clean_teardown=True,
|
||||
no_secret_leak=True,
|
||||
finished_ts=0.0,
|
||||
expected_na={"backup_restore": "stateless static file server"},
|
||||
)
|
||||
# backup_restore skip still caps at L2 (never inflates) — even though functional passes above it,
|
||||
# the skip caps the climb — but it's the declared (intentional) rung that capped.
|
||||
assert data["level"] == 2
|
||||
assert "L3" in data["level_cap_reason"]
|
||||
assert data["level_cap_rung"] == "backup_restore"
|
||||
assert data["rungs"]["functional"] == "pass"
|
||||
assert data["skips"]["intentional"]["backup_restore"] == "stateless static file server"
|
||||
assert data["skips"]["unintentional"] == [] # backup_restore declared; functional passed → clean
|
||||
|
||||
|
||||
def test_write_results_roundtrip(tmp_path):
|
||||
data = {"run_id": "42", "level": 3, "stages": []}
|
||||
path = R.write_results(data, runs_dir_override=str(tmp_path))
|
||||
|
||||
Reference in New Issue
Block a user