feat(harness): declare intentional N/A tiers + custom-html-tiny functional test
Some checks failed
continuous-integration/drone/push Build is failing
Some checks failed
continuous-integration/drone/push Build is failing
Two changes the operator asked for after noticing custom-html-tiny PR #6 has no backup/restore or functional coverage: 1) Intentional-vs-accidental N/A. A recipe can now declare recipe_meta.EXPECTED_NA = {rung: reason} to mark a tier as deliberately not applicable (e.g. a stateless static server has no backup surface). N/A still caps the level — the harness never claims a rung it did not verify — but the run is now annotated 'intentional · <reason>' instead of being indistinguishable from a forgotten test. An *undeclared* N/A on a gap-sensitive rung (backup_restore, functional) is surfaced as a 'possible coverage gap', and a stale EXPECTED_NA (declared N/A but actually exercised) is surfaced too. All non-blocking (R7): results.json gains level_cap_intent + an block, the summary card shows the clause, and the CI log prints the gap/stale warnings. (results.classify_na/cap_intent are pure + unit-tested; level.py untouched.) custom-html-tiny declares backup_restore intentionally N/A. 2) custom-html-tiny functional test: writes a random file into the served content volume (via the volume mountpoint, like install_steps.sh, since the SWS image is shell-less), asserts exact-byte round-trip + a real 404 on a missing path — proving the static-web-server actually serves the volume, not a 200-everything fallback. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@ -257,6 +257,104 @@ def test_build_results_capped_at_L1_on_upgrade_fail(tmp_path):
|
||||
assert "L2" in data["level_cap_reason"]
|
||||
|
||||
|
||||
# ---- classify_na / cap_intent: intentional-vs-accidental N/A (operator request) ----
|
||||
|
||||
|
||||
def _rungs(**kw):
|
||||
base = {
|
||||
"install": "pass",
|
||||
"upgrade": "pass",
|
||||
"backup_restore": "pass",
|
||||
"functional": "pass",
|
||||
"integration": "na",
|
||||
"recipe_local": "na",
|
||||
}
|
||||
base.update(kw)
|
||||
return base
|
||||
|
||||
|
||||
def test_classify_na_declared_vs_undeclared():
|
||||
rungs = _rungs(backup_restore="na", functional="na")
|
||||
info = R.classify_na(rungs, {"backup_restore": "stateless static server"})
|
||||
# backup_restore is declared intentional; functional is an undeclared gap-sensitive N/A.
|
||||
assert info["rungs"]["backup_restore"] == {
|
||||
"intent": "declared",
|
||||
"reason": "stateless static server",
|
||||
}
|
||||
assert info["rungs"]["functional"]["intent"] == "undeclared"
|
||||
assert info["gaps"] == ["functional"] # backup_restore declared → not a gap
|
||||
assert info["stale_declared"] == []
|
||||
# structurally-optional N/A (integration, recipe_local) are recorded but never flagged as gaps.
|
||||
assert info["rungs"]["integration"]["intent"] == "undeclared"
|
||||
assert "integration" not in info["gaps"]
|
||||
|
||||
|
||||
def test_classify_na_stale_declaration():
|
||||
# backup_restore actually ran (pass) but is declared N/A → stale opt-out, surfaced.
|
||||
rungs = _rungs(backup_restore="pass")
|
||||
info = R.classify_na(rungs, {"backup_restore": "stale reason"})
|
||||
assert info["stale_declared"] == ["backup_restore"]
|
||||
assert "backup_restore" not in info["rungs"] # not N/A, so not in the per-rung N/A map
|
||||
|
||||
|
||||
def test_cap_intent_declared_explains_cap():
|
||||
# install+upgrade pass, backup_restore declared-N/A → caps at L2 with an intentional clause.
|
||||
rungs = _rungs(backup_restore="na")
|
||||
info = R.classify_na(rungs, {"backup_restore": "no persistent data"})
|
||||
intent = R.cap_intent(rungs, 2, "L3 backup/restore (data integrity) N/A", info)
|
||||
assert intent == "intentional · no persistent data"
|
||||
|
||||
|
||||
def test_cap_intent_undeclared_gap():
|
||||
rungs = _rungs(backup_restore="na")
|
||||
info = R.classify_na(rungs, None)
|
||||
intent = R.cap_intent(rungs, 2, "L3 backup/restore (data integrity) N/A", info)
|
||||
assert "possible coverage gap" in intent
|
||||
|
||||
|
||||
def test_cap_intent_blank_when_not_capped_on_na():
|
||||
rungs = _rungs() # full clean climb, capped only at integration (na, structurally optional)
|
||||
info = R.classify_na(rungs, None)
|
||||
# capping rung is integration (level 4) — structurally optional, so no intent clause.
|
||||
assert R.cap_intent(rungs, 4, "L5 integration N/A", info) == ""
|
||||
# and no cap at all → blank.
|
||||
assert R.cap_intent(rungs, 6, "", info) == ""
|
||||
|
||||
|
||||
def test_build_results_threads_expected_na(tmp_path):
|
||||
recs = [
|
||||
{
|
||||
"tier": "install",
|
||||
"source": "generic",
|
||||
"file": "g/test_install.py",
|
||||
"rc": 0,
|
||||
"junit": _write(tmp_path, "i.xml", JUNIT_PASS),
|
||||
}
|
||||
]
|
||||
data = R.build_results(
|
||||
recipe="custom-html-tiny",
|
||||
version="1.1.0",
|
||||
pr="0",
|
||||
ref=None,
|
||||
records=recs,
|
||||
results=_results(backup="skip", restore="skip", custom="skip"),
|
||||
backup_capable=False, # no backupbot label → backup_restore N/A
|
||||
declared=[],
|
||||
deps_ready=True,
|
||||
sso_unverified=False,
|
||||
clean_teardown=True,
|
||||
no_secret_leak=True,
|
||||
finished_ts=0.0,
|
||||
expected_na={"backup_restore": "stateless static file server"},
|
||||
)
|
||||
# N/A still caps at L2 (never inflates), but now annotated intentional rather than flagged.
|
||||
assert data["level"] == 2
|
||||
assert "L3" in data["level_cap_reason"]
|
||||
assert data["level_cap_intent"] == "intentional · stateless static file server"
|
||||
assert data["na"]["rungs"]["backup_restore"]["intent"] == "declared"
|
||||
assert data["na"]["gaps"] == []
|
||||
|
||||
|
||||
def test_write_results_roundtrip(tmp_path):
|
||||
data = {"run_id": "42", "level": 3, "stages": []}
|
||||
path = R.write_results(data, runs_dir_override=str(tmp_path))
|
||||
|
||||
Reference in New Issue
Block a user