refactor: simplify to a list of intentionally-skipped rungs
Some checks failed
continuous-integration/drone/push Build is failing
Some checks failed
continuous-integration/drone/push Build is failing
Per operator: drop the gap-sensitivity / cap-intent-clause / stale-detection
machinery. Model is now dead simple — recipe_meta.EXPECTED_NA = {rung: reason}
lists the rungs a recipe intentionally skips; ANY rung skipped (N/A) and not in
that list is unintentional.
results.json: replace the 'na' block + level_cap_intent with
skips: { intentional: {rung: reason}, unintentional: [rung] }
plus level_cap_rung (which rung capped). Badge/card derive intentional-vs-
unintentional from whether the capping rung is in the intentional list. Skips
still cap the level (never inflate). custom-html-tiny lists all three rungs it
intentionally skips (backup_restore, integration, recipe_local).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@ -4,14 +4,14 @@
|
||||
DEPLOY_TIMEOUT = 120
|
||||
HTTP_TIMEOUT = 90
|
||||
|
||||
# Intentionally-N/A tiers (reviewed opt-out, NOT a coverage gap). custom-html-tiny is a stateless
|
||||
# static-web-server: it serves an ephemeral `content` volume that the harness seeds at deploy time
|
||||
# (install_steps.sh) and holds no persistent or user data, so there is nothing to back up or restore.
|
||||
# The recipe therefore declares no `backupbot.backup` label and the L3 backup/restore rung is N/A.
|
||||
# Declaring it here marks that N/A as deliberate, so the run is annotated "intentional" instead of
|
||||
# being flagged as a possible missing-coverage gap. (N/A still caps the level — the harness never
|
||||
# claims a rung it did not verify; this only explains *why* the cap is expected.)
|
||||
# Rungs this recipe INTENTIONALLY skips, each with a reason. Any rung that is skipped (N/A) and is
|
||||
# NOT listed here is reported as an *unintentional* skip (a coverage gap to fill or declare). A skip
|
||||
# still caps the level either way — the harness never claims a rung it did not verify; this only
|
||||
# records that the skip is deliberate. custom-html-tiny is a stateless static-web-server, so:
|
||||
EXPECTED_NA = {
|
||||
"backup_restore": "stateless static file server: serves an ephemeral content volume seeded at "
|
||||
"deploy, with no persistent/user data to back up or restore (no backupbot.backup label)",
|
||||
"integration": "no SSO/OIDC or cross-app surface — a static file server has no auth integration",
|
||||
"recipe_local": "the upstream recipe ships no tests/ of its own; coverage is the cc-ci generic "
|
||||
"install tier + the functional serve test",
|
||||
}
|
||||
|
||||
@ -55,13 +55,13 @@ def test_badge_svg_wellformed():
|
||||
assert "expected" not in svg and "gap?" not in svg
|
||||
|
||||
|
||||
def test_badge_svg_differentiates_expected_vs_unexpected_skip():
|
||||
# declared intentional N/A capped the climb → muted "expected" third segment
|
||||
exp = C.level_badge_svg(2, "L3 backup/restore N/A", "intentional · no persistent data")
|
||||
def test_badge_svg_differentiates_intentional_vs_unintentional_skip():
|
||||
# an intentional (declared) skip capped the climb → muted "expected" third segment
|
||||
exp = C.level_badge_svg(2, "L3 backup/restore N/A", "intentional")
|
||||
assert "level 2" in exp and "expected" in exp and C.EXPECT_COLOR in exp
|
||||
assert "gap?" not in exp
|
||||
# undeclared gap-sensitive N/A → amber "gap?" third segment (an UNEXPECTED skip)
|
||||
gap = C.level_badge_svg(2, "L3 backup/restore N/A", "undeclared N/A — possible coverage gap")
|
||||
# an unintentional skip (not declared) → amber "gap?" third segment
|
||||
gap = C.level_badge_svg(2, "L3 backup/restore N/A", "unintentional")
|
||||
assert "level 2" in gap and "gap?" in gap and C.GAP_COLOR in gap
|
||||
assert "expected" not in gap
|
||||
|
||||
|
||||
@ -257,7 +257,7 @@ def test_build_results_capped_at_L1_on_upgrade_fail(tmp_path):
|
||||
assert "L2" in data["level_cap_reason"]
|
||||
|
||||
|
||||
# ---- classify_na / cap_intent: intentional-vs-accidental N/A (operator request) ----
|
||||
# ---- skips: intentional (declared) vs unintentional (everything else skipped) ----
|
||||
|
||||
|
||||
def _rungs(**kw):
|
||||
@ -273,57 +273,32 @@ def _rungs(**kw):
|
||||
return base
|
||||
|
||||
|
||||
def test_classify_na_declared_vs_undeclared():
|
||||
def test_skips_intentional_vs_unintentional():
|
||||
rungs = _rungs(backup_restore="na", functional="na")
|
||||
info = R.classify_na(rungs, {"backup_restore": "stateless static server"})
|
||||
# backup_restore is declared intentional; functional is an undeclared gap-sensitive N/A.
|
||||
assert info["rungs"]["backup_restore"] == {
|
||||
"intent": "declared",
|
||||
"reason": "stateless static server",
|
||||
}
|
||||
assert info["rungs"]["functional"]["intent"] == "undeclared"
|
||||
assert info["gaps"] == ["functional"] # backup_restore declared → not a gap
|
||||
assert info["stale_declared"] == []
|
||||
# structurally-optional N/A (integration, recipe_local) are recorded but never flagged as gaps.
|
||||
assert info["rungs"]["integration"]["intent"] == "undeclared"
|
||||
assert "integration" not in info["gaps"]
|
||||
sk = R.skips(rungs, {"backup_restore": "stateless static server"})
|
||||
# backup_restore is declared (intentional, with reason); everything else skipped is unintentional.
|
||||
assert sk["intentional"] == {"backup_restore": "stateless static server"}
|
||||
assert sk["unintentional"] == ["functional", "integration", "recipe_local"]
|
||||
|
||||
|
||||
def test_classify_na_stale_declaration():
|
||||
# backup_restore actually ran (pass) but is declared N/A → stale opt-out, surfaced.
|
||||
def test_skips_none_declared_all_unintentional():
|
||||
rungs = _rungs(backup_restore="na")
|
||||
sk = R.skips(rungs, None)
|
||||
assert sk["intentional"] == {}
|
||||
assert sk["unintentional"] == ["backup_restore", "integration", "recipe_local"]
|
||||
|
||||
|
||||
def test_skips_declaration_only_counts_when_actually_skipped():
|
||||
# backup_restore actually ran (pass) → not a skip, so a declaration for it is simply inert.
|
||||
rungs = _rungs(backup_restore="pass")
|
||||
info = R.classify_na(rungs, {"backup_restore": "stale reason"})
|
||||
assert info["stale_declared"] == ["backup_restore"]
|
||||
assert "backup_restore" not in info["rungs"] # not N/A, so not in the per-rung N/A map
|
||||
|
||||
|
||||
def test_cap_intent_declared_explains_cap():
|
||||
# install+upgrade pass, backup_restore declared-N/A → caps at L2 with an intentional clause.
|
||||
rungs = _rungs(backup_restore="na")
|
||||
info = R.classify_na(rungs, {"backup_restore": "no persistent data"})
|
||||
intent = R.cap_intent(rungs, 2, "L3 backup/restore (data integrity) N/A", info)
|
||||
assert intent == "intentional · no persistent data"
|
||||
|
||||
|
||||
def test_cap_intent_undeclared_gap():
|
||||
rungs = _rungs(backup_restore="na")
|
||||
info = R.classify_na(rungs, None)
|
||||
intent = R.cap_intent(rungs, 2, "L3 backup/restore (data integrity) N/A", info)
|
||||
assert "possible coverage gap" in intent
|
||||
|
||||
|
||||
def test_cap_intent_blank_when_not_capped_on_na():
|
||||
rungs = _rungs() # full clean climb, capped only at integration (na, structurally optional)
|
||||
info = R.classify_na(rungs, None)
|
||||
# capping rung is integration (level 4) — structurally optional, so no intent clause.
|
||||
assert R.cap_intent(rungs, 4, "L5 integration N/A", info) == ""
|
||||
# and no cap at all → blank.
|
||||
assert R.cap_intent(rungs, 6, "", info) == ""
|
||||
sk = R.skips(rungs, {"backup_restore": "reason"})
|
||||
assert "backup_restore" not in sk["intentional"]
|
||||
assert "backup_restore" not in sk["unintentional"]
|
||||
|
||||
|
||||
def test_build_results_threads_expected_na(tmp_path):
|
||||
# Mirrors custom-html-tiny post-change: install + a passing functional (custom) test, but no
|
||||
# backup surface (backup_restore declared intentionally N/A).
|
||||
# backup surface (backup_restore declared intentionally skipped).
|
||||
recs = [
|
||||
{
|
||||
"tier": "install",
|
||||
@ -347,23 +322,27 @@ def test_build_results_threads_expected_na(tmp_path):
|
||||
ref=None,
|
||||
records=recs,
|
||||
results=_results(backup="skip", restore="skip"), # custom=pass (default) → functional pass
|
||||
backup_capable=False, # no backupbot label → backup_restore N/A
|
||||
backup_capable=False, # no backupbot label → backup_restore skipped (N/A)
|
||||
declared=[],
|
||||
deps_ready=True,
|
||||
sso_unverified=False,
|
||||
clean_teardown=True,
|
||||
no_secret_leak=True,
|
||||
finished_ts=0.0,
|
||||
expected_na={"backup_restore": "stateless static file server"},
|
||||
expected_na={
|
||||
"backup_restore": "stateless static file server",
|
||||
"integration": "no SSO surface",
|
||||
"recipe_local": "no upstream tests/",
|
||||
},
|
||||
)
|
||||
# backup_restore N/A still caps at L2 (never inflates) — even though functional passes above it,
|
||||
# the gap caps the climb — but the cap is now annotated intentional rather than flagged.
|
||||
# backup_restore skip still caps at L2 (never inflates) — even though functional passes above it,
|
||||
# the skip caps the climb — but it's the declared (intentional) rung that capped.
|
||||
assert data["level"] == 2
|
||||
assert "L3" in data["level_cap_reason"]
|
||||
assert data["level_cap_intent"] == "intentional · stateless static file server"
|
||||
assert data["na"]["rungs"]["backup_restore"]["intent"] == "declared"
|
||||
assert data["level_cap_rung"] == "backup_restore"
|
||||
assert data["rungs"]["functional"] == "pass"
|
||||
assert data["na"]["gaps"] == [] # functional now covered; backup_restore declared → no gaps
|
||||
assert data["skips"]["intentional"]["backup_restore"] == "stateless static file server"
|
||||
assert data["skips"]["unintentional"] == [] # every skip accounted for → fully clean
|
||||
|
||||
|
||||
def test_write_results_roundtrip(tmp_path):
|
||||
|
||||
Reference in New Issue
Block a user