recipe-maintainers/cc-ci
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

refactor(level): four essential rungs only — integration & recipe-local are optional
Some checks failed
continuous-integration/drone/push Build is failing
Details

Browse Source 
Per operator: the level ladder is now the FOUR essential rungs every recipe is
held to — install, upgrade (essential), backup/restore, functional (top = L4).
Integration (SSO/OIDC) and recipe-local are OPTIONAL capabilities: they no longer
appear as level rungs or skip rows and never cap the level. SSO is still enforced
for the run VERDICT (unchanged in run_recipe_ci.py); it just doesn't affect the
level. derive_rungs simplified accordingly (drops declared/deps/sso/repo-local
inputs). custom-html-tiny's EXPECTED_NA is back to just backup_restore.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
autonomic-bot
2026-06-09 02:55:47 +00:00
parent 3980340727
commit 46e2cdb93e
9 changed files with 63 additions and 204 deletions
Download Patch File Download Diff File Expand all files Collapse all files

13
tests/custom-html-tiny/recipe_meta.py
View File

@ -4,14 +4,13 @@
DEPLOY_TIMEOUT = 120
HTTP_TIMEOUT = 90
# Rungs this recipe INTENTIONALLY skips, each with a reason. Any rung that is skipped (N/A) and is
# NOT listed here is reported as an *unintentional* skip (a coverage gap to fill or declare). A skip
# still caps the level either way — the harness never claims a rung it did not verify; this only
# records that the skip is deliberate. custom-html-tiny is a stateless static-web-server, so:
# Rungs this recipe INTENTIONALLY skips, each with a reason. Any essential rung skipped (N/A) and NOT
# listed here is reported as an *unintentional* skip (a coverage gap to fill or declare). A skip still
# caps the level either way — the harness never claims a rung it did not verify; this only records
# that the skip is deliberate. (The level ladder is the four essential rungs install/upgrade/
# backup_restore/functional; integration + recipe-local are optional and not leveled.)
# custom-html-tiny is a stateless static-web-server, so it has no backup surface:
EXPECTED_NA = {
"backup_restore": "stateless static file server: serves an ephemeral content volume seeded at "
"deploy, with no persistent/user data to back up or restore (no backupbot.backup label)",
"integration": "no SSO/OIDC or cross-app surface — a static file server has no auth integration",
"recipe_local": "the upstream recipe ships no tests/ of its own; coverage is the cc-ci generic "
"install tier + the functional serve test",
}

2
tests/unit/test_card.py
View File

@ -14,7 +14,7 @@ sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner")
from harness import card as C # noqa: E402
def _data(level=4, cap="L5 integration (SSO/OIDC + cross-app) N/A"):
def _data(level=3, cap="L4 functional (recipe-specific tests) N/A"):
return {
"recipe": "uptime-kuma",
"version": "1.23.0",

2
tests/unit/test_dashboard.py
View File

@ -24,7 +24,7 @@ import dashboard # noqa: E402
def _row(**kw):
base = {
"recipe": "custom-html", "status": "success", "number": 4, "ref": "db9a9502",
"version": "db9a95024e9d", "level": 4, "level_cap_reason": "L5 integration N/A",
"version": "db9a95024e9d", "level": 4, "level_cap_reason": "",
"has_screenshot": True, "flags": {"clean_teardown": True, "no_secret_leak": True},
"finished": 0, "url": "https://drone.x/cc-ci/4",
}

41
tests/unit/test_level.py
View File

@ -19,33 +19,23 @@ def _rungs(
upgrade="pass",
backup_restore="pass",
functional="pass",
integration="pass",
recipe_local="pass",
):
return {
"install": install,
"upgrade": upgrade,
"backup_restore": backup_restore,
"functional": functional,
"integration": integration,
"recipe_local": recipe_local,
}
# ---- the U0 gate: L4-pass and L2-cap ----
# ---- the ladder: four essential rungs, top is L4 (functional) ----
def test_full_clean_climb_to_L6():
def test_full_clean_climb_to_L4():
# All four essential rungs pass → L4 (the top; integration/recipe-local are optional, not leveled).
lvl, reason = L.compute_level(_rungs())
assert lvl == 6
assert reason == ""
def test_climbs_through_L4_then_no_integration_surface_caps_at_L4():
# GATE: a recipe whose functional tests pass but has no SSO/integration surface caps at L4.
lvl, reason = L.compute_level(_rungs(integration="na", recipe_local="na"))
assert lvl == 4
assert "L5" in reason and "N/A" in reason
assert reason == ""
def test_fails_at_L2_capped_at_L1():
@ -69,34 +59,27 @@ def test_install_fail_is_L0():
def test_higher_pass_does_not_rescue_lower_na():
# backup/restore N/A (stateless app) caps at L2 even though functional would pass.
lvl, reason = L.compute_level(_rungs(backup_restore="na", functional="pass", integration="na"))
lvl, reason = L.compute_level(_rungs(backup_restore="na", functional="pass"))
assert lvl == 2
assert "L3" in reason and "N/A" in reason
def test_upgrade_na_caps_at_L1():
# only one published version → no upgrade possible → N/A caps at L1.
# only one published version → no upgrade possible → N/A caps at L1 (upgrade is essential).
lvl, reason = L.compute_level(_rungs(upgrade="na"))
assert lvl == 1
assert "L2" in reason and "N/A" in reason
def test_integration_fail_caps_at_L4():
# SSO declared but unverified (failed) → integration rung fails → cap at L4.
lvl, reason = L.compute_level(_rungs(integration="fail", recipe_local="na"))
assert lvl == 4
assert "L5" in reason and "FAILED" in reason
def test_recipe_local_na_caps_at_L5():
# SSO passes but no recipe-local tests → cap at L5 (L6 N/A).
lvl, reason = L.compute_level(_rungs(recipe_local="na"))
assert lvl == 5
assert "L6" in reason and "N/A" in reason
def test_functional_na_caps_at_L3():
# no recipe-specific functional tests → functional N/A caps at L3.
lvl, reason = L.compute_level(_rungs(functional="na"))
assert lvl == 3
assert "L4" in reason and "N/A" in reason
def test_functional_fail_caps_at_L3():
lvl, reason = L.compute_level(_rungs(functional="fail", integration="na"))
lvl, reason = L.compute_level(_rungs(functional="fail"))
assert lvl == 3
assert "L4" in reason and "FAILED" in reason

97
tests/unit/test_results.py
View File

@ -105,83 +105,31 @@ def _results(**kw):
return base
def test_derive_rungs_full_stateful_sso():
rungs = R.derive_rungs(
_results(),
backup_capable=True,
declared=["keycloak"],
deps_ready=True,
sso_unverified=False,
has_custom=True,
has_repo_local=False,
repo_local_passed=False,
)
def test_derive_rungs_full_climb_four_essential():
rungs = R.derive_rungs(_results(), backup_capable=True, has_custom=True)
# only the four essential rungs — integration/recipe-local are optional, not produced here.
assert rungs == {
"install": "pass",
"upgrade": "pass",
"backup_restore": "pass",
"functional": "pass",
"integration": "pass",
"recipe_local": "na",
}
def test_derive_rungs_no_sso_surface_is_integration_na():
rungs = R.derive_rungs(
_results(),
backup_capable=True,
declared=[],
deps_ready=True,
sso_unverified=False,
has_custom=True,
has_repo_local=False,
repo_local_passed=False,
)
assert rungs["integration"] == "na"
assert rungs["functional"] == "pass"
def test_derive_rungs_stateless_backup_na():
def test_derive_rungs_stateless_backup_and_functional_na():
rungs = R.derive_rungs(
_results(backup="skip", restore="skip", custom="skip"),
backup_capable=False,
declared=[],
deps_ready=True,
sso_unverified=False,
has_custom=False,
has_repo_local=False,
repo_local_passed=False,
)
assert rungs["backup_restore"] == "na"
assert rungs["functional"] == "na"
assert "integration" not in rungs and "recipe_local" not in rungs
def test_derive_rungs_sso_unverified_is_integration_fail():
rungs = R.derive_rungs(
_results(),
backup_capable=True,
declared=["keycloak"],
deps_ready=False,
sso_unverified=True,
has_custom=True,
has_repo_local=False,
repo_local_passed=False,
)
assert rungs["integration"] == "fail"
def test_derive_rungs_repo_local_pass():
rungs = R.derive_rungs(
_results(),
backup_capable=True,
declared=[],
deps_ready=True,
sso_unverified=False,
has_custom=True,
has_repo_local=True,
repo_local_passed=True,
)
assert rungs["recipe_local"] == "pass"
def test_derive_rungs_functional_fail():
rungs = R.derive_rungs(_results(custom="fail"), backup_capable=True, has_custom=True)
assert rungs["functional"] == "fail"
# ---- build_results: end-to-end incl level + flags ----
@ -212,16 +160,13 @@ def test_build_results_level_and_flags(tmp_path):
records=recs,
results=_results(),
backup_capable=True,
declared=[],
deps_ready=True,
sso_unverified=False,
clean_teardown=True,
no_secret_leak=True,
finished_ts=1234.0,
)
# stateful, functional pass, no SSO surface, no repo-local → caps at L4
# all four essential rungs pass → full climb to L4 (the top), no cap
assert data["level"] == 4
assert "L5" in data["level_cap_reason"]
assert data["level_cap_reason"] == ""
assert data["recipe"] == "hedgedoc"
assert data["ref"] == "deadbeefcafe"
assert data["flags"] == {"clean_teardown": True, "no_secret_leak": True}
@ -246,9 +191,6 @@ def test_build_results_capped_at_L1_on_upgrade_fail(tmp_path):
records=recs,
results=_results(upgrade="fail"),
backup_capable=True,
declared=[],
deps_ready=True,
sso_unverified=False,
clean_teardown=True,
no_secret_leak=True,
finished_ts=0.0,
@ -266,8 +208,6 @@ def _rungs(**kw):
"upgrade": "pass",
"backup_restore": "pass",
"functional": "pass",
"integration": "na",
"recipe_local": "na",
}
base.update(kw)
return base
@ -276,16 +216,16 @@ def _rungs(**kw):
def test_skips_intentional_vs_unintentional():
rungs = _rungs(backup_restore="na", functional="na")
sk = R.skips(rungs, {"backup_restore": "stateless static server"})
# backup_restore is declared (intentional, with reason); everything else skipped is unintentional.
# backup_restore is declared (intentional, with reason); functional skipped but not declared.
assert sk["intentional"] == {"backup_restore": "stateless static server"}
assert sk["unintentional"] == ["functional", "integration", "recipe_local"]
assert sk["unintentional"] == ["functional"]
def test_skips_none_declared_all_unintentional():
rungs = _rungs(backup_restore="na")
sk = R.skips(rungs, None)
assert sk["intentional"] == {}
assert sk["unintentional"] == ["backup_restore", "integration", "recipe_local"]
assert sk["unintentional"] == ["backup_restore"]
def test_skips_declaration_only_counts_when_actually_skipped():
@ -323,17 +263,10 @@ def test_build_results_threads_expected_na(tmp_path):
records=recs,
results=_results(backup="skip", restore="skip"), # custom=pass (default) → functional pass
backup_capable=False, # no backupbot label → backup_restore skipped (N/A)
declared=[],
deps_ready=True,
sso_unverified=False,
clean_teardown=True,
no_secret_leak=True,
finished_ts=0.0,
expected_na={
"backup_restore": "stateless static file server",
"integration": "no SSO surface",
"recipe_local": "no upstream tests/",
},
expected_na={"backup_restore": "stateless static file server"},
)
# backup_restore skip still caps at L2 (never inflates) — even though functional passes above it,
# the skip caps the climb — but it's the declared (intentional) rung that capped.
@ -342,7 +275,7 @@ def test_build_results_threads_expected_na(tmp_path):
assert data["level_cap_rung"] == "backup_restore"
assert data["rungs"]["functional"] == "pass"
assert data["skips"]["intentional"]["backup_restore"] == "stateless static file server"
assert data["skips"]["unintentional"] == [] # every skip accounted for → fully clean
assert data["skips"]["unintentional"] == [] # backup_restore declared; functional passed → clean
def test_write_results_roundtrip(tmp_path):