52e5d210d8
harness/results.py: JUnit-XML parsing (stdlib) → per-stage/per-test rows; derive_rungs (documented
tier+deps/SSO → rung mapping); build_results assembles results.json {recipe,version,pr,ref,run_id,
stages[],level,level_cap_reason,rungs,flags{clean_teardown,no_secret_leak},screenshot,summary_card};
write_results (atomic). run_recipe_ci.py: tiers emit --junitxml + append {tier,source,file,rc,junit}
records; main() assembles+writes results.json wrapped so a failure NEVER changes the verdict (R7),
incl. a narrow leak-scan of the serialised artifact. 17 new unit tests (test_results.py).
Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
146 lines
4.3 KiB
Python
146 lines
4.3 KiB
Python
"""Unit tests for the Phase-3 level ladder (harness.level), plan-phase3-results-ux.md §4.1 / R1.
|
|
|
|
Pure function — no I/O. Proves the YunoHost gap-caps-the-level semantics, including the U0 gate
|
|
acceptance: a recipe that climbs through L4 reports 4, and one that fails at L2 is capped at 1
|
|
(the level just below the failed rung). Run cold with: cc-ci-run -m pytest tests/unit/test_level.py -q
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
|
|
from harness import level as L # noqa: E402
|
|
|
|
|
|
def _rungs(
|
|
install="pass",
|
|
upgrade="pass",
|
|
backup_restore="pass",
|
|
functional="pass",
|
|
integration="pass",
|
|
recipe_local="pass",
|
|
):
|
|
return {
|
|
"install": install,
|
|
"upgrade": upgrade,
|
|
"backup_restore": backup_restore,
|
|
"functional": functional,
|
|
"integration": integration,
|
|
"recipe_local": recipe_local,
|
|
}
|
|
|
|
|
|
# ---- the U0 gate: L4-pass and L2-cap ----
|
|
|
|
|
|
def test_full_clean_climb_to_L6():
|
|
lvl, reason = L.compute_level(_rungs())
|
|
assert lvl == 6
|
|
assert reason == ""
|
|
|
|
|
|
def test_climbs_through_L4_then_no_integration_surface_caps_at_L4():
|
|
# GATE: a recipe whose functional tests pass but has no SSO/integration surface caps at L4.
|
|
lvl, reason = L.compute_level(_rungs(integration="na", recipe_local="na"))
|
|
assert lvl == 4
|
|
assert "L5" in reason and "N/A" in reason
|
|
|
|
|
|
def test_fails_at_L2_capped_at_L1():
|
|
# GATE: upgrade fails → capped at L1 even though higher rungs would pass.
|
|
lvl, reason = L.compute_level(_rungs(upgrade="fail", backup_restore="pass", functional="pass"))
|
|
assert lvl == 1
|
|
assert "L2" in reason and "FAILED" in reason
|
|
|
|
|
|
# ---- L0 / install ----
|
|
|
|
|
|
def test_install_fail_is_L0():
|
|
lvl, reason = L.compute_level(_rungs(install="fail"))
|
|
assert lvl == 0
|
|
assert "L1" in reason and "FAILED" in reason
|
|
|
|
|
|
# ---- gap-caps semantics: a higher pass can't rescue a lower gap ----
|
|
|
|
|
|
def test_higher_pass_does_not_rescue_lower_na():
|
|
# backup/restore N/A (stateless app) caps at L2 even though functional would pass.
|
|
lvl, reason = L.compute_level(_rungs(backup_restore="na", functional="pass", integration="na"))
|
|
assert lvl == 2
|
|
assert "L3" in reason and "N/A" in reason
|
|
|
|
|
|
def test_upgrade_na_caps_at_L1():
|
|
# only one published version → no upgrade possible → N/A caps at L1.
|
|
lvl, reason = L.compute_level(_rungs(upgrade="na"))
|
|
assert lvl == 1
|
|
assert "L2" in reason and "N/A" in reason
|
|
|
|
|
|
def test_integration_fail_caps_at_L4():
|
|
# SSO declared but unverified (failed) → integration rung fails → cap at L4.
|
|
lvl, reason = L.compute_level(_rungs(integration="fail", recipe_local="na"))
|
|
assert lvl == 4
|
|
assert "L5" in reason and "FAILED" in reason
|
|
|
|
|
|
def test_recipe_local_na_caps_at_L5():
|
|
# SSO passes but no recipe-local tests → cap at L5 (L6 N/A).
|
|
lvl, reason = L.compute_level(_rungs(recipe_local="na"))
|
|
assert lvl == 5
|
|
assert "L6" in reason and "N/A" in reason
|
|
|
|
|
|
def test_functional_fail_caps_at_L3():
|
|
lvl, reason = L.compute_level(_rungs(functional="fail", integration="na"))
|
|
assert lvl == 3
|
|
assert "L4" in reason and "FAILED" in reason
|
|
|
|
|
|
# ---- input validation ----
|
|
|
|
|
|
def test_invalid_status_raises():
|
|
bad = _rungs()
|
|
bad["functional"] = "passed" # not in the vocabulary
|
|
try:
|
|
L.compute_level(bad)
|
|
except ValueError:
|
|
return
|
|
raise AssertionError("expected ValueError on invalid rung status")
|
|
|
|
|
|
# ---- helpers: backup_restore_status ----
|
|
|
|
|
|
def test_backup_restore_status_pass():
|
|
assert L.backup_restore_status("pass", "pass", True) == "pass"
|
|
|
|
|
|
def test_backup_restore_status_not_capable_is_na():
|
|
assert L.backup_restore_status("skip", "skip", False) == "na"
|
|
|
|
|
|
def test_backup_restore_status_fail_on_either():
|
|
assert L.backup_restore_status("pass", "fail", True) == "fail"
|
|
assert L.backup_restore_status("fail", "pass", True) == "fail"
|
|
|
|
|
|
def test_backup_restore_partial_is_na():
|
|
# backup-capable but restore didn't run cleanly (not pass, not fail) → cannot claim L3
|
|
assert L.backup_restore_status("pass", "skip", True) == "na"
|
|
|
|
|
|
# ---- helpers: tier_to_rung ----
|
|
|
|
|
|
def test_tier_to_rung_mapping():
|
|
assert L.tier_to_rung("pass") == "pass"
|
|
assert L.tier_to_rung("fail") == "fail"
|
|
assert L.tier_to_rung("skip") == "na"
|
|
assert L.tier_to_rung(None) == "na"
|