b3ab68a9dd
Some checks failed
continuous-integration/drone/push Build is failing
Per operator: drop the gap-sensitivity / cap-intent-clause / stale-detection
machinery. Model is now dead simple — recipe_meta.EXPECTED_NA = {rung: reason}
lists the rungs a recipe intentionally skips; ANY rung skipped (N/A) and not in
that list is unintentional.
results.json: replace the 'na' block + level_cap_intent with
skips: { intentional: {rung: reason}, unintentional: [rung] }
plus level_cap_rung (which rung capped). Badge/card derive intentional-vs-
unintentional from whether the capping rung is in the intentional list. Skips
still cap the level (never inflate). custom-html-tiny lists all three rungs it
intentionally skips (backup_restore, integration, recipe_local).
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
354 lines
11 KiB
Python
354 lines
11 KiB
Python
"""Unit tests for Phase-3 results assembly (harness.results), plan-phase3-results-ux.md §4.2 / R1/R3.
|
|
|
|
Covers JUnit parsing, stage roll-up, the tier→rung derivation (the documented mapping the level
|
|
depends on), and full results.json assembly incl. the U0 gate cases. Pure / tmp-file only. Run cold:
|
|
cc-ci-run -m pytest tests/unit/test_results.py -q
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
|
|
from harness import results as R # noqa: E402
|
|
|
|
JUNIT_PASS = """<?xml version="1.0"?>
|
|
<testsuites><testsuite name="pytest" tests="2">
|
|
<testcase classname="tests.x" name="test_a" time="0.012"/>
|
|
<testcase classname="tests.x" name="test_b" time="1.5"/>
|
|
</testsuite></testsuites>"""
|
|
|
|
JUNIT_MIXED = """<?xml version="1.0"?>
|
|
<testsuites><testsuite name="pytest" tests="3">
|
|
<testcase classname="tests.y" name="test_ok" time="0.1"/>
|
|
<testcase classname="tests.y" name="test_bad" time="0.2"><failure message="boom">trace</failure></testcase>
|
|
<testcase classname="tests.y" name="test_skipped" time="0"><skipped message="no deps"/></testcase>
|
|
</testsuite></testsuites>"""
|
|
|
|
|
|
def _write(tmp_path, name, content):
|
|
p = tmp_path / name
|
|
p.write_text(content)
|
|
return str(p)
|
|
|
|
|
|
def test_parse_junit_pass(tmp_path):
|
|
rows = R.parse_junit(_write(tmp_path, "p.xml", JUNIT_PASS))
|
|
assert len(rows) == 2
|
|
assert {r["status"] for r in rows} == {"pass"}
|
|
assert rows[1]["ms"] == 1500
|
|
|
|
|
|
def test_parse_junit_mixed(tmp_path):
|
|
rows = R.parse_junit(_write(tmp_path, "m.xml", JUNIT_MIXED))
|
|
by = {r["name"]: r["status"] for r in rows}
|
|
assert by == {"test_ok": "pass", "test_bad": "fail", "test_skipped": "skip"}
|
|
|
|
|
|
def test_parse_junit_missing_file_is_empty():
|
|
assert R.parse_junit("/nonexistent/x.xml") == []
|
|
|
|
|
|
def test_collect_stages_orders_and_rolls_up(tmp_path):
|
|
recs = [
|
|
{
|
|
"tier": "install",
|
|
"source": "generic",
|
|
"file": "g/test_install.py",
|
|
"rc": 0,
|
|
"junit": _write(tmp_path, "i.xml", JUNIT_PASS),
|
|
},
|
|
{
|
|
"tier": "custom",
|
|
"source": "cc-ci",
|
|
"file": "c/test_x.py",
|
|
"rc": 1,
|
|
"junit": _write(tmp_path, "c.xml", JUNIT_MIXED),
|
|
},
|
|
]
|
|
stages = R.collect_stages(recs)
|
|
assert [s["name"] for s in stages] == ["install", "custom"] # install before custom
|
|
assert stages[0]["status"] == "pass"
|
|
assert stages[1]["status"] == "fail" # the failure in JUNIT_MIXED
|
|
assert len(stages[1]["tests"]) == 3
|
|
|
|
|
|
def test_collect_stages_synthesizes_when_no_junit():
|
|
recs = [
|
|
{
|
|
"tier": "install",
|
|
"source": "generic",
|
|
"file": "g/test_install.py",
|
|
"rc": 1,
|
|
"junit": None,
|
|
}
|
|
]
|
|
stages = R.collect_stages(recs)
|
|
assert stages[0]["status"] == "fail"
|
|
assert len(stages[0]["tests"]) == 1
|
|
|
|
|
|
# ---- derive_rungs: the documented mapping ----
|
|
|
|
|
|
def _results(**kw):
|
|
base = {
|
|
"install": "pass",
|
|
"upgrade": "pass",
|
|
"backup": "pass",
|
|
"restore": "pass",
|
|
"custom": "pass",
|
|
}
|
|
base.update(kw)
|
|
return base
|
|
|
|
|
|
def test_derive_rungs_full_stateful_sso():
|
|
rungs = R.derive_rungs(
|
|
_results(),
|
|
backup_capable=True,
|
|
declared=["keycloak"],
|
|
deps_ready=True,
|
|
sso_unverified=False,
|
|
has_custom=True,
|
|
has_repo_local=False,
|
|
repo_local_passed=False,
|
|
)
|
|
assert rungs == {
|
|
"install": "pass",
|
|
"upgrade": "pass",
|
|
"backup_restore": "pass",
|
|
"functional": "pass",
|
|
"integration": "pass",
|
|
"recipe_local": "na",
|
|
}
|
|
|
|
|
|
def test_derive_rungs_no_sso_surface_is_integration_na():
|
|
rungs = R.derive_rungs(
|
|
_results(),
|
|
backup_capable=True,
|
|
declared=[],
|
|
deps_ready=True,
|
|
sso_unverified=False,
|
|
has_custom=True,
|
|
has_repo_local=False,
|
|
repo_local_passed=False,
|
|
)
|
|
assert rungs["integration"] == "na"
|
|
assert rungs["functional"] == "pass"
|
|
|
|
|
|
def test_derive_rungs_stateless_backup_na():
|
|
rungs = R.derive_rungs(
|
|
_results(backup="skip", restore="skip", custom="skip"),
|
|
backup_capable=False,
|
|
declared=[],
|
|
deps_ready=True,
|
|
sso_unverified=False,
|
|
has_custom=False,
|
|
has_repo_local=False,
|
|
repo_local_passed=False,
|
|
)
|
|
assert rungs["backup_restore"] == "na"
|
|
assert rungs["functional"] == "na"
|
|
|
|
|
|
def test_derive_rungs_sso_unverified_is_integration_fail():
|
|
rungs = R.derive_rungs(
|
|
_results(),
|
|
backup_capable=True,
|
|
declared=["keycloak"],
|
|
deps_ready=False,
|
|
sso_unverified=True,
|
|
has_custom=True,
|
|
has_repo_local=False,
|
|
repo_local_passed=False,
|
|
)
|
|
assert rungs["integration"] == "fail"
|
|
|
|
|
|
def test_derive_rungs_repo_local_pass():
|
|
rungs = R.derive_rungs(
|
|
_results(),
|
|
backup_capable=True,
|
|
declared=[],
|
|
deps_ready=True,
|
|
sso_unverified=False,
|
|
has_custom=True,
|
|
has_repo_local=True,
|
|
repo_local_passed=True,
|
|
)
|
|
assert rungs["recipe_local"] == "pass"
|
|
|
|
|
|
# ---- build_results: end-to-end incl level + flags ----
|
|
|
|
|
|
def test_build_results_level_and_flags(tmp_path):
|
|
recs = [
|
|
{
|
|
"tier": "install",
|
|
"source": "generic",
|
|
"file": "g/test_install.py",
|
|
"rc": 0,
|
|
"junit": _write(tmp_path, "i.xml", JUNIT_PASS),
|
|
},
|
|
{
|
|
"tier": "custom",
|
|
"source": "cc-ci",
|
|
"file": "c/test_func.py",
|
|
"rc": 0,
|
|
"junit": _write(tmp_path, "c.xml", JUNIT_PASS),
|
|
},
|
|
]
|
|
data = R.build_results(
|
|
recipe="hedgedoc",
|
|
version="1.2.3",
|
|
pr="7",
|
|
ref="deadbeefcafe0000",
|
|
records=recs,
|
|
results=_results(),
|
|
backup_capable=True,
|
|
declared=[],
|
|
deps_ready=True,
|
|
sso_unverified=False,
|
|
clean_teardown=True,
|
|
no_secret_leak=True,
|
|
finished_ts=1234.0,
|
|
)
|
|
# stateful, functional pass, no SSO surface, no repo-local → caps at L4
|
|
assert data["level"] == 4
|
|
assert "L5" in data["level_cap_reason"]
|
|
assert data["recipe"] == "hedgedoc"
|
|
assert data["ref"] == "deadbeefcafe"
|
|
assert data["flags"] == {"clean_teardown": True, "no_secret_leak": True}
|
|
assert [s["name"] for s in data["stages"]] == ["install", "custom"]
|
|
|
|
|
|
def test_build_results_capped_at_L1_on_upgrade_fail(tmp_path):
|
|
recs = [
|
|
{
|
|
"tier": "install",
|
|
"source": "generic",
|
|
"file": "g/test_install.py",
|
|
"rc": 0,
|
|
"junit": _write(tmp_path, "i.xml", JUNIT_PASS),
|
|
}
|
|
]
|
|
data = R.build_results(
|
|
recipe="x",
|
|
version=None,
|
|
pr="0",
|
|
ref=None,
|
|
records=recs,
|
|
results=_results(upgrade="fail"),
|
|
backup_capable=True,
|
|
declared=[],
|
|
deps_ready=True,
|
|
sso_unverified=False,
|
|
clean_teardown=True,
|
|
no_secret_leak=True,
|
|
finished_ts=0.0,
|
|
)
|
|
assert data["level"] == 1
|
|
assert "L2" in data["level_cap_reason"]
|
|
|
|
|
|
# ---- skips: intentional (declared) vs unintentional (everything else skipped) ----
|
|
|
|
|
|
def _rungs(**kw):
|
|
base = {
|
|
"install": "pass",
|
|
"upgrade": "pass",
|
|
"backup_restore": "pass",
|
|
"functional": "pass",
|
|
"integration": "na",
|
|
"recipe_local": "na",
|
|
}
|
|
base.update(kw)
|
|
return base
|
|
|
|
|
|
def test_skips_intentional_vs_unintentional():
|
|
rungs = _rungs(backup_restore="na", functional="na")
|
|
sk = R.skips(rungs, {"backup_restore": "stateless static server"})
|
|
# backup_restore is declared (intentional, with reason); everything else skipped is unintentional.
|
|
assert sk["intentional"] == {"backup_restore": "stateless static server"}
|
|
assert sk["unintentional"] == ["functional", "integration", "recipe_local"]
|
|
|
|
|
|
def test_skips_none_declared_all_unintentional():
|
|
rungs = _rungs(backup_restore="na")
|
|
sk = R.skips(rungs, None)
|
|
assert sk["intentional"] == {}
|
|
assert sk["unintentional"] == ["backup_restore", "integration", "recipe_local"]
|
|
|
|
|
|
def test_skips_declaration_only_counts_when_actually_skipped():
|
|
# backup_restore actually ran (pass) → not a skip, so a declaration for it is simply inert.
|
|
rungs = _rungs(backup_restore="pass")
|
|
sk = R.skips(rungs, {"backup_restore": "reason"})
|
|
assert "backup_restore" not in sk["intentional"]
|
|
assert "backup_restore" not in sk["unintentional"]
|
|
|
|
|
|
def test_build_results_threads_expected_na(tmp_path):
|
|
# Mirrors custom-html-tiny post-change: install + a passing functional (custom) test, but no
|
|
# backup surface (backup_restore declared intentionally skipped).
|
|
recs = [
|
|
{
|
|
"tier": "install",
|
|
"source": "generic",
|
|
"file": "g/test_install.py",
|
|
"rc": 0,
|
|
"junit": _write(tmp_path, "i.xml", JUNIT_PASS),
|
|
},
|
|
{
|
|
"tier": "custom",
|
|
"source": "cc-ci",
|
|
"file": "c/test_serves_content.py",
|
|
"rc": 0,
|
|
"junit": _write(tmp_path, "c.xml", JUNIT_PASS),
|
|
},
|
|
]
|
|
data = R.build_results(
|
|
recipe="custom-html-tiny",
|
|
version="1.1.0",
|
|
pr="0",
|
|
ref=None,
|
|
records=recs,
|
|
results=_results(backup="skip", restore="skip"), # custom=pass (default) → functional pass
|
|
backup_capable=False, # no backupbot label → backup_restore skipped (N/A)
|
|
declared=[],
|
|
deps_ready=True,
|
|
sso_unverified=False,
|
|
clean_teardown=True,
|
|
no_secret_leak=True,
|
|
finished_ts=0.0,
|
|
expected_na={
|
|
"backup_restore": "stateless static file server",
|
|
"integration": "no SSO surface",
|
|
"recipe_local": "no upstream tests/",
|
|
},
|
|
)
|
|
# backup_restore skip still caps at L2 (never inflates) — even though functional passes above it,
|
|
# the skip caps the climb — but it's the declared (intentional) rung that capped.
|
|
assert data["level"] == 2
|
|
assert "L3" in data["level_cap_reason"]
|
|
assert data["level_cap_rung"] == "backup_restore"
|
|
assert data["rungs"]["functional"] == "pass"
|
|
assert data["skips"]["intentional"]["backup_restore"] == "stateless static file server"
|
|
assert data["skips"]["unintentional"] == [] # every skip accounted for → fully clean
|
|
|
|
|
|
def test_write_results_roundtrip(tmp_path):
|
|
data = {"run_id": "42", "level": 3, "stages": []}
|
|
path = R.write_results(data, runs_dir_override=str(tmp_path))
|
|
assert path.endswith("/42/results.json")
|
|
with open(path) as f:
|
|
assert json.load(f)["level"] == 3
|