feat(lvl5): P1 — 5-rung ladder (L5=abra recipe lint) + de-capped level semantics
continuous-integration/drone/push Build is passing
continuous-integration/drone/push Build is passing
level.py: RUNGS += lint; statuses {pass,fail,skip,unver}; compute_level = max passed
rung with all below pass-or-skip (fail/unver block); cap_reason/capped DELETED.
harness/lint.py: lint executor — pristine scratch clone of the per-run tree at the
exact tested ref (mirror-origin + untracked-overlay pollution solved by context, no
rule filtered), PTY via script -qec, 60s hard budget, lint.txt artifact, table-parse
classifier (rc only signals FATA), unver on any non-run (never silent pass).
results.py: derive_rungs classifies every N/A source (structural/declared → skip,
else unver), lint rung + synthetic lint stage + lint block in results.json, schema 2,
cap fields removed. run_recipe_ci.py: lint call before tiers (double-wrapped,
verdict-neutral), badge = level only. card/dashboard: 0-5 ramp, cap line → 'level N
of {4|5}', unverified rows, badge number+colour only, lint.txt servable, old schema-1
artifacts render untouched. Unit suite rewritten: 245 passed on cc-ci venv.
This commit is contained in:
@@ -0,0 +1,188 @@
|
||||
"""Unit tests for the L5 lint executor (harness.lint) — phase lvl5.
|
||||
|
||||
Covers the table parser + classifier against real abra-0.13 output shapes (probed on the CI
|
||||
host 2026-06-11, JOURNAL-lvl5), and run_lint's never-raise / never-silent-pass guarantees via
|
||||
a fake-PATH `script` shim (no real abra needed). Run cold:
|
||||
cc-ci-run -m pytest tests/unit/test_lint.py -q
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
|
||||
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
|
||||
from harness import lint as L # noqa: E402
|
||||
|
||||
# Realistic abra lint table rows (unicode box drawing, ✅/❌ marks), as captured on cc-ci.
|
||||
TABLE_OK = (
|
||||
"┏━━━━━━┳━━━━━━┓\r\n"
|
||||
"│ R001 │ compose config has expected version │ warn │ ✅ │ - │ ensure │\r\n"
|
||||
"│ R015 │ long secret names │ warn │ ❌ │ - │ reduce │\r\n"
|
||||
"│ R008 │ .env.sample provided │ error │ ✅ │ - │ create │\r\n"
|
||||
"│ R014 │ only annotated tags used for recipe version │ error │ ✅ │ - │ retag │\r\n"
|
||||
"┗━━━━━━┻━━━━━━┛\r\n"
|
||||
"WARN secret session_secret is longer than 12 characters\r\n"
|
||||
)
|
||||
|
||||
TABLE_R014_FAIL = (
|
||||
TABLE_OK.replace(
|
||||
"│ R014 │ only annotated tags used for recipe version │ error │ ✅",
|
||||
"│ R014 │ only annotated tags used for recipe version │ error │ ❌",
|
||||
)
|
||||
+ "WARN critical errors present in hedgedoc config\r\n"
|
||||
)
|
||||
|
||||
TABLE_SKIPPED_ERROR = TABLE_OK.replace(
|
||||
"│ R014 │ only annotated tags used for recipe version │ error │ ✅ │ - │",
|
||||
"│ R014 │ only annotated tags used for recipe version │ error │ ❌ │ skipped │",
|
||||
)
|
||||
|
||||
|
||||
# ---- parse_table ----
|
||||
|
||||
|
||||
def test_parse_table_rows_and_marks():
|
||||
rows = L.parse_table(TABLE_OK)
|
||||
by = {r["rule"]: r for r in rows}
|
||||
assert set(by) == {"R001", "R015", "R008", "R014"}
|
||||
assert by["R001"]["severity"] == "warn" and by["R001"]["satisfied"]
|
||||
assert by["R015"]["severity"] == "warn" and not by["R015"]["satisfied"]
|
||||
assert by["R014"]["severity"] == "error" and by["R014"]["satisfied"]
|
||||
assert not any(r["skipped"] for r in rows)
|
||||
|
||||
|
||||
def test_parse_table_strips_ansi():
|
||||
rows = L.parse_table("\x1b[1m" + TABLE_OK + "\x1b[0m")
|
||||
assert len(rows) == 4
|
||||
|
||||
|
||||
def test_parse_table_garbage_is_empty():
|
||||
assert L.parse_table("FATA something exploded\r\n") == []
|
||||
assert L.parse_table("") == []
|
||||
|
||||
|
||||
# ---- classify ----
|
||||
|
||||
|
||||
def test_classify_pass_with_warn_misses_only():
|
||||
# warn-severity ❌ (R015) does NOT fail the rung — only error-severity rules do.
|
||||
assert L.classify(0, TABLE_OK) == ("pass", "", [])
|
||||
|
||||
|
||||
def test_classify_error_rule_fails():
|
||||
status, detail, failed = L.classify(0, TABLE_R014_FAIL)
|
||||
assert status == "fail"
|
||||
assert failed == ["R014"]
|
||||
assert "R014" in detail
|
||||
|
||||
|
||||
def test_classify_skipped_error_rule_does_not_fail_but_sentinel_guards():
|
||||
# a skipped error rule isn't counted as failed by the parser, but abra's own sentinel line
|
||||
# (if present) still forces fail — the classifier never out-greens abra.
|
||||
status, _, failed = L.classify(0, TABLE_SKIPPED_ERROR)
|
||||
assert failed == []
|
||||
assert status == "pass"
|
||||
status2, detail2, _ = L.classify(
|
||||
0, TABLE_SKIPPED_ERROR + "WARN critical errors present in x config\r\n"
|
||||
)
|
||||
assert status2 == "fail"
|
||||
assert "critical errors" in detail2
|
||||
|
||||
|
||||
def test_classify_rc0_without_table_is_unver():
|
||||
# rc=0 but nothing parseable → cannot claim pass.
|
||||
assert L.classify(0, "weird output")[0] == "unver"
|
||||
|
||||
|
||||
def test_classify_content_fata_is_fail():
|
||||
out = "FATA unable to validate recipe: .env.sample for x couldn't be read\r\n"
|
||||
status, detail, _ = L.classify(1, out)
|
||||
assert status == "fail"
|
||||
assert "unable to validate recipe" in detail
|
||||
|
||||
|
||||
def test_classify_environment_fata_is_unver():
|
||||
out = "FATA unable to fetch tags in /x: repository not found: Not found.\r\n"
|
||||
status, detail, _ = L.classify(1, out)
|
||||
assert status == "unver"
|
||||
assert "fetch tags" in detail
|
||||
|
||||
|
||||
def test_classify_did_not_run_is_unver():
|
||||
assert L.classify(None, "")[0] == "unver"
|
||||
|
||||
|
||||
# ---- run_lint: never raises, never silently passes ----
|
||||
|
||||
|
||||
def _mkrecipe(tmp_path):
|
||||
repo = tmp_path / "abra" / "recipes" / "fakerec"
|
||||
repo.mkdir(parents=True)
|
||||
(repo / "compose.yml").write_text("version: '3.8'\n")
|
||||
for cmd in (
|
||||
["git", "init", "-q"],
|
||||
["git", "add", "."],
|
||||
["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "x"],
|
||||
):
|
||||
subprocess.run(cmd, cwd=repo, check=True)
|
||||
return repo
|
||||
|
||||
|
||||
def _shim(tmp_path, body):
|
||||
"""Drop a fake `script` executable on PATH (run_lint invokes `script -qec "abra ..."`)."""
|
||||
bindir = tmp_path / "bin"
|
||||
bindir.mkdir(exist_ok=True)
|
||||
sh = bindir / "script"
|
||||
sh.write_text("#!/bin/sh\n" + body)
|
||||
sh.chmod(sh.stat().st_mode | stat.S_IEXEC)
|
||||
return str(bindir)
|
||||
|
||||
|
||||
def test_run_lint_pass_via_shim(tmp_path, monkeypatch):
|
||||
_mkrecipe(tmp_path)
|
||||
monkeypatch.setenv("ABRA_DIR", str(tmp_path / "abra"))
|
||||
out = TABLE_OK.replace("\r\n", "\\n")
|
||||
monkeypatch.setenv(
|
||||
"PATH", _shim(tmp_path, f'printf "{out}"\nexit 0\n') + os.pathsep + os.environ["PATH"]
|
||||
)
|
||||
res = L.run_lint("fakerec", None, str(tmp_path / "artifacts"))
|
||||
assert res["status"] == "pass"
|
||||
txt = (tmp_path / "artifacts" / "lint.txt").read_text()
|
||||
assert "abra recipe lint -n fakerec" in txt and "R001" in txt
|
||||
|
||||
|
||||
def test_run_lint_fail_via_shim(tmp_path, monkeypatch):
|
||||
_mkrecipe(tmp_path)
|
||||
monkeypatch.setenv("ABRA_DIR", str(tmp_path / "abra"))
|
||||
out = TABLE_R014_FAIL.replace("\r\n", "\\n")
|
||||
monkeypatch.setenv(
|
||||
"PATH", _shim(tmp_path, f'printf "{out}"\nexit 0\n') + os.pathsep + os.environ["PATH"]
|
||||
)
|
||||
res = L.run_lint("fakerec", None, str(tmp_path / "artifacts"))
|
||||
assert res["status"] == "fail"
|
||||
assert res["rules_failed"] == ["R014"]
|
||||
|
||||
|
||||
def test_run_lint_missing_recipe_is_unver_not_raise(tmp_path, monkeypatch):
|
||||
monkeypatch.setenv("ABRA_DIR", str(tmp_path / "abra-none"))
|
||||
res = L.run_lint("no-such-recipe", None, str(tmp_path / "artifacts"))
|
||||
assert res["status"] == "unver"
|
||||
assert res["detail"]
|
||||
# lint.txt still written with the failure context (loud, never silent)
|
||||
assert (tmp_path / "artifacts" / "lint.txt").exists()
|
||||
|
||||
|
||||
def test_run_lint_abra_blowup_is_unver(tmp_path, monkeypatch):
|
||||
_mkrecipe(tmp_path)
|
||||
monkeypatch.setenv("ABRA_DIR", str(tmp_path / "abra"))
|
||||
monkeypatch.setenv(
|
||||
"PATH",
|
||||
_shim(tmp_path, 'echo "FATA inappropriate ioctl for device"\nexit 1\n')
|
||||
+ os.pathsep
|
||||
+ os.environ["PATH"],
|
||||
)
|
||||
res = L.run_lint("fakerec", None, None)
|
||||
assert res["status"] == "unver"
|
||||
Reference in New Issue
Block a user