232 lines
9.0 KiB
Python
232 lines
9.0 KiB
Python
"""Unit tests for the L5 lint executor (harness.lint) — phase lvl5.
|
|
|
|
Covers the table parser + classifier against real abra-0.13 output shapes (probed on the CI
|
|
host 2026-06-11, JOURNAL-lvl5), and run_lint's never-raise / never-silent-pass guarantees via
|
|
a fake-PATH `script` shim (no real abra needed). Run cold:
|
|
cc-ci-run -m pytest tests/unit/test_lint.py -q
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import stat
|
|
import subprocess
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
|
|
from harness import lint as L # noqa: E402
|
|
|
|
# Realistic abra lint table rows, as captured on cc-ci: abra renders HEAVY box-drawing
|
|
# verticals (┃ U+2503) — the parser must match those, not just the light │.
|
|
TABLE_OK = (
|
|
"┏━━━━━━┳━━━━━━┓\r\n"
|
|
"┃ R001 ┃ compose config has expected version ┃ warn ┃ ✅ ┃ - ┃ ensure ┃\r\n"
|
|
"┃ R015 ┃ long secret names ┃ warn ┃ ❌ ┃ - ┃ reduce ┃\r\n"
|
|
"┃ R008 ┃ .env.sample provided ┃ error ┃ ✅ ┃ - ┃ create ┃\r\n"
|
|
"┃ R014 ┃ only annotated tags used for recipe version ┃ error ┃ ✅ ┃ - ┃ retag ┃\r\n"
|
|
"┗━━━━━━┻━━━━━━┛\r\n"
|
|
"WARN secret session_secret is longer than 12 characters\r\n"
|
|
)
|
|
|
|
# The light-vertical variant must parse identically (defensive: abra theme/version drift).
|
|
TABLE_OK_LIGHT = TABLE_OK.replace("┃", "│")
|
|
|
|
TABLE_R014_FAIL = (
|
|
TABLE_OK.replace(
|
|
"┃ R014 ┃ only annotated tags used for recipe version ┃ error ┃ ✅",
|
|
"┃ R014 ┃ only annotated tags used for recipe version ┃ error ┃ ❌",
|
|
)
|
|
+ "WARN critical errors present in hedgedoc config\r\n"
|
|
)
|
|
|
|
TABLE_SKIPPED_ERROR = TABLE_OK.replace(
|
|
"┃ R014 ┃ only annotated tags used for recipe version ┃ error ┃ ✅ ┃ - ┃",
|
|
"┃ R014 ┃ only annotated tags used for recipe version ┃ error ┃ ❌ ┃ skipped ┃",
|
|
)
|
|
|
|
|
|
# ---- parse_table ----
|
|
|
|
|
|
def test_parse_table_rows_and_marks():
|
|
rows = L.parse_table(TABLE_OK)
|
|
by = {r["rule"]: r for r in rows}
|
|
assert set(by) == {"R001", "R015", "R008", "R014"}
|
|
assert by["R001"]["severity"] == "warn" and by["R001"]["satisfied"]
|
|
assert by["R015"]["severity"] == "warn" and not by["R015"]["satisfied"]
|
|
assert by["R014"]["severity"] == "error" and by["R014"]["satisfied"]
|
|
assert not any(r["skipped"] for r in rows)
|
|
|
|
|
|
def test_parse_table_strips_ansi():
|
|
rows = L.parse_table("\x1b[1m" + TABLE_OK + "\x1b[0m")
|
|
assert len(rows) == 4
|
|
|
|
|
|
def test_parse_table_light_verticals_too():
|
|
assert L.parse_table(TABLE_OK_LIGHT) == L.parse_table(TABLE_OK)
|
|
|
|
|
|
def test_parse_table_garbage_is_empty():
|
|
assert L.parse_table("FATA something exploded\r\n") == []
|
|
assert L.parse_table("") == []
|
|
|
|
|
|
# ---- classify ----
|
|
|
|
|
|
def test_classify_pass_with_warn_misses_only():
|
|
# warn-severity ❌ (R015) does NOT fail the rung — only error-severity rules do.
|
|
assert L.classify(0, TABLE_OK) == ("pass", "", [])
|
|
|
|
|
|
def test_classify_error_rule_fails():
|
|
status, detail, failed = L.classify(0, TABLE_R014_FAIL)
|
|
assert status == "fail"
|
|
assert failed == ["R014"]
|
|
assert "R014" in detail
|
|
|
|
|
|
def test_classify_skipped_error_rule_does_not_fail_but_sentinel_guards():
|
|
# a skipped error rule isn't counted as failed by the parser, but abra's own sentinel line
|
|
# (if present) still forces fail — the classifier never out-greens abra.
|
|
status, _, failed = L.classify(0, TABLE_SKIPPED_ERROR)
|
|
assert failed == []
|
|
assert status == "pass"
|
|
status2, detail2, _ = L.classify(
|
|
0, TABLE_SKIPPED_ERROR + "WARN critical errors present in x config\r\n"
|
|
)
|
|
assert status2 == "fail"
|
|
assert "critical errors" in detail2
|
|
|
|
|
|
def test_classify_rc0_without_table_is_unver():
|
|
# rc=0 but nothing parseable → cannot claim pass.
|
|
assert L.classify(0, "weird output")[0] == "unver"
|
|
|
|
|
|
def test_classify_content_fata_is_fail():
|
|
out = "FATA unable to validate recipe: .env.sample for x couldn't be read\r\n"
|
|
status, detail, _ = L.classify(1, out)
|
|
assert status == "fail"
|
|
assert "unable to validate recipe" in detail
|
|
|
|
|
|
def test_classify_environment_fata_is_unver():
|
|
out = "FATA unable to fetch tags in /x: repository not found: Not found.\r\n"
|
|
status, detail, _ = L.classify(1, out)
|
|
assert status == "unver"
|
|
assert "fetch tags" in detail
|
|
|
|
|
|
def test_classify_did_not_run_is_unver():
|
|
assert L.classify(None, "")[0] == "unver"
|
|
|
|
|
|
# ---- run_lint: never raises, never silently passes ----
|
|
|
|
|
|
def _mkrecipe(tmp_path):
|
|
repo = tmp_path / "abra" / "recipes" / "fakerec"
|
|
repo.mkdir(parents=True)
|
|
(repo / "compose.yml").write_text("version: '3.8'\n")
|
|
for cmd in (
|
|
["git", "init", "-q"],
|
|
["git", "add", "."],
|
|
["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "x"],
|
|
):
|
|
subprocess.run(cmd, cwd=repo, check=True)
|
|
return repo
|
|
|
|
|
|
def _shim(tmp_path, body):
|
|
"""Drop a fake `script` executable on PATH (run_lint invokes `script -qec "abra ..."`)."""
|
|
bindir = tmp_path / "bin"
|
|
bindir.mkdir(exist_ok=True)
|
|
sh = bindir / "script"
|
|
sh.write_text("#!/bin/sh\n" + body)
|
|
sh.chmod(sh.stat().st_mode | stat.S_IEXEC)
|
|
return str(bindir)
|
|
|
|
|
|
def test_run_lint_pass_via_shim(tmp_path, monkeypatch):
|
|
_mkrecipe(tmp_path)
|
|
monkeypatch.setenv("ABRA_DIR", str(tmp_path / "abra"))
|
|
out = TABLE_OK.replace("\r\n", "\\n")
|
|
monkeypatch.setenv(
|
|
"PATH", _shim(tmp_path, f'printf "{out}"\nexit 0\n') + os.pathsep + os.environ["PATH"]
|
|
)
|
|
res = L.run_lint("fakerec", None, str(tmp_path / "artifacts"))
|
|
assert res["status"] == "pass"
|
|
txt = (tmp_path / "artifacts" / "lint.txt").read_text()
|
|
assert "abra recipe lint -n fakerec" in txt and "R001" in txt
|
|
|
|
|
|
def test_run_lint_fail_via_shim(tmp_path, monkeypatch):
|
|
_mkrecipe(tmp_path)
|
|
monkeypatch.setenv("ABRA_DIR", str(tmp_path / "abra"))
|
|
out = TABLE_R014_FAIL.replace("\r\n", "\\n")
|
|
monkeypatch.setenv(
|
|
"PATH", _shim(tmp_path, f'printf "{out}"\nexit 0\n') + os.pathsep + os.environ["PATH"]
|
|
)
|
|
res = L.run_lint("fakerec", None, str(tmp_path / "artifacts"))
|
|
assert res["status"] == "fail"
|
|
assert res["rules_failed"] == ["R014"]
|
|
|
|
|
|
def test_run_lint_missing_recipe_is_unver_not_raise(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("ABRA_DIR", str(tmp_path / "abra-none"))
|
|
res = L.run_lint("no-such-recipe", None, str(tmp_path / "artifacts"))
|
|
assert res["status"] == "unver"
|
|
assert res["detail"]
|
|
# lint.txt still written with the failure context (loud, never silent)
|
|
assert (tmp_path / "artifacts" / "lint.txt").exists()
|
|
|
|
|
|
def test_run_lint_detached_pr_tree_lints_exact_ref(tmp_path, monkeypatch):
|
|
# PR-path regression (live builds 400-402): the per-run tree sits at a DETACHED HEAD (the PR
|
|
# sha), and abra lint selects+checks out the repo's DEFAULT BRANCH before linting. run_lint
|
|
# must (a) not FATA on the branchless clone, and (b) make the default branch BE the tested
|
|
# ref — never some other commit's content. Source repo: branch main at C1, detached at C2
|
|
# (the "PR head", which adds marker-c2). The shim passes only if the linted tree contains
|
|
# marker-c2 AND HEAD is a local branch.
|
|
repo = _mkrecipe(tmp_path)
|
|
(repo / "marker-c2.txt").write_text("pr head\n")
|
|
subprocess.run(["git", "add", "."], cwd=repo, check=True)
|
|
subprocess.run(
|
|
["git", "-c", "user.email=t@t", "-c", "user.name=t", "commit", "-qm", "c2"],
|
|
cwd=repo,
|
|
check=True,
|
|
)
|
|
c2 = subprocess.run(
|
|
["git", "rev-parse", "HEAD"], cwd=repo, check=True, capture_output=True, text=True
|
|
).stdout.strip()
|
|
subprocess.run(["git", "branch", "-f", "main", "HEAD^"], cwd=repo, check=True)
|
|
subprocess.run(["git", "checkout", "-q", c2], cwd=repo, check=True) # detached, like a PR run
|
|
monkeypatch.setenv("ABRA_DIR", str(tmp_path / "abra"))
|
|
out = TABLE_OK.replace("\r\n", "\\n")
|
|
shim = (
|
|
'C="$ABRA_DIR/recipes/fakerec"\n'
|
|
'git -C "$C" symbolic-ref HEAD >&2 || { echo "FATA no default branch" >&2; exit 1; }\n'
|
|
'[ -f "$C/marker-c2.txt" ] || { echo "FATA wrong ref linted" >&2; exit 1; }\n'
|
|
f'printf "{out}"\nexit 0\n'
|
|
)
|
|
monkeypatch.setenv("PATH", _shim(tmp_path, shim) + os.pathsep + os.environ["PATH"])
|
|
res = L.run_lint("fakerec", c2, str(tmp_path / "artifacts"))
|
|
assert res == {"status": "pass", "detail": "", "rules_failed": []}
|
|
txt = (tmp_path / "artifacts" / "lint.txt").read_text()
|
|
assert "refs/heads/main" in txt # HEAD was a local default branch when abra ran
|
|
|
|
|
|
def test_run_lint_abra_blowup_is_unver(tmp_path, monkeypatch):
|
|
_mkrecipe(tmp_path)
|
|
monkeypatch.setenv("ABRA_DIR", str(tmp_path / "abra"))
|
|
monkeypatch.setenv(
|
|
"PATH",
|
|
_shim(tmp_path, 'echo "FATA inappropriate ioctl for device"\nexit 1\n')
|
|
+ os.pathsep
|
|
+ os.environ["PATH"],
|
|
)
|
|
res = L.run_lint("fakerec", None, None)
|
|
assert res["status"] == "unver"
|