fix(2): F2-11 — SSO-dep deps-not-ready SKIP no longer yields GREEN !testme
When a DEPS-declaring recipe's setup_custom_tests fails, its @requires_deps (SSO/OIDC) tests skip; a skip-only pytest file exits 0 so the run previously reported overall=0 (GREEN) while the only SSO test never ran (violates P7). Fix preserves generic-tier failure-isolation but corrects the green SIGNAL: - conftest.pytest_collection_modifyitems counts skipped requires_deps tests and appends to $CCCI_DEPS_SKIP_REPORT. - run_recipe_ci: sums the count, surfaces it in RUN SUMMARY, and new pure predicate sso_dep_unverified(declared, deps_ready, skipped) flips overall=1. - 7 new unit tests (tests/unit/test_f211_sso_skip.py). Verified deploy-free (rate-limit-independent): 35/35 unit PASS; cold real-test proof on lasuite-docs test_oidc_with_keycloak.py -> 1 skipped + skip-report==1 -> orchestrator would set overall=1. Full e2e deferred until Docker Hub rate limit lifts. Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
@ -45,6 +45,17 @@ from harness import deps as deps_mod, discovery, generic, lifecycle, naming # n
|
||||
ALL_STAGES = ("install", "upgrade", "backup", "restore", "custom")
|
||||
|
||||
|
||||
def sso_dep_unverified(declared, deps_ready: bool, requires_deps_skipped: int) -> bool:
|
||||
"""F2-11 gate predicate (pure, unit-tested). True when a recipe declares DEPS but its
|
||||
setup_custom_tests failed (deps not ready) AND that caused ≥1 `requires_deps` (SSO/OIDC) test
|
||||
to SKIP. In that case the recipe's characteristic SSO claim was NOT verified, so the run must
|
||||
NOT report GREEN — even though a skip-only pytest file exits 0 and leaves every tier 'pass'.
|
||||
Generic-tier failure-isolation is preserved (those results stand); only the green SIGNAL is
|
||||
corrected. Gated on skip>0 so a deps-declaring recipe with no requires_deps tests isn't
|
||||
false-failed."""
|
||||
return bool(declared) and not deps_ready and requires_deps_skipped > 0
|
||||
|
||||
|
||||
def _truthy(v: str | None) -> bool:
|
||||
return str(v or "").strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
@ -427,6 +438,11 @@ def main() -> int:
|
||||
with open(depsfile, "w") as f:
|
||||
json.dump({}, f)
|
||||
os.environ["CCCI_DEPS_FILE"] = depsfile
|
||||
# F2-11: conftest appends the count of requires_deps tests it skips (deps-not-ready) here.
|
||||
skipfile = os.path.join(tempfile.gettempdir(), f"ccci-depskip-{domain}.txt")
|
||||
with contextlib.suppress(OSError):
|
||||
os.remove(skipfile)
|
||||
os.environ["CCCI_DEPS_SKIP_REPORT"] = skipfile
|
||||
declared = deps_mod.declared_deps(recipe)
|
||||
if declared:
|
||||
print(f"\n===== DEPS declared (deploy AFTER generic tiers): {declared} =====", flush=True)
|
||||
@ -562,6 +578,15 @@ def main() -> int:
|
||||
os.remove(statefile)
|
||||
with contextlib.suppress(OSError):
|
||||
os.remove(depsfile)
|
||||
# F2-11: sum the requires_deps skip counts conftest recorded across the custom files.
|
||||
requires_deps_skipped = 0
|
||||
try:
|
||||
with open(skipfile) as f:
|
||||
requires_deps_skipped = sum(int(x) for x in f.read().split() if x.strip())
|
||||
except OSError:
|
||||
pass
|
||||
with contextlib.suppress(OSError):
|
||||
os.remove(skipfile)
|
||||
|
||||
# ---- per-op summary (DG6 feed) ----
|
||||
# SSO-dep plan §1: DG4.1 generalised — one `abra app new` per app in the run (recipe + each
|
||||
@ -582,7 +607,12 @@ def main() -> int:
|
||||
print(f" deps-not-ready: {deps_not_ready_reason}")
|
||||
order = [s for s in ALL_STAGES if s in results]
|
||||
for op in order:
|
||||
print(f" {op:8s}: {results[op]}")
|
||||
suffix = ""
|
||||
# F2-11: annotate the custom tier when requires_deps (SSO) tests were skipped, so a reader
|
||||
# of the summary can't mistake a green custom tier for "SSO verified".
|
||||
if op == "custom" and requires_deps_skipped:
|
||||
suffix = f" ({requires_deps_skipped} requires_deps SKIPPED: deps-not-ready — SSO UNVERIFIED)"
|
||||
print(f" {op:8s}: {results[op]}{suffix}")
|
||||
|
||||
overall = 0
|
||||
if deploy_count != expected_deploy_count:
|
||||
@ -597,6 +627,17 @@ def main() -> int:
|
||||
overall = 1
|
||||
if any(v == "fail" for v in results.values()):
|
||||
overall = 1
|
||||
# F2-11: a deps-declaring recipe whose setup_custom_tests failed has NOT verified its SSO/OIDC
|
||||
# claim — its requires_deps tests SKIPPED (a skip-only file exits 0, so without this the run
|
||||
# would report GREEN). Fail the run for that recipe; generic-tier results above are untouched.
|
||||
if sso_dep_unverified(declared, deps_ready, requires_deps_skipped):
|
||||
print(
|
||||
f"!! recipe declares DEPS={declared} but setup_custom_tests failed and "
|
||||
f"{requires_deps_skipped} requires_deps (SSO) test(s) were SKIPPED — SSO claim NOT "
|
||||
f"verified; failing run (F2-11). deps-not-ready: {deps_not_ready_reason}",
|
||||
file=sys.stderr,
|
||||
)
|
||||
overall = 1
|
||||
if not results:
|
||||
print("no tiers ran", file=sys.stderr)
|
||||
return 1
|
||||
|
||||
Reference in New Issue
Block a user