fix(2): F2-11 — SSO-dep deps-not-ready SKIP no longer yields GREEN !testme

When a DEPS-declaring recipe's setup_custom_tests fails, its @requires_deps (SSO/OIDC)
tests skip; a skip-only pytest file exits 0 so the run previously reported overall=0
(GREEN) while the only SSO test never ran (violates P7). Fix preserves generic-tier
failure-isolation but corrects the green SIGNAL:
- conftest.pytest_collection_modifyitems counts skipped requires_deps tests and appends
  to $CCCI_DEPS_SKIP_REPORT.
- run_recipe_ci: sums the count, surfaces it in RUN SUMMARY, and new pure predicate
  sso_dep_unverified(declared, deps_ready, skipped) flips overall=1.
- 7 new unit tests (tests/unit/test_f211_sso_skip.py).

Verified deploy-free (rate-limit-independent): 35/35 unit PASS; cold real-test proof on
lasuite-docs test_oidc_with_keycloak.py -> 1 skipped + skip-report==1 -> orchestrator
would set overall=1. Full e2e deferred until Docker Hub rate limit lifts.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
2026-05-28 21:25:27 +01:00
parent 10d2a13031
commit 5b34496557
5 changed files with 248 additions and 1 deletions

View File

@ -45,6 +45,17 @@ from harness import deps as deps_mod, discovery, generic, lifecycle, naming # n
ALL_STAGES = ("install", "upgrade", "backup", "restore", "custom")
def sso_dep_unverified(declared, deps_ready: bool, requires_deps_skipped: int) -> bool:
"""F2-11 gate predicate (pure, unit-tested). True when a recipe declares DEPS but its
setup_custom_tests failed (deps not ready) AND that caused ≥1 `requires_deps` (SSO/OIDC) test
to SKIP. In that case the recipe's characteristic SSO claim was NOT verified, so the run must
NOT report GREEN — even though a skip-only pytest file exits 0 and leaves every tier 'pass'.
Generic-tier failure-isolation is preserved (those results stand); only the green SIGNAL is
corrected. Gated on skip>0 so a deps-declaring recipe with no requires_deps tests isn't
false-failed."""
return bool(declared) and not deps_ready and requires_deps_skipped > 0
def _truthy(v: str | None) -> bool:
return str(v or "").strip().lower() in ("1", "true", "yes", "on")
@ -427,6 +438,11 @@ def main() -> int:
with open(depsfile, "w") as f:
json.dump({}, f)
os.environ["CCCI_DEPS_FILE"] = depsfile
# F2-11: conftest appends the count of requires_deps tests it skips (deps-not-ready) here.
skipfile = os.path.join(tempfile.gettempdir(), f"ccci-depskip-{domain}.txt")
with contextlib.suppress(OSError):
os.remove(skipfile)
os.environ["CCCI_DEPS_SKIP_REPORT"] = skipfile
declared = deps_mod.declared_deps(recipe)
if declared:
print(f"\n===== DEPS declared (deploy AFTER generic tiers): {declared} =====", flush=True)
@ -562,6 +578,15 @@ def main() -> int:
os.remove(statefile)
with contextlib.suppress(OSError):
os.remove(depsfile)
# F2-11: sum the requires_deps skip counts conftest recorded across the custom files.
requires_deps_skipped = 0
try:
with open(skipfile) as f:
requires_deps_skipped = sum(int(x) for x in f.read().split() if x.strip())
except OSError:
pass
with contextlib.suppress(OSError):
os.remove(skipfile)
# ---- per-op summary (DG6 feed) ----
# SSO-dep plan §1: DG4.1 generalised — one `abra app new` per app in the run (recipe + each
@ -582,7 +607,12 @@ def main() -> int:
print(f" deps-not-ready: {deps_not_ready_reason}")
order = [s for s in ALL_STAGES if s in results]
for op in order:
print(f" {op:8s}: {results[op]}")
suffix = ""
# F2-11: annotate the custom tier when requires_deps (SSO) tests were skipped, so a reader
# of the summary can't mistake a green custom tier for "SSO verified".
if op == "custom" and requires_deps_skipped:
suffix = f" ({requires_deps_skipped} requires_deps SKIPPED: deps-not-ready — SSO UNVERIFIED)"
print(f" {op:8s}: {results[op]}{suffix}")
overall = 0
if deploy_count != expected_deploy_count:
@ -597,6 +627,17 @@ def main() -> int:
overall = 1
if any(v == "fail" for v in results.values()):
overall = 1
# F2-11: a deps-declaring recipe whose setup_custom_tests failed has NOT verified its SSO/OIDC
# claim — its requires_deps tests SKIPPED (a skip-only file exits 0, so without this the run
# would report GREEN). Fail the run for that recipe; generic-tier results above are untouched.
if sso_dep_unverified(declared, deps_ready, requires_deps_skipped):
print(
f"!! recipe declares DEPS={declared} but setup_custom_tests failed and "
f"{requires_deps_skipped} requires_deps (SSO) test(s) were SKIPPED — SSO claim NOT "
f"verified; failing run (F2-11). deps-not-ready: {deps_not_ready_reason}",
file=sys.stderr,
)
overall = 1
if not results:
print("no tiers ran", file=sys.stderr)
return 1