fix(2): F2-11 — SSO-dep deps-not-ready SKIP no longer yields GREEN !testme

When a DEPS-declaring recipe's setup_custom_tests fails, its @requires_deps (SSO/OIDC)
tests skip; a skip-only pytest file exits 0 so the run previously reported overall=0
(GREEN) while the only SSO test never ran (violates P7). Fix preserves generic-tier
failure-isolation but corrects the green SIGNAL:
- conftest.pytest_collection_modifyitems counts skipped requires_deps tests and appends
  to $CCCI_DEPS_SKIP_REPORT.
- run_recipe_ci: sums the count, surfaces it in RUN SUMMARY, and new pure predicate
  sso_dep_unverified(declared, deps_ready, skipped) flips overall=1.
- 7 new unit tests (tests/unit/test_f211_sso_skip.py).

Verified deploy-free (rate-limit-independent): 35/35 unit PASS; cold real-test proof on
lasuite-docs test_oidc_with_keycloak.py -> 1 skipped + skip-report==1 -> orchestrator
would set overall=1. Full e2e deferred until Docker Hub rate limit lifts.

Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>

tests/conftest.py

@@ -107,9 +107,23 @@ def pytest_collection_modifyitems(config, items):
         return
     reason = os.environ.get("CCCI_DEPS_NOT_READY_REASON", "(no reason given)")
     skip_mark = pytest.mark.skip(reason=f"deps-not-ready: {reason}")
+    skipped = 0
     for item in items:
         if "requires_deps" in item.keywords:
             item.add_marker(skip_mark)
+            skipped += 1
+    # F2-11: a skip-only pytest file exits 0, so without this the orchestrator can't tell
+    # "SSO verified" from "SSO test silently skipped because deps weren't ready". Record the count
+    # of requires_deps tests we skipped to a report file the orchestrator reads — it surfaces the
+    # count in RUN SUMMARY and FAILS the recipe's SSO claim (a green exit must not mask an unrun
+    # SSO test). Appended one line per pytest invocation (one per custom file); orchestrator sums.
+    report = os.environ.get("CCCI_DEPS_SKIP_REPORT")
+    if report and skipped:
+        try:
+            with open(report, "a") as f:
+                f.write(f"{skipped}\n")
+        except OSError:
+            pass
 
 
 def pytest_configure(config):

tests/unit/test_f211_sso_skip.py

@@ -0,0 +1,115 @@
+"""Unit tests for F2-11 — SSO-dep "deps-not-ready" SKIP must NOT yield a GREEN run.
+
+Two halves of the fix are tested without any real deploy:
+1. `run_recipe_ci.sso_dep_unverified` — the pure gate predicate the orchestrator uses to flip
+   `overall` to fail when a deps-declaring recipe's SSO tests were skipped because deps weren't ready.
+2. `conftest.pytest_collection_modifyitems` — when CCCI_DEPS_READY=0 it (a) skips every
+   `requires_deps` test and (b) records the skipped count to `$CCCI_DEPS_SKIP_REPORT` so the
+   orchestrator can surface it + gate on it.
+
+The end-to-end hazard (a real SSO-dep recipe going green on a skip) is exercised by e2e against
+cc-ci; here we lock the decision logic + the conftest→orchestrator signal that drives it.
+"""
+
+from __future__ import annotations
+
+import importlib.util
+import os
+import sys
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
+import run_recipe_ci  # noqa: E402
+
+
+# ---- 1. the pure gate predicate ----
+
+def test_sso_dep_unverified_true_when_declared_notready_and_skipped():
+    """declares DEPS + deps not ready + ≥1 requires_deps test skipped → run must FAIL (F2-11)."""
+    assert run_recipe_ci.sso_dep_unverified(["keycloak"], deps_ready=False, requires_deps_skipped=1)
+    assert run_recipe_ci.sso_dep_unverified(["keycloak"], deps_ready=False, requires_deps_skipped=3)
+
+
+def test_sso_dep_unverified_false_when_deps_ready():
+    """deps ready (setup_custom_tests succeeded) → SSO tests actually ran → not a failure."""
+    assert not run_recipe_ci.sso_dep_unverified(["keycloak"], deps_ready=True, requires_deps_skipped=0)
+
+
+def test_sso_dep_unverified_false_when_no_deps_declared():
+    """A recipe with no DEPS can never trip the SSO-skip gate."""
+    assert not run_recipe_ci.sso_dep_unverified([], deps_ready=False, requires_deps_skipped=0)
+    assert not run_recipe_ci.sso_dep_unverified(None, deps_ready=False, requires_deps_skipped=2)
+
+
+def test_sso_dep_unverified_false_when_nothing_skipped():
+    """Deps declared + not ready but ZERO requires_deps tests skipped → don't false-fail
+    (the recipe has no SSO-marked tests to have been masked)."""
+    assert not run_recipe_ci.sso_dep_unverified(["keycloak"], deps_ready=False, requires_deps_skipped=0)
+
+
+# ---- 2. conftest skip + record behavior ----
+
+def _load_conftest():
+    """Load tests/conftest.py under a private module name (avoid clashing with pytest's own
+    loaded `conftest`), so we can call pytest_collection_modifyitems directly with fakes."""
+    path = os.path.join(os.path.dirname(__file__), "..", "conftest.py")
+    spec = importlib.util.spec_from_file_location("ccci_conftest_under_test", path)
+    mod = importlib.util.module_from_spec(spec)
+    spec.loader.exec_module(mod)
+    return mod
+
+
+class _FakeItem:
+    def __init__(self, keywords):
+        # pytest `item.keywords` supports `in`; a dict suffices.
+        self.keywords = {k: True for k in keywords}
+        self.markers = []
+
+    def add_marker(self, mark):
+        self.markers.append(mark)
+
+
+def test_conftest_skips_and_records_requires_deps_when_not_ready(tmp_path, monkeypatch):
+    conftest = _load_conftest()
+    report = tmp_path / "skip.txt"
+    monkeypatch.setenv("CCCI_DEPS_READY", "0")
+    monkeypatch.setenv("CCCI_DEPS_NOT_READY_REASON", "keycloak realm setup boom")
+    monkeypatch.setenv("CCCI_DEPS_SKIP_REPORT", str(report))
+
+    sso1 = _FakeItem(["requires_deps"])
+    sso2 = _FakeItem(["requires_deps"])
+    plain = _FakeItem([])  # a non-deps custom test — must NOT be skipped
+    conftest.pytest_collection_modifyitems(config=None, items=[sso1, sso2, plain])
+
+    # Both requires_deps items got a skip marker; the plain one did not.
+    assert len(sso1.markers) == 1 and len(sso2.markers) == 1
+    assert plain.markers == []
+    # The skipped count was recorded for the orchestrator.
+    assert report.read_text().split() == ["2"]
+
+
+def test_conftest_appends_across_invocations(tmp_path, monkeypatch):
+    """The orchestrator runs one pytest per custom file; counts must accumulate (append)."""
+    conftest = _load_conftest()
+    report = tmp_path / "skip.txt"
+    monkeypatch.setenv("CCCI_DEPS_READY", "0")
+    monkeypatch.setenv("CCCI_DEPS_SKIP_REPORT", str(report))
+
+    conftest.pytest_collection_modifyitems(None, [_FakeItem(["requires_deps"])])
+    conftest.pytest_collection_modifyitems(None, [_FakeItem(["requires_deps"]), _FakeItem(["requires_deps"])])
+
+    total = sum(int(x) for x in report.read_text().split())
+    assert total == 3
+
+
+def test_conftest_noop_and_no_record_when_deps_ready(tmp_path, monkeypatch):
+    """deps ready → no skips, no report file written (early return)."""
+    conftest = _load_conftest()
+    report = tmp_path / "skip.txt"
+    monkeypatch.setenv("CCCI_DEPS_READY", "1")
+    monkeypatch.setenv("CCCI_DEPS_SKIP_REPORT", str(report))
+
+    item = _FakeItem(["requires_deps"])
+    conftest.pytest_collection_modifyitems(None, [item])
+
+    assert item.markers == []  # not skipped
+    assert not report.exists()  # nothing recorded