"""Unit tests for the pure helpers in harness.screenshot (Phase 3 U1). The Playwright capture itself needs a live app (exercised in the U1 live demo); here we cover the pure bits: the artifact path and the SCREENSHOT-hook resolution. Run cold: cc-ci-run -m pytest tests/unit/test_screenshot.py -q """ from __future__ import annotations import os import sys sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner")) from harness import meta as meta_mod # noqa: E402 from harness import screenshot as S # noqa: E402 def test_screenshot_path(): assert S.screenshot_path("/var/lib/cc-ci-runs/42") == "/var/lib/cc-ci-runs/42/screenshot.png" def test_hook_none_when_absent(): assert S._load_screenshot_hook(None) is None assert S._load_screenshot_hook({}) is None assert S._load_screenshot_hook({"SCREENSHOT": "not-callable"}) is None def test_hook_returned_when_callable(): def hook(page, domain, meta): pass assert S._load_screenshot_hook({"SCREENSHOT": hook}) is hook class _FakePage: """Minimal Playwright-page stand-in for the settle/blank-retry helpers (no browser needed).""" def __init__(self, shot_sizes, idle_raises=False): self._shot_sizes = list(shot_sizes) # bytes written per successive screenshot() call self._idle_raises = idle_raises self.idle_waits = [] # (state, timeout) per wait_for_load_state call self.timeout_waits = [] # ms per wait_for_timeout call self.shots = 0 def wait_for_load_state(self, state, timeout=None): self.idle_waits.append((state, timeout)) if self._idle_raises: raise TimeoutError(f"page kept polling past {timeout}ms") def wait_for_timeout(self, ms): self.timeout_waits.append(ms) def screenshot(self, path, full_page=False): self.shots += 1 with open(path, "wb") as f: f.write(b"\x89PNG" + b"\0" * (self._shot_sizes.pop(0) - 4)) def test_settle_swallows_never_idle_pages(): """R7: an app that never reaches network-idle (continuous polling) must not raise — the timeout cap IS the wait.""" page = _FakePage([], idle_raises=True) S._settle(page, 1234) # must not raise assert page.idle_waits == [("networkidle", 1234)] assert page.timeout_waits == [S.RENDER_GRACE_MS] def test_snap_retries_blank_frame(tmp_path): """A blank-sized first frame (audit fingerprint: 4801 B) triggers exactly one retry with a longer settle, overwriting the tiny frame with the later (painted) one.""" out = str(tmp_path / "shot.png") page = _FakePage([4801, 30256]) S._snap_with_blank_retry(page, out) assert page.shots == 2 assert page.idle_waits == [("networkidle", S.BLANK_RETRY_SETTLE_MS)] assert os.path.getsize(out) == 30256 def test_snap_no_retry_for_real_frame(tmp_path): """A real-sized first frame is kept as-is — no second screenshot, no extra waiting.""" out = str(tmp_path / "shot.png") page = _FakePage([35707]) S._snap_with_blank_retry(page, out) assert page.shots == 1 assert page.idle_waits == [] assert os.path.getsize(out) == 35707 def test_snap_retry_keeps_late_frame_even_if_still_blank(tmp_path): """If the retry frame is still tiny we keep it (honest best-effort) — exactly one retry, never a loop.""" out = str(tmp_path / "shot.png") page = _FakePage([4801, 4801]) S._snap_with_blank_retry(page, out) assert page.shots == 2 assert os.path.getsize(out) == 4801 assert not os.path.exists(out + ".retry"), "temp retry frame must be cleaned up" def test_snap_retry_never_regresses_to_smaller_frame(tmp_path): """Adversary finding A1: a partial-but-real first frame (just under the threshold) must survive a retry that comes back WORSE (page regressed to blank during the extra settle) — the larger frame wins.""" out = str(tmp_path / "shot.png") page = _FakePage([9999, 4801]) S._snap_with_blank_retry(page, out) assert page.shots == 2 assert os.path.getsize(out) == 9999, "retry must never overwrite a larger frame (A1)" assert not os.path.exists(out + ".retry"), "temp retry frame must be cleaned up" def test_blank_threshold_brackets_observed_sizes(): """Threshold sits between the audited defect sizes (blank 4801-2 B, lone spinners up to 8764 B) and the smallest real page (custom-html-tiny, 12950 B).""" for defect in (4801, 4802, 5895, 6022, 7913, 8764): assert defect < S.BLANK_SIZE_BYTES assert S.BLANK_SIZE_BYTES < 12950 def test_wait_budget_within_step_cap(): """plan-phase-shot §3 P3: the screenshot step's bounded waiting must stay ≤ ~60s worst case.""" total_ms = ( S.NAV_DEADLINE_S * 1000 + S.SETTLE_TIMEOUT_MS + S.RENDER_GRACE_MS + S.BLANK_RETRY_SETTLE_MS + S.RENDER_GRACE_MS ) assert total_ms <= 60_000, f"screenshot wait budget {total_ms}ms exceeds the ~60s step cap" def test_mattermost_screenshot_hook_lands_login(): """phase-shot: mattermost-lts ships the first real SCREENSHOT hook — `/` serves the desktop-or-browser interstitial, so the hook must navigate to /login (the representative, credential-free sign-in form) and settle; the harness then snaps the PNG.""" class _Resp: status = 200 class _NavPage(_FakePage): def __init__(self, click_raises=False): super().__init__([]) self.urls = [] self.clicks = [] self._click_raises = click_raises def goto(self, url, wait_until=None, timeout=None): self.urls.append(url) return _Resp() def click(self, selector, timeout=None): self.clicks.append(selector) if self._click_raises: raise TimeoutError("no interstitial") tests_dir = os.path.join(os.path.dirname(__file__), "..") meta = meta_mod.load("mattermost-lts", tests_dir=tests_dir) hook = S._load_screenshot_hook(meta) assert callable(hook), "mattermost-lts SCREENSHOT hook missing from the real load path" page = _NavPage() hook(page, meta_mod.hook_ctx("mm.example.org", meta)) assert page.urls == ["https://mm.example.org/login"] assert page.clicks == ["text=View in Browser"], "hook must click through the interstitial" assert len(page.idle_waits) == 2, "hook must settle after nav AND after the click" # no interstitial (already on the form): the click times out and the hook still succeeds page2 = _NavPage(click_raises=True) hook(page2, meta_mod.hook_ctx("mm.example.org", meta)) assert page2.clicks == ["text=View in Browser"] assert len(page2.idle_waits) == 1, "failed click must skip the second settle, not raise" def test_screenshot_reachable_through_real_load_path(tmp_path): """R2 proof (rcust P1): a recipe SCREENSHOT hook declared in recipe_meta.py arrives at screenshot._load_screenshot_hook through the REAL orchestrator load path (meta.load — the object run_recipe_ci passes to capture()). Under the old six-loader world the orchestrator's L1 allowlist dropped SCREENSHOT, so the hook was unreachable (spec §8 R2).""" d = tmp_path / "shotrecipe" d.mkdir() (d / "recipe_meta.py").write_text( "def SCREENSHOT(page, ctx):\n return None\n", ) meta = meta_mod.load("shotrecipe", tests_dir=str(tmp_path)) hook = S._load_screenshot_hook(meta) assert callable(hook), "SCREENSHOT hook did not survive the orchestrator load path (R2)" assert S._load_screenshot_hook(meta_mod.load("no-such", tests_dir=str(tmp_path))) is None