187 lines
7.5 KiB
Python
187 lines
7.5 KiB
Python
"""Unit tests for the pure helpers in harness.screenshot (Phase 3 U1).
|
|
|
|
The Playwright capture itself needs a live app (exercised in the U1 live demo); here we cover the
|
|
pure bits: the artifact path and the SCREENSHOT-hook resolution. Run cold:
|
|
cc-ci-run -m pytest tests/unit/test_screenshot.py -q
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
|
|
sys.path.insert(0, os.path.join(os.path.dirname(__file__), "..", "..", "runner"))
|
|
from harness import meta as meta_mod # noqa: E402
|
|
from harness import screenshot as S # noqa: E402
|
|
|
|
|
|
def test_screenshot_path():
|
|
assert S.screenshot_path("/var/lib/cc-ci-runs/42") == "/var/lib/cc-ci-runs/42/screenshot.png"
|
|
|
|
|
|
def test_hook_none_when_absent():
|
|
assert S._load_screenshot_hook(None) is None
|
|
assert S._load_screenshot_hook({}) is None
|
|
assert S._load_screenshot_hook({"SCREENSHOT": "not-callable"}) is None
|
|
|
|
|
|
def test_hook_returned_when_callable():
|
|
def hook(page, domain, meta):
|
|
pass
|
|
|
|
assert S._load_screenshot_hook({"SCREENSHOT": hook}) is hook
|
|
|
|
|
|
class _FakePage:
|
|
"""Minimal Playwright-page stand-in for the settle/blank-retry helpers (no browser needed)."""
|
|
|
|
def __init__(self, shot_sizes, idle_raises=False):
|
|
self._shot_sizes = list(shot_sizes) # bytes written per successive screenshot() call
|
|
self._idle_raises = idle_raises
|
|
self.idle_waits = [] # (state, timeout) per wait_for_load_state call
|
|
self.timeout_waits = [] # ms per wait_for_timeout call
|
|
self.shots = 0
|
|
|
|
def wait_for_load_state(self, state, timeout=None):
|
|
self.idle_waits.append((state, timeout))
|
|
if self._idle_raises:
|
|
raise TimeoutError(f"page kept polling past {timeout}ms")
|
|
|
|
def wait_for_timeout(self, ms):
|
|
self.timeout_waits.append(ms)
|
|
|
|
def screenshot(self, path, full_page=False):
|
|
self.shots += 1
|
|
with open(path, "wb") as f:
|
|
f.write(b"\x89PNG" + b"\0" * (self._shot_sizes.pop(0) - 4))
|
|
|
|
|
|
def test_settle_swallows_never_idle_pages():
|
|
"""R7: an app that never reaches network-idle (continuous polling) must not raise — the
|
|
timeout cap IS the wait."""
|
|
page = _FakePage([], idle_raises=True)
|
|
S._settle(page, 1234) # must not raise
|
|
assert page.idle_waits == [("networkidle", 1234)]
|
|
assert page.timeout_waits == [S.RENDER_GRACE_MS]
|
|
|
|
|
|
def test_snap_retries_blank_frame(tmp_path):
|
|
"""A blank-sized first frame (audit fingerprint: 4801 B) triggers exactly one retry with a
|
|
longer settle, overwriting the tiny frame with the later (painted) one."""
|
|
out = str(tmp_path / "shot.png")
|
|
page = _FakePage([4801, 30256])
|
|
S._snap_with_blank_retry(page, out)
|
|
assert page.shots == 2
|
|
assert page.idle_waits == [("networkidle", S.BLANK_RETRY_SETTLE_MS)]
|
|
assert os.path.getsize(out) == 30256
|
|
|
|
|
|
def test_snap_no_retry_for_real_frame(tmp_path):
|
|
"""A real-sized first frame is kept as-is — no second screenshot, no extra waiting."""
|
|
out = str(tmp_path / "shot.png")
|
|
page = _FakePage([35707])
|
|
S._snap_with_blank_retry(page, out)
|
|
assert page.shots == 1
|
|
assert page.idle_waits == []
|
|
assert os.path.getsize(out) == 35707
|
|
|
|
|
|
def test_snap_retry_keeps_late_frame_even_if_still_blank(tmp_path):
|
|
"""If the retry frame is still tiny we keep it (honest best-effort) — exactly one retry,
|
|
never a loop."""
|
|
out = str(tmp_path / "shot.png")
|
|
page = _FakePage([4801, 4801])
|
|
S._snap_with_blank_retry(page, out)
|
|
assert page.shots == 2
|
|
assert os.path.getsize(out) == 4801
|
|
assert not os.path.exists(out + ".retry"), "temp retry frame must be cleaned up"
|
|
|
|
|
|
def test_snap_retry_never_regresses_to_smaller_frame(tmp_path):
|
|
"""Adversary finding A1: a partial-but-real first frame (just under the threshold) must
|
|
survive a retry that comes back WORSE (page regressed to blank during the extra settle) —
|
|
the larger frame wins."""
|
|
out = str(tmp_path / "shot.png")
|
|
page = _FakePage([9999, 4801])
|
|
S._snap_with_blank_retry(page, out)
|
|
assert page.shots == 2
|
|
assert os.path.getsize(out) == 9999, "retry must never overwrite a larger frame (A1)"
|
|
assert not os.path.exists(out + ".retry"), "temp retry frame must be cleaned up"
|
|
|
|
|
|
def test_blank_threshold_brackets_observed_sizes():
|
|
"""Threshold sits between the audited defect sizes (blank 4801-2 B, lone spinners up to
|
|
8764 B) and the smallest real page (custom-html-tiny, 12950 B)."""
|
|
for defect in (4801, 4802, 5895, 6022, 7913, 8764):
|
|
assert defect < S.BLANK_SIZE_BYTES
|
|
assert S.BLANK_SIZE_BYTES < 12950
|
|
|
|
|
|
def test_wait_budget_within_step_cap():
|
|
"""plan-phase-shot §3 P3: the screenshot step's bounded waiting must stay ≤ ~60s worst case."""
|
|
total_ms = (
|
|
S.NAV_DEADLINE_S * 1000
|
|
+ S.SETTLE_TIMEOUT_MS
|
|
+ S.RENDER_GRACE_MS
|
|
+ S.BLANK_RETRY_SETTLE_MS
|
|
+ S.RENDER_GRACE_MS
|
|
)
|
|
assert total_ms <= 60_000, f"screenshot wait budget {total_ms}ms exceeds the ~60s step cap"
|
|
|
|
|
|
def test_mattermost_screenshot_hook_lands_login():
|
|
"""phase-shot: mattermost-lts ships the first real SCREENSHOT hook — `/` serves the
|
|
desktop-or-browser interstitial, so the hook must navigate to /login (the representative,
|
|
credential-free sign-in form) and settle; the harness then snaps the PNG."""
|
|
|
|
class _Resp:
|
|
status = 200
|
|
|
|
class _NavPage(_FakePage):
|
|
def __init__(self, click_raises=False):
|
|
super().__init__([])
|
|
self.urls = []
|
|
self.clicks = []
|
|
self._click_raises = click_raises
|
|
|
|
def goto(self, url, wait_until=None, timeout=None):
|
|
self.urls.append(url)
|
|
return _Resp()
|
|
|
|
def click(self, selector, timeout=None):
|
|
self.clicks.append(selector)
|
|
if self._click_raises:
|
|
raise TimeoutError("no interstitial")
|
|
|
|
tests_dir = os.path.join(os.path.dirname(__file__), "..")
|
|
meta = meta_mod.load("mattermost-lts", tests_dir=tests_dir)
|
|
hook = S._load_screenshot_hook(meta)
|
|
assert callable(hook), "mattermost-lts SCREENSHOT hook missing from the real load path"
|
|
page = _NavPage()
|
|
hook(page, meta_mod.hook_ctx("mm.example.org", meta))
|
|
assert page.urls == ["https://mm.example.org/login"]
|
|
assert page.clicks == ["text=View in Browser"], "hook must click through the interstitial"
|
|
assert len(page.idle_waits) == 2, "hook must settle after nav AND after the click"
|
|
|
|
# no interstitial (already on the form): the click times out and the hook still succeeds
|
|
page2 = _NavPage(click_raises=True)
|
|
hook(page2, meta_mod.hook_ctx("mm.example.org", meta))
|
|
assert page2.clicks == ["text=View in Browser"]
|
|
assert len(page2.idle_waits) == 1, "failed click must skip the second settle, not raise"
|
|
|
|
|
|
def test_screenshot_reachable_through_real_load_path(tmp_path):
|
|
"""R2 proof (rcust P1): a recipe SCREENSHOT hook declared in recipe_meta.py arrives at
|
|
screenshot._load_screenshot_hook through the REAL orchestrator load path (meta.load — the
|
|
object run_recipe_ci passes to capture()). Under the old six-loader world the orchestrator's
|
|
L1 allowlist dropped SCREENSHOT, so the hook was unreachable (spec §8 R2)."""
|
|
d = tmp_path / "shotrecipe"
|
|
d.mkdir()
|
|
(d / "recipe_meta.py").write_text(
|
|
"def SCREENSHOT(page, ctx):\n return None\n",
|
|
)
|
|
meta = meta_mod.load("shotrecipe", tests_dir=str(tmp_path))
|
|
hook = S._load_screenshot_hook(meta)
|
|
assert callable(hook), "SCREENSHOT hook did not survive the orchestrator load path (R2)"
|
|
assert S._load_screenshot_hook(meta_mod.load("no-such", tests_dir=str(tmp_path))) is None
|