diff --git a/runner/harness/generic.py b/runner/harness/generic.py index 7db0af3..0c1b94f 100644 --- a/runner/harness/generic.py +++ b/runner/harness/generic.py @@ -93,9 +93,15 @@ def assert_serving(domain: str, meta) -> None: Steps 1–2 are BOUNDED POLLS (no bare sleep), so a state-mutating op (upgrade/restore) that leaves the app briefly reconverging settles, while a persistent failure still fails within the timeout.""" deadline = time.time() + meta.DEPLOY_TIMEOUT - while time.time() < deadline and not lifecycle.services_converged(domain): + converged = False + while time.time() < deadline: + # Keep both observations inside the retry budget: a service may restart + # between the initial convergence sample and its confirmation. + if lifecycle.services_converged(domain) and lifecycle.services_converged(domain): + converged = True + break time.sleep(5) - assert lifecycle.services_converged(domain), f"{domain}: services did not converge" + assert converged, f"{domain}: services did not converge" path = meta.HEALTH_PATH ok = tuple(meta.HEALTH_OK) diff --git a/tests/unit/test_serving_retry.py b/tests/unit/test_serving_retry.py new file mode 100644 index 0000000..26b9c92 --- /dev/null +++ b/tests/unit/test_serving_retry.py @@ -0,0 +1,30 @@ +"""Convergence confirmation must retry a transient second-sample failure.""" + +from types import SimpleNamespace + +import pytest + +from harness import generic + + +def _setup(monkeypatch, samples): + clock = [0] + monkeypatch.setattr(generic.time, "time", lambda: clock[0]) + monkeypatch.setattr(generic.time, "sleep", lambda n: clock.__setitem__(0, clock[0] + n)) + monkeypatch.setattr(generic.lifecycle, "services_converged", lambda d: next(samples)) + monkeypatch.setattr(generic.lifecycle, "http_fetch", lambda *a: (200, "application")) + monkeypatch.setattr(generic, "served_cert", lambda d: (True, "*.ci.autonomic.zone")) + return SimpleNamespace(DEPLOY_TIMEOUT=20, HTTP_TIMEOUT=20, HEALTH_PATH="/", HEALTH_OK=(200,)) + + +def test_confirmation_failure_retries_until_services_settle(monkeypatch): + meta = _setup(monkeypatch, iter([True, False, True, True])) + generic.assert_serving("example.ci.autonomic.zone", meta) + + +def test_persistent_confirmation_failure_still_fails(monkeypatch): + import itertools + + meta = _setup(monkeypatch, itertools.cycle([True, False])) + with pytest.raises(AssertionError, match="services did not converge"): + generic.assert_serving("example.ci.autonomic.zone", meta)