runner/harness/generic.py

@@ -93,9 +93,15 @@ def assert_serving(domain: str, meta) -> None:
     Steps 1–2 are BOUNDED POLLS (no bare sleep), so a state-mutating op (upgrade/restore) that leaves
     the app briefly reconverging settles, while a persistent failure still fails within the timeout."""
     deadline = time.time() + meta.DEPLOY_TIMEOUT
-    while time.time() < deadline and not lifecycle.services_converged(domain):
+    converged = False
+    while time.time() < deadline:
+        # Keep both observations inside the retry budget: a service may restart
+        # between the initial convergence sample and its confirmation.
+        if lifecycle.services_converged(domain) and lifecycle.services_converged(domain):
+            converged = True
+            break
         time.sleep(5)
-    assert lifecycle.services_converged(domain), f"{domain}: services did not converge"
+    assert converged, f"{domain}: services did not converge"
 
     path = meta.HEALTH_PATH
     ok = tuple(meta.HEALTH_OK)

tests/unit/test_serving_retry.py

@@ -0,0 +1,30 @@
+"""Convergence confirmation must retry a transient second-sample failure."""
+
+from types import SimpleNamespace
+
+import pytest
+
+from harness import generic
+
+
+def _setup(monkeypatch, samples):
+    clock = [0]
+    monkeypatch.setattr(generic.time, "time", lambda: clock[0])
+    monkeypatch.setattr(generic.time, "sleep", lambda n: clock.__setitem__(0, clock[0] + n))
+    monkeypatch.setattr(generic.lifecycle, "services_converged", lambda d: next(samples))
+    monkeypatch.setattr(generic.lifecycle, "http_fetch", lambda *a: (200, "application"))
+    monkeypatch.setattr(generic, "served_cert", lambda d: (True, "*.ci.autonomic.zone"))
+    return SimpleNamespace(DEPLOY_TIMEOUT=20, HTTP_TIMEOUT=20, HEALTH_PATH="/", HEALTH_OK=(200,))
+
+
+def test_confirmation_failure_retries_until_services_settle(monkeypatch):
+    meta = _setup(monkeypatch, iter([True, False, True, True]))
+    generic.assert_serving("example.ci.autonomic.zone", meta)
+
+
+def test_persistent_confirmation_failure_still_fails(monkeypatch):
+    import itertools
+
+    meta = _setup(monkeypatch, itertools.cycle([True, False]))
+    with pytest.raises(AssertionError, match="services did not converge"):
+        generic.assert_serving("example.ci.autonomic.zone", meta)