watchdog: proof-of-life outranks the agent's own WAITING-UNTIL deadline

Completes b73af35. That commit stopped the cap from killing a live build, but the OTHER branch still
rebooted past the stated deadline regardless of a running build — and an agent blocked on a shell cannot
re-emit a fresh marker to extend its estimate. So a cargo-mutants run that simply overran its guess would
still be killed at the guess, throwing the work away. The deadline is an ESTIMATE; a running build is a
FACT. Order is now: absolute cap (waiting_until_max) > live build > stated deadline. The cap is the one
bound that reboots even mid-build — which is exactly how a genuinely hung build gets caught, so a runaway
still cannot park forever. Tests: live-build-defers-past-deadline, past-deadline-no-build-reboots,
cap-is-absolute-and-reboots-a-hung-build. 68 pass.
This commit is contained in:
2026-07-11 13:53:39 +00:00
parent b73af35792
commit aa1b625732
2 changed files with 36 additions and 20 deletions
+24 -13
View File
@@ -709,23 +709,34 @@ class TestBuildAwareStall(unittest.TestCase):
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, [self.agent["name"]]) # idle past cap, no build → rebooted
def test_waiting_until_cap_yields_to_a_live_build(self):
# cargo-mutants / coverage runs legitimately take hours: a build still running is proof of
# life, so the cap must NOT guillotine it — the stated deadline governs instead
def test_live_build_defers_past_the_stated_deadline(self):
# the deadline is only the agent's ESTIMATE, and an agent blocked on a shell cannot re-emit a
# fresh marker — so a still-running build (cargo-mutants overrunning its guess) must not be
# killed at the estimate. Proof of life outranks the deadline.
self.cfg["watchdog"]["waiting_until_max"] = 7200
self.patch("_build_running", lambda *a, **k: True)
self.patch("_parse_waiting_until", lambda *a, **k: time.time() - 1000) # deadline already past
self._set_idle(3000) # but still under the absolute cap
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, []) # build running → deferred
def test_past_deadline_with_no_build_reboots(self):
# no build, deadline blown → the self-wake did not fire → reboot
self.patch("_build_running", lambda *a, **k: False)
self.patch("_parse_waiting_until", lambda *a, **k: time.time() - 1000)
self._set_idle(3000)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, [self.agent["name"]])
def test_cap_is_absolute_and_reboots_a_hung_build(self):
# the cap is the ONE absolute bound: past it we reboot even mid-build — that is precisely how
# a genuinely HUNG build gets caught, and why a runaway can never park forever
self.cfg["watchdog"]["waiting_until_max"] = 7200
self.patch("_build_running", lambda *a, **k: True)
self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000)
self._set_idle(8000) # idle > cap, but a build IS running
self._set_idle(8000) # idle > cap, build "running" (hung)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, []) # survives the cap
def test_waiting_until_past_deadline_reboots_even_with_a_build(self):
# the deadline is still the hard bound — a build running does not let it park forever
self.patch("_build_running", lambda *a, **k: True)
self.patch("_parse_waiting_until", lambda *a, **k: time.time() - 1000) # deadline long past
self._set_idle(3000)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, [self.agent["name"]]) # past WAITING-UNTIL → rebooted
self.assertEqual(self.reboots, [self.agent["name"]])
def test_no_build_check_below_base_threshold(self):
def boom(*a, **k):