diff --git a/agents.py b/agents.py index 51cf6c8..1d52cf0 100755 --- a/agents.py +++ b/agents.py @@ -582,14 +582,19 @@ def stall_check_one(cfg, agent): # of killing it mid-run. Cap how far out it can push its own reboot, so a runaway can't park # itself forever ("some max no matter what"). wu_max = int(cfg["watchdog"].get("waiting_until_max", 7200)) - # The cap guards against a session that parked itself and is genuinely stuck. A build still - # running under the session (cargo-mutants / a coverage run / a remote ssh) is proof of life — - # those legitimately run for hours — so it defers to the agent's stated deadline instead of - # being guillotined by the cap. Past the deadline (+grace) it reboots either way, so a runaway - # can never park forever. - if wu_max and idle > wu_max and not _build_running(cfg, agent): - reason = (f"WAITING-UNTIL exceeded the {wu_max}s cap (idle {int(idle)}s, no build running) " + building = _build_running(cfg, agent) + # A build still running under the session (cargo-mutants, a coverage run, a remote ssh) is + # PROOF OF LIFE, and it outranks the agent's own deadline: the deadline is only an estimate, + # and an agent blocked on a shell cannot re-emit a fresh marker to extend it — so killing a + # live build at its estimate throws the work away for nothing. `waiting_until_max` is the one + # absolute bound: past it we reboot even mid-build, which is what catches a genuinely HUNG + # build (the case the cap exists for). + if wu_max and idle > wu_max: + reason = (f"idle {int(idle)}s past the {wu_max}s WAITING-UNTIL cap " + f"({'build still running — treating it as hung' if building else 'no build running'}) " f"— rebooting regardless") + elif building: + return elif now <= until + grace: return else: diff --git a/tests/test_unit.py b/tests/test_unit.py index 0b75aa4..1f340e8 100755 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -709,23 +709,34 @@ class TestBuildAwareStall(unittest.TestCase): agents.stall_check_one(self.cfg, self.agent) self.assertEqual(self.reboots, [self.agent["name"]]) # idle past cap, no build → rebooted - def test_waiting_until_cap_yields_to_a_live_build(self): - # cargo-mutants / coverage runs legitimately take hours: a build still running is proof of - # life, so the cap must NOT guillotine it — the stated deadline governs instead + def test_live_build_defers_past_the_stated_deadline(self): + # the deadline is only the agent's ESTIMATE, and an agent blocked on a shell cannot re-emit a + # fresh marker — so a still-running build (cargo-mutants overrunning its guess) must not be + # killed at the estimate. Proof of life outranks the deadline. + self.cfg["watchdog"]["waiting_until_max"] = 7200 + self.patch("_build_running", lambda *a, **k: True) + self.patch("_parse_waiting_until", lambda *a, **k: time.time() - 1000) # deadline already past + self._set_idle(3000) # but still under the absolute cap + agents.stall_check_one(self.cfg, self.agent) + self.assertEqual(self.reboots, []) # build running → deferred + + def test_past_deadline_with_no_build_reboots(self): + # no build, deadline blown → the self-wake did not fire → reboot + self.patch("_build_running", lambda *a, **k: False) + self.patch("_parse_waiting_until", lambda *a, **k: time.time() - 1000) + self._set_idle(3000) + agents.stall_check_one(self.cfg, self.agent) + self.assertEqual(self.reboots, [self.agent["name"]]) + + def test_cap_is_absolute_and_reboots_a_hung_build(self): + # the cap is the ONE absolute bound: past it we reboot even mid-build — that is precisely how + # a genuinely HUNG build gets caught, and why a runaway can never park forever self.cfg["watchdog"]["waiting_until_max"] = 7200 self.patch("_build_running", lambda *a, **k: True) self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000) - self._set_idle(8000) # idle > cap, but a build IS running + self._set_idle(8000) # idle > cap, build "running" (hung) agents.stall_check_one(self.cfg, self.agent) - self.assertEqual(self.reboots, []) # survives the cap - - def test_waiting_until_past_deadline_reboots_even_with_a_build(self): - # the deadline is still the hard bound — a build running does not let it park forever - self.patch("_build_running", lambda *a, **k: True) - self.patch("_parse_waiting_until", lambda *a, **k: time.time() - 1000) # deadline long past - self._set_idle(3000) - agents.stall_check_one(self.cfg, self.agent) - self.assertEqual(self.reboots, [self.agent["name"]]) # past WAITING-UNTIL → rebooted + self.assertEqual(self.reboots, [self.agent["name"]]) def test_no_build_check_below_base_threshold(self): def boom(*a, **k):