diff --git a/agents.py b/agents.py index 23799a2..51cf6c8 100755 --- a/agents.py +++ b/agents.py @@ -582,10 +582,14 @@ def stall_check_one(cfg, agent): # of killing it mid-run. Cap how far out it can push its own reboot, so a runaway can't park # itself forever ("some max no matter what"). wu_max = int(cfg["watchdog"].get("waiting_until_max", 7200)) - if wu_max and idle > wu_max: - # idle here ≈ how long the pane has sat quiet since the agent emitted the marker; once - # that exceeds the cap we reboot no matter how far out the stated deadline is. - reason = f"WAITING-UNTIL exceeded the {wu_max}s cap (idle {int(idle)}s) — rebooting regardless" + # The cap guards against a session that parked itself and is genuinely stuck. A build still + # running under the session (cargo-mutants / a coverage run / a remote ssh) is proof of life — + # those legitimately run for hours — so it defers to the agent's stated deadline instead of + # being guillotined by the cap. Past the deadline (+grace) it reboots either way, so a runaway + # can never park forever. + if wu_max and idle > wu_max and not _build_running(cfg, agent): + reason = (f"WAITING-UNTIL exceeded the {wu_max}s cap (idle {int(idle)}s, no build running) " + f"— rebooting regardless") elif now <= until + grace: return else: diff --git a/tests/test_unit.py b/tests/test_unit.py index 3d9718c..0b75aa4 100755 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -699,15 +699,33 @@ class TestBuildAwareStall(unittest.TestCase): agents.stall_check_one(self.cfg, self.agent) self.assertEqual(self.reboots, []) # deferred until the stated deadline - def test_waiting_until_capped_reboots(self): - # a runaway that stays idle past the cap still reboots ("max no matter what"), however far - # out its stated deadline is + def test_waiting_until_capped_reboots_when_no_build(self): + # a runaway that parked itself and is doing NOTHING still reboots at the cap ("max no matter + # what"), however far out its stated deadline is self.cfg["watchdog"]["waiting_until_max"] = 7200 self.patch("_build_running", lambda *a, **k: False) self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000) - self._set_idle(8000) # idle > cap(7200) + self._set_idle(8000) # idle > cap(7200), nothing running agents.stall_check_one(self.cfg, self.agent) - self.assertEqual(self.reboots, [self.agent["name"]]) # idle past cap → rebooted + self.assertEqual(self.reboots, [self.agent["name"]]) # idle past cap, no build → rebooted + + def test_waiting_until_cap_yields_to_a_live_build(self): + # cargo-mutants / coverage runs legitimately take hours: a build still running is proof of + # life, so the cap must NOT guillotine it — the stated deadline governs instead + self.cfg["watchdog"]["waiting_until_max"] = 7200 + self.patch("_build_running", lambda *a, **k: True) + self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000) + self._set_idle(8000) # idle > cap, but a build IS running + agents.stall_check_one(self.cfg, self.agent) + self.assertEqual(self.reboots, []) # survives the cap + + def test_waiting_until_past_deadline_reboots_even_with_a_build(self): + # the deadline is still the hard bound — a build running does not let it park forever + self.patch("_build_running", lambda *a, **k: True) + self.patch("_parse_waiting_until", lambda *a, **k: time.time() - 1000) # deadline long past + self._set_idle(3000) + agents.stall_check_one(self.cfg, self.agent) + self.assertEqual(self.reboots, [self.agent["name"]]) # past WAITING-UNTIL → rebooted def test_no_build_check_below_base_threshold(self): def boom(*a, **k):