watchdog: a live build is proof of life — don't let waiting_until_max guillotine a long legitimate run
Found in production: rust-mutation emitted WAITING-UNTIL 3h11m out for a cargo-mutants run over the whole workspace (a genuinely multi-hour job, cargo running the whole time), but the marker branch skipped the build-aware check entirely and went straight to the cap — so the 7200s cap would have killed a live run and thrown away hours of work. The cap exists to catch a session that PARKED itself and is stuck; a build still running under the session is proof it is not. Now the cap only fires when idle exceeds it AND no build is running. The stated deadline remains the hard bound either way, so a runaway still cannot park forever. Tests: cap-reboots-when-no-build, cap-yields-to-a-live-build, past-deadline-reboots-even-with-a-build. 68 pass.
This commit is contained in:
+23
-5
@@ -699,15 +699,33 @@ class TestBuildAwareStall(unittest.TestCase):
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, []) # deferred until the stated deadline
|
||||
|
||||
def test_waiting_until_capped_reboots(self):
|
||||
# a runaway that stays idle past the cap still reboots ("max no matter what"), however far
|
||||
# out its stated deadline is
|
||||
def test_waiting_until_capped_reboots_when_no_build(self):
|
||||
# a runaway that parked itself and is doing NOTHING still reboots at the cap ("max no matter
|
||||
# what"), however far out its stated deadline is
|
||||
self.cfg["watchdog"]["waiting_until_max"] = 7200
|
||||
self.patch("_build_running", lambda *a, **k: False)
|
||||
self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000)
|
||||
self._set_idle(8000) # idle > cap(7200)
|
||||
self._set_idle(8000) # idle > cap(7200), nothing running
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, [self.agent["name"]]) # idle past cap → rebooted
|
||||
self.assertEqual(self.reboots, [self.agent["name"]]) # idle past cap, no build → rebooted
|
||||
|
||||
def test_waiting_until_cap_yields_to_a_live_build(self):
|
||||
# cargo-mutants / coverage runs legitimately take hours: a build still running is proof of
|
||||
# life, so the cap must NOT guillotine it — the stated deadline governs instead
|
||||
self.cfg["watchdog"]["waiting_until_max"] = 7200
|
||||
self.patch("_build_running", lambda *a, **k: True)
|
||||
self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000)
|
||||
self._set_idle(8000) # idle > cap, but a build IS running
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, []) # survives the cap
|
||||
|
||||
def test_waiting_until_past_deadline_reboots_even_with_a_build(self):
|
||||
# the deadline is still the hard bound — a build running does not let it park forever
|
||||
self.patch("_build_running", lambda *a, **k: True)
|
||||
self.patch("_parse_waiting_until", lambda *a, **k: time.time() - 1000) # deadline long past
|
||||
self._set_idle(3000)
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, [self.agent["name"]]) # past WAITING-UNTIL → rebooted
|
||||
|
||||
def test_no_build_check_below_base_threshold(self):
|
||||
def boom(*a, **k):
|
||||
|
||||
Reference in New Issue
Block a user