watchdog: a live build is proof of life — don't let waiting_until_max guillotine a long legitimate run

Found in production: rust-mutation emitted WAITING-UNTIL 3h11m out for a cargo-mutants run over the whole
workspace (a genuinely multi-hour job, cargo running the whole time), but the marker branch skipped the
build-aware check entirely and went straight to the cap — so the 7200s cap would have killed a live run and
thrown away hours of work. The cap exists to catch a session that PARKED itself and is stuck; a build still
running under the session is proof it is not. Now the cap only fires when idle exceeds it AND no build is
running. The stated deadline remains the hard bound either way, so a runaway still cannot park forever.
Tests: cap-reboots-when-no-build, cap-yields-to-a-live-build, past-deadline-reboots-even-with-a-build. 68 pass.

agents.py

@@ -582,10 +582,14 @@ def stall_check_one(cfg, agent):
         # of killing it mid-run. Cap how far out it can push its own reboot, so a runaway can't park
         # itself forever ("some max no matter what").
         wu_max = int(cfg["watchdog"].get("waiting_until_max", 7200))
-        if wu_max and idle > wu_max:
-            # idle here ≈ how long the pane has sat quiet since the agent emitted the marker; once
-            # that exceeds the cap we reboot no matter how far out the stated deadline is.
-            reason = f"WAITING-UNTIL exceeded the {wu_max}s cap (idle {int(idle)}s) — rebooting regardless"
+        # The cap guards against a session that parked itself and is genuinely stuck. A build still
+        # running under the session (cargo-mutants / a coverage run / a remote ssh) is proof of life —
+        # those legitimately run for hours — so it defers to the agent's stated deadline instead of
+        # being guillotined by the cap. Past the deadline (+grace) it reboots either way, so a runaway
+        # can never park forever.
+        if wu_max and idle > wu_max and not _build_running(cfg, agent):
+            reason = (f"WAITING-UNTIL exceeded the {wu_max}s cap (idle {int(idle)}s, no build running) "
+                      f"— rebooting regardless")
         elif now <= until + grace:
             return
         else:

tests/test_unit.py

@@ -699,15 +699,33 @@ class TestBuildAwareStall(unittest.TestCase):
         agents.stall_check_one(self.cfg, self.agent)
         self.assertEqual(self.reboots, [])                     # deferred until the stated deadline
 
-    def test_waiting_until_capped_reboots(self):
-        # a runaway that stays idle past the cap still reboots ("max no matter what"), however far
-        # out its stated deadline is
+    def test_waiting_until_capped_reboots_when_no_build(self):
+        # a runaway that parked itself and is doing NOTHING still reboots at the cap ("max no matter
+        # what"), however far out its stated deadline is
         self.cfg["watchdog"]["waiting_until_max"] = 7200
         self.patch("_build_running", lambda *a, **k: False)
         self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000)
-        self._set_idle(8000)                                   # idle > cap(7200)
+        self._set_idle(8000)                                   # idle > cap(7200), nothing running
         agents.stall_check_one(self.cfg, self.agent)
-        self.assertEqual(self.reboots, [self.agent["name"]])   # idle past cap → rebooted
+        self.assertEqual(self.reboots, [self.agent["name"]])   # idle past cap, no build → rebooted
+
+    def test_waiting_until_cap_yields_to_a_live_build(self):
+        # cargo-mutants / coverage runs legitimately take hours: a build still running is proof of
+        # life, so the cap must NOT guillotine it — the stated deadline governs instead
+        self.cfg["watchdog"]["waiting_until_max"] = 7200
+        self.patch("_build_running", lambda *a, **k: True)
+        self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000)
+        self._set_idle(8000)                                   # idle > cap, but a build IS running
+        agents.stall_check_one(self.cfg, self.agent)
+        self.assertEqual(self.reboots, [])                     # survives the cap
+
+    def test_waiting_until_past_deadline_reboots_even_with_a_build(self):
+        # the deadline is still the hard bound — a build running does not let it park forever
+        self.patch("_build_running", lambda *a, **k: True)
+        self.patch("_parse_waiting_until", lambda *a, **k: time.time() - 1000)   # deadline long past
+        self._set_idle(3000)
+        agents.stall_check_one(self.cfg, self.agent)
+        self.assertEqual(self.reboots, [self.agent["name"]])   # past WAITING-UNTIL → rebooted
 
     def test_no_build_check_below_base_threshold(self):
         def boom(*a, **k):