From 582f392ef56c22e596b898a8c6080a65587e839c Mon Sep 17 00:00:00 2001 From: notplants-bot Date: Fri, 10 Jul 2026 20:39:51 +0000 Subject: [PATCH] watchdog: make WAITING-UNTIL work for footer_ui backends + cap runaway defers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit _parse_waiting_until scanned only the pane's last non-empty line for footer_ui backends (claude/ opencode) — but their input-box footer always renders BELOW the agent's final message, so the marker was never seen and WAITING-UNTIL was effectively dead for claude agents. It's only consulted once the pane is already idle, so scan the whole capture and take the most-recent marker (the footer never contains it). Add waiting_until_max (default 7200s) so an agent can't park its own reboot forever. Tests: footer-honors-marker-above-footer, takes-most-recent, defer + cap in stall_check_one; make the stall harness's patch() idempotent so a re-patched name doesn't leak into tearDown. 66 pass. --- agents.py | 31 ++++++++++++++++++++----------- tests/test_unit.py | 39 +++++++++++++++++++++++++++++++-------- 2 files changed, 51 insertions(+), 19 deletions(-) diff --git a/agents.py b/agents.py index 96ff151..23799a2 100755 --- a/agents.py +++ b/agents.py @@ -541,17 +541,16 @@ def _last_nonempty_line(text): return "" def _parse_waiting_until(cfg, agent, pane): - if backend_of(cfg, agent).get("footer_ui"): - line = _last_nonempty_line(pane) - if not line.startswith("WAITING-UNTIL:"): - return None - m = re.search(r"WAITING-UNTIL:\s*(\S+)", line) - else: - m = re.search(r"WAITING-UNTIL:\s*(\S+)", pane) - if not m: + # Only consulted once the pane is already idle (see stall_check_one), so scanning the whole + # capture and taking the MOST-RECENT marker is safe — and it's the only thing that works for a + # footer_ui backend (claude/opencode), whose input-box footer always renders BELOW the agent's + # final message. The footer never contains the marker, so the last match is the agent's own + # signal, whether or not a status footer follows it. + matches = re.findall(r"WAITING-UNTIL:\s*(\S+)", pane) + if not matches: return None try: - return datetime.fromisoformat(m.group(1).replace("Z", "+00:00")).timestamp() + return datetime.fromisoformat(matches[-1].replace("Z", "+00:00")).timestamp() except Exception: return None @@ -578,9 +577,19 @@ def stall_check_one(cfg, agent): grace = int(cfg["watchdog"].get("stall_grace", 180)) until = _parse_waiting_until(cfg, agent, pane) if until is not None: - if now <= until + grace: + # An agent that starts a long remote/async run (remote cargo build, terraform apply, VM + # provision, long ssh) prints `WAITING-UNTIL: ` so the watchdog holds off instead + # of killing it mid-run. Cap how far out it can push its own reboot, so a runaway can't park + # itself forever ("some max no matter what"). + wu_max = int(cfg["watchdog"].get("waiting_until_max", 7200)) + if wu_max and idle > wu_max: + # idle here ≈ how long the pane has sat quiet since the agent emitted the marker; once + # that exceeds the cap we reboot no matter how far out the stated deadline is. + reason = f"WAITING-UNTIL exceeded the {wu_max}s cap (idle {int(idle)}s) — rebooting regardless" + elif now <= until + grace: return - reason = f"past its WAITING-UNTIL by {int(now-until)}s — self-wake did not fire" + else: + reason = f"past its WAITING-UNTIL by {int(now-until)}s — self-wake did not fire" else: stall_idle = int(backend_of(cfg, agent).get("stall_idle", 300)) if idle < stall_idle: diff --git a/tests/test_unit.py b/tests/test_unit.py index 557496f..3d9718c 100755 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -454,15 +454,19 @@ class TestWaitingUntil(unittest.TestCase): self.assertIsNone(agents._parse_waiting_until( self.cfg, self.claude_agent, "just working, no marker")) - def test_footer_requires_marker_as_last_line(self): - # marker present but NOT the last non-empty line → ignored for a footer UI + def test_footer_honors_marker_above_the_footer(self): + # A footer_ui backend renders its input-box/status footer BELOW the agent's message, so the + # marker is never the literal last line. It must still be honored (this is the real-claude case). pane = "WAITING-UNTIL: 2030-06-13T12:00:00Z\n ▣ Build · GPT · 2m 19s\n" - self.assertIsNone(agents._parse_waiting_until(self.cfg, self.oc_agent, pane)) - - def test_footer_honors_marker_when_last_line(self): - pane = "some work\nWAITING-UNTIL: 2030-06-13T12:00:00Z\n\n" ep = agents._parse_waiting_until(self.cfg, self.oc_agent, pane) self.assertIsNotNone(ep) + self.assertEqual(ep, datetime.fromisoformat("2030-06-13T12:00:00+00:00").timestamp()) + + def test_takes_most_recent_marker(self): + pane = ("WAITING-UNTIL: 2030-01-01T00:00:00Z\nwork\n" + "WAITING-UNTIL: 2031-06-13T12:00:00Z\n footer\n") + ep = agents._parse_waiting_until(self.cfg, self.oc_agent, pane) + self.assertEqual(ep, datetime.fromisoformat("2031-06-13T12:00:00+00:00").timestamp()) def test_bad_timestamp_none(self): self.assertIsNone(agents._parse_waiting_until( @@ -644,8 +648,9 @@ class TestBuildAwareStall(unittest.TestCase): self.reboots = [] self._orig = {} def patch(name, fn): - self._orig[name] = getattr(agents, name) - setattr(agents, name, fn) + if name not in self._orig: # capture the TRUE original once, so a + self._orig[name] = getattr(agents, name) # test that re-patches doesn't leak the + setattr(agents, name, fn) # earlier patch into tearDown's restore patch("session_alive", lambda s: True) patch("capture_pane", lambda *a, **k: "") patch("limit_tick", lambda *a, **k: False) @@ -686,6 +691,24 @@ class TestBuildAwareStall(unittest.TestCase): agents.stall_check_one(self.cfg, self.agent) self.assertEqual(self.reboots, [self.agent["name"]]) # cap reached → reboot despite build + def test_waiting_until_defers_reboot(self): + # agent signalled a remote run in progress → hold off even with no local build + long idle + self.patch("_build_running", lambda *a, **k: False) + self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 600) + self._set_idle(3000) + agents.stall_check_one(self.cfg, self.agent) + self.assertEqual(self.reboots, []) # deferred until the stated deadline + + def test_waiting_until_capped_reboots(self): + # a runaway that stays idle past the cap still reboots ("max no matter what"), however far + # out its stated deadline is + self.cfg["watchdog"]["waiting_until_max"] = 7200 + self.patch("_build_running", lambda *a, **k: False) + self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000) + self._set_idle(8000) # idle > cap(7200) + agents.stall_check_one(self.cfg, self.agent) + self.assertEqual(self.reboots, [self.agent["name"]]) # idle past cap → rebooted + def test_no_build_check_below_base_threshold(self): def boom(*a, **k): raise AssertionError("_build_running must not be consulted below stall_idle")