watchdog: make WAITING-UNTIL work for footer_ui backends + cap runaway defers
_parse_waiting_until scanned only the pane's last non-empty line for footer_ui backends (claude/ opencode) — but their input-box footer always renders BELOW the agent's final message, so the marker was never seen and WAITING-UNTIL was effectively dead for claude agents. It's only consulted once the pane is already idle, so scan the whole capture and take the most-recent marker (the footer never contains it). Add waiting_until_max (default 7200s) so an agent can't park its own reboot forever. Tests: footer-honors-marker-above-footer, takes-most-recent, defer + cap in stall_check_one; make the stall harness's patch() idempotent so a re-patched name doesn't leak into tearDown. 66 pass.
This commit is contained in:
@@ -541,17 +541,16 @@ def _last_nonempty_line(text):
|
||||
return ""
|
||||
|
||||
def _parse_waiting_until(cfg, agent, pane):
|
||||
if backend_of(cfg, agent).get("footer_ui"):
|
||||
line = _last_nonempty_line(pane)
|
||||
if not line.startswith("WAITING-UNTIL:"):
|
||||
return None
|
||||
m = re.search(r"WAITING-UNTIL:\s*(\S+)", line)
|
||||
else:
|
||||
m = re.search(r"WAITING-UNTIL:\s*(\S+)", pane)
|
||||
if not m:
|
||||
# Only consulted once the pane is already idle (see stall_check_one), so scanning the whole
|
||||
# capture and taking the MOST-RECENT marker is safe — and it's the only thing that works for a
|
||||
# footer_ui backend (claude/opencode), whose input-box footer always renders BELOW the agent's
|
||||
# final message. The footer never contains the marker, so the last match is the agent's own
|
||||
# signal, whether or not a status footer follows it.
|
||||
matches = re.findall(r"WAITING-UNTIL:\s*(\S+)", pane)
|
||||
if not matches:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(m.group(1).replace("Z", "+00:00")).timestamp()
|
||||
return datetime.fromisoformat(matches[-1].replace("Z", "+00:00")).timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -578,9 +577,19 @@ def stall_check_one(cfg, agent):
|
||||
grace = int(cfg["watchdog"].get("stall_grace", 180))
|
||||
until = _parse_waiting_until(cfg, agent, pane)
|
||||
if until is not None:
|
||||
if now <= until + grace:
|
||||
# An agent that starts a long remote/async run (remote cargo build, terraform apply, VM
|
||||
# provision, long ssh) prints `WAITING-UNTIL: <ISO8601>` so the watchdog holds off instead
|
||||
# of killing it mid-run. Cap how far out it can push its own reboot, so a runaway can't park
|
||||
# itself forever ("some max no matter what").
|
||||
wu_max = int(cfg["watchdog"].get("waiting_until_max", 7200))
|
||||
if wu_max and idle > wu_max:
|
||||
# idle here ≈ how long the pane has sat quiet since the agent emitted the marker; once
|
||||
# that exceeds the cap we reboot no matter how far out the stated deadline is.
|
||||
reason = f"WAITING-UNTIL exceeded the {wu_max}s cap (idle {int(idle)}s) — rebooting regardless"
|
||||
elif now <= until + grace:
|
||||
return
|
||||
reason = f"past its WAITING-UNTIL by {int(now-until)}s — self-wake did not fire"
|
||||
else:
|
||||
reason = f"past its WAITING-UNTIL by {int(now-until)}s — self-wake did not fire"
|
||||
else:
|
||||
stall_idle = int(backend_of(cfg, agent).get("stall_idle", 300))
|
||||
if idle < stall_idle:
|
||||
|
||||
+31
-8
@@ -454,15 +454,19 @@ class TestWaitingUntil(unittest.TestCase):
|
||||
self.assertIsNone(agents._parse_waiting_until(
|
||||
self.cfg, self.claude_agent, "just working, no marker"))
|
||||
|
||||
def test_footer_requires_marker_as_last_line(self):
|
||||
# marker present but NOT the last non-empty line → ignored for a footer UI
|
||||
def test_footer_honors_marker_above_the_footer(self):
|
||||
# A footer_ui backend renders its input-box/status footer BELOW the agent's message, so the
|
||||
# marker is never the literal last line. It must still be honored (this is the real-claude case).
|
||||
pane = "WAITING-UNTIL: 2030-06-13T12:00:00Z\n ▣ Build · GPT · 2m 19s\n"
|
||||
self.assertIsNone(agents._parse_waiting_until(self.cfg, self.oc_agent, pane))
|
||||
|
||||
def test_footer_honors_marker_when_last_line(self):
|
||||
pane = "some work\nWAITING-UNTIL: 2030-06-13T12:00:00Z\n\n"
|
||||
ep = agents._parse_waiting_until(self.cfg, self.oc_agent, pane)
|
||||
self.assertIsNotNone(ep)
|
||||
self.assertEqual(ep, datetime.fromisoformat("2030-06-13T12:00:00+00:00").timestamp())
|
||||
|
||||
def test_takes_most_recent_marker(self):
|
||||
pane = ("WAITING-UNTIL: 2030-01-01T00:00:00Z\nwork\n"
|
||||
"WAITING-UNTIL: 2031-06-13T12:00:00Z\n footer\n")
|
||||
ep = agents._parse_waiting_until(self.cfg, self.oc_agent, pane)
|
||||
self.assertEqual(ep, datetime.fromisoformat("2031-06-13T12:00:00+00:00").timestamp())
|
||||
|
||||
def test_bad_timestamp_none(self):
|
||||
self.assertIsNone(agents._parse_waiting_until(
|
||||
@@ -644,8 +648,9 @@ class TestBuildAwareStall(unittest.TestCase):
|
||||
self.reboots = []
|
||||
self._orig = {}
|
||||
def patch(name, fn):
|
||||
self._orig[name] = getattr(agents, name)
|
||||
setattr(agents, name, fn)
|
||||
if name not in self._orig: # capture the TRUE original once, so a
|
||||
self._orig[name] = getattr(agents, name) # test that re-patches doesn't leak the
|
||||
setattr(agents, name, fn) # earlier patch into tearDown's restore
|
||||
patch("session_alive", lambda s: True)
|
||||
patch("capture_pane", lambda *a, **k: "")
|
||||
patch("limit_tick", lambda *a, **k: False)
|
||||
@@ -686,6 +691,24 @@ class TestBuildAwareStall(unittest.TestCase):
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, [self.agent["name"]]) # cap reached → reboot despite build
|
||||
|
||||
def test_waiting_until_defers_reboot(self):
|
||||
# agent signalled a remote run in progress → hold off even with no local build + long idle
|
||||
self.patch("_build_running", lambda *a, **k: False)
|
||||
self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 600)
|
||||
self._set_idle(3000)
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, []) # deferred until the stated deadline
|
||||
|
||||
def test_waiting_until_capped_reboots(self):
|
||||
# a runaway that stays idle past the cap still reboots ("max no matter what"), however far
|
||||
# out its stated deadline is
|
||||
self.cfg["watchdog"]["waiting_until_max"] = 7200
|
||||
self.patch("_build_running", lambda *a, **k: False)
|
||||
self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000)
|
||||
self._set_idle(8000) # idle > cap(7200)
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, [self.agent["name"]]) # idle past cap → rebooted
|
||||
|
||||
def test_no_build_check_below_base_threshold(self):
|
||||
def boom(*a, **k):
|
||||
raise AssertionError("_build_running must not be consulted below stall_idle")
|
||||
|
||||
Reference in New Issue
Block a user