watchdog: make WAITING-UNTIL work for footer_ui backends + cap runaway defers
_parse_waiting_until scanned only the pane's last non-empty line for footer_ui backends (claude/ opencode) — but their input-box footer always renders BELOW the agent's final message, so the marker was never seen and WAITING-UNTIL was effectively dead for claude agents. It's only consulted once the pane is already idle, so scan the whole capture and take the most-recent marker (the footer never contains it). Add waiting_until_max (default 7200s) so an agent can't park its own reboot forever. Tests: footer-honors-marker-above-footer, takes-most-recent, defer + cap in stall_check_one; make the stall harness's patch() idempotent so a re-patched name doesn't leak into tearDown. 66 pass.
This commit is contained in:
@@ -541,17 +541,16 @@ def _last_nonempty_line(text):
|
|||||||
return ""
|
return ""
|
||||||
|
|
||||||
def _parse_waiting_until(cfg, agent, pane):
|
def _parse_waiting_until(cfg, agent, pane):
|
||||||
if backend_of(cfg, agent).get("footer_ui"):
|
# Only consulted once the pane is already idle (see stall_check_one), so scanning the whole
|
||||||
line = _last_nonempty_line(pane)
|
# capture and taking the MOST-RECENT marker is safe — and it's the only thing that works for a
|
||||||
if not line.startswith("WAITING-UNTIL:"):
|
# footer_ui backend (claude/opencode), whose input-box footer always renders BELOW the agent's
|
||||||
return None
|
# final message. The footer never contains the marker, so the last match is the agent's own
|
||||||
m = re.search(r"WAITING-UNTIL:\s*(\S+)", line)
|
# signal, whether or not a status footer follows it.
|
||||||
else:
|
matches = re.findall(r"WAITING-UNTIL:\s*(\S+)", pane)
|
||||||
m = re.search(r"WAITING-UNTIL:\s*(\S+)", pane)
|
if not matches:
|
||||||
if not m:
|
|
||||||
return None
|
return None
|
||||||
try:
|
try:
|
||||||
return datetime.fromisoformat(m.group(1).replace("Z", "+00:00")).timestamp()
|
return datetime.fromisoformat(matches[-1].replace("Z", "+00:00")).timestamp()
|
||||||
except Exception:
|
except Exception:
|
||||||
return None
|
return None
|
||||||
|
|
||||||
@@ -578,9 +577,19 @@ def stall_check_one(cfg, agent):
|
|||||||
grace = int(cfg["watchdog"].get("stall_grace", 180))
|
grace = int(cfg["watchdog"].get("stall_grace", 180))
|
||||||
until = _parse_waiting_until(cfg, agent, pane)
|
until = _parse_waiting_until(cfg, agent, pane)
|
||||||
if until is not None:
|
if until is not None:
|
||||||
if now <= until + grace:
|
# An agent that starts a long remote/async run (remote cargo build, terraform apply, VM
|
||||||
|
# provision, long ssh) prints `WAITING-UNTIL: <ISO8601>` so the watchdog holds off instead
|
||||||
|
# of killing it mid-run. Cap how far out it can push its own reboot, so a runaway can't park
|
||||||
|
# itself forever ("some max no matter what").
|
||||||
|
wu_max = int(cfg["watchdog"].get("waiting_until_max", 7200))
|
||||||
|
if wu_max and idle > wu_max:
|
||||||
|
# idle here ≈ how long the pane has sat quiet since the agent emitted the marker; once
|
||||||
|
# that exceeds the cap we reboot no matter how far out the stated deadline is.
|
||||||
|
reason = f"WAITING-UNTIL exceeded the {wu_max}s cap (idle {int(idle)}s) — rebooting regardless"
|
||||||
|
elif now <= until + grace:
|
||||||
return
|
return
|
||||||
reason = f"past its WAITING-UNTIL by {int(now-until)}s — self-wake did not fire"
|
else:
|
||||||
|
reason = f"past its WAITING-UNTIL by {int(now-until)}s — self-wake did not fire"
|
||||||
else:
|
else:
|
||||||
stall_idle = int(backend_of(cfg, agent).get("stall_idle", 300))
|
stall_idle = int(backend_of(cfg, agent).get("stall_idle", 300))
|
||||||
if idle < stall_idle:
|
if idle < stall_idle:
|
||||||
|
|||||||
+31
-8
@@ -454,15 +454,19 @@ class TestWaitingUntil(unittest.TestCase):
|
|||||||
self.assertIsNone(agents._parse_waiting_until(
|
self.assertIsNone(agents._parse_waiting_until(
|
||||||
self.cfg, self.claude_agent, "just working, no marker"))
|
self.cfg, self.claude_agent, "just working, no marker"))
|
||||||
|
|
||||||
def test_footer_requires_marker_as_last_line(self):
|
def test_footer_honors_marker_above_the_footer(self):
|
||||||
# marker present but NOT the last non-empty line → ignored for a footer UI
|
# A footer_ui backend renders its input-box/status footer BELOW the agent's message, so the
|
||||||
|
# marker is never the literal last line. It must still be honored (this is the real-claude case).
|
||||||
pane = "WAITING-UNTIL: 2030-06-13T12:00:00Z\n ▣ Build · GPT · 2m 19s\n"
|
pane = "WAITING-UNTIL: 2030-06-13T12:00:00Z\n ▣ Build · GPT · 2m 19s\n"
|
||||||
self.assertIsNone(agents._parse_waiting_until(self.cfg, self.oc_agent, pane))
|
|
||||||
|
|
||||||
def test_footer_honors_marker_when_last_line(self):
|
|
||||||
pane = "some work\nWAITING-UNTIL: 2030-06-13T12:00:00Z\n\n"
|
|
||||||
ep = agents._parse_waiting_until(self.cfg, self.oc_agent, pane)
|
ep = agents._parse_waiting_until(self.cfg, self.oc_agent, pane)
|
||||||
self.assertIsNotNone(ep)
|
self.assertIsNotNone(ep)
|
||||||
|
self.assertEqual(ep, datetime.fromisoformat("2030-06-13T12:00:00+00:00").timestamp())
|
||||||
|
|
||||||
|
def test_takes_most_recent_marker(self):
|
||||||
|
pane = ("WAITING-UNTIL: 2030-01-01T00:00:00Z\nwork\n"
|
||||||
|
"WAITING-UNTIL: 2031-06-13T12:00:00Z\n footer\n")
|
||||||
|
ep = agents._parse_waiting_until(self.cfg, self.oc_agent, pane)
|
||||||
|
self.assertEqual(ep, datetime.fromisoformat("2031-06-13T12:00:00+00:00").timestamp())
|
||||||
|
|
||||||
def test_bad_timestamp_none(self):
|
def test_bad_timestamp_none(self):
|
||||||
self.assertIsNone(agents._parse_waiting_until(
|
self.assertIsNone(agents._parse_waiting_until(
|
||||||
@@ -644,8 +648,9 @@ class TestBuildAwareStall(unittest.TestCase):
|
|||||||
self.reboots = []
|
self.reboots = []
|
||||||
self._orig = {}
|
self._orig = {}
|
||||||
def patch(name, fn):
|
def patch(name, fn):
|
||||||
self._orig[name] = getattr(agents, name)
|
if name not in self._orig: # capture the TRUE original once, so a
|
||||||
setattr(agents, name, fn)
|
self._orig[name] = getattr(agents, name) # test that re-patches doesn't leak the
|
||||||
|
setattr(agents, name, fn) # earlier patch into tearDown's restore
|
||||||
patch("session_alive", lambda s: True)
|
patch("session_alive", lambda s: True)
|
||||||
patch("capture_pane", lambda *a, **k: "")
|
patch("capture_pane", lambda *a, **k: "")
|
||||||
patch("limit_tick", lambda *a, **k: False)
|
patch("limit_tick", lambda *a, **k: False)
|
||||||
@@ -686,6 +691,24 @@ class TestBuildAwareStall(unittest.TestCase):
|
|||||||
agents.stall_check_one(self.cfg, self.agent)
|
agents.stall_check_one(self.cfg, self.agent)
|
||||||
self.assertEqual(self.reboots, [self.agent["name"]]) # cap reached → reboot despite build
|
self.assertEqual(self.reboots, [self.agent["name"]]) # cap reached → reboot despite build
|
||||||
|
|
||||||
|
def test_waiting_until_defers_reboot(self):
|
||||||
|
# agent signalled a remote run in progress → hold off even with no local build + long idle
|
||||||
|
self.patch("_build_running", lambda *a, **k: False)
|
||||||
|
self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 600)
|
||||||
|
self._set_idle(3000)
|
||||||
|
agents.stall_check_one(self.cfg, self.agent)
|
||||||
|
self.assertEqual(self.reboots, []) # deferred until the stated deadline
|
||||||
|
|
||||||
|
def test_waiting_until_capped_reboots(self):
|
||||||
|
# a runaway that stays idle past the cap still reboots ("max no matter what"), however far
|
||||||
|
# out its stated deadline is
|
||||||
|
self.cfg["watchdog"]["waiting_until_max"] = 7200
|
||||||
|
self.patch("_build_running", lambda *a, **k: False)
|
||||||
|
self.patch("_parse_waiting_until", lambda *a, **k: time.time() + 100000)
|
||||||
|
self._set_idle(8000) # idle > cap(7200)
|
||||||
|
agents.stall_check_one(self.cfg, self.agent)
|
||||||
|
self.assertEqual(self.reboots, [self.agent["name"]]) # idle past cap → rebooted
|
||||||
|
|
||||||
def test_no_build_check_below_base_threshold(self):
|
def test_no_build_check_below_base_threshold(self):
|
||||||
def boom(*a, **k):
|
def boom(*a, **k):
|
||||||
raise AssertionError("_build_running must not be consulted below stall_idle")
|
raise AssertionError("_build_running must not be consulted below stall_idle")
|
||||||
|
|||||||
Reference in New Issue
Block a user