watchdog: make WAITING-UNTIL work for footer_ui backends + cap runaway defers

_parse_waiting_until scanned only the pane's last non-empty line for footer_ui backends (claude/
opencode) — but their input-box footer always renders BELOW the agent's final message, so the marker
was never seen and WAITING-UNTIL was effectively dead for claude agents. It's only consulted once the
pane is already idle, so scan the whole capture and take the most-recent marker (the footer never
contains it). Add waiting_until_max (default 7200s) so an agent can't park its own reboot forever.
Tests: footer-honors-marker-above-footer, takes-most-recent, defer + cap in stall_check_one; make the
stall harness's patch() idempotent so a re-patched name doesn't leak into tearDown. 66 pass.
This commit is contained in:
2026-07-10 20:39:51 +00:00
parent c591f5430d
commit 582f392ef5
2 changed files with 51 additions and 19 deletions
+20 -11
View File
@@ -541,17 +541,16 @@ def _last_nonempty_line(text):
return ""
def _parse_waiting_until(cfg, agent, pane):
if backend_of(cfg, agent).get("footer_ui"):
line = _last_nonempty_line(pane)
if not line.startswith("WAITING-UNTIL:"):
return None
m = re.search(r"WAITING-UNTIL:\s*(\S+)", line)
else:
m = re.search(r"WAITING-UNTIL:\s*(\S+)", pane)
if not m:
# Only consulted once the pane is already idle (see stall_check_one), so scanning the whole
# capture and taking the MOST-RECENT marker is safe — and it's the only thing that works for a
# footer_ui backend (claude/opencode), whose input-box footer always renders BELOW the agent's
# final message. The footer never contains the marker, so the last match is the agent's own
# signal, whether or not a status footer follows it.
matches = re.findall(r"WAITING-UNTIL:\s*(\S+)", pane)
if not matches:
return None
try:
return datetime.fromisoformat(m.group(1).replace("Z", "+00:00")).timestamp()
return datetime.fromisoformat(matches[-1].replace("Z", "+00:00")).timestamp()
except Exception:
return None
@@ -578,9 +577,19 @@ def stall_check_one(cfg, agent):
grace = int(cfg["watchdog"].get("stall_grace", 180))
until = _parse_waiting_until(cfg, agent, pane)
if until is not None:
if now <= until + grace:
# An agent that starts a long remote/async run (remote cargo build, terraform apply, VM
# provision, long ssh) prints `WAITING-UNTIL: <ISO8601>` so the watchdog holds off instead
# of killing it mid-run. Cap how far out it can push its own reboot, so a runaway can't park
# itself forever ("some max no matter what").
wu_max = int(cfg["watchdog"].get("waiting_until_max", 7200))
if wu_max and idle > wu_max:
# idle here ≈ how long the pane has sat quiet since the agent emitted the marker; once
# that exceeds the cap we reboot no matter how far out the stated deadline is.
reason = f"WAITING-UNTIL exceeded the {wu_max}s cap (idle {int(idle)}s) — rebooting regardless"
elif now <= until + grace:
return
reason = f"past its WAITING-UNTIL by {int(now-until)}s — self-wake did not fire"
else:
reason = f"past its WAITING-UNTIL by {int(now-until)}s — self-wake did not fire"
else:
stall_idle = int(backend_of(cfg, agent).get("stall_idle", 300))
if idle < stall_idle: