watchdog: make WAITING-UNTIL work for footer_ui backends + cap runaway defers
_parse_waiting_until scanned only the pane's last non-empty line for footer_ui backends (claude/ opencode) — but their input-box footer always renders BELOW the agent's final message, so the marker was never seen and WAITING-UNTIL was effectively dead for claude agents. It's only consulted once the pane is already idle, so scan the whole capture and take the most-recent marker (the footer never contains it). Add waiting_until_max (default 7200s) so an agent can't park its own reboot forever. Tests: footer-honors-marker-above-footer, takes-most-recent, defer + cap in stall_check_one; make the stall harness's patch() idempotent so a re-patched name doesn't leak into tearDown. 66 pass.
This commit is contained in:
@@ -541,17 +541,16 @@ def _last_nonempty_line(text):
|
||||
return ""
|
||||
|
||||
def _parse_waiting_until(cfg, agent, pane):
|
||||
if backend_of(cfg, agent).get("footer_ui"):
|
||||
line = _last_nonempty_line(pane)
|
||||
if not line.startswith("WAITING-UNTIL:"):
|
||||
return None
|
||||
m = re.search(r"WAITING-UNTIL:\s*(\S+)", line)
|
||||
else:
|
||||
m = re.search(r"WAITING-UNTIL:\s*(\S+)", pane)
|
||||
if not m:
|
||||
# Only consulted once the pane is already idle (see stall_check_one), so scanning the whole
|
||||
# capture and taking the MOST-RECENT marker is safe — and it's the only thing that works for a
|
||||
# footer_ui backend (claude/opencode), whose input-box footer always renders BELOW the agent's
|
||||
# final message. The footer never contains the marker, so the last match is the agent's own
|
||||
# signal, whether or not a status footer follows it.
|
||||
matches = re.findall(r"WAITING-UNTIL:\s*(\S+)", pane)
|
||||
if not matches:
|
||||
return None
|
||||
try:
|
||||
return datetime.fromisoformat(m.group(1).replace("Z", "+00:00")).timestamp()
|
||||
return datetime.fromisoformat(matches[-1].replace("Z", "+00:00")).timestamp()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -578,9 +577,19 @@ def stall_check_one(cfg, agent):
|
||||
grace = int(cfg["watchdog"].get("stall_grace", 180))
|
||||
until = _parse_waiting_until(cfg, agent, pane)
|
||||
if until is not None:
|
||||
if now <= until + grace:
|
||||
# An agent that starts a long remote/async run (remote cargo build, terraform apply, VM
|
||||
# provision, long ssh) prints `WAITING-UNTIL: <ISO8601>` so the watchdog holds off instead
|
||||
# of killing it mid-run. Cap how far out it can push its own reboot, so a runaway can't park
|
||||
# itself forever ("some max no matter what").
|
||||
wu_max = int(cfg["watchdog"].get("waiting_until_max", 7200))
|
||||
if wu_max and idle > wu_max:
|
||||
# idle here ≈ how long the pane has sat quiet since the agent emitted the marker; once
|
||||
# that exceeds the cap we reboot no matter how far out the stated deadline is.
|
||||
reason = f"WAITING-UNTIL exceeded the {wu_max}s cap (idle {int(idle)}s) — rebooting regardless"
|
||||
elif now <= until + grace:
|
||||
return
|
||||
reason = f"past its WAITING-UNTIL by {int(now-until)}s — self-wake did not fire"
|
||||
else:
|
||||
reason = f"past its WAITING-UNTIL by {int(now-until)}s — self-wake did not fire"
|
||||
else:
|
||||
stall_idle = int(backend_of(cfg, agent).get("stall_idle", 300))
|
||||
if idle < stall_idle:
|
||||
|
||||
Reference in New Issue
Block a user