watchdog: reboot idle-wedged loops via self-reported WAITING-UNTIL markers
The builder wedged at the context limit (garbled output) — alive but matching none of heal_session's signals (dead/FATAL/limit), so the watchdog left it stuck. Fix: loops now declare every wait, and the watchdog reboots a wait that never resumes. - plan.md §7 + both prompts: cap every wait at 10 min (chunk longer waits); before going idle, the loop's FINAL line must be `WAITING-UNTIL: <ISO8601 UTC>` (the resume time, matching its ScheduleWakeup); run /compact proactively at ~80% context to avoid wedging near the limit. - launch.sh: new stall_check (runs every 30s signal tick) — reboots a loop idle >= STALL_IDLE (300s) when it has NO current WAITING-UNTIL marker as its last message OR is past the time the marker named; a healthy paced wait (marker present, before its time) is left alone. Complements heal_session's dead/FATAL/limit cases. Reboot is safe — loops re-orient from git + STATUS. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -42,6 +42,8 @@ LOG_DIR="${LOG_DIR:-/srv/cc-ci/.cc-ci-logs}"
|
||||
|
||||
WATCH_INTERVAL="${WATCH_INTERVAL:-300}" # seconds between HEAVY checks (phase DONE / restart dead loops)
|
||||
SIGNAL_INTERVAL="${SIGNAL_INTERVAL:-30}" # seconds between HANDOFF checks (ping the waiting loop)
|
||||
STALL_IDLE="${STALL_IDLE:-300}" # seconds a loop may sit idle past its WAITING-UNTIL marker
|
||||
# (or with no marker at all) before the watchdog reboots it
|
||||
|
||||
BUILDER_SESSION="cc-ci-builder"
|
||||
ADV_SESSION="cc-ci-adv"
|
||||
@@ -178,6 +180,52 @@ heal_session() {
|
||||
fi
|
||||
}
|
||||
|
||||
# --- Idle-wedge detection (complements heal_session's dead/FATAL/limit cases) ----------------------
|
||||
# A loop can sit ALIVE but wedged — e.g. garbled output at the context limit — showing none of the
|
||||
# heal_session signals (not dead, no FATAL string, no limit notice). The loops therefore DECLARE every
|
||||
# wait with a final-line marker `WAITING-UNTIL: <ISO-8601 UTC>` and cap each wait at 10 min (plan §7).
|
||||
# A healthy idle loop ALWAYS has a current marker as its last message; a wedge does not (or has one
|
||||
# whose time has already passed). So: reboot a loop that has been idle (no "esc to interrupt") for
|
||||
# >= STALL_IDLE seconds AND (has no WAITING-UNTIL marker OR is now past the time that marker named).
|
||||
# Runs every signal tick (30 s) for fine resolution; rebooting is safe — the loop re-orients from
|
||||
# git + its phase STATUS/REVIEW files.
|
||||
declare -A _wd_idle_since # session -> epoch first seen idle this stretch (0/unset = working)
|
||||
|
||||
_parse_waiting_until() { # arg1 = pane text; echoes epoch seconds of the last marker, or nothing
|
||||
local line ts
|
||||
line="$(printf '%s\n' "$1" | grep -oE 'WAITING-UNTIL:[[:space:]]*[0-9][0-9T:Z+-]+' | tail -1)"
|
||||
[[ -n "$line" ]] || return 0
|
||||
ts="$(printf '%s' "${line#WAITING-UNTIL:}" | tr -d '[:space:]')"
|
||||
date -u -d "$ts" +%s 2>/dev/null || true
|
||||
}
|
||||
|
||||
stall_check_one() {
|
||||
local role="$1" s="$2" dir="$3" pane now until idle since
|
||||
session_alive "$s" || { _wd_idle_since[$s]=0; return 0; } # dead => heal_session handles it
|
||||
now="$(printf '%(%s)T' -1)"
|
||||
pane="$(tmux capture-pane -pt "$s" 2>/dev/null | tail -40 || true)"
|
||||
if printf '%s\n' "$pane" | grep -q 'esc to interrupt'; then
|
||||
_wd_idle_since[$s]=0; return 0 # actively working — not idle
|
||||
fi
|
||||
since="${_wd_idle_since[$s]:-0}"
|
||||
if [[ "$since" == 0 ]]; then since="$now"; _wd_idle_since[$s]="$now"; fi
|
||||
idle=$(( now - since ))
|
||||
(( idle >= STALL_IDLE )) || return 0
|
||||
until="$(_parse_waiting_until "$pane")"
|
||||
if [[ -n "$until" ]] && (( now < until )); then
|
||||
return 0 # legitimately waiting, before its time
|
||||
fi
|
||||
log "stall: $role ($s) idle ${idle}s, $([[ -n "$until" ]] && echo "past its WAITING-UNTIL" || echo "no WAITING-UNTIL marker") — kill + reboot (re-orients from repo)"
|
||||
tmux kill-session -t "$s" 2>/dev/null || true
|
||||
start_agent "$role" "$s" "$dir"
|
||||
_wd_idle_since[$s]=0
|
||||
}
|
||||
|
||||
stall_check() {
|
||||
stall_check_one builder "$BUILDER_SESSION" "$BUILDER_DIR"
|
||||
stall_check_one adversary "$ADV_SESSION" "$ADV_DIR"
|
||||
}
|
||||
|
||||
# Is an orchestrator process alive ANYWHERE? Conflict-safety: we must NEVER launch a second
|
||||
# orchestrator that resumes the same conversation while one is already running (that double-resume is
|
||||
# the likely cause of the "thinking blocks cannot be modified" crashes). The orchestrator may be
|
||||
@@ -289,6 +337,7 @@ watchdog_loop() {
|
||||
local elapsed="$WATCH_INTERVAL"
|
||||
while true; do
|
||||
handoff_check
|
||||
stall_check
|
||||
if (( elapsed >= WATCH_INTERVAL )); then
|
||||
elapsed=0
|
||||
idx="$(cur_idx)"; pid="$(phase_id "$idx")"; status="$(phase_status "$idx")"
|
||||
|
||||
Reference in New Issue
Block a user