feat(orchestrator): fold hourly supervision wake into the watchdog

The standalone ai-progress-monitor.sh waker pinged a hardcoded
orchestrator session every 15m. Move that into the watchdog loop:
ORCH_WAKE_INTERVAL (default 3600s) types the supervision prompt into
the live orchestrator session, retrying each tick until it lands so a
busy or briefly-absent orchestrator is never interrupted and no hour is
skipped. Delete the now-redundant waker script; the prompt file is now
driven by the watchdog. Reboot-safe by inheritance (the watchdog is
started by cc-ci-loops.service).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
autonomic-bot
2026-06-01 21:46:20 +00:00
co-authored by Claude Opus 4.8
parent 8f7265e948
commit ca6e68c08d
4 changed files with 60 additions and 54 deletions
+40
View File
@@ -26,6 +26,8 @@ Env (all optional — defaults shown):
PHASE_IDX_FILE $LOG_DIR/.phase-idx
WATCH_INTERVAL 300 (seconds between heavy checks: phase DONE / heal sessions)
SIGNAL_INTERVAL 30 (seconds between handoff / stall checks)
ORCH_WAKE_INTERVAL 3600 (seconds between supervision wakes typed into the orchestrator session)
ORCH_WAKE_PROMPT $PLAN_DIR/ai-progress-monitor-prompt.txt (the supervision prompt)
STALL_IDLE 300 (idle seconds without a WAITING-UNTIL before reboot)
STALL_GRACE 180 (seconds past a WAITING-UNTIL before reboot)
"""
@@ -78,6 +80,11 @@ WATCHDOG_SESSION = "cc-ci-watchdog"
WATCH_INTERVAL = int(os.environ.get("WATCH_INTERVAL", 300))
SIGNAL_INTERVAL = int(os.environ.get("SIGNAL_INTERVAL", 30))
# Hourly supervision wake: the watchdog types this prompt into the orchestrator session
# so it reviews the loops and nudges as needed (replaces the standalone ai-progress-monitor waker).
ORCH_WAKE_INTERVAL = int(os.environ.get("ORCH_WAKE_INTERVAL", 3600))
ORCH_WAKE_PROMPT = os.environ.get("ORCH_WAKE_PROMPT", f"{PLAN_DIR}/ai-progress-monitor-prompt.txt")
STALL_IDLE = int(os.environ.get("STALL_IDLE", 300))
STALL_GRACE = int(os.environ.get("STALL_GRACE", 180))
@@ -438,6 +445,31 @@ def heal_orchestrator():
log(f"orchestrator not running — restarting via {ORCH_LAUNCHER}")
subprocess.run([ORCH_LAUNCHER, "start"], capture_output=True)
def wake_orchestrator():
"""Hourly supervision nudge: type the progress-monitor prompt into the orchestrator
session so it reviews the loops. Returns True when the wake was delivered (or is moot),
False when it should be retried on a later tick.
Skips (retry later) if the orchestrator is absent — heal_orchestrator restarts it — or
actively working, so we never interrupt a turn; the wake lands the moment it goes idle.
"""
if not WATCH_ORCHESTRATOR:
return True # feature off — treat as handled so the timer doesn't spin
if not session_alive(ORCH_SESSION):
return False
if ACTIVE_RE.search(capture_pane(ORCH_SESSION, 25)):
return False # busy — don't interrupt; retry when idle
try:
msg = " ".join(Path(ORCH_WAKE_PROMPT).read_text().split())
except FileNotFoundError:
log(f"orchestrator wake skipped — prompt file missing: {ORCH_WAKE_PROMPT}")
return True
if not msg:
return True
log(f"waking orchestrator ({ORCH_SESSION}) for scheduled supervision pass")
ping_session(ORCH_SESSION, msg, submit_key=_SUBMIT)
return True
# ── handoff signalling ────────────────────────────────────────────────────────
_last_sha = ""
@@ -529,10 +561,17 @@ def watchdog_loop():
f"seq='{all_ids()}' signal={SIGNAL_INTERVAL}s heavy={WATCH_INTERVAL}s")
elapsed = WATCH_INTERVAL # force a heavy check on the first tick
wake_elapsed = 0 # first orchestrator wake fires after a full interval, not at startup
while True:
handoff_check()
stall_check()
if wake_elapsed >= ORCH_WAKE_INTERVAL:
# Reset only once the wake actually lands; if the orchestrator is busy/absent,
# leave the timer tripped so we retry each tick until it's idle.
if wake_orchestrator():
wake_elapsed = 0
if elapsed >= WATCH_INTERVAL:
elapsed = 0
idx = cur_idx()
@@ -563,6 +602,7 @@ def watchdog_loop():
time.sleep(SIGNAL_INTERVAL)
elapsed += SIGNAL_INTERVAL
wake_elapsed += SIGNAL_INTERVAL
def start_watchdog():
if session_alive(WATCHDOG_SESSION):