feat(orchestrator): fold hourly supervision wake into the watchdog
The standalone ai-progress-monitor.sh waker pinged a hardcoded orchestrator session every 15m. Move that into the watchdog loop: ORCH_WAKE_INTERVAL (default 3600s) types the supervision prompt into the live orchestrator session, retrying each tick until it lands so a busy or briefly-absent orchestrator is never interrupted and no hour is skipped. Delete the now-redundant waker script; the prompt file is now driven by the watchdog. Reboot-safe by inheritance (the watchdog is started by cc-ci-loops.service). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Opus 4.8
parent
8f7265e948
commit
ca6e68c08d
@@ -26,6 +26,8 @@ Env (all optional — defaults shown):
|
||||
PHASE_IDX_FILE $LOG_DIR/.phase-idx
|
||||
WATCH_INTERVAL 300 (seconds between heavy checks: phase DONE / heal sessions)
|
||||
SIGNAL_INTERVAL 30 (seconds between handoff / stall checks)
|
||||
ORCH_WAKE_INTERVAL 3600 (seconds between supervision wakes typed into the orchestrator session)
|
||||
ORCH_WAKE_PROMPT $PLAN_DIR/ai-progress-monitor-prompt.txt (the supervision prompt)
|
||||
STALL_IDLE 300 (idle seconds without a WAITING-UNTIL before reboot)
|
||||
STALL_GRACE 180 (seconds past a WAITING-UNTIL before reboot)
|
||||
"""
|
||||
@@ -78,6 +80,11 @@ WATCHDOG_SESSION = "cc-ci-watchdog"
|
||||
|
||||
WATCH_INTERVAL = int(os.environ.get("WATCH_INTERVAL", 300))
|
||||
SIGNAL_INTERVAL = int(os.environ.get("SIGNAL_INTERVAL", 30))
|
||||
|
||||
# Hourly supervision wake: the watchdog types this prompt into the orchestrator session
|
||||
# so it reviews the loops and nudges as needed (replaces the standalone ai-progress-monitor waker).
|
||||
ORCH_WAKE_INTERVAL = int(os.environ.get("ORCH_WAKE_INTERVAL", 3600))
|
||||
ORCH_WAKE_PROMPT = os.environ.get("ORCH_WAKE_PROMPT", f"{PLAN_DIR}/ai-progress-monitor-prompt.txt")
|
||||
STALL_IDLE = int(os.environ.get("STALL_IDLE", 300))
|
||||
STALL_GRACE = int(os.environ.get("STALL_GRACE", 180))
|
||||
|
||||
@@ -438,6 +445,31 @@ def heal_orchestrator():
|
||||
log(f"orchestrator not running — restarting via {ORCH_LAUNCHER}")
|
||||
subprocess.run([ORCH_LAUNCHER, "start"], capture_output=True)
|
||||
|
||||
def wake_orchestrator():
|
||||
"""Hourly supervision nudge: type the progress-monitor prompt into the orchestrator
|
||||
session so it reviews the loops. Returns True when the wake was delivered (or is moot),
|
||||
False when it should be retried on a later tick.
|
||||
|
||||
Skips (retry later) if the orchestrator is absent — heal_orchestrator restarts it — or
|
||||
actively working, so we never interrupt a turn; the wake lands the moment it goes idle.
|
||||
"""
|
||||
if not WATCH_ORCHESTRATOR:
|
||||
return True # feature off — treat as handled so the timer doesn't spin
|
||||
if not session_alive(ORCH_SESSION):
|
||||
return False
|
||||
if ACTIVE_RE.search(capture_pane(ORCH_SESSION, 25)):
|
||||
return False # busy — don't interrupt; retry when idle
|
||||
try:
|
||||
msg = " ".join(Path(ORCH_WAKE_PROMPT).read_text().split())
|
||||
except FileNotFoundError:
|
||||
log(f"orchestrator wake skipped — prompt file missing: {ORCH_WAKE_PROMPT}")
|
||||
return True
|
||||
if not msg:
|
||||
return True
|
||||
log(f"waking orchestrator ({ORCH_SESSION}) for scheduled supervision pass")
|
||||
ping_session(ORCH_SESSION, msg, submit_key=_SUBMIT)
|
||||
return True
|
||||
|
||||
# ── handoff signalling ────────────────────────────────────────────────────────
|
||||
|
||||
_last_sha = ""
|
||||
@@ -529,10 +561,17 @@ def watchdog_loop():
|
||||
f"seq='{all_ids()}' signal={SIGNAL_INTERVAL}s heavy={WATCH_INTERVAL}s")
|
||||
|
||||
elapsed = WATCH_INTERVAL # force a heavy check on the first tick
|
||||
wake_elapsed = 0 # first orchestrator wake fires after a full interval, not at startup
|
||||
while True:
|
||||
handoff_check()
|
||||
stall_check()
|
||||
|
||||
if wake_elapsed >= ORCH_WAKE_INTERVAL:
|
||||
# Reset only once the wake actually lands; if the orchestrator is busy/absent,
|
||||
# leave the timer tripped so we retry each tick until it's idle.
|
||||
if wake_orchestrator():
|
||||
wake_elapsed = 0
|
||||
|
||||
if elapsed >= WATCH_INTERVAL:
|
||||
elapsed = 0
|
||||
idx = cur_idx()
|
||||
@@ -563,6 +602,7 @@ def watchdog_loop():
|
||||
|
||||
time.sleep(SIGNAL_INTERVAL)
|
||||
elapsed += SIGNAL_INTERVAL
|
||||
wake_elapsed += SIGNAL_INTERVAL
|
||||
|
||||
def start_watchdog():
|
||||
if session_alive(WATCHDOG_SESSION):
|
||||
|
||||
Reference in New Issue
Block a user