orchestrator: reboot-resilience + session auto-resume + full session plan/tooling
Reboot survival for the Pi orchestrator host: - systemd unit cc-ci-plan/systemd/cc-ci-loops.service (installed + enabled): on boot records the reboot, starts loops+watchdog (RESUME_PHASE=1), and resumes the orchestrator session. - reboot-log.sh: boot_id-gated reboot record -> REBOOTS.md (manual restarts don't count). - launch-orchestrator.sh: injects an AGENTS.md startup nudge so an auto-resumed orchestrator announces itself (PushNotification) + reports reboots. - AGENTS.md: on-startup notify routine documented. Plans/tooling accumulated this session: - plan-phase1d (generic suite), 1e (harness corrections), phase4 (final review), sso-dep-testing, orchestrator-migration (parked), test-e2e-testme-acceptance. - launch.sh: 1d/1e/2/2b/3/4 phase sequence, machine-docs-aware state resolution, limit-stall re-nudge, INBOX side-channel detection. - plan.md §6.1/§7: artifact-layer isolation, INBOX, 5-min long-run polling, DEFERRED. - prompts: isolation discipline + INBOX + pacing. - .gitignore: harden (.sops/, cc-ci-secrets/, .claude/, *.tmp.*). Co-Authored-By: Claude Opus 4.8 (1M context) <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
+59
-11
@@ -7,10 +7,10 @@
|
||||
# • Adversary (tmux session: cc-ci-adv) working clone /srv/cc-ci/cc-ci-adv
|
||||
# coordinating only through the git repo on git.autonomic.zone.
|
||||
#
|
||||
# PHASES: the watchdog runs an ordered sequence of sub-phases (default: 1c then 1b). Each phase
|
||||
# has its own plan + phase-namespaced loop-state files (STATUS-<id>.md etc.). When a phase's
|
||||
# PHASES: the watchdog runs an ordered sequence of sub-phases (default: 1c → 1b → 1d → 1e → 2 → 2b → 3 → 4).
|
||||
# Each phase has its own plan + phase-namespaced loop-state files (STATUS-<id>.md etc.). When a phase's
|
||||
# STATUS-<id>.md shows "## DONE", the watchdog AUTO-TRANSITIONS to the next phase; after the LAST
|
||||
# phase it STOPS the loops and exits (a manual gate — e.g. check in before Phase 2).
|
||||
# phase (4, final review/polish/cleanup) it STOPS the loops and exits (end of the whole build).
|
||||
#
|
||||
# Three jobs: ITERATION (each agent's /loop), RESILIENCE (restart a dead loop), HANDOFF SIGNALLING
|
||||
# (ping the waiting loop the moment its counterpart hands off), PHASE SEQUENCING (this file).
|
||||
@@ -49,7 +49,7 @@ WATCHDOG_SESSION="cc-ci-watchdog"
|
||||
# Ordered phase sequence: each entry "id|planfile|statusbasename". The watchdog runs them in order,
|
||||
# auto-transitions on the phase's "## DONE" (in BUILDER_DIR/<statusbasename>), and STOPS after the
|
||||
# last one (manual gate). Override PHASES_SPEC (semicolon-separated) to change the sequence.
|
||||
PHASES_SPEC="${PHASES_SPEC:-1c|plan-phase1c-full-reproducibility.md|STATUS-1c.md;1b|plan-phase1b-review-lint.md|STATUS-1b.md}"
|
||||
PHASES_SPEC="${PHASES_SPEC:-1c|plan-phase1c-full-reproducibility.md|STATUS-1c.md;1b|plan-phase1b-review-lint.md|STATUS-1b.md;1d|plan-phase1d-generic-test-suite.md|STATUS-1d.md;1e|plan-phase1e-harness-corrections.md|STATUS-1e.md;2|plan-phase2-recipe-tests.md|STATUS-2.md;2b|plan-phase2b-test-performance.md|STATUS-2b.md;3|plan-phase3-results-ux.md|STATUS-3.md;4|plan-phase4-final-review-polish-cleanup.md|STATUS-4.md}"
|
||||
IFS=';' read -r -a PHASES <<< "$PHASES_SPEC"
|
||||
PHASE_IDX_FILE="${PHASE_IDX_FILE:-$LOG_DIR/.phase-idx}"
|
||||
# --------------------------------------------------------------------------
|
||||
@@ -64,7 +64,10 @@ phase_id() { echo "${PHASES[$1]}" | cut -d'|' -f1; }
|
||||
phase_plan() { echo "${PHASES[$1]}" | cut -d'|' -f2; }
|
||||
phase_status() { echo "${PHASES[$1]}" | cut -d'|' -f3; }
|
||||
phase_review() { echo "REVIEW-$(phase_id "$1").md"; }
|
||||
phase_done() { grep -qE '^##[[:space:]]+DONE' "$BUILDER_DIR/$1" 2>/dev/null; } # $1 = status basename (read locally)
|
||||
# Loop-state files may sit at the repo root OR under machine-docs/ (the 1b RL6 move). Prefer
|
||||
# machine-docs/ if present, else root — so the watchdog survives the move whenever it happens.
|
||||
resolve_state() { local dir="$1" base="$2"; if [[ -f "$dir/machine-docs/$base" ]]; then echo "$dir/machine-docs/$base"; else echo "$dir/$base"; fi; }
|
||||
phase_done() { grep -qE '^##[[:space:]]+DONE' "$(resolve_state "$BUILDER_DIR" "$1")" 2>/dev/null; } # $1 = status basename (read locally)
|
||||
all_ids() { local p; for p in "${PHASES[@]}"; do printf '%s ' "$(echo "$p" | cut -d'|' -f1)"; done; }
|
||||
|
||||
preflight() {
|
||||
@@ -133,15 +136,32 @@ ping_session() {
|
||||
tmux send-keys -t "$s" -l -- "$msg" 2>/dev/null && { sleep 0.3; tmux send-keys -t "$s" Enter 2>/dev/null; }
|
||||
}
|
||||
|
||||
# A loop can stall ALIVE on a usage/spend-limit notice: the claude process stays up (so the
|
||||
# dead-session restart never fires) but makes no progress, and the /loop self-pacing is dead because
|
||||
# the limit interrupted the turn that would have scheduled the next tick. Detect that signature
|
||||
# (limit text present + no active-turn marker) and re-nudge it each heavy tick — once the limit resets
|
||||
# the next nudge lands and the loop resumes. Gated on the limit text so we NEVER nudge a loop that is
|
||||
# just legitimately idle-waiting on a handoff.
|
||||
LIMIT_RE='spend limit|usage limit|limit reached|reached your .*limit|out of (credits|tokens)'
|
||||
nudge_if_limit_stalled() {
|
||||
local s="$1" pane
|
||||
pane="$(tmux capture-pane -pt "$s" 2>/dev/null | tail -25 || true)"
|
||||
if printf '%s\n' "$pane" | grep -q 'esc to interrupt'; then return 0; fi # actively working
|
||||
if ! printf '%s\n' "$pane" | grep -qiE "$LIMIT_RE"; then return 0; fi # not a limit stall
|
||||
log "limit-stall detected on $s — re-nudging to resume"
|
||||
ping_session "$s" "watchdog: the usage/spend limit appears lifted — RESUME your loop now. Pull latest, re-read your phase STATUS/REVIEW files, and continue from where you stopped; re-arm your loop pacing."
|
||||
}
|
||||
|
||||
# Edge-triggered handoff signalling for the CURRENT phase. Reads the loops' local clones.
|
||||
# Ping the Adversary only when a gate id NEWLY appears on a "CLAIMED … awaiting" line (never on
|
||||
# the baseline / restart / a passed-but-kept line). Ping the Builder when the phase REVIEW changes.
|
||||
_wd_awaiting=""; _wd_baselined=""; _wd_last_review=""
|
||||
handoff_reset() { _wd_awaiting=""; _wd_baselined=""; _wd_last_review=""; } # call on phase transition
|
||||
_wd_adv_inbox_seen=""; _wd_builder_inbox_seen=""
|
||||
handoff_reset() { _wd_awaiting=""; _wd_baselined=""; _wd_last_review=""; _wd_adv_inbox_seen=""; _wd_builder_inbox_seen=""; } # call on phase transition
|
||||
handoff_check() {
|
||||
local idx sf rf cur now added
|
||||
idx="$(cur_idx)"
|
||||
sf="$BUILDER_DIR/$(phase_status "$idx")"; rf="$ADV_DIR/$(phase_review "$idx")"
|
||||
sf="$(resolve_state "$BUILDER_DIR" "$(phase_status "$idx")")"; rf="$(resolve_state "$ADV_DIR" "$(phase_review "$idx")")"
|
||||
if [[ -f "$sf" ]]; then
|
||||
now="$(grep -iE 'CLAIMED.*awaiting' "$sf" 2>/dev/null | grep -oiE 'M[0-9]+(\.[0-9]+)?|[A-Z][0-9]+' | tr '[:lower:]' '[:upper:]' | sort -u || true)"
|
||||
if [[ -n "$_wd_baselined" ]]; then
|
||||
@@ -163,6 +183,34 @@ handoff_check() {
|
||||
_wd_last_review="$cur"
|
||||
fi
|
||||
fi
|
||||
|
||||
# INBOX side-channel (§6.1). The sender writes the receiver's inbox in their OWN clone, so we
|
||||
# detect from the sender side. Edge-trigger on content hash so a fresh message (sender re-wrote
|
||||
# before receiver consumed) re-pings. Receiver deletes after processing => hash empty => next
|
||||
# write re-triggers.
|
||||
local adv_inbox builder_inbox h
|
||||
adv_inbox="$(resolve_state "$BUILDER_DIR" "ADVERSARY-INBOX.md")"
|
||||
if [[ -f "$adv_inbox" ]]; then
|
||||
h="$(md5sum "$adv_inbox" 2>/dev/null | awk '{print $1}' || true)"
|
||||
if [[ -n "$h" && "$h" != "$_wd_adv_inbox_seen" ]]; then
|
||||
log "handoff: ADVERSARY-INBOX.md new/changed -> pinging Adversary"
|
||||
ping_session "$ADV_SESSION" "watchdog ping: the Builder wrote machine-docs/ADVERSARY-INBOX.md — pull, read the message, act on it, then delete the file (commit + push) to mark it consumed."
|
||||
_wd_adv_inbox_seen="$h"
|
||||
fi
|
||||
else
|
||||
_wd_adv_inbox_seen="" # consumed; ready for the next write
|
||||
fi
|
||||
builder_inbox="$(resolve_state "$ADV_DIR" "BUILDER-INBOX.md")"
|
||||
if [[ -f "$builder_inbox" ]]; then
|
||||
h="$(md5sum "$builder_inbox" 2>/dev/null | awk '{print $1}' || true)"
|
||||
if [[ -n "$h" && "$h" != "$_wd_builder_inbox_seen" ]]; then
|
||||
log "handoff: BUILDER-INBOX.md new/changed -> pinging Builder"
|
||||
ping_session "$BUILDER_SESSION" "watchdog ping: the Adversary wrote machine-docs/BUILDER-INBOX.md — pull, read the message, act on it, then delete the file (commit + push) to mark it consumed."
|
||||
_wd_builder_inbox_seen="$h"
|
||||
fi
|
||||
else
|
||||
_wd_builder_inbox_seen=""
|
||||
fi
|
||||
}
|
||||
|
||||
watchdog_loop() {
|
||||
@@ -184,15 +232,15 @@ watchdog_loop() {
|
||||
handoff_reset
|
||||
start_loops
|
||||
else
|
||||
log "PHASE SEQUENCE COMPLETE (last phase $pid DONE). Stopping loops — MANUAL CHECK-IN required before Phase 2."
|
||||
log "PHASE SEQUENCE COMPLETE (last phase $pid DONE). Stopping loops — entire build (1c→3) finished."
|
||||
stop_loops
|
||||
printf 'cc-ci phase sequence complete %(%F %T)T. Phases: %s. Loops stopped; manual check-in required before Phase 2.\n' -1 "$(all_ids)" > "$LOG_DIR/SEQUENCE-COMPLETE"
|
||||
printf 'cc-ci phase sequence complete %(%F %T)T. Phases: %s. Loops stopped; entire build finished.\n' -1 "$(all_ids)" > "$LOG_DIR/SEQUENCE-COMPLETE"
|
||||
log "watchdog exiting."
|
||||
exit 0
|
||||
fi
|
||||
else
|
||||
session_alive "$BUILDER_SESSION" || { log "builder gone — restarting (phase $pid)"; start_agent builder "$BUILDER_SESSION" "$BUILDER_DIR"; }
|
||||
session_alive "$ADV_SESSION" || { log "adversary gone — restarting (phase $pid)"; start_agent adversary "$ADV_SESSION" "$ADV_DIR"; }
|
||||
if session_alive "$BUILDER_SESSION"; then nudge_if_limit_stalled "$BUILDER_SESSION"; else log "builder gone — restarting (phase $pid)"; start_agent builder "$BUILDER_SESSION" "$BUILDER_DIR"; fi
|
||||
if session_alive "$ADV_SESSION"; then nudge_if_limit_stalled "$ADV_SESSION"; else log "adversary gone — restarting (phase $pid)"; start_agent adversary "$ADV_SESSION" "$ADV_DIR"; fi
|
||||
fi
|
||||
fi
|
||||
sleep "$SIGNAL_INTERVAL"
|
||||
|
||||
Reference in New Issue
Block a user