autonomic-bot
d441c6caaf
Root cause chain, all confirmed on the live system: 1. _run_pids() substring-matched the WHOLE cmdline for the session name. An agent's kickoff PROMPT is an argv element, and the supervisor's prompt text contains the literal 'cc-ci-upgrader' — so the supervisor's OWN billing-hung agent matched as a live upgrader run. Now matches FLAG VALUES only (--title <SESSION> / -s <sid>). 2. The gate treated 'a live proc exists' as progress. A provider-walled run keeps its process alive and SPINNING while emitting nothing (verified: 3 days, zero session output, still burning CPU). Progress now REQUIRES the session tree to have advanced within STALL_MIN; a live-but-idle proc is explicitly logged as stalled. Result was ~60 consecutive false 'run progressing — leaving it' no-ops while the weekly run sat unfinished and unreported. Billing-walled runs are REPORTED, never killed (operator policy 2026-08-10): they may resume when the wall lifts and their context is the run's state. New _billing_blocked() detects the wall from the log tail and the gate surfaces 'run BLOCKED on a provider billing/usage wall — NOT killing; operator action required'. Verified live: _run_pids no longer matches the hung Aug-7 supervisor (1497561) while still matching the real finisher; gate now reasons 'session advanced Nm ago'.
664 lines
34 KiB
Python
664 lines
34 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
cc-ci upgrader launcher — one-shot weekly recipe-upgrade job agent.
|
|
|
|
The upgrader runs /upgrade-all to completion, then stops and stays idle so the
|
|
run + summary remain viewable in the web UI. The next weekly run starts a fresh
|
|
session (start clears any idle/finished session).
|
|
|
|
Usage:
|
|
launch-upgrader.py start use-or-create: leave an in-flight run alone, else start fresh
|
|
launch-upgrader.py fresh always kill any existing session and start fresh
|
|
launch-upgrader.py stop kill the session
|
|
launch-upgrader.py status show session state
|
|
launch-upgrader.py attach tmux attach to the session
|
|
|
|
Env:
|
|
LOOP_BACKEND opencode (default) | claude — also accepts UPGRADER_BACKEND
|
|
LOOP_TIER opencode subscription tier: "zen" (OpenCode ZEN, default) or "go"
|
|
(OpenCode Go). Selects the default model + the usage-limit probe
|
|
endpoint/key. Go hit a monthly limit; ZEN is the working alternative.
|
|
Only affects the opencode backend (ignored for claude).
|
|
LOOP_MODEL model flag (overrides UPGRADER_MODEL); default tracks backend+tier —
|
|
opencode+zen→opencode/glm-5.2, opencode+go→opencode-go/glm-5.2, claude→sonnet
|
|
UPGRADER_MODEL provider/model for opencode, e.g. opencode/glm-5.2 (OpenCode ZEN),
|
|
opencode-go/glm-5.2 (OpenCode Go), or tinfoil/deepseek-v4-pro; sonnet for claude
|
|
UPGRADER_ARGS extra args passed to /upgrade-all (e.g. "n8n ghost", "--dry-run")
|
|
|
|
claude backend:
|
|
CLAUDE_BIN, CLAUDE_FLAGS, REMOTE_CONTROL
|
|
opencode backend:
|
|
OPENCODE_BIN, OPENCODE_SERVER, OPENCODE_SHARE (1=attach to web server + public --share link)
|
|
"""
|
|
|
|
import os, sys, subprocess, re
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
# ── config ────────────────────────────────────────────────────────────────────
|
|
|
|
SESSION = os.environ.get("UPGRADER_SESSION", "cc-ci-upgrader")
|
|
WORKDIR = os.environ.get("UPGRADER_DIR", "/srv/cc-ci")
|
|
LOG_DIR = os.environ.get("LOG_DIR", "/srv/cc-ci/.cc-ci-logs")
|
|
|
|
# LOOP_BACKEND / LOOP_MODEL take precedence (unified control from the operator).
|
|
# LOOP_TIER selects the OpenCode subscription: "zen" (default) or "go". Go hit a monthly
|
|
# usage limit; ZEN is the working alternative. Set LOOP_TIER=go when the Go limit resets.
|
|
# Only affects the opencode backend; claude ignores the tier.
|
|
BACKEND = os.environ.get("LOOP_BACKEND", os.environ.get("UPGRADER_BACKEND", "opencode"))
|
|
TIER = os.environ.get("LOOP_TIER", os.environ.get("UPGRADER_TIER", "zen"))
|
|
_TIER_CFG = {
|
|
"go": {"model": "opencode-go/glm-5.2", "endpoint": "https://opencode.ai/zen/go/v1/chat/completions",
|
|
"auth": "opencode-go", "label": "OpenCode Go"},
|
|
"zen": {"model": "opencode/glm-5.2", "endpoint": "https://opencode.ai/zen/v1/chat/completions",
|
|
"auth": "opencode", "label": "OpenCode ZEN"},
|
|
}
|
|
if BACKEND == "opencode" and TIER not in _TIER_CFG:
|
|
print(f"[upgrader] ERROR: unknown LOOP_TIER '{TIER}' — use 'go' or 'zen'", flush=True)
|
|
sys.exit(1)
|
|
_tier = _TIER_CFG.get(TIER, _TIER_CFG["zen"])
|
|
_DEFAULT_MODEL = _tier["model"] if BACKEND == "opencode" else "sonnet"
|
|
MODEL = os.environ.get("LOOP_MODEL", os.environ.get("UPGRADER_MODEL", _DEFAULT_MODEL))
|
|
|
|
CLAUDE_BIN = os.environ.get("CLAUDE_BIN", "claude")
|
|
CLAUDE_FLAGS = os.environ.get("CLAUDE_FLAGS", "--dangerously-skip-permissions")
|
|
REMOTE_CONTROL = os.environ.get("REMOTE_CONTROL", "1") == "1"
|
|
|
|
OPENCODE_BIN = os.environ.get("OPENCODE_BIN", "/home/loops/.local/bin/opencode")
|
|
OPENCODE_SERVER = os.environ.get("OPENCODE_SERVER", "http://127.0.0.1:4096")
|
|
# Web visibility for the opencode backend: attach the session to the shared opencode
|
|
# web server (viewable at http://oc.commoninternet.net, tailnet-only) AND optionally
|
|
# create a public opencode.ai --share link. Default both on so the run is monitorable.
|
|
OPENCODE_SHARE = os.environ.get("OPENCODE_SHARE", "1") == "1"
|
|
|
|
UPGRADER_ARGS = os.environ.get("UPGRADER_ARGS", "")
|
|
|
|
# First step of the weekly run: reclaim STALE docker images on the cc-ci server BEFORE the run so a
|
|
# heavy run can't fill the disk mid-flight (root cause of the 2026-07-03 stall — 100% ENOSPC killed
|
|
# lasuite-drive + wedged the run). "Stale" = unused by any container AND older than PRERECLAIM_UNTIL,
|
|
# so recently-built/pulled images (the ones this week's tests will reuse) are KEPT — we only evict
|
|
# leftovers from prior weeks. Best-effort; never fails the run.
|
|
PRERECLAIM = os.environ.get("UPGRADER_PRERECLAIM", "1") == "1"
|
|
PRERECLAIM_UNTIL = os.environ.get("UPGRADER_PRERECLAIM_UNTIL", "168h") # 7d: older than one run ago
|
|
PRERECLAIM_HOST = os.environ.get("UPGRADER_PRERECLAIM_HOST", "cc-ci")
|
|
|
|
# ── helpers ───────────────────────────────────────────────────────────────────
|
|
|
|
def log(msg):
|
|
ts = datetime.now().strftime("%H:%M:%S")
|
|
print(f"[upgrader {ts}] {msg}", flush=True)
|
|
|
|
def die(msg):
|
|
log(f"ERROR: {msg}")
|
|
sys.exit(1)
|
|
|
|
def session_alive():
|
|
return subprocess.run(
|
|
["tmux", "has-session", "-t", SESSION], capture_output=True
|
|
).returncode == 0
|
|
|
|
def session_busy():
|
|
"""True while a turn is actively in flight (not idle/finished/wedged)."""
|
|
r = subprocess.run(["tmux", "capture-pane", "-pt", SESSION],
|
|
capture_output=True, text=True)
|
|
pane = r.stdout if r.returncode == 0 else ""
|
|
return bool(re.search(r"esc to interrupt|⠋|⠙|⠹|⠸|⠼|⠴|⠦|⠧|⠇|⠏|Running tool", pane))
|
|
|
|
def kill_session():
|
|
subprocess.run(["tmux", "kill-session", "-t", SESSION], capture_output=True)
|
|
|
|
def _watchdog_alive():
|
|
return subprocess.run(["tmux", "has-session", "-t", f"{SESSION}-watchdog"],
|
|
capture_output=True).returncode == 0
|
|
|
|
def prereclaim_cc_ci():
|
|
"""Weekly-run step 0: prune STALE (unused AND older than PRERECLAIM_UNTIL) docker images on the
|
|
cc-ci server so the run has disk headroom. Keeps recent images (reused this week); only clears
|
|
prior-weeks' leftovers. Best-effort — a reclaim failure must never abort the run."""
|
|
if not PRERECLAIM:
|
|
return
|
|
filt = f"--filter until={PRERECLAIM_UNTIL}"
|
|
remote = (f"docker image prune -af {filt} 2>&1 | tail -1; "
|
|
f"docker builder prune -af {filt} >/dev/null 2>&1 || true; "
|
|
f"df -h / | tail -1")
|
|
log(f" step 0: pre-reclaim stale docker images on {PRERECLAIM_HOST} (unused & >{PRERECLAIM_UNTIL})")
|
|
try:
|
|
r = subprocess.run(["ssh", "-o", "ConnectTimeout=15", PRERECLAIM_HOST, remote],
|
|
capture_output=True, text=True, timeout=900)
|
|
out = (r.stdout or r.stderr or "").strip()
|
|
for ln in out.splitlines():
|
|
if ln.strip():
|
|
log(f" {ln.strip()}")
|
|
except Exception as e:
|
|
log(f" pre-reclaim skipped (non-fatal): {e}")
|
|
|
|
# ── kickoff prompt ────────────────────────────────────────────────────────────
|
|
|
|
def build_kickoff():
|
|
args_note = f" with arguments: {UPGRADER_ARGS}" if UPGRADER_ARGS else ""
|
|
return f"""\
|
|
*** cc-ci UPGRADER — weekly recipe-upgrade job ***
|
|
You are the cc-ci Upgrader: a ONE-SHOT job agent, NOT a perpetual loop. Run the
|
|
recipe-upgrade sequence to completion, then STOP. Your cwd is {WORKDIR}; reach the CI
|
|
server with `ssh cc-ci`; creds are in {WORKDIR}/.testenv; skills in {WORKDIR}/.claude/skills/.
|
|
|
|
DO THIS:
|
|
1. Invoke the /upgrade-all skill in DEFAULT mode{args_note}
|
|
(read {WORKDIR}/.claude/skills/upgrade-all/SKILL.md for the full procedure). It surveys
|
|
every enrolled recipe and, for each upgradeable one, runs /recipe-upgrade in DEFAULT
|
|
mode — recipe PR only, verified by posting `!testme` on the PR (results visible in the
|
|
PR, iterate up to 3x). A genuinely stale test gets an explanatory PR COMMENT, never a
|
|
test edit.
|
|
2. Process recipes via per-recipe SUBAGENTS so your own context stays light. If your
|
|
context usage climbs (~80%), run /compact before continuing.
|
|
3. Write + push the weekly summary (the PR list is the actionable output for the operator).
|
|
4. WHEN THE RUN IS COMPLETE: STOP. Print the final summary (lead with the PR list) and an
|
|
`UPGRADE RUN COMPLETE` line, then go idle. Do NOT loop, do NOT re-run, and do NOT kill
|
|
your own session — leave it up so the operator can review the output in the web UI.
|
|
Next week's run starts a fresh session (the launcher clears this idle one).
|
|
|
|
GUARDRAILS: NEVER merge any PR. NEVER weaken a test. DEFAULT mode only — do NOT pass
|
|
--with-tests (updating cc-ci tests is the operator's per-recipe opt-in). Single-writer:
|
|
dedicated branches + separate clones, never push main, never touch the build loops'
|
|
/cc-ci /cc-ci-adv clones. The shared Swarm is stateful — go sequentially.
|
|
"""
|
|
|
|
# ── launch ────────────────────────────────────────────────────────────────────
|
|
|
|
def start(mode="use-or-create"):
|
|
import shutil
|
|
if not shutil.which("tmux"):
|
|
die("tmux not found")
|
|
Path(LOG_DIR).mkdir(parents=True, exist_ok=True)
|
|
|
|
if session_alive():
|
|
if mode == "use-or-create" and session_busy():
|
|
log(f"{SESSION} already running a job (busy) — leaving it")
|
|
return
|
|
log(f"{SESSION} exists but idle/stale (or fresh requested) — killing it first")
|
|
kill_session()
|
|
import time; time.sleep(1)
|
|
|
|
# Step 0 of the weekly run: clear STALE cc-ci docker images so a heavy run can't run the disk
|
|
# out mid-flight (root cause of the 2026-07-03 stall). Only for the actual upgrade run.
|
|
if SESSION == "cc-ci-upgrader":
|
|
prereclaim_cc_ci()
|
|
|
|
# Unique-name invariant: the run we are about to launch becomes THE 'cc-ci-upgrader' in the
|
|
# web UI; every older run gets an archive title. Also snapshot existing ids so the new
|
|
# session can be pinned unambiguously after launch.
|
|
_archive_stale_titles()
|
|
_prev_rows = _server_get("/session") or []
|
|
_prev_rows = _prev_rows if isinstance(_prev_rows, list) else _prev_rows.get("data", [])
|
|
_prev_ids = {s.get("id") for s in _prev_rows}
|
|
try:
|
|
STATE_SID_FILE.unlink()
|
|
except OSError:
|
|
pass
|
|
|
|
kf = Path(LOG_DIR) / f".kickoff-{SESSION}.txt"
|
|
kf.write_text(build_kickoff())
|
|
|
|
model_flag = f"--model '{MODEL}'" if MODEL else ""
|
|
tier_note = f", tier={TIER} ({_tier['label']})" if BACKEND == "opencode" else ""
|
|
log(f"starting {SESSION} (backend={BACKEND}{tier_note}, model={MODEL}, args='{UPGRADER_ARGS or '<none>'}')")
|
|
|
|
if BACKEND == "claude":
|
|
if not shutil.which(CLAUDE_BIN):
|
|
die(f"claude CLI not found — set CLAUDE_BIN (currently: {CLAUDE_BIN})")
|
|
rc = f"--remote-control '{SESSION}'" if REMOTE_CONTROL else ""
|
|
cmd = f"{CLAUDE_BIN} {rc} {model_flag} {CLAUDE_FLAGS} \"$(cat '{kf}')\""
|
|
|
|
elif BACKEND == "opencode":
|
|
if not Path(OPENCODE_BIN).exists():
|
|
die(f"opencode not found at {OPENCODE_BIN}")
|
|
# NOTE: -m/--model and --attach/--title/--share are flags on the `run` SUBCOMMAND,
|
|
# so they must come AFTER `run` (a global `opencode --model X run` is ignored).
|
|
share_flag = "--share" if OPENCODE_SHARE else ""
|
|
cmd = (
|
|
f"set -a; . /srv/cc-ci/.testenv; set +a; "
|
|
f"{OPENCODE_BIN} run {model_flag} {share_flag} --attach '{OPENCODE_SERVER}' "
|
|
f"--title '{SESSION}' \"$(cat '{kf}')\""
|
|
)
|
|
log(f" attached to {OPENCODE_SERVER} → http://oc.commoninternet.net (tailnet only)"
|
|
+ (" +public --share link (printed in the session)" if OPENCODE_SHARE else ""))
|
|
else:
|
|
die(f"unknown LOOP_BACKEND '{BACKEND}' — use 'claude' or 'opencode'")
|
|
|
|
subprocess.run(["tmux", "new-session", "-d", "-s", SESSION, "-c", WORKDIR, cmd])
|
|
subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION,
|
|
f"cat >> '{LOG_DIR}/{SESSION}.log'"])
|
|
log(f"started. attach: tmux attach -t {SESSION} log: {LOG_DIR}/{SESSION}.log")
|
|
if BACKEND == "opencode":
|
|
_pin_new_session(_prev_ids)
|
|
# For the opencode backend, spawn a watchdog that auto-resumes the run if the opencode
|
|
# usage-limit (429) stalls it mid-run (it does NOT self-resume). See watchdog().
|
|
if BACKEND == "opencode" and os.environ.get("UPGRADER_WATCHDOG", "1") == "1":
|
|
_spawn_watchdog()
|
|
|
|
# ── opencode stall-detect + auto-resume watchdog ────────────────────────────────
|
|
# The opencode subscription (Go or ZEN tier) enforces a rolling usage-limit (HTTP 429 + retry-after). When it
|
|
# trips mid-run, the `opencode run` agent loop ENDS and does NOT self-resume. This watchdog detects
|
|
# the stall (the session log stops growing), waits out the limit, and resumes the SAME session —
|
|
# context preserved — via `opencode run -s <id> --continue`. Standalone: launch-upgrader.py {resume|watchdog}.
|
|
import json as _json, urllib.request as _ureq, time as _time
|
|
|
|
STALL_MIN = float(os.environ.get("UPGRADER_STALL_MIN", "15")) # log-idle minutes ⇒ stalled
|
|
CHECK_EVERY = int(os.environ.get("UPGRADER_CHECK_SEC", "180")) # watchdog poll cadence
|
|
# Generic so the SAME watchdog also covers the report job (launch-report.py points it at the
|
|
# cc-ci-report session with its own marker + resume prompt via these env vars).
|
|
DONE_MARKER = os.environ.get("UPGRADER_DONE_MARKER", "UPGRADE RUN COMPLETE")
|
|
RESUME_FILE = os.environ.get("UPGRADER_RESUME_FILE") # optional path to a custom resume prompt
|
|
LIMIT_ENDPOINT = _tier["endpoint"] # tier-aware: zen or go
|
|
AUTH_JSON = os.path.expanduser("~/.local/share/opencode/auth.json")
|
|
LOG_FILE = f"{LOG_DIR}/{SESSION}.log"
|
|
|
|
def _server_get(path):
|
|
try:
|
|
with _ureq.urlopen(OPENCODE_SERVER + path, timeout=15) as r:
|
|
return _json.load(r)
|
|
except Exception:
|
|
return None
|
|
|
|
STATE_SID_FILE = Path(LOG_DIR) / f".{SESSION}-session-id"
|
|
|
|
def _server_patch(path, body):
|
|
req = _ureq.Request(OPENCODE_SERVER + path, method="PATCH",
|
|
headers={"Content-Type": "application/json"},
|
|
data=_json.dumps(body).encode())
|
|
try:
|
|
with _ureq.urlopen(req, timeout=15) as r:
|
|
return r.status
|
|
except Exception:
|
|
return None
|
|
|
|
def _db_created_ms(sid):
|
|
"""Session creation time from the opencode sqlite DB — the /session API rows carry NO
|
|
time fields, which is exactly how the 2026-08-04 watchdog resumed the WRONG (old, giant)
|
|
session: sorting on a missing key degraded to server list order."""
|
|
try:
|
|
import sqlite3
|
|
db = sqlite3.connect("file:" + os.path.expanduser(
|
|
"~/.local/share/opencode/opencode.db") + "?mode=ro", uri=True)
|
|
row = db.execute("SELECT time_created FROM session WHERE id=?", (sid,)).fetchone()
|
|
db.close()
|
|
return row[0] if row else 0
|
|
except Exception:
|
|
return 0
|
|
|
|
def _archive_stale_titles(title=None, label=None):
|
|
"""Rename every existing top-level session with the given canonical title to a dated
|
|
archive title, so EXACTLY ONE session ever carries the canonical name (the one the next
|
|
launch creates). Keeps the run trivially findable in the opencode web UI and makes the
|
|
title lookup in _session_id() unambiguous. Old runs stay browsable under
|
|
'<label> — <date>'. Also used by launch-supervisor.py for its own session name."""
|
|
title = title or SESSION
|
|
# Archive names always start with 'archive-' (operator convention 2026-08-04) so they
|
|
# sort/filter together in the web UI: 'archive-<original-title> — <date>'.
|
|
label = label or f"archive-{title} —"
|
|
rows = _server_get("/session") or []
|
|
rows = rows if isinstance(rows, list) else rows.get("data", [])
|
|
for s in rows:
|
|
if s.get("title") == title and not (s.get("parentID") or s.get("parentId")):
|
|
created = _db_created_ms(s["id"])
|
|
d = datetime.fromtimestamp(created / 1000).strftime("%Y-%m-%d") if created else "unknown-date"
|
|
_server_patch(f"/session/{s['id']}", {"title": f"{label} {d}"})
|
|
log(f" archived old session {s['id'][:20]} → '{label} {d}'")
|
|
|
|
def _session_id():
|
|
"""The opencode session this launcher manages. Prefers the pinned id recorded at launch
|
|
(LOG_DIR/.{SESSION}-session-id) — title lookup is only the fallback, and thanks to
|
|
_archive_stale_titles() at most one top-level session carries the title. Never trust
|
|
server list order (see _db_created_ms)."""
|
|
try:
|
|
pinned = STATE_SID_FILE.read_text().strip()
|
|
# Validate directly — the /session LIST is paginated (~100 rows), so membership
|
|
# scans miss older/newer sessions; a direct GET is authoritative.
|
|
if pinned and (_server_get(f"/session/{pinned}") or {}).get("id") == pinned:
|
|
return pinned
|
|
except OSError:
|
|
pass
|
|
rows = _server_get("/session") or []
|
|
rows = rows if isinstance(rows, list) else rows.get("data", [])
|
|
cands = [s for s in rows if s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId"))]
|
|
cands.sort(key=lambda s: _db_created_ms(s.get("id")), reverse=True)
|
|
if cands:
|
|
try:
|
|
STATE_SID_FILE.write_text(cands[0]["id"])
|
|
except OSError:
|
|
pass
|
|
return cands[0]["id"]
|
|
return None
|
|
|
|
def _pin_new_session(prev_ids, wait_sec=45):
|
|
"""After launching a fresh run, discover the NEW top-level SESSION-titled session (one not
|
|
in prev_ids) and pin its id to the state file, so the watchdog can never grab an old one."""
|
|
deadline = _time.time() + wait_sec
|
|
while _time.time() < deadline:
|
|
rows = _server_get("/session") or []
|
|
rows = rows if isinstance(rows, list) else rows.get("data", [])
|
|
for s in rows:
|
|
if (s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId"))
|
|
and s.get("id") not in prev_ids):
|
|
try:
|
|
STATE_SID_FILE.write_text(s["id"])
|
|
except OSError:
|
|
pass
|
|
log(f" pinned managed session id {s['id'][:20]} → {STATE_SID_FILE.name}")
|
|
return s["id"]
|
|
_time.sleep(3)
|
|
log(" WARNING: could not discover the new session id to pin (title lookup remains the fallback)")
|
|
return None
|
|
|
|
def _session_idle_min():
|
|
"""Minutes since the managed run last ADVANCED — measured across the whole session TREE (the
|
|
top-level cc-ci-upgrader session AND every descendant subagent). Uses the opencode SERVER's
|
|
time.updated (authoritative — bumps on every new message/part) rather than the tmux log mtime,
|
|
which FREEZES when a headless `opencode run --continue` doesn't stream to the pane (that froze
|
|
signal made the watchdog resume-storm). Crucially it must include CHILDREN: while a per-recipe
|
|
subagent runs (a deploy can take 20-40min) the PARENT's updated time is stale, so reading the
|
|
parent alone would look 'idle' and trigger a false resume that kills the productive run. Falls
|
|
back to the log mtime only when the server is unreachable."""
|
|
sid = _session_id()
|
|
if not sid:
|
|
try:
|
|
return (_time.time() - os.path.getmtime(LOG_FILE)) / 60.0
|
|
except Exception:
|
|
return None
|
|
rows = _server_get("/session") or []
|
|
rows = rows if isinstance(rows, list) else rows.get("data", [])
|
|
def _parent(s): return s.get("parentID") or s.get("parentId")
|
|
# Transitive closure of sid over parent links → the managed session + all its descendants.
|
|
tree = {sid}
|
|
for _ in range(8): # bounded fixpoint (subagents nest only a few deep)
|
|
grew = False
|
|
for s in rows:
|
|
if s.get("id") not in tree and _parent(s) in tree:
|
|
tree.add(s.get("id")); grew = True
|
|
if not grew:
|
|
break
|
|
newest = 0
|
|
for s in rows:
|
|
if s.get("id") in tree:
|
|
newest = max(newest, (s.get("time") or {}).get("updated") or 0)
|
|
if newest:
|
|
return max(0.0, (_time.time() * 1000 - newest) / 60000.0)
|
|
try:
|
|
return (_time.time() - os.path.getmtime(LOG_FILE)) / 60.0
|
|
except Exception:
|
|
return None
|
|
|
|
# Back-compat alias (older callers / the report watchdog import this name).
|
|
_log_idle_min = _session_idle_min
|
|
|
|
def _limit_key():
|
|
try:
|
|
return (_json.load(open(AUTH_JSON)).get(_tier["auth"]) or {}).get("key")
|
|
except Exception:
|
|
return None
|
|
|
|
def _limit_retry_after():
|
|
"""0 if the tier's opencode endpoint is available (HTTP 200); else the 429 retry-after seconds."""
|
|
key = _limit_key()
|
|
if not key:
|
|
return 0
|
|
body = _json.dumps({"model": (MODEL or "").split("/")[-1] or "glm-5.2", "max_tokens": 8,
|
|
"messages": [{"role": "user", "content": "hi"}]}).encode()
|
|
req = _ureq.Request(LIMIT_ENDPOINT, data=body, method="POST",
|
|
headers={"Authorization": "Bearer " + key, "content-type": "application/json"})
|
|
try:
|
|
_ureq.urlopen(req, timeout=20).read(); return 0
|
|
except _ureq.HTTPError as e:
|
|
if e.code == 429:
|
|
try: return max(1, int(e.headers.get("retry-after", "300")))
|
|
except Exception: return 300
|
|
return 0
|
|
except Exception:
|
|
return 0
|
|
|
|
def _run_pids(sid=None):
|
|
"""PIDs of live `opencode run` procs for THIS session (via /proc scan — never matches self).
|
|
|
|
Matches on FLAG VALUES (`--title <SESSION>` / `-s|--session <sid>`), never a substring of the
|
|
whole cmdline. The substring form caused the 2026-08-07 three-day deadlock: an agent's kickoff
|
|
PROMPT is passed as an argv element, and the supervisor's prompt text contains the literal
|
|
"cc-ci-upgrader", so the supervisor's own (billing-hung) agent matched as a live upgrader run —
|
|
the hourly gate then read its own corpse as "run progressing" and no-opped ~60 times."""
|
|
me, out = os.getpid(), []
|
|
for p in os.listdir("/proc"):
|
|
if not p.isdigit() or int(p) == me:
|
|
continue
|
|
try:
|
|
cl = [c for c in open(f"/proc/{p}/cmdline", "rb").read().split(b"\0") if c]
|
|
except Exception:
|
|
continue
|
|
if not cl or b"opencode" not in cl[0] or b"run" not in cl or b"--attach" not in cl:
|
|
continue
|
|
|
|
def _flag(names):
|
|
for i, a in enumerate(cl[:-1]):
|
|
if a in names:
|
|
return cl[i + 1]
|
|
return None
|
|
|
|
title = _flag((b"--title",))
|
|
s_val = _flag((b"-s", b"--session"))
|
|
if title == SESSION.encode() or (sid and s_val == sid.encode()):
|
|
out.append(int(p))
|
|
return out
|
|
|
|
|
|
def _billing_blocked(window_bytes=4000):
|
|
"""True when the tail of this run's log shows a provider billing/usage wall.
|
|
|
|
Such a process must NEVER be killed: it may resume when the wall lifts, and killing it can
|
|
lose in-flight work. It is also NOT progress — it can spin for days emitting nothing (observed
|
|
2026-08-07..10), so the gate reports it as BLOCKED and leaves recovery to the operator."""
|
|
try:
|
|
with open(LOG_FILE, "rb") as f:
|
|
f.seek(0, os.SEEK_END)
|
|
f.seek(max(0, f.tell() - window_bytes))
|
|
tail = f.read().decode(errors="replace").lower()
|
|
except OSError:
|
|
return False
|
|
return any(
|
|
s in tail
|
|
for s in ("spending limit", "insufficient balance", "usage limit", "usagelimiterror")
|
|
)
|
|
|
|
def _completed():
|
|
# Done only when the MODEL signs off with DONE_MARKER as its FINAL word: the marker in the LAST
|
|
# assistant TEXT (prose) message. This guards THREE false-positives that each abandoned a run:
|
|
# (1) the kickoff/resume PROMPT contains the marker — but that's a USER message (skipped).
|
|
# (2) the marker inside a TOOL part — a subagent `task` prompt or a bash command that echoes
|
|
# "print <marker>" — so we look ONLY at type=='text' prose, never tool-call args.
|
|
# (3) the model ECHOING the instruction mid-run ("…then I'll print <marker>") — only the FINAL
|
|
# assistant prose counts, so any further work after the echo means it's NOT done.
|
|
sid = _session_id()
|
|
msgs = _server_get(f"/session/{sid}/message") if sid else None
|
|
if msgs is not None:
|
|
msgs = msgs if isinstance(msgs, list) else msgs.get("data", [])
|
|
last_prose = None
|
|
for m in msgs:
|
|
if ((m.get("info") or {}).get("role")) != "assistant":
|
|
continue
|
|
prose = "".join(p.get("text", "") for p in (m.get("parts") or [])
|
|
if p.get("type") == "text" and isinstance(p.get("text"), str))
|
|
if prose.strip():
|
|
last_prose = prose
|
|
return bool(last_prose and DONE_MARKER in last_prose)
|
|
# Server unreachable → conservative log fallback that excludes the prompt's own mention.
|
|
try:
|
|
with open(LOG_FILE, errors="ignore") as f:
|
|
f.seek(0, 2); f.seek(max(0, f.tell() - 20000))
|
|
return any(DONE_MARKER in ln and "print" not in ln and "'" not in ln for ln in f)
|
|
except Exception:
|
|
return False
|
|
|
|
def resume(reason="manual"):
|
|
"""Resume the managed opencode session from where it stopped (context preserved)."""
|
|
import signal
|
|
sid = _session_id()
|
|
if not sid:
|
|
log(f"resume: no top-level '{SESSION}' session on {OPENCODE_SERVER} — cannot resume"); return False
|
|
log(f"resume ({reason}): continuing session {sid}")
|
|
for pid in _run_pids(sid):
|
|
try: os.kill(pid, signal.SIGTERM)
|
|
except Exception: pass
|
|
_time.sleep(2); kill_session(); _time.sleep(1)
|
|
kf = Path(LOG_DIR) / f".kickoff-{SESSION}-resume.txt"
|
|
if RESUME_FILE and os.path.exists(RESUME_FILE):
|
|
kf.write_text(open(RESUME_FILE).read()) # caller-supplied (e.g. the report job)
|
|
else:
|
|
kf.write_text(
|
|
"The opencode usage limit has reset (or the run stalled). You were mid-way through the weekly "
|
|
"cc-ci /upgrade-all run. CONTINUE from where you left off — do NOT start over. Process the enrolled "
|
|
"recipes not yet done this week, alphabetically; SKIP ones already done (their PRs exist — extend, "
|
|
"never duplicate). Per recipe: run /recipe-upgrade in DEFAULT mode via a subagent, verify with "
|
|
"!testme, open/extend the recipe PR (NEVER merge, NEVER weaken a test), <= DRONE_RUNNER_CAPACITY "
|
|
"concurrent. immich has a tag+digest image abra can't parse — do the upstream-direct cross-check "
|
|
"(recipe-upgrade SKILL §1), don't silently skip it. When all remaining recipes are done: "
|
|
"write+push the weekly summary, then `python3 /srv/cc-ci/cc-ci-plan/launch-report.py fresh`, print "
|
|
"'" + DONE_MARKER + "', and go idle.")
|
|
share = "--share" if OPENCODE_SHARE else ""
|
|
cmd = (f"set -a; . /srv/cc-ci/.testenv; set +a; {OPENCODE_BIN} run -s {sid} --continue "
|
|
f"--model '{MODEL}' {share} --attach '{OPENCODE_SERVER}' --dir '{WORKDIR}' \"$(cat '{kf}')\"")
|
|
subprocess.run(["tmux", "new-session", "-d", "-s", SESSION, "-c", WORKDIR, cmd])
|
|
subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION, f"cat >> '{LOG_FILE}'"])
|
|
try:
|
|
STATE_SID_FILE.write_text(sid) # a resume continues the SAME session — keep it pinned
|
|
except OSError:
|
|
pass
|
|
log(f"resume: relaunched {SESSION} (session {sid})")
|
|
# Every resume must be self-healing: ensure a watchdog is watching this run. Skip if one is
|
|
# already alive — notably when the watchdog ITSELF called resume (it lives in {SESSION}-watchdog),
|
|
# so this never spawns a duplicate watchdog-of-a-watchdog.
|
|
if os.environ.get("UPGRADER_WATCHDOG", "1") == "1" and not _watchdog_alive():
|
|
_spawn_watchdog()
|
|
return True
|
|
|
|
def _spawn_watchdog():
|
|
"""Start the watchdog inside the persistent tmux server (NOT a Popen child). A systemd-timer
|
|
`start` is a Type=oneshot whose cgroup is reaped on exit, which would kill a Popen child; a
|
|
tmux session lives under the long-running tmux server and survives. Env is passed explicitly so
|
|
the watchdog gets THIS run's config regardless of the tmux server's ambient environment."""
|
|
import shlex
|
|
wsess = f"{SESSION}-watchdog"
|
|
wlog = f"{LOG_DIR}/{SESSION}-watchdog.log"
|
|
env = {"HOME": os.environ.get("HOME") or os.path.expanduser("~"),
|
|
"UPGRADER_SESSION": SESSION, "UPGRADER_DIR": WORKDIR, "LOG_DIR": LOG_DIR,
|
|
"UPGRADER_BACKEND": "opencode", "UPGRADER_MODEL": MODEL, "LOOP_TIER": TIER,
|
|
"OPENCODE_BIN": OPENCODE_BIN, "OPENCODE_SERVER": OPENCODE_SERVER,
|
|
"OPENCODE_SHARE": "1" if OPENCODE_SHARE else "0"}
|
|
for k in ("UPGRADER_RESUME_FILE", "UPGRADER_DONE_MARKER", "UPGRADER_STALL_MIN", "UPGRADER_CHECK_SEC"):
|
|
if os.environ.get(k):
|
|
env[k] = os.environ[k]
|
|
envstr = " ".join(f"{k}={shlex.quote(str(v))}" for k, v in env.items())
|
|
cmd = f"env {envstr} python3 {shlex.quote(os.path.realpath(__file__))} watchdog >> {shlex.quote(wlog)} 2>&1"
|
|
subprocess.run(["tmux", "kill-session", "-t", wsess], capture_output=True)
|
|
subprocess.run(["tmux", "new-session", "-d", "-s", wsess, "-c", WORKDIR, cmd])
|
|
log(f" watchdog spawned in tmux '{wsess}' — auto-resume on usage-limit stalls (survives the oneshot)")
|
|
|
|
def watchdog():
|
|
"""Watch the opencode upgrader and keep it alive to completion. Two stall modes:
|
|
(a) PROC-DEATH — `opencode run` exits when the model ENDS ITS TURN (or crashes). For a long
|
|
autonomous /upgrade-all this happens repeatedly before the whole run is done; the log mtime
|
|
also freezes, so log-idle alone would take 15min to notice a run that died in 5. We detect
|
|
it directly: no live `opencode run` proc for the session + not completed ⇒ resume promptly.
|
|
(b) LOG-IDLE — a proc is alive but wedged (no output > STALL_MIN); resume after confirming.
|
|
Either way, wait out an opencode usage-limit (429) first rather than hammering. Exits when the
|
|
model prints DONE_MARKER, or after MAX_RESUMES consecutive resumes fail to get a live proc going
|
|
(truly broken — hand back to the hourly supervisor / operator). Spawned by start()/resume()."""
|
|
MAX_RESUMES = int(os.environ.get("UPGRADER_MAX_RESUMES", "20"))
|
|
log(f"watchdog: watching {SESSION} (proc-death + stall>{STALL_MIN}min session-idle, poll {CHECK_EVERY}s)")
|
|
misses = 0; resumes = 0
|
|
while True:
|
|
_time.sleep(CHECK_EVERY)
|
|
# A transient error (server blip, race) must NEVER kill the watchdog — that would silently
|
|
# abandon the run. Log and keep polling instead of letting the exception exit the loop.
|
|
try:
|
|
if _completed():
|
|
log("watchdog: run completed — exiting"); return
|
|
sid = _session_id()
|
|
pids = _run_pids(sid) if sid else []
|
|
idle = _session_idle_min()
|
|
dead = not pids
|
|
stalled = idle is not None and idle > STALL_MIN
|
|
if not dead and not stalled:
|
|
misses = 0; resumes = 0; continue # alive + session advancing — healthy
|
|
# Something's wrong (dead or wedged). Wait out a usage-limit before touching it.
|
|
retry = _limit_retry_after()
|
|
if retry > 0:
|
|
wait = min(retry + 30, 3600)
|
|
log(f"watchdog: {'proc dead' if dead else f'stalled {idle:.0f}min'} + usage-limited "
|
|
f"(retry-after {retry}s) — waiting {wait}s")
|
|
_time.sleep(wait); continue
|
|
if not dead:
|
|
# Alive but session not advancing — confirm really wedged (two misses) before acting.
|
|
misses += 1
|
|
if misses < 2:
|
|
continue
|
|
if resumes >= MAX_RESUMES:
|
|
log(f"watchdog: {MAX_RESUMES} resumes without completion — giving up (supervisor/operator needed)")
|
|
return
|
|
why = "run proc exited (turn ended/crashed)" if dead else f"stalled {idle:.0f}min, limit clear"
|
|
log(f"watchdog: {why} — auto-resuming (#{resumes + 1})")
|
|
resume("watchdog auto-resume"); resumes += 1; misses = 0
|
|
except Exception as e:
|
|
log(f"watchdog: poll error ({type(e).__name__}: {e}) — continuing")
|
|
|
|
# ── main ──────────────────────────────────────────────────────────────────────
|
|
|
|
def main():
|
|
cmd = sys.argv[1] if len(sys.argv) > 1 else "start"
|
|
|
|
if cmd == "start":
|
|
start("use-or-create")
|
|
elif cmd == "fresh":
|
|
start("fresh")
|
|
elif cmd == "stop":
|
|
if session_alive():
|
|
log(f"killing {SESSION}")
|
|
kill_session()
|
|
else:
|
|
log(f"{SESSION} not running")
|
|
elif cmd == "status":
|
|
if session_alive():
|
|
busy = "busy" if session_busy() else "idle/finishing"
|
|
log(f"{SESSION}: RUNNING ({busy})")
|
|
subprocess.run(
|
|
f"ps -eo pid,etime,args | grep '[r]emote-control {SESSION}' || true",
|
|
shell=True)
|
|
else:
|
|
log(f"{SESSION}: stopped")
|
|
log(f"backend: {BACKEND} tier: {TIER} model: {MODEL} args: '{UPGRADER_ARGS or '<none>'}'")
|
|
elif cmd == "attach":
|
|
os.execvp("tmux", ["tmux", "attach", "-t", SESSION])
|
|
elif cmd == "resume":
|
|
resume("manual")
|
|
elif cmd == "watchdog":
|
|
watchdog()
|
|
else:
|
|
print(f"""cc-ci upgrader launcher — one-shot weekly recipe-upgrade job
|
|
|
|
launch-upgrader.py start use-or-create (leave busy run alone, else start fresh)
|
|
launch-upgrader.py fresh always kill existing + start fresh
|
|
launch-upgrader.py stop kill the session
|
|
launch-upgrader.py status show session state
|
|
launch-upgrader.py attach tmux attach
|
|
launch-upgrader.py resume continue the opencode session from where it stalled (-s <id> --continue)
|
|
launch-upgrader.py watchdog watch + auto-resume the opencode run across usage-limit (429) stalls
|
|
|
|
Backend: {BACKEND} (LOOP_BACKEND or UPGRADER_BACKEND env var)
|
|
Tier: {TIER} (LOOP_TIER or UPGRADER_TIER env var; 'go' or 'zen'; only for opencode)
|
|
Model: {MODEL} (LOOP_MODEL or UPGRADER_MODEL env var)
|
|
Args: {UPGRADER_ARGS or '<none>'} (UPGRADER_ARGS env var, passed to /upgrade-all)
|
|
|
|
claude: viewable at claude.ai/code
|
|
opencode: viewable at http://oc.commoninternet.net server={OPENCODE_SERVER}
|
|
""")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|