From 28ef7e44ab19feef26e81fd388363d64feec0027 Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Tue, 23 Jun 2026 01:26:24 +0000 Subject: [PATCH] launch-upgrader: add stall-detect + auto-resume watchdog (opencode-go limit) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The opencode-go subscription's rolling usage-limit (429) ends the 'opencode run' agent loop mid-run; it does NOT self-resume. Add: - resume: continue the SAME session (context preserved) via 'opencode run -s --continue' — finds the session from the web server, kills the idle proc safely (via /proc scan, never pkill -f self-match), relaunches in the tmux session. - babysit: poll the session log; on a stall (>15min idle) wait out any 429 retry-after then auto-resume. Spawned automatically by an opencode 'start'. So a usage-limit pause now self-heals instead of needing a manual nudge. Co-Authored-By: Claude Opus 4.8 --- cc-ci-plan/launch-upgrader.py | 147 ++++++++++++++++++++++++++++++++++ 1 file changed, 147 insertions(+) diff --git a/cc-ci-plan/launch-upgrader.py b/cc-ci-plan/launch-upgrader.py index 9ccf109..090eea0 100644 --- a/cc-ci-plan/launch-upgrader.py +++ b/cc-ci-plan/launch-upgrader.py @@ -162,6 +162,147 @@ def start(mode="use-or-create"): subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION, f"cat >> '{LOG_DIR}/{SESSION}.log'"]) log(f"started. attach: tmux attach -t {SESSION} log: {LOG_DIR}/{SESSION}.log") + # For the opencode backend, spawn a detached babysitter that auto-resumes the run if the + # opencode-go usage-limit (429) stalls it mid-run (it does NOT self-resume). See babysit(). + if BACKEND == "opencode" and os.environ.get("UPGRADER_BABYSIT", "1") == "1": + subprocess.Popen(["python3", os.path.realpath(__file__), "babysit"], + stdout=open(f"{LOG_DIR}/{SESSION}-babysit.log", "a"), + stderr=subprocess.STDOUT, start_new_session=True) + log(" babysitter spawned — auto-resume on usage-limit stalls") + +# ── opencode stall-detect + auto-resume watchdog ──────────────────────────────── +# The opencode-go subscription enforces a rolling usage-limit (HTTP 429 + retry-after). When it +# trips mid-run, the `opencode run` agent loop ENDS and does NOT self-resume. This watchdog detects +# the stall (the session log stops growing), waits out the limit, and resumes the SAME session — +# context preserved — via `opencode run -s --continue`. Standalone: launch-upgrader.py {resume|babysit}. +import json as _json, urllib.request as _ureq, time as _time + +STALL_MIN = float(os.environ.get("UPGRADER_STALL_MIN", "15")) # log-idle minutes ⇒ stalled +CHECK_EVERY = int(os.environ.get("UPGRADER_CHECK_SEC", "180")) # babysitter poll cadence +DONE_MARKER = "UPGRADE RUN COMPLETE" +GO_ENDPOINT = "https://opencode.ai/zen/go/v1/chat/completions" +AUTH_JSON = os.path.expanduser("~/.local/share/opencode/auth.json") +LOG_FILE = f"{LOG_DIR}/{SESSION}.log" + +def _server_get(path): + try: + with _ureq.urlopen(OPENCODE_SERVER + path, timeout=15) as r: + return _json.load(r) + except Exception: + return None + +def _session_id(): + """Newest top-level opencode session titled like SESSION (the run we manage).""" + rows = _server_get("/session") or [] + rows = rows if isinstance(rows, list) else rows.get("data", []) + cands = [s for s in rows if s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId"))] + cands.sort(key=lambda s: (s.get("time") or {}).get("created") or 0, reverse=True) + return cands[0]["id"] if cands else None + +def _log_idle_min(): + try: + return (_time.time() - os.path.getmtime(LOG_FILE)) / 60.0 + except Exception: + return None + +def _go_key(): + try: + return (_json.load(open(AUTH_JSON)).get("opencode-go") or {}).get("key") + except Exception: + return None + +def _limit_retry_after(): + """0 if the opencode-go endpoint is available (HTTP 200); else the 429 retry-after seconds.""" + key = _go_key() + if not key: + return 0 + body = _json.dumps({"model": (MODEL or "").split("/")[-1] or "glm-5.2", "max_tokens": 8, + "messages": [{"role": "user", "content": "hi"}]}).encode() + req = _ureq.Request(GO_ENDPOINT, data=body, method="POST", + headers={"Authorization": "Bearer " + key, "content-type": "application/json"}) + try: + _ureq.urlopen(req, timeout=20).read(); return 0 + except _ureq.HTTPError as e: + if e.code == 429: + try: return max(1, int(e.headers.get("retry-after", "300"))) + except Exception: return 300 + return 0 + except Exception: + return 0 + +def _run_pids(): + """PIDs of live `opencode run` procs (via /proc scan — never matches this process).""" + me, out = os.getpid(), [] + for p in os.listdir("/proc"): + if not p.isdigit() or int(p) == me: + continue + try: + cl = open(f"/proc/{p}/cmdline", "rb").read().split(b"\0") + except Exception: + continue + if b"opencode" in (b" ".join(cl)) and b"run" in cl and b"--attach" in cl: + out.append(int(p)) + return out + +def _completed(): + try: + with open(LOG_FILE, errors="ignore") as f: + f.seek(0, 2); f.seek(max(0, f.tell() - 20000)); return DONE_MARKER in f.read() + except Exception: + return False + +def resume(reason="manual"): + """Resume the managed opencode session from where it stopped (context preserved).""" + import signal + sid = _session_id() + if not sid: + log(f"resume: no top-level '{SESSION}' session on {OPENCODE_SERVER} — cannot resume"); return False + log(f"resume ({reason}): continuing session {sid}") + for pid in _run_pids(): + try: os.kill(pid, signal.SIGTERM) + except Exception: pass + _time.sleep(2); kill_session(); _time.sleep(1) + kf = Path(LOG_DIR) / f".kickoff-{SESSION}-resume.txt" + kf.write_text( + "The opencode-go usage limit has reset (or the run stalled). You were mid-way through the weekly " + "cc-ci /upgrade-all run. CONTINUE from where you left off — do NOT start over. Process the enrolled " + "recipes not yet done this week, alphabetically; SKIP ones already done (their PRs exist — extend, " + "never duplicate). Per recipe: run /recipe-upgrade in DEFAULT mode via a subagent, verify with " + "!testme, open/extend the recipe PR (NEVER merge, NEVER weaken a test), <= DRONE_RUNNER_CAPACITY " + "concurrent. immich has a tag+digest image abra can't parse — do the upstream-direct cross-check " + "(recipe-upgrade SKILL §1), don't silently skip it. When all remaining recipes are done: " + "write+push the weekly summary, then `python3 /srv/cc-ci/cc-ci-plan/launch-report.py fresh`, print " + "'" + DONE_MARKER + "', and go idle.") + share = "--share" if OPENCODE_SHARE else "" + cmd = (f"set -a; . /srv/cc-ci/.testenv; set +a; {OPENCODE_BIN} run -s {sid} --continue " + f"--model '{MODEL}' {share} --attach '{OPENCODE_SERVER}' \"$(cat '{kf}')\"") + subprocess.run(["tmux", "new-session", "-d", "-s", SESSION, "-c", WORKDIR, cmd]) + subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION, f"cat >> '{LOG_FILE}'"]) + log(f"resume: relaunched {SESSION} (session {sid})"); return True + +def babysit(): + """Watch the opencode upgrader; on a stall, wait out any usage-limit then resume the session. + Exits when the run prints UPGRADE RUN COMPLETE. Spawned by an opencode `start`; also standalone.""" + log(f"babysit: watching {SESSION} (stall>{STALL_MIN}min log-idle, poll {CHECK_EVERY}s)") + misses = 0 + while True: + _time.sleep(CHECK_EVERY) + if _completed(): + log("babysit: run completed — exiting"); return + idle = _log_idle_min() + if idle is None or idle <= STALL_MIN: + misses = 0; continue + # Log has been static > STALL_MIN — the run is stalled (limit or wedge), not working. + retry = _limit_retry_after() + if retry > 0: + wait = min(retry + 30, 3600) + log(f"babysit: stalled {idle:.0f}min + usage-limited (retry-after {retry}s) — waiting {wait}s") + _time.sleep(wait); continue + # Not limited but stalled — confirm it's really wedged (two consecutive misses), then resume. + misses += 1 + if misses >= 2: + log(f"babysit: stalled {idle:.0f}min, limit clear — auto-resuming") + resume("babysit auto-resume"); misses = 0 # ── main ────────────────────────────────────────────────────────────────────── @@ -190,6 +331,10 @@ def main(): log(f"backend: {BACKEND} model: {MODEL} args: '{UPGRADER_ARGS or ''}'") elif cmd == "attach": os.execvp("tmux", ["tmux", "attach", "-t", SESSION]) + elif cmd == "resume": + resume("manual") + elif cmd == "babysit": + babysit() else: print(f"""cc-ci upgrader launcher — one-shot weekly recipe-upgrade job @@ -198,6 +343,8 @@ def main(): launch-upgrader.py stop kill the session launch-upgrader.py status show session state launch-upgrader.py attach tmux attach + launch-upgrader.py resume continue the opencode session from where it stalled (-s --continue) + launch-upgrader.py babysit watch + auto-resume the opencode run across usage-limit (429) stalls Backend: {BACKEND} (LOOP_BACKEND or UPGRADER_BACKEND env var) Model: {MODEL} (LOOP_MODEL or UPGRADER_MODEL env var)