diff --git a/cc-ci-plan/launch-supervisor.py b/cc-ci-plan/launch-supervisor.py index dc8b287..eef0549 100644 --- a/cc-ci-plan/launch-supervisor.py +++ b/cc-ci-plan/launch-supervisor.py @@ -143,9 +143,24 @@ def _gate(): # a live `opencode run … -s --attach` proc, or a log touched within the stall window. pids = lu._run_pids(sid) idle = lu._session_idle_min() - if pids or (idle is not None and idle < lu.STALL_MIN): - via = f"{len(pids)} live run proc(s)" if pids else f"log idle {idle:.0f}m < {lu.STALL_MIN:.0f}m" - return False, sid, f"upgrader run progressing ({via}) — leaving it" + advancing = idle is not None and idle < lu.STALL_MIN + # PROGRESS REQUIRES THE SESSION TO BE ADVANCING — a live proc alone is not enough. A run walled + # by the provider keeps its process alive and spinning while emitting nothing (2026-08-07: 3 + # days, zero output). Treating "proc exists" as progress is what deadlocked the gate. + if advancing: + return False, sid, f"upgrader run progressing (session advanced {idle:.0f}m ago) — leaving it" + if pids and lu._billing_blocked(): + # NEVER kill it: it may resume when the wall lifts, and its context is the run's state. + # Surface it loudly instead — this is an operator-actionable condition, not self-healing. + return False, sid, ( + f"run BLOCKED on a provider billing/usage wall ({len(pids)} proc(s) alive, session idle " + f"{idle:.0f}m) — NOT killing (resumable once the wall lifts); operator action required" + ) + if pids: + log( + f"note: {len(pids)} live run proc(s) but session idle {idle:.0f}m ≥ " + f"{lu.STALL_MIN:.0f}m — treating as stalled, not progressing" + ) # The per-run watchdog owns PROMPT recovery (resume on proc-death/stall) and is the single writer # while it lives. Defer to it — it gives up (exits its tmux) only after MAX_RESUMES fail, i.e. the # run is stuck in a way a bare resume can't fix (e.g. disk-full). THEN the supervisor takes over. diff --git a/cc-ci-plan/launch-upgrader.py b/cc-ci-plan/launch-upgrader.py index 3de72a5..f637f4f 100644 --- a/cc-ci-plan/launch-upgrader.py +++ b/cc-ci-plan/launch-upgrader.py @@ -417,25 +417,55 @@ def _limit_retry_after(): return 0 def _run_pids(sid=None): - """PIDs of live `opencode run` procs for THIS session (via /proc scan — never matches self).""" + """PIDs of live `opencode run` procs for THIS session (via /proc scan — never matches self). + + Matches on FLAG VALUES (`--title ` / `-s|--session `), never a substring of the + whole cmdline. The substring form caused the 2026-08-07 three-day deadlock: an agent's kickoff + PROMPT is passed as an argv element, and the supervisor's prompt text contains the literal + "cc-ci-upgrader", so the supervisor's own (billing-hung) agent matched as a live upgrader run — + the hourly gate then read its own corpse as "run progressing" and no-opped ~60 times.""" me, out = os.getpid(), [] for p in os.listdir("/proc"): if not p.isdigit() or int(p) == me: continue try: - cl = open(f"/proc/{p}/cmdline", "rb").read().split(b"\0") + cl = [c for c in open(f"/proc/{p}/cmdline", "rb").read().split(b"\0") if c] except Exception: continue - joined = b" ".join(cl) - if not (b"opencode" in joined and b"run" in cl and b"--attach" in cl): + if not cl or b"opencode" not in cl[0] or b"run" not in cl or b"--attach" not in cl: continue - # Scope to THIS managed session only: a fresh run carries `--title `, a resumed - # run carries `-s `. Without this, the report watchdog would kill the idle upgrader - # run (and vice-versa) since both are `opencode run … --attach`. - if SESSION.encode() in joined or (sid and sid.encode() in joined): + + def _flag(names): + for i, a in enumerate(cl[:-1]): + if a in names: + return cl[i + 1] + return None + + title = _flag((b"--title",)) + s_val = _flag((b"-s", b"--session")) + if title == SESSION.encode() or (sid and s_val == sid.encode()): out.append(int(p)) return out + +def _billing_blocked(window_bytes=4000): + """True when the tail of this run's log shows a provider billing/usage wall. + + Such a process must NEVER be killed: it may resume when the wall lifts, and killing it can + lose in-flight work. It is also NOT progress — it can spin for days emitting nothing (observed + 2026-08-07..10), so the gate reports it as BLOCKED and leaves recovery to the operator.""" + try: + with open(LOG_FILE, "rb") as f: + f.seek(0, os.SEEK_END) + f.seek(max(0, f.tell() - window_bytes)) + tail = f.read().decode(errors="replace").lower() + except OSError: + return False + return any( + s in tail + for s in ("spending limit", "insufficient balance", "usage limit", "usagelimiterror") + ) + def _completed(): # Done only when the MODEL signs off with DONE_MARKER as its FINAL word: the marker in the LAST # assistant TEXT (prose) message. This guards THREE false-positives that each abandoned a run: