From 0b6cc632d4a10ce26c43155aa0c9750fce5ad46e Mon Sep 17 00:00:00 2001 From: autonomic-bot Date: Tue, 4 Aug 2026 16:43:26 +0000 Subject: [PATCH] launch-upgrader: fix watchdog wrong-session resume + unique web-UI name invariant MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bug (2026-08-04 16:00): _session_id() sorted candidates on (s.time.created) which the /session API rows DON'T carry — every key was 0, 'newest' degraded to server list order, and the watchdog resumed the old giant unresumable session, kill_session()ing the healthy fresh run mid-work. Fixes: - Pin the managed session id at launch/resume to LOG_DIR/.{SESSION}-session-id; _session_id() prefers the pin, validated via direct GET /session/ (the LIST is paginated ~100 rows, membership scans lie). Title lookup is only the fallback and now sorts on authoritative sqlite time_created. - _archive_stale_titles() at start: every older top-level session titled cc-ci-upgrader is renamed 'upgrader archive — weekly ', so EXACTLY ONE session ever carries the canonical name in the opencode web UI (easy to find; finished runs stay browsable under archive names). 11 historical sessions renamed live today; the in-flight finisher pinned. Verified live: _session_id() returns the pinned running session; tree-idle 0.0min while subagents active. Full synthetic-stall watchdog confirmation queued post-run (task #13). --- cc-ci-plan/launch-upgrader.py | 101 +++++++++++++++++++++++++++++++++- 1 file changed, 98 insertions(+), 3 deletions(-) diff --git a/cc-ci-plan/launch-upgrader.py b/cc-ci-plan/launch-upgrader.py index 8f95d04..5783039 100644 --- a/cc-ci-plan/launch-upgrader.py +++ b/cc-ci-plan/launch-upgrader.py @@ -184,6 +184,18 @@ def start(mode="use-or-create"): if SESSION == "cc-ci-upgrader": prereclaim_cc_ci() + # Unique-name invariant: the run we are about to launch becomes THE 'cc-ci-upgrader' in the + # web UI; every older run gets an archive title. Also snapshot existing ids so the new + # session can be pinned unambiguously after launch. + _archive_stale_titles() + _prev_rows = _server_get("/session") or [] + _prev_rows = _prev_rows if isinstance(_prev_rows, list) else _prev_rows.get("data", []) + _prev_ids = {s.get("id") for s in _prev_rows} + try: + STATE_SID_FILE.unlink() + except OSError: + pass + kf = Path(LOG_DIR) / f".kickoff-{SESSION}.txt" kf.write_text(build_kickoff()) @@ -217,6 +229,8 @@ def start(mode="use-or-create"): subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION, f"cat >> '{LOG_DIR}/{SESSION}.log'"]) log(f"started. attach: tmux attach -t {SESSION} log: {LOG_DIR}/{SESSION}.log") + if BACKEND == "opencode": + _pin_new_session(_prev_ids) # For the opencode backend, spawn a watchdog that auto-resumes the run if the opencode # usage-limit (429) stalls it mid-run (it does NOT self-resume). See watchdog(). if BACKEND == "opencode" and os.environ.get("UPGRADER_WATCHDOG", "1") == "1": @@ -246,13 +260,90 @@ def _server_get(path): except Exception: return None +STATE_SID_FILE = Path(LOG_DIR) / f".{SESSION}-session-id" + +def _server_patch(path, body): + req = _ureq.Request(OPENCODE_SERVER + path, method="PATCH", + headers={"Content-Type": "application/json"}, + data=_json.dumps(body).encode()) + try: + with _ureq.urlopen(req, timeout=15) as r: + return r.status + except Exception: + return None + +def _db_created_ms(sid): + """Session creation time from the opencode sqlite DB — the /session API rows carry NO + time fields, which is exactly how the 2026-08-04 watchdog resumed the WRONG (old, giant) + session: sorting on a missing key degraded to server list order.""" + try: + import sqlite3 + db = sqlite3.connect("file:" + os.path.expanduser( + "~/.local/share/opencode/opencode.db") + "?mode=ro", uri=True) + row = db.execute("SELECT time_created FROM session WHERE id=?", (sid,)).fetchone() + db.close() + return row[0] if row else 0 + except Exception: + return 0 + +def _archive_stale_titles(): + """Rename every existing top-level session titled SESSION to a dated archive title, so + EXACTLY ONE session ever carries the canonical name (the one this launch creates). This + keeps the run trivially findable in the opencode web UI and makes the title lookup in + _session_id() unambiguous. Old runs stay browsable under 'upgrader archive — weekly '.""" + rows = _server_get("/session") or [] + rows = rows if isinstance(rows, list) else rows.get("data", []) + for s in rows: + if s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId")): + created = _db_created_ms(s["id"]) + d = datetime.fromtimestamp(created / 1000).strftime("%Y-%m-%d") if created else "unknown-date" + _server_patch(f"/session/{s['id']}", {"title": f"upgrader archive — weekly {d}"}) + log(f" archived old session {s['id'][:20]} → 'upgrader archive — weekly {d}'") + def _session_id(): - """Newest top-level opencode session titled like SESSION (the run we manage).""" + """The opencode session this launcher manages. Prefers the pinned id recorded at launch + (LOG_DIR/.{SESSION}-session-id) — title lookup is only the fallback, and thanks to + _archive_stale_titles() at most one top-level session carries the title. Never trust + server list order (see _db_created_ms).""" + try: + pinned = STATE_SID_FILE.read_text().strip() + # Validate directly — the /session LIST is paginated (~100 rows), so membership + # scans miss older/newer sessions; a direct GET is authoritative. + if pinned and (_server_get(f"/session/{pinned}") or {}).get("id") == pinned: + return pinned + except OSError: + pass rows = _server_get("/session") or [] rows = rows if isinstance(rows, list) else rows.get("data", []) cands = [s for s in rows if s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId"))] - cands.sort(key=lambda s: (s.get("time") or {}).get("created") or 0, reverse=True) - return cands[0]["id"] if cands else None + cands.sort(key=lambda s: _db_created_ms(s.get("id")), reverse=True) + if cands: + try: + STATE_SID_FILE.write_text(cands[0]["id"]) + except OSError: + pass + return cands[0]["id"] + return None + +def _pin_new_session(prev_ids, wait_sec=45): + """After launching a fresh run, discover the NEW top-level SESSION-titled session (one not + in prev_ids) and pin its id to the state file, so the watchdog can never grab an old one.""" + deadline = _time.time() + wait_sec + while _time.time() < deadline: + rows = _server_get("/session") or [] + rows = rows if isinstance(rows, list) else rows.get("data", []) + for s in rows: + if (s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId")) + and s.get("id") not in prev_ids): + try: + STATE_SID_FILE.write_text(s["id"]) + except OSError: + pass + log(f" pinned managed session id {s['id'][:20]} → {STATE_SID_FILE.name}") + return s["id"] + _time.sleep(3) + log(" WARNING: could not discover the new session id to pin (title lookup remains the fallback)") + return None def _session_idle_min(): """Minutes since the managed run last ADVANCED — measured across the whole session TREE (the @@ -399,6 +490,10 @@ def resume(reason="manual"): f"--model '{MODEL}' {share} --attach '{OPENCODE_SERVER}' --dir '{WORKDIR}' \"$(cat '{kf}')\"") subprocess.run(["tmux", "new-session", "-d", "-s", SESSION, "-c", WORKDIR, cmd]) subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION, f"cat >> '{LOG_FILE}'"]) + try: + STATE_SID_FILE.write_text(sid) # a resume continues the SAME session — keep it pinned + except OSError: + pass log(f"resume: relaunched {SESSION} (session {sid})") # Every resume must be self-healing: ensure a watchdog is watching this run. Skip if one is # already alive — notably when the watchdog ITSELF called resume (it lives in {SESSION}-watchdog),