launch-upgrader: fix watchdog wrong-session resume + unique web-UI name invariant

Bug (2026-08-04 16:00): _session_id() sorted candidates on (s.time.created) which the
/session API rows DON'T carry — every key was 0, 'newest' degraded to server list
order, and the watchdog resumed the old giant unresumable session, kill_session()ing
the healthy fresh run mid-work.

Fixes:
- Pin the managed session id at launch/resume to LOG_DIR/.{SESSION}-session-id;
  _session_id() prefers the pin, validated via direct GET /session/<id> (the LIST is
  paginated ~100 rows, membership scans lie). Title lookup is only the fallback and
  now sorts on authoritative sqlite time_created.
- _archive_stale_titles() at start: every older top-level session titled
  cc-ci-upgrader is renamed 'upgrader archive — weekly <date>', so EXACTLY ONE
  session ever carries the canonical name in the opencode web UI (easy to find;
  finished runs stay browsable under archive names). 11 historical sessions
  renamed live today; the in-flight finisher pinned.
Verified live: _session_id() returns the pinned running session; tree-idle 0.0min
while subagents active. Full synthetic-stall watchdog confirmation queued post-run
(task #13).
This commit is contained in:
autonomic-bot
2026-08-04 16:43:26 +00:00
parent e8d7d09445
commit 0b6cc632d4
+98 -3
View File
@@ -184,6 +184,18 @@ def start(mode="use-or-create"):
if SESSION == "cc-ci-upgrader":
prereclaim_cc_ci()
# Unique-name invariant: the run we are about to launch becomes THE 'cc-ci-upgrader' in the
# web UI; every older run gets an archive title. Also snapshot existing ids so the new
# session can be pinned unambiguously after launch.
_archive_stale_titles()
_prev_rows = _server_get("/session") or []
_prev_rows = _prev_rows if isinstance(_prev_rows, list) else _prev_rows.get("data", [])
_prev_ids = {s.get("id") for s in _prev_rows}
try:
STATE_SID_FILE.unlink()
except OSError:
pass
kf = Path(LOG_DIR) / f".kickoff-{SESSION}.txt"
kf.write_text(build_kickoff())
@@ -217,6 +229,8 @@ def start(mode="use-or-create"):
subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION,
f"cat >> '{LOG_DIR}/{SESSION}.log'"])
log(f"started. attach: tmux attach -t {SESSION} log: {LOG_DIR}/{SESSION}.log")
if BACKEND == "opencode":
_pin_new_session(_prev_ids)
# For the opencode backend, spawn a watchdog that auto-resumes the run if the opencode
# usage-limit (429) stalls it mid-run (it does NOT self-resume). See watchdog().
if BACKEND == "opencode" and os.environ.get("UPGRADER_WATCHDOG", "1") == "1":
@@ -246,13 +260,90 @@ def _server_get(path):
except Exception:
return None
STATE_SID_FILE = Path(LOG_DIR) / f".{SESSION}-session-id"
def _server_patch(path, body):
req = _ureq.Request(OPENCODE_SERVER + path, method="PATCH",
headers={"Content-Type": "application/json"},
data=_json.dumps(body).encode())
try:
with _ureq.urlopen(req, timeout=15) as r:
return r.status
except Exception:
return None
def _db_created_ms(sid):
"""Session creation time from the opencode sqlite DB — the /session API rows carry NO
time fields, which is exactly how the 2026-08-04 watchdog resumed the WRONG (old, giant)
session: sorting on a missing key degraded to server list order."""
try:
import sqlite3
db = sqlite3.connect("file:" + os.path.expanduser(
"~/.local/share/opencode/opencode.db") + "?mode=ro", uri=True)
row = db.execute("SELECT time_created FROM session WHERE id=?", (sid,)).fetchone()
db.close()
return row[0] if row else 0
except Exception:
return 0
def _archive_stale_titles():
"""Rename every existing top-level session titled SESSION to a dated archive title, so
EXACTLY ONE session ever carries the canonical name (the one this launch creates). This
keeps the run trivially findable in the opencode web UI and makes the title lookup in
_session_id() unambiguous. Old runs stay browsable under 'upgrader archive — weekly <date>'."""
rows = _server_get("/session") or []
rows = rows if isinstance(rows, list) else rows.get("data", [])
for s in rows:
if s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId")):
created = _db_created_ms(s["id"])
d = datetime.fromtimestamp(created / 1000).strftime("%Y-%m-%d") if created else "unknown-date"
_server_patch(f"/session/{s['id']}", {"title": f"upgrader archive — weekly {d}"})
log(f" archived old session {s['id'][:20]}'upgrader archive — weekly {d}'")
def _session_id():
"""Newest top-level opencode session titled like SESSION (the run we manage)."""
"""The opencode session this launcher manages. Prefers the pinned id recorded at launch
(LOG_DIR/.{SESSION}-session-id) — title lookup is only the fallback, and thanks to
_archive_stale_titles() at most one top-level session carries the title. Never trust
server list order (see _db_created_ms)."""
try:
pinned = STATE_SID_FILE.read_text().strip()
# Validate directly — the /session LIST is paginated (~100 rows), so membership
# scans miss older/newer sessions; a direct GET is authoritative.
if pinned and (_server_get(f"/session/{pinned}") or {}).get("id") == pinned:
return pinned
except OSError:
pass
rows = _server_get("/session") or []
rows = rows if isinstance(rows, list) else rows.get("data", [])
cands = [s for s in rows if s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId"))]
cands.sort(key=lambda s: (s.get("time") or {}).get("created") or 0, reverse=True)
return cands[0]["id"] if cands else None
cands.sort(key=lambda s: _db_created_ms(s.get("id")), reverse=True)
if cands:
try:
STATE_SID_FILE.write_text(cands[0]["id"])
except OSError:
pass
return cands[0]["id"]
return None
def _pin_new_session(prev_ids, wait_sec=45):
"""After launching a fresh run, discover the NEW top-level SESSION-titled session (one not
in prev_ids) and pin its id to the state file, so the watchdog can never grab an old one."""
deadline = _time.time() + wait_sec
while _time.time() < deadline:
rows = _server_get("/session") or []
rows = rows if isinstance(rows, list) else rows.get("data", [])
for s in rows:
if (s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId"))
and s.get("id") not in prev_ids):
try:
STATE_SID_FILE.write_text(s["id"])
except OSError:
pass
log(f" pinned managed session id {s['id'][:20]}{STATE_SID_FILE.name}")
return s["id"]
_time.sleep(3)
log(" WARNING: could not discover the new session id to pin (title lookup remains the fallback)")
return None
def _session_idle_min():
"""Minutes since the managed run last ADVANCED — measured across the whole session TREE (the
@@ -399,6 +490,10 @@ def resume(reason="manual"):
f"--model '{MODEL}' {share} --attach '{OPENCODE_SERVER}' --dir '{WORKDIR}' \"$(cat '{kf}')\"")
subprocess.run(["tmux", "new-session", "-d", "-s", SESSION, "-c", WORKDIR, cmd])
subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION, f"cat >> '{LOG_FILE}'"])
try:
STATE_SID_FILE.write_text(sid) # a resume continues the SAME session — keep it pinned
except OSError:
pass
log(f"resume: relaunched {SESSION} (session {sid})")
# Every resume must be self-healing: ensure a watchdog is watching this run. Skip if one is
# already alive — notably when the watchdog ITSELF called resume (it lives in {SESSION}-watchdog),