launch-upgrader: fix watchdog wrong-session resume + unique web-UI name invariant
Bug (2026-08-04 16:00): _session_id() sorted candidates on (s.time.created) which the
/session API rows DON'T carry — every key was 0, 'newest' degraded to server list
order, and the watchdog resumed the old giant unresumable session, kill_session()ing
the healthy fresh run mid-work.
Fixes:
- Pin the managed session id at launch/resume to LOG_DIR/.{SESSION}-session-id;
_session_id() prefers the pin, validated via direct GET /session/<id> (the LIST is
paginated ~100 rows, membership scans lie). Title lookup is only the fallback and
now sorts on authoritative sqlite time_created.
- _archive_stale_titles() at start: every older top-level session titled
cc-ci-upgrader is renamed 'upgrader archive — weekly <date>', so EXACTLY ONE
session ever carries the canonical name in the opencode web UI (easy to find;
finished runs stay browsable under archive names). 11 historical sessions
renamed live today; the in-flight finisher pinned.
Verified live: _session_id() returns the pinned running session; tree-idle 0.0min
while subagents active. Full synthetic-stall watchdog confirmation queued post-run
(task #13).
This commit is contained in:
@@ -184,6 +184,18 @@ def start(mode="use-or-create"):
|
||||
if SESSION == "cc-ci-upgrader":
|
||||
prereclaim_cc_ci()
|
||||
|
||||
# Unique-name invariant: the run we are about to launch becomes THE 'cc-ci-upgrader' in the
|
||||
# web UI; every older run gets an archive title. Also snapshot existing ids so the new
|
||||
# session can be pinned unambiguously after launch.
|
||||
_archive_stale_titles()
|
||||
_prev_rows = _server_get("/session") or []
|
||||
_prev_rows = _prev_rows if isinstance(_prev_rows, list) else _prev_rows.get("data", [])
|
||||
_prev_ids = {s.get("id") for s in _prev_rows}
|
||||
try:
|
||||
STATE_SID_FILE.unlink()
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
kf = Path(LOG_DIR) / f".kickoff-{SESSION}.txt"
|
||||
kf.write_text(build_kickoff())
|
||||
|
||||
@@ -217,6 +229,8 @@ def start(mode="use-or-create"):
|
||||
subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION,
|
||||
f"cat >> '{LOG_DIR}/{SESSION}.log'"])
|
||||
log(f"started. attach: tmux attach -t {SESSION} log: {LOG_DIR}/{SESSION}.log")
|
||||
if BACKEND == "opencode":
|
||||
_pin_new_session(_prev_ids)
|
||||
# For the opencode backend, spawn a watchdog that auto-resumes the run if the opencode
|
||||
# usage-limit (429) stalls it mid-run (it does NOT self-resume). See watchdog().
|
||||
if BACKEND == "opencode" and os.environ.get("UPGRADER_WATCHDOG", "1") == "1":
|
||||
@@ -246,13 +260,90 @@ def _server_get(path):
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
STATE_SID_FILE = Path(LOG_DIR) / f".{SESSION}-session-id"
|
||||
|
||||
def _server_patch(path, body):
|
||||
req = _ureq.Request(OPENCODE_SERVER + path, method="PATCH",
|
||||
headers={"Content-Type": "application/json"},
|
||||
data=_json.dumps(body).encode())
|
||||
try:
|
||||
with _ureq.urlopen(req, timeout=15) as r:
|
||||
return r.status
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _db_created_ms(sid):
|
||||
"""Session creation time from the opencode sqlite DB — the /session API rows carry NO
|
||||
time fields, which is exactly how the 2026-08-04 watchdog resumed the WRONG (old, giant)
|
||||
session: sorting on a missing key degraded to server list order."""
|
||||
try:
|
||||
import sqlite3
|
||||
db = sqlite3.connect("file:" + os.path.expanduser(
|
||||
"~/.local/share/opencode/opencode.db") + "?mode=ro", uri=True)
|
||||
row = db.execute("SELECT time_created FROM session WHERE id=?", (sid,)).fetchone()
|
||||
db.close()
|
||||
return row[0] if row else 0
|
||||
except Exception:
|
||||
return 0
|
||||
|
||||
def _archive_stale_titles():
|
||||
"""Rename every existing top-level session titled SESSION to a dated archive title, so
|
||||
EXACTLY ONE session ever carries the canonical name (the one this launch creates). This
|
||||
keeps the run trivially findable in the opencode web UI and makes the title lookup in
|
||||
_session_id() unambiguous. Old runs stay browsable under 'upgrader archive — weekly <date>'."""
|
||||
rows = _server_get("/session") or []
|
||||
rows = rows if isinstance(rows, list) else rows.get("data", [])
|
||||
for s in rows:
|
||||
if s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId")):
|
||||
created = _db_created_ms(s["id"])
|
||||
d = datetime.fromtimestamp(created / 1000).strftime("%Y-%m-%d") if created else "unknown-date"
|
||||
_server_patch(f"/session/{s['id']}", {"title": f"upgrader archive — weekly {d}"})
|
||||
log(f" archived old session {s['id'][:20]} → 'upgrader archive — weekly {d}'")
|
||||
|
||||
def _session_id():
|
||||
"""Newest top-level opencode session titled like SESSION (the run we manage)."""
|
||||
"""The opencode session this launcher manages. Prefers the pinned id recorded at launch
|
||||
(LOG_DIR/.{SESSION}-session-id) — title lookup is only the fallback, and thanks to
|
||||
_archive_stale_titles() at most one top-level session carries the title. Never trust
|
||||
server list order (see _db_created_ms)."""
|
||||
try:
|
||||
pinned = STATE_SID_FILE.read_text().strip()
|
||||
# Validate directly — the /session LIST is paginated (~100 rows), so membership
|
||||
# scans miss older/newer sessions; a direct GET is authoritative.
|
||||
if pinned and (_server_get(f"/session/{pinned}") or {}).get("id") == pinned:
|
||||
return pinned
|
||||
except OSError:
|
||||
pass
|
||||
rows = _server_get("/session") or []
|
||||
rows = rows if isinstance(rows, list) else rows.get("data", [])
|
||||
cands = [s for s in rows if s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId"))]
|
||||
cands.sort(key=lambda s: (s.get("time") or {}).get("created") or 0, reverse=True)
|
||||
return cands[0]["id"] if cands else None
|
||||
cands.sort(key=lambda s: _db_created_ms(s.get("id")), reverse=True)
|
||||
if cands:
|
||||
try:
|
||||
STATE_SID_FILE.write_text(cands[0]["id"])
|
||||
except OSError:
|
||||
pass
|
||||
return cands[0]["id"]
|
||||
return None
|
||||
|
||||
def _pin_new_session(prev_ids, wait_sec=45):
|
||||
"""After launching a fresh run, discover the NEW top-level SESSION-titled session (one not
|
||||
in prev_ids) and pin its id to the state file, so the watchdog can never grab an old one."""
|
||||
deadline = _time.time() + wait_sec
|
||||
while _time.time() < deadline:
|
||||
rows = _server_get("/session") or []
|
||||
rows = rows if isinstance(rows, list) else rows.get("data", [])
|
||||
for s in rows:
|
||||
if (s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId"))
|
||||
and s.get("id") not in prev_ids):
|
||||
try:
|
||||
STATE_SID_FILE.write_text(s["id"])
|
||||
except OSError:
|
||||
pass
|
||||
log(f" pinned managed session id {s['id'][:20]} → {STATE_SID_FILE.name}")
|
||||
return s["id"]
|
||||
_time.sleep(3)
|
||||
log(" WARNING: could not discover the new session id to pin (title lookup remains the fallback)")
|
||||
return None
|
||||
|
||||
def _session_idle_min():
|
||||
"""Minutes since the managed run last ADVANCED — measured across the whole session TREE (the
|
||||
@@ -399,6 +490,10 @@ def resume(reason="manual"):
|
||||
f"--model '{MODEL}' {share} --attach '{OPENCODE_SERVER}' --dir '{WORKDIR}' \"$(cat '{kf}')\"")
|
||||
subprocess.run(["tmux", "new-session", "-d", "-s", SESSION, "-c", WORKDIR, cmd])
|
||||
subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION, f"cat >> '{LOG_FILE}'"])
|
||||
try:
|
||||
STATE_SID_FILE.write_text(sid) # a resume continues the SAME session — keep it pinned
|
||||
except OSError:
|
||||
pass
|
||||
log(f"resume: relaunched {SESSION} (session {sid})")
|
||||
# Every resume must be self-healing: ensure a watchdog is watching this run. Skip if one is
|
||||
# already alive — notably when the watchdog ITSELF called resume (it lives in {SESSION}-watchdog),
|
||||
|
||||
Reference in New Issue
Block a user