#!/usr/bin/env python3
"""
cc-ci upgrader launcher — one-shot weekly recipe-upgrade job agent.

The upgrader runs /upgrade-all to completion, then stops and stays idle so the
run + summary remain viewable in the web UI. The next weekly run starts a fresh
session (start clears any idle/finished session).

Usage:
  launch-upgrader.py start    use-or-create: leave an in-flight run alone, else start fresh
  launch-upgrader.py fresh    always kill any existing session and start fresh
  launch-upgrader.py stop     kill the session
  launch-upgrader.py status   show session state
  launch-upgrader.py attach   tmux attach to the session

Env:
  LOOP_BACKEND     opencode (default) | claude   — also accepts UPGRADER_BACKEND
  LOOP_TIER       opencode subscription tier: "zen" (OpenCode ZEN, default) or "go"
                   (OpenCode Go). Selects the default model + the usage-limit probe
                   endpoint/key. Go hit a monthly limit; ZEN is the working alternative.
                   Only affects the opencode backend (ignored for claude).
  LOOP_MODEL       model flag (overrides UPGRADER_MODEL); default tracks backend+tier —
                   opencode+zen→opencode/glm-5.2, opencode+go→opencode-go/glm-5.2, claude→sonnet
  UPGRADER_MODEL   provider/model for opencode, e.g. opencode/glm-5.2 (OpenCode ZEN),
                   opencode-go/glm-5.2 (OpenCode Go), or tinfoil/deepseek-v4-pro; sonnet for claude
  UPGRADER_ARGS    extra args passed to /upgrade-all (e.g. "n8n ghost", "--dry-run")

  claude backend:
    CLAUDE_BIN, CLAUDE_FLAGS, REMOTE_CONTROL
  opencode backend:
    OPENCODE_BIN, OPENCODE_SERVER, OPENCODE_SHARE (1=attach to web server + public --share link)
"""

import os, sys, subprocess, re
from datetime import datetime
from pathlib import Path

# ── config ────────────────────────────────────────────────────────────────────

SESSION = os.environ.get("UPGRADER_SESSION", "cc-ci-upgrader")
WORKDIR = os.environ.get("UPGRADER_DIR",     "/srv/cc-ci")
LOG_DIR = os.environ.get("LOG_DIR",          "/srv/cc-ci/.cc-ci-logs")

# LOOP_BACKEND / LOOP_MODEL take precedence (unified control from the operator).
# LOOP_TIER selects the OpenCode subscription: "zen" (default) or "go". Go hit a monthly
# usage limit; ZEN is the working alternative. Set LOOP_TIER=go when the Go limit resets.
# Only affects the opencode backend; claude ignores the tier.
BACKEND = os.environ.get("LOOP_BACKEND", os.environ.get("UPGRADER_BACKEND", "opencode"))
TIER = os.environ.get("LOOP_TIER", os.environ.get("UPGRADER_TIER", "zen"))
_TIER_CFG = {
    "go":  {"model": "opencode-go/glm-5.2", "endpoint": "https://opencode.ai/zen/go/v1/chat/completions",
            "auth": "opencode-go", "label": "OpenCode Go"},
    "zen": {"model": "opencode/glm-5.2",    "endpoint": "https://opencode.ai/zen/v1/chat/completions",
            "auth": "opencode",    "label": "OpenCode ZEN"},
}
if BACKEND == "opencode" and TIER not in _TIER_CFG:
    print(f"[upgrader] ERROR: unknown LOOP_TIER '{TIER}' — use 'go' or 'zen'", flush=True)
    sys.exit(1)
_tier = _TIER_CFG.get(TIER, _TIER_CFG["zen"])
_DEFAULT_MODEL = _tier["model"] if BACKEND == "opencode" else "sonnet"
MODEL   = os.environ.get("LOOP_MODEL",   os.environ.get("UPGRADER_MODEL",   _DEFAULT_MODEL))

CLAUDE_BIN     = os.environ.get("CLAUDE_BIN",   "claude")
CLAUDE_FLAGS   = os.environ.get("CLAUDE_FLAGS", "--dangerously-skip-permissions")
REMOTE_CONTROL = os.environ.get("REMOTE_CONTROL", "1") == "1"

OPENCODE_BIN    = os.environ.get("OPENCODE_BIN",    "/home/loops/.local/bin/opencode")
OPENCODE_SERVER = os.environ.get("OPENCODE_SERVER", "http://127.0.0.1:4096")
# Web visibility for the opencode backend: attach the session to the shared opencode
# web server (viewable at http://oc.commoninternet.net, tailnet-only) AND optionally
# create a public opencode.ai --share link. Default both on so the run is monitorable.
OPENCODE_SHARE = os.environ.get("OPENCODE_SHARE", "1") == "1"

UPGRADER_ARGS = os.environ.get("UPGRADER_ARGS", "")

# First step of the weekly run: reclaim STALE docker images on the cc-ci server BEFORE the run so a
# heavy run can't fill the disk mid-flight (root cause of the 2026-07-03 stall — 100% ENOSPC killed
# lasuite-drive + wedged the run). "Stale" = unused by any container AND older than PRERECLAIM_UNTIL,
# so recently-built/pulled images (the ones this week's tests will reuse) are KEPT — we only evict
# leftovers from prior weeks. Best-effort; never fails the run.
PRERECLAIM       = os.environ.get("UPGRADER_PRERECLAIM", "1") == "1"
PRERECLAIM_UNTIL = os.environ.get("UPGRADER_PRERECLAIM_UNTIL", "168h")  # 7d: older than one run ago
PRERECLAIM_HOST  = os.environ.get("UPGRADER_PRERECLAIM_HOST", "cc-ci")

# ── helpers ───────────────────────────────────────────────────────────────────

def log(msg):
    ts = datetime.now().strftime("%H:%M:%S")
    print(f"[upgrader {ts}] {msg}", flush=True)

def die(msg):
    log(f"ERROR: {msg}")
    sys.exit(1)

def session_alive():
    return subprocess.run(
        ["tmux", "has-session", "-t", SESSION], capture_output=True
    ).returncode == 0

def session_busy():
    """True while a turn is actively in flight (not idle/finished/wedged)."""
    r = subprocess.run(["tmux", "capture-pane", "-pt", SESSION],
                       capture_output=True, text=True)
    pane = r.stdout if r.returncode == 0 else ""
    return bool(re.search(r"esc to interrupt|⠋|⠙|⠹|⠸|⠼|⠴|⠦|⠧|⠇|⠏|Running tool", pane))

def kill_session():
    subprocess.run(["tmux", "kill-session", "-t", SESSION], capture_output=True)

def _watchdog_alive():
    return subprocess.run(["tmux", "has-session", "-t", f"{SESSION}-watchdog"],
                          capture_output=True).returncode == 0

def prereclaim_cc_ci():
    """Weekly-run step 0: prune STALE (unused AND older than PRERECLAIM_UNTIL) docker images on the
    cc-ci server so the run has disk headroom. Keeps recent images (reused this week); only clears
    prior-weeks' leftovers. Best-effort — a reclaim failure must never abort the run."""
    if not PRERECLAIM:
        return
    filt = f"--filter until={PRERECLAIM_UNTIL}"
    remote = (f"docker image prune -af {filt} 2>&1 | tail -1; "
              f"docker builder prune -af {filt} >/dev/null 2>&1 || true; "
              f"df -h / | tail -1")
    log(f"  step 0: pre-reclaim stale docker images on {PRERECLAIM_HOST} (unused & >{PRERECLAIM_UNTIL})")
    try:
        r = subprocess.run(["ssh", "-o", "ConnectTimeout=15", PRERECLAIM_HOST, remote],
                           capture_output=True, text=True, timeout=900)
        out = (r.stdout or r.stderr or "").strip()
        for ln in out.splitlines():
            if ln.strip():
                log(f"    {ln.strip()}")
    except Exception as e:
        log(f"    pre-reclaim skipped (non-fatal): {e}")

# ── kickoff prompt ────────────────────────────────────────────────────────────

def build_kickoff():
    args_note = f" with arguments: {UPGRADER_ARGS}" if UPGRADER_ARGS else ""
    return f"""\
*** cc-ci UPGRADER — weekly recipe-upgrade job ***
You are the cc-ci Upgrader: a ONE-SHOT job agent, NOT a perpetual loop. Run the
recipe-upgrade sequence to completion, then STOP. Your cwd is {WORKDIR}; reach the CI
server with `ssh cc-ci`; creds are in {WORKDIR}/.testenv; skills in {WORKDIR}/.claude/skills/.

DO THIS:
1. Invoke the /upgrade-all skill in DEFAULT mode{args_note}
   (read {WORKDIR}/.claude/skills/upgrade-all/SKILL.md for the full procedure). It surveys
   every enrolled recipe and, for each upgradeable one, runs /recipe-upgrade in DEFAULT
   mode — recipe PR only, verified by posting `!testme` on the PR (results visible in the
   PR, iterate up to 3x). A genuinely stale test gets an explanatory PR COMMENT, never a
   test edit.
2. Process recipes via per-recipe SUBAGENTS so your own context stays light. If your
   context usage climbs (~80%), run /compact before continuing.
3. Write + push the weekly summary (the PR list is the actionable output for the operator).
4. WHEN THE RUN IS COMPLETE: STOP. Print the final summary (lead with the PR list) and an
   `UPGRADE RUN COMPLETE` line, then go idle. Do NOT loop, do NOT re-run, and do NOT kill
   your own session — leave it up so the operator can review the output in the web UI.
   Next week's run starts a fresh session (the launcher clears this idle one).

GUARDRAILS: NEVER merge any PR. NEVER weaken a test. DEFAULT mode only — do NOT pass
--with-tests (updating cc-ci tests is the operator's per-recipe opt-in). Single-writer:
dedicated branches + separate clones, never push main, never touch the build loops'
/cc-ci /cc-ci-adv clones. The shared Swarm is stateful — go sequentially.
"""

# ── launch ────────────────────────────────────────────────────────────────────

def start(mode="use-or-create"):
    import shutil
    if not shutil.which("tmux"):
        die("tmux not found")
    Path(LOG_DIR).mkdir(parents=True, exist_ok=True)

    if session_alive():
        if mode == "use-or-create" and session_busy():
            log(f"{SESSION} already running a job (busy) — leaving it")
            return
        log(f"{SESSION} exists but idle/stale (or fresh requested) — killing it first")
        kill_session()
        import time; time.sleep(1)

    # Step 0 of the weekly run: clear STALE cc-ci docker images so a heavy run can't run the disk
    # out mid-flight (root cause of the 2026-07-03 stall). Only for the actual upgrade run.
    if SESSION == "cc-ci-upgrader":
        prereclaim_cc_ci()

    # Unique-name invariant: the run we are about to launch becomes THE 'cc-ci-upgrader' in the
    # web UI; every older run gets an archive title. Also snapshot existing ids so the new
    # session can be pinned unambiguously after launch.
    _archive_stale_titles()
    _prev_rows = _server_get("/session") or []
    _prev_rows = _prev_rows if isinstance(_prev_rows, list) else _prev_rows.get("data", [])
    _prev_ids = {s.get("id") for s in _prev_rows}
    try:
        STATE_SID_FILE.unlink()
    except OSError:
        pass

    kf = Path(LOG_DIR) / f".kickoff-{SESSION}.txt"
    kf.write_text(build_kickoff())

    model_flag = f"--model '{MODEL}'" if MODEL else ""
    tier_note = f", tier={TIER} ({_tier['label']})" if BACKEND == "opencode" else ""
    log(f"starting {SESSION} (backend={BACKEND}{tier_note}, model={MODEL}, args='{UPGRADER_ARGS or '<none>'}')")

    if BACKEND == "claude":
        if not shutil.which(CLAUDE_BIN):
            die(f"claude CLI not found — set CLAUDE_BIN (currently: {CLAUDE_BIN})")
        rc  = f"--remote-control '{SESSION}'" if REMOTE_CONTROL else ""
        cmd = f"{CLAUDE_BIN} {rc} {model_flag} {CLAUDE_FLAGS} \"$(cat '{kf}')\""

    elif BACKEND == "opencode":
        if not Path(OPENCODE_BIN).exists():
            die(f"opencode not found at {OPENCODE_BIN}")
        # NOTE: -m/--model and --attach/--title/--share are flags on the `run` SUBCOMMAND,
        # so they must come AFTER `run` (a global `opencode --model X run` is ignored).
        share_flag = "--share" if OPENCODE_SHARE else ""
        cmd = (
            f"set -a; . /srv/cc-ci/.testenv; set +a; "
            f"{OPENCODE_BIN} run {model_flag} {share_flag} --attach '{OPENCODE_SERVER}' "
            f"--title '{SESSION}' \"$(cat '{kf}')\""
        )
        log(f"  attached to {OPENCODE_SERVER} → http://oc.commoninternet.net (tailnet only)"
            + ("  +public --share link (printed in the session)" if OPENCODE_SHARE else ""))
    else:
        die(f"unknown LOOP_BACKEND '{BACKEND}' — use 'claude' or 'opencode'")

    subprocess.run(["tmux", "new-session", "-d", "-s", SESSION, "-c", WORKDIR, cmd])
    subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION,
                    f"cat >> '{LOG_DIR}/{SESSION}.log'"])
    log(f"started. attach: tmux attach -t {SESSION}  log: {LOG_DIR}/{SESSION}.log")
    if BACKEND == "opencode":
        _pin_new_session(_prev_ids)
    # For the opencode backend, spawn a watchdog that auto-resumes the run if the opencode
    # usage-limit (429) stalls it mid-run (it does NOT self-resume). See watchdog().
    if BACKEND == "opencode" and os.environ.get("UPGRADER_WATCHDOG", "1") == "1":
        _spawn_watchdog()

# ── opencode stall-detect + auto-resume watchdog ────────────────────────────────
# The opencode subscription (Go or ZEN tier) enforces a rolling usage-limit (HTTP 429 + retry-after). When it
# trips mid-run, the `opencode run` agent loop ENDS and does NOT self-resume. This watchdog detects
# the stall (the session log stops growing), waits out the limit, and resumes the SAME session —
# context preserved — via `opencode run -s <id> --continue`. Standalone: launch-upgrader.py {resume|watchdog}.
import json as _json, urllib.request as _ureq, time as _time

STALL_MIN   = float(os.environ.get("UPGRADER_STALL_MIN", "15"))   # log-idle minutes ⇒ stalled
CHECK_EVERY = int(os.environ.get("UPGRADER_CHECK_SEC", "180"))    # watchdog poll cadence
# Generic so the SAME watchdog also covers the report job (launch-report.py points it at the
# cc-ci-report session with its own marker + resume prompt via these env vars).
DONE_MARKER = os.environ.get("UPGRADER_DONE_MARKER", "UPGRADE RUN COMPLETE")
RESUME_FILE = os.environ.get("UPGRADER_RESUME_FILE")  # optional path to a custom resume prompt
LIMIT_ENDPOINT = _tier["endpoint"]   # tier-aware: zen or go
AUTH_JSON   = os.path.expanduser("~/.local/share/opencode/auth.json")
LOG_FILE    = f"{LOG_DIR}/{SESSION}.log"

def _server_get(path):
    try:
        with _ureq.urlopen(OPENCODE_SERVER + path, timeout=15) as r:
            return _json.load(r)
    except Exception:
        return None

STATE_SID_FILE = Path(LOG_DIR) / f".{SESSION}-session-id"

def _server_patch(path, body):
    req = _ureq.Request(OPENCODE_SERVER + path, method="PATCH",
                        headers={"Content-Type": "application/json"},
                        data=_json.dumps(body).encode())
    try:
        with _ureq.urlopen(req, timeout=15) as r:
            return r.status
    except Exception:
        return None

def _db_created_ms(sid):
    """Session creation time from the opencode sqlite DB — the /session API rows carry NO
    time fields, which is exactly how the 2026-08-04 watchdog resumed the WRONG (old, giant)
    session: sorting on a missing key degraded to server list order."""
    try:
        import sqlite3
        db = sqlite3.connect("file:" + os.path.expanduser(
            "~/.local/share/opencode/opencode.db") + "?mode=ro", uri=True)
        row = db.execute("SELECT time_created FROM session WHERE id=?", (sid,)).fetchone()
        db.close()
        return row[0] if row else 0
    except Exception:
        return 0

def _archive_stale_titles(title=None, label=None):
    """Rename every existing top-level session with the given canonical title to a dated
    archive title, so EXACTLY ONE session ever carries the canonical name (the one the next
    launch creates). Keeps the run trivially findable in the opencode web UI and makes the
    title lookup in _session_id() unambiguous. Old runs stay browsable under
    '<label> — <date>'. Also used by launch-supervisor.py for its own session name."""
    title = title or SESSION
    # Archive names always start with 'archive-' (operator convention 2026-08-04) so they
    # sort/filter together in the web UI: 'archive-<original-title> — <date>'.
    label = label or f"archive-{title} —"
    rows = _server_get("/session") or []
    rows = rows if isinstance(rows, list) else rows.get("data", [])
    for s in rows:
        if s.get("title") == title and not (s.get("parentID") or s.get("parentId")):
            created = _db_created_ms(s["id"])
            d = datetime.fromtimestamp(created / 1000).strftime("%Y-%m-%d") if created else "unknown-date"
            _server_patch(f"/session/{s['id']}", {"title": f"{label} {d}"})
            log(f"  archived old session {s['id'][:20]} → '{label} {d}'")

def _session_id():
    """The opencode session this launcher manages. Prefers the pinned id recorded at launch
    (LOG_DIR/.{SESSION}-session-id) — title lookup is only the fallback, and thanks to
    _archive_stale_titles() at most one top-level session carries the title. Never trust
    server list order (see _db_created_ms)."""
    try:
        pinned = STATE_SID_FILE.read_text().strip()
        # Validate directly — the /session LIST is paginated (~100 rows), so membership
        # scans miss older/newer sessions; a direct GET is authoritative.
        if pinned and (_server_get(f"/session/{pinned}") or {}).get("id") == pinned:
            return pinned
    except OSError:
        pass
    rows = _server_get("/session") or []
    rows = rows if isinstance(rows, list) else rows.get("data", [])
    cands = [s for s in rows if s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId"))]
    cands.sort(key=lambda s: _db_created_ms(s.get("id")), reverse=True)
    if cands:
        try:
            STATE_SID_FILE.write_text(cands[0]["id"])
        except OSError:
            pass
        return cands[0]["id"]
    return None

def _pin_new_session(prev_ids, wait_sec=45):
    """After launching a fresh run, discover the NEW top-level SESSION-titled session (one not
    in prev_ids) and pin its id to the state file, so the watchdog can never grab an old one."""
    deadline = _time.time() + wait_sec
    while _time.time() < deadline:
        rows = _server_get("/session") or []
        rows = rows if isinstance(rows, list) else rows.get("data", [])
        for s in rows:
            if (s.get("title") == SESSION and not (s.get("parentID") or s.get("parentId"))
                    and s.get("id") not in prev_ids):
                try:
                    STATE_SID_FILE.write_text(s["id"])
                except OSError:
                    pass
                log(f"  pinned managed session id {s['id'][:20]} → {STATE_SID_FILE.name}")
                return s["id"]
        _time.sleep(3)
    log("  WARNING: could not discover the new session id to pin (title lookup remains the fallback)")
    return None

def _session_idle_min():
    """Minutes since the managed run last ADVANCED — measured across the whole session TREE (the
    top-level cc-ci-upgrader session AND every descendant subagent). Uses the opencode SERVER's
    time.updated (authoritative — bumps on every new message/part) rather than the tmux log mtime,
    which FREEZES when a headless `opencode run --continue` doesn't stream to the pane (that froze
    signal made the watchdog resume-storm). Crucially it must include CHILDREN: while a per-recipe
    subagent runs (a deploy can take 20-40min) the PARENT's updated time is stale, so reading the
    parent alone would look 'idle' and trigger a false resume that kills the productive run. Falls
    back to the log mtime only when the server is unreachable."""
    sid = _session_id()
    if not sid:
        try:
            return (_time.time() - os.path.getmtime(LOG_FILE)) / 60.0
        except Exception:
            return None
    rows = _server_get("/session") or []
    rows = rows if isinstance(rows, list) else rows.get("data", [])
    def _parent(s): return s.get("parentID") or s.get("parentId")
    # Transitive closure of sid over parent links → the managed session + all its descendants.
    tree = {sid}
    for _ in range(8):  # bounded fixpoint (subagents nest only a few deep)
        grew = False
        for s in rows:
            if s.get("id") not in tree and _parent(s) in tree:
                tree.add(s.get("id")); grew = True
        if not grew:
            break
    newest = 0
    for s in rows:
        if s.get("id") in tree:
            newest = max(newest, (s.get("time") or {}).get("updated") or 0)
    if newest:
        return max(0.0, (_time.time() * 1000 - newest) / 60000.0)
    try:
        return (_time.time() - os.path.getmtime(LOG_FILE)) / 60.0
    except Exception:
        return None

# Back-compat alias (older callers / the report watchdog import this name).
_log_idle_min = _session_idle_min

def _limit_key():
    try:
        return (_json.load(open(AUTH_JSON)).get(_tier["auth"]) or {}).get("key")
    except Exception:
        return None

def _limit_retry_after():
    """0 if the tier's opencode endpoint is available (HTTP 200); else the 429 retry-after seconds."""
    key = _limit_key()
    if not key:
        return 0
    body = _json.dumps({"model": (MODEL or "").split("/")[-1] or "glm-5.2", "max_tokens": 8,
                        "messages": [{"role": "user", "content": "hi"}]}).encode()
    req = _ureq.Request(LIMIT_ENDPOINT, data=body, method="POST",
                        headers={"Authorization": "Bearer " + key, "content-type": "application/json"})
    try:
        _ureq.urlopen(req, timeout=20).read(); return 0
    except _ureq.HTTPError as e:
        if e.code == 429:
            try: return max(1, int(e.headers.get("retry-after", "300")))
            except Exception: return 300
        return 0
    except Exception:
        return 0

def _run_pids(sid=None):
    """PIDs of live `opencode run` procs for THIS session (via /proc scan — never matches self).

    Matches on FLAG VALUES (`--title <SESSION>` / `-s|--session <sid>`), never a substring of the
    whole cmdline. The substring form caused the 2026-08-07 three-day deadlock: an agent's kickoff
    PROMPT is passed as an argv element, and the supervisor's prompt text contains the literal
    "cc-ci-upgrader", so the supervisor's own (billing-hung) agent matched as a live upgrader run —
    the hourly gate then read its own corpse as "run progressing" and no-opped ~60 times."""
    me, out = os.getpid(), []
    for p in os.listdir("/proc"):
        if not p.isdigit() or int(p) == me:
            continue
        try:
            cl = [c for c in open(f"/proc/{p}/cmdline", "rb").read().split(b"\0") if c]
        except Exception:
            continue
        if not cl or b"opencode" not in cl[0] or b"run" not in cl or b"--attach" not in cl:
            continue

        def _flag(names):
            for i, a in enumerate(cl[:-1]):
                if a in names:
                    return cl[i + 1]
            return None

        title = _flag((b"--title",))
        s_val = _flag((b"-s", b"--session"))
        if title == SESSION.encode() or (sid and s_val == sid.encode()):
            out.append(int(p))
    return out


def _billing_blocked(window_bytes=4000):
    """True when the tail of this run's log shows a provider billing/usage wall.

    Such a process must NEVER be killed: it may resume when the wall lifts, and killing it can
    lose in-flight work. It is also NOT progress — it can spin for days emitting nothing (observed
    2026-08-07..10), so the gate reports it as BLOCKED and leaves recovery to the operator."""
    try:
        with open(LOG_FILE, "rb") as f:
            f.seek(0, os.SEEK_END)
            f.seek(max(0, f.tell() - window_bytes))
            tail = f.read().decode(errors="replace").lower()
    except OSError:
        return False
    return any(
        s in tail
        for s in ("spending limit", "insufficient balance", "usage limit", "usagelimiterror")
    )

def _completed():
    # Done only when the MODEL signs off with DONE_MARKER as its FINAL word: the marker in the LAST
    # assistant TEXT (prose) message. This guards THREE false-positives that each abandoned a run:
    #   (1) the kickoff/resume PROMPT contains the marker — but that's a USER message (skipped).
    #   (2) the marker inside a TOOL part — a subagent `task` prompt or a bash command that echoes
    #       "print <marker>" — so we look ONLY at type=='text' prose, never tool-call args.
    #   (3) the model ECHOING the instruction mid-run ("…then I'll print <marker>") — only the FINAL
    #       assistant prose counts, so any further work after the echo means it's NOT done.
    sid = _session_id()
    msgs = _server_get(f"/session/{sid}/message") if sid else None
    if msgs is not None:
        msgs = msgs if isinstance(msgs, list) else msgs.get("data", [])
        last_prose = None
        for m in msgs:
            if ((m.get("info") or {}).get("role")) != "assistant":
                continue
            prose = "".join(p.get("text", "") for p in (m.get("parts") or [])
                            if p.get("type") == "text" and isinstance(p.get("text"), str))
            if prose.strip():
                last_prose = prose
        return bool(last_prose and DONE_MARKER in last_prose)
    # Server unreachable → conservative log fallback that excludes the prompt's own mention.
    try:
        with open(LOG_FILE, errors="ignore") as f:
            f.seek(0, 2); f.seek(max(0, f.tell() - 20000))
            return any(DONE_MARKER in ln and "print" not in ln and "'" not in ln for ln in f)
    except Exception:
        return False

def resume(reason="manual"):
    """Resume the managed opencode session from where it stopped (context preserved)."""
    import signal
    sid = _session_id()
    if not sid:
        log(f"resume: no top-level '{SESSION}' session on {OPENCODE_SERVER} — cannot resume"); return False
    log(f"resume ({reason}): continuing session {sid}")
    for pid in _run_pids(sid):
        try: os.kill(pid, signal.SIGTERM)
        except Exception: pass
    _time.sleep(2); kill_session(); _time.sleep(1)
    kf = Path(LOG_DIR) / f".kickoff-{SESSION}-resume.txt"
    if RESUME_FILE and os.path.exists(RESUME_FILE):
        kf.write_text(open(RESUME_FILE).read())   # caller-supplied (e.g. the report job)
    else:
        kf.write_text(
            "The opencode usage limit has reset (or the run stalled). You were mid-way through the weekly "
            "cc-ci /upgrade-all run. CONTINUE from where you left off — do NOT start over. Process the enrolled "
            "recipes not yet done this week, alphabetically; SKIP ones already done (their PRs exist — extend, "
            "never duplicate). Per recipe: run /recipe-upgrade in DEFAULT mode via a subagent, verify with "
            "!testme, open/extend the recipe PR (NEVER merge, NEVER weaken a test), <= DRONE_RUNNER_CAPACITY "
            "concurrent. immich has a tag+digest image abra can't parse — do the upstream-direct cross-check "
            "(recipe-upgrade SKILL §1), don't silently skip it. When all remaining recipes are done: "
            "write+push the weekly summary, then `python3 /srv/cc-ci/cc-ci-plan/launch-report.py fresh`, print "
            "'" + DONE_MARKER + "', and go idle.")
    share = "--share" if OPENCODE_SHARE else ""
    cmd = (f"set -a; . /srv/cc-ci/.testenv; set +a; {OPENCODE_BIN} run -s {sid} --continue "
           f"--model '{MODEL}' {share} --attach '{OPENCODE_SERVER}' --dir '{WORKDIR}' \"$(cat '{kf}')\"")
    subprocess.run(["tmux", "new-session", "-d", "-s", SESSION, "-c", WORKDIR, cmd])
    subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION, f"cat >> '{LOG_FILE}'"])
    try:
        STATE_SID_FILE.write_text(sid)   # a resume continues the SAME session — keep it pinned
    except OSError:
        pass
    log(f"resume: relaunched {SESSION} (session {sid})")
    # Every resume must be self-healing: ensure a watchdog is watching this run. Skip if one is
    # already alive — notably when the watchdog ITSELF called resume (it lives in {SESSION}-watchdog),
    # so this never spawns a duplicate watchdog-of-a-watchdog.
    if os.environ.get("UPGRADER_WATCHDOG", "1") == "1" and not _watchdog_alive():
        _spawn_watchdog()
    return True

def _spawn_watchdog():
    """Start the watchdog inside the persistent tmux server (NOT a Popen child). A systemd-timer
    `start` is a Type=oneshot whose cgroup is reaped on exit, which would kill a Popen child; a
    tmux session lives under the long-running tmux server and survives. Env is passed explicitly so
    the watchdog gets THIS run's config regardless of the tmux server's ambient environment."""
    import shlex
    wsess = f"{SESSION}-watchdog"
    wlog = f"{LOG_DIR}/{SESSION}-watchdog.log"
    env = {"HOME": os.environ.get("HOME") or os.path.expanduser("~"),
            "UPGRADER_SESSION": SESSION, "UPGRADER_DIR": WORKDIR, "LOG_DIR": LOG_DIR,
            "UPGRADER_BACKEND": "opencode", "UPGRADER_MODEL": MODEL, "LOOP_TIER": TIER,
           "OPENCODE_BIN": OPENCODE_BIN, "OPENCODE_SERVER": OPENCODE_SERVER,
           "OPENCODE_SHARE": "1" if OPENCODE_SHARE else "0"}
    for k in ("UPGRADER_RESUME_FILE", "UPGRADER_DONE_MARKER", "UPGRADER_STALL_MIN", "UPGRADER_CHECK_SEC"):
        if os.environ.get(k):
            env[k] = os.environ[k]
    envstr = " ".join(f"{k}={shlex.quote(str(v))}" for k, v in env.items())
    cmd = f"env {envstr} python3 {shlex.quote(os.path.realpath(__file__))} watchdog >> {shlex.quote(wlog)} 2>&1"
    subprocess.run(["tmux", "kill-session", "-t", wsess], capture_output=True)
    subprocess.run(["tmux", "new-session", "-d", "-s", wsess, "-c", WORKDIR, cmd])
    log(f"  watchdog spawned in tmux '{wsess}' — auto-resume on usage-limit stalls (survives the oneshot)")

def watchdog():
    """Watch the opencode upgrader and keep it alive to completion. Two stall modes:
      (a) PROC-DEATH — `opencode run` exits when the model ENDS ITS TURN (or crashes). For a long
          autonomous /upgrade-all this happens repeatedly before the whole run is done; the log mtime
          also freezes, so log-idle alone would take 15min to notice a run that died in 5. We detect
          it directly: no live `opencode run` proc for the session + not completed ⇒ resume promptly.
      (b) LOG-IDLE — a proc is alive but wedged (no output > STALL_MIN); resume after confirming.
    Either way, wait out an opencode usage-limit (429) first rather than hammering. Exits when the
    model prints DONE_MARKER, or after MAX_RESUMES consecutive resumes fail to get a live proc going
    (truly broken — hand back to the hourly supervisor / operator). Spawned by start()/resume()."""
    MAX_RESUMES = int(os.environ.get("UPGRADER_MAX_RESUMES", "20"))
    log(f"watchdog: watching {SESSION} (proc-death + stall>{STALL_MIN}min session-idle, poll {CHECK_EVERY}s)")
    misses = 0; resumes = 0
    while True:
        _time.sleep(CHECK_EVERY)
        # A transient error (server blip, race) must NEVER kill the watchdog — that would silently
        # abandon the run. Log and keep polling instead of letting the exception exit the loop.
        try:
            if _completed():
                log("watchdog: run completed — exiting"); return
            sid = _session_id()
            pids = _run_pids(sid) if sid else []
            idle = _session_idle_min()
            dead = not pids
            stalled = idle is not None and idle > STALL_MIN
            if not dead and not stalled:
                misses = 0; resumes = 0; continue      # alive + session advancing — healthy
            # Something's wrong (dead or wedged). Wait out a usage-limit before touching it.
            retry = _limit_retry_after()
            if retry > 0:
                wait = min(retry + 30, 3600)
                log(f"watchdog: {'proc dead' if dead else f'stalled {idle:.0f}min'} + usage-limited "
                    f"(retry-after {retry}s) — waiting {wait}s")
                _time.sleep(wait); continue
            if not dead:
                # Alive but session not advancing — confirm really wedged (two misses) before acting.
                misses += 1
                if misses < 2:
                    continue
            if resumes >= MAX_RESUMES:
                log(f"watchdog: {MAX_RESUMES} resumes without completion — giving up (supervisor/operator needed)")
                return
            why = "run proc exited (turn ended/crashed)" if dead else f"stalled {idle:.0f}min, limit clear"
            log(f"watchdog: {why} — auto-resuming (#{resumes + 1})")
            resume("watchdog auto-resume"); resumes += 1; misses = 0
        except Exception as e:
            log(f"watchdog: poll error ({type(e).__name__}: {e}) — continuing")

# ── main ──────────────────────────────────────────────────────────────────────

def main():
    cmd = sys.argv[1] if len(sys.argv) > 1 else "start"

    if cmd == "start":
        start("use-or-create")
    elif cmd == "fresh":
        start("fresh")
    elif cmd == "stop":
        if session_alive():
            log(f"killing {SESSION}")
            kill_session()
        else:
            log(f"{SESSION} not running")
    elif cmd == "status":
        if session_alive():
            busy = "busy" if session_busy() else "idle/finishing"
            log(f"{SESSION}: RUNNING ({busy})")
            subprocess.run(
                f"ps -eo pid,etime,args | grep '[r]emote-control {SESSION}' || true",
                shell=True)
        else:
            log(f"{SESSION}: stopped")
        log(f"backend: {BACKEND}  tier: {TIER}  model: {MODEL}  args: '{UPGRADER_ARGS or '<none>'}'")
    elif cmd == "attach":
        os.execvp("tmux", ["tmux", "attach", "-t", SESSION])
    elif cmd == "resume":
        resume("manual")
    elif cmd == "watchdog":
        watchdog()
    else:
        print(f"""cc-ci upgrader launcher — one-shot weekly recipe-upgrade job

  launch-upgrader.py start    use-or-create (leave busy run alone, else start fresh)
  launch-upgrader.py fresh    always kill existing + start fresh
  launch-upgrader.py stop     kill the session
  launch-upgrader.py status   show session state
  launch-upgrader.py attach   tmux attach
  launch-upgrader.py resume   continue the opencode session from where it stalled (-s <id> --continue)
  launch-upgrader.py watchdog  watch + auto-resume the opencode run across usage-limit (429) stalls

Backend: {BACKEND}  (LOOP_BACKEND or UPGRADER_BACKEND env var)
Tier:    {TIER}  (LOOP_TIER or UPGRADER_TIER env var; 'go' or 'zen'; only for opencode)
Model:   {MODEL}  (LOOP_MODEL or UPGRADER_MODEL env var)
Args:    {UPGRADER_ARGS or '<none>'}  (UPGRADER_ARGS env var, passed to /upgrade-all)

claude:   viewable at claude.ai/code
opencode: viewable at http://oc.commoninternet.net  server={OPENCODE_SERVER}
""")


if __name__ == "__main__":
    main()