watchdog: cover all parts of the weekly run + survive the systemd oneshot

Two gaps for the scheduled Thursday glm-5.2 run:
1. Survival: the watchdog was a Popen child of the Type=oneshot service, which
   systemd's cgroup cleanup kills on exit. Spawn it under the persistent tmux
   server instead (_spawn_watchdog), like the run sessions — survives the oneshot.
2. The report runs on glm-5.2 sharing the same opencode-go budget the upgrade run
   drains, so it can 429-stall with no recovery. launch-report.py now spawns the
   SAME watchdog pointed at the cc-ci-report session (generic via UPGRADER_SESSION/
   _MODEL/_DONE_MARKER/_RESUME_FILE), with a report-specific resume prompt.

Also: _run_pids() is now scoped to the managed session (title or -s <sid>) so the
report watchdog can't kill the idle upgrader process and vice-versa; resume() adds
--dir and honors a custom resume prompt file.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

cc-ci-plan/launch-report.py

@@ -87,8 +87,35 @@ def start(mode, date):
     log(f"starting {SESSION} (backend={BACKEND}, model={MODEL}, date={date or 'today'})")
     subprocess.run(["tmux", "new-session", "-d", "-s", SESSION, "-c", cwd, cmd])
     subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION, f"cat >> '{LOG_DIR}/{SESSION}.log'"])
-    if BACKEND == "opencode" and OPENCODE_SHARE:
+    if BACKEND == "opencode":
-        log(f"  attached to {OPENCODE_SERVER} → http://oc.commoninternet.net  +public --share link")
+        if OPENCODE_SHARE:
+            log(f"  attached to {OPENCODE_SERVER} → http://oc.commoninternet.net  +public --share link")
+        # Watchdog for the report too: it runs on glm-5.2 sharing the same opencode-go budget the
+        # upgrade run just drained, so a 429 stall is likely. Reuse launch-upgrader.py's watchdog,
+        # pointed at THIS (cc-ci-report) session with a report-specific marker + resume prompt. It
+        # runs under the tmux server (survives the systemd oneshot like the run sessions).
+        if os.environ.get("REPORT_WATCHDOG", "1") == "1":
+            import shlex
+            LU = "/srv/cc-ci/cc-ci-plan/launch-upgrader.py"
+            rf = Path(LOG_DIR) / f".kickoff-{SESSION}-resume.txt"
+            rf.write_text(
+                "The opencode-go usage limit has reset (or the report stalled). CONTINUE generating and "
+                "publishing this week's public Recipe Report per /recipe-report: survey → write the spec "
+                "JSON → render with recipe-report.py → publish. Do NOT hand-write HTML (render() owns all "
+                "formatting). When the page is live, print 'RECIPE REPORT COMPLETE' and go idle.")
+            wenv = {"HOME": os.environ.get("HOME") or "/home/loops",
+                    "UPGRADER_SESSION": SESSION, "UPGRADER_DIR": cwd, "LOG_DIR": LOG_DIR,
+                    "UPGRADER_BACKEND": "opencode", "UPGRADER_MODEL": MODEL,
+                    "OPENCODE_BIN": OPENCODE_BIN, "OPENCODE_SERVER": OPENCODE_SERVER,
+                    "OPENCODE_SHARE": "1" if OPENCODE_SHARE else "0",
+                    "UPGRADER_DONE_MARKER": "RECIPE REPORT COMPLETE",
+                    "UPGRADER_RESUME_FILE": str(rf)}
+            envstr = " ".join(f"{k}={shlex.quote(str(v))}" for k, v in wenv.items())
+            wlog = f"{LOG_DIR}/{SESSION}-watchdog.log"
+            wcmd = f"env {envstr} python3 {shlex.quote(LU)} watchdog >> {shlex.quote(wlog)} 2>&1"
+            subprocess.run(["tmux", "kill-session", "-t", f"{SESSION}-watchdog"], capture_output=True)
+            subprocess.run(["tmux", "new-session", "-d", "-s", f"{SESSION}-watchdog", "-c", cwd, wcmd])
+            log(f"  report watchdog spawned in tmux '{SESSION}-watchdog' — auto-resume on usage-limit stalls")
     log(f"started. attach: tmux attach -t {SESSION}")
 
 

cc-ci-plan/launch-upgrader.py

@@ -162,13 +162,10 @@ def start(mode="use-or-create"):
     subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION,
                     f"cat >> '{LOG_DIR}/{SESSION}.log'"])
     log(f"started. attach: tmux attach -t {SESSION}  log: {LOG_DIR}/{SESSION}.log")
-    # For the opencode backend, spawn a detached watchdog that auto-resumes the run if the
+    # For the opencode backend, spawn a watchdog that auto-resumes the run if the opencode-go
-    # opencode-go usage-limit (429) stalls it mid-run (it does NOT self-resume). See watchdog().
+    # usage-limit (429) stalls it mid-run (it does NOT self-resume). See watchdog().
     if BACKEND == "opencode" and os.environ.get("UPGRADER_WATCHDOG", "1") == "1":
-        subprocess.Popen(["python3", os.path.realpath(__file__), "watchdog"],
+        _spawn_watchdog()
-                         stdout=open(f"{LOG_DIR}/{SESSION}-watchdog.log", "a"),
-                         stderr=subprocess.STDOUT, start_new_session=True)
-        log("  watchdog spawned — auto-resume on usage-limit stalls")
 
 # ── opencode stall-detect + auto-resume watchdog ────────────────────────────────
 # The opencode-go subscription enforces a rolling usage-limit (HTTP 429 + retry-after). When it
@@ -179,7 +176,10 @@ import json as _json, urllib.request as _ureq, time as _time
 
 STALL_MIN   = float(os.environ.get("UPGRADER_STALL_MIN", "15"))   # log-idle minutes ⇒ stalled
 CHECK_EVERY = int(os.environ.get("UPGRADER_CHECK_SEC", "180"))    # watchdog poll cadence
-DONE_MARKER = "UPGRADE RUN COMPLETE"
+# Generic so the SAME watchdog also covers the report job (launch-report.py points it at the
+# cc-ci-report session with its own marker + resume prompt via these env vars).
+DONE_MARKER = os.environ.get("UPGRADER_DONE_MARKER", "UPGRADE RUN COMPLETE")
+RESUME_FILE = os.environ.get("UPGRADER_RESUME_FILE")  # optional path to a custom resume prompt
 GO_ENDPOINT = "https://opencode.ai/zen/go/v1/chat/completions"
 AUTH_JSON   = os.path.expanduser("~/.local/share/opencode/auth.json")
 LOG_FILE    = f"{LOG_DIR}/{SESSION}.log"
@@ -230,8 +230,8 @@ def _limit_retry_after():
     except Exception:
         return 0
 
-def _run_pids():
+def _run_pids(sid=None):
-    """PIDs of live `opencode run` procs (via /proc scan — never matches this process)."""
+    """PIDs of live `opencode run` procs for THIS session (via /proc scan — never matches self)."""
     me, out = os.getpid(), []
     for p in os.listdir("/proc"):
         if not p.isdigit() or int(p) == me:
@@ -240,7 +240,13 @@ def _run_pids():
             cl = open(f"/proc/{p}/cmdline", "rb").read().split(b"\0")
         except Exception:
             continue
-        if b"opencode" in (b" ".join(cl)) and b"run" in cl and b"--attach" in cl:
+        joined = b" ".join(cl)
+        if not (b"opencode" in joined and b"run" in cl and b"--attach" in cl):
+            continue
+        # Scope to THIS managed session only: a fresh run carries `--title <SESSION>`, a resumed
+        # run carries `-s <sid>`. Without this, the report watchdog would kill the idle upgrader
+        # run (and vice-versa) since both are `opencode run … --attach`.
+        if SESSION.encode() in joined or (sid and sid.encode() in joined):
             out.append(int(p))
     return out
 
@@ -275,31 +281,56 @@ def resume(reason="manual"):
     if not sid:
         log(f"resume: no top-level '{SESSION}' session on {OPENCODE_SERVER} — cannot resume"); return False
     log(f"resume ({reason}): continuing session {sid}")
-    for pid in _run_pids():
+    for pid in _run_pids(sid):
         try: os.kill(pid, signal.SIGTERM)
         except Exception: pass
     _time.sleep(2); kill_session(); _time.sleep(1)
     kf = Path(LOG_DIR) / f".kickoff-{SESSION}-resume.txt"
-    kf.write_text(
+    if RESUME_FILE and os.path.exists(RESUME_FILE):
-        "The opencode-go usage limit has reset (or the run stalled). You were mid-way through the weekly "
+        kf.write_text(open(RESUME_FILE).read())   # caller-supplied (e.g. the report job)
-        "cc-ci /upgrade-all run. CONTINUE from where you left off — do NOT start over. Process the enrolled "
+    else:
-        "recipes not yet done this week, alphabetically; SKIP ones already done (their PRs exist — extend, "
+        kf.write_text(
-        "never duplicate). Per recipe: run /recipe-upgrade in DEFAULT mode via a subagent, verify with "
+            "The opencode-go usage limit has reset (or the run stalled). You were mid-way through the weekly "
-        "!testme, open/extend the recipe PR (NEVER merge, NEVER weaken a test), <= DRONE_RUNNER_CAPACITY "
+            "cc-ci /upgrade-all run. CONTINUE from where you left off — do NOT start over. Process the enrolled "
-        "concurrent. immich has a tag+digest image abra can't parse — do the upstream-direct cross-check "
+            "recipes not yet done this week, alphabetically; SKIP ones already done (their PRs exist — extend, "
-        "(recipe-upgrade SKILL §1), don't silently skip it. When all remaining recipes are done: "
+            "never duplicate). Per recipe: run /recipe-upgrade in DEFAULT mode via a subagent, verify with "
-        "write+push the weekly summary, then `python3 /srv/cc-ci/cc-ci-plan/launch-report.py fresh`, print "
+            "!testme, open/extend the recipe PR (NEVER merge, NEVER weaken a test), <= DRONE_RUNNER_CAPACITY "
-        "'" + DONE_MARKER + "', and go idle.")
+            "concurrent. immich has a tag+digest image abra can't parse — do the upstream-direct cross-check "
+            "(recipe-upgrade SKILL §1), don't silently skip it. When all remaining recipes are done: "
+            "write+push the weekly summary, then `python3 /srv/cc-ci/cc-ci-plan/launch-report.py fresh`, print "
+            "'" + DONE_MARKER + "', and go idle.")
     share = "--share" if OPENCODE_SHARE else ""
     cmd = (f"set -a; . /srv/cc-ci/.testenv; set +a; {OPENCODE_BIN} run -s {sid} --continue "
-           f"--model '{MODEL}' {share} --attach '{OPENCODE_SERVER}' \"$(cat '{kf}')\"")
+           f"--model '{MODEL}' {share} --attach '{OPENCODE_SERVER}' --dir '{WORKDIR}' \"$(cat '{kf}')\"")
     subprocess.run(["tmux", "new-session", "-d", "-s", SESSION, "-c", WORKDIR, cmd])
     subprocess.run(["tmux", "pipe-pane", "-o", "-t", SESSION, f"cat >> '{LOG_FILE}'"])
     log(f"resume: relaunched {SESSION} (session {sid})"); return True
 
+def _spawn_watchdog():
+    """Start the watchdog inside the persistent tmux server (NOT a Popen child). A systemd-timer
+    `start` is a Type=oneshot whose cgroup is reaped on exit, which would kill a Popen child; a
+    tmux session lives under the long-running tmux server and survives. Env is passed explicitly so
+    the watchdog gets THIS run's config regardless of the tmux server's ambient environment."""
+    import shlex
+    wsess = f"{SESSION}-watchdog"
+    wlog = f"{LOG_DIR}/{SESSION}-watchdog.log"
+    env = {"HOME": os.environ.get("HOME") or os.path.expanduser("~"),
+           "UPGRADER_SESSION": SESSION, "UPGRADER_DIR": WORKDIR, "LOG_DIR": LOG_DIR,
+           "UPGRADER_BACKEND": "opencode", "UPGRADER_MODEL": MODEL,
+           "OPENCODE_BIN": OPENCODE_BIN, "OPENCODE_SERVER": OPENCODE_SERVER,
+           "OPENCODE_SHARE": "1" if OPENCODE_SHARE else "0"}
+    for k in ("UPGRADER_RESUME_FILE", "UPGRADER_DONE_MARKER", "UPGRADER_STALL_MIN", "UPGRADER_CHECK_SEC"):
+        if os.environ.get(k):
+            env[k] = os.environ[k]
+    envstr = " ".join(f"{k}={shlex.quote(str(v))}" for k, v in env.items())
+    cmd = f"env {envstr} python3 {shlex.quote(os.path.realpath(__file__))} watchdog >> {shlex.quote(wlog)} 2>&1"
+    subprocess.run(["tmux", "kill-session", "-t", wsess], capture_output=True)
+    subprocess.run(["tmux", "new-session", "-d", "-s", wsess, "-c", WORKDIR, cmd])
+    log(f"  watchdog spawned in tmux '{wsess}' — auto-resume on usage-limit stalls (survives the oneshot)")
+
 def watchdog():
     """Watch the opencode upgrader; on a stall, wait out any usage-limit then resume the session.
-    Exits when the run prints UPGRADE RUN COMPLETE. Spawned by an opencode `start`; also standalone."""
+    Exits when the model prints DONE_MARKER. Spawned by an opencode `start`; also standalone."""
     log(f"watchdog: watching {SESSION} (stall>{STALL_MIN}min log-idle, poll {CHECK_EVERY}s)")
     misses = 0
     while True: