From ce669482455ddbc64c01dfae9e0b2930507cce7b Mon Sep 17 00:00:00 2001 From: mfowler Date: Wed, 8 Jul 2026 16:42:57 +0000 Subject: [PATCH] =?UTF-8?q?watchdog:=20build-aware=20stall=20detection=20?= =?UTF-8?q?=E2=80=94=20defer=20reboot=20while=20a=20real=20build/test=20ru?= =?UTF-8?q?ns=20(scoped=20to=20the=20watched=20session's=20child=20procs),?= =?UTF-8?q?=20with=20stall=5Fidle=5Fmax=20hard=20cap?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A silent pane whose claude session has a live descendant compile/coverage/test process (cargo, rustc, cc1, llvm-cov, lichen-server, chromium, …) is a running build, not a stall. _build_running() inspects ONLY the descendants of that session's pane_pid (never the claude root, whose args embed the prompt), matching process comm. Defers the kill+reboot until the build finishes, but never past stall_idle_max (default 1800s) so a hung build still recovers. Configurable via build_procs_re / stall_idle_max. --- agents.py | 57 ++++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 56 insertions(+), 1 deletion(-) diff --git a/agents.py b/agents.py index bc6a414..96ff151 100755 --- a/agents.py +++ b/agents.py @@ -468,6 +468,48 @@ def limit_tick(cfg, agent, pane): _idle_since: dict[str, float] = {} _done_nudged: dict[str, bool] = {} # per-session: sent the one-time "write the done marker" nudge this phase +_build_deferred: set = set() # per-session: currently deferring a stall because a real build is running + +# Default set of process names (comm) that mean "genuine long-running work is happening" — a compile, +# coverage run, mutation run, or a real e2e test driving the server/browser. If one of these is a +# descendant of the agent's tmux pane, a silent pane is a running build, NOT a stall. High-signal names +# only (no bare python/node/bash — those match the orchestrator's own engine + would false-positive). +DEFAULT_BUILD_PROCS_RE = (r"^(cargo|cargo-llvm-cov|cargo-mutants|cargo-nextest|nextest|rustc|rustdoc|" + r"cc1|cc1plus|collect2|lld|ld\.lld|llvm-cov|llvm-profdata|" + r"lichen-server|lichen-cms|lichen-shell|lichen-cli|chromium|chrome|playwright)$") + +def _proc_descendants(roots): + """All descendant PIDs of the given root PIDs (BFS via pgrep -P), EXCLUDING the roots themselves — + the pane root is the claude process whose args embed the prompt (which mentions cargo/rustc/…), so + matching it would false-positive; we only inspect its real child processes.""" + roots = [p for p in roots if str(p).isdigit()] + seen, stack = set(), list(roots) + while stack: + pid = stack.pop() + if pid in seen: + continue + seen.add(pid) + c = subprocess.run(f"pgrep -P {pid}", shell=True, capture_output=True, text=True) + stack += [p for p in c.stdout.split() if p.isdigit()] + return seen - set(roots) + +def _build_running(cfg, agent): + """True if a real build/coverage/test process is running under the agent's tmux session, so a long + silent pane isn't mistaken for a stall. Matches process comm (never the claude root's args). Bounded + by stall_idle_max upstream so a genuinely hung build still gets rebooted eventually.""" + session = agent["session"] + r = subprocess.run(f"tmux list-panes -t {session!r} -F '#{{pane_pid}}'", + shell=True, capture_output=True, text=True) + kids = _proc_descendants(r.stdout.split()) + if not kids: + return False + try: + rx = re.compile(cfg["watchdog"].get("build_procs_re", DEFAULT_BUILD_PROCS_RE)) + except re.error: + rx = re.compile(DEFAULT_BUILD_PROCS_RE) + ps = subprocess.run("ps -o comm= -p " + ",".join(sorted(kids)), + shell=True, capture_output=True, text=True) + return any(rx.match(c.strip()) for c in ps.stdout.splitlines() if c.strip()) def _done_nudge_msg(cfg, ph): """The DONE-nudge: prompts a stalled loop agent to finalize a built-but-unmarked phase.""" @@ -526,6 +568,7 @@ def stall_check_one(cfg, agent): return if pane_active(cfg, agent, pane): _idle_since[session] = 0.0 + _build_deferred.discard(session) return # Seed from the pane's real last-activity (not `now`), so a watchdog that just (re)started still # sees an already-idle pane as idle-for-its-true-duration instead of resetting the clock. @@ -542,7 +585,18 @@ def stall_check_one(cfg, agent): stall_idle = int(backend_of(cfg, agent).get("stall_idle", 300)) if idle < stall_idle: return - reason = f"idle {int(idle)}s with no WAITING-UNTIL marker" + # Build-aware: a silent pane with a real compile/coverage/test process running is NOT a stall — + # defer the reboot until it finishes, but never past stall_idle_max (hard cap for a hung build). + stall_idle_max = int(backend_of(cfg, agent).get("stall_idle_max", 1800)) + if idle < stall_idle_max and _build_running(cfg, agent): + if session not in _build_deferred: + log(f"stall-defer: {agent['name']} ({session}) idle {int(idle)}s but a build/test is " + f"running — waiting (hard cap {stall_idle_max}s)") + _build_deferred.add(session) + return + reason = (f"idle {int(idle)}s past build-aware hard cap {stall_idle_max}s — rebooting regardless" + if idle >= stall_idle_max else + f"idle {int(idle)}s with no WAITING-UNTIL marker and no build running") # Ceremony-lag guard: a loop agent idling in a phase that's built but NOT marked done won't let the # phase advance (the recurring "all gates PASS but no ## DONE written" stall). Nudge it ONCE per phase # to finalize (write the done marker if the DoD is met) before falling back to the blunt kill+reboot. @@ -557,6 +611,7 @@ def stall_check_one(cfg, agent): log(f"stall: {agent['name']} ({session}) {reason} — kill + reboot") start_agent(cfg, agent, force=True) _idle_since[session] = 0.0 + _build_deferred.discard(session) # ── healing ──────────────────────────────────────────────────────────────────────