watchdog: build-aware stall detection — defer reboot while a real build/test runs (scoped to the watched session's child procs), with stall_idle_max hard cap

A silent pane whose claude session has a live descendant compile/coverage/test process (cargo, rustc,
cc1, llvm-cov, lichen-server, chromium, …) is a running build, not a stall. _build_running() inspects
ONLY the descendants of that session's pane_pid (never the claude root, whose args embed the prompt),
matching process comm. Defers the kill+reboot until the build finishes, but never past stall_idle_max
(default 1800s) so a hung build still recovers. Configurable via build_procs_re / stall_idle_max.

agents.py

@@ -468,6 +468,48 @@ def limit_tick(cfg, agent, pane):
 
 _idle_since: dict[str, float] = {}
 _done_nudged: dict[str, bool] = {}   # per-session: sent the one-time "write the done marker" nudge this phase
+_build_deferred: set = set()         # per-session: currently deferring a stall because a real build is running
+
+# Default set of process names (comm) that mean "genuine long-running work is happening" — a compile,
+# coverage run, mutation run, or a real e2e test driving the server/browser. If one of these is a
+# descendant of the agent's tmux pane, a silent pane is a running build, NOT a stall. High-signal names
+# only (no bare python/node/bash — those match the orchestrator's own engine + would false-positive).
+DEFAULT_BUILD_PROCS_RE = (r"^(cargo|cargo-llvm-cov|cargo-mutants|cargo-nextest|nextest|rustc|rustdoc|"
+                          r"cc1|cc1plus|collect2|lld|ld\.lld|llvm-cov|llvm-profdata|"
+                          r"lichen-server|lichen-cms|lichen-shell|lichen-cli|chromium|chrome|playwright)$")
+
+def _proc_descendants(roots):
+    """All descendant PIDs of the given root PIDs (BFS via pgrep -P), EXCLUDING the roots themselves —
+    the pane root is the claude process whose args embed the prompt (which mentions cargo/rustc/…), so
+    matching it would false-positive; we only inspect its real child processes."""
+    roots = [p for p in roots if str(p).isdigit()]
+    seen, stack = set(), list(roots)
+    while stack:
+        pid = stack.pop()
+        if pid in seen:
+            continue
+        seen.add(pid)
+        c = subprocess.run(f"pgrep -P {pid}", shell=True, capture_output=True, text=True)
+        stack += [p for p in c.stdout.split() if p.isdigit()]
+    return seen - set(roots)
+
+def _build_running(cfg, agent):
+    """True if a real build/coverage/test process is running under the agent's tmux session, so a long
+    silent pane isn't mistaken for a stall. Matches process comm (never the claude root's args). Bounded
+    by stall_idle_max upstream so a genuinely hung build still gets rebooted eventually."""
+    session = agent["session"]
+    r = subprocess.run(f"tmux list-panes -t {session!r} -F '#{{pane_pid}}'",
+                       shell=True, capture_output=True, text=True)
+    kids = _proc_descendants(r.stdout.split())
+    if not kids:
+        return False
+    try:
+        rx = re.compile(cfg["watchdog"].get("build_procs_re", DEFAULT_BUILD_PROCS_RE))
+    except re.error:
+        rx = re.compile(DEFAULT_BUILD_PROCS_RE)
+    ps = subprocess.run("ps -o comm= -p " + ",".join(sorted(kids)),
+                        shell=True, capture_output=True, text=True)
+    return any(rx.match(c.strip()) for c in ps.stdout.splitlines() if c.strip())
 
 def _done_nudge_msg(cfg, ph):
     """The DONE-nudge: prompts a stalled loop agent to finalize a built-but-unmarked phase."""
@@ -526,6 +568,7 @@ def stall_check_one(cfg, agent):
         return
     if pane_active(cfg, agent, pane):
         _idle_since[session] = 0.0
+        _build_deferred.discard(session)
         return
     # Seed from the pane's real last-activity (not `now`), so a watchdog that just (re)started still
     # sees an already-idle pane as idle-for-its-true-duration instead of resetting the clock.
@@ -542,7 +585,18 @@ def stall_check_one(cfg, agent):
         stall_idle = int(backend_of(cfg, agent).get("stall_idle", 300))
         if idle < stall_idle:
             return
-        reason = f"idle {int(idle)}s with no WAITING-UNTIL marker"
+        # Build-aware: a silent pane with a real compile/coverage/test process running is NOT a stall —
+        # defer the reboot until it finishes, but never past stall_idle_max (hard cap for a hung build).
+        stall_idle_max = int(backend_of(cfg, agent).get("stall_idle_max", 1800))
+        if idle < stall_idle_max and _build_running(cfg, agent):
+            if session not in _build_deferred:
+                log(f"stall-defer: {agent['name']} ({session}) idle {int(idle)}s but a build/test is "
+                    f"running — waiting (hard cap {stall_idle_max}s)")
+                _build_deferred.add(session)
+            return
+        reason = (f"idle {int(idle)}s past build-aware hard cap {stall_idle_max}s — rebooting regardless"
+                  if idle >= stall_idle_max else
+                  f"idle {int(idle)}s with no WAITING-UNTIL marker and no build running")
     # Ceremony-lag guard: a loop agent idling in a phase that's built but NOT marked done won't let the
     # phase advance (the recurring "all gates PASS but no ## DONE written" stall). Nudge it ONCE per phase
     # to finalize (write the done marker if the DoD is met) before falling back to the blunt kill+reboot.
@@ -557,6 +611,7 @@ def stall_check_one(cfg, agent):
     log(f"stall: {agent['name']} ({session}) {reason} — kill + reboot")
     start_agent(cfg, agent, force=True)
     _idle_since[session] = 0.0
+    _build_deferred.discard(session)
 
 # ── healing ──────────────────────────────────────────────────────────────────────