tests: cover build-aware stall detection

13 tests across 4 classes: the build-proc match set (build tools match; python/node/bash/claude and
substring look-alikes don't), _proc_descendants (real child tree, roots excluded), _build_running
(session-scoped, comm-matched, never the claude root, custom regex override), and stall_check_one's
defer/reboot/hard-cap behavior. Full suite 64 pass.
This commit is contained in:
2026-07-08 16:53:08 +00:00
parent ce66948245
commit c591f5430d

View File

@ -17,6 +17,9 @@ import time
import textwrap
import tempfile
import shutil
import re
import signal
import subprocess
import unittest
from datetime import datetime, timedelta
from pathlib import Path
@ -522,5 +525,175 @@ class TestActivityDetection(unittest.TestCase):
self.assertFalse(agents.pane_active(self.cfg, self.oc_agent, idle, use_log=True))
# ── build-aware stall detection ─────────────────────────────────────────────────────
# A silent pane whose claude session has a real compile/coverage/test process running is a
# running build, NOT a stall. These cover the three moving parts: the process-name match set,
# the session-scoped descendant walk, _build_running end-to-end, and stall_check_one's defer.
class TestBuildProcRegex(unittest.TestCase):
"""High-signal build/test tool names match; generic interpreters + the harness itself must
NOT (bare python/node/bash would false-positive on the orchestrator's own engine, and the
claude root's args embed the prompt which names cargo/rustc/…)."""
def setUp(self):
self.rx = re.compile(agents.DEFAULT_BUILD_PROCS_RE)
def test_build_tools_match(self):
for name in ["cargo", "cargo-llvm-cov", "cargo-mutants", "cargo-nextest", "nextest",
"rustc", "rustdoc", "cc1", "cc1plus", "collect2", "lld", "llvm-cov",
"llvm-profdata", "lichen-server", "lichen-cms", "lichen-shell",
"lichen-cli", "chromium", "chrome", "playwright"]:
self.assertTrue(self.rx.match(name), f"{name!r} should be treated as a build proc")
def test_generic_procs_do_not_match(self):
for name in ["python", "python3", "bash", "sh", "node", "claude", "tmux",
"sleep", "grep", "git", "ssh", "vim", "pgrep", "ps"]:
self.assertIsNone(self.rx.match(name), f"{name!r} must NOT be treated as a build proc")
def test_anchored_no_substring_false_positives(self):
# ^...$ anchored — a longer name merely containing a tool word must not match
for name in ["cargotool", "xrustc", "mycc1", "playwrightish", "notchromium", "rustcx"]:
self.assertIsNone(self.rx.match(name), f"{name!r} must not match (substring)")
class TestProcDescendants(unittest.TestCase):
"""_proc_descendants returns the real child tree of the given roots and EXCLUDES the roots."""
def test_finds_children_excludes_root(self):
p = subprocess.Popen("sleep 30 & sleep 30 & wait", shell=True, start_new_session=True)
try:
time.sleep(0.4) # let the two children spawn
kids = agents._proc_descendants([str(p.pid)])
self.assertNotIn(str(p.pid), kids) # root excluded
self.assertGreaterEqual(len(kids), 2) # the two sleeps
comms = subprocess.run("ps -o comm= -p " + ",".join(sorted(kids)),
shell=True, capture_output=True, text=True).stdout
self.assertIn("sleep", comms)
finally:
os.killpg(os.getpgid(p.pid), signal.SIGKILL); p.wait()
def test_leaf_process_has_no_descendants(self):
p = subprocess.Popen(["sleep", "30"], start_new_session=True)
try:
time.sleep(0.2)
self.assertEqual(agents._proc_descendants([str(p.pid)]), set())
finally:
os.killpg(os.getpgid(p.pid), signal.SIGKILL); p.wait()
class TestBuildRunning(unittest.TestCase):
"""_build_running is scoped to the watched session's pane_pid descendants (never the root
claude process) and matches on process comm."""
def setUp(self):
self.tmp = tempfile.mkdtemp(prefix="aotest-ut-")
self.cfg = agents.load_config(_make_project(self.tmp))
self.agent = self.cfg["agents"]["cl"]
self._orig_run = agents.subprocess.run
self.ps_targets = ""
def tearDown(self):
agents.subprocess.run = self._orig_run
shutil.rmtree(self.tmp, ignore_errors=True)
def _patch(self, descendant_comms):
"""Fake tmux/pgrep/ps: pane_pid=1000; its children are 1001,1002 with the given comms."""
outer = self
class R:
def __init__(self, out): self.stdout = out; self.returncode = 0
def fake_run(cmd, *a, **k):
if "list-panes" in cmd:
return R("1000\n")
if "pgrep -P 1000" in cmd:
return R("1001\n1002\n")
if "pgrep -P" in cmd:
return R("") # 1001/1002 are leaves
if cmd.startswith("ps -o comm="):
outer.ps_targets = cmd.split("-p", 1)[1].strip()
return R("\n".join(descendant_comms) + "\n")
return R("")
agents.subprocess.run = fake_run
def test_detects_running_build(self):
self._patch(["bash", "cargo"])
self.assertTrue(agents._build_running(self.cfg, self.agent))
def test_no_build_when_only_shells(self):
self._patch(["bash", "vim"])
self.assertFalse(agents._build_running(self.cfg, self.agent))
def test_only_inspects_descendants_not_the_claude_root(self):
self._patch(["bash", "cargo"])
agents._build_running(self.cfg, self.agent)
self.assertNotIn("1000", self.ps_targets) # root pane_pid never ps-inspected
self.assertIn("1001", self.ps_targets)
def test_custom_build_procs_re_override(self):
cfg = dict(self.cfg)
cfg["watchdog"] = dict(self.cfg["watchdog"], build_procs_re=r"^mybuild$")
self._patch(["mybuild"])
self.assertTrue(agents._build_running(cfg, self.agent))
self._patch(["cargo"]) # default tool no longer counts
self.assertFalse(agents._build_running(cfg, self.agent))
class TestBuildAwareStall(unittest.TestCase):
"""stall_check_one defers the kill+reboot while a build runs, up to the stall_idle_max cap."""
def setUp(self):
self.tmp = tempfile.mkdtemp(prefix="aotest-ut-")
self.cfg = agents.load_config(_make_project(self.tmp))
self.agent = self.cfg["agents"]["cl"] # stall_idle=300, stall_idle_max default 1800
self.session = self.agent["session"]
self.reboots = []
self._orig = {}
def patch(name, fn):
self._orig[name] = getattr(agents, name)
setattr(agents, name, fn)
patch("session_alive", lambda s: True)
patch("capture_pane", lambda *a, **k: "")
patch("limit_tick", lambda *a, **k: False)
patch("pane_active", lambda *a, **k: False)
patch("_parse_waiting_until", lambda *a, **k: None)
patch("_pane_last_active", lambda s: None)
patch("phases", lambda cfg: []) # skip the DONE-nudge branch
patch("log", lambda *a, **k: None)
patch("start_agent", lambda cfg, a, force=False: self.reboots.append(a["name"]))
self.patch = patch
agents._idle_since.clear(); agents._build_deferred.discard(self.session)
def tearDown(self):
for name, fn in self._orig.items():
setattr(agents, name, fn)
agents._idle_since.clear(); agents._build_deferred.discard(self.session)
shutil.rmtree(self.tmp, ignore_errors=True)
def _set_idle(self, seconds):
agents._idle_since[self.session] = time.time() - seconds
def test_defers_reboot_while_building(self):
self.patch("_build_running", lambda *a, **k: True)
self._set_idle(700) # > stall_idle(300), < max(1800)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, []) # deferred, not rebooted
self.assertIn(self.session, agents._build_deferred)
def test_reboots_when_idle_and_no_build(self):
self.patch("_build_running", lambda *a, **k: False)
self._set_idle(700)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, [self.agent["name"]]) # genuinely idle → rebooted
def test_hard_cap_reboots_even_while_building(self):
self.patch("_build_running", lambda *a, **k: True)
self._set_idle(2000) # > stall_idle_max(1800)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, [self.agent["name"]]) # cap reached → reboot despite build
def test_no_build_check_below_base_threshold(self):
def boom(*a, **k):
raise AssertionError("_build_running must not be consulted below stall_idle")
self.patch("_build_running", boom)
self._set_idle(100) # < stall_idle(300)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, [])
if __name__ == "__main__":
unittest.main(verbosity=2)