tests: cover build-aware stall detection

13 tests across 4 classes: the build-proc match set (build tools match; python/node/bash/claude and
substring look-alikes don't), _proc_descendants (real child tree, roots excluded), _build_running
(session-scoped, comm-matched, never the claude root, custom regex override), and stall_check_one's
defer/reboot/hard-cap behavior. Full suite 64 pass.
This commit is contained in:
2026-07-08 16:53:08 +00:00
parent ce66948245
commit c591f5430d
+173
View File
@@ -17,6 +17,9 @@ import time
import textwrap import textwrap
import tempfile import tempfile
import shutil import shutil
import re
import signal
import subprocess
import unittest import unittest
from datetime import datetime, timedelta from datetime import datetime, timedelta
from pathlib import Path from pathlib import Path
@@ -522,5 +525,175 @@ class TestActivityDetection(unittest.TestCase):
self.assertFalse(agents.pane_active(self.cfg, self.oc_agent, idle, use_log=True)) self.assertFalse(agents.pane_active(self.cfg, self.oc_agent, idle, use_log=True))
# ── build-aware stall detection ─────────────────────────────────────────────────────
# A silent pane whose claude session has a real compile/coverage/test process running is a
# running build, NOT a stall. These cover the three moving parts: the process-name match set,
# the session-scoped descendant walk, _build_running end-to-end, and stall_check_one's defer.
class TestBuildProcRegex(unittest.TestCase):
"""High-signal build/test tool names match; generic interpreters + the harness itself must
NOT (bare python/node/bash would false-positive on the orchestrator's own engine, and the
claude root's args embed the prompt which names cargo/rustc/…)."""
def setUp(self):
self.rx = re.compile(agents.DEFAULT_BUILD_PROCS_RE)
def test_build_tools_match(self):
for name in ["cargo", "cargo-llvm-cov", "cargo-mutants", "cargo-nextest", "nextest",
"rustc", "rustdoc", "cc1", "cc1plus", "collect2", "lld", "llvm-cov",
"llvm-profdata", "lichen-server", "lichen-cms", "lichen-shell",
"lichen-cli", "chromium", "chrome", "playwright"]:
self.assertTrue(self.rx.match(name), f"{name!r} should be treated as a build proc")
def test_generic_procs_do_not_match(self):
for name in ["python", "python3", "bash", "sh", "node", "claude", "tmux",
"sleep", "grep", "git", "ssh", "vim", "pgrep", "ps"]:
self.assertIsNone(self.rx.match(name), f"{name!r} must NOT be treated as a build proc")
def test_anchored_no_substring_false_positives(self):
# ^...$ anchored — a longer name merely containing a tool word must not match
for name in ["cargotool", "xrustc", "mycc1", "playwrightish", "notchromium", "rustcx"]:
self.assertIsNone(self.rx.match(name), f"{name!r} must not match (substring)")
class TestProcDescendants(unittest.TestCase):
"""_proc_descendants returns the real child tree of the given roots and EXCLUDES the roots."""
def test_finds_children_excludes_root(self):
p = subprocess.Popen("sleep 30 & sleep 30 & wait", shell=True, start_new_session=True)
try:
time.sleep(0.4) # let the two children spawn
kids = agents._proc_descendants([str(p.pid)])
self.assertNotIn(str(p.pid), kids) # root excluded
self.assertGreaterEqual(len(kids), 2) # the two sleeps
comms = subprocess.run("ps -o comm= -p " + ",".join(sorted(kids)),
shell=True, capture_output=True, text=True).stdout
self.assertIn("sleep", comms)
finally:
os.killpg(os.getpgid(p.pid), signal.SIGKILL); p.wait()
def test_leaf_process_has_no_descendants(self):
p = subprocess.Popen(["sleep", "30"], start_new_session=True)
try:
time.sleep(0.2)
self.assertEqual(agents._proc_descendants([str(p.pid)]), set())
finally:
os.killpg(os.getpgid(p.pid), signal.SIGKILL); p.wait()
class TestBuildRunning(unittest.TestCase):
"""_build_running is scoped to the watched session's pane_pid descendants (never the root
claude process) and matches on process comm."""
def setUp(self):
self.tmp = tempfile.mkdtemp(prefix="aotest-ut-")
self.cfg = agents.load_config(_make_project(self.tmp))
self.agent = self.cfg["agents"]["cl"]
self._orig_run = agents.subprocess.run
self.ps_targets = ""
def tearDown(self):
agents.subprocess.run = self._orig_run
shutil.rmtree(self.tmp, ignore_errors=True)
def _patch(self, descendant_comms):
"""Fake tmux/pgrep/ps: pane_pid=1000; its children are 1001,1002 with the given comms."""
outer = self
class R:
def __init__(self, out): self.stdout = out; self.returncode = 0
def fake_run(cmd, *a, **k):
if "list-panes" in cmd:
return R("1000\n")
if "pgrep -P 1000" in cmd:
return R("1001\n1002\n")
if "pgrep -P" in cmd:
return R("") # 1001/1002 are leaves
if cmd.startswith("ps -o comm="):
outer.ps_targets = cmd.split("-p", 1)[1].strip()
return R("\n".join(descendant_comms) + "\n")
return R("")
agents.subprocess.run = fake_run
def test_detects_running_build(self):
self._patch(["bash", "cargo"])
self.assertTrue(agents._build_running(self.cfg, self.agent))
def test_no_build_when_only_shells(self):
self._patch(["bash", "vim"])
self.assertFalse(agents._build_running(self.cfg, self.agent))
def test_only_inspects_descendants_not_the_claude_root(self):
self._patch(["bash", "cargo"])
agents._build_running(self.cfg, self.agent)
self.assertNotIn("1000", self.ps_targets) # root pane_pid never ps-inspected
self.assertIn("1001", self.ps_targets)
def test_custom_build_procs_re_override(self):
cfg = dict(self.cfg)
cfg["watchdog"] = dict(self.cfg["watchdog"], build_procs_re=r"^mybuild$")
self._patch(["mybuild"])
self.assertTrue(agents._build_running(cfg, self.agent))
self._patch(["cargo"]) # default tool no longer counts
self.assertFalse(agents._build_running(cfg, self.agent))
class TestBuildAwareStall(unittest.TestCase):
"""stall_check_one defers the kill+reboot while a build runs, up to the stall_idle_max cap."""
def setUp(self):
self.tmp = tempfile.mkdtemp(prefix="aotest-ut-")
self.cfg = agents.load_config(_make_project(self.tmp))
self.agent = self.cfg["agents"]["cl"] # stall_idle=300, stall_idle_max default 1800
self.session = self.agent["session"]
self.reboots = []
self._orig = {}
def patch(name, fn):
self._orig[name] = getattr(agents, name)
setattr(agents, name, fn)
patch("session_alive", lambda s: True)
patch("capture_pane", lambda *a, **k: "")
patch("limit_tick", lambda *a, **k: False)
patch("pane_active", lambda *a, **k: False)
patch("_parse_waiting_until", lambda *a, **k: None)
patch("_pane_last_active", lambda s: None)
patch("phases", lambda cfg: []) # skip the DONE-nudge branch
patch("log", lambda *a, **k: None)
patch("start_agent", lambda cfg, a, force=False: self.reboots.append(a["name"]))
self.patch = patch
agents._idle_since.clear(); agents._build_deferred.discard(self.session)
def tearDown(self):
for name, fn in self._orig.items():
setattr(agents, name, fn)
agents._idle_since.clear(); agents._build_deferred.discard(self.session)
shutil.rmtree(self.tmp, ignore_errors=True)
def _set_idle(self, seconds):
agents._idle_since[self.session] = time.time() - seconds
def test_defers_reboot_while_building(self):
self.patch("_build_running", lambda *a, **k: True)
self._set_idle(700) # > stall_idle(300), < max(1800)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, []) # deferred, not rebooted
self.assertIn(self.session, agents._build_deferred)
def test_reboots_when_idle_and_no_build(self):
self.patch("_build_running", lambda *a, **k: False)
self._set_idle(700)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, [self.agent["name"]]) # genuinely idle → rebooted
def test_hard_cap_reboots_even_while_building(self):
self.patch("_build_running", lambda *a, **k: True)
self._set_idle(2000) # > stall_idle_max(1800)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, [self.agent["name"]]) # cap reached → reboot despite build
def test_no_build_check_below_base_threshold(self):
def boom(*a, **k):
raise AssertionError("_build_running must not be consulted below stall_idle")
self.patch("_build_running", boom)
self._set_idle(100) # < stall_idle(300)
agents.stall_check_one(self.cfg, self.agent)
self.assertEqual(self.reboots, [])
if __name__ == "__main__": if __name__ == "__main__":
unittest.main(verbosity=2) unittest.main(verbosity=2)