From c591f5430db550c3974057b01b5bffd6f06fed87 Mon Sep 17 00:00:00 2001 From: mfowler Date: Wed, 8 Jul 2026 16:53:08 +0000 Subject: [PATCH] tests: cover build-aware stall detection 13 tests across 4 classes: the build-proc match set (build tools match; python/node/bash/claude and substring look-alikes don't), _proc_descendants (real child tree, roots excluded), _build_running (session-scoped, comm-matched, never the claude root, custom regex override), and stall_check_one's defer/reboot/hard-cap behavior. Full suite 64 pass. --- tests/test_unit.py | 173 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 173 insertions(+) diff --git a/tests/test_unit.py b/tests/test_unit.py index 646d3ac..557496f 100755 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -17,6 +17,9 @@ import time import textwrap import tempfile import shutil +import re +import signal +import subprocess import unittest from datetime import datetime, timedelta from pathlib import Path @@ -522,5 +525,175 @@ class TestActivityDetection(unittest.TestCase): self.assertFalse(agents.pane_active(self.cfg, self.oc_agent, idle, use_log=True)) +# ── build-aware stall detection ───────────────────────────────────────────────────── +# A silent pane whose claude session has a real compile/coverage/test process running is a +# running build, NOT a stall. These cover the three moving parts: the process-name match set, +# the session-scoped descendant walk, _build_running end-to-end, and stall_check_one's defer. + +class TestBuildProcRegex(unittest.TestCase): + """High-signal build/test tool names match; generic interpreters + the harness itself must + NOT (bare python/node/bash would false-positive on the orchestrator's own engine, and the + claude root's args embed the prompt which names cargo/rustc/…).""" + def setUp(self): + self.rx = re.compile(agents.DEFAULT_BUILD_PROCS_RE) + + def test_build_tools_match(self): + for name in ["cargo", "cargo-llvm-cov", "cargo-mutants", "cargo-nextest", "nextest", + "rustc", "rustdoc", "cc1", "cc1plus", "collect2", "lld", "llvm-cov", + "llvm-profdata", "lichen-server", "lichen-cms", "lichen-shell", + "lichen-cli", "chromium", "chrome", "playwright"]: + self.assertTrue(self.rx.match(name), f"{name!r} should be treated as a build proc") + + def test_generic_procs_do_not_match(self): + for name in ["python", "python3", "bash", "sh", "node", "claude", "tmux", + "sleep", "grep", "git", "ssh", "vim", "pgrep", "ps"]: + self.assertIsNone(self.rx.match(name), f"{name!r} must NOT be treated as a build proc") + + def test_anchored_no_substring_false_positives(self): + # ^...$ anchored — a longer name merely containing a tool word must not match + for name in ["cargotool", "xrustc", "mycc1", "playwrightish", "notchromium", "rustcx"]: + self.assertIsNone(self.rx.match(name), f"{name!r} must not match (substring)") + + +class TestProcDescendants(unittest.TestCase): + """_proc_descendants returns the real child tree of the given roots and EXCLUDES the roots.""" + def test_finds_children_excludes_root(self): + p = subprocess.Popen("sleep 30 & sleep 30 & wait", shell=True, start_new_session=True) + try: + time.sleep(0.4) # let the two children spawn + kids = agents._proc_descendants([str(p.pid)]) + self.assertNotIn(str(p.pid), kids) # root excluded + self.assertGreaterEqual(len(kids), 2) # the two sleeps + comms = subprocess.run("ps -o comm= -p " + ",".join(sorted(kids)), + shell=True, capture_output=True, text=True).stdout + self.assertIn("sleep", comms) + finally: + os.killpg(os.getpgid(p.pid), signal.SIGKILL); p.wait() + + def test_leaf_process_has_no_descendants(self): + p = subprocess.Popen(["sleep", "30"], start_new_session=True) + try: + time.sleep(0.2) + self.assertEqual(agents._proc_descendants([str(p.pid)]), set()) + finally: + os.killpg(os.getpgid(p.pid), signal.SIGKILL); p.wait() + + +class TestBuildRunning(unittest.TestCase): + """_build_running is scoped to the watched session's pane_pid descendants (never the root + claude process) and matches on process comm.""" + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="aotest-ut-") + self.cfg = agents.load_config(_make_project(self.tmp)) + self.agent = self.cfg["agents"]["cl"] + self._orig_run = agents.subprocess.run + self.ps_targets = "" + + def tearDown(self): + agents.subprocess.run = self._orig_run + shutil.rmtree(self.tmp, ignore_errors=True) + + def _patch(self, descendant_comms): + """Fake tmux/pgrep/ps: pane_pid=1000; its children are 1001,1002 with the given comms.""" + outer = self + class R: + def __init__(self, out): self.stdout = out; self.returncode = 0 + def fake_run(cmd, *a, **k): + if "list-panes" in cmd: + return R("1000\n") + if "pgrep -P 1000" in cmd: + return R("1001\n1002\n") + if "pgrep -P" in cmd: + return R("") # 1001/1002 are leaves + if cmd.startswith("ps -o comm="): + outer.ps_targets = cmd.split("-p", 1)[1].strip() + return R("\n".join(descendant_comms) + "\n") + return R("") + agents.subprocess.run = fake_run + + def test_detects_running_build(self): + self._patch(["bash", "cargo"]) + self.assertTrue(agents._build_running(self.cfg, self.agent)) + + def test_no_build_when_only_shells(self): + self._patch(["bash", "vim"]) + self.assertFalse(agents._build_running(self.cfg, self.agent)) + + def test_only_inspects_descendants_not_the_claude_root(self): + self._patch(["bash", "cargo"]) + agents._build_running(self.cfg, self.agent) + self.assertNotIn("1000", self.ps_targets) # root pane_pid never ps-inspected + self.assertIn("1001", self.ps_targets) + + def test_custom_build_procs_re_override(self): + cfg = dict(self.cfg) + cfg["watchdog"] = dict(self.cfg["watchdog"], build_procs_re=r"^mybuild$") + self._patch(["mybuild"]) + self.assertTrue(agents._build_running(cfg, self.agent)) + self._patch(["cargo"]) # default tool no longer counts + self.assertFalse(agents._build_running(cfg, self.agent)) + + +class TestBuildAwareStall(unittest.TestCase): + """stall_check_one defers the kill+reboot while a build runs, up to the stall_idle_max cap.""" + def setUp(self): + self.tmp = tempfile.mkdtemp(prefix="aotest-ut-") + self.cfg = agents.load_config(_make_project(self.tmp)) + self.agent = self.cfg["agents"]["cl"] # stall_idle=300, stall_idle_max default 1800 + self.session = self.agent["session"] + self.reboots = [] + self._orig = {} + def patch(name, fn): + self._orig[name] = getattr(agents, name) + setattr(agents, name, fn) + patch("session_alive", lambda s: True) + patch("capture_pane", lambda *a, **k: "") + patch("limit_tick", lambda *a, **k: False) + patch("pane_active", lambda *a, **k: False) + patch("_parse_waiting_until", lambda *a, **k: None) + patch("_pane_last_active", lambda s: None) + patch("phases", lambda cfg: []) # skip the DONE-nudge branch + patch("log", lambda *a, **k: None) + patch("start_agent", lambda cfg, a, force=False: self.reboots.append(a["name"])) + self.patch = patch + agents._idle_since.clear(); agents._build_deferred.discard(self.session) + + def tearDown(self): + for name, fn in self._orig.items(): + setattr(agents, name, fn) + agents._idle_since.clear(); agents._build_deferred.discard(self.session) + shutil.rmtree(self.tmp, ignore_errors=True) + + def _set_idle(self, seconds): + agents._idle_since[self.session] = time.time() - seconds + + def test_defers_reboot_while_building(self): + self.patch("_build_running", lambda *a, **k: True) + self._set_idle(700) # > stall_idle(300), < max(1800) + agents.stall_check_one(self.cfg, self.agent) + self.assertEqual(self.reboots, []) # deferred, not rebooted + self.assertIn(self.session, agents._build_deferred) + + def test_reboots_when_idle_and_no_build(self): + self.patch("_build_running", lambda *a, **k: False) + self._set_idle(700) + agents.stall_check_one(self.cfg, self.agent) + self.assertEqual(self.reboots, [self.agent["name"]]) # genuinely idle → rebooted + + def test_hard_cap_reboots_even_while_building(self): + self.patch("_build_running", lambda *a, **k: True) + self._set_idle(2000) # > stall_idle_max(1800) + agents.stall_check_one(self.cfg, self.agent) + self.assertEqual(self.reboots, [self.agent["name"]]) # cap reached → reboot despite build + + def test_no_build_check_below_base_threshold(self): + def boom(*a, **k): + raise AssertionError("_build_running must not be consulted below stall_idle") + self.patch("_build_running", boom) + self._set_idle(100) # < stall_idle(300) + agents.stall_check_one(self.cfg, self.agent) + self.assertEqual(self.reboots, []) + + if __name__ == "__main__": unittest.main(verbosity=2)