tests: cover build-aware stall detection
13 tests across 4 classes: the build-proc match set (build tools match; python/node/bash/claude and substring look-alikes don't), _proc_descendants (real child tree, roots excluded), _build_running (session-scoped, comm-matched, never the claude root, custom regex override), and stall_check_one's defer/reboot/hard-cap behavior. Full suite 64 pass.
This commit is contained in:
@ -17,6 +17,9 @@ import time
|
||||
import textwrap
|
||||
import tempfile
|
||||
import shutil
|
||||
import re
|
||||
import signal
|
||||
import subprocess
|
||||
import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from pathlib import Path
|
||||
@ -522,5 +525,175 @@ class TestActivityDetection(unittest.TestCase):
|
||||
self.assertFalse(agents.pane_active(self.cfg, self.oc_agent, idle, use_log=True))
|
||||
|
||||
|
||||
# ── build-aware stall detection ─────────────────────────────────────────────────────
|
||||
# A silent pane whose claude session has a real compile/coverage/test process running is a
|
||||
# running build, NOT a stall. These cover the three moving parts: the process-name match set,
|
||||
# the session-scoped descendant walk, _build_running end-to-end, and stall_check_one's defer.
|
||||
|
||||
class TestBuildProcRegex(unittest.TestCase):
|
||||
"""High-signal build/test tool names match; generic interpreters + the harness itself must
|
||||
NOT (bare python/node/bash would false-positive on the orchestrator's own engine, and the
|
||||
claude root's args embed the prompt which names cargo/rustc/…)."""
|
||||
def setUp(self):
|
||||
self.rx = re.compile(agents.DEFAULT_BUILD_PROCS_RE)
|
||||
|
||||
def test_build_tools_match(self):
|
||||
for name in ["cargo", "cargo-llvm-cov", "cargo-mutants", "cargo-nextest", "nextest",
|
||||
"rustc", "rustdoc", "cc1", "cc1plus", "collect2", "lld", "llvm-cov",
|
||||
"llvm-profdata", "lichen-server", "lichen-cms", "lichen-shell",
|
||||
"lichen-cli", "chromium", "chrome", "playwright"]:
|
||||
self.assertTrue(self.rx.match(name), f"{name!r} should be treated as a build proc")
|
||||
|
||||
def test_generic_procs_do_not_match(self):
|
||||
for name in ["python", "python3", "bash", "sh", "node", "claude", "tmux",
|
||||
"sleep", "grep", "git", "ssh", "vim", "pgrep", "ps"]:
|
||||
self.assertIsNone(self.rx.match(name), f"{name!r} must NOT be treated as a build proc")
|
||||
|
||||
def test_anchored_no_substring_false_positives(self):
|
||||
# ^...$ anchored — a longer name merely containing a tool word must not match
|
||||
for name in ["cargotool", "xrustc", "mycc1", "playwrightish", "notchromium", "rustcx"]:
|
||||
self.assertIsNone(self.rx.match(name), f"{name!r} must not match (substring)")
|
||||
|
||||
|
||||
class TestProcDescendants(unittest.TestCase):
|
||||
"""_proc_descendants returns the real child tree of the given roots and EXCLUDES the roots."""
|
||||
def test_finds_children_excludes_root(self):
|
||||
p = subprocess.Popen("sleep 30 & sleep 30 & wait", shell=True, start_new_session=True)
|
||||
try:
|
||||
time.sleep(0.4) # let the two children spawn
|
||||
kids = agents._proc_descendants([str(p.pid)])
|
||||
self.assertNotIn(str(p.pid), kids) # root excluded
|
||||
self.assertGreaterEqual(len(kids), 2) # the two sleeps
|
||||
comms = subprocess.run("ps -o comm= -p " + ",".join(sorted(kids)),
|
||||
shell=True, capture_output=True, text=True).stdout
|
||||
self.assertIn("sleep", comms)
|
||||
finally:
|
||||
os.killpg(os.getpgid(p.pid), signal.SIGKILL); p.wait()
|
||||
|
||||
def test_leaf_process_has_no_descendants(self):
|
||||
p = subprocess.Popen(["sleep", "30"], start_new_session=True)
|
||||
try:
|
||||
time.sleep(0.2)
|
||||
self.assertEqual(agents._proc_descendants([str(p.pid)]), set())
|
||||
finally:
|
||||
os.killpg(os.getpgid(p.pid), signal.SIGKILL); p.wait()
|
||||
|
||||
|
||||
class TestBuildRunning(unittest.TestCase):
|
||||
"""_build_running is scoped to the watched session's pane_pid descendants (never the root
|
||||
claude process) and matches on process comm."""
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="aotest-ut-")
|
||||
self.cfg = agents.load_config(_make_project(self.tmp))
|
||||
self.agent = self.cfg["agents"]["cl"]
|
||||
self._orig_run = agents.subprocess.run
|
||||
self.ps_targets = ""
|
||||
|
||||
def tearDown(self):
|
||||
agents.subprocess.run = self._orig_run
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def _patch(self, descendant_comms):
|
||||
"""Fake tmux/pgrep/ps: pane_pid=1000; its children are 1001,1002 with the given comms."""
|
||||
outer = self
|
||||
class R:
|
||||
def __init__(self, out): self.stdout = out; self.returncode = 0
|
||||
def fake_run(cmd, *a, **k):
|
||||
if "list-panes" in cmd:
|
||||
return R("1000\n")
|
||||
if "pgrep -P 1000" in cmd:
|
||||
return R("1001\n1002\n")
|
||||
if "pgrep -P" in cmd:
|
||||
return R("") # 1001/1002 are leaves
|
||||
if cmd.startswith("ps -o comm="):
|
||||
outer.ps_targets = cmd.split("-p", 1)[1].strip()
|
||||
return R("\n".join(descendant_comms) + "\n")
|
||||
return R("")
|
||||
agents.subprocess.run = fake_run
|
||||
|
||||
def test_detects_running_build(self):
|
||||
self._patch(["bash", "cargo"])
|
||||
self.assertTrue(agents._build_running(self.cfg, self.agent))
|
||||
|
||||
def test_no_build_when_only_shells(self):
|
||||
self._patch(["bash", "vim"])
|
||||
self.assertFalse(agents._build_running(self.cfg, self.agent))
|
||||
|
||||
def test_only_inspects_descendants_not_the_claude_root(self):
|
||||
self._patch(["bash", "cargo"])
|
||||
agents._build_running(self.cfg, self.agent)
|
||||
self.assertNotIn("1000", self.ps_targets) # root pane_pid never ps-inspected
|
||||
self.assertIn("1001", self.ps_targets)
|
||||
|
||||
def test_custom_build_procs_re_override(self):
|
||||
cfg = dict(self.cfg)
|
||||
cfg["watchdog"] = dict(self.cfg["watchdog"], build_procs_re=r"^mybuild$")
|
||||
self._patch(["mybuild"])
|
||||
self.assertTrue(agents._build_running(cfg, self.agent))
|
||||
self._patch(["cargo"]) # default tool no longer counts
|
||||
self.assertFalse(agents._build_running(cfg, self.agent))
|
||||
|
||||
|
||||
class TestBuildAwareStall(unittest.TestCase):
|
||||
"""stall_check_one defers the kill+reboot while a build runs, up to the stall_idle_max cap."""
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.mkdtemp(prefix="aotest-ut-")
|
||||
self.cfg = agents.load_config(_make_project(self.tmp))
|
||||
self.agent = self.cfg["agents"]["cl"] # stall_idle=300, stall_idle_max default 1800
|
||||
self.session = self.agent["session"]
|
||||
self.reboots = []
|
||||
self._orig = {}
|
||||
def patch(name, fn):
|
||||
self._orig[name] = getattr(agents, name)
|
||||
setattr(agents, name, fn)
|
||||
patch("session_alive", lambda s: True)
|
||||
patch("capture_pane", lambda *a, **k: "")
|
||||
patch("limit_tick", lambda *a, **k: False)
|
||||
patch("pane_active", lambda *a, **k: False)
|
||||
patch("_parse_waiting_until", lambda *a, **k: None)
|
||||
patch("_pane_last_active", lambda s: None)
|
||||
patch("phases", lambda cfg: []) # skip the DONE-nudge branch
|
||||
patch("log", lambda *a, **k: None)
|
||||
patch("start_agent", lambda cfg, a, force=False: self.reboots.append(a["name"]))
|
||||
self.patch = patch
|
||||
agents._idle_since.clear(); agents._build_deferred.discard(self.session)
|
||||
|
||||
def tearDown(self):
|
||||
for name, fn in self._orig.items():
|
||||
setattr(agents, name, fn)
|
||||
agents._idle_since.clear(); agents._build_deferred.discard(self.session)
|
||||
shutil.rmtree(self.tmp, ignore_errors=True)
|
||||
|
||||
def _set_idle(self, seconds):
|
||||
agents._idle_since[self.session] = time.time() - seconds
|
||||
|
||||
def test_defers_reboot_while_building(self):
|
||||
self.patch("_build_running", lambda *a, **k: True)
|
||||
self._set_idle(700) # > stall_idle(300), < max(1800)
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, []) # deferred, not rebooted
|
||||
self.assertIn(self.session, agents._build_deferred)
|
||||
|
||||
def test_reboots_when_idle_and_no_build(self):
|
||||
self.patch("_build_running", lambda *a, **k: False)
|
||||
self._set_idle(700)
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, [self.agent["name"]]) # genuinely idle → rebooted
|
||||
|
||||
def test_hard_cap_reboots_even_while_building(self):
|
||||
self.patch("_build_running", lambda *a, **k: True)
|
||||
self._set_idle(2000) # > stall_idle_max(1800)
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, [self.agent["name"]]) # cap reached → reboot despite build
|
||||
|
||||
def test_no_build_check_below_base_threshold(self):
|
||||
def boom(*a, **k):
|
||||
raise AssertionError("_build_running must not be consulted below stall_idle")
|
||||
self.patch("_build_running", boom)
|
||||
self._set_idle(100) # < stall_idle(300)
|
||||
agents.stall_check_one(self.cfg, self.agent)
|
||||
self.assertEqual(self.reboots, [])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main(verbosity=2)
|
||||
|
||||
Reference in New Issue
Block a user