diff --git a/README.md b/README.md index a9566e2..4692692 100644 --- a/README.md +++ b/README.md @@ -178,6 +178,24 @@ preamble = "set -a; . ./.env; set +a" # shell run before launch (e.g. l active_re = "esc interrupt|thinking|running tool|preparing patch" limit_re = "usage limit|limit reached" +[backend.codex] # Codex TUI, exposed through Codex Remote Control +bin = "codex" +preamble = "{bin} remote-control start --json >/dev/null" +flags = "--dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust --no-alt-screen" +supports_resume = false # thread-ID persistence is not implemented by this harness +prompt_delivery = "arg" +process_name = "codex" +footer_ui = true +log_grace = 180 +submit_key = "Enter" +startup_prompt_re = "Trusting the directory|Do you trust" +startup_prompt_response = "1" # explicitly trust the agent's configured working directory +startup_prompt_delay = 2 +stall_idle = 300 +active_re = "esc to interrupt|working|thinking|running|searching|exploring|implementing" +limit_re = "usage limit|limit reached|reached your .*limit|out of (credits|tokens)" +fatal_re = "connection is errored|not logged in|authentication required" + [backend.demo] # a dependency-free backend for testing the harness mechanics bin = "echo '[demo] {session} up'; exec sleep 1000000" prompt_delivery = "exec" # {kickoff}=prompt file, {session}=session name, {model}=model @@ -187,6 +205,13 @@ For an `"arg"` backend the flag *templates* are configurable (so you can point a CLI): `resume_flag` (default `--resume '{id}'`), `model_flag` (default `--model '{model}'`), `remote_control_flag` (default `--remote-control '{session}'`). A backend that sets `process_name` participates in backend-mismatch healing; one that doesn't (e.g. `demo`) never does. +An optional `preamble` runs first and must succeed before the TUI starts; it supports `{bin}`, +`{session}`, `{model}`, and `{dir}` templates. The Codex backend uses it to idempotently start the +shared Remote Control daemon before launching each tmux-hosted Codex TUI. +For an unattended TUI with a deterministic first-run gate, `startup_prompt_re` opts into one +screen check after `startup_prompt_delay` seconds; on a match, the harness types the explicitly +configured `startup_prompt_response` and `submit_key`. Codex uses this to trust the agent's +configured project directory so its argv kickoff can proceed. ### `[[agent]]` — one block per agent @@ -438,7 +463,7 @@ nix develop -c python3 agents.py selftest # or run one command in it nix flake check # evaluate + build the devShell ``` -The agent CLIs themselves (`claude`, `opencode`) are **external, non-Nix tools** — install them +The agent CLIs themselves (`claude`, `codex`, `opencode`) are **external, non-Nix tools** — install them per their own docs and make sure they are on `PATH` before launching live agents. The devShell documents this in its banner. @@ -461,13 +486,15 @@ What it runs: parsing, `WAITING-UNTIL` / stall parsing, and the per-backend activity detectors (claude + opencode footers). Always run; a failure fails the suite. Run them alone with `python3 -m unittest discover -s tests` (or `python3 tests/test_unit.py`). -- **Live backend smokes** (`tests/smoke_claude.sh`, `tests/smoke_opencode.sh`) — each brings a +- **Live backend smokes** (`tests/smoke_claude.sh`, `tests/smoke_codex.sh`, + `tests/smoke_opencode.sh`) — each brings a throwaway scratch project up **through `agents.py`** on a real backend, in a fully isolated sandbox (its own unique `session_prefix`, a temp `log_dir`, and — for opencode — a dedicated server on a non-default port `AOTEST_OC_PORT`, default `4097`), confirms the session attaches and `status` reports it RUNNING, then `down`s it and cleans up (no leftover sessions, port freed). Each **SKIPs gracefully** (exit 0) when its backend's binary or creds are unavailable. Useful env: - `CLAUDE_BIN` / `OPENCODE_BIN`, `AOTEST_MODEL`, `AOTEST_OC_PORT`, `AOTEST_OC_CREDS`. + `CLAUDE_BIN` / `CODEX_BIN` / `OPENCODE_BIN`, `AOTEST_MODEL`, `AOTEST_OC_PORT`, + `AOTEST_OC_CREDS`. - **Isolation sanity** — after the live runs, the runner asserts no `aotest-*` tmux sessions leaked and reports that any live sessions are untouched. diff --git a/agents.example.toml b/agents.example.toml index c0af01e..c74a68f 100644 --- a/agents.example.toml +++ b/agents.example.toml @@ -6,7 +6,7 @@ # # This example is self-contained: its agents use a dependency-free `demo` backend (a shell that # just idles), so the whole project can be brought up and torn down with no external agent CLI — -# see ./smoke.sh. The `claude` and `opencode` backends below are the real ones; point an agent at +# see ./smoke.sh. The `claude`, `codex`, and `opencode` backends below are real; point an agent at # them with `backend = "claude"` for a live run. # ─────────────────────────── global watchdog cadence ─────────────────────────── @@ -42,11 +42,29 @@ supports_resume = true prompt_delivery = "arg" process_name = "claude" # used for backend-mismatch healing submit_key = "Enter" +startup_prompt_re = "Trusting the directory|Do you trust" +startup_prompt_response = "1" +startup_prompt_delay = 2 stall_idle = 300 active_re = "esc to interrupt|Running tool|⠇|⠙|· \\d+" limit_re = "spend limit|usage limit|limit reached|reached your .*limit|out of (credits|tokens)" fatal_re = "redacted_thinking|blocks cannot be modified|cannot be modified" +[backend.codex] # Codex TUI + the shared Remote Control app-server daemon +bin = "codex" +preamble = "{bin} remote-control start --json >/dev/null" +flags = "--dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust --no-alt-screen" +supports_resume = false # the harness does not yet persist Codex thread IDs +prompt_delivery = "arg" +process_name = "codex" +footer_ui = true +log_grace = 180 +submit_key = "Enter" +stall_idle = 300 +active_re = "esc to interrupt|working|thinking|running|searching|exploring|implementing" +limit_re = "usage limit|limit reached|reached your .*limit|out of (credits|tokens)" +fatal_re = "connection is errored|not logged in|authentication required" + [backend.opencode] # the real opencode backend (a TUI; prompt typed after connect) bin = "opencode" attach = "{bin} attach {server} --dir {dir}" diff --git a/agents.py b/agents.py index 1d52cf0..5da8314 100755 --- a/agents.py +++ b/agents.py @@ -355,9 +355,29 @@ def start_agent(cfg, agent, *, force=False): parts.append(backend["flags"]) parts.append(f"\"$(cat '{kf}')\"") cmd = " ".join(p for p in parts if p) + # Some interactive CLIs need an idempotent service prepared before their TUI starts. + # Codex Remote Control is one such service: `remote-control start` ensures the shared + # app-server daemon is enrolled and connected, then the pane runs the ordinary Codex TUI. + if backend.get("preamble"): + preamble = _render_template(backend["preamble"], { + "bin": backend["bin"], "session": session, "model": model, + "dir": shlex.quote(cwd), + }) + cmd = f"{preamble} && {cmd}" log(f"starting {session} ({agent['backend']}, kind={agent['kind']}, phase={pid}, " f"model={model or 'default'}{', resume' if rid else ''})") new_session(session, cwd, cmd, log_path) + # An unattended TUI may present a deterministic first-run gate before it processes the + # prompt already supplied on argv (Codex asks whether to trust a new project directory). + # Backends opt in with both the screen regex and response; nothing is accepted implicitly. + startup_re = backend.get("startup_prompt_re") + if startup_re: + time.sleep(int(backend.get("startup_prompt_delay", 2))) + if re.search(startup_re, capture_pane(session, 40), re.I): + _run(["tmux", "send-keys", "-t", TP(session), "-l", "--", + str(backend.get("startup_prompt_response", ""))]) + _run(["tmux", "send-keys", "-t", TP(session), + backend.get("submit_key", "Enter")]) def start_service(cfg, svc): session = svc["session"] diff --git a/flake.nix b/flake.nix index 210a693..50aabf1 100644 --- a/flake.nix +++ b/flake.nix @@ -13,7 +13,7 @@ { # Reproducible devShell with the harness runtime deps. The driver itself is pure Python # stdlib (it needs tomllib, so python >= 3.11); the rest is what the agents/watchdog shell - # out to. Make the agent CLIs (claude / opencode) available on PATH separately — they are + # out to. Make the agent CLIs (claude / codex / opencode) available on PATH separately — they are # external, non-Nix tools; install them per their own docs, then `nix develop` here. devShells = forAllSystems (pkgs: { default = pkgs.mkShell { diff --git a/tests/run.sh b/tests/run.sh index 560b0c8..c347c52 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -4,7 +4,8 @@ # # • UNIT tests — always run (pure logic, no agents spawned). A failure fails the suite. # • CLAUDE smoke — live, run when the `claude` CLI is available; SKIPs otherwise. -# • OPENCODE smoke — live, run when `opencode` + creds are available; SKIPs otherwise. +# • CODEX smoke — live, run when Codex is installed, logged in, and Remote Control connects. +# • OPENCODE smoke — live, run when `opencode` + creds are available; SKIPs otherwise. # • ISOLATION sanity — after the live runs: assert no leftover aotest-* tmux sessions, and that # the live cc-ci-* sessions are untouched. # @@ -19,7 +20,7 @@ set -uo pipefail HERE="$(cd "$(dirname "$0")" && pwd)" REPO="$(cd "$HERE/.." && pwd)" RC=0 -UNIT=FAIL CLAUDE=SKIP OPENCODE=SKIP ISO=PASS +UNIT=FAIL CLAUDE=SKIP CODEX=SKIP OPENCODE=SKIP ISO=PASS echo "######################################################################" echo "# agent-orchestrator test suite" @@ -47,6 +48,7 @@ run_smoke() { # ── live smoke tests (when backends available) ────────────────────────────────────── run_smoke "CLAUDE" "$HERE/smoke_claude.sh"; case $? in 0) CLAUDE=PASS;; 2) CLAUDE=SKIP;; *) CLAUDE=FAIL; RC=1;; esac +run_smoke "CODEX" "$HERE/smoke_codex.sh"; case $? in 0) CODEX=PASS;; 2) CODEX=SKIP;; *) CODEX=FAIL; RC=1;; esac run_smoke "OPENCODE" "$HERE/smoke_opencode.sh"; case $? in 0) OPENCODE=PASS;; 2) OPENCODE=SKIP;; *) OPENCODE=FAIL; RC=1;; esac # ── isolation sanity ──────────────────────────────────────────────────────────────── @@ -69,7 +71,7 @@ fi # ── summary ───────────────────────────────────────────────────────────────────────── echo; echo "######################################################################" -echo "# SUMMARY: unit=$UNIT claude=$CLAUDE opencode=$OPENCODE isolation=$ISO" +echo "# SUMMARY: unit=$UNIT claude=$CLAUDE codex=$CODEX opencode=$OPENCODE isolation=$ISO" echo "######################################################################" [ "$RC" -eq 0 ] && echo "ALL RUN TESTS PASSED (skips are OK)" || echo "SUITE FAILED" exit "$RC" diff --git a/tests/smoke_codex.sh b/tests/smoke_codex.sh new file mode 100755 index 0000000..17a1c7c --- /dev/null +++ b/tests/smoke_codex.sh @@ -0,0 +1,113 @@ +#!/usr/bin/env bash +# Isolated live smoke of the Codex backend, driven entirely through agents.py. +# It starts one uniquely-named tmux session, verifies Remote Control and the Codex TUI, then +# removes only that session. The shared Remote Control daemon intentionally remains available. +set -uo pipefail + +HERE="$(cd "$(dirname "$0")" && pwd)" +REPO="$(cd "$HERE/.." && pwd)" +CODEX_BIN="${CODEX_BIN:-$(command -v codex 2>/dev/null || echo "$HOME/.local/bin/codex")}" +MODEL="${AOTEST_CODEX_MODEL:-}" +PREFIX="aotest-x-$$-" +SANDBOX="$(mktemp -d)" +CFG="$SANDBOX/agents.toml" +FAILED=0 + +pass(){ echo " PASS: $*"; } +fail(){ echo " FAIL: $*"; FAILED=1; } + +cleanup(){ + local rc=$? + python3 "$REPO/agents.py" --config "$CFG" down probe >/dev/null 2>&1 || true + if command -v tmux >/dev/null 2>&1; then + tmux ls 2>/dev/null | sed 's/:.*//' | grep "^${PREFIX}" | while read -r s; do + tmux kill-session -t "=$s" 2>/dev/null || true + done || true + fi + rm -rf "$SANDBOX" + exit "$rc" +} +trap cleanup EXIT INT TERM + +echo "=== codex backend smoke (isolated: prefix=${PREFIX}) ===" +command -v tmux >/dev/null 2>&1 || { echo "SKIP: tmux not on PATH"; exit 0; } +[ -x "$CODEX_BIN" ] || command -v "$CODEX_BIN" >/dev/null 2>&1 \ + || { echo "SKIP: codex binary not found ($CODEX_BIN)"; exit 0; } +"$CODEX_BIN" login status >/dev/null 2>&1 \ + || { echo "SKIP: Codex is not logged in"; exit 0; } +remote_status="$("$CODEX_BIN" remote-control start --json 2>&1)" +echo "$remote_status" | grep -q '"status":"connected"' \ + || { echo "SKIP: Codex Remote Control is not connected: $remote_status"; exit 0; } +pass "Codex Remote Control reports connected" + +model_line="" +[ -n "$MODEL" ] && model_line="model = \"$MODEL\"" +cat > "$CFG" </dev/null; then + cmd="$(tmux display-message -p -t "=${PREFIX}probe:" '#{pane_current_command}')" + pass "session ${PREFIX}probe created via agents.py (pane command: ${cmd})" +else + fail "${PREFIX}probe session was not created" + echo "=== CODEX BACKEND SMOKE: FAIL ===" + exit 1 +fi + +pane="$(tmux capture-pane -p -t "=${PREFIX}probe:" -S -200 2>/dev/null)" +if [ "$cmd" = "codex" ] && echo "$pane" | grep -q "CODEX_BACKEND_READY"; then + pass "Codex TUI attached and completed the probe" +else + fail "Codex probe did not complete (cmd=${cmd}); tail: $(echo "$pane" | grep -vE '^\s*$' | tail -5)" +fi + +if python3 "$REPO/agents.py" --config "$CFG" status \ + | grep -E '^\s*probe\b' | grep -q RUNNING; then + pass "agents.py status reports probe RUNNING" +else + fail "agents.py status did not report probe RUNNING" +fi + +python3 "$REPO/agents.py" --config "$CFG" down probe >/dev/null 2>&1 +sleep 2 +if tmux has-session -t "=${PREFIX}probe" 2>/dev/null; then + fail "${PREFIX}probe still alive after agents.py down" +else + pass "agents.py down cleanly removed the session" +fi + +if [ "$FAILED" = 0 ]; then echo "=== CODEX BACKEND SMOKE: PASS ==="; exit 0 +else echo "=== CODEX BACKEND SMOKE: FAIL ==="; exit 1; fi diff --git a/tests/test_unit.py b/tests/test_unit.py index 1f340e8..e91954b 100755 --- a/tests/test_unit.py +++ b/tests/test_unit.py @@ -74,6 +74,22 @@ stall_idle = 900 active_re = "esc interrupt|thinking|inferring|running tool|tool call|preparing patch|reading|searching" limit_re = "usage limit|limit reached" +[backend.codex] +bin = "codex" +preamble = "{bin} remote-control start --json >/dev/null" +flags = "--dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust --no-alt-screen" +supports_resume = false +prompt_delivery = "arg" +process_name = "codex" +footer_ui = true +log_grace = 180 +submit_key = "Enter" +startup_prompt_re = "Trusting the directory|Do you trust" +startup_prompt_response = "1" +startup_prompt_delay = 0 +active_re = "working|thinking|running" +limit_re = "usage limit|limit reached" + [backend.demo] bin = "echo up; exec sleep 100000" prompt_delivery = "exec" @@ -102,6 +118,13 @@ kind = "persistent" backend = "opencode" prompt = "hi" +[[agent]] +name = "cx" +kind = "persistent" +backend = "codex" +model = "gpt-test" +prompt = "hi from codex" + [[agent]] name = "custom" kind = "persistent" @@ -233,10 +256,39 @@ class TestExampleConfig(unittest.TestCase): cfg = agents.load_config(ex) self.assertIn("builder", cfg["agents"]) self.assertIn("adversary", cfg["agents"]) - for be in ("demo", "claude", "opencode"): + for be in ("demo", "claude", "codex", "opencode"): self.assertIn(be, cfg["backends"], f"backend {be} missing from example") self.assertEqual(len(agents.phases(cfg)), 2) + def test_codex_launch_prepares_remote_control_then_starts_tui(self): + tmp = tempfile.mkdtemp(prefix="aotest-ut-codex-") + old_alive, old_new, old_log = agents.session_alive, agents.new_session, agents.log + old_capture, old_run = agents.capture_pane, agents._run + try: + cfg = agents.load_config(_make_project(tmp)) + launched = [] + sent = [] + agents.session_alive = lambda _session: False + agents.new_session = lambda session, cwd, cmd, log_path: launched.append(cmd) + agents.log = lambda _msg: None + agents.capture_pane = lambda *_args, **_kwargs: "Trusting the directory" + agents._run = lambda command: sent.append(command) + agents.start_agent(cfg, cfg["agents"]["cx"]) + self.assertEqual(len(launched), 1) + cmd = launched[0] + self.assertTrue(cmd.startswith( + "codex remote-control start --json >/dev/null && codex ")) + self.assertIn("--model 'gpt-test'", cmd) + self.assertIn( + "--dangerously-bypass-approvals-and-sandbox --dangerously-bypass-hook-trust --no-alt-screen", + cmd) + self.assertIn("kickoff-aotest-ut-cx.txt", cmd) + self.assertTrue(any("1" in command for command in sent)) + finally: + agents.session_alive, agents.new_session, agents.log = old_alive, old_new, old_log + agents.capture_pane, agents._run = old_capture, old_run + shutil.rmtree(tmp, ignore_errors=True) + # ── kickoff-template assembly ──────────────────────────────────────────────────────