mfowler
27df2c7b55
A standalone repo (engine vendored as a submodule at the examples commit) that runs a head-to-head between the builder-adversary and builder-adversary-min example variants: same task, independent headless runs, both on Sonnet, with token counts. Includes the roman-numeral test problem and run-bench.sh. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
165 lines
8.2 KiB
Bash
Executable File
165 lines
8.2 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# agent-orchestrator-benchmark — head-to-head prompt comparison.
|
|
#
|
|
# Compares two example variants that ship in the vendored engine (engine/examples/):
|
|
# • builder-adversary (original prompts)
|
|
# • builder-adversary-min (minimal prompts)
|
|
# on the same task, INDEPENDENTLY (no shared context), both on Sonnet. Confirms each succeeds and
|
|
# clocks the tokens each uses.
|
|
#
|
|
# Methodology (controlled, reproducible — not the full self-paced watchdog loop):
|
|
# For each version we assemble EXACTLY the prompt the harness would send a loop agent
|
|
# (kickoff_template with {phase_id}/{plan}/{status}/{role} substituted, then the role prompt),
|
|
# and drive one Builder pass then one Adversary pass as separate headless `claude -p` sessions
|
|
# (fresh context each → "no shared context"). The Builder builds+commits in its own repo; the
|
|
# Adversary cold-verifies from its OWN clone. We then independently re-run the DoD check and read
|
|
# the Adversary's verdict. Tokens come from `claude -p --output-format json`.
|
|
#
|
|
# A short identical DRIVER NOTE is appended to BOTH versions' prompts so the agents finish in one
|
|
# headless session (no /loop, no waiting). Being identical, it doesn't bias the comparison.
|
|
#
|
|
# Usage: ./run-bench.sh (writes RESULTS.md here; work dirs under a temp dir)
|
|
set -u
|
|
|
|
BENCH_DIR="$(cd "$(dirname "$0")" && pwd)"
|
|
ENGINE_EX="$BENCH_DIR/engine/examples"
|
|
PLAN_SRC="$BENCH_DIR/plans/roman.md"
|
|
MODEL="claude-sonnet-4-6"
|
|
RUNROOT="$(mktemp -d /tmp/ao-benchmark.XXXXXX)"
|
|
RESULTS="$BENCH_DIR/RESULTS.md"
|
|
GIT_ID=(-c user.email=bench@example.com -c user.name=bench)
|
|
|
|
DRIVER_NOTE='[HEADLESS BENCH — single session, NO loop] Do the ENTIRE phase now, in THIS one session: implement the code and tests, run them, then `git add -A && git commit -m "..."`. Builder: also write machine-docs/STATUS-roman.md and make a claim commit (prefix `claim(`). Adversary: cold-verify and write machine-docs/REVIEW-roman.md with `roman: PASS` or `roman: FAIL`, then commit (prefix `review(`). Do NOT invoke /loop, do NOT ScheduleWakeup, do NOT wait — finish and exit. No git remote exists; commit locally only (do not push).'
|
|
|
|
# assemble_prompt <version> <role> <plan-relpath> -> stdout
|
|
assemble_prompt() {
|
|
local version="$1" role="$2" planrel="$3"
|
|
sed -e "s/{phase_id}/roman/g" \
|
|
-e "s#{plan}#$planrel#g" \
|
|
-e "s/{status}/STATUS-roman.md/g" \
|
|
-e "s/{role}/$role/g" \
|
|
"$ENGINE_EX/$version/prompts/kickoff.md"
|
|
printf '\n'
|
|
cat "$ENGINE_EX/$version/prompts/$role.md"
|
|
printf '\n\n%s\n' "$DRIVER_NOTE"
|
|
}
|
|
|
|
# usage_of <json-file> -> "in out cache_create cache_read cost turns"
|
|
usage_of() {
|
|
python3 - "$1" <<'PY'
|
|
import json,sys
|
|
try:
|
|
d=json.load(open(sys.argv[1]))
|
|
except Exception:
|
|
print("0 0 0 0 0 0"); sys.exit()
|
|
u=d.get("usage",{}) or {}
|
|
print(u.get("input_tokens",0), u.get("output_tokens",0),
|
|
u.get("cache_creation_input_tokens",0), u.get("cache_read_input_tokens",0),
|
|
d.get("total_cost_usd",0), d.get("num_turns",0))
|
|
PY
|
|
}
|
|
|
|
declare -A SUM_TOK SUM_COST SUM_OK
|
|
|
|
run_version() {
|
|
local version="$1"
|
|
local W="$RUNROOT/$version"
|
|
local A="$RUNROOT/$version-adv"
|
|
echo "===== $version ====="
|
|
|
|
# --- Builder: its own work repo ---
|
|
mkdir -p "$W/plans"; cp "$PLAN_SRC" "$W/plans/roman.md"
|
|
git "${GIT_ID[@]}" -C "$W" init -q
|
|
git "${GIT_ID[@]}" -C "$W" add -A && git "${GIT_ID[@]}" -C "$W" commit -q -m "chore: seed plan"
|
|
echo "[$version] builder running…"
|
|
( cd "$W" && timeout 600 claude -p "$(assemble_prompt "$version" builder plans/roman.md)" \
|
|
--output-format json --model "$MODEL" --dangerously-skip-permissions ) > "$RUNROOT/$version.builder.json" 2>"$RUNROOT/$version.builder.err"
|
|
|
|
# --- Adversary: its OWN clone, cold ---
|
|
git "${GIT_ID[@]}" clone -q "$W" "$A"
|
|
echo "[$version] adversary running…"
|
|
( cd "$A" && timeout 600 claude -p "$(assemble_prompt "$version" adversary plans/roman.md)" \
|
|
--output-format json --model "$MODEL" --dangerously-skip-permissions ) > "$RUNROOT/$version.adv.json" 2>"$RUNROOT/$version.adv.err"
|
|
|
|
# --- independent success check (re-run DoD ourselves, in the adversary's clone) ---
|
|
local tests_ok="no" cli_ok="no" verdict="missing"
|
|
( cd "$A" && python -m unittest -q ) >"$RUNROOT/$version.unittest.txt" 2>&1 && tests_ok="yes"
|
|
local out; out="$( cd "$A" && python roman.py 1994 2>/dev/null )"
|
|
[ "$out" = "MCMXCIV" ] && cli_ok="yes"
|
|
if [ -f "$A/machine-docs/REVIEW-roman.md" ]; then
|
|
if grep -qiE 'roman:?\s*PASS' "$A/machine-docs/REVIEW-roman.md" && ! grep -qi 'VETO' "$A/machine-docs/REVIEW-roman.md"; then
|
|
verdict="PASS"; else verdict="FAIL/none"; fi
|
|
fi
|
|
local success="NO"
|
|
[ "$tests_ok" = yes ] && [ "$cli_ok" = yes ] && [ "$verdict" = PASS ] && success="YES"
|
|
|
|
# --- tally tokens ---
|
|
read -r bi bo bcc bcr bcost bturns <<<"$(usage_of "$RUNROOT/$version.builder.json")"
|
|
read -r ai ao acc acr acost aturns <<<"$(usage_of "$RUNROOT/$version.adv.json")"
|
|
local btok=$((bi+bo+bcc+bcr)) atok=$((ai+ao+acc+acr))
|
|
local vtok=$((btok+atok))
|
|
local vcost; vcost=$(python3 -c "print(round(${bcost:-0}+${acost:-0},4))")
|
|
SUM_TOK[$version]=$vtok; SUM_COST[$version]=$vcost; SUM_OK[$version]=$success
|
|
|
|
{
|
|
echo "### $version"
|
|
echo "- **success:** $success (tests=$tests_ok, cli=$cli_ok, adversary-verdict=$verdict)"
|
|
echo "- **builder:** in=$bi out=$bo cache_create=$bcc cache_read=$bcr → ${btok} tok, \$${bcost}, turns=$bturns"
|
|
echo "- **adversary:** in=$ai out=$ao cache_create=$acc cache_read=$acr → ${atok} tok, \$${acost}, turns=$aturns"
|
|
echo "- **total:** ${vtok} tokens, \$${vcost}"
|
|
echo
|
|
} >>"$RESULTS.tmp"
|
|
}
|
|
|
|
# static prompt size (chars: kickoff + role, what gets sent each kickoff)
|
|
prompt_chars() { cat "$ENGINE_EX/$1/prompts/kickoff.md" "$ENGINE_EX/$1/prompts/$2.md" | wc -c | tr -d ' '; }
|
|
|
|
: >"$RESULTS.tmp"
|
|
run_version builder-adversary
|
|
run_version builder-adversary-min
|
|
|
|
# ---- write RESULTS.md ----
|
|
{
|
|
echo "# Benchmark results — original vs minimal prompts"
|
|
echo
|
|
echo "Engine pinned at: \`$(git -C "$BENCH_DIR/engine" rev-parse --short HEAD)\`. Task:"
|
|
echo "\`plans/roman.md\` (integer → Roman numeral). Model: **$MODEL** for Builder and Adversary in"
|
|
echo "both versions. Runs are independent (separate headless \`claude -p\` sessions, no shared"
|
|
echo "context). Methodology + caveats: see \`run-bench.sh\` header and the note below."
|
|
echo
|
|
echo "## Static prompt size (chars: kickoff + role, what gets sent each kickoff)"
|
|
echo
|
|
echo "| version | builder prompt | adversary prompt |"
|
|
echo "|---|--:|--:|"
|
|
echo "| builder-adversary (orig) | $(prompt_chars builder-adversary builder) | $(prompt_chars builder-adversary adversary) |"
|
|
echo "| builder-adversary-min | $(prompt_chars builder-adversary-min builder) | $(prompt_chars builder-adversary-min adversary) |"
|
|
echo
|
|
echo "## Per-run tokens & cost"
|
|
echo
|
|
cat "$RESULTS.tmp"
|
|
echo "## Summary"
|
|
echo
|
|
echo "| version | success | total tokens | total cost |"
|
|
echo "|---|:--:|--:|--:|"
|
|
echo "| builder-adversary (orig) | ${SUM_OK[builder-adversary]} | ${SUM_TOK[builder-adversary]} | \$${SUM_COST[builder-adversary]} |"
|
|
echo "| builder-adversary-min | ${SUM_OK[builder-adversary-min]} | ${SUM_TOK[builder-adversary-min]} | \$${SUM_COST[builder-adversary-min]} |"
|
|
echo
|
|
echo "> Note: each \`claude -p\` call carries a fixed ~24k-token cached Claude Code system-prompt +"
|
|
echo "> tool-schema overhead, and most tokens come from the agentic work itself (reading the plan,"
|
|
echo "> writing/running code, tool results). The role/kickoff prompt is a small slice — so the"
|
|
echo "> headline token totals are close; the minimisation shows up in the static prompt size above"
|
|
echo "> and the (smaller) input/cache-creation portion. This bench is a single controlled pass per"
|
|
echo "> version (N=1; expect run-to-run variance); it exercises task effectiveness + prompt cost,"
|
|
echo "> NOT the live watchdog loop / handoff machinery (that needs a full \`agents.py up\` run)."
|
|
echo
|
|
echo "_Work dirs for this run: \`$RUNROOT\`_"
|
|
} >"$RESULTS"
|
|
rm -f "$RESULTS.tmp"
|
|
|
|
echo
|
|
echo "===== DONE ====="
|
|
echo "orig: success=${SUM_OK[builder-adversary]} tokens=${SUM_TOK[builder-adversary]} cost=\$${SUM_COST[builder-adversary]}"
|
|
echo "min : success=${SUM_OK[builder-adversary-min]} tokens=${SUM_TOK[builder-adversary-min]} cost=\$${SUM_COST[builder-adversary-min]}"
|
|
echo "Results: $RESULTS"
|
|
echo "Run dirs: $RUNROOT"
|