feat: multi-phase calculator problem + full-harness benchmark runner
- plans/calc/{lex,parse,eval}.md: a 3-phase calculator with multiple gates per
phase (tokenizer → recursive-descent parser → evaluator+CLI), rich adversarial
edge cases (precedence/associativity/unary/div-zero)
- run-harness-bench.sh: stands up a real agents.py up Builder/Adversary loop pair
+ watchdog over a shared work repo per variant, runs to SEQUENCE-COMPLETE, and
clocks tokens from the session transcripts (AI-as-adversary kept intact)
- RESULTS.md: baseline single-pass roman-numeral run (prompt size had ~0 token
effect; cache-read of the working context dominates)
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 4.8
Executable
+217
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env bash
|
||||
# run-harness-bench.sh — FULL harness benchmark (real agents.py up loop, not headless single-pass).
|
||||
#
|
||||
# For each variant (builder-adversary, builder-adversary-min) this stands up a real Builder/Adversary
|
||||
# loop pair + watchdog over a shared work repo and lets them run autonomously through the multi-phase
|
||||
# calculator (plans/calc/{lex,parse,eval}.md) to SEQUENCE-COMPLETE. Both loops on Sonnet. Then it
|
||||
# clocks the tokens each loop used (summed from the Claude Code session transcripts) and re-runs the
|
||||
# final Definition-of-Done itself.
|
||||
#
|
||||
# Long, autonomous, nondeterministic (N=1). Per-variant wall-clock timeout below. Usage: ./run-harness-bench.sh
|
||||
set -u
|
||||
|
||||
BENCH_DIR="$(cd "$(dirname "$0")" && pwd)"
|
||||
ENGINE="$BENCH_DIR/engine"
|
||||
PLANS="$BENCH_DIR/plans/calc"
|
||||
AGENTS_PY="$ENGINE/agents.py"
|
||||
MODEL="claude-sonnet-4-6"
|
||||
RUNROOT="$(mktemp -d /tmp/ao-harness-XXXXXX)" # no dot → clean transcript-dir mapping
|
||||
RESULTS="$BENCH_DIR/RESULTS-harness.md"
|
||||
TIMEOUT="${BENCH_TIMEOUT:-3000}" # seconds per variant
|
||||
POLL=60
|
||||
GIT_ID=(-c user.email=bench@example.com -c user.name=bench)
|
||||
|
||||
log() { echo "[$(date -u +%H:%M:%S)] $*"; }
|
||||
|
||||
# transcript token sum for an agent's working dir -> "in out cache_create cache_read"
|
||||
collect_tokens() {
|
||||
python3 - "$1" <<'PY'
|
||||
import json,sys,os,glob
|
||||
wd=sys.argv[1].rstrip('/')
|
||||
name=wd.replace('/','-').replace('.','-') # '/tmp/x' -> '-tmp-x'
|
||||
tdir=os.path.expanduser("~/.claude/projects/"+name)
|
||||
ti=to=tcc=tcr=0
|
||||
for f in glob.glob(tdir+"/*.jsonl"):
|
||||
for line in open(f, errors="ignore"):
|
||||
try: o=json.loads(line)
|
||||
except Exception: continue
|
||||
if o.get("type")=="assistant":
|
||||
u=(o.get("message",{}) or {}).get("usage",{}) or {}
|
||||
ti+=u.get("input_tokens",0) or 0; to+=u.get("output_tokens",0) or 0
|
||||
tcc+=u.get("cache_creation_input_tokens",0) or 0; tcr+=u.get("cache_read_input_tokens",0) or 0
|
||||
print(ti,to,tcc,tcr)
|
||||
PY
|
||||
}
|
||||
|
||||
gen_config() { # <variant> <run> <prefix>
|
||||
local v="$1" run="$2" prefix="$3"
|
||||
cat > "$run/agents.toml" <<EOF
|
||||
[watchdog]
|
||||
signal_interval = 15
|
||||
heavy_interval = 60
|
||||
limit_probe_fallback = 300
|
||||
limit_reset_slack = 45
|
||||
stall_grace = 180
|
||||
|
||||
[defaults]
|
||||
session_prefix = "$prefix"
|
||||
log_dir = "$run/.ao-state"
|
||||
backend = "claude"
|
||||
model = "$MODEL"
|
||||
watch = "heal"
|
||||
|
||||
[backend.claude]
|
||||
bin = "claude"
|
||||
flags = "--dangerously-skip-permissions"
|
||||
remote_control = true
|
||||
supports_resume = true
|
||||
prompt_delivery = "arg"
|
||||
process_name = "claude"
|
||||
submit_key = "Enter"
|
||||
stall_idle = 300
|
||||
active_re = "esc to interrupt|Running tool|⠇|⠙|· \\\\d+"
|
||||
limit_re = "spend limit|usage limit|limit reached|reached your .*limit|out of (credits|tokens)"
|
||||
fatal_re = "redacted_thinking|blocks cannot be modified|cannot be modified"
|
||||
|
||||
[[agent]]
|
||||
name = "builder"
|
||||
kind = "loop"
|
||||
role = "builder"
|
||||
dir = "$run/work"
|
||||
watch = "heal+stall"
|
||||
|
||||
[[agent]]
|
||||
name = "adversary"
|
||||
session = "${prefix}adv"
|
||||
kind = "loop"
|
||||
role = "adversary"
|
||||
dir = "$run/work-adv"
|
||||
watch = "heal+stall"
|
||||
|
||||
[loop]
|
||||
state_file = "phase-idx"
|
||||
resume_phase = true
|
||||
auto_advance = true
|
||||
done_marker = "## DONE"
|
||||
kickoff_template = "$ENGINE/examples/$v/prompts/kickoff.md"
|
||||
roles_dir = "$ENGINE/examples/$v/prompts"
|
||||
handoff = { repo = "$run/work", claim_pings = "adversary", review_pings = "builder", inboxes = ["ADVERSARY-INBOX.md", "BUILDER-INBOX.md"], claim_pattern = "^claim", review_pattern = "^review", state_subdir = "machine-docs" }
|
||||
phases = [
|
||||
{ id = "lex", plan = "$PLANS/lex.md", status = "STATUS-lex.md" },
|
||||
{ id = "parse", plan = "$PLANS/parse.md", status = "STATUS-parse.md" },
|
||||
{ id = "eval", plan = "$PLANS/eval.md", status = "STATUS-eval.md" },
|
||||
]
|
||||
EOF
|
||||
}
|
||||
|
||||
declare -A SUM_TOK SUM_OK SUM_PHASES
|
||||
|
||||
run_variant() {
|
||||
local v="$1"
|
||||
local run="$RUNROOT/$v"
|
||||
local rtag; rtag=$(basename "$RUNROOT"); rtag=${rtag##*-} # mktemp suffix → unique per invocation
|
||||
local prefix="b${rtag}$(echo "$v" | cksum | cut -c1-2)-" # unique per (run, variant); avoids tmux collisions
|
||||
mkdir -p "$run"
|
||||
log "===== $v (run dir: $run, prefix: $prefix) ====="
|
||||
|
||||
# shared bare 'origin' + two clones
|
||||
git "${GIT_ID[@]}" init -q --bare "$run/origin.git"
|
||||
git -C "$run/origin.git" symbolic-ref HEAD refs/heads/main # so clones check out 'main' (we push main, not master)
|
||||
git "${GIT_ID[@]}" init -q -b main "$run/seed"
|
||||
( cd "$run/seed" && mkdir -p machine-docs && echo "# calc work repo" > README.md \
|
||||
&& : > machine-docs/.gitkeep \
|
||||
&& git "${GIT_ID[@]}" add -A && git "${GIT_ID[@]}" commit -q -m "chore: seed work repo" \
|
||||
&& git "${GIT_ID[@]}" remote add origin "$run/origin.git" && git "${GIT_ID[@]}" push -q -u origin main )
|
||||
git "${GIT_ID[@]}" clone -q "$run/origin.git" "$run/work"
|
||||
git "${GIT_ID[@]}" clone -q "$run/origin.git" "$run/work-adv"
|
||||
for c in work work-adv; do ( cd "$run/$c" && git config user.email bench@example.com && git config user.name bench ); done
|
||||
|
||||
gen_config "$v" "$run" "$prefix"
|
||||
|
||||
log "[$v] agents.py up …"
|
||||
python3 "$AGENTS_PY" up --config "$run/agents.toml" >"$run/up.log" 2>&1
|
||||
log "[$v] up done; status:"; python3 "$AGENTS_PY" status --config "$run/agents.toml" 2>&1 | sed 's/^/ /'
|
||||
|
||||
# poll for SEQUENCE-COMPLETE or timeout
|
||||
local marker="$run/.ao-state/SEQUENCE-COMPLETE" t=0 done="no"
|
||||
while [ $t -lt "$TIMEOUT" ]; do
|
||||
if [ -f "$marker" ]; then done="yes"; break; fi
|
||||
sleep "$POLL"; t=$((t+POLL))
|
||||
local idx commits
|
||||
idx="$(cat "$run/.ao-state/state/phase-idx" 2>/dev/null || echo '?')"
|
||||
commits="$(git -C "$run/origin.git" rev-list --count main 2>/dev/null || echo '?')"
|
||||
log "[$v] t=${t}s phase-idx=$idx origin-commits=$commits"
|
||||
done
|
||||
log "[$v] loop finished (sequence-complete=$done after ${t}s); tearing down"
|
||||
python3 "$AGENTS_PY" down --config "$run/agents.toml" >"$run/down.log" 2>&1
|
||||
|
||||
# final DoD check from the adversary's clone (pull latest first)
|
||||
( cd "$run/work-adv" && git "${GIT_ID[@]}" pull -q --no-rebase origin main 2>/dev/null )
|
||||
local tests=no cli=no
|
||||
( cd "$run/work-adv" && python -m unittest -q ) >"$run/final-unittest.txt" 2>&1 && tests=yes
|
||||
local out; out="$( cd "$run/work-adv" && python calc.py '2+3*4' 2>/dev/null )"
|
||||
[ "$out" = "14" ] && cli=yes
|
||||
local reviews; reviews="$(grep -rhoiE '(lex|parse|eval)/D[0-9]+:?\s*PASS' "$run/work-adv/machine-docs/" 2>/dev/null | sort -u | wc -l)"
|
||||
local veto="no"; grep -rqi 'VETO' "$run/work-adv/machine-docs/" 2>/dev/null && veto=yes
|
||||
SUM_PHASES[$v]="$(cat "$run/.ao-state/state/phase-idx" 2>/dev/null || echo '?')"
|
||||
|
||||
local success=NO
|
||||
[ "$done" = yes ] && [ "$tests" = yes ] && [ "$cli" = yes ] && [ "$veto" = no ] && success=YES
|
||||
SUM_OK[$v]=$success
|
||||
|
||||
# tokens
|
||||
read -r bi bo bcc bcr <<<"$(collect_tokens "$run/work")"
|
||||
read -r ai ao acc acr <<<"$(collect_tokens "$run/work-adv")"
|
||||
local btok=$((bi+bo+bcc+bcr)) atok=$((ai+ao+acc+acr)) vtok=$(( bi+bo+bcc+bcr + ai+ao+acc+acr ))
|
||||
SUM_TOK[$v]=$vtok
|
||||
|
||||
{
|
||||
echo "### $v"
|
||||
echo "- **success:** $success (sequence-complete=$done, tests=$tests, cli('2+3*4'→'$out')=$cli, gates-passed=$reviews, veto=$veto, final phase-idx=${SUM_PHASES[$v]})"
|
||||
echo "- **builder loop:** in=$bi out=$bo cache_create=$bcc cache_read=$bcr → **${btok}** tok"
|
||||
echo "- **adversary loop:** in=$ai out=$ao cache_create=$acc cache_read=$acr → **${atok}** tok"
|
||||
echo "- **total:** **${vtok}** tokens"
|
||||
echo
|
||||
} >>"$RESULTS.tmp"
|
||||
log "[$v] DONE success=$success tokens=$vtok gates-passed=$reviews"
|
||||
}
|
||||
|
||||
prompt_chars() { cat "$ENGINE/examples/$1/prompts/kickoff.md" "$ENGINE/examples/$1/prompts/$2.md" | wc -c | tr -d ' '; }
|
||||
|
||||
: >"$RESULTS.tmp"
|
||||
run_variant builder-adversary
|
||||
run_variant builder-adversary-min
|
||||
|
||||
{
|
||||
echo "# Full-harness benchmark — original vs minimal prompts"
|
||||
echo
|
||||
echo "Real \`agents.py up\` Builder/Adversary loop pair + watchdog, run autonomously through the"
|
||||
echo "multi-phase calculator (\`plans/calc/{lex,parse,eval}.md\`) to SEQUENCE-COMPLETE. Engine pinned"
|
||||
echo "at \`$(git -C "$ENGINE" rev-parse --short HEAD)\`. Both loops on **$MODEL**. Per-variant timeout"
|
||||
echo "${TIMEOUT}s. Tokens summed from the Claude Code session transcripts of each loop's clone."
|
||||
echo
|
||||
echo "## Static prompt size (chars: kickoff + role)"
|
||||
echo "| version | builder | adversary |"
|
||||
echo "|---|--:|--:|"
|
||||
echo "| builder-adversary (orig) | $(prompt_chars builder-adversary builder) | $(prompt_chars builder-adversary adversary) |"
|
||||
echo "| builder-adversary-min | $(prompt_chars builder-adversary-min builder) | $(prompt_chars builder-adversary-min adversary) |"
|
||||
echo
|
||||
echo "## Per-variant"
|
||||
echo
|
||||
cat "$RESULTS.tmp"
|
||||
echo "## Summary"
|
||||
echo "| version | success | total tokens |"
|
||||
echo "|---|:--:|--:|"
|
||||
echo "| builder-adversary (orig) | ${SUM_OK[builder-adversary]:-?} | ${SUM_TOK[builder-adversary]:-?} |"
|
||||
echo "| builder-adversary-min | ${SUM_OK[builder-adversary-min]:-?} | ${SUM_TOK[builder-adversary-min]:-?} |"
|
||||
echo
|
||||
echo "_N=1 per variant; the autonomous loop is nondeterministic (number of review rounds varies)._"
|
||||
echo "_Run dirs: \`$RUNROOT\`_"
|
||||
} >"$RESULTS"
|
||||
rm -f "$RESULTS.tmp"
|
||||
|
||||
echo; echo "===== ALL DONE ====="
|
||||
echo "orig: success=${SUM_OK[builder-adversary]:-?} tokens=${SUM_TOK[builder-adversary]:-?}"
|
||||
echo "min : success=${SUM_OK[builder-adversary-min]:-?} tokens=${SUM_TOK[builder-adversary-min]:-?}"
|
||||
echo "Results: $RESULTS"
|
||||
echo "Run dirs: $RUNROOT"
|
||||
Reference in New Issue
Block a user