recipe-maintainers/agent-orchestrator-benchmark
0
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: keep run repos + record commits/LOC per run

Browse Source 
Per request: stop deleting each run's git repo after tallying — keep work/,
work-adv/, origin.git under the run root so differences can be analysed. Record
commit count (origin rev-list) and calc/*.py LOC in each data row; aggregation
now reports per-variant median commits/LOC and tokens~{duration,commits,LOC}
correlations over successful runs, plus the full raw table.
This commit is contained in:
mfowler
2026-06-15 00:13:08 +00:00
parent 37032ee363
commit dbe9ef9c72
1 changed files with 48 additions and 29 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
run-harness-bench.sh
View File

@ -163,17 +163,19 @@ run_one() { # <variant> <rep>
read -r bi bo bcc bcr <<<"$(collect_tokens "$run/work")"
read -r ai ao acc acr <<<"$(collect_tokens "$run/work-adv")"
local btok=$((bi+bo+bcc+bcr)) atok=$((ai+ao+acc+acr)) total=$(( bi+bo+bcc+bcr + ai+ao+acc+acr ))
local commits loc
commits=$(git -C "$run/origin.git" rev-list --count main 2>/dev/null || echo 0)
loc=$(cat "$run/work-adv"/calc/*.py 2>/dev/null | grep -cve '^[[:space:]]*$')
# append one row immediately (survives a kill): variant rep success btok atok total dur
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$v" "$rep" "$success" "$btok" "$atok" "$total" "$t" >>"$DATA"
log " -> $v r$rep: success=$success total=$total dur=${t}s"
# free disk: drop this run's transcripts + repo now that tokens are tallied
rm -rf "$run/work/.git" "$run/work-adv/.git" "$run/origin.git" "$run/seed" 2>/dev/null
# append one row immediately (survives a kill): variant rep success btok atok total dur commits loc
printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$v" "$rep" "$success" "$btok" "$atok" "$total" "$t" "$commits" "$loc" >>"$DATA"
log " -> $v r$rep: success=$success total=$total dur=${t}s commits=$commits loc=$loc"
# NOTE: repos are KEPT under "$run" (work/, work-adv/, origin.git) for later analysis — not deleted.
}
# ---- collect ----
: > "$DATA"
printf 'variant\trep\tsuccess\tbuilder\tadversary\ttotal\tduration_s\n' > "$DATA.hdr"
printf 'variant\trep\tsuccess\tbuilder\tadversary\ttotal\tduration_s\tcommits\tloc\n' > "$DATA.hdr"
for v in "${VARIANTS[@]}"; do
for rep in $(seq 1 "$REPEATS"); do run_one "$v" "$rep"; done
done
@ -185,41 +187,58 @@ data, out, model, reps, eng, runroot = sys.argv[1:7]
rows=[]
for line in open(data):
p=line.rstrip("\n").split("\t")
if len(p)!=7: continue
v,rep,ok,b,a,tot,dur=p
rows.append((v,int(rep),ok,int(b),int(a),int(tot),int(dur)))
if len(p)!=9: continue
v,rep,ok,b,a,tot,dur,commits,loc=p
rows.append(dict(v=v, rep=int(rep), ok=ok, builder=int(b), adversary=int(a),
total=int(tot), dur=int(dur), commits=int(commits), loc=int(loc)))
variants=[]
for r in rows:
if r[0] not in variants: variants.append(r[0])
if r["v"] not in variants: variants.append(r["v"])
def fmt(n): return f"{n:,}"
def stats(xs):
if not xs: return None
return dict(n=len(xs), min=min(xs), max=max(xs), mean=int(st.mean(xs)),
median=int(st.median(xs)), stdev=int(st.pstdev(xs)) if len(xs)>1 else 0)
def pearson(a,b):
if len(a)<2: return float('nan')
ma,mb=st.mean(a),st.mean(b)
den=(sum((x-ma)**2 for x in a)*sum((y-mb)**2 for y in b))**0.5
return (sum((x-ma)*(y-mb) for x,y in zip(a,b))/den) if den else float('nan')
ok_rows=[r for r in rows if r["ok"]=="YES"]
with open(out,"w") as f:
f.write("# Full-harness benchmark — campaign (5× per variant)\n\n")
f.write("# Full-harness benchmark — campaign\n\n")
f.write(f"Real `agents.py up` Builder/Adversary loop pair + watchdog through the 3-phase calculator "
f"to SEQUENCE-COMPLETE, **{reps}× per variant**. Both loops on **{model}**. Engine `{eng}`. "
f"Tokens summed from each loop's session transcript.\n\n")
f.write("## Per-variant total tokens (across repeats)\n\n")
f.write("| variant | runs | success | median | mean | min | max | stdev | spread (max/min) |\n")
f"Tokens summed from each loop's session transcript; commits = work-repo commit count; "
f"LOC = non-blank lines in `calc/*.py` (code + tests). Stats over SUCCESSFUL runs.\n\n")
f.write("## Per-variant — total tokens (successful runs)\n\n")
f.write("| variant | runs(ok) | median | mean | min | max | stdev | spread |\n")
f.write("|---|:--:|--:|--:|--:|--:|--:|--:|\n")
for v in variants:
tots=[r["total"] for r in ok_rows if r["v"]==v]
n=sum(1 for r in rows if r["v"]==v); s=stats(tots)
if not s: f.write(f"| {v} | 0/{n} | — | — | — | — | — | — |\n"); continue
spread=f"{s['max']/s['min']:.2f}x" if s['min'] else "—"
f.write(f"| {v} | {s['n']}/{n} | {fmt(s['median'])} | {fmt(s['mean'])} | {fmt(s['min'])} | "
f"{fmt(s['max'])} | {fmt(s['stdev'])} | {spread} |\n")
f.write("\n## Per-variant — medians (commits / LOC / duration)\n\n")
f.write("| variant | median commits | median LOC | median dur(s) |\n|---|--:|--:|--:|\n")
for v in variants:
sub=[r for r in ok_rows if r["v"]==v]
if not sub: f.write(f"| {v} | — | — | — |\n"); continue
f.write(f"| {v} | {int(st.median([r['commits'] for r in sub]))} | "
f"{int(st.median([r['loc'] for r in sub]))} | {int(st.median([r['dur'] for r in sub]))} |\n")
f.write("\n## Correlations with total tokens (pooled over all successful runs)\n\n")
f.write(f"n = {len(ok_rows)} successful runs.\n\n| tokens vs | Pearson r |\n|---|--:|\n")
for k,lab in [("dur","duration"),("commits","commits"),("loc","LOC (code+tests)")]:
f.write(f"| {lab} | {pearson([r['total'] for r in ok_rows],[r[k] for r in ok_rows]):+.2f} |\n")
f.write("\n## All runs (raw)\n\n")
f.write("| variant | rep | ok | builder | adversary | total | dur(s) | commits | LOC |\n")
f.write("|---|:--:|:--:|--:|--:|--:|--:|--:|--:|\n")
for v in variants:
tots=[r[5] for r in rows if r[0]==v]
oks=sum(1 for r in rows if r[0]==v and r[2]=="YES")
s=stats(tots)
spread=f"{s['max']/s['min']:.2f}x" if s and s['min'] else "—"
f.write(f"| {v} | {s['n']} | {oks}/{s['n']} | {fmt(s['median'])} | {fmt(s['mean'])} | "
f"{fmt(s['min'])} | {fmt(s['max'])} | {fmt(s['stdev'])} | {spread} |\n")
f.write("\n## Median duration\n\n| variant | median dur (s) |\n|---|--:|\n")
for v in variants:
durs=[r[6] for r in rows if r[0]==v]
f.write(f"| {v} | {int(st.median(durs)) if durs else '—'} |\n")
f.write("\n## All runs (raw)\n\n| variant | rep | success | builder | adversary | total | dur(s) |\n")
f.write("|---|:--:|:--:|--:|--:|--:|--:|\n")
for r in rows:
f.write(f"| {r[0]} | {r[1]} | {r[2]} | {fmt(r[3])} | {fmt(r[4])} | {fmt(r[5])} | {r[6]} |\n")
f.write(f"\n_Run root: `{runroot}`. Raw data: `RESULTS-campaign.md.data`._\n")
f.write(f"| {r['v']} | {r['rep']} | {r['ok']} | {fmt(r['builder'])} | {fmt(r['adversary'])} | "
f"{fmt(r['total'])} | {r['dur']} | {r['commits']} | {r['loc']} |\n")
f.write(f"\n_Run root (repos kept here for analysis): `{runroot}`. Raw data: `RESULTS-campaign.md.data`._\n")
print("wrote", out)
PY