diff --git a/run-harness-bench.sh b/run-harness-bench.sh index 3b749ff..93373fc 100755 --- a/run-harness-bench.sh +++ b/run-harness-bench.sh @@ -163,17 +163,19 @@ run_one() { # read -r bi bo bcc bcr <<<"$(collect_tokens "$run/work")" read -r ai ao acc acr <<<"$(collect_tokens "$run/work-adv")" local btok=$((bi+bo+bcc+bcr)) atok=$((ai+ao+acc+acr)) total=$(( bi+bo+bcc+bcr + ai+ao+acc+acr )) + local commits loc + commits=$(git -C "$run/origin.git" rev-list --count main 2>/dev/null || echo 0) + loc=$(cat "$run/work-adv"/calc/*.py 2>/dev/null | grep -cve '^[[:space:]]*$') - # append one row immediately (survives a kill): variant rep success btok atok total dur - printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$v" "$rep" "$success" "$btok" "$atok" "$total" "$t" >>"$DATA" - log " -> $v r$rep: success=$success total=$total dur=${t}s" - # free disk: drop this run's transcripts + repo now that tokens are tallied - rm -rf "$run/work/.git" "$run/work-adv/.git" "$run/origin.git" "$run/seed" 2>/dev/null + # append one row immediately (survives a kill): variant rep success btok atok total dur commits loc + printf '%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\t%s\n' "$v" "$rep" "$success" "$btok" "$atok" "$total" "$t" "$commits" "$loc" >>"$DATA" + log " -> $v r$rep: success=$success total=$total dur=${t}s commits=$commits loc=$loc" + # NOTE: repos are KEPT under "$run" (work/, work-adv/, origin.git) for later analysis — not deleted. } # ---- collect ---- : > "$DATA" -printf 'variant\trep\tsuccess\tbuilder\tadversary\ttotal\tduration_s\n' > "$DATA.hdr" +printf 'variant\trep\tsuccess\tbuilder\tadversary\ttotal\tduration_s\tcommits\tloc\n' > "$DATA.hdr" for v in "${VARIANTS[@]}"; do for rep in $(seq 1 "$REPEATS"); do run_one "$v" "$rep"; done done @@ -185,41 +187,58 @@ data, out, model, reps, eng, runroot = sys.argv[1:7] rows=[] for line in open(data): p=line.rstrip("\n").split("\t") - if len(p)!=7: continue - v,rep,ok,b,a,tot,dur=p - rows.append((v,int(rep),ok,int(b),int(a),int(tot),int(dur))) + if len(p)!=9: continue + v,rep,ok,b,a,tot,dur,commits,loc=p + rows.append(dict(v=v, rep=int(rep), ok=ok, builder=int(b), adversary=int(a), + total=int(tot), dur=int(dur), commits=int(commits), loc=int(loc))) variants=[] for r in rows: - if r[0] not in variants: variants.append(r[0]) + if r["v"] not in variants: variants.append(r["v"]) def fmt(n): return f"{n:,}" def stats(xs): if not xs: return None return dict(n=len(xs), min=min(xs), max=max(xs), mean=int(st.mean(xs)), median=int(st.median(xs)), stdev=int(st.pstdev(xs)) if len(xs)>1 else 0) +def pearson(a,b): + if len(a)<2: return float('nan') + ma,mb=st.mean(a),st.mean(b) + den=(sum((x-ma)**2 for x in a)*sum((y-mb)**2 for y in b))**0.5 + return (sum((x-ma)*(y-mb) for x,y in zip(a,b))/den) if den else float('nan') +ok_rows=[r for r in rows if r["ok"]=="YES"] with open(out,"w") as f: - f.write("# Full-harness benchmark — campaign (5× per variant)\n\n") + f.write("# Full-harness benchmark — campaign\n\n") f.write(f"Real `agents.py up` Builder/Adversary loop pair + watchdog through the 3-phase calculator " f"to SEQUENCE-COMPLETE, **{reps}× per variant**. Both loops on **{model}**. Engine `{eng}`. " - f"Tokens summed from each loop's session transcript.\n\n") - f.write("## Per-variant total tokens (across repeats)\n\n") - f.write("| variant | runs | success | median | mean | min | max | stdev | spread (max/min) |\n") + f"Tokens summed from each loop's session transcript; commits = work-repo commit count; " + f"LOC = non-blank lines in `calc/*.py` (code + tests). Stats over SUCCESSFUL runs.\n\n") + f.write("## Per-variant — total tokens (successful runs)\n\n") + f.write("| variant | runs(ok) | median | mean | min | max | stdev | spread |\n") + f.write("|---|:--:|--:|--:|--:|--:|--:|--:|\n") + for v in variants: + tots=[r["total"] for r in ok_rows if r["v"]==v] + n=sum(1 for r in rows if r["v"]==v); s=stats(tots) + if not s: f.write(f"| {v} | 0/{n} | — | — | — | — | — | — |\n"); continue + spread=f"{s['max']/s['min']:.2f}x" if s['min'] else "—" + f.write(f"| {v} | {s['n']}/{n} | {fmt(s['median'])} | {fmt(s['mean'])} | {fmt(s['min'])} | " + f"{fmt(s['max'])} | {fmt(s['stdev'])} | {spread} |\n") + f.write("\n## Per-variant — medians (commits / LOC / duration)\n\n") + f.write("| variant | median commits | median LOC | median dur(s) |\n|---|--:|--:|--:|\n") + for v in variants: + sub=[r for r in ok_rows if r["v"]==v] + if not sub: f.write(f"| {v} | — | — | — |\n"); continue + f.write(f"| {v} | {int(st.median([r['commits'] for r in sub]))} | " + f"{int(st.median([r['loc'] for r in sub]))} | {int(st.median([r['dur'] for r in sub]))} |\n") + f.write("\n## Correlations with total tokens (pooled over all successful runs)\n\n") + f.write(f"n = {len(ok_rows)} successful runs.\n\n| tokens vs | Pearson r |\n|---|--:|\n") + for k,lab in [("dur","duration"),("commits","commits"),("loc","LOC (code+tests)")]: + f.write(f"| {lab} | {pearson([r['total'] for r in ok_rows],[r[k] for r in ok_rows]):+.2f} |\n") + f.write("\n## All runs (raw)\n\n") + f.write("| variant | rep | ok | builder | adversary | total | dur(s) | commits | LOC |\n") f.write("|---|:--:|:--:|--:|--:|--:|--:|--:|--:|\n") - for v in variants: - tots=[r[5] for r in rows if r[0]==v] - oks=sum(1 for r in rows if r[0]==v and r[2]=="YES") - s=stats(tots) - spread=f"{s['max']/s['min']:.2f}x" if s and s['min'] else "—" - f.write(f"| {v} | {s['n']} | {oks}/{s['n']} | {fmt(s['median'])} | {fmt(s['mean'])} | " - f"{fmt(s['min'])} | {fmt(s['max'])} | {fmt(s['stdev'])} | {spread} |\n") - f.write("\n## Median duration\n\n| variant | median dur (s) |\n|---|--:|\n") - for v in variants: - durs=[r[6] for r in rows if r[0]==v] - f.write(f"| {v} | {int(st.median(durs)) if durs else '—'} |\n") - f.write("\n## All runs (raw)\n\n| variant | rep | success | builder | adversary | total | dur(s) |\n") - f.write("|---|:--:|:--:|--:|--:|--:|--:|\n") for r in rows: - f.write(f"| {r[0]} | {r[1]} | {r[2]} | {fmt(r[3])} | {fmt(r[4])} | {fmt(r[5])} | {r[6]} |\n") - f.write(f"\n_Run root: `{runroot}`. Raw data: `RESULTS-campaign.md.data`._\n") + f.write(f"| {r['v']} | {r['rep']} | {r['ok']} | {fmt(r['builder'])} | {fmt(r['adversary'])} | " + f"{fmt(r['total'])} | {r['dur']} | {r['commits']} | {r['loc']} |\n") + f.write(f"\n_Run root (repos kept here for analysis): `{runroot}`. Raw data: `RESULTS-campaign.md.data`._\n") print("wrote", out) PY