diff --git a/scripts/analyze-browser-metrics.py b/scripts/analyze-browser-metrics.py index b0b425b4..6ab5a802 100755 --- a/scripts/analyze-browser-metrics.py +++ b/scripts/analyze-browser-metrics.py @@ -13,6 +13,7 @@ Defaults to $OPENSWARM_BROWSER_METRICS_DIR, else import json import os +import statistics import sys from collections import Counter, defaultdict @@ -170,6 +171,33 @@ def playbook_report(tasks): print(f" {h[:40]:40} cold avg {cold:.1f} turns -> seeded avg {seeded:.1f} turns {verdict}") +def latency_report(tasks): + """Where a run's time actually goes, and which part a code change can move. + + Live-site wall clocks swing 5-12x on model turn count alone, so a wall-clock A/B needs ~30 runs + per arm to say anything at all. Split it: llm_ms is the model's, tools_ms is the browser's, and + the remainder is OURS (perception, waits, dispatch, bookkeeping). The spread column is the point. + If wall swings 8x while ours swings 1.4x, then ours is the only column worth optimising against, + and any speed claim measured on the wall was measuring the model's mood. + """ + runs = [t for t in tasks + if t.get("completed") and t.get("llm_ms") and t.get("path") in ("llm", "llm_fallback")] + print("\n=== WHERE THE TIME GOES (completed model-path runs) ===") + if len(runs) < 2: + print(f" {len(runs)} run(s) carry a latency split; need at least 2. " + "Runs recorded before this was persisted have no split and are skipped.") + return + cols = [("wall", "total_ms"), ("llm", "llm_ms"), ("browser", "tools_ms"), ("OURS", "other_ms")] + print(f" {'part':<9}{'median':>9}{'min':>9}{'max':>9}{'spread':>9} share") + med_total = statistics.median([r["total_ms"] for r in runs]) or 1 + for label, key in cols: + vals = sorted(r.get(key, 0) for r in runs) + med, lo, hi = statistics.median(vals), vals[0], vals[-1] + spread = f"{hi / lo:.1f}x" if lo > 0 else "n/a" + print(f" {label:<9}{round(med):>8}ms{lo:>8}ms{hi:>8}ms{spread:>9} {round(100 * med / med_total):>3}%") + print(f" n={len(runs)} runs. A change to our code can only move the OURS row; judge it there.") + + def main(): d = sys.argv[1] if len(sys.argv) > 1 else _default_dir() events = load(os.path.join(d, "events.jsonl")) @@ -235,6 +263,7 @@ def main(): print(f"honest completion rate: {round(100*(completed-ghosts)/n,1)}% " f"(completed minus ghosts)") + latency_report(tasks) skill_layer_report(tasks, skill_events) playbook_report(tasks)