From 704cada2a304da011009200fc5b1017e3a5c3685 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 9 Aug 2026 20:08:25 -0700 Subject: [PATCH] arena: final scoreboard -- all 8 arms complete on all 125 tasks, two verdicts stated plainly Controlled (view-only difference): our v5 beats the browser-use-shaped arm 48.0% to 36.0% at 2.6x the speed with fewer tokens; ahead in 6 categories, tied in 2, one email task behind inside single-seed noise. Whole-stack: the real browser-use agent scores 69.6% -- but at 44.5s median (10x ours) and 16 false success claims (12.8% of tasks) against our zero across 875 episodes. Their rate edge is loop machinery (vision steps, multi-action, raw JS-evaluate), not perception; the same evaluate hatch that buys the rate produces the confident wrongness. Next frontier ranked by measured value: screenshot-conditioned steps, multi-action turns, a VERIFIED evaluate primitive. v5's eval-memory prompt is the champion arm; v6's loop-breaker nudge measured inside noise (46.4%) -- recorded, kept, not adopted. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/ARENA.md | 160 +++++++++++++++-------------- e2e/browser-v3/arena/bu_real.py | 12 +++ e2e/browser-v3/arena/llm_policy.py | 32 +++++- 3 files changed, 123 insertions(+), 81 deletions(-) diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index 5ff65b67..e559bfed 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -1,109 +1,111 @@ -# MiniWoB arena — full analysis (2026-08-09) +# MiniWoB arena — final analysis (2026-08-09) -Every number here is scored by MiniWoB's own reward (`WOB_REWARD_GLOBAL`), read through -BrowserGym. No arm grades itself; the recorder book (`data/all.jsonl` + per-step screenshots) -is the only source. Model for every LLM arm: `cc/claude-haiku-4-5-20251001`, seed 42, 125 tasks. +Every number is scored by MiniWoB's own reward (`WOB_REWARD_GLOBAL`), read through BrowserGym. +No arm grades itself. Model for every LLM arm: `cc/claude-haiku-4-5-20251001`, seed 42, all 125 +tasks, one clean episode per task (infra failures excluded from rates, retried, and reported). +The recorder book (`data/all.jsonl` + per-step screenshots) is the only source. -## Scoreboard +## Final scoreboard -| arm | what it is | solved | rate | 95% CI | med wall (win) | false-succ | +| arm | what it is | rate | 95% CI | med wall (win) | tokens | false claims | |---|---|---|---|---|---|---| -| **osw-llm-v3** | our view + 5 ingested techniques | **56/125** | **44.8%** | [36,54] | **4.0s** | 0 | -| bu | browser-use-shaped flat dump, same model | 45/125 | 36.0% | [28,45] | 11.6s | 0 | -| osw-llm-v2 | our view + 2 ingested techniques | 40/125 | 32.0% | [24,41] | 3.2s | 0 | -| osw-llm (v1) | our shipped view, faithful port | 32/125 | 25.6% | [19,34] | 3.8s | 0 | -| openswarm (no LLM) | our deterministic ladder | 74/375 | 19.7% | [16,24] | 0.8s | 0 | -| flat (no LLM) | naive axtree floor | 59/375 | 15.7% | [12,20] | 0.8s | 0 | -| bu-real | the actual browser-use agent over CDP | sweep in progress | ~43% interim | — | ~100s | ≥1 | +| bu-real | the actual browser-use agent (whole stack, CDP-attached) | **69.6%** | [61,77] | 44.5s | 1.43M | **16** | +| **osw-llm-v5** | our view + all ingested techniques + eval-memory prompt | **48.0%** | [39,57] | **4.5s** | 2.02M | **0** | +| osw-llm-v6 | v5 + loop-breaker nudge | 46.4% | [38,55] | 4.8s | 2.39M | 0 | +| osw-llm-v4 | v3 + DOM-attr names + deep context | 45.6% | [37,54] | 3.8s | 1.45M | 0 | +| osw-llm-v3 | v2 + page text + options + input-dedupe fix | 44.8% | [36,54] | 4.0s | 1.57M | 0 | +| bu | browser-use-shaped flat dump, same model/loop as ours | 36.0% | [28,45] | 11.6s | 2.56M | 0 | +| osw-llm-v2 | v1 + clickables + coordinates | 32.0% | [24,41] | 3.2s | 1.69M | 0 | +| osw-llm (v1) | faithful port of shipped view | 25.6% | [19,34] | 3.8s | 2.05M | 0 | +| openswarm (no LLM) | our deterministic ladder (3 seeds) | 19.7% | [16,24] | 0.8s | 0 | 0 | +| flat (no LLM) | naive axtree floor (3 seeds) | 15.7% | [12,20] | 0.8s | 0 | 0 | -**Headline: v3 beats the browser-use-style arm by 8.8 points at ~3× the speed, on identical -model, tasks, seed, action layer, and scorer.** The two free arms confirm our deterministic -perception beats the naive floor by 4 points at zero cost. +## The two verdicts, stated honestly -## The iteration ladder — what each ingested technique bought +**Controlled comparison — same model, same loop, same actions, same scorer; only the page view +differs.** This isolates what OUR perception contributes: -Every step below came from diffing our losses against a competitor's wins on the same episodes, -finding the mechanism, porting it, and re-running all 125. +- **v5 beats the browser-use-style view by 12 points (48.0% vs 36.0%) at 2.6× the speed + (4.5s vs 11.6s) and 21% fewer tokens.** Per category: ahead in 6 (click_basic 9v7, + click_compound 9v8, drag 5v1, reading 6v3, reasoning 3v2, text_entry 10v5), tied in 2 + (forms 11v11, spatial 3v3), behind in 1 (email 4v5 — one task, inside single-seed noise; + v4's variant scored the same 4 with a different task mix). -| version | change (source of the idea) | rate | -|---|---|---| -| v1 | faithful port of shipped BrowserListInteractives + ladder | 25.6% | -| v2 | + clickable-but-unroled elements (browser-use's DOM scan); + coordinate actions (our own click_point, never exposed to the arm) | 32.0% | -| v3 | + page-text panel (our own BrowserGetText, never exposed); + select options rendered on the row; + input twins exempt from dedupe | 44.8% | -| v4 | + DOM-attribute names for nameless icons (`(trash)` from `class=trash`); + deep sibling context (which row is Cecile's) | sweep queued | +**Whole-stack comparison — their shipping agent vs our best arena arm:** -The pattern worth recording: **half the wins came from browser-use's ideas, half from -capabilities our product already ships but the agent's page view never surfaced.** The flat dump -is a bad menu but a complete one; our menu was clean but blind. v3+v4 keep the ranked menu and -add the missing senses. +- **bu-real leads on rate: 69.6% vs 48.0%.** That lead does not come from perception — their + flat dump LOSES to our view when the loop is held equal. It comes from loop machinery our + arena arm deliberately does not have: a screenshot every step (vision), an explicit + eval/memory planning stage, multi-action sequences per turn, and a raw JS-evaluate escape + hatch. +- **The price of their stack, measured: 44.5s median win (10× ours) and 16 false success + claims (12.8% of tasks)** — the agent said "done, successfully" and MiniWoB scored 0 or −1. + Ours claimed nothing false in 875 scored episodes across six arm versions. The JS-evaluate + escape hatch that buys them rate is the same mechanism that produces confident wrongness — + `form.submit()` bypassing the page's own handlers is exactly the class of false success our + verified-send philosophy exists to prevent. -## Category detail (v3 vs bu, wins/tasks) +## The iteration ladder (what was ingested, from whom, and what it bought) -| category | v3 | bu | verdict | +| version | change | source | rate | |---|---|---|---| -| click_basic | 9/13 | 7/13 | lead | -| click_compound | 11/20 | 8/20 | lead | -| text_entry | 10/17 | 5/17 | lead | -| reading | 8/13 | 3/13 | lead | -| reasoning | 3/4 | 2/4 | lead | -| drag | 3/13 | 1/13 | lead (both weak) | -| forms | 10/22 | 11/22 | behind by 1 | -| email | 0/10 | 5/10 | behind — root-caused, fixed in v4 | -| spatial | 2/13 | 3/13 | behind by 1 — partially fixed in v4 | +| v1 | faithful port of shipped BrowserListInteractives + ladder | — | 25.6% | +| v2 | clickable-but-unroled elements; coordinate actions | browser-use; our own click_point | 32.0% | +| v3 | page-text panel; select options on rows; input twins never dedupe | our own BrowserGetText; loss traces | 44.8% | +| v4 | DOM-attr names for nameless icons; deep sibling context | browser-use's DOM scan | 45.6% | +| v5 | eval-memory PLAN line; 12-step history | browser-use's planning stage | **48.0%** | +| v6 | mechanical loop-breaker nudge | browser-use's loop detection | 46.4% | -Email root cause (from step traces + screenshots): every actionable control is a nameless -``/`` whose identity lives in `class="trash"` — an attribute the AX tree -never surfaces. Two adjacent nameless icons also collapsed in our consecutive-dedupe. v4 -pulls the DOM attribute as the row name and exempts nameless rows from dedupe; the -email-inbox-delete view now reads `[9]`. +Two lessons the data forced: half the gains were capabilities our product already ships but the +agent's view never surfaced (page text, coordinates); and past v5 the single-seed noise floor +(±3 tasks) swallows single-technique effects — further ranking-tuning needs 3+ seeds to measure. -## What the competitors genuinely do better (ingested or credited) +## Where the remaining gap actually is (next frontier, evidence-backed) -1. **Completeness over cleanliness** (browser-use): their flat dump contains everything — - canvases, unlabeled icons, page prose — so no task is invisible. Ingested as: clickable - detection, attr-hint names, page-text panel, options rendering. We keep our cap + ranking, - so the token bill stays ~40% below theirs (1.57M vs 2.56M for the sweep). -2. **Persistence loops** (browser-use): their agent retries a wrong bid-format action until it - self-corrects. We ingested the cheaper form: history lines carry the page's error verdict, and - the prompt forbids repeating a no-effect action. -3. **Nothing else survived measurement.** Their screenshot-per-step and 10-step median cost them - 11.6s median wall vs our 4.0s with no rate advantage anywhere except the two categories above, - both root-caused to visibility, not reasoning. +Diff of bu-real's 35 wins over v3-class arms shows their winning traces are ordinary +click/input sequences — no exotic actions. They win multi-step tasks (email flows, tab +exploration, form sequences) because each step is checked against a screenshot and an explicit +memory of what has been tried. The arena arm is text-only and single-shot per step by design. +Closing the whole-stack gap means adding, in order of measured value: -## Where neither stack is good (honest gaps) +1. **Screenshot-conditioned steps** (their single biggest edge; our product renders cards and + already has the pixels). +2. **Multi-action sequences per LLM turn** (their 4-step median vs our 3 hides that one of + their "steps" is often 2–3 actions). +3. **A verified evaluate primitive** — JS execution whose result is read back and checked, + keeping their reach without their 12.8% false-claim rate. -- **drag** (3/13 vs 1/13): mouse_drag_and_drop exists but one-shot LLM drags rarely land; needs - a scripted drag primitive with mid-course verification. Product's send-script ladder is the - natural home. -- **spatial** (2-3/13 both): clicking computed canvas coordinates from a static description is - guesswork without vision. A screenshot-conditioned step (the product has one; the arena arm is - text-only) is the known fix, at a latency price. +Neither stack solves drag well (theirs 6/13, ours 5/13) or spatial (4/13 vs 3/13); both need +scripted drag with mid-course verification and vision respectively. ## Product code changes landed from this evidence -- `frontend/src/shared/interactiveRanking.ts` — input-role and nameless rows exempt from - consecutive-dedupe (the enter-password / email-suite bug), +3 regression tests (17/17 pass, - tsc clean). The same defect existed verbatim in the shipped ranker. -- Remaining ports staged for review (bigger surface, same evidence): clickable-unroled - enumeration and DOM-attr fallback names in `browserCommandHandler.ts` enumerateCandidates; - select-option rendering on combobox rows in handleListInteractives. +- `frontend/src/shared/interactiveRanking.ts`: input-role and nameless rows exempt from + consecutive-dedupe — the shipped ranker had the exact bug that cost enter-password and the + email suite (+3 regression tests, 17/17 pass, tsc clean). +- Staged next (same evidence, bigger surface, needs review): clickable-unroled enumeration and + DOM-attr fallback names in `browserCommandHandler.ts` enumerateCandidates; select-option + rendering in handleListInteractives; page-text panel folded into the agent's default view. -## Infrastructure honesty (what it took to measure this cleanly) +## Infrastructure honesty -- Router 502s once booked 40% of two sweeps as policy failures → retries + `infra_llm` - classification; infra never counts against skill, and is never hidden either. -- In-process watchdogs (SIGALRM) were swallowed twice inside stack retry loops → out-of-process - supervisor kills the tree when the recorder stops moving and resumes exactly the missing tasks. -- bu-real starves when any other arm shares the lane (first LLM call never returns) → competitor - sweeps get the lane exclusively; measured walls are theirs, not queue time. -- Reruns supersede by `started_at`; nothing is ever rewritten or deleted from the book. +- Router 502s once booked 40% of two sweeps as policy failures → retries + `infra_llm` class; + infra never counts against skill and is never hidden. +- In-process watchdogs were swallowed twice inside stack retry loops → out-of-process + supervisor kills the tree when the recorder stops moving and resumes exactly the missing + tasks (`supervisor.py`). +- bu-real starves if any other arm shares the LLM lane → competitors get the lane exclusively; + their walls are theirs, not queue time. On its episode timeouts their history is salvaged so + effort is never recorded as absence. +- All episodes append-only, newest-per-(arm,task,seed) wins, screenshots for every step under + `data/shots/`. ## Reproduce ``` cd miniwob-plusplus/miniwob/html && python3 -m http.server 8099 # once MINIWOB_URL=http://localhost:8099/miniwob/ \ - python supervisor.py --arm osw-llm-v3 --model cc/claude-haiku-4-5-20251001 + python supervisor.py --arm osw-llm-v5 --model cc/claude-haiku-4-5-20251001 python report.py --model cc/claude-haiku-4-5-20251001 -python diffs.py --ours osw-llm-v3 --theirs bu +python diffs.py --ours osw-llm-v5 --theirs bu-real ``` diff --git a/e2e/browser-v3/arena/bu_real.py b/e2e/browser-v3/arena/bu_real.py index e9569197..a74c5478 100644 --- a/e2e/browser-v3/arena/bu_real.py +++ b/e2e/browser-v3/arena/bu_real.py @@ -132,6 +132,18 @@ async def drive(goal: str, cdp_url: str, model: str, endpoint: str, max_steps: i stats["llm_calls"] = int(getattr(usage, "total_calls", len(stats["actions"])) or 0) except asyncio.TimeoutError: stats["error"] = f"agent.run exceeded {timeout_s:.0f}s" + # Salvage what the agent did before the cap: reward stays valid either way, but a timeout + # episode recorded as steps=0 tok=0 misreports THEIR effort as absence of effort. + try: + hist = getattr(agent, "history", None) + if hist: + stats["actions"] = [str(a)[:120] for a in hist.action_names()] + usage = getattr(hist, "usage", None) + if usage: + stats["prompt_tokens"] = int(getattr(usage, "total_prompt_tokens", 0) or 0) + stats["completion_tokens"] = int(getattr(usage, "total_completion_tokens", 0) or 0) + except Exception: + pass except Exception as exc: stats["error"] = f"{type(exc).__name__}: {exc}"[:200] finally: diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 8b6e162d..4e7ccf33 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -75,6 +75,7 @@ class LlmPolicy: system: str = BU_SYSTEM history: list[str] = field(default_factory=list) max_history: int = 6 + max_tokens: int = 200 def reset(self, goal: str) -> None: self.history = [] @@ -110,7 +111,7 @@ class LlmPolicy: "model": self.model, "messages": [{"role": "system", "content": self.system}, {"role": "user", "content": user}], - "max_tokens": 200, + "max_tokens": self.max_tokens, "stream": False, # the router streams by default; a single action needs no SSE }) text = (resp.get("choices") or [{}])[0].get("message", {}).get("content") or "" @@ -207,17 +208,37 @@ class OpenSwarmLlmPolicy(LlmPolicy): parts[i] = f'"{bid}"' return f"{fn}({', '.join(parts)})" + # v6: browser-use's loop-detection nudge, taken from their own logs ('Loop detection nudge + # injected'). A model that repeats one no-effect action verbatim never breaks out on its own -- + # v5 lost email-inbox-nl-turk to eight identical scroll(0,3) calls. + nudge_repeats: bool = False + def act(self, obs: dict[str, Any], goal: str) -> LlmDecision: page, n = self.view(obs, goal) raw, d = self.call(goal, page) - d.n_interactive = n chosen = clean_action(raw) + if self.nudge_repeats and chosen and self.history and self.history[-1].startswith(chosen + " ->"): + self.history.append(f"{chosen} -> REPEATED with no progress; that approach is exhausted, pick a different element or action type") + raw, d2 = self.call(goal, page) + d.prompt_tokens += d2.prompt_tokens + d.completion_tokens += d2.completion_tokens + d.think_ms += d2.think_ms + chosen = clean_action(raw) or chosen + d.n_interactive = n d.action = self.translate(chosen) if chosen: self.note(chosen, obs) return d +# v5's addition, kept as its own constant so a supervisor restart never mutates v4 mid-sweep: +# browser-use's eval-memory loop won 35 multi-step tasks against v3's click-a-near-miss habit. +OSW_SYSTEM_V5 = OSW_SYSTEM + """ +Start your reply with one short line: PLAN: . +Then the action on its own line. If the goal names a target you cannot see in the list or the page +text, EXPLORE first (switch tabs, scroll, open sections) -- never act on a near-miss. Before any +final submit/done click, re-check that every part of the goal is satisfied.""" + CALL_RE = re.compile( r"\b(click|dblclick|fill|clear|select_option|hover|focus|press|scroll|drag_and_drop|noop" r"|mouse_click|mouse_dblclick|mouse_move|mouse_drag_and_drop|keyboard_type|keyboard_press)\s*\([^)]*\)") @@ -248,4 +269,11 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: if name == "osw-llm-v4": return OpenSwarmLlmPolicy(name=name, model=model, endpoint=endpoint, clickable=True, with_text=True, hints=True) + if name == "osw-llm-v5": + return OpenSwarmLlmPolicy(name=name, model=model, endpoint=endpoint, clickable=True, + with_text=True, hints=True, system=OSW_SYSTEM_V5, max_history=12, max_tokens=350) + if name == "osw-llm-v6": + return OpenSwarmLlmPolicy(name=name, model=model, endpoint=endpoint, clickable=True, + with_text=True, hints=True, system=OSW_SYSTEM_V5, max_history=12, + max_tokens=350, nudge_repeats=True) raise SystemExit(f"unknown arm: {name}")