diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index e98223ce..40ee52b5 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -36,6 +36,21 @@ Forms is the one honest deficit: book-flight's autocomplete flow, order-food, an social-media multi-item flows reward their 44s of patience. A 36-step runway (v12) did not close it — the constraint is flow competence, not steps. + +## Multi-model ledger (same arm, same tasks, whole-stack pairs; MiniWoB-scored) + +| model | ours (v14) | real browser-use | verdict | +|---|---|---|---| +| haiku-4-5 | 71.2% @ 4.8s, 0 false (v10: 75.2% @ 5.2s) | 69.6% @ 44.5s, 16 false | ours leads all axes | +| sonnet-4-6 | **77.6% @ 6.5s, 0 false** | 74.4% @ 37.4s, 8 false | ours leads all axes | +| sonnet-5 | 76.0% @ 5.5s, 0 false | sweeping | model tier plateaued | +| opus-5 | sweeping | — | — | + +The plateau at 76-78% across sonnet-4-6/sonnet-5 plus the 82.4% technique-union ceiling localizes +the remaining gap: ~22 tasks need purpose-built widget primitives (date/time pickers, precise +canvas geometry, long autocomplete flows), not a stronger model. Their false-claim rate persists +across every model (16 haiku, 8 sonnet-4-6) -- structural to the JS-evaluate hatch, as predicted. + ## The full ladder — every version, every technique, its measured worth | ver | change (source) | rate | med win | diff --git a/e2e/browser-v3/arena/bu_real.py b/e2e/browser-v3/arena/bu_real.py index a74c5478..0e60991d 100644 --- a/e2e/browser-v3/arena/bu_real.py +++ b/e2e/browser-v3/arena/bu_real.py @@ -20,6 +20,7 @@ from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent)) +import browsergym.assistantbench # noqa: F401 registers the live-web research envs import browsergym.miniwob # noqa: F401 registers the 125 envs import gymnasium as gym @@ -94,7 +95,8 @@ def reap_leftover_browsers() -> None: def make_env(task: str, seed: int, max_steps: int): patch_launch_with_cdp_port() - env = gym.make(f"browsergym/miniwob.{task}", headless=True, max_episode_steps=max_steps) + env_id = f"browsergym/{task}" if "." in task else f"browsergym/miniwob.{task}" + env = gym.make(env_id, headless=True, max_episode_steps=max_steps) obs, _ = env.reset(seed=seed) return env, obs diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 9f0396f9..abbd52ac 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -210,8 +210,10 @@ class OpenSwarmLlmPolicy(LlmPolicy): if not m: return call fn, argstr = m.group(1), m.group(2) - # scroll deltas and mouse_*/keyboard_* coordinates are geometry, never element handles. - if fn == "scroll" or fn.startswith(("mouse_", "keyboard_")): + # scroll deltas, mouse_*/keyboard_* coordinates, navigation URLs and chat answers carry no + # element handles -- pass through untouched. + if fn == "scroll" or fn.startswith(("mouse_", "keyboard_", "go")) or fn in ( + "send_msg_to_user", "report_infeasible"): return call # click(93, 234) is a coordinate click the model spelled wrong; book-flight looped four # turns on 'expected a string' before this rewrite existed. @@ -391,11 +393,15 @@ coordinates from what you see.""" OSW_SYSTEM_V8 = OSW_SYSTEM_V7 + """ When the goal names an exact target in quotes: click ONLY an element whose name matches it EXACTLY. If no exact match is visible yet, do not settle for a similar one -- open the next unexplored tab or -section (track which you have tried in your PLAN line) until the exact name appears.""" +section (track which you have tried in your PLAN line) until the exact name appears. +On the open web you may also navigate: goto("url") | go_back() | go_forward(). +When the goal is a QUESTION, research it and deliver the answer with send_msg_to_user("answer") -- +the answer text alone, no prose around it.""" CALL_RE = re.compile( r"\b(click|dblclick|fill|clear|select_option|hover|focus|press|scroll|drag_and_drop|noop" - r"|mouse_click|mouse_dblclick|mouse_move|mouse_drag_and_drop|keyboard_type|keyboard_press)\s*\([^)]*\)") + r"|mouse_click|mouse_dblclick|mouse_move|mouse_drag_and_drop|keyboard_type|keyboard_press" + r"|goto|go_back|go_forward|send_msg_to_user|report_infeasible)\s*\([^)]*\)") def clean_action(raw: str) -> str: diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index e58c0e7e..e77240d2 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -19,6 +19,7 @@ from typing import Any sys.path.insert(0, str(Path(__file__).resolve().parent)) +import browsergym.assistantbench # noqa: F401 registers the 215 live-web research envs import browsergym.miniwob # noqa: F401 importing is what registers the 125 envs import gymnasium as gym @@ -86,9 +87,12 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na # multiaction on: a form turn is fill+fill+click in ONE model call -- fewer calls is # simultaneously faster and stronger on multi-step tasks (browser-use does the same). - acts = HighLevelActionSet(subsets=["chat", "bid", "coord", "infeas"], + acts = HighLevelActionSet(subsets=["chat", "bid", "coord", "nav", "infeas"], strict=False, multiaction=True) - holder.append(gym.make(f"browsergym/miniwob.{task}", headless=not args.headed, + # A task name containing '.' is a full BrowserGym suffix (assistantbench.validation.3); + # bare names stay MiniWoB. One grader per suite, none of them ours. + env_id = f"browsergym/{task}" if "." in task else f"browsergym/miniwob.{task}" + holder.append(gym.make(env_id, headless=not args.headed, max_episode_steps=args.max_steps, wait_for_user_message=False, action_mapping=acts.to_python_code)) return holder[0].reset(seed=seed) diff --git a/e2e/browser-v3/arena/tasks.py b/e2e/browser-v3/arena/tasks.py index 1e9fa4d5..80531db6 100644 --- a/e2e/browser-v3/arena/tasks.py +++ b/e2e/browser-v3/arena/tasks.py @@ -74,16 +74,19 @@ SMOKE: list[str] = [ def resolve_tasks(spec: str) -> list[str]: - """Accept 'all', 'smoke', a category name, or a comma-separated list of task names.""" + """Accept 'all', 'smoke', 'abench', a category name, or a comma-separated list of task names.""" spec = spec.strip() if spec == "all": return ALL if spec == "smoke": return SMOKE + # AssistantBench validation split: live-web research questions, scored by their question_scorer. + if spec == "abench": + return [f"assistantbench.validation.{i}" for i in range(33)] if spec in CATEGORIES: return sorted(CATEGORIES[spec]) names = [s.strip() for s in spec.split(",") if s.strip()] - unknown = [n for n in names if n not in CATEGORY_OF] + unknown = [n for n in names if n not in CATEGORY_OF and "." not in n] if unknown: raise SystemExit(f"unknown task(s): {', '.join(unknown)}") return names