From 5b93ca2e8408546438c3d9e6a2be977c2f4666e7 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Mon, 10 Aug 2026 02:54:29 -0700 Subject: [PATCH] arena: v8 ties v7's 56.8 with a different shape -- drag and spatial now BEAT the real browser-use Scripted exact-match fastpath + exact-name guard moved the wins where near-miss clicks were terminal: drag 8/13 vs their 6/13, spatial 7/13 vs their 4/13, at 4.8s and zero false claims. The remaining systematic gap is runway, not ability: 21 of 54 losses die exactly at the 12-step cap, all in the email/forms/long-flow cluster their 44s median happily outlasts. v9 sweeps with a 24-step cap (wins end at median 3 steps, so the win path pays nothing) plus progressive vision -- any episode that burns 6 actions gets a screenshot every turn thereafter. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/llm_policy.py | 11 +++++++++-- e2e/browser-v3/arena/supervisor.py | 4 +++- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 0150ea70..06800476 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -270,8 +270,12 @@ class OpenSwarmLlmPolicy(LlmPolicy): return d last_err = bool(str(obs.get("last_action_error") or "").strip()) image = "" - if self.vision == "always" or (self.vision == "adaptive" - and self.stuck_or_spatial(page, goal, last_err)): + # progressive: adaptive triggers PLUS any task that has already burned 6 actions gets eyes + # every turn -- a dragging episode is by definition one the text view is not solving. + struggling = self.vision == "progressive" and len(self.history) >= 6 + if (self.vision == "always" + or (self.vision in ("adaptive", "progressive") + and (self.stuck_or_spatial(page, goal, last_err) or struggling))): image = encode_screenshot(obs) raw, d = self.call(goal, page, image_b64=image) d.vision = 1 if image else 0 @@ -398,4 +402,7 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: if name == "osw-llm-v8": v8 = dict(v7, system=OSW_SYSTEM_V8) return OpenSwarmLlmPolicy(name=name, multi=True, vision="adaptive", fastpath=True, **v8) + if name == "osw-llm-v9": # v8 + progressive vision; run with --max-steps 24 (21 losses were step-capped) + v9 = dict(v7, system=OSW_SYSTEM_V8) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, **v9) raise SystemExit(f"unknown arm: {name}") diff --git a/e2e/browser-v3/arena/supervisor.py b/e2e/browser-v3/arena/supervisor.py index 08784757..d74fa6e2 100644 --- a/e2e/browser-v3/arena/supervisor.py +++ b/e2e/browser-v3/arena/supervisor.py @@ -54,7 +54,8 @@ def spawn(arm: str, tasks: list[str], args: argparse.Namespace) -> subprocess.Po "--episode-timeout", str(args.episode_timeout)] else: cmd = [PY, str(ARENA / "run.py"), "--arm", arm, "--tasks", ",".join(tasks), "--seeds", "1", - "--seed-base", str(args.seed), "--model", args.model, "--shots", args.shots] + "--seed-base", str(args.seed), "--model", args.model, "--shots", args.shots, + "--max-steps", str(args.max_steps)] env = dict(os.environ) env.setdefault("MINIWOB_URL", "http://localhost:8099/miniwob/") env.setdefault("BROWSER_USE_LOGGING_LEVEL", "warning") @@ -81,6 +82,7 @@ def main() -> None: ap.add_argument("--stall", type=float, default=240.0, help="kill if no new episode lands this long") ap.add_argument("--rounds", type=int, default=12) ap.add_argument("--episode-timeout", type=float, default=100.0) + ap.add_argument("--max-steps", type=int, default=12) ap.add_argument("--shots", default="first-last") ap.add_argument("--log", default="") # Resume window: episodes recorded after this epoch count as done, so a restarted supervisor