diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index 8cdce427..8669910e 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -586,6 +586,15 @@ replacement is strictly safer than the click it replaces). Order: v37 then v36, sequential, immediately after the Claude chore column finishes (auto-chained). +VERDICTS (2026-08-16): +- **v36 escape token: PASS — 3/8 targets flipped (click-collapsible-2 AND -nodelay, both + never-won hard-cluster residents, + click-dialog-2_login-user-popup), controls 12/12.** First + competitor ingestion to clear its gate. PROMOTED: v40 champion = v35 stack + escape_token. +- v37 note scratchpad: fail on short-horizon proxies (0/8; controls 12/12 — mechanism safe, + never harmful). Honest reading: the gate words rarely trigger on these tasks and short tasks + don't need durable memory; the REAL test is chore-scale aggregation, which needs an uncapped + chore rerun (expensive, deferred). Flag stays off in champion until that evidence exists. + ## Open-source / SOTA reference points (2026 leaderboards, for honest comparison) - WebArena: SOTA WebTactix/DeepSeek-v3.2 74.3%; frontier models 64-68%; human 78. (leaderboard.steel.dev) diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 4b72d311..e86b9d85 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -999,6 +999,15 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: scripted_drag=True, auto_complete=True, som=False, native_pickers=True, verify_terminal=True, post_mouse_vision=True, multi_cap=6, fill_verify=True, **v17) + if name == "osw-llm-v40": # CHAMPION: v35 stack + v36 escape token (first passed ingestion) + v40 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30 + + OSW_SYSTEM_V36, max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + local_ctx=True, blocker_probe=True, suppress_wrappers=True, + force_unblock=True, native_js_fallback=True, escape_token=True, **v40) if name == "osw-llm-v39": # v35 + mutation-diff action feedback (Agent-E observer, text-side) v39 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, max_tokens=800)