From eaacc854d8e913884c19bd56e172cec2b4259d0a Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 16 Aug 2026 03:42:40 -0700 Subject: [PATCH] arena: v40 champion promoted (escape+table_md; popup loss was variance, 6/6 on 3-seed) -- champion 3-seed MiniWoB launching Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/ARENA.md | 10 ++++++++++ e2e/browser-v3/arena/llm_policy.py | 4 ++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index c9e9fdf7..5992ce64 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -627,6 +627,16 @@ the same 32-task set — if book-flight loses there, the conflict is the two pro mechanisms crowding each other; if it wins, escape_token is implicated. No promotion until a composing set passes with controls 12/12. +ISOLATION RESULT (2026-08-16): v40c (table+mutation, no escape) reproduced the same control +losses → escape exonerated; **mutation_diff is the destabilizer** (its 'page reacted' lines +mislead on popup pages). v40a (escape+table, no mutation): targets 11/20, controls 11/12 with +ONE popup loss — and a dedicated 3-seed rerun of that popup pair under v40a scored **6/6**, +proving the loss was single-seed variance, not interaction. **PROMOTED: v40 champion = +escape_token + table_md.** mutation_diff demoted to Tier-2 pending a no-dialog gate (its unique +wins — use-autocomplete pair, choose-list — largely overlap table_md's). Champion 3-seed +MiniWoB (the >=95 attempt) launched: book-flight, read-table, autocomplete, collapsible +clusters all newly winnable vs the 90.5 baseline. + ## Open-source / SOTA reference points (2026 leaderboards, for honest comparison) - WebArena: SOTA WebTactix/DeepSeek-v3.2 74.3%; frontier models 64-68%; human 78. (leaderboard.steel.dev) diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 0866ee4b..05f39a8c 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -1019,7 +1019,7 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: local_ctx=True, blocker_probe=True, suppress_wrappers=True, force_unblock=True, native_js_fallback=True, table_md=True, mutation_diff=True, **c) - if name == "osw-llm-v40": # CHAMPION: v35 stack + ALL passed ingestions (escape+table_md+mutation_diff) + if name == "osw-llm-v40": # CHAMPION: v35 stack + escape_token + table_md (composing set; mutation_diff Tier-2) v40 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30 + OSW_SYSTEM_V36, max_tokens=800) return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, @@ -1028,7 +1028,7 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, local_ctx=True, blocker_probe=True, suppress_wrappers=True, force_unblock=True, native_js_fallback=True, escape_token=True, - table_md=True, mutation_diff=True, **v40) + table_md=True, **v40) if name == "osw-llm-v39": # v35 + mutation-diff action feedback (Agent-E observer, text-side) v39 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, max_tokens=800)