From 76eada698727eaacded1802f6bc804815f4431a4 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 16 Aug 2026 02:33:55 -0700 Subject: [PATCH] arena: v40 union FAILS confirm (interaction regression) -- pairwise isolation running (v40c first) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/ARENA.md | 8 ++++++++ e2e/browser-v3/arena/llm_policy.py | 10 ++++++++++ 2 files changed, 18 insertions(+) diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index 90bed8e4..c9e9fdf7 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -619,6 +619,14 @@ champion re-registered as the UNION (escape_token + table_md + mutation_diff). P a combined CONFIRMATION pilot (union of the three target sets + 12 controls) runs before the champion 3-seed — passed-alone does not guarantee passed-together (interaction risk). +CONFIRM VERDICT (2026-08-16): **interaction regression — gate FAILS.** Targets 11/20 but +controls 10/12 (login-popup pair lost) and book-flight regressed vs BOTH individual pilots +(won under v38-alone and v39-alone, lost combined). The union does not compose. Isolation +running pairwise, cheapest discriminator first: v40c = table_md+mutation_diff (no escape) on +the same 32-task set — if book-flight loses there, the conflict is the two prompt-additive +mechanisms crowding each other; if it wins, escape_token is implicated. No promotion until a +composing set passes with controls 12/12. + ## Open-source / SOTA reference points (2026 leaderboards, for honest comparison) - WebArena: SOTA WebTactix/DeepSeek-v3.2 74.3%; frontier models 64-68%; human 78. (leaderboard.steel.dev) diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 0fdf6b03..756e791b 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -999,6 +999,16 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: scripted_drag=True, auto_complete=True, som=False, native_pickers=True, verify_terminal=True, post_mouse_vision=True, multi_cap=6, fill_verify=True, **v17) + if name == "osw-llm-v40c": # pair isolate: table_md + mutation_diff (no escape token) + c = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, + max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + local_ctx=True, blocker_probe=True, suppress_wrappers=True, + force_unblock=True, native_js_fallback=True, + table_md=True, mutation_diff=True, **c) if name == "osw-llm-v40": # CHAMPION: v35 stack + ALL passed ingestions (escape+table_md+mutation_diff) v40 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30 + OSW_SYSTEM_V36, max_tokens=800)