From 4cdda1ccc00ac191fad4cdc5b6553e47bef5e29a Mon Sep 17 00:00:00 2001 From: ciregenz Date: Fri, 14 Aug 2026 12:57:30 -0700 Subject: [PATCH] arena: v33 verdict (fastpath is the reverse-task killer) + v34 inversion gate (pre-registered) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ --- e2e/browser-v3/arena/ARENA.md | 14 ++++++++++++++ e2e/browser-v3/arena/llm_policy.py | 15 +++++++++++++++ 2 files changed, 29 insertions(+) diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index be1b1dc9..da7938dd 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -407,6 +407,20 @@ history. Targets: exactly the 12 tasks they win and we lose (v33_pilot.json) + 1 controls. Prediction: >=5 of 12 targets (the 7 reverse-order ones are the core candidates), controls 12/12. Pilot runs only after the MiniWoB regression frees the LLM lane. +VERDICT v33 (2026-08-14): **fail — targets 1/12, controls 12/12.** Deferral never fires on +reverse tasks because nothing ERRORS: the fastpath instantly clicks the goal's first-QUOTED +target, which under 'X, after doing Y' grammar is the last-executed step — 1-step terminal +losses, no model call involved. The model, when consulted, parses the inversion correctly +(it opened the dialog case in the right order unaided). Mechanism withdrawn. + +## PRE-REGISTERED (2026-08-14, before any v34 episode): fastpath inversion gate + +Mechanism: v34 = v32 + one gate — fastpath stands down when the goal contains subordinate +order conjunctions (after/before/once), sending those sentences to the model. Linguistic +feature-trigger, no task names, no new scaffolding; normal goals keep the scripted speed path +(unit-checked both ways). Prediction: >=3 of the 7 reverse-order targets flip (the two 1-step +losses at minimum), controls 12/12 (no control goal contains the conjunctions). + ## Benchmark roadmap (2026 landscape survey, method-filtered) Rules: third-party scoring, reproducible from a committed artifact, no LLM-judge (or deterministic diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 45d2d2d6..b6f0d5f7 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -424,6 +424,12 @@ class OpenSwarmLlmPolicy(LlmPolicy): targets = [t for t in quoted(goal) if t and t not in self.fastpath_used] if not targets: return "" + # v34: subordinate order clauses ('X, after doing Y' / 'before' / 'once') make stated + # order != execution order. A scripted first-quoted click is then a coin flip at best and + # an instant terminal loss at worst (measured: 1-step losses on every reverse variant). + # The model parses these sentences correctly when consulted -- so consult it. + if re_.search(r"\b(after|before|once)\b", goal, re_.I): + return "" # ORDER IS PART OF THE TASK: composed goals ("click Ok, then the link") enforce sequence, # and skipping to a later quoted target because the first is ambiguous clicked things out # of order -- an instant, unrecoverable loss (measured on ordered pairs). The fastpath may @@ -924,6 +930,15 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: scripted_drag=True, auto_complete=True, som=False, native_pickers=True, verify_terminal=True, post_mouse_vision=True, multi_cap=6, fill_verify=True, **v17) + if name == "osw-llm-v34": # v32 + fastpath inversion gate (subordinate-clause goals go to the model) + v34 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, + max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + local_ctx=True, blocker_probe=True, suppress_wrappers=True, + force_unblock=True, **v34) if name == "osw-llm-v33": # v32 + opportunistic ordering (deferral doctrine + defer-nudge) v33 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30 + OSW_SYSTEM_V33, max_tokens=800)