diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index 5826fd78..c922526c 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -348,6 +348,24 @@ DIV.ui-dialog-titlebar). Also fixed alongside: empty LLM completions (thinking e max_tokens) are now retried with doubled budget instead of being booked as no-action turns — this was the entire login-composition 0-step failure mode. Prediction unchanged; pilot rerun. +RERUN VERDICT (2026-08-14): **fail on targets (0/8 pattern continuing; controls clean).** The +BLOCKED message now demonstrably reaches the model and changes behavior — the trace shows it +attempting to move the dialog (dragging the Close row: the titlebar itself is not an +addressable element) and then resuming the clause sequence — but it never RETRIES the blocked +clause-1 click, so the composition still scores 0. Model-side remediation of occlusion is now +0-for-2 (instruction alone, instruction + named blocker). + +## PRE-REGISTERED (2026-08-14, before any v32 episode): forced dispatch on blocked clicks + +Mechanism: `force_unblock` (v32 = v31 + this) — when a click is BLOCKED, run.py dispatches a +synthetic el.click() on the target immediately and the error says so ('the click was DISPATCHED +anyway ... check the page before repeating it'). Rationale: CDP-driven stacks (browser-use) +have exactly these semantics natively — occlusion never stops their clicks — so this is +actionability PARITY between stacks, not a cheat; the blocked attempt and the forced dispatch +are both surfaced honestly in history and in the book. Prediction: >=3 of 8 targets (the +click-widget+click-dialog compositions), controls 12/12 (fires only on blocked clicks, which +controls never hit). + ## Positioning vs public generic-harness baselines (user-supplied 2026 survey) The comparable class is generic agents, NOT MiniWoB-specialized systems (HTML-T5++ 95.2 trained diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 917b40fb..285facfc 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -137,6 +137,10 @@ class LlmPolicy: return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n" f"{rows}\nBegin your PLAN line with 'CLAUSE :' stating the clause you are working on.") + # v32: when a click is BLOCKED by an overlay, run.py dispatches a synthetic click on the + # target anyway (the actionability semantics CDP-driven stacks have natively) and says so. + force_unblock: bool = False + # v31: suppress weak-named wrapper rows that shadow exactly one real child. suppress_wrappers: bool = False @@ -905,6 +909,15 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: scripted_drag=True, auto_complete=True, som=False, native_pickers=True, verify_terminal=True, post_mouse_vision=True, multi_cap=6, fill_verify=True, **v17) + if name == "osw-llm-v32": # v31 + forced dispatch on blocked clicks (occlusion parity with CDP stacks) + v32 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, + max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + local_ctx=True, blocker_probe=True, suppress_wrappers=True, + force_unblock=True, **v32) if name == "osw-llm-v31": # v30 + wrapper suppression (trap rows shadowing one real child) v31 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, max_tokens=800) diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index 9affa9b8..bf0b9fb5 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -231,11 +231,22 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na " (e.className&&typeof e.className==='string'?'.'+e.className.split(' ')[0]:''); }", m_bid.group(1)), 8) if top: + forced = "" + if getattr(policy, "force_unblock", False): + # v32: dispatch the click on the element anyway (what CDP-driven + # stacks do natively -- occlusion never stops them). The page's + # listener decides if it counts; the model is told either way. + ok = with_deadline(lambda: env.unwrapped.page.evaluate( + "(bid) => { const el = document.querySelector(`[bid=\"${bid}\"]`);" + " if (!el) return false; el.click(); return true; }", + m_bid.group(1)), 8) + if ok: + forced = ("; the click was DISPATCHED anyway and may have " + "registered -- check the page before repeating it") # PREPENDED: a suffix after Playwright's multi-line call log gets # truncated out of both the record and the model's history. - err = (f"BLOCKED: {top} is covering the element you clicked -- move or " - f"close the cover first (drag its titlebar or dismiss it), then " - f"retry | {err}") + err = (f"BLOCKED: {top} is covering the element you clicked{forced} -- " + f"if it did not register, move or close the cover and retry | {err}") obs["last_action_error"] = err except Exception: pass