diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index eb54d8fe..66f0bda3 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -447,6 +447,18 @@ MiniWoB, +12.6 CompWoB, +2 more CompWoB tasks under v34** — strongly positive, per-benchmark configs. The readonly-picker dead end is a named future primitive (JS value-set fallback when a picker fill bounces); the >=95 MiniWoB clause remains open. +## PRE-REGISTERED (2026-08-15, before any v35 episode): readonly-picker fallback pilot + +Mechanism: `native_js_fallback` (v35 = v34 + this) — a fill that bounces off a READONLY input +is applied through the page's OWN widget machinery (jQuery datepicker setDate when present; +value+input/change events otherwise), and the model is told to verify. Unit-verified against +the live page WITHOUT any model: fallback + submit scores 1.0 from MiniWoB's own reward. +Built during the quota outage (all lanes 429 — the plan window; sweeps paused ~9h so far). +Prediction: choose-date / choose-date-nodelay flip (both lost 2/3 seeds to exactly this dead +end), book-flight improves; controls (non-picker tasks) untouched — the mechanism fires only +on readonly fill timeouts. Pilot: the picker-loss cluster + 12 standard controls, when the +lane returns; then a full 3-seed to re-measure the MiniWoB clause. + ## Benchmark roadmap (2026 landscape survey, method-filtered) Rules: third-party scoring, reproducible from a committed artifact, no LLM-judge (or deterministic diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 47363a59..8c24a230 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -137,6 +137,10 @@ class LlmPolicy: return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n" f"{rows}\nBegin your PLAN line with 'CLAUSE :' stating the clause you are working on.") + # v35: readonly-picker value fallback (runner-side). A fill that bounces off a READONLY + # picker input is applied through the page's own widget API (jQuery datepicker etc.). + native_js_fallback: bool = False + # v33: deferral doctrine (system rung) + a mechanical defer-nudge when an action fails on a # missing/blocked target: the model is reminded to defer and continue, never stall or skip. defer_nudge: bool = False @@ -939,6 +943,15 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: scripted_drag=True, auto_complete=True, som=False, native_pickers=True, verify_terminal=True, post_mouse_vision=True, multi_cap=6, fill_verify=True, **v17) + if name == "osw-llm-v35": # v34 + readonly-picker value fallback + v35 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, + max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + local_ctx=True, blocker_probe=True, suppress_wrappers=True, + force_unblock=True, native_js_fallback=True, **v35) if name == "osw-llm-v34": # v32 + fastpath inversion gate (subordinate-clause goals go to the model) v34 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, max_tokens=800) diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index 075a77d0..bf292eba 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -53,6 +53,28 @@ def with_deadline(fn: Any, timeout_s: float) -> Any: signal.signal(signal.SIGALRM, prev) +def picker_value_fallback(page: Any, bid: str, value: str) -> str: + """READONLY pickers reject fill by design -- the page wants you to use its widget. So use its + widget: drive the picker's own API with the attempted value. Returns a note or '' if n/a.""" + try: + ok = page.evaluate( + """([bid, val]) => { + const el = document.querySelector(`[bid="${bid}"]`); + if (!el || !el.readOnly) return ''; + if (typeof jQuery !== 'undefined' && jQuery(el).hasClass('hasDatepicker')) { + const m = val.match(/(\\d{4})-(\\d{2})-(\\d{2})/); + if (m) { jQuery(el).datepicker('setDate', new Date(+m[1], +m[2]-1, +m[3])); return 'datepicker'; } + } + el.value = val; + el.dispatchEvent(new Event('input', {bubbles: true})); + el.dispatchEvent(new Event('change', {bubbles: true})); + return 'value+events'; + }""", [bid, value]) + return str(ok or "") + except Exception: + return "" + + def classify(exc: BaseException) -> str: """Separate a harness/browser failure from a policy failure so infra noise never scores as skill.""" name = type(exc).__name__ @@ -223,6 +245,15 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na # v30 (gated by the arm): a click that times out on actionability is usually COVERED by # an overlay (dialog, banner, sticky bar). Playwright knows; the model only hears # 'TimeoutError'. Name the blocker so the model can move/close it and retry. + if (getattr(policy, "native_js_fallback", False) and "Timeout" in err + and re.match(r"fill\(", decision.action)): + m_f = re.match(r'fill\(\s*"([^"]+)"\s*,\s*"([^"]*)"', decision.action) + if m_f: + how = picker_value_fallback(env.unwrapped.page, m_f.group(1), m_f.group(2)) + if how: + err = (f"READONLY input: your value was applied through the page's own " + f"{how} machinery instead -- verify it on the page and continue | {err}") + obs["last_action_error"] = err if (getattr(policy, "blocker_probe", False) and "Timeout" in err and re.match(r"(?:dbl)?click\(", decision.action)): m_bid = re.search(r'"([^"]+)"', decision.action)