arena: v35 readonly-picker fallback (unit-verified 1.0 without a model) -- pre-registered for lane return

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ
This commit is contained in:
ciregenz
2026-08-15 08:42:14 -07:00
co-authored by Claude Fable 5
parent e6ecb83744
commit 1bb3b0810a
3 changed files with 56 additions and 0 deletions
+12
View File
@@ -447,6 +447,18 @@ MiniWoB, +12.6 CompWoB, +2 more CompWoB tasks under v34** — strongly positive,
per-benchmark configs. The readonly-picker dead end is a named future primitive (JS value-set
fallback when a picker fill bounces); the >=95 MiniWoB clause remains open.
## PRE-REGISTERED (2026-08-15, before any v35 episode): readonly-picker fallback pilot
Mechanism: `native_js_fallback` (v35 = v34 + this) — a fill that bounces off a READONLY input
is applied through the page's OWN widget machinery (jQuery datepicker setDate when present;
value+input/change events otherwise), and the model is told to verify. Unit-verified against
the live page WITHOUT any model: fallback + submit scores 1.0 from MiniWoB's own reward.
Built during the quota outage (all lanes 429 — the plan window; sweeps paused ~9h so far).
Prediction: choose-date / choose-date-nodelay flip (both lost 2/3 seeds to exactly this dead
end), book-flight improves; controls (non-picker tasks) untouched — the mechanism fires only
on readonly fill timeouts. Pilot: the picker-loss cluster + 12 standard controls, when the
lane returns; then a full 3-seed to re-measure the MiniWoB clause.
## Benchmark roadmap (2026 landscape survey, method-filtered)
Rules: third-party scoring, reproducible from a committed artifact, no LLM-judge (or deterministic
+13
View File
@@ -137,6 +137,10 @@ class LlmPolicy:
return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n"
f"{rows}\nBegin your PLAN line with 'CLAUSE <n>:' stating the clause you are working on.")
# v35: readonly-picker value fallback (runner-side). A fill that bounces off a READONLY
# picker input is applied through the page's own widget API (jQuery datepicker etc.).
native_js_fallback: bool = False
# v33: deferral doctrine (system rung) + a mechanical defer-nudge when an action fails on a
# missing/blocked target: the model is reminded to defer and continue, never stall or skip.
defer_nudge: bool = False
@@ -939,6 +943,15 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any:
scripted_drag=True, auto_complete=True, som=False,
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
multi_cap=6, fill_verify=True, **v17)
if name == "osw-llm-v35": # v34 + readonly-picker value fallback
v35 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30,
max_tokens=800)
return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True,
scripted_drag=True, auto_complete=True, som=False,
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
multi_cap=6, fill_verify=True, dispatch=True, offscreen=True,
local_ctx=True, blocker_probe=True, suppress_wrappers=True,
force_unblock=True, native_js_fallback=True, **v35)
if name == "osw-llm-v34": # v32 + fastpath inversion gate (subordinate-clause goals go to the model)
v34 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30,
max_tokens=800)
+31
View File
@@ -53,6 +53,28 @@ def with_deadline(fn: Any, timeout_s: float) -> Any:
signal.signal(signal.SIGALRM, prev)
def picker_value_fallback(page: Any, bid: str, value: str) -> str:
"""READONLY pickers reject fill by design -- the page wants you to use its widget. So use its
widget: drive the picker's own API with the attempted value. Returns a note or '' if n/a."""
try:
ok = page.evaluate(
"""([bid, val]) => {
const el = document.querySelector(`[bid="${bid}"]`);
if (!el || !el.readOnly) return '';
if (typeof jQuery !== 'undefined' && jQuery(el).hasClass('hasDatepicker')) {
const m = val.match(/(\\d{4})-(\\d{2})-(\\d{2})/);
if (m) { jQuery(el).datepicker('setDate', new Date(+m[1], +m[2]-1, +m[3])); return 'datepicker'; }
}
el.value = val;
el.dispatchEvent(new Event('input', {bubbles: true}));
el.dispatchEvent(new Event('change', {bubbles: true}));
return 'value+events';
}""", [bid, value])
return str(ok or "")
except Exception:
return ""
def classify(exc: BaseException) -> str:
"""Separate a harness/browser failure from a policy failure so infra noise never scores as skill."""
name = type(exc).__name__
@@ -223,6 +245,15 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na
# v30 (gated by the arm): a click that times out on actionability is usually COVERED by
# an overlay (dialog, banner, sticky bar). Playwright knows; the model only hears
# 'TimeoutError'. Name the blocker so the model can move/close it and retry.
if (getattr(policy, "native_js_fallback", False) and "Timeout" in err
and re.match(r"fill\(", decision.action)):
m_f = re.match(r'fill\(\s*"([^"]+)"\s*,\s*"([^"]*)"', decision.action)
if m_f:
how = picker_value_fallback(env.unwrapped.page, m_f.group(1), m_f.group(2))
if how:
err = (f"READONLY input: your value was applied through the page's own "
f"{how} machinery instead -- verify it on the page and continue | {err}")
obs["last_action_error"] = err
if (getattr(policy, "blocker_probe", False) and "Timeout" in err
and re.match(r"(?:dbl)?click\(", decision.action)):
m_bid = re.search(r'"([^"]+)"', decision.action)