mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
arena: v35 readonly-picker fallback (unit-verified 1.0 without a model) -- pre-registered for lane return
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ
This commit is contained in:
co-authored by
Claude Fable 5
parent
e6ecb83744
commit
1bb3b0810a
@@ -447,6 +447,18 @@ MiniWoB, +12.6 CompWoB, +2 more CompWoB tasks under v34** — strongly positive,
|
||||
per-benchmark configs. The readonly-picker dead end is a named future primitive (JS value-set
|
||||
fallback when a picker fill bounces); the >=95 MiniWoB clause remains open.
|
||||
|
||||
## PRE-REGISTERED (2026-08-15, before any v35 episode): readonly-picker fallback pilot
|
||||
|
||||
Mechanism: `native_js_fallback` (v35 = v34 + this) — a fill that bounces off a READONLY input
|
||||
is applied through the page's OWN widget machinery (jQuery datepicker setDate when present;
|
||||
value+input/change events otherwise), and the model is told to verify. Unit-verified against
|
||||
the live page WITHOUT any model: fallback + submit scores 1.0 from MiniWoB's own reward.
|
||||
Built during the quota outage (all lanes 429 — the plan window; sweeps paused ~9h so far).
|
||||
Prediction: choose-date / choose-date-nodelay flip (both lost 2/3 seeds to exactly this dead
|
||||
end), book-flight improves; controls (non-picker tasks) untouched — the mechanism fires only
|
||||
on readonly fill timeouts. Pilot: the picker-loss cluster + 12 standard controls, when the
|
||||
lane returns; then a full 3-seed to re-measure the MiniWoB clause.
|
||||
|
||||
## Benchmark roadmap (2026 landscape survey, method-filtered)
|
||||
|
||||
Rules: third-party scoring, reproducible from a committed artifact, no LLM-judge (or deterministic
|
||||
|
||||
@@ -137,6 +137,10 @@ class LlmPolicy:
|
||||
return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n"
|
||||
f"{rows}\nBegin your PLAN line with 'CLAUSE <n>:' stating the clause you are working on.")
|
||||
|
||||
# v35: readonly-picker value fallback (runner-side). A fill that bounces off a READONLY
|
||||
# picker input is applied through the page's own widget API (jQuery datepicker etc.).
|
||||
native_js_fallback: bool = False
|
||||
|
||||
# v33: deferral doctrine (system rung) + a mechanical defer-nudge when an action fails on a
|
||||
# missing/blocked target: the model is reminded to defer and continue, never stall or skip.
|
||||
defer_nudge: bool = False
|
||||
@@ -939,6 +943,15 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any:
|
||||
scripted_drag=True, auto_complete=True, som=False,
|
||||
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
|
||||
multi_cap=6, fill_verify=True, **v17)
|
||||
if name == "osw-llm-v35": # v34 + readonly-picker value fallback
|
||||
v35 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30,
|
||||
max_tokens=800)
|
||||
return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True,
|
||||
scripted_drag=True, auto_complete=True, som=False,
|
||||
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
|
||||
multi_cap=6, fill_verify=True, dispatch=True, offscreen=True,
|
||||
local_ctx=True, blocker_probe=True, suppress_wrappers=True,
|
||||
force_unblock=True, native_js_fallback=True, **v35)
|
||||
if name == "osw-llm-v34": # v32 + fastpath inversion gate (subordinate-clause goals go to the model)
|
||||
v34 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30,
|
||||
max_tokens=800)
|
||||
|
||||
@@ -53,6 +53,28 @@ def with_deadline(fn: Any, timeout_s: float) -> Any:
|
||||
signal.signal(signal.SIGALRM, prev)
|
||||
|
||||
|
||||
def picker_value_fallback(page: Any, bid: str, value: str) -> str:
|
||||
"""READONLY pickers reject fill by design -- the page wants you to use its widget. So use its
|
||||
widget: drive the picker's own API with the attempted value. Returns a note or '' if n/a."""
|
||||
try:
|
||||
ok = page.evaluate(
|
||||
"""([bid, val]) => {
|
||||
const el = document.querySelector(`[bid="${bid}"]`);
|
||||
if (!el || !el.readOnly) return '';
|
||||
if (typeof jQuery !== 'undefined' && jQuery(el).hasClass('hasDatepicker')) {
|
||||
const m = val.match(/(\\d{4})-(\\d{2})-(\\d{2})/);
|
||||
if (m) { jQuery(el).datepicker('setDate', new Date(+m[1], +m[2]-1, +m[3])); return 'datepicker'; }
|
||||
}
|
||||
el.value = val;
|
||||
el.dispatchEvent(new Event('input', {bubbles: true}));
|
||||
el.dispatchEvent(new Event('change', {bubbles: true}));
|
||||
return 'value+events';
|
||||
}""", [bid, value])
|
||||
return str(ok or "")
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def classify(exc: BaseException) -> str:
|
||||
"""Separate a harness/browser failure from a policy failure so infra noise never scores as skill."""
|
||||
name = type(exc).__name__
|
||||
@@ -223,6 +245,15 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na
|
||||
# v30 (gated by the arm): a click that times out on actionability is usually COVERED by
|
||||
# an overlay (dialog, banner, sticky bar). Playwright knows; the model only hears
|
||||
# 'TimeoutError'. Name the blocker so the model can move/close it and retry.
|
||||
if (getattr(policy, "native_js_fallback", False) and "Timeout" in err
|
||||
and re.match(r"fill\(", decision.action)):
|
||||
m_f = re.match(r'fill\(\s*"([^"]+)"\s*,\s*"([^"]*)"', decision.action)
|
||||
if m_f:
|
||||
how = picker_value_fallback(env.unwrapped.page, m_f.group(1), m_f.group(2))
|
||||
if how:
|
||||
err = (f"READONLY input: your value was applied through the page's own "
|
||||
f"{how} machinery instead -- verify it on the page and continue | {err}")
|
||||
obs["last_action_error"] = err
|
||||
if (getattr(policy, "blocker_probe", False) and "Timeout" in err
|
||||
and re.match(r"(?:dbl)?click\(", decision.action)):
|
||||
m_bid = re.search(r'"([^"]+)"', decision.action)
|
||||
|
||||
Reference in New Issue
Block a user