arena: v30 verdict (trap wrapper rows) + v31 wrapper suppression (pre-registered) + raw-reply capture

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ
This commit is contained in:
ciregenz
2026-08-14 05:28:33 -07:00
co-authored by Claude Fable 5
parent e87fd7545b
commit b2c0d2be08
4 changed files with 65 additions and 3 deletions
+19
View File
@@ -320,6 +320,25 @@ blocked clicks. Real-world analogue: cookie banners, modals, sticky headers. Pre
8 pilot targets (the 5 click-widget+click-dialog compositions are the candidates), controls
12/12 (mechanism cannot fire on them).
VERDICT (same day): **targets 0/8, controls 12/12 — the blocker path never fired because the
model sidestepped it into a TRAP ROW.** Labeled trace: step 1 clicked the row named '(widget)'
— a weak-named clickable WRAPPER div that shadows its single real child; the click lands (no
timeout, no error), the page ignores it, the clause silently stays undone. The composed pages
render every widget twice (generic mirror + real element); we showed both. Second finding:
login-composition targets die to 'policy produced no action' (unparseable replies) — raw
replies are now captured on no-action turns (LlmDecision.raw_tail) to make that diagnosable.
## PRE-REGISTERED (2026-08-14, before any v31 episode): wrapper-suppression pilot
Mechanism: `suppress_wrappers` (v31 = v30 + this) — a weak-named clickable (empty or
'(attr-hint)' name) whose subtree holds exactly ONE other picked element is that element's
wrapper: drop the shell, keep the properly-roled child. Acid test: the trap '(widget)' row and
all four generic widget mirrors vanish; the five real § widget elements remain (31 -> 25 rows).
Generic: wrapper-shadowing is how most real pages wire icon buttons and custom controls.
Prediction: >=3 of 8 targets (click-widget compositions specifically; the blocker rung from v30
now actually gets exercised when the model clicks the real covered input), controls 12/12
(suppression only fires on single-child weak-named shells).
## Positioning vs public generic-harness baselines (user-supplied 2026 survey)
The comparable class is generic agents, NOT MiniWoB-specialized systems (HTML-T5++ 95.2 trained
+17 -1
View File
@@ -67,6 +67,8 @@ class LlmDecision(Decision):
llm_error: str = ""
retries: int = 0
vision: int = 0
# The model's verbatim reply when no action parsed from it -- the only way to debug WHY.
raw_tail: str = ""
@dataclass
@@ -135,6 +137,9 @@ class LlmPolicy:
return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n"
f"{rows}\nBegin your PLAN line with 'CLAUSE <n>:' stating the clause you are working on.")
# v31: suppress weak-named wrapper rows that shadow exactly one real child.
suppress_wrappers: bool = False
# v30: blocked-click intelligence. When a click times out on actionability, run.py names the
# covering element in last_action_error; the system rung teaches the move-the-cover response.
blocker_probe: bool = False
@@ -318,7 +323,8 @@ class OpenSwarmLlmPolicy(LlmPolicy):
def view(self, obs: dict[str, Any], goal: str) -> tuple[str, int]:
raw_items: list[RankItem] = perception.interactives(
obs, include_clickable=self.clickable, attr_hints=self.hints,
include_offscreen=self.offscreen, local_ctx=self.local_ctx)
include_offscreen=self.offscreen, local_ctx=self.local_ctx,
suppress_wrappers=self.suppress_wrappers)
shown, truncated = rank_and_cap(raw_items, goal=goal)
new = {it.bid for it in shown} - self.prev_bids if self.prev_bids else set()
self.prev_bids = {it.bid for it in shown}
@@ -706,6 +712,8 @@ class OpenSwarmLlmPolicy(LlmPolicy):
if self.scripted_drag:
translated = [self.decompose_drag(t) for t in translated]
d.action = "\n".join(translated)
if not translated:
d.raw_tail = (raw or "")[:300]
for c in chosen_list:
self.note(c, obs)
self.all_actions.append(c)
@@ -891,6 +899,14 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any:
scripted_drag=True, auto_complete=True, som=False,
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
multi_cap=6, fill_verify=True, **v17)
if name == "osw-llm-v31": # v30 + wrapper suppression (trap rows shadowing one real child)
v31 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30,
max_tokens=800)
return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True,
scripted_drag=True, auto_complete=True, som=False,
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
multi_cap=6, fill_verify=True, dispatch=True, offscreen=True,
local_ctx=True, blocker_probe=True, suppress_wrappers=True, **v31)
if name == "osw-llm-v30": # v29 + blocked-click intelligence (named blockers + move-the-cover rung)
v30 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30,
max_tokens=800)
+28 -1
View File
@@ -159,7 +159,8 @@ def build_local_context(by_id: dict[str, dict[str, Any]], node: dict[str, Any],
def interactives(obs: dict[str, Any], include_hidden: bool = False,
include_clickable: bool = False, attr_hints: bool = False,
include_offscreen: bool = False, local_ctx: bool = False) -> list[RankItem]:
include_offscreen: bool = False, local_ctx: bool = False,
suppress_wrappers: bool = False) -> list[RankItem]:
"""Every actionable node in document order, before any ranking or capping is applied.
include_clickable is the technique ingested from browser-use: elements the page wires for
@@ -205,6 +206,32 @@ def interactives(obs: dict[str, Any], include_hidden: bool = False,
return nm
inter_ids = {n.get("nodeId") for n, *_ in picked}
if suppress_wrappers and picked:
# A weak-named clickable whose subtree holds exactly ONE other picked element is that
# element's wrapper: a trap row ('(widget)') that looks like the thing and eats the click
# the page only counts on the child. Keep the properly-roled child, drop the shell.
memo: dict[str, int] = {}
def pdesc(nid: str) -> int:
if nid in memo:
return memo[nid]
memo[nid] = 0
node = by_id.get(nid) or {}
c = sum((1 if cid in inter_ids else 0) + pdesc(cid) for cid in node.get("childIds") or [])
memo[nid] = c
return c
kept = []
for tup in picked:
n = tup[0]
nm = node_name(n).strip()
weak = not nm or nm.startswith("(")
if weak and n.get("nodeId") and pdesc(n["nodeId"]) == 1:
continue
kept.append(tup)
if kept:
picked = kept
inter_ids = {n.get("nodeId") for n, *_ in picked}
grp = dom_group_hints(obs) if local_ctx else {}
out: list[RankItem] = []
for n, role, bid, props, onscreen in picked:
+1 -1
View File
@@ -197,7 +197,7 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na
dest = rec.shot_path(arm, task, seed, step)
rec_step.shot = save_screenshot(obs, dest)
if not decision.action:
rec_step.action_error = "policy produced no action"
rec_step.action_error = ("policy produced no action | RAW: " + getattr(decision, "raw_tail", ""))[:400]
ep.add(rec_step)
# A no-action turn caused by a dead LLM lane is the harness's problem, not the arm's.
if getattr(decision, "llm_error", ""):