diff --git a/e2e/browser-v3/arena/ARENA.md b/e2e/browser-v3/arena/ARENA.md index d8b55d86..a1e08439 100644 --- a/e2e/browser-v3/arena/ARENA.md +++ b/e2e/browser-v3/arena/ARENA.md @@ -320,6 +320,25 @@ blocked clicks. Real-world analogue: cookie banners, modals, sticky headers. Pre 8 pilot targets (the 5 click-widget+click-dialog compositions are the candidates), controls 12/12 (mechanism cannot fire on them). +VERDICT (same day): **targets 0/8, controls 12/12 — the blocker path never fired because the +model sidestepped it into a TRAP ROW.** Labeled trace: step 1 clicked the row named '(widget)' +— a weak-named clickable WRAPPER div that shadows its single real child; the click lands (no +timeout, no error), the page ignores it, the clause silently stays undone. The composed pages +render every widget twice (generic mirror + real element); we showed both. Second finding: +login-composition targets die to 'policy produced no action' (unparseable replies) — raw +replies are now captured on no-action turns (LlmDecision.raw_tail) to make that diagnosable. + +## PRE-REGISTERED (2026-08-14, before any v31 episode): wrapper-suppression pilot + +Mechanism: `suppress_wrappers` (v31 = v30 + this) — a weak-named clickable (empty or +'(attr-hint)' name) whose subtree holds exactly ONE other picked element is that element's +wrapper: drop the shell, keep the properly-roled child. Acid test: the trap '(widget)' row and +all four generic widget mirrors vanish; the five real § widget elements remain (31 -> 25 rows). +Generic: wrapper-shadowing is how most real pages wire icon buttons and custom controls. +Prediction: >=3 of 8 targets (click-widget compositions specifically; the blocker rung from v30 +now actually gets exercised when the model clicks the real covered input), controls 12/12 +(suppression only fires on single-child weak-named shells). + ## Positioning vs public generic-harness baselines (user-supplied 2026 survey) The comparable class is generic agents, NOT MiniWoB-specialized systems (HTML-T5++ 95.2 trained diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 7326ad87..c664def0 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -67,6 +67,8 @@ class LlmDecision(Decision): llm_error: str = "" retries: int = 0 vision: int = 0 + # The model's verbatim reply when no action parsed from it -- the only way to debug WHY. + raw_tail: str = "" @dataclass @@ -135,6 +137,9 @@ class LlmPolicy: return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n" f"{rows}\nBegin your PLAN line with 'CLAUSE :' stating the clause you are working on.") + # v31: suppress weak-named wrapper rows that shadow exactly one real child. + suppress_wrappers: bool = False + # v30: blocked-click intelligence. When a click times out on actionability, run.py names the # covering element in last_action_error; the system rung teaches the move-the-cover response. blocker_probe: bool = False @@ -318,7 +323,8 @@ class OpenSwarmLlmPolicy(LlmPolicy): def view(self, obs: dict[str, Any], goal: str) -> tuple[str, int]: raw_items: list[RankItem] = perception.interactives( obs, include_clickable=self.clickable, attr_hints=self.hints, - include_offscreen=self.offscreen, local_ctx=self.local_ctx) + include_offscreen=self.offscreen, local_ctx=self.local_ctx, + suppress_wrappers=self.suppress_wrappers) shown, truncated = rank_and_cap(raw_items, goal=goal) new = {it.bid for it in shown} - self.prev_bids if self.prev_bids else set() self.prev_bids = {it.bid for it in shown} @@ -706,6 +712,8 @@ class OpenSwarmLlmPolicy(LlmPolicy): if self.scripted_drag: translated = [self.decompose_drag(t) for t in translated] d.action = "\n".join(translated) + if not translated: + d.raw_tail = (raw or "")[:300] for c in chosen_list: self.note(c, obs) self.all_actions.append(c) @@ -891,6 +899,14 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: scripted_drag=True, auto_complete=True, som=False, native_pickers=True, verify_terminal=True, post_mouse_vision=True, multi_cap=6, fill_verify=True, **v17) + if name == "osw-llm-v31": # v30 + wrapper suppression (trap rows shadowing one real child) + v31 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, + max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + local_ctx=True, blocker_probe=True, suppress_wrappers=True, **v31) if name == "osw-llm-v30": # v29 + blocked-click intelligence (named blockers + move-the-cover rung) v30 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, max_tokens=800) diff --git a/e2e/browser-v3/arena/perception.py b/e2e/browser-v3/arena/perception.py index b474f17b..72387e02 100644 --- a/e2e/browser-v3/arena/perception.py +++ b/e2e/browser-v3/arena/perception.py @@ -159,7 +159,8 @@ def build_local_context(by_id: dict[str, dict[str, Any]], node: dict[str, Any], def interactives(obs: dict[str, Any], include_hidden: bool = False, include_clickable: bool = False, attr_hints: bool = False, - include_offscreen: bool = False, local_ctx: bool = False) -> list[RankItem]: + include_offscreen: bool = False, local_ctx: bool = False, + suppress_wrappers: bool = False) -> list[RankItem]: """Every actionable node in document order, before any ranking or capping is applied. include_clickable is the technique ingested from browser-use: elements the page wires for @@ -205,6 +206,32 @@ def interactives(obs: dict[str, Any], include_hidden: bool = False, return nm inter_ids = {n.get("nodeId") for n, *_ in picked} + if suppress_wrappers and picked: + # A weak-named clickable whose subtree holds exactly ONE other picked element is that + # element's wrapper: a trap row ('(widget)') that looks like the thing and eats the click + # the page only counts on the child. Keep the properly-roled child, drop the shell. + memo: dict[str, int] = {} + + def pdesc(nid: str) -> int: + if nid in memo: + return memo[nid] + memo[nid] = 0 + node = by_id.get(nid) or {} + c = sum((1 if cid in inter_ids else 0) + pdesc(cid) for cid in node.get("childIds") or []) + memo[nid] = c + return c + + kept = [] + for tup in picked: + n = tup[0] + nm = node_name(n).strip() + weak = not nm or nm.startswith("(") + if weak and n.get("nodeId") and pdesc(n["nodeId"]) == 1: + continue + kept.append(tup) + if kept: + picked = kept + inter_ids = {n.get("nodeId") for n, *_ in picked} grp = dom_group_hints(obs) if local_ctx else {} out: list[RankItem] = [] for n, role, bid, props, onscreen in picked: diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index bcfe8bee..2334f2bd 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -197,7 +197,7 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na dest = rec.shot_path(arm, task, seed, step) rec_step.shot = save_screenshot(obs, dest) if not decision.action: - rec_step.action_error = "policy produced no action" + rec_step.action_error = ("policy produced no action | RAW: " + getattr(decision, "raw_tail", ""))[:400] ep.add(rec_step) # A no-action turn caused by a dead LLM lane is the harness's problem, not the arm's. if getattr(decision, "llm_error", ""):