diff --git a/e2e/browser-v3/arena/llm_policy.py b/e2e/browser-v3/arena/llm_policy.py index 6a076746..7326ad87 100644 --- a/e2e/browser-v3/arena/llm_policy.py +++ b/e2e/browser-v3/arena/llm_policy.py @@ -135,6 +135,10 @@ class LlmPolicy: return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n" f"{rows}\nBegin your PLAN line with 'CLAUSE :' stating the clause you are working on.") + # v30: blocked-click intelligence. When a click times out on actionability, run.py names the + # covering element in last_action_error; the system rung teaches the move-the-cover response. + blocker_probe: bool = False + # v29: discriminative local-group row context (perception layer). See # perception.build_local_context -- same-role nameless twins in different page sections must # render distinct, or the model's choice between them is a coin flip it cannot know it's making. @@ -187,7 +191,9 @@ class LlmPolicy: five times. Feeding an agent a memory it cannot read is a harness bug, not an agent failure. """ err = str(obs.get("last_action_error") or "").strip() - line = f"{action} -> {'ERROR: ' + err[:120] if err else 'ok'}" + # A named blocker is the actionable half of the message; never truncate it away. + cap = 240 if "BLOCKED:" in err else 120 + line = f"{action} -> {'ERROR: ' + err[:cap] if err else 'ok'}" if self.echo_feedback: now = "" try: @@ -752,6 +758,13 @@ On the open web you may also navigate: goto("url") | go_back() | go_forward(). When the goal is a QUESTION, research it and deliver the answer with send_msg_to_user("answer") -- the answer text alone, no prose around it.""" +OSW_SYSTEM_V30 = """ +A click that errors with BLOCKED means another element physically covers the target (a dialog, +banner, or sticky bar). Do NOT retry the same click and do NOT guess coordinates. First get the +cover out of the way -- drag it aside by its titlebar with drag_and_drop if the goal needs it +kept open, or close/dismiss it if the goal doesn't -- then retry the original click. If the goal +says to interact with the covering element LATER, moving it aside now keeps that order intact.""" + CALL_RE = re.compile( r"\b(click|dblclick|fill|clear|select_option|hover|focus|press|scroll|drag_and_drop|noop" r"|mouse_click|mouse_dblclick|mouse_move|mouse_drag_and_drop|keyboard_type|keyboard_press" @@ -878,6 +891,14 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any: scripted_drag=True, auto_complete=True, som=False, native_pickers=True, verify_terminal=True, post_mouse_vision=True, multi_cap=6, fill_verify=True, **v17) + if name == "osw-llm-v30": # v29 + blocked-click intelligence (named blockers + move-the-cover rung) + v30 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30, + max_tokens=800) + return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, + scripted_drag=True, auto_complete=True, som=False, + native_pickers=True, verify_terminal=True, post_mouse_vision=True, + multi_cap=6, fill_verify=True, dispatch=True, offscreen=True, + local_ctx=True, blocker_probe=True, **v30) if name == "osw-llm-v29": # v22 + discriminative local-group row context (perception primitive) v29 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16, max_tokens=800) return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True, diff --git a/e2e/browser-v3/arena/run.py b/e2e/browser-v3/arena/run.py index a21c4c9b..bcfe8bee 100644 --- a/e2e/browser-v3/arena/run.py +++ b/e2e/browser-v3/arena/run.py @@ -211,7 +211,27 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na lambda: env.step(decision.action), args.step_timeout) rec_step.action_ms = (time.time() - t_act) * 1000 rec_step.reward = float(reward or 0) - rec_step.action_error = str(obs.get("last_action_error") or "")[:200] + err = str(obs.get("last_action_error") or "") + # v30 (gated by the arm): a click that times out on actionability is usually COVERED by + # an overlay (dialog, banner, sticky bar). Playwright knows; the model only hears + # 'TimeoutError'. Name the blocker so the model can move/close it and retry. + if (getattr(policy, "blocker_probe", False) and "Timeout" in err + and re.match(r"(?:dbl)?click\(", decision.action)): + m_bid = re.search(r'"([^"]+)"', decision.action) + bbox = ((obs.get("extra_element_properties") or {}).get(m_bid.group(1)) or {}).get("bbox") if m_bid else None + if bbox: + try: + top = with_deadline(lambda: env.unwrapped.page.evaluate( + "([x,y]) => { const e = document.elementFromPoint(x,y);" + " return e ? e.tagName + (e.id?'#'+e.id:'') +" + " (e.className&&typeof e.className==='string'?'.'+e.className.split(' ')[0]:'') : ''; }", + [bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2]), 8) + if top: + err += f" | BLOCKED: {top} is covering this element -- move or close the cover first (drag its titlebar or dismiss it), then retry" + obs["last_action_error"] = err + except Exception: + pass + rec_step.action_error = err[:260] ep.add(rec_step) ep.reward = max(ep.reward, float(reward or 0)) ep.steps = step