mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
arena: v29 verdict (controls clean, targets 0/8 -- model now picks RIGHT element, click blocked by overlay) + v30 blocked-click intelligence (pre-registered)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ
This commit is contained in:
co-authored by
Claude Fable 5
parent
60318f2c1f
commit
7f10e100a9
@@ -135,6 +135,10 @@ class LlmPolicy:
|
||||
return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n"
|
||||
f"{rows}\nBegin your PLAN line with 'CLAUSE <n>:' stating the clause you are working on.")
|
||||
|
||||
# v30: blocked-click intelligence. When a click times out on actionability, run.py names the
|
||||
# covering element in last_action_error; the system rung teaches the move-the-cover response.
|
||||
blocker_probe: bool = False
|
||||
|
||||
# v29: discriminative local-group row context (perception layer). See
|
||||
# perception.build_local_context -- same-role nameless twins in different page sections must
|
||||
# render distinct, or the model's choice between them is a coin flip it cannot know it's making.
|
||||
@@ -187,7 +191,9 @@ class LlmPolicy:
|
||||
five times. Feeding an agent a memory it cannot read is a harness bug, not an agent failure.
|
||||
"""
|
||||
err = str(obs.get("last_action_error") or "").strip()
|
||||
line = f"{action} -> {'ERROR: ' + err[:120] if err else 'ok'}"
|
||||
# A named blocker is the actionable half of the message; never truncate it away.
|
||||
cap = 240 if "BLOCKED:" in err else 120
|
||||
line = f"{action} -> {'ERROR: ' + err[:cap] if err else 'ok'}"
|
||||
if self.echo_feedback:
|
||||
now = ""
|
||||
try:
|
||||
@@ -752,6 +758,13 @@ On the open web you may also navigate: goto("url") | go_back() | go_forward().
|
||||
When the goal is a QUESTION, research it and deliver the answer with send_msg_to_user("answer") --
|
||||
the answer text alone, no prose around it."""
|
||||
|
||||
OSW_SYSTEM_V30 = """
|
||||
A click that errors with BLOCKED means another element physically covers the target (a dialog,
|
||||
banner, or sticky bar). Do NOT retry the same click and do NOT guess coordinates. First get the
|
||||
cover out of the way -- drag it aside by its titlebar with drag_and_drop if the goal needs it
|
||||
kept open, or close/dismiss it if the goal doesn't -- then retry the original click. If the goal
|
||||
says to interact with the covering element LATER, moving it aside now keeps that order intact."""
|
||||
|
||||
CALL_RE = re.compile(
|
||||
r"\b(click|dblclick|fill|clear|select_option|hover|focus|press|scroll|drag_and_drop|noop"
|
||||
r"|mouse_click|mouse_dblclick|mouse_move|mouse_drag_and_drop|keyboard_type|keyboard_press"
|
||||
@@ -878,6 +891,14 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any:
|
||||
scripted_drag=True, auto_complete=True, som=False,
|
||||
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
|
||||
multi_cap=6, fill_verify=True, **v17)
|
||||
if name == "osw-llm-v30": # v29 + blocked-click intelligence (named blockers + move-the-cover rung)
|
||||
v30 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30,
|
||||
max_tokens=800)
|
||||
return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True,
|
||||
scripted_drag=True, auto_complete=True, som=False,
|
||||
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
|
||||
multi_cap=6, fill_verify=True, dispatch=True, offscreen=True,
|
||||
local_ctx=True, blocker_probe=True, **v30)
|
||||
if name == "osw-llm-v29": # v22 + discriminative local-group row context (perception primitive)
|
||||
v29 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16, max_tokens=800)
|
||||
return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True,
|
||||
|
||||
@@ -211,7 +211,27 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na
|
||||
lambda: env.step(decision.action), args.step_timeout)
|
||||
rec_step.action_ms = (time.time() - t_act) * 1000
|
||||
rec_step.reward = float(reward or 0)
|
||||
rec_step.action_error = str(obs.get("last_action_error") or "")[:200]
|
||||
err = str(obs.get("last_action_error") or "")
|
||||
# v30 (gated by the arm): a click that times out on actionability is usually COVERED by
|
||||
# an overlay (dialog, banner, sticky bar). Playwright knows; the model only hears
|
||||
# 'TimeoutError'. Name the blocker so the model can move/close it and retry.
|
||||
if (getattr(policy, "blocker_probe", False) and "Timeout" in err
|
||||
and re.match(r"(?:dbl)?click\(", decision.action)):
|
||||
m_bid = re.search(r'"([^"]+)"', decision.action)
|
||||
bbox = ((obs.get("extra_element_properties") or {}).get(m_bid.group(1)) or {}).get("bbox") if m_bid else None
|
||||
if bbox:
|
||||
try:
|
||||
top = with_deadline(lambda: env.unwrapped.page.evaluate(
|
||||
"([x,y]) => { const e = document.elementFromPoint(x,y);"
|
||||
" return e ? e.tagName + (e.id?'#'+e.id:'') +"
|
||||
" (e.className&&typeof e.className==='string'?'.'+e.className.split(' ')[0]:'') : ''; }",
|
||||
[bbox[0] + bbox[2] / 2, bbox[1] + bbox[3] / 2]), 8)
|
||||
if top:
|
||||
err += f" | BLOCKED: {top} is covering this element -- move or close the cover first (drag its titlebar or dismiss it), then retry"
|
||||
obs["last_action_error"] = err
|
||||
except Exception:
|
||||
pass
|
||||
rec_step.action_error = err[:260]
|
||||
ep.add(rec_step)
|
||||
ep.reward = max(ep.reward, float(reward or 0))
|
||||
ep.steps = step
|
||||
|
||||
Reference in New Issue
Block a user