mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-28 10:49:46 +02:00
arena: v30 verdict (trap wrapper rows) + v31 wrapper suppression (pre-registered) + raw-reply capture
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01WsbS5x2rYsMDxP2kW3qqmQ
This commit is contained in:
co-authored by
Claude Fable 5
parent
e87fd7545b
commit
b2c0d2be08
@@ -320,6 +320,25 @@ blocked clicks. Real-world analogue: cookie banners, modals, sticky headers. Pre
|
||||
8 pilot targets (the 5 click-widget+click-dialog compositions are the candidates), controls
|
||||
12/12 (mechanism cannot fire on them).
|
||||
|
||||
VERDICT (same day): **targets 0/8, controls 12/12 — the blocker path never fired because the
|
||||
model sidestepped it into a TRAP ROW.** Labeled trace: step 1 clicked the row named '(widget)'
|
||||
— a weak-named clickable WRAPPER div that shadows its single real child; the click lands (no
|
||||
timeout, no error), the page ignores it, the clause silently stays undone. The composed pages
|
||||
render every widget twice (generic mirror + real element); we showed both. Second finding:
|
||||
login-composition targets die to 'policy produced no action' (unparseable replies) — raw
|
||||
replies are now captured on no-action turns (LlmDecision.raw_tail) to make that diagnosable.
|
||||
|
||||
## PRE-REGISTERED (2026-08-14, before any v31 episode): wrapper-suppression pilot
|
||||
|
||||
Mechanism: `suppress_wrappers` (v31 = v30 + this) — a weak-named clickable (empty or
|
||||
'(attr-hint)' name) whose subtree holds exactly ONE other picked element is that element's
|
||||
wrapper: drop the shell, keep the properly-roled child. Acid test: the trap '(widget)' row and
|
||||
all four generic widget mirrors vanish; the five real § widget elements remain (31 -> 25 rows).
|
||||
Generic: wrapper-shadowing is how most real pages wire icon buttons and custom controls.
|
||||
Prediction: >=3 of 8 targets (click-widget compositions specifically; the blocker rung from v30
|
||||
now actually gets exercised when the model clicks the real covered input), controls 12/12
|
||||
(suppression only fires on single-child weak-named shells).
|
||||
|
||||
## Positioning vs public generic-harness baselines (user-supplied 2026 survey)
|
||||
|
||||
The comparable class is generic agents, NOT MiniWoB-specialized systems (HTML-T5++ 95.2 trained
|
||||
|
||||
@@ -67,6 +67,8 @@ class LlmDecision(Decision):
|
||||
llm_error: str = ""
|
||||
retries: int = 0
|
||||
vision: int = 0
|
||||
# The model's verbatim reply when no action parsed from it -- the only way to debug WHY.
|
||||
raw_tail: str = ""
|
||||
|
||||
|
||||
@dataclass
|
||||
@@ -135,6 +137,9 @@ class LlmPolicy:
|
||||
return (f"\nINSTRUCTION CLAUSES (complete IN ORDER; you last reported clause {self.cur_clause}):\n"
|
||||
f"{rows}\nBegin your PLAN line with 'CLAUSE <n>:' stating the clause you are working on.")
|
||||
|
||||
# v31: suppress weak-named wrapper rows that shadow exactly one real child.
|
||||
suppress_wrappers: bool = False
|
||||
|
||||
# v30: blocked-click intelligence. When a click times out on actionability, run.py names the
|
||||
# covering element in last_action_error; the system rung teaches the move-the-cover response.
|
||||
blocker_probe: bool = False
|
||||
@@ -318,7 +323,8 @@ class OpenSwarmLlmPolicy(LlmPolicy):
|
||||
def view(self, obs: dict[str, Any], goal: str) -> tuple[str, int]:
|
||||
raw_items: list[RankItem] = perception.interactives(
|
||||
obs, include_clickable=self.clickable, attr_hints=self.hints,
|
||||
include_offscreen=self.offscreen, local_ctx=self.local_ctx)
|
||||
include_offscreen=self.offscreen, local_ctx=self.local_ctx,
|
||||
suppress_wrappers=self.suppress_wrappers)
|
||||
shown, truncated = rank_and_cap(raw_items, goal=goal)
|
||||
new = {it.bid for it in shown} - self.prev_bids if self.prev_bids else set()
|
||||
self.prev_bids = {it.bid for it in shown}
|
||||
@@ -706,6 +712,8 @@ class OpenSwarmLlmPolicy(LlmPolicy):
|
||||
if self.scripted_drag:
|
||||
translated = [self.decompose_drag(t) for t in translated]
|
||||
d.action = "\n".join(translated)
|
||||
if not translated:
|
||||
d.raw_tail = (raw or "")[:300]
|
||||
for c in chosen_list:
|
||||
self.note(c, obs)
|
||||
self.all_actions.append(c)
|
||||
@@ -891,6 +899,14 @@ def build(name: str, model: str = "", endpoint: str = "", **_: Any) -> Any:
|
||||
scripted_drag=True, auto_complete=True, som=False,
|
||||
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
|
||||
multi_cap=6, fill_verify=True, **v17)
|
||||
if name == "osw-llm-v31": # v30 + wrapper suppression (trap rows shadowing one real child)
|
||||
v31 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30,
|
||||
max_tokens=800)
|
||||
return OpenSwarmLlmPolicy(name=name, multi=True, vision="progressive", fastpath=True,
|
||||
scripted_drag=True, auto_complete=True, som=False,
|
||||
native_pickers=True, verify_terminal=True, post_mouse_vision=True,
|
||||
multi_cap=6, fill_verify=True, dispatch=True, offscreen=True,
|
||||
local_ctx=True, blocker_probe=True, suppress_wrappers=True, **v31)
|
||||
if name == "osw-llm-v30": # v29 + blocked-click intelligence (named blockers + move-the-cover rung)
|
||||
v30 = dict(v7, system=OSW_SYSTEM_V8 + OSW_SYSTEM_V9_WIDGETS + OSW_SYSTEM_V16 + OSW_SYSTEM_V30,
|
||||
max_tokens=800)
|
||||
|
||||
@@ -159,7 +159,8 @@ def build_local_context(by_id: dict[str, dict[str, Any]], node: dict[str, Any],
|
||||
|
||||
def interactives(obs: dict[str, Any], include_hidden: bool = False,
|
||||
include_clickable: bool = False, attr_hints: bool = False,
|
||||
include_offscreen: bool = False, local_ctx: bool = False) -> list[RankItem]:
|
||||
include_offscreen: bool = False, local_ctx: bool = False,
|
||||
suppress_wrappers: bool = False) -> list[RankItem]:
|
||||
"""Every actionable node in document order, before any ranking or capping is applied.
|
||||
|
||||
include_clickable is the technique ingested from browser-use: elements the page wires for
|
||||
@@ -205,6 +206,32 @@ def interactives(obs: dict[str, Any], include_hidden: bool = False,
|
||||
return nm
|
||||
|
||||
inter_ids = {n.get("nodeId") for n, *_ in picked}
|
||||
if suppress_wrappers and picked:
|
||||
# A weak-named clickable whose subtree holds exactly ONE other picked element is that
|
||||
# element's wrapper: a trap row ('(widget)') that looks like the thing and eats the click
|
||||
# the page only counts on the child. Keep the properly-roled child, drop the shell.
|
||||
memo: dict[str, int] = {}
|
||||
|
||||
def pdesc(nid: str) -> int:
|
||||
if nid in memo:
|
||||
return memo[nid]
|
||||
memo[nid] = 0
|
||||
node = by_id.get(nid) or {}
|
||||
c = sum((1 if cid in inter_ids else 0) + pdesc(cid) for cid in node.get("childIds") or [])
|
||||
memo[nid] = c
|
||||
return c
|
||||
|
||||
kept = []
|
||||
for tup in picked:
|
||||
n = tup[0]
|
||||
nm = node_name(n).strip()
|
||||
weak = not nm or nm.startswith("(")
|
||||
if weak and n.get("nodeId") and pdesc(n["nodeId"]) == 1:
|
||||
continue
|
||||
kept.append(tup)
|
||||
if kept:
|
||||
picked = kept
|
||||
inter_ids = {n.get("nodeId") for n, *_ in picked}
|
||||
grp = dom_group_hints(obs) if local_ctx else {}
|
||||
out: list[RankItem] = []
|
||||
for n, role, bid, props, onscreen in picked:
|
||||
|
||||
@@ -197,7 +197,7 @@ def run_episode(arm: str, task: str, seed: int, rec: Recorder, args: argparse.Na
|
||||
dest = rec.shot_path(arm, task, seed, step)
|
||||
rec_step.shot = save_screenshot(obs, dest)
|
||||
if not decision.action:
|
||||
rec_step.action_error = "policy produced no action"
|
||||
rec_step.action_error = ("policy produced no action | RAW: " + getattr(decision, "raw_tail", ""))[:400]
|
||||
ep.add(rec_step)
|
||||
# A no-action turn caused by a dead LLM lane is the harness's problem, not the arm's.
|
||||
if getattr(decision, "llm_error", ""):
|
||||
|
||||
Reference in New Issue
Block a user