mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-26 14:32:22 +02:00
[eric] browser: settle on the click target before a replay step so a recorded click never fires before paint
This commit is contained in:
@@ -631,8 +631,15 @@ async def run_browser_agent(
|
||||
return None
|
||||
|
||||
async def _exec_step(step: dict) -> dict | None:
|
||||
"""One replay step; an off-screen click gets one scroll-and-retry
|
||||
(recorded elements often sit below the fold on a fresh page)."""
|
||||
"""One replay step; settle on the click target first so a recorded
|
||||
click never fires before the page paints it (the premature-click miss
|
||||
that quarantined skills), then an off-screen click gets one
|
||||
scroll-and-retry (recorded elements often sit below the fold)."""
|
||||
_settle = browser_skills.replay_settle_target(step)
|
||||
if _settle:
|
||||
async def _w(t, p, b, tid):
|
||||
return await _cancellable(execute_browser_tool(t, p, b, tid))
|
||||
await browser_wait.smart_wait(_w, browser_id, tab_id, 1500, until=_settle)
|
||||
res = await _cancellable(execute_browser_tool(step["tool"], step.get("params", {}), browser_id, tab_id))
|
||||
if res is not None and "box model" in str(res.get("error", "")):
|
||||
logger.info(f"[browser-skills] replay step off-screen ({step['tool']}); scrolling and retrying once")
|
||||
|
||||
@@ -328,6 +328,19 @@ def _prune_detours(steps: list[dict]) -> list[dict]:
|
||||
return out
|
||||
|
||||
|
||||
def replay_settle_target(step: dict) -> str | None:
|
||||
"""What a replay should WAIT for before running this step, or None. For a
|
||||
click-by-name, that's the name itself: a recorded click can fire before the
|
||||
target paints on a fresh page (the premature-click miss that quarantined
|
||||
skills), so settling on the name first makes replay robust without changing
|
||||
what it does. Only short, literal names are useful settle targets."""
|
||||
if step.get("tool") != "BrowserClickByName":
|
||||
return None
|
||||
name = (step.get("params", {}) or {}).get("name") or ""
|
||||
name = name.strip()
|
||||
return name if 0 < len(name) <= 60 else None
|
||||
|
||||
|
||||
def first_unsafe_step(steps: list[dict]) -> tuple[int, str]:
|
||||
"""Index of the first outward-facing step (click Send/Submit/Pay, type into
|
||||
a composer), -1 if none. Reuses the batch replayer's wordlist so there is
|
||||
|
||||
@@ -697,3 +697,19 @@ def test_distill_keeps_navigate_that_was_acted_on():
|
||||
steps = sk.distill_steps(log)
|
||||
urls = [s["params"].get("url") for s in steps if s["tool"] == "BrowserNavigate"]
|
||||
assert urls == ["https://x.com/form", "https://x.com/results"] # both kept
|
||||
|
||||
|
||||
# --- replay settle-before-click target ---------------------------------------
|
||||
def test_replay_settle_target_for_click_by_name():
|
||||
assert sk.replay_settle_target(
|
||||
{"tool": "BrowserClickByName", "params": {"role": "button", "name": "Send"}}) == "Send"
|
||||
# other tools have no settle target
|
||||
assert sk.replay_settle_target(
|
||||
{"tool": "BrowserType", "params": {"selector": "#m", "text": "hi"}}) is None
|
||||
assert sk.replay_settle_target(
|
||||
{"tool": "BrowserNavigate", "params": {"url": "https://x.com"}}) is None
|
||||
# a too-long/blob name is not a useful settle target
|
||||
assert sk.replay_settle_target(
|
||||
{"tool": "BrowserClickByName", "params": {"name": "x" * 80}}) is None
|
||||
assert sk.replay_settle_target(
|
||||
{"tool": "BrowserClickByName", "params": {"name": ""}}) is None
|
||||
|
||||
Reference in New Issue
Block a user