[eric] browser: settle on the click target before a replay step so a recorded click never fires before paint

This commit is contained in:
ciregenz
2026-06-06 05:01:25 -07:00
parent 4e320033ad
commit d4e4327f78
3 changed files with 38 additions and 2 deletions
+9 -2
View File
@@ -631,8 +631,15 @@ async def run_browser_agent(
return None
async def _exec_step(step: dict) -> dict | None:
"""One replay step; an off-screen click gets one scroll-and-retry
(recorded elements often sit below the fold on a fresh page)."""
"""One replay step; settle on the click target first so a recorded
click never fires before the page paints it (the premature-click miss
that quarantined skills), then an off-screen click gets one
scroll-and-retry (recorded elements often sit below the fold)."""
_settle = browser_skills.replay_settle_target(step)
if _settle:
async def _w(t, p, b, tid):
return await _cancellable(execute_browser_tool(t, p, b, tid))
await browser_wait.smart_wait(_w, browser_id, tab_id, 1500, until=_settle)
res = await _cancellable(execute_browser_tool(step["tool"], step.get("params", {}), browser_id, tab_id))
if res is not None and "box model" in str(res.get("error", "")):
logger.info(f"[browser-skills] replay step off-screen ({step['tool']}); scrolling and retrying once")
@@ -328,6 +328,19 @@ def _prune_detours(steps: list[dict]) -> list[dict]:
return out
def replay_settle_target(step: dict) -> str | None:
"""What a replay should WAIT for before running this step, or None. For a
click-by-name, that's the name itself: a recorded click can fire before the
target paints on a fresh page (the premature-click miss that quarantined
skills), so settling on the name first makes replay robust without changing
what it does. Only short, literal names are useful settle targets."""
if step.get("tool") != "BrowserClickByName":
return None
name = (step.get("params", {}) or {}).get("name") or ""
name = name.strip()
return name if 0 < len(name) <= 60 else None
def first_unsafe_step(steps: list[dict]) -> tuple[int, str]:
"""Index of the first outward-facing step (click Send/Submit/Pay, type into
a composer), -1 if none. Reuses the batch replayer's wordlist so there is
+16
View File
@@ -697,3 +697,19 @@ def test_distill_keeps_navigate_that_was_acted_on():
steps = sk.distill_steps(log)
urls = [s["params"].get("url") for s in steps if s["tool"] == "BrowserNavigate"]
assert urls == ["https://x.com/form", "https://x.com/results"] # both kept
# --- replay settle-before-click target ---------------------------------------
def test_replay_settle_target_for_click_by_name():
assert sk.replay_settle_target(
{"tool": "BrowserClickByName", "params": {"role": "button", "name": "Send"}}) == "Send"
# other tools have no settle target
assert sk.replay_settle_target(
{"tool": "BrowserType", "params": {"selector": "#m", "text": "hi"}}) is None
assert sk.replay_settle_target(
{"tool": "BrowserNavigate", "params": {"url": "https://x.com"}}) is None
# a too-long/blob name is not a useful settle target
assert sk.replay_settle_target(
{"tool": "BrowserClickByName", "params": {"name": "x" * 80}}) is None
assert sk.replay_settle_target(
{"tool": "BrowserClickByName", "params": {"name": ""}}) is None