diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 855bf61d..7f63a314 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -987,6 +987,7 @@ async def run_browser_agent( return await _cancellable(execute_browser_tool(tool, params, bid, tid)) result = await browser_wait.smart_wait( _wait_exec, browser_id, tab_id, tu.input.get("milliseconds"), + until=(tu.input.get("until") or ""), ) else: result = await _cancellable(execute_browser_tool( diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index d439ad3c..b4f08763 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -289,18 +289,29 @@ BROWSER_TOOLS_SCHEMA = [ { "name": "BrowserWait", "description": ( - "Wait for the page to settle after navigation or an action that loads " - "async content. This is SMART: it returns as soon as the page's network " - "goes quiet, so the duration you give is just an upper bound, not a fixed " - "sleep. Pass a generous cap (e.g. 4000) without worrying about wasted " - "time; you usually get control back in a few hundred ms. Min 100, max 10000." + "Wait for the page to be READY after navigation or an action. This is SMART: " + "it returns as soon as the page settles visually (its DOM stops changing), so " + "the duration is just an upper bound, not a fixed sleep, pass a generous cap " + "(e.g. 4000) without worrying about wasted time. Best of all, pass `until` with " + "the thing you expect to appear (a button label, result text, the compose box) " + "and it returns the INSTANT that shows up, so you wait for what you actually " + "need instead of guessing. Min 100, max 10000." ), "input_schema": { "type": "object", "properties": { "milliseconds": { "type": "number", - "description": "Duration to wait in milliseconds. Defaults to 1000.", + "description": "Upper-bound wait in milliseconds. Defaults to 1000.", + }, + "until": { + "type": "string", + "description": ( + "Optional. A specific button label, visible text, or CSS selector " + "you expect to appear (e.g. 'Haik Decie', 'Write a message', " + "'button[type=submit]'). The wait ends the moment it's present and " + "visible. Be specific, not a generic word like 'Message'." + ), }, }, "required": [], @@ -584,7 +595,9 @@ SYSTEM_PROMPT = ( "- Do NOT screenshot after every single action. Screenshot ONLY when you genuinely " "don't know the page state (start of task, after navigation, after a failure).\n" "- Don't BrowserWait if what you need is already on screen; just act (the wait is for " - "content that hasn't loaded yet, not a reflex after every action).\n" + "content that hasn't loaded yet, not a reflex after every action). When you DO wait, " + "pass `until` with the specific thing you expect (a name, the exact button label, the " + "compose box) so it returns the instant that appears instead of waiting blind.\n" "- When scrolling, stop as soon as BrowserScroll reports atTop/atBottom or a 0 delta; " "don't loop past the end.\n" "- Do NOT call BrowserGetElements on the entire body if you already know roughly " diff --git a/backend/apps/agents/browser/browser_wait.py b/backend/apps/agents/browser/browser_wait.py index 46f2527e..8288c50c 100644 --- a/backend/apps/agents/browser/browser_wait.py +++ b/backend/apps/agents/browser/browser_wait.py @@ -28,15 +28,27 @@ import time logger = logging.getLogger(__name__) -# One probe: is the document complete, and how long since the last network -# resource finished/started? Returns a JSON string (BrowserEvaluate hands back -# string results verbatim). -PROBE_JS = ( - "(()=>{const n=performance.now();" - "const es=performance.getEntriesByType('resource');let last=0;" - "for(const e of es){const t=Math.max(e.responseEnd||0,e.startTime||0);if(t>last)last=t;}" - "return JSON.stringify({ready:document.readyState==='complete',quiet:Math.round(n-last)});})()" -) +# One probe, built per wait so it can also look for the agent's target. Returns: +# ready - document.readyState === 'complete' +# quiet - ms since the last network resource (the old network-idle signal) +# elems - element count; the loop watches it stop changing = DOM/visual settle +# found - the agent's `until` target is present + visible (visible text or selector) +# `until` is JSON-encoded into a string literal, so it's data, never executable. +def _probe_js(until: str) -> str: + spec = json.dumps(until or "") + return ( + "(()=>{const n=performance.now();" + "const es=performance.getEntriesByType('resource');let last=0;" + "for(const e of es){const t=Math.max(e.responseEnd||0,e.startTime||0);if(t>last)last=t;}" + f"let found=false;const spec={spec};" + "if(spec){try{const low=spec.toLowerCase();" + "let el=[...document.querySelectorAll('button,a,[role],input,textarea,[contenteditable],[aria-label],h1,h2')]" + ".find(e=>((e.innerText||e.value||e.getAttribute('aria-label')||'')+'').toLowerCase().includes(low));" + "if(!el){try{el=document.querySelector(spec);}catch(_){}}" + "if(el){const r=el.getBoundingClientRect();found=r.width>0&&r.height>0;}}catch(_){}}" + "return JSON.stringify({ready:document.readyState==='complete'," + "quiet:Math.round(n-last),elems:document.getElementsByTagName('*').length,found});})()" + ) _QUIET_WINDOW_MS = 400 # network must be silent this long to count as settled _FLOOR_MS = 250 # never return before this (a momentary gap isn't 'settled') @@ -51,30 +63,43 @@ _PROBE_TIMEOUT_S = 2.5 _MAX_PROBE_TIMEOUTS = 3 -def decide_stop(ready, quiet_ms, elapsed_ms, - floor_ms=_FLOOR_MS, quiet_window_ms=_QUIET_WINDOW_MS) -> bool: - """Pure decision: stop waiting once we're past the floor AND the document is - complete AND the network has been quiet for the settle window.""" +def decide_stop(ready, quiet_ms, dom_stable_ms, found, elapsed_ms, + floor_ms=_FLOOR_MS, settle_window_ms=_QUIET_WINDOW_MS) -> bool: + """Pure decision. Stop the INSTANT the agent's target is present (no floor, it's + exactly what we were waiting for). Otherwise, past the floor and once the document + is complete, stop as soon as it has gone quiet by EITHER the network OR the DOM + settling. Watching DOM-settle as well as network is what stops beacon-heavy SPAs + (LinkedIn, Gmail) riding to the cap while visually done: their network never idles, + but their DOM does.""" + if found: + return True if elapsed_ms < floor_ms: return False - return bool(ready) and (quiet_ms or 0) >= quiet_window_ms + if not ready: + return False + return (quiet_ms or 0) >= settle_window_ms or (dom_stable_ms or 0) >= settle_window_ms -async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *, +async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *, until="", poll_ms=_POLL_MS, floor_ms=_FLOOR_MS, quiet_window_ms=_QUIET_WINDOW_MS, probe_timeout_s=_PROBE_TIMEOUT_S) -> dict: - """Wait up to `max_ms`, returning early once the page settles. `execute_fn` - is an async (tool, params, browser_id, tab_id) -> result|None (None = the run - was cancelled). Never raises into the caller. If the page stops responding to - probes (hung tab), returns fast with hung=True so the caller can bail instead - of blocking on the underlying long command timeout.""" + """Wait up to `max_ms`, returning early once the page is ready. `until` (optional) + is a label / visible text / selector the agent expects to appear; the wait ends the + INSTANT it's present, so the agent isn't waiting blind. `execute_fn` is an async + (tool, params, browser_id, tab_id) -> result|None (None = cancelled). Never raises. + If the page stops responding to probes (hung tab), returns fast with hung=True so the + caller can bail instead of inheriting the long command timeout.""" max_ms = max(100, min(int(max_ms or 1000), 10000)) + probe_js = _probe_js(until) start = time.monotonic() settled = False + found = False hung = False last_url = "" probe_timeouts = 0 + last_elems = None + elems_changed_at = start # DOM-settle clock: when the element count last changed def _elapsed(): return (time.monotonic() - start) * 1000 @@ -89,7 +114,7 @@ async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *, # timeout error is a different problem, treated as 'keep waiting'. try: res = await asyncio.wait_for( - execute_fn("BrowserEvaluate", {"expression": PROBE_JS}, browser_id, tab_id), + execute_fn("BrowserEvaluate", {"expression": probe_js}, browser_id, tab_id), timeout=probe_timeout_s, ) except asyncio.TimeoutError: @@ -111,13 +136,27 @@ async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *, probe = json.loads(res.get("text") or "{}") except Exception: continue - if decide_stop(probe.get("ready"), probe.get("quiet", 0), _elapsed(), - floor_ms=floor_ms, quiet_window_ms=quiet_window_ms): + elems = probe.get("elems") + if elems != last_elems: + last_elems = elems + elems_changed_at = time.monotonic() + dom_stable_ms = (time.monotonic() - elems_changed_at) * 1000 + if decide_stop(probe.get("ready"), probe.get("quiet", 0), dom_stable_ms, + probe.get("found"), _elapsed(), + floor_ms=floor_ms, settle_window_ms=quiet_window_ms): settled = True + found = bool(probe.get("found")) break waited = round(_elapsed()) - state = "page settled" if settled else ("page not responding" if hung else "reached cap") + if found: + state = "found target" + elif settled: + state = "page settled" + elif hung: + state = "page not responding" + else: + state = "reached cap" text = f"Waited {waited}ms ({state})." if hung: text += " The page or tab appears unresponsive." diff --git a/backend/tests/test_browser_wait.py b/backend/tests/test_browser_wait.py index 1ff26ee4..e51f0c73 100644 --- a/backend/tests/test_browser_wait.py +++ b/backend/tests/test_browser_wait.py @@ -20,26 +20,37 @@ from backend.apps.agents.browser import browser_wait as bw def test_decide_stop_waits_until_past_the_floor(): # even a fully-settled page must not return before the floor (a momentary gap # between two requests would otherwise look 'settled') - assert bw.decide_stop(ready=True, quiet_ms=9999, elapsed_ms=100, floor_ms=250) is False - assert bw.decide_stop(ready=True, quiet_ms=9999, elapsed_ms=300, floor_ms=250) is True + assert bw.decide_stop(True, 9999, 0, False, 100, floor_ms=250) is False + assert bw.decide_stop(True, 9999, 0, False, 300, floor_ms=250) is True def test_decide_stop_needs_ready_and_quiet(): # past floor, but document not complete -> keep waiting - assert bw.decide_stop(ready=False, quiet_ms=9999, elapsed_ms=500) is False - # past floor, ready, but network still active (quiet below the window) -> wait - assert bw.decide_stop(ready=True, quiet_ms=100, elapsed_ms=500, quiet_window_ms=400) is False - # past floor, ready, quiet long enough -> stop - assert bw.decide_stop(ready=True, quiet_ms=400, elapsed_ms=500, quiet_window_ms=400) is True + assert bw.decide_stop(False, 9999, 9999, False, 500) is False + # past floor, ready, but neither network nor DOM quiet long enough -> wait + assert bw.decide_stop(True, 100, 100, False, 500, settle_window_ms=400) is False + # past floor, ready, network quiet long enough -> stop + assert bw.decide_stop(True, 400, 0, False, 500, settle_window_ms=400) is True -def test_decide_stop_handles_missing_quiet(): - assert bw.decide_stop(ready=True, quiet_ms=None, elapsed_ms=500) is False +def test_decide_stop_target_found_short_circuits_everything(): + # the target is present -> stop NOW, even before the floor and with a busy network + assert bw.decide_stop(False, 0, 0, True, 10) is True + + +def test_decide_stop_dom_settle_when_network_never_idles(): + # beacon-heavy SPA: network never idle (quiet tiny) but the DOM has stopped -> stop + assert bw.decide_stop(True, 5, 400, False, 600, settle_window_ms=400) is True + + +def test_decide_stop_handles_missing_signals(): + assert bw.decide_stop(True, None, None, False, 500) is False # --- the async loop with a scripted probe ----------------------------------- -def _probe(ready, quiet): - return {"text": json.dumps({"ready": ready, "quiet": quiet}), "url": "https://x.com"} +def _probe(ready, quiet, elems=1000, found=False): + return {"text": json.dumps({"ready": ready, "quiet": quiet, "elems": elems, "found": found}), + "url": "https://x.com"} class FakeExec: @@ -88,6 +99,28 @@ async def test_rides_to_cap_when_page_never_settles(): assert "reached cap" in out["text"] +@pytest.mark.asyncio +async def test_settles_on_dom_stable_when_network_never_idles(): + # the LinkedIn case: network always busy (quiet tiny) but the DOM count is + # constant -> DOM-settle fires instead of riding to the cap + ex = FakeExec([_probe(True, 5, elems=500)]) + out = await bw.smart_wait(ex, "b", "", 3000, poll_ms=20, floor_ms=20, quiet_window_ms=200) + assert out["settled"] is True and out["waited_ms"] < 3000 + assert "page settled" in out["text"] + + +@pytest.mark.asyncio +async def test_returns_the_instant_target_is_found(): + # network busy AND DOM churning, but the agent's target appears on probe 2 -> + # stop immediately, bypassing even the floor + ex = FakeExec([_probe(False, 5, elems=100, found=False), + _probe(False, 5, elems=200, found=True)]) + out = await bw.smart_wait(ex, "b", "", 5000, until="Send", + poll_ms=20, floor_ms=800, quiet_window_ms=999) + assert out["settled"] is True and "found target" in out["text"] + assert out["waited_ms"] < 800 # bypassed the floor because the target was there + + @pytest.mark.asyncio async def test_never_returns_before_the_floor(): # settled from the very first probe, but the floor must still be respected diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 53073018..4b99f569 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -2,7 +2,7 @@ import { getWebview, type BrowserWebview } from './browserRegistry'; import { dashboardWs } from './ws/WebSocketManager'; import { resolveInput } from './resolveUrl'; import { rankAndCapInteractives, type RankItem } from './interactiveRanking'; -import { shouldStopWaiting, SETTLE_FLOOR_MS, SETTLE_POLL_MS, SETTLE_PROBE_JS } from './browserSettle'; +import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettle'; let initialized = false; @@ -688,19 +688,27 @@ async function handleScroll(wv: BrowserWebview, params: Record): Pr async function handleWait(wv: BrowserWebview, params: Record): Promise> { const ms = Math.min(Math.max((params.milliseconds as number) || 1000, 100), 10000); + const until = typeof params.until === 'string' ? params.until : ''; + const probeJs = settleProbeJs(until); const start = Date.now(); let settled = false; + let found = false; let probeErrors = 0; + let lastElems: number | null = null; + let elemsChangedAt = start; // DOM-settle clock while (Date.now() - start < ms) { const remaining = ms - (Date.now() - start); await new Promise((resolve) => setTimeout(resolve, Math.min(SETTLE_POLL_MS, Math.max(0, remaining)))); const elapsed = Date.now() - start; if (elapsed >= ms) break; - if (elapsed < SETTLE_FLOOR_MS) continue; try { - const probe = JSON.parse(await wv.executeJavaScript(SETTLE_PROBE_JS)); + const probe = JSON.parse(await wv.executeJavaScript(probeJs)); probeErrors = 0; - if (shouldStopWaiting(probe.ready, probe.quiet || 0, elapsed)) { settled = true; break; } + if (probe.elems !== lastElems) { lastElems = probe.elems; elemsChangedAt = Date.now(); } + const domStable = Date.now() - elemsChangedAt; + if (shouldStopWaiting(probe.ready, probe.quiet || 0, domStable, !!probe.found, elapsed)) { + settled = true; found = !!probe.found; break; + } } catch { // Mid-navigation pages aren't evaluable yet; a few misses is normal, but a // wedged tab shouldn't make us burn the whole cap, so bail after a short streak. @@ -708,8 +716,9 @@ async function handleWait(wv: BrowserWebview, params: Record): Prom } } const waited = Date.now() - start; + const state = found ? 'found target' : settled ? 'page settled' : 'reached cap'; return { - text: `Waited ${waited}ms (${settled ? 'page settled' : 'reached cap'}). Current URL: ${wv.getURL()}`, + text: `Waited ${waited}ms (${state}). Current URL: ${wv.getURL()}`, url: wv.getURL(), title: wv.getTitle(), }; diff --git a/frontend/src/shared/browserSettle.test.ts b/frontend/src/shared/browserSettle.test.ts index 61c56a7e..decc6702 100644 --- a/frontend/src/shared/browserSettle.test.ts +++ b/frontend/src/shared/browserSettle.test.ts @@ -4,26 +4,32 @@ import assert from 'node:assert/strict'; import { shouldStopWaiting, SETTLE_FLOOR_MS, SETTLE_QUIET_MS } from './browserSettle.ts'; test('does not settle before the floor, even when fully quiet', () => { - assert.equal(shouldStopWaiting(true, 5000, SETTLE_FLOOR_MS - 1), false); + assert.equal(shouldStopWaiting(true, 5000, 5000, false, SETTLE_FLOOR_MS - 1), false); }); -test('settles once past the floor with a complete doc and a long-quiet network', () => { - assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS, SETTLE_FLOOR_MS), true); - assert.equal(shouldStopWaiting(true, 2000, 1000), true); +test('settles past the floor when the network is quiet', () => { + assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS, 0, false, SETTLE_FLOOR_MS), true); +}); + +test('settles past the floor on DOM-stable even when the network never idles', () => { + assert.equal(shouldStopWaiting(true, 5, SETTLE_QUIET_MS, false, 1000), true); +}); + +test('target found short-circuits the floor and a busy network', () => { + assert.equal(shouldStopWaiting(false, 0, 0, true, 10), true); }); test('does not settle while the document is still loading', () => { - assert.equal(shouldStopWaiting(false, 5000, 1000), false); + assert.equal(shouldStopWaiting(false, 5000, 5000, false, 1000), false); }); -test('does not settle when the network was quiet for less than the window', () => { - assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS - 1, 1000), false); +test('does not settle when neither network nor DOM has been quiet long enough', () => { + assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS - 1, SETTLE_QUIET_MS - 1, false, 1000), false); }); -test('a page that keeps fetching (quiet=0) rides to the cap', () => { - assert.equal(shouldStopWaiting(true, 0, 9000), false); -}); - -test('missing/NaN quiet is treated as not-quiet, not as settled', () => { - assert.equal(shouldStopWaiting(true, undefined as unknown as number, 1000), false); +test('missing signals are treated as not-quiet, not as settled', () => { + assert.equal( + shouldStopWaiting(true, undefined as unknown as number, undefined as unknown as number, false, 1000), + false, + ); }); diff --git a/frontend/src/shared/browserSettle.ts b/frontend/src/shared/browserSettle.ts index d755f7e1..6c92e5d3 100644 --- a/frontend/src/shared/browserSettle.ts +++ b/frontend/src/shared/browserSettle.ts @@ -1,30 +1,50 @@ // Renderer-side mirror of the backend's smart-wait (browser_wait.py): a `wait` -// that returns the instant the page's network goes quiet instead of blind-sleeping -// the full duration. A top-level BrowserWait already gets this on the backend, but -// a `wait` INSIDE a BrowserBatch runs entirely here in the renderer, so without -// this it would fall back to a dumb sleep and make batching slower than not batching. +// that returns the instant the page is READY instead of blind-sleeping. A top-level +// BrowserWait gets this on the backend, but a `wait` INSIDE a BrowserBatch runs +// entirely here, so without this it would fall back to a dumb sleep. +// +// Two readiness signals plus a target: settle when the agent's `until` target is +// present (no waiting blind), else when the page goes quiet by EITHER the network +// OR the DOM settling (beacon-heavy SPAs never idle the network but their DOM does). -export const SETTLE_FLOOR_MS = 250; // never settle before this (a momentary gap isn't 'settled') -export const SETTLE_QUIET_MS = 400; // network must be silent this long to count as settled +export const SETTLE_FLOOR_MS = 250; // never settle before this unless the target is already there +export const SETTLE_QUIET_MS = 400; // network OR DOM must be quiet this long to count as settled export const SETTLE_POLL_MS = 150; -// One probe: is the document complete, and how long since the last network resource -// finished/started? Returns a JSON string. No interpolation, so it's injection-safe. -export const SETTLE_PROBE_JS = - "(()=>{const n=performance.now();" + - "const es=performance.getEntriesByType('resource');let last=0;" + - "for(const e of es){const t=Math.max(e.responseEnd||0,e.startTime||0);if(t>last)last=t;}" + - "return JSON.stringify({ready:document.readyState==='complete',quiet:Math.round(n-last)});})()"; +// Probe built per wait so it can also look for the agent's target. Returns ready + +// quiet (network-idle ms) + elems (element count; the caller watches it stop changing +// = DOM settle) + found (the `until` target is present + visible). `until` is JSON- +// encoded into a string literal, so it is data, never executable. +export function settleProbeJs(until: string): string { + const spec = JSON.stringify(until || ''); + return ( + '(()=>{const n=performance.now();' + + "const es=performance.getEntriesByType('resource');let last=0;" + + 'for(const e of es){const t=Math.max(e.responseEnd||0,e.startTime||0);if(t>last)last=t;}' + + 'let found=false;const spec=' + spec + ';' + + 'if(spec){try{const low=spec.toLowerCase();' + + "let el=[...document.querySelectorAll('button,a,[role],input,textarea,[contenteditable],[aria-label],h1,h2')]" + + ".find(e=>((e.innerText||e.value||e.getAttribute('aria-label')||'')+'').toLowerCase().includes(low));" + + 'if(!el){try{el=document.querySelector(spec);}catch(_){}}' + + 'if(el){const r=el.getBoundingClientRect();found=r.width>0&&r.height>0;}}catch(_){}}' + + "return JSON.stringify({ready:document.readyState==='complete'," + + 'quiet:Math.round(n-last),elems:document.getElementsByTagName(\'*\').length,found});})()' + ); +} -// Pure decision: stop waiting once we're past the floor AND the document is complete -// AND the network has been quiet for the settle window. +// Pure decision. Stop the instant the target is present; otherwise, past the floor and +// once the document is complete, stop as soon as it's quiet by EITHER network OR DOM. export function shouldStopWaiting( ready: boolean, quietMs: number, + domStableMs: number, + found: boolean, elapsedMs: number, floorMs = SETTLE_FLOOR_MS, - quietWindowMs = SETTLE_QUIET_MS, + windowMs = SETTLE_QUIET_MS, ): boolean { + if (found) return true; if (elapsedMs < floorMs) return false; - return !!ready && (quietMs || 0) >= quietWindowMs; + if (!ready) return false; + return (quietMs || 0) >= windowMs || (domStableMs || 0) >= windowMs; }