[eric] browser: end the wait on the target or DOM-settle, not network-idle, so it stops waiting blind

backend/apps/agents/browser/browser_agent.py

@@ -987,6 +987,7 @@ async def run_browser_agent(
                         return await _cancellable(execute_browser_tool(tool, params, bid, tid))
                     result = await browser_wait.smart_wait(
                         _wait_exec, browser_id, tab_id, tu.input.get("milliseconds"),
+                        until=(tu.input.get("until") or ""),
                     )
                 else:
                     result = await _cancellable(execute_browser_tool(

backend/apps/agents/browser/browser_schema.py

@@ -289,18 +289,29 @@ BROWSER_TOOLS_SCHEMA = [
     {
         "name": "BrowserWait",
         "description": (
-            "Wait for the page to settle after navigation or an action that loads "
-            "async content. This is SMART: it returns as soon as the page's network "
-            "goes quiet, so the duration you give is just an upper bound, not a fixed "
-            "sleep. Pass a generous cap (e.g. 4000) without worrying about wasted "
-            "time; you usually get control back in a few hundred ms. Min 100, max 10000."
+            "Wait for the page to be READY after navigation or an action. This is SMART: "
+            "it returns as soon as the page settles visually (its DOM stops changing), so "
+            "the duration is just an upper bound, not a fixed sleep, pass a generous cap "
+            "(e.g. 4000) without worrying about wasted time. Best of all, pass `until` with "
+            "the thing you expect to appear (a button label, result text, the compose box) "
+            "and it returns the INSTANT that shows up, so you wait for what you actually "
+            "need instead of guessing. Min 100, max 10000."
         ),
         "input_schema": {
             "type": "object",
             "properties": {
                 "milliseconds": {
                     "type": "number",
-                    "description": "Duration to wait in milliseconds. Defaults to 1000.",
+                    "description": "Upper-bound wait in milliseconds. Defaults to 1000.",
+                },
+                "until": {
+                    "type": "string",
+                    "description": (
+                        "Optional. A specific button label, visible text, or CSS selector "
+                        "you expect to appear (e.g. 'Haik Decie', 'Write a message', "
+                        "'button[type=submit]'). The wait ends the moment it's present and "
+                        "visible. Be specific, not a generic word like 'Message'."
+                    ),
                 },
             },
             "required": [],
@@ -584,7 +595,9 @@ SYSTEM_PROMPT = (
     "- Do NOT screenshot after every single action. Screenshot ONLY when you genuinely "
     "don't know the page state (start of task, after navigation, after a failure).\n"
     "- Don't BrowserWait if what you need is already on screen; just act (the wait is for "
-    "content that hasn't loaded yet, not a reflex after every action).\n"
+    "content that hasn't loaded yet, not a reflex after every action). When you DO wait, "
+    "pass `until` with the specific thing you expect (a name, the exact button label, the "
+    "compose box) so it returns the instant that appears instead of waiting blind.\n"
     "- When scrolling, stop as soon as BrowserScroll reports atTop/atBottom or a 0 delta; "
     "don't loop past the end.\n"
     "- Do NOT call BrowserGetElements on the entire body if you already know roughly "

backend/apps/agents/browser/browser_wait.py

@@ -28,15 +28,27 @@ import time
 
 logger = logging.getLogger(__name__)
 
-# One probe: is the document complete, and how long since the last network
-# resource finished/started? Returns a JSON string (BrowserEvaluate hands back
-# string results verbatim).
-PROBE_JS = (
-    "(()=>{const n=performance.now();"
-    "const es=performance.getEntriesByType('resource');let last=0;"
-    "for(const e of es){const t=Math.max(e.responseEnd||0,e.startTime||0);if(t>last)last=t;}"
-    "return JSON.stringify({ready:document.readyState==='complete',quiet:Math.round(n-last)});})()"
-)
+# One probe, built per wait so it can also look for the agent's target. Returns:
+#   ready  - document.readyState === 'complete'
+#   quiet  - ms since the last network resource (the old network-idle signal)
+#   elems  - element count; the loop watches it stop changing = DOM/visual settle
+#   found  - the agent's `until` target is present + visible (visible text or selector)
+# `until` is JSON-encoded into a string literal, so it's data, never executable.
+def _probe_js(until: str) -> str:
+    spec = json.dumps(until or "")
+    return (
+        "(()=>{const n=performance.now();"
+        "const es=performance.getEntriesByType('resource');let last=0;"
+        "for(const e of es){const t=Math.max(e.responseEnd||0,e.startTime||0);if(t>last)last=t;}"
+        f"let found=false;const spec={spec};"
+        "if(spec){try{const low=spec.toLowerCase();"
+        "let el=[...document.querySelectorAll('button,a,[role],input,textarea,[contenteditable],[aria-label],h1,h2')]"
+        ".find(e=>((e.innerText||e.value||e.getAttribute('aria-label')||'')+'').toLowerCase().includes(low));"
+        "if(!el){try{el=document.querySelector(spec);}catch(_){}}"
+        "if(el){const r=el.getBoundingClientRect();found=r.width>0&&r.height>0;}}catch(_){}}"
+        "return JSON.stringify({ready:document.readyState==='complete',"
+        "quiet:Math.round(n-last),elems:document.getElementsByTagName('*').length,found});})()"
+    )
 
 _QUIET_WINDOW_MS = 400   # network must be silent this long to count as settled
 _FLOOR_MS = 250          # never return before this (a momentary gap isn't 'settled')
@@ -51,30 +63,43 @@ _PROBE_TIMEOUT_S = 2.5
 _MAX_PROBE_TIMEOUTS = 3
 
 
-def decide_stop(ready, quiet_ms, elapsed_ms,
-                floor_ms=_FLOOR_MS, quiet_window_ms=_QUIET_WINDOW_MS) -> bool:
-    """Pure decision: stop waiting once we're past the floor AND the document is
-    complete AND the network has been quiet for the settle window."""
+def decide_stop(ready, quiet_ms, dom_stable_ms, found, elapsed_ms,
+                floor_ms=_FLOOR_MS, settle_window_ms=_QUIET_WINDOW_MS) -> bool:
+    """Pure decision. Stop the INSTANT the agent's target is present (no floor, it's
+    exactly what we were waiting for). Otherwise, past the floor and once the document
+    is complete, stop as soon as it has gone quiet by EITHER the network OR the DOM
+    settling. Watching DOM-settle as well as network is what stops beacon-heavy SPAs
+    (LinkedIn, Gmail) riding to the cap while visually done: their network never idles,
+    but their DOM does."""
+    if found:
+        return True
     if elapsed_ms < floor_ms:
         return False
-    return bool(ready) and (quiet_ms or 0) >= quiet_window_ms
+    if not ready:
+        return False
+    return (quiet_ms or 0) >= settle_window_ms or (dom_stable_ms or 0) >= settle_window_ms
 
 
-async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *,
+async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *, until="",
                      poll_ms=_POLL_MS, floor_ms=_FLOOR_MS,
                      quiet_window_ms=_QUIET_WINDOW_MS,
                      probe_timeout_s=_PROBE_TIMEOUT_S) -> dict:
-    """Wait up to `max_ms`, returning early once the page settles. `execute_fn`
-    is an async (tool, params, browser_id, tab_id) -> result|None (None = the run
-    was cancelled). Never raises into the caller. If the page stops responding to
-    probes (hung tab), returns fast with hung=True so the caller can bail instead
-    of blocking on the underlying long command timeout."""
+    """Wait up to `max_ms`, returning early once the page is ready. `until` (optional)
+    is a label / visible text / selector the agent expects to appear; the wait ends the
+    INSTANT it's present, so the agent isn't waiting blind. `execute_fn` is an async
+    (tool, params, browser_id, tab_id) -> result|None (None = cancelled). Never raises.
+    If the page stops responding to probes (hung tab), returns fast with hung=True so the
+    caller can bail instead of inheriting the long command timeout."""
     max_ms = max(100, min(int(max_ms or 1000), 10000))
+    probe_js = _probe_js(until)
     start = time.monotonic()
     settled = False
+    found = False
     hung = False
     last_url = ""
     probe_timeouts = 0
+    last_elems = None
+    elems_changed_at = start  # DOM-settle clock: when the element count last changed
 
     def _elapsed():
         return (time.monotonic() - start) * 1000
@@ -89,7 +114,7 @@ async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *,
         # timeout error is a different problem, treated as 'keep waiting'.
         try:
             res = await asyncio.wait_for(
-                execute_fn("BrowserEvaluate", {"expression": PROBE_JS}, browser_id, tab_id),
+                execute_fn("BrowserEvaluate", {"expression": probe_js}, browser_id, tab_id),
                 timeout=probe_timeout_s,
             )
         except asyncio.TimeoutError:
@@ -111,13 +136,27 @@ async def smart_wait(execute_fn, browser_id, tab_id, max_ms, *,
             probe = json.loads(res.get("text") or "{}")
         except Exception:
             continue
-        if decide_stop(probe.get("ready"), probe.get("quiet", 0), _elapsed(),
-                       floor_ms=floor_ms, quiet_window_ms=quiet_window_ms):
+        elems = probe.get("elems")
+        if elems != last_elems:
+            last_elems = elems
+            elems_changed_at = time.monotonic()
+        dom_stable_ms = (time.monotonic() - elems_changed_at) * 1000
+        if decide_stop(probe.get("ready"), probe.get("quiet", 0), dom_stable_ms,
+                       probe.get("found"), _elapsed(),
+                       floor_ms=floor_ms, settle_window_ms=quiet_window_ms):
             settled = True
+            found = bool(probe.get("found"))
             break
 
     waited = round(_elapsed())
-    state = "page settled" if settled else ("page not responding" if hung else "reached cap")
+    if found:
+        state = "found target"
+    elif settled:
+        state = "page settled"
+    elif hung:
+        state = "page not responding"
+    else:
+        state = "reached cap"
     text = f"Waited {waited}ms ({state})."
     if hung:
         text += " The page or tab appears unresponsive."

backend/tests/test_browser_wait.py

@@ -20,26 +20,37 @@ from backend.apps.agents.browser import browser_wait as bw
 def test_decide_stop_waits_until_past_the_floor():
     # even a fully-settled page must not return before the floor (a momentary gap
     # between two requests would otherwise look 'settled')
-    assert bw.decide_stop(ready=True, quiet_ms=9999, elapsed_ms=100, floor_ms=250) is False
-    assert bw.decide_stop(ready=True, quiet_ms=9999, elapsed_ms=300, floor_ms=250) is True
+    assert bw.decide_stop(True, 9999, 0, False, 100, floor_ms=250) is False
+    assert bw.decide_stop(True, 9999, 0, False, 300, floor_ms=250) is True
 
 
 def test_decide_stop_needs_ready_and_quiet():
     # past floor, but document not complete -> keep waiting
-    assert bw.decide_stop(ready=False, quiet_ms=9999, elapsed_ms=500) is False
-    # past floor, ready, but network still active (quiet below the window) -> wait
-    assert bw.decide_stop(ready=True, quiet_ms=100, elapsed_ms=500, quiet_window_ms=400) is False
-    # past floor, ready, quiet long enough -> stop
-    assert bw.decide_stop(ready=True, quiet_ms=400, elapsed_ms=500, quiet_window_ms=400) is True
+    assert bw.decide_stop(False, 9999, 9999, False, 500) is False
+    # past floor, ready, but neither network nor DOM quiet long enough -> wait
+    assert bw.decide_stop(True, 100, 100, False, 500, settle_window_ms=400) is False
+    # past floor, ready, network quiet long enough -> stop
+    assert bw.decide_stop(True, 400, 0, False, 500, settle_window_ms=400) is True
 
 
-def test_decide_stop_handles_missing_quiet():
-    assert bw.decide_stop(ready=True, quiet_ms=None, elapsed_ms=500) is False
+def test_decide_stop_target_found_short_circuits_everything():
+    # the target is present -> stop NOW, even before the floor and with a busy network
+    assert bw.decide_stop(False, 0, 0, True, 10) is True
+
+
+def test_decide_stop_dom_settle_when_network_never_idles():
+    # beacon-heavy SPA: network never idle (quiet tiny) but the DOM has stopped -> stop
+    assert bw.decide_stop(True, 5, 400, False, 600, settle_window_ms=400) is True
+
+
+def test_decide_stop_handles_missing_signals():
+    assert bw.decide_stop(True, None, None, False, 500) is False
 
 
 # --- the async loop with a scripted probe -----------------------------------
-def _probe(ready, quiet):
-    return {"text": json.dumps({"ready": ready, "quiet": quiet}), "url": "https://x.com"}
+def _probe(ready, quiet, elems=1000, found=False):
+    return {"text": json.dumps({"ready": ready, "quiet": quiet, "elems": elems, "found": found}),
+            "url": "https://x.com"}
 
 
 class FakeExec:
@@ -88,6 +99,28 @@ async def test_rides_to_cap_when_page_never_settles():
     assert "reached cap" in out["text"]
 
 
+@pytest.mark.asyncio
+async def test_settles_on_dom_stable_when_network_never_idles():
+    # the LinkedIn case: network always busy (quiet tiny) but the DOM count is
+    # constant -> DOM-settle fires instead of riding to the cap
+    ex = FakeExec([_probe(True, 5, elems=500)])
+    out = await bw.smart_wait(ex, "b", "", 3000, poll_ms=20, floor_ms=20, quiet_window_ms=200)
+    assert out["settled"] is True and out["waited_ms"] < 3000
+    assert "page settled" in out["text"]
+
+
+@pytest.mark.asyncio
+async def test_returns_the_instant_target_is_found():
+    # network busy AND DOM churning, but the agent's target appears on probe 2 ->
+    # stop immediately, bypassing even the floor
+    ex = FakeExec([_probe(False, 5, elems=100, found=False),
+                   _probe(False, 5, elems=200, found=True)])
+    out = await bw.smart_wait(ex, "b", "", 5000, until="Send",
+                              poll_ms=20, floor_ms=800, quiet_window_ms=999)
+    assert out["settled"] is True and "found target" in out["text"]
+    assert out["waited_ms"] < 800  # bypassed the floor because the target was there
+
+
 @pytest.mark.asyncio
 async def test_never_returns_before_the_floor():
     # settled from the very first probe, but the floor must still be respected

frontend/src/shared/browserCommandHandler.ts

@@ -2,7 +2,7 @@ import { getWebview, type BrowserWebview } from './browserRegistry';
 import { dashboardWs } from './ws/WebSocketManager';
 import { resolveInput } from './resolveUrl';
 import { rankAndCapInteractives, type RankItem } from './interactiveRanking';
-import { shouldStopWaiting, SETTLE_FLOOR_MS, SETTLE_POLL_MS, SETTLE_PROBE_JS } from './browserSettle';
+import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettle';
 
 let initialized = false;
 
@@ -688,19 +688,27 @@ async function handleScroll(wv: BrowserWebview, params: Record<string, any>): Pr
 
 async function handleWait(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
   const ms = Math.min(Math.max((params.milliseconds as number) || 1000, 100), 10000);
+  const until = typeof params.until === 'string' ? params.until : '';
+  const probeJs = settleProbeJs(until);
   const start = Date.now();
   let settled = false;
+  let found = false;
   let probeErrors = 0;
+  let lastElems: number | null = null;
+  let elemsChangedAt = start; // DOM-settle clock
   while (Date.now() - start < ms) {
     const remaining = ms - (Date.now() - start);
     await new Promise((resolve) => setTimeout(resolve, Math.min(SETTLE_POLL_MS, Math.max(0, remaining))));
     const elapsed = Date.now() - start;
     if (elapsed >= ms) break;
-    if (elapsed < SETTLE_FLOOR_MS) continue;
     try {
-      const probe = JSON.parse(await wv.executeJavaScript(SETTLE_PROBE_JS));
+      const probe = JSON.parse(await wv.executeJavaScript(probeJs));
       probeErrors = 0;
-      if (shouldStopWaiting(probe.ready, probe.quiet || 0, elapsed)) { settled = true; break; }
+      if (probe.elems !== lastElems) { lastElems = probe.elems; elemsChangedAt = Date.now(); }
+      const domStable = Date.now() - elemsChangedAt;
+      if (shouldStopWaiting(probe.ready, probe.quiet || 0, domStable, !!probe.found, elapsed)) {
+        settled = true; found = !!probe.found; break;
+      }
     } catch {
       // Mid-navigation pages aren't evaluable yet; a few misses is normal, but a
       // wedged tab shouldn't make us burn the whole cap, so bail after a short streak.
@@ -708,8 +716,9 @@ async function handleWait(wv: BrowserWebview, params: Record<string, any>): Prom
     }
   }
   const waited = Date.now() - start;
+  const state = found ? 'found target' : settled ? 'page settled' : 'reached cap';
   return {
-    text: `Waited ${waited}ms (${settled ? 'page settled' : 'reached cap'}). Current URL: ${wv.getURL()}`,
+    text: `Waited ${waited}ms (${state}). Current URL: ${wv.getURL()}`,
     url: wv.getURL(),
     title: wv.getTitle(),
   };

frontend/src/shared/browserSettle.test.ts

@@ -4,26 +4,32 @@ import assert from 'node:assert/strict';
 import { shouldStopWaiting, SETTLE_FLOOR_MS, SETTLE_QUIET_MS } from './browserSettle.ts';
 
 test('does not settle before the floor, even when fully quiet', () => {
-  assert.equal(shouldStopWaiting(true, 5000, SETTLE_FLOOR_MS - 1), false);
+  assert.equal(shouldStopWaiting(true, 5000, 5000, false, SETTLE_FLOOR_MS - 1), false);
 });
 
-test('settles once past the floor with a complete doc and a long-quiet network', () => {
-  assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS, SETTLE_FLOOR_MS), true);
-  assert.equal(shouldStopWaiting(true, 2000, 1000), true);
+test('settles past the floor when the network is quiet', () => {
+  assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS, 0, false, SETTLE_FLOOR_MS), true);
+});
+
+test('settles past the floor on DOM-stable even when the network never idles', () => {
+  assert.equal(shouldStopWaiting(true, 5, SETTLE_QUIET_MS, false, 1000), true);
+});
+
+test('target found short-circuits the floor and a busy network', () => {
+  assert.equal(shouldStopWaiting(false, 0, 0, true, 10), true);
 });
 
 test('does not settle while the document is still loading', () => {
-  assert.equal(shouldStopWaiting(false, 5000, 1000), false);
+  assert.equal(shouldStopWaiting(false, 5000, 5000, false, 1000), false);
 });
 
-test('does not settle when the network was quiet for less than the window', () => {
-  assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS - 1, 1000), false);
+test('does not settle when neither network nor DOM has been quiet long enough', () => {
+  assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS - 1, SETTLE_QUIET_MS - 1, false, 1000), false);
 });
 
-test('a page that keeps fetching (quiet=0) rides to the cap', () => {
-  assert.equal(shouldStopWaiting(true, 0, 9000), false);
-});
-
-test('missing/NaN quiet is treated as not-quiet, not as settled', () => {
-  assert.equal(shouldStopWaiting(true, undefined as unknown as number, 1000), false);
+test('missing signals are treated as not-quiet, not as settled', () => {
+  assert.equal(
+    shouldStopWaiting(true, undefined as unknown as number, undefined as unknown as number, false, 1000),
+    false,
+  );
 });

frontend/src/shared/browserSettle.ts

@@ -1,30 +1,50 @@
 // Renderer-side mirror of the backend's smart-wait (browser_wait.py): a `wait`
-// that returns the instant the page's network goes quiet instead of blind-sleeping
-// the full duration. A top-level BrowserWait already gets this on the backend, but
-// a `wait` INSIDE a BrowserBatch runs entirely here in the renderer, so without
-// this it would fall back to a dumb sleep and make batching slower than not batching.
+// that returns the instant the page is READY instead of blind-sleeping. A top-level
+// BrowserWait gets this on the backend, but a `wait` INSIDE a BrowserBatch runs
+// entirely here, so without this it would fall back to a dumb sleep.
+//
+// Two readiness signals plus a target: settle when the agent's `until` target is
+// present (no waiting blind), else when the page goes quiet by EITHER the network
+// OR the DOM settling (beacon-heavy SPAs never idle the network but their DOM does).
 
-export const SETTLE_FLOOR_MS = 250; // never settle before this (a momentary gap isn't 'settled')
-export const SETTLE_QUIET_MS = 400; // network must be silent this long to count as settled
+export const SETTLE_FLOOR_MS = 250; // never settle before this unless the target is already there
+export const SETTLE_QUIET_MS = 400; // network OR DOM must be quiet this long to count as settled
 export const SETTLE_POLL_MS = 150;
 
-// One probe: is the document complete, and how long since the last network resource
-// finished/started? Returns a JSON string. No interpolation, so it's injection-safe.
-export const SETTLE_PROBE_JS =
-  "(()=>{const n=performance.now();" +
-  "const es=performance.getEntriesByType('resource');let last=0;" +
-  "for(const e of es){const t=Math.max(e.responseEnd||0,e.startTime||0);if(t>last)last=t;}" +
-  "return JSON.stringify({ready:document.readyState==='complete',quiet:Math.round(n-last)});})()";
+// Probe built per wait so it can also look for the agent's target. Returns ready +
+// quiet (network-idle ms) + elems (element count; the caller watches it stop changing
+// = DOM settle) + found (the `until` target is present + visible). `until` is JSON-
+// encoded into a string literal, so it is data, never executable.
+export function settleProbeJs(until: string): string {
+  const spec = JSON.stringify(until || '');
+  return (
+    '(()=>{const n=performance.now();' +
+    "const es=performance.getEntriesByType('resource');let last=0;" +
+    'for(const e of es){const t=Math.max(e.responseEnd||0,e.startTime||0);if(t>last)last=t;}' +
+    'let found=false;const spec=' + spec + ';' +
+    'if(spec){try{const low=spec.toLowerCase();' +
+    "let el=[...document.querySelectorAll('button,a,[role],input,textarea,[contenteditable],[aria-label],h1,h2')]" +
+    ".find(e=>((e.innerText||e.value||e.getAttribute('aria-label')||'')+'').toLowerCase().includes(low));" +
+    'if(!el){try{el=document.querySelector(spec);}catch(_){}}' +
+    'if(el){const r=el.getBoundingClientRect();found=r.width>0&&r.height>0;}}catch(_){}}' +
+    "return JSON.stringify({ready:document.readyState==='complete'," +
+    'quiet:Math.round(n-last),elems:document.getElementsByTagName(\'*\').length,found});})()'
+  );
+}
 
-// Pure decision: stop waiting once we're past the floor AND the document is complete
-// AND the network has been quiet for the settle window.
+// Pure decision. Stop the instant the target is present; otherwise, past the floor and
+// once the document is complete, stop as soon as it's quiet by EITHER network OR DOM.
 export function shouldStopWaiting(
   ready: boolean,
   quietMs: number,
+  domStableMs: number,
+  found: boolean,
   elapsedMs: number,
   floorMs = SETTLE_FLOOR_MS,
-  quietWindowMs = SETTLE_QUIET_MS,
+  windowMs = SETTLE_QUIET_MS,
 ): boolean {
+  if (found) return true;
   if (elapsedMs < floorMs) return false;
-  return !!ready && (quietMs || 0) >= quietWindowMs;
+  if (!ready) return false;
+  return (quietMs || 0) >= windowMs || (domStableMs || 0) >= windowMs;
 }