[eric] browser: settle-aware in-batch wait and a terminal read sub-action

backend/apps/agents/browser/browser_schema.py

@@ -214,10 +214,14 @@ BROWSER_TOOLS_SCHEMA = [
             "- click: { selector: str }\n"
             "- scroll: { direction?: 'up'|'down', amount?: int }\n"
             "- wait: { milliseconds?: int }\n"
-            "- navigate: { url: str }\n\n"
+            "- navigate: { url: str }\n"
+            "- list_interactives: { } (read the page; ONLY valid as the LAST sub-action)\n\n"
+            "End a batch with list_interactives to fold a click -> wait -> read into "
+            "ONE turn: e.g. click a button, wait for it to settle, then read the "
+            "result, all without a second round-trip.\n"
             "Example: { actions: [{type: 'click_index', params: {index: 1}}, "
-            "{type: 'wait', params: {milliseconds: 500}}, "
-            "{type: 'press_key', params: {key: 'ArrowRight'}}] }"
+            "{type: 'wait', params: {milliseconds: 4000}}, "
+            "{type: 'list_interactives', params: {}}] }"
         ),
         "input_schema": {
             "type": "object",
@@ -230,7 +234,7 @@ BROWSER_TOOLS_SCHEMA = [
                         "properties": {
                             "type": {
                                 "type": "string",
-                                "enum": ["click_index", "press_key", "type", "wait", "scroll", "navigate", "click"],
+                                "enum": ["click_index", "press_key", "type", "wait", "scroll", "navigate", "click", "list_interactives"],
                             },
                             "params": {"type": "object"},
                         },
@@ -523,8 +527,11 @@ SYSTEM_PROMPT = (
     "to be fast is FEWER TURNS, not faster tools. Once you can see the page, plan "
     "the whole remaining sequence and emit it in ONE BrowserBatch instead of one "
     "action per turn. A 3-step form (type, type, click Send) should be a single "
-    "batch turn, not three. Only break the batch when a later step genuinely "
-    "depends on reading what an earlier step produced.\n\n"
+    "batch turn, not three. And when you DO need to read after acting, put "
+    "list_interactives as the batch's LAST sub-action (click -> wait -> "
+    "list_interactives) so the click, the settle, and the read are one turn "
+    "instead of three. Only truly break the batch when a step needs to read "
+    "what an EARLIER step produced (a mid-sequence read, not a final one).\n\n"
 
     "## Batch known sequences with BrowserBatch\n"
     "When you have a known sequence of actions; typing then pressing Enter, "
@@ -535,8 +542,9 @@ SYSTEM_PROMPT = (
     "Use BrowserBatch when:\n"
     "- You're doing the same action repeatedly (5 swipes, 3 scrolls)\n"
     "- You have a deterministic flow (type query → press Enter → click first result)\n"
+    "- You act then need to see the result (click → wait → list_interactives as the last step)\n"
     "Don't use BrowserBatch when:\n"
-    "- You need to read the page state between actions\n"
+    "- You need to read the page state BETWEEN actions to decide the next one (a final read is fine)\n"
     "- You're uncertain about what comes next\n"
     "- An action might trigger an unexpected popup or navigation\n\n"
 

frontend/src/shared/browserCommandHandler.ts

@@ -2,6 +2,7 @@ import { getWebview, type BrowserWebview } from './browserRegistry';
 import { dashboardWs } from './ws/WebSocketManager';
 import { resolveInput } from './resolveUrl';
 import { rankAndCapInteractives, type RankItem } from './interactiveRanking';
+import { shouldStopWaiting, SETTLE_FLOOR_MS, SETTLE_POLL_MS, SETTLE_PROBE_JS } from './browserSettle';
 
 let initialized = false;
 
@@ -18,7 +19,15 @@ type ActivityListener = (browserId: string, activity: BrowserActivity | null) =>
 const activityMap = new Map<string, BrowserActivity>();
 const listeners = new Set<ActivityListener>();
 
+// A webview keeps churning for a beat after an action lands; capturing it into the
+// dashboard snapshot during that churn is what crashes the renderer (SharedImage
+// 'non-existent mailbox' -> V8 ToLocalChecked), so we hold "busy" this long past
+// the last command before letting the thumbnail capture run again.
+const BUSY_COOLDOWN_MS = 1500;
+let lastActivityAt = 0;
+
 function setActivity(browserId: string, activity: BrowserActivity | null) {
+  lastActivityAt = Date.now();
   if (activity) {
     activityMap.set(browserId, activity);
   } else {
@@ -31,6 +40,14 @@ export function getActivity(browserId: string): BrowserActivity | null {
   return activityMap.get(browserId) ?? null;
 }
 
+// True while an agent is actively driving any browser webview (a command is in
+// flight, or one finished within the cooldown). The dashboard thumbnail capture
+// checks this and skips rather than screenshot a live, churning webview.
+export function isAnyBrowserBusy(): boolean {
+  if (activityMap.size > 0) return true;
+  return Date.now() - lastActivityAt < BUSY_COOLDOWN_MS;
+}
+
 export function subscribeActivity(fn: ActivityListener): () => void {
   listeners.add(fn);
   return () => { listeners.delete(fn); };
@@ -67,23 +84,13 @@ async function handleScreenshot(wv: BrowserWebview): Promise<Record<string, any>
     try {
       const nativeImage = await wv.capturePage();
       if (!nativeImage.isEmpty()) {
-        // Send a downscaled JPEG, not a full-res PNG: on real pages JPEG cuts the
-        // wire/upload bytes ~10x (the model reads images by dimensions, so this is
-        // a network + memory win), and capping near 1280 actual px keeps text
-        // legible while trimming tokens a little. Native ops, sub-10ms.
-        // Electron-42 retina gotchas, verified empirically: toJPEG() on a raw
-        // scaleFactor-2 capture returns an EMPTY image, and resize({width}) emits
-        // `width` ACTUAL pixels at scaleFactor 1, EXCEPT resizing to the source's
-        // own logical width is a no-op that leaves it retina (and unencodable). So
-        // we ALWAYS resize to a distinct width to force a clean scaleFactor-1 image.
-        const TARGET_W = 1280;
-        const dpr = (typeof window !== 'undefined' && window.devicePixelRatio) || 1;
-        const { width: dipW } = nativeImage.getSize();
-        const backingW = Math.round(dipW * dpr);
-        let target = Math.min(TARGET_W, backingW);
-        if (target === dipW) target = Math.max(1, target - 1); // dodge the retina no-op
-        const base64 = nativeImage.resize({ width: target, quality: 'good' }).toJPEG(72).toString('base64');
-        return { image: base64, image_mime: 'image/jpeg', url: wv.getURL(), title: wv.getTitle() };
+        // Stable PNG capture. The resize()+toJPEG() variant was reverted: it's the
+        // prime suspect for the renderer "V8 Empty MaybeLocal" crash, NativeImage's
+        // JPEG codec returns an empty image on some retina captures, which is the
+        // shape of that native fault. A stable app beats a faster screenshot.
+        const dataUrl = nativeImage.toDataURL();
+        const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, '');
+        return { image: base64, url: wv.getURL(), title: wv.getTitle() };
       }
       lastErr = new Error('capturePage returned an empty image (frame not painted yet)');
     } catch (err: any) {
@@ -498,7 +505,7 @@ const MAX_BATCH_ACTIONS = 5;
 
 type SubActionType =
   | 'click_index' | 'press_key' | 'type' | 'wait'
-  | 'scroll' | 'navigate' | 'click';
+  | 'scroll' | 'navigate' | 'click' | 'list_interactives';
 
 const BATCH_DISPATCH: Record<SubActionType, (wv: BrowserWebview, p: Record<string, any>) => Promise<Record<string, any>>> = {
   click_index: handleClickIndex,
@@ -508,6 +515,8 @@ const BATCH_DISPATCH: Record<SubActionType, (wv: BrowserWebview, p: Record<strin
   scroll: handleScroll,
   navigate: handleNavigate,
   click: handleClick,
+  // Allowed as the LAST sub-action so a click->wait->read folds into one turn.
+  list_interactives: handleListInteractives,
 };
 
 async function handleBatch(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
@@ -643,9 +652,28 @@ async function handleScroll(wv: BrowserWebview, params: Record<string, any>): Pr
 
 async function handleWait(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
   const ms = Math.min(Math.max((params.milliseconds as number) || 1000, 100), 10000);
-  await new Promise((resolve) => setTimeout(resolve, ms));
+  const start = Date.now();
+  let settled = false;
+  let probeErrors = 0;
+  while (Date.now() - start < ms) {
+    const remaining = ms - (Date.now() - start);
+    await new Promise((resolve) => setTimeout(resolve, Math.min(SETTLE_POLL_MS, Math.max(0, remaining))));
+    const elapsed = Date.now() - start;
+    if (elapsed >= ms) break;
+    if (elapsed < SETTLE_FLOOR_MS) continue;
+    try {
+      const probe = JSON.parse(await wv.executeJavaScript(SETTLE_PROBE_JS));
+      probeErrors = 0;
+      if (shouldStopWaiting(probe.ready, probe.quiet || 0, elapsed)) { settled = true; break; }
+    } catch {
+      // Mid-navigation pages aren't evaluable yet; a few misses is normal, but a
+      // wedged tab shouldn't make us burn the whole cap, so bail after a short streak.
+      if (++probeErrors >= 3) break;
+    }
+  }
+  const waited = Date.now() - start;
   return {
-    text: `Waited ${ms}ms. Current URL: ${wv.getURL()}`,
+    text: `Waited ${waited}ms (${settled ? 'page settled' : 'reached cap'}). Current URL: ${wv.getURL()}`,
     url: wv.getURL(),
     title: wv.getTitle(),
   };

frontend/src/shared/browserSettle.test.ts

@@ -0,0 +1,29 @@
+// Run: node --test frontend/src/shared/browserSettle.test.ts
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { shouldStopWaiting, SETTLE_FLOOR_MS, SETTLE_QUIET_MS } from './browserSettle.ts';
+
+test('does not settle before the floor, even when fully quiet', () => {
+  assert.equal(shouldStopWaiting(true, 5000, SETTLE_FLOOR_MS - 1), false);
+});
+
+test('settles once past the floor with a complete doc and a long-quiet network', () => {
+  assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS, SETTLE_FLOOR_MS), true);
+  assert.equal(shouldStopWaiting(true, 2000, 1000), true);
+});
+
+test('does not settle while the document is still loading', () => {
+  assert.equal(shouldStopWaiting(false, 5000, 1000), false);
+});
+
+test('does not settle when the network was quiet for less than the window', () => {
+  assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS - 1, 1000), false);
+});
+
+test('a page that keeps fetching (quiet=0) rides to the cap', () => {
+  assert.equal(shouldStopWaiting(true, 0, 9000), false);
+});
+
+test('missing/NaN quiet is treated as not-quiet, not as settled', () => {
+  assert.equal(shouldStopWaiting(true, undefined as unknown as number, 1000), false);
+});

frontend/src/shared/browserSettle.ts

@@ -0,0 +1,30 @@
+// Renderer-side mirror of the backend's smart-wait (browser_wait.py): a `wait`
+// that returns the instant the page's network goes quiet instead of blind-sleeping
+// the full duration. A top-level BrowserWait already gets this on the backend, but
+// a `wait` INSIDE a BrowserBatch runs entirely here in the renderer, so without
+// this it would fall back to a dumb sleep and make batching slower than not batching.
+
+export const SETTLE_FLOOR_MS = 250; // never settle before this (a momentary gap isn't 'settled')
+export const SETTLE_QUIET_MS = 400; // network must be silent this long to count as settled
+export const SETTLE_POLL_MS = 150;
+
+// One probe: is the document complete, and how long since the last network resource
+// finished/started? Returns a JSON string. No interpolation, so it's injection-safe.
+export const SETTLE_PROBE_JS =
+  "(()=>{const n=performance.now();" +
+  "const es=performance.getEntriesByType('resource');let last=0;" +
+  "for(const e of es){const t=Math.max(e.responseEnd||0,e.startTime||0);if(t>last)last=t;}" +
+  "return JSON.stringify({ready:document.readyState==='complete',quiet:Math.round(n-last)});})()";
+
+// Pure decision: stop waiting once we're past the floor AND the document is complete
+// AND the network has been quiet for the settle window.
+export function shouldStopWaiting(
+  ready: boolean,
+  quietMs: number,
+  elapsedMs: number,
+  floorMs = SETTLE_FLOOR_MS,
+  quietWindowMs = SETTLE_QUIET_MS,
+): boolean {
+  if (elapsedMs < floorMs) return false;
+  return !!ready && (quietMs || 0) >= quietWindowMs;
+}