From 27686e270b330c6127b616f8a0af2bf34c924dbd Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 4 Jun 2026 04:04:44 -0700 Subject: [PATCH] [eric] browser: settle-aware in-batch wait and a terminal read sub-action --- backend/apps/agents/browser/browser_schema.py | 22 ++++-- frontend/src/shared/browserCommandHandler.ts | 68 +++++++++++++------ frontend/src/shared/browserSettle.test.ts | 29 ++++++++ frontend/src/shared/browserSettle.ts | 30 ++++++++ 4 files changed, 122 insertions(+), 27 deletions(-) create mode 100644 frontend/src/shared/browserSettle.test.ts create mode 100644 frontend/src/shared/browserSettle.ts diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index e04e747d..c6fdeafe 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -214,10 +214,14 @@ BROWSER_TOOLS_SCHEMA = [ "- click: { selector: str }\n" "- scroll: { direction?: 'up'|'down', amount?: int }\n" "- wait: { milliseconds?: int }\n" - "- navigate: { url: str }\n\n" + "- navigate: { url: str }\n" + "- list_interactives: { } (read the page; ONLY valid as the LAST sub-action)\n\n" + "End a batch with list_interactives to fold a click -> wait -> read into " + "ONE turn: e.g. click a button, wait for it to settle, then read the " + "result, all without a second round-trip.\n" "Example: { actions: [{type: 'click_index', params: {index: 1}}, " - "{type: 'wait', params: {milliseconds: 500}}, " - "{type: 'press_key', params: {key: 'ArrowRight'}}] }" + "{type: 'wait', params: {milliseconds: 4000}}, " + "{type: 'list_interactives', params: {}}] }" ), "input_schema": { "type": "object", @@ -230,7 +234,7 @@ BROWSER_TOOLS_SCHEMA = [ "properties": { "type": { "type": "string", - "enum": ["click_index", "press_key", "type", "wait", "scroll", "navigate", "click"], + "enum": ["click_index", "press_key", "type", "wait", "scroll", "navigate", "click", "list_interactives"], }, "params": {"type": "object"}, }, @@ -523,8 +527,11 @@ SYSTEM_PROMPT = ( "to be fast is FEWER TURNS, not faster tools. Once you can see the page, plan " "the whole remaining sequence and emit it in ONE BrowserBatch instead of one " "action per turn. A 3-step form (type, type, click Send) should be a single " - "batch turn, not three. Only break the batch when a later step genuinely " - "depends on reading what an earlier step produced.\n\n" + "batch turn, not three. And when you DO need to read after acting, put " + "list_interactives as the batch's LAST sub-action (click -> wait -> " + "list_interactives) so the click, the settle, and the read are one turn " + "instead of three. Only truly break the batch when a step needs to read " + "what an EARLIER step produced (a mid-sequence read, not a final one).\n\n" "## Batch known sequences with BrowserBatch\n" "When you have a known sequence of actions; typing then pressing Enter, " @@ -535,8 +542,9 @@ SYSTEM_PROMPT = ( "Use BrowserBatch when:\n" "- You're doing the same action repeatedly (5 swipes, 3 scrolls)\n" "- You have a deterministic flow (type query → press Enter → click first result)\n" + "- You act then need to see the result (click → wait → list_interactives as the last step)\n" "Don't use BrowserBatch when:\n" - "- You need to read the page state between actions\n" + "- You need to read the page state BETWEEN actions to decide the next one (a final read is fine)\n" "- You're uncertain about what comes next\n" "- An action might trigger an unexpected popup or navigation\n\n" diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 14bbc0bd..82312f0f 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -2,6 +2,7 @@ import { getWebview, type BrowserWebview } from './browserRegistry'; import { dashboardWs } from './ws/WebSocketManager'; import { resolveInput } from './resolveUrl'; import { rankAndCapInteractives, type RankItem } from './interactiveRanking'; +import { shouldStopWaiting, SETTLE_FLOOR_MS, SETTLE_POLL_MS, SETTLE_PROBE_JS } from './browserSettle'; let initialized = false; @@ -18,7 +19,15 @@ type ActivityListener = (browserId: string, activity: BrowserActivity | null) => const activityMap = new Map(); const listeners = new Set(); +// A webview keeps churning for a beat after an action lands; capturing it into the +// dashboard snapshot during that churn is what crashes the renderer (SharedImage +// 'non-existent mailbox' -> V8 ToLocalChecked), so we hold "busy" this long past +// the last command before letting the thumbnail capture run again. +const BUSY_COOLDOWN_MS = 1500; +let lastActivityAt = 0; + function setActivity(browserId: string, activity: BrowserActivity | null) { + lastActivityAt = Date.now(); if (activity) { activityMap.set(browserId, activity); } else { @@ -31,6 +40,14 @@ export function getActivity(browserId: string): BrowserActivity | null { return activityMap.get(browserId) ?? null; } +// True while an agent is actively driving any browser webview (a command is in +// flight, or one finished within the cooldown). The dashboard thumbnail capture +// checks this and skips rather than screenshot a live, churning webview. +export function isAnyBrowserBusy(): boolean { + if (activityMap.size > 0) return true; + return Date.now() - lastActivityAt < BUSY_COOLDOWN_MS; +} + export function subscribeActivity(fn: ActivityListener): () => void { listeners.add(fn); return () => { listeners.delete(fn); }; @@ -67,23 +84,13 @@ async function handleScreenshot(wv: BrowserWebview): Promise try { const nativeImage = await wv.capturePage(); if (!nativeImage.isEmpty()) { - // Send a downscaled JPEG, not a full-res PNG: on real pages JPEG cuts the - // wire/upload bytes ~10x (the model reads images by dimensions, so this is - // a network + memory win), and capping near 1280 actual px keeps text - // legible while trimming tokens a little. Native ops, sub-10ms. - // Electron-42 retina gotchas, verified empirically: toJPEG() on a raw - // scaleFactor-2 capture returns an EMPTY image, and resize({width}) emits - // `width` ACTUAL pixels at scaleFactor 1, EXCEPT resizing to the source's - // own logical width is a no-op that leaves it retina (and unencodable). So - // we ALWAYS resize to a distinct width to force a clean scaleFactor-1 image. - const TARGET_W = 1280; - const dpr = (typeof window !== 'undefined' && window.devicePixelRatio) || 1; - const { width: dipW } = nativeImage.getSize(); - const backingW = Math.round(dipW * dpr); - let target = Math.min(TARGET_W, backingW); - if (target === dipW) target = Math.max(1, target - 1); // dodge the retina no-op - const base64 = nativeImage.resize({ width: target, quality: 'good' }).toJPEG(72).toString('base64'); - return { image: base64, image_mime: 'image/jpeg', url: wv.getURL(), title: wv.getTitle() }; + // Stable PNG capture. The resize()+toJPEG() variant was reverted: it's the + // prime suspect for the renderer "V8 Empty MaybeLocal" crash, NativeImage's + // JPEG codec returns an empty image on some retina captures, which is the + // shape of that native fault. A stable app beats a faster screenshot. + const dataUrl = nativeImage.toDataURL(); + const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, ''); + return { image: base64, url: wv.getURL(), title: wv.getTitle() }; } lastErr = new Error('capturePage returned an empty image (frame not painted yet)'); } catch (err: any) { @@ -498,7 +505,7 @@ const MAX_BATCH_ACTIONS = 5; type SubActionType = | 'click_index' | 'press_key' | 'type' | 'wait' - | 'scroll' | 'navigate' | 'click'; + | 'scroll' | 'navigate' | 'click' | 'list_interactives'; const BATCH_DISPATCH: Record) => Promise>> = { click_index: handleClickIndex, @@ -508,6 +515,8 @@ const BATCH_DISPATCH: Recordwait->read folds into one turn. + list_interactives: handleListInteractives, }; async function handleBatch(wv: BrowserWebview, params: Record): Promise> { @@ -643,9 +652,28 @@ async function handleScroll(wv: BrowserWebview, params: Record): Pr async function handleWait(wv: BrowserWebview, params: Record): Promise> { const ms = Math.min(Math.max((params.milliseconds as number) || 1000, 100), 10000); - await new Promise((resolve) => setTimeout(resolve, ms)); + const start = Date.now(); + let settled = false; + let probeErrors = 0; + while (Date.now() - start < ms) { + const remaining = ms - (Date.now() - start); + await new Promise((resolve) => setTimeout(resolve, Math.min(SETTLE_POLL_MS, Math.max(0, remaining)))); + const elapsed = Date.now() - start; + if (elapsed >= ms) break; + if (elapsed < SETTLE_FLOOR_MS) continue; + try { + const probe = JSON.parse(await wv.executeJavaScript(SETTLE_PROBE_JS)); + probeErrors = 0; + if (shouldStopWaiting(probe.ready, probe.quiet || 0, elapsed)) { settled = true; break; } + } catch { + // Mid-navigation pages aren't evaluable yet; a few misses is normal, but a + // wedged tab shouldn't make us burn the whole cap, so bail after a short streak. + if (++probeErrors >= 3) break; + } + } + const waited = Date.now() - start; return { - text: `Waited ${ms}ms. Current URL: ${wv.getURL()}`, + text: `Waited ${waited}ms (${settled ? 'page settled' : 'reached cap'}). Current URL: ${wv.getURL()}`, url: wv.getURL(), title: wv.getTitle(), }; diff --git a/frontend/src/shared/browserSettle.test.ts b/frontend/src/shared/browserSettle.test.ts new file mode 100644 index 00000000..61c56a7e --- /dev/null +++ b/frontend/src/shared/browserSettle.test.ts @@ -0,0 +1,29 @@ +// Run: node --test frontend/src/shared/browserSettle.test.ts +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { shouldStopWaiting, SETTLE_FLOOR_MS, SETTLE_QUIET_MS } from './browserSettle.ts'; + +test('does not settle before the floor, even when fully quiet', () => { + assert.equal(shouldStopWaiting(true, 5000, SETTLE_FLOOR_MS - 1), false); +}); + +test('settles once past the floor with a complete doc and a long-quiet network', () => { + assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS, SETTLE_FLOOR_MS), true); + assert.equal(shouldStopWaiting(true, 2000, 1000), true); +}); + +test('does not settle while the document is still loading', () => { + assert.equal(shouldStopWaiting(false, 5000, 1000), false); +}); + +test('does not settle when the network was quiet for less than the window', () => { + assert.equal(shouldStopWaiting(true, SETTLE_QUIET_MS - 1, 1000), false); +}); + +test('a page that keeps fetching (quiet=0) rides to the cap', () => { + assert.equal(shouldStopWaiting(true, 0, 9000), false); +}); + +test('missing/NaN quiet is treated as not-quiet, not as settled', () => { + assert.equal(shouldStopWaiting(true, undefined as unknown as number, 1000), false); +}); diff --git a/frontend/src/shared/browserSettle.ts b/frontend/src/shared/browserSettle.ts new file mode 100644 index 00000000..d755f7e1 --- /dev/null +++ b/frontend/src/shared/browserSettle.ts @@ -0,0 +1,30 @@ +// Renderer-side mirror of the backend's smart-wait (browser_wait.py): a `wait` +// that returns the instant the page's network goes quiet instead of blind-sleeping +// the full duration. A top-level BrowserWait already gets this on the backend, but +// a `wait` INSIDE a BrowserBatch runs entirely here in the renderer, so without +// this it would fall back to a dumb sleep and make batching slower than not batching. + +export const SETTLE_FLOOR_MS = 250; // never settle before this (a momentary gap isn't 'settled') +export const SETTLE_QUIET_MS = 400; // network must be silent this long to count as settled +export const SETTLE_POLL_MS = 150; + +// One probe: is the document complete, and how long since the last network resource +// finished/started? Returns a JSON string. No interpolation, so it's injection-safe. +export const SETTLE_PROBE_JS = + "(()=>{const n=performance.now();" + + "const es=performance.getEntriesByType('resource');let last=0;" + + "for(const e of es){const t=Math.max(e.responseEnd||0,e.startTime||0);if(t>last)last=t;}" + + "return JSON.stringify({ready:document.readyState==='complete',quiet:Math.round(n-last)});})()"; + +// Pure decision: stop waiting once we're past the floor AND the document is complete +// AND the network has been quiet for the settle window. +export function shouldStopWaiting( + ready: boolean, + quietMs: number, + elapsedMs: number, + floorMs = SETTLE_FLOOR_MS, + quietWindowMs = SETTLE_QUIET_MS, +): boolean { + if (elapsedMs < floorMs) return false; + return !!ready && (quietMs || 0) >= quietWindowMs; +}