From 0992e25bf27f20fcdc01c511e00eca1b722b64eb Mon Sep 17 00:00:00 2001 From: ciregenz Date: Fri, 5 Jun 2026 11:03:20 -0700 Subject: [PATCH] [eric] browser: numbered index overlays on screenshots, vision and clicks share one index language --- backend/apps/agents/browser/browser_schema.py | 15 +++- frontend/src/shared/browserCommandHandler.ts | 89 ++++++++++++++++++- 2 files changed, 100 insertions(+), 4 deletions(-) diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index c42f12ed..06b057bc 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -71,11 +71,22 @@ BROWSER_TOOLS_SCHEMA = [ "name": "BrowserScreenshot", "description": ( "Capture a screenshot of the browser page. Returns the screenshot as a " - "base64-encoded PNG image. Use this to see what is currently displayed." + "base64-encoded PNG image. Use this to see what is currently displayed. " + "Elements from your last BrowserListInteractives come back with numbered " + "colored boxes drawn on them, the same numbers you click with, so you can " + "go straight from what you see to BrowserClickIndex." ), "input_schema": { "type": "object", - "properties": {}, + "properties": { + "annotate": { + "type": "boolean", + "description": ( + "Default true. Pass false for a clean, unannotated shot, e.g. " + "when capturing proof of a completed action." + ), + }, + }, "required": [], }, }, diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index e33c2ed0..890d4738 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -74,7 +74,92 @@ export function getActionLabel(action: string): string { return ACTION_LABELS[action] ?? 'Working...'; } -async function handleScreenshot(wv: BrowserWebview): Promise> { +// Draw each cached element's index as a colored chip on the live page right +// before capture (browser-use's trick): the screenshot then speaks the same +// numbers as BrowserListInteractives, so the vision side can act by index. +const _ANNOTATION_COLORS = ['#e5484d', '#0091ff', '#30a46c', '#f76b15', '#8e4ec6', '#00a2c7']; +const _ANNOTATE_BUDGET_MS = 1500; + +// One unsettled CDP bridge promise must never hang the screenshot; race each call. +function _cdpTimeout(p: Promise, ms: number): Promise { + return Promise.race([p, new Promise((_, rej) => setTimeout(() => rej(new Error('cdp call timed out')), ms))]); +} + +async function annotateElements(wv: BrowserWebview): Promise { + const cacheBridge = (window as any).openswarm?.cdpCacheGet; + const cached = cacheBridge ? await _cdpTimeout(cacheBridge(wv.getWebContentsId()), 500) : null; + if (!cached || typeof cached !== 'object') return 0; + const deadline = Date.now() + _ANNOTATE_BUDGET_MS; + const drawOne = async (idxStr: string, entry: any): Promise => { + const backendNodeId = typeof entry === 'number' ? entry : entry?.backendNodeId; + // v1 is root-frame only: OOPIF rects are frame-local and would land wrong + if (!backendNodeId || entry?.sessionId) return false; + try { + const t = await _cdpTimeout(sendCdp(wv, 'DOM.resolveNode', { backendNodeId }), 300); + const r = await _cdpTimeout(sendCdp(wv, 'Runtime.callFunctionOn', { + objectId: t.object.objectId, + functionDeclaration: + 'function(idx, color) {' + + ' const r = this.getBoundingClientRect();' + + ' if (r.width <= 0 || r.height <= 0) return false;' + + ' if (r.bottom < 0 || r.top > innerHeight || r.right < 0 || r.left > innerWidth) return false;' + + ' let c = document.getElementById("__osw_annotations__");' + + ' if (!c) { c = document.createElement("div"); c.id = "__osw_annotations__";' + + ' c.style.cssText = "position:fixed;inset:0;z-index:2147483647;pointer-events:none;";' + + ' document.documentElement.appendChild(c); }' + + ' const box = document.createElement("div");' + + ' box.style.cssText = "position:fixed;left:" + r.left + "px;top:" + r.top + "px;width:" + r.width + "px;height:" + r.height + "px;border:2px solid " + color + ";box-sizing:border-box;";' + + ' const tag = document.createElement("span");' + + ' tag.textContent = String(idx);' + + ' tag.style.cssText = "position:absolute;left:-2px;top:-16px;background:" + color + ";color:#fff;font:bold 11px/14px monospace;padding:0 4px;border-radius:2px;";' + + ' if (r.top < 18) { tag.style.top = "-2px"; }' + + ' box.appendChild(tag); c.appendChild(box); return true;' + + ' }', + arguments: [{ value: Number(idxStr) }, { value: _ANNOTATION_COLORS[Number(idxStr) % _ANNOTATION_COLORS.length] }], + returnByValue: true, + }), 300); + return r?.result?.value === true; + } catch { return false; } // node gone or call timed out; skip + }; + let drawn = 0; + const entries = Object.entries(cached); + const CHUNK = 10; + for (let i = 0; i < entries.length && Date.now() < deadline; i += CHUNK) { + const results = await Promise.allSettled( + entries.slice(i, i + CHUNK).map(([idxStr, e]) => drawOne(idxStr, e)), + ); + drawn += results.filter((r) => r.status === 'fulfilled' && r.value === true).length; + } + return drawn; +} + +async function removeAnnotations(wv: BrowserWebview): Promise { + try { + await _cdpTimeout(sendCdp(wv, 'Runtime.evaluate', { + expression: 'document.getElementById("__osw_annotations__")?.remove()', + }), 800); + } catch { /* page navigated mid-capture; the overlay died with it */ } +} + +async function handleScreenshot(wv: BrowserWebview, params?: Record): Promise> { + if (params?.annotate !== false) { + let drawn = 0; + try { + drawn = await annotateElements(wv); + if (drawn > 0) { + const shot = await captureRetry(wv); + if (shot.image) shot.text = `Screenshot with ${drawn} numbered boxes matching your element list (pass annotate:false for a clean shot).`; + return shot; + } + } catch { /* annotation is decoration; a plain shot always beats an error */ + } finally { + if (drawn > 0) await removeAnnotations(wv); + } + } + return captureRetry(wv); +} + +async function captureRetry(wv: BrowserWebview): Promise> { // capturePage throws UnknownVizError if the webview hasn't composited a frame // yet (the Viz compositor races the first paint, reliably bit turn-0 captures). // Retry a few times with a short backoff so a cold first screenshot succeeds @@ -1305,7 +1390,7 @@ async function runBrowserCommand( try { switch (action) { case 'screenshot': - result = await handleScreenshot(wv); + result = await handleScreenshot(wv, params); break; case 'get_text': result = await handleGetText(wv);