diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index 5243fd0c..bbc2fae6 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -284,6 +284,37 @@ BROWSER_TOOLS_SCHEMA = [ "required": [], }, }, + { + "name": "BrowserListRoutes", + "description": ( + "List the site's own API endpoints that were captured while you " + "browsed it (GET routes, safe to call directly). When you need to " + "re-fetch data you already loaded once (search results, a list, a " + "detail page), calling the API with BrowserReplayRoute is far faster " + "than re-navigating and re-scraping the UI. Returns nothing until " + "you've actually used the page. Read-only." + ), + "input_schema": {"type": "object", "properties": {}, "required": []}, + }, + { + "name": "BrowserReplayRoute", + "description": ( + "Directly call one of the site's captured GET endpoints (from " + "BrowserListRoutes) and get the raw response, skipping the UI. " + "ONLY safe read-only GET/HEAD requests on the current site are " + "allowed; anything that changes data (add to cart, send, delete, " + "post) must be done through the UI by clicking. Use this to read " + "data fast, not to perform actions." + ), + "input_schema": { + "type": "object", + "properties": { + "url": {"type": "string", "description": "The endpoint URL to GET (from BrowserListRoutes; same site only)."}, + "method": {"type": "string", "enum": ["GET", "HEAD"], "description": "Defaults to GET."}, + }, + "required": ["url"], + }, + }, { "name": "BrowserDetectWebMCP", "description": ( @@ -341,6 +372,8 @@ ACTION_MAP = { "BrowserClickIndex": "click_index", "BrowserBatch": "batch", "BrowserDetectWebMCP": "detect_webmcp", + "BrowserListRoutes": "list_routes", + "BrowserReplayRoute": "replay_route", } SYSTEM_PROMPT = ( @@ -424,7 +457,11 @@ SYSTEM_PROMPT = ( "- Do NOT call the same failing tool twice with identical parameters. If selector " "X failed, try a DIFFERENT selector or a DIFFERENT strategy.\n" "- For repeated actions (swiping through profiles, going through inbox messages), " - "use BrowserPressKey if available; it's an order of magnitude faster than DOM clicks.\n\n" + "use BrowserPressKey if available; it's an order of magnitude faster than DOM clicks.\n" + "- To RE-READ data you already loaded once (search results, a list, a detail page), " + "check BrowserListRoutes and use BrowserReplayRoute to fetch it straight from the " + "site's API instead of re-navigating and re-scraping; it's much faster. This is for " + "reading only, never for actions that change data (those go through the UI).\n\n" "## When you genuinely cannot proceed\n" "Use RequestHumanIntervention for:\n" diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 02ffb1ef..b6f31d55 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -5,7 +5,7 @@ import { rankAndCapInteractives, type RankItem } from './interactiveRanking'; let initialized = false; -export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'batch' | 'detect_webmcp'; +export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route'; export interface BrowserActivity { action: BrowserAction; @@ -684,6 +684,66 @@ async function handleDetectWebMCP(wv: BrowserWebview): Promise> { + const bridge = (window as any).openswarm?.cdpRoutesGet as + | ((id: number, origin?: string) => Promise) | undefined; + if (!bridge) return { error: 'Route capture not available, restart the app.' }; + let origin = ''; + try { origin = new URL(wv.getURL()).origin; } catch {} + let routes: any[] = []; + try { routes = (await bridge(wv.getWebContentsId(), origin)) || []; } catch {} + const safe = routes.filter((r) => r && r.safe); + if (!safe.length) { + return { text: 'No replayable (GET) API routes captured for this site yet. Use the page first so they get recorded, then try again.', url: wv.getURL() }; + } + const lines = safe.slice(0, 40).map((r) => `${r.method} ${r.template} (x${r.hits})`); + return { + text: `Replayable API routes for this site (safe GETs, call with BrowserReplayRoute):\n${lines.join('\n')}`, + routes: safe.slice(0, 40), + url: wv.getURL(), + }; +} + +// Tier 2: replay a captured endpoint directly. GET/HEAD only (idempotent) and +// same-origin only; the fetch runs IN the page so cookies/CSRF come for free. +// Mutating methods are intentionally refused, those must go through the UI. +async function handleReplayRoute(wv: BrowserWebview, params: Record): Promise> { + const rawUrl = params.url as string; + const method = String(params.method || 'GET').toUpperCase(); + if (!rawUrl) return { error: 'url parameter is required' }; + if (method !== 'GET' && method !== 'HEAD') { + return { error: `BrowserReplayRoute only runs safe GET/HEAD requests. ${method} changes data, do that through the UI (click the button) instead.` }; + } + let absUrl: string; + let pageOrigin: string; + try { + pageOrigin = new URL(wv.getURL()).origin; + absUrl = new URL(rawUrl, wv.getURL()).href; + } catch { + return { error: 'invalid url' }; + } + if (new URL(absUrl).origin !== pageOrigin) { + return { error: "BrowserReplayRoute can only call the current site's own API (same origin)." }; + } + const code = `(async () => { + try { + const r = await fetch(${JSON.stringify(absUrl)}, { method: ${JSON.stringify(method)}, credentials: 'include' }); + const body = await r.text(); + return { status: r.status, body: body.slice(0, 15000) }; + } catch (e) { return { error: String((e && e.message) || e) }; } + })()`; + try { + const res = await wv.executeJavaScript(code); + if (res.error) return { error: `Replay failed: ${res.error}` }; + return { text: `${method} ${absUrl} -> HTTP ${res.status}\n${res.body}`, status: res.status, url: wv.getURL() }; + } catch (err: any) { + return { error: `Replay failed: ${err?.message || String(err)}` }; + } +} + async function handleEvaluate(wv: BrowserWebview, params: Record): Promise> { const expression = params.expression as string; if (!expression) return { error: 'expression parameter is required' }; @@ -785,6 +845,12 @@ async function handleBrowserCommand(data: Record) { case 'detect_webmcp': result = await handleDetectWebMCP(wv); break; + case 'list_routes': + result = await handleListRoutes(wv); + break; + case 'replay_route': + result = await handleReplayRoute(wv, params); + break; default: result = { error: `Unknown browser action: ${action}` }; }