mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-25 05:52:22 +02:00
2464 lines
124 KiB
TypeScript
2464 lines
124 KiB
TypeScript
import { getWebview, findWebviewByDomain, hasDomReady, markDomReady, isPendingLoad, wakePendingLoad, clearPendingLoad, type BrowserWebview } from './browserRegistry';
|
|
import { shouldSelfHealClick } from './selfHealClick';
|
|
import { FP_EXPR, clickEffect } from './clickEffect';
|
|
import { store } from './state/store';
|
|
import { resumeBrowserCard } from './state/dashboardLayoutSlice';
|
|
import { dashboardWs } from './ws/WebSocketManager';
|
|
import { resolveInput } from './resolveUrl';
|
|
import { rankAndCapInteractives, type RankItem } from './interactiveRanking';
|
|
import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettle';
|
|
import { unwrapCdpEval } from './cdpEval';
|
|
import { typeChars, type TypedKeys } from './typeChars';
|
|
|
|
let initialized = false;
|
|
|
|
export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'upload_file' | 'click_point' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name' | 'find_composer';
|
|
|
|
export interface BrowserActivity {
|
|
action: BrowserAction;
|
|
detail?: string;
|
|
coords?: { xPercent: number; yPercent: number };
|
|
}
|
|
|
|
type ActivityListener = (browserId: string, activity: BrowserActivity | null) => void;
|
|
|
|
const activityMap = new Map<string, BrowserActivity>();
|
|
const listeners = new Set<ActivityListener>();
|
|
|
|
// A webview keeps churning for a beat after an action lands; capturing it into the dashboard snapshot during that churn is what crashes the renderer (SharedImage 'non-existent mailbox' -> V8 ToLocalChecked), so we hold "busy" this long past the last command before letting the thumbnail capture run again.
|
|
const BUSY_COOLDOWN_MS = 1500;
|
|
let lastActivityAt = 0;
|
|
|
|
function setActivity(browserId: string, activity: BrowserActivity | null) {
|
|
lastActivityAt = Date.now();
|
|
if (activity) {
|
|
activityMap.set(browserId, activity);
|
|
} else {
|
|
activityMap.delete(browserId);
|
|
}
|
|
listeners.forEach((fn) => fn(browserId, activity));
|
|
}
|
|
|
|
export function getActivity(browserId: string): BrowserActivity | null {
|
|
return activityMap.get(browserId) ?? null;
|
|
}
|
|
|
|
// True while an agent is actively driving any browser webview (a command is in flight, or one finished within the cooldown). The dashboard thumbnail capture checks this and skips rather than screenshot a live, churning webview.
|
|
export function isAnyBrowserBusy(): boolean {
|
|
if (activityMap.size > 0) return true;
|
|
return Date.now() - lastActivityAt < BUSY_COOLDOWN_MS;
|
|
}
|
|
|
|
export function subscribeActivity(fn: ActivityListener): () => void {
|
|
listeners.add(fn);
|
|
return () => { listeners.delete(fn); };
|
|
}
|
|
|
|
const ACTION_LABELS: Record<string, string> = {
|
|
screenshot: 'Capturing...',
|
|
get_text: 'Reading...',
|
|
navigate: 'Navigating...',
|
|
click: 'Clicking...',
|
|
type: 'Typing...',
|
|
evaluate: 'Evaluating...',
|
|
get_elements: 'Inspecting...',
|
|
scroll: 'Scrolling...',
|
|
wait: 'Waiting...',
|
|
press_key: 'Pressing key...',
|
|
list_interactives: 'Reading page structure...',
|
|
click_index: 'Clicking element...',
|
|
click_point: 'Tapping screen...',
|
|
click_by_name: 'Clicking element...',
|
|
batch: 'Running batch...',
|
|
};
|
|
|
|
export function getActionLabel(action: string): string {
|
|
return ACTION_LABELS[action] ?? 'Working...';
|
|
}
|
|
|
|
// Draw each cached element's index as a colored chip on the live page right before capture (browser-use's trick): the screenshot then speaks the same numbers as BrowserListInteractives, so the vision side can act by index.
|
|
const _ANNOTATION_COLORS = ['#e5484d', '#0091ff', '#30a46c', '#f76b15', '#8e4ec6', '#00a2c7'];
|
|
const _ANNOTATE_BUDGET_MS = 1500;
|
|
|
|
// One unsettled CDP bridge promise must never hang the screenshot; race each call.
|
|
function _cdpTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
|
|
return Promise.race([p, new Promise<T>((_, rej) => setTimeout(() => rej(new Error('cdp call timed out')), ms))]);
|
|
}
|
|
|
|
async function annotateElements(wv: BrowserWebview): Promise<number> {
|
|
const cacheBridge = (window as any).openswarm?.cdpCacheGet;
|
|
const cached = cacheBridge ? await _cdpTimeout(cacheBridge(wv.getWebContentsId()), 500) : null;
|
|
if (!cached || typeof cached !== 'object') return 0;
|
|
const deadline = Date.now() + _ANNOTATE_BUDGET_MS;
|
|
const drawOne = async (idxStr: string, entry: any): Promise<boolean> => {
|
|
const backendNodeId = typeof entry === 'number' ? entry : entry?.backendNodeId;
|
|
// v1 is root-frame only: OOPIF rects are frame-local and would land wrong
|
|
if (!backendNodeId || entry?.sessionId) return false;
|
|
try {
|
|
const t = await _cdpTimeout(sendCdp(wv, 'DOM.resolveNode', { backendNodeId }), 300);
|
|
const r = await _cdpTimeout(sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: t.object.objectId,
|
|
functionDeclaration:
|
|
'function(idx, color) {'
|
|
+ ' const r = this.getBoundingClientRect();'
|
|
+ ' if (r.width <= 0 || r.height <= 0) return false;'
|
|
+ ' if (r.bottom < 0 || r.top > innerHeight || r.right < 0 || r.left > innerWidth) return false;'
|
|
+ ' let c = document.getElementById("__osw_annotations__");'
|
|
+ ' if (!c) { c = document.createElement("div"); c.id = "__osw_annotations__";'
|
|
+ ' c.style.cssText = "position:fixed;inset:0;z-index:2147483647;pointer-events:none;";'
|
|
+ ' document.documentElement.appendChild(c); }'
|
|
+ ' const box = document.createElement("div");'
|
|
+ ' box.style.cssText = "position:fixed;left:" + r.left + "px;top:" + r.top + "px;width:" + r.width + "px;height:" + r.height + "px;border:2px solid " + color + ";box-sizing:border-box;";'
|
|
+ ' const tag = document.createElement("span");'
|
|
+ ' tag.textContent = String(idx);'
|
|
+ ' tag.style.cssText = "position:absolute;left:-2px;top:-16px;background:" + color + ";color:#fff;font:bold 11px/14px monospace;padding:0 4px;border-radius:2px;";'
|
|
+ ' if (r.top < 18) { tag.style.top = "-2px"; }'
|
|
+ ' box.appendChild(tag); c.appendChild(box); return true;'
|
|
+ ' }',
|
|
arguments: [{ value: Number(idxStr) }, { value: _ANNOTATION_COLORS[Number(idxStr) % _ANNOTATION_COLORS.length] }],
|
|
returnByValue: true,
|
|
}), 300);
|
|
return r?.result?.value === true;
|
|
} catch { return false; } // node gone or call timed out; skip
|
|
};
|
|
let drawn = 0;
|
|
const entries = Object.entries(cached);
|
|
const CHUNK = 10;
|
|
for (let i = 0; i < entries.length && Date.now() < deadline; i += CHUNK) {
|
|
const results = await Promise.allSettled(
|
|
entries.slice(i, i + CHUNK).map(([idxStr, e]) => drawOne(idxStr, e)),
|
|
);
|
|
drawn += results.filter((r) => r.status === 'fulfilled' && r.value === true).length;
|
|
}
|
|
return drawn;
|
|
}
|
|
|
|
async function removeAnnotations(wv: BrowserWebview): Promise<void> {
|
|
try {
|
|
await _cdpTimeout(sendCdp(wv, 'Runtime.evaluate', {
|
|
expression: 'document.getElementById("__osw_annotations__")?.remove()',
|
|
}), 800);
|
|
} catch { /* page navigated mid-capture; the overlay died with it */ }
|
|
}
|
|
|
|
// Chromium only produces compositor frames for a guest that is actually on screen, so a browser
|
|
// card parked outside the canvas viewport has no frame for anyone to copy and EVERY capture path
|
|
// hangs instead of failing. Measured in one window, one instant: a card at x=657 captured in 58ms
|
|
// while a card at x=-15185 timed out on guest capturePage, on host capturePage, AND on CDP
|
|
// Page.captureScreenshot in both fromSurface modes. Panning the card into view is not a nicety,
|
|
// it is the only thing that makes the pixels exist.
|
|
// A sliver poking over the edge composites the sliver, not the page, so ask for a real chunk of card.
|
|
const ONSCREEN_MIN_PX = 80;
|
|
|
|
function p_overlap(lo: number, hi: number, limit: number): number {
|
|
return Math.min(hi, limit) - Math.max(lo, 0);
|
|
}
|
|
|
|
function p_cardIsOnScreen(wv: BrowserWebview): boolean {
|
|
const r = wv.getBoundingClientRect();
|
|
return p_overlap(r.left, r.right, window.innerWidth) >= ONSCREEN_MIN_PX
|
|
&& p_overlap(r.top, r.bottom, window.innerHeight) >= ONSCREEN_MIN_PX;
|
|
}
|
|
|
|
/** '' when the card is visible enough to capture, else why not. Never moves the camera: yanking the user's view because the agent wants a screenshot was the top annoyance report, so an off-screen card degrades to reading tools instead. */
|
|
function ensureCardOnScreen(wv: BrowserWebview): string {
|
|
if (p_cardIsOnScreen(wv)) return '';
|
|
return 'the browser card is off screen (screenshots need it visible; the user can bring it into view)';
|
|
}
|
|
|
|
async function handleScreenshot(wv: BrowserWebview, browserId: string, params?: Record<string, any>): Promise<Record<string, any>> {
|
|
const p_t0 = Date.now();
|
|
const p_stages: string[] = [];
|
|
const p_mark = (stage: string): void => { p_stages.push(`${stage}:${Date.now() - p_t0}`); };
|
|
const p_hidden = ensureCardOnScreen(wv);
|
|
p_mark('onscreen');
|
|
if (p_hidden) {
|
|
return {
|
|
error: `Screenshot unavailable: ${p_hidden}. Reading tools (get_text, list_interactives, get_elements) do not need the card on screen and work right now.`,
|
|
stages: p_stages.join(' '),
|
|
};
|
|
}
|
|
if (params?.annotate !== false) {
|
|
let drawn = 0;
|
|
try {
|
|
drawn = await annotateElements(wv);
|
|
p_mark('annotated');
|
|
if (drawn > 0) {
|
|
const shot = await captureRetry(wv);
|
|
p_mark('captured');
|
|
if (shot.image) shot.text = `Screenshot with ${drawn} numbered boxes matching your element list (pass annotate:false for a clean shot).`;
|
|
return { ...shot, stages: p_stages.join(' ') };
|
|
}
|
|
} catch { /* annotation is decoration; a plain shot always beats an error */
|
|
} finally {
|
|
if (drawn > 0) await removeAnnotations(wv);
|
|
p_mark('annotations removed');
|
|
}
|
|
}
|
|
const p_plain = await captureRetry(wv);
|
|
p_mark('captured');
|
|
return { ...p_plain, stages: p_stages.join(' ') };
|
|
}
|
|
|
|
// Longest one capturePage attempt may take. Every attempt plus backoff must finish inside the
|
|
// backend's 15s command budget, or the honest error never gets a chance to be sent.
|
|
const CAPTURE_ATTEMPT_MS = 2200;
|
|
// Two is enough now that the card is guaranteed on screen: the only thing left to wait out is a
|
|
// cold turn-0 capture racing the first paint. When there is genuinely no frame, extra attempts
|
|
// only burn the budget, because none of them can succeed.
|
|
const CAPTURE_ATTEMPTS = 2;
|
|
|
|
async function captureRetry(wv: BrowserWebview): Promise<Record<string, any>> {
|
|
// capturePage throws UnknownVizError if the webview hasn't composited a frame yet (the Viz compositor races the first paint, reliably bit turn-0 captures). Retry a few times with a short backoff so a cold first screenshot succeeds instead of burning a whole agent turn on a transient error.
|
|
let lastErr: any;
|
|
for (let attempt = 0; attempt < CAPTURE_ATTEMPTS; attempt++) {
|
|
try {
|
|
// capturePage was the ONE unbounded await left on this path, and with no frame to copy it
|
|
// does not throw, it HANGS: the main process logs "GUEST_VIEW_MANAGER_CALL: UnknownVizError"
|
|
// while the renderer promise never settles, so handleScreenshot never returns, no result is
|
|
// ever sent, and the backend kills the command at 15s having learned nothing (one hang ran
|
|
// 244s). The leash keeps that from ever being silent again, and costs a healthy capture
|
|
// nothing.
|
|
const nativeImage = await _cdpTimeout(wv.capturePage(), CAPTURE_ATTEMPT_MS);
|
|
if (!nativeImage.isEmpty()) {
|
|
// Stable PNG capture. The resize()+toJPEG() variant was reverted: it's the prime suspect for the renderer "V8 Empty MaybeLocal" crash, NativeImage's JPEG codec returns an empty image on some retina captures, which is the shape of that native fault. A stable app beats a faster screenshot.
|
|
const dataUrl = nativeImage.toDataURL();
|
|
const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, '');
|
|
return { image: base64, url: wv.getURL(), title: wv.getTitle() };
|
|
}
|
|
lastErr = new Error('capturePage returned an empty image (frame not painted yet)');
|
|
} catch (err: any) {
|
|
lastErr = err;
|
|
}
|
|
await new Promise((r) => setTimeout(r, 250 * (attempt + 1)));
|
|
}
|
|
// Never refuse up front on this: a covered window often still composites, and predicting the
|
|
// failure would blind the agent every time the user tabs to something else. Attempt, then explain.
|
|
const p_why = document.visibilityState === 'hidden'
|
|
? ' The OpenSwarm window is minimized or fully covered, so nothing is being drawn; the reading tools (get_text, list_interactives) still work.'
|
|
: '';
|
|
return { error: `Screenshot failed after retries: ${lastErr?.message || String(lastErr)}.${p_why}` };
|
|
}
|
|
|
|
// Count the safe (GET) API endpoints captured for this site so the backend can nudge the agent toward the fast network path. Best-effort, never throws.
|
|
async function countSafeRoutes(wv: BrowserWebview): Promise<number> {
|
|
try {
|
|
const bridge = (window as any).openswarm?.cdpRoutesGet as
|
|
| ((id: number, origin?: string) => Promise<any[]>) | undefined;
|
|
if (!bridge) return 0;
|
|
let origin = '';
|
|
try { origin = new URL(wv.getURL()).origin; } catch {}
|
|
const routes = (await bridge(wv.getWebContentsId(), origin)) || [];
|
|
return routes.filter((r) => r && r.safe).length;
|
|
} catch { return 0; }
|
|
}
|
|
|
|
// Electron queues executeJavaScript until the page "stops loading", and pages with straggler subresources (recaptcha/tracker iframes) can stay isLoading for minutes, starving EVERY command into its backend timeout (the wedged-webview tail). Once the document itself is ready, wv.stop() cancels only the stragglers and fires did-stop-loading, which flushes the queue; a genuinely-still-loading document (no dom-ready yet) is left alone.
|
|
const STUCK_EVAL_GRACE_MS = 2500;
|
|
const STUCK_EVAL_LIMIT_MS = 9000;
|
|
|
|
// Run `code` in the guest page. In Electron we prefer CDP Runtime.evaluate: it runs in the
|
|
// browser process, so it is NOT suspended while the page is still loading, the way
|
|
// webContents.executeJavaScript is (that suspend, on a page whose trackers never let it "stop
|
|
// loading", is the 15s command wedge). When the CDP bridge isn't there (dev Chrome, or a forced
|
|
// A/B via window.__OSW_CDP_EVAL__ = false) we fall back to the executeJavaScript path unchanged,
|
|
// so behavior never regresses where CDP can't run. Both paths keep the same contract: return the
|
|
// value, throw on a page-side error, mark dom-ready on success.
|
|
async function evalInPage(wv: BrowserWebview, code: string): Promise<any> {
|
|
const cdpBridge = (window as any).openswarm?.sendCdpCommand;
|
|
if (cdpBridge && (window as any).__OSW_CDP_EVAL__ !== false) {
|
|
let cdp: any;
|
|
try {
|
|
cdp = await sendCdp(wv, 'Runtime.evaluate',
|
|
{ expression: code, returnByValue: true, awaitPromise: true });
|
|
} catch {
|
|
// CDP INFRA failure (the debugger can't attach because DevTools or a remote-debugging
|
|
// port already holds this webContents, or the bridge errored). Never worse than today:
|
|
// fall through to the executeJavaScript path. A real PAGE exception is NOT an infra
|
|
// failure, it rides exceptionDetails below, so it still surfaces as a throw.
|
|
cdp = undefined;
|
|
}
|
|
if (cdp !== undefined) {
|
|
const value = unwrapCdpEval(cdp); // throws on a real page-side exception
|
|
markDomReady(wv);
|
|
return value;
|
|
}
|
|
}
|
|
return await evalViaExecuteJs(wv, code);
|
|
}
|
|
|
|
// The original webContents.executeJavaScript path, kept as the dev-Chrome / bridge-absent
|
|
// fallback: it suspends until the page stops loading, so a grace/limit race cancels stragglers
|
|
// with wv.stop() once the document is ready and flushes the queue.
|
|
async function evalViaExecuteJs(wv: BrowserWebview, code: string): Promise<any> {
|
|
const run = wv.executeJavaScript(code).then((v) => {
|
|
markDomReady(wv);
|
|
return { done: true as const, value: v };
|
|
});
|
|
const grace = new Promise<{ done: false }>((r) => setTimeout(() => r({ done: false }), STUCK_EVAL_GRACE_MS));
|
|
let first = await Promise.race([run, grace]);
|
|
if (!first.done) {
|
|
let stopped = false;
|
|
try {
|
|
if (wv.isLoading() && hasDomReady(wv)) {
|
|
wv.stop();
|
|
stopped = true;
|
|
}
|
|
} catch {
|
|
// torn-down webview; the limit below surfaces it
|
|
}
|
|
const limit = new Promise<{ done: false }>((r) => setTimeout(() => r({ done: false }), STUCK_EVAL_LIMIT_MS));
|
|
first = await Promise.race([run, limit]);
|
|
if (!first.done) {
|
|
throw new Error(stopped
|
|
? 'page never finished loading even after cancelling stragglers'
|
|
: 'page is still loading; retry shortly');
|
|
}
|
|
}
|
|
return first.value;
|
|
}
|
|
|
|
// What the MAIN model loop gets. Every read lands in its context, so this stays modest on purpose.
|
|
const GET_TEXT_DEFAULT_CHARS = 15000;
|
|
// Ceiling for a caller that asks for more. The read script (one cheap aux call with its own context)
|
|
// asks for its full budget: measured, 9 of 18 of its reads came back at EXACTLY 15000 chars, meaning
|
|
// cut off, and on a reddit thread the comment scores sit after the post body, so they were never in
|
|
// the text it was handed. It then declined, and a 100-220s model loop went scrolling for what we had
|
|
// truncated ourselves.
|
|
const GET_TEXT_MAX_CHARS = 30000;
|
|
|
|
async function handleGetText(wv: BrowserWebview, params: Record<string, any> = {}): Promise<Record<string, any>> {
|
|
const asked = Number(params.max_chars) || GET_TEXT_DEFAULT_CHARS;
|
|
const cap = Math.min(Math.max(asked, 1000), GET_TEXT_MAX_CHARS);
|
|
const text: string = await evalInPage(wv,
|
|
`document.body.innerText.substring(0, ${cap})`
|
|
);
|
|
// Sampled HERE (on a read), not on navigate: by the time the agent reads the page, the SPA's XHR/fetch have fired, so routes are actually captured.
|
|
const routes_available = await countSafeRoutes(wv);
|
|
return { text, url: wv.getURL(), title: wv.getTitle(), routes_available };
|
|
}
|
|
|
|
// Recent warn+error console output for this webview (captured in main.js). Lets a stuck agent see the page's OWN errors (JS exceptions, failed loads) instead of guessing. Read-only, fail-safe: any miss returns an empty, honest result.
|
|
async function handleGetConsole(wv: BrowserWebview): Promise<Record<string, any>> {
|
|
try {
|
|
const bridge = (window as any).openswarm?.getWebviewConsole as
|
|
| ((id: number) => Promise<Array<{ level: string; message: string; source?: string; line?: number }>>)
|
|
| undefined;
|
|
if (!bridge) return { text: 'Console capture is unavailable here.', errors: [] };
|
|
const errors = (await bridge(wv.getWebContentsId())) || [];
|
|
if (errors.length === 0) {
|
|
return { text: 'No console warnings or errors recorded on this page.', errors: [], url: wv.getURL() };
|
|
}
|
|
const lines = errors.map(
|
|
(e) => `[${e.level}] ${e.message}${e.source ? ` (${e.source}:${e.line ?? '?'})` : ''}`,
|
|
);
|
|
return {
|
|
text: `Page console, ${errors.length} recent warning(s)/error(s), newest last:\n${lines.join('\n')}`,
|
|
errors,
|
|
url: wv.getURL(),
|
|
};
|
|
} catch (err: any) {
|
|
return { text: `Could not read console: ${err?.message || String(err)}`, errors: [] };
|
|
}
|
|
}
|
|
|
|
async function handleNavigate(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const raw = params.url as string;
|
|
if (!raw) return { error: 'url parameter is required' };
|
|
const url = resolveInput(raw);
|
|
// loadURL resolves only on the full 'load' event, which heavy SPAs (LinkedIn, Gmail) hold open with persistent connections long past our timeout even though the page is usable in a second. Return the moment the DOM is ready and let the agent's next wait settle the rest, the way a person clicks before every background request has finished.
|
|
let removeReady = () => {};
|
|
const domReady = new Promise<void>((resolve) => {
|
|
const onReady = () => resolve();
|
|
wv.addEventListener('dom-ready', onReady, { once: true });
|
|
removeReady = () => wv.removeEventListener('dom-ready', onReady);
|
|
});
|
|
const fullyLoaded = wv.loadURL(url).catch((err: any) => {
|
|
// A superseded navigation aborts the old load; that's normal, not a failure.
|
|
if (err?.message?.includes('ERR_ABORTED')) return;
|
|
throw err;
|
|
});
|
|
fullyLoaded.catch(() => {}); // a late load failure shouldn't throw once dom-ready returned
|
|
try {
|
|
await Promise.race([fullyLoaded, domReady]);
|
|
} finally {
|
|
removeReady();
|
|
}
|
|
// A navigate that lands on a raw JSON/API document (Instagram's topsearch, any /api/... GET)
|
|
// paints an unreadable wall in the card and reads as a crash to the user. Hand the data to the
|
|
// agent as the result instead, and quietly get the card off the wall, so a person never sees
|
|
// raw JSON where a page should be.
|
|
const data = await readDataDocument(wv);
|
|
if (data) {
|
|
recoverCardOffDataWall(wv, url);
|
|
return {
|
|
text: `Fetched ${data.contentType} data from ${url} (this URL is a raw API endpoint, not a page):\n${data.body}`,
|
|
url: wv.getURL(),
|
|
data_document: true,
|
|
};
|
|
}
|
|
// Route-count is sampled on the next READ (handleGetText), not here: at navigate-return the SPA's XHRs haven't fired yet, so this would always be ~0.
|
|
return { text: `Navigated to ${url}`, url };
|
|
}
|
|
|
|
// Chromium renders any JSON (or JSON-shaped text) response with its built-in viewer, so a navigate
|
|
// to an API endpoint leaves the card showing a wall of raw data. contentType is the reliable tell;
|
|
// re-fetch same-origin (the just-loaded GET, idempotent) to hand the agent the exact bytes.
|
|
// Exported so BrowserCard can reuse the exact same detection for the card's own initial load.
|
|
export async function readDataDocument(wv: BrowserWebview): Promise<{ contentType: string; body: string } | null> {
|
|
const code = `(async () => {
|
|
const ct = String(document.contentType || '').toLowerCase();
|
|
const isJson = ct.startsWith('application/json');
|
|
const maybePlain = ct.startsWith('text/plain');
|
|
if (!isJson && !maybePlain) return null;
|
|
try {
|
|
const r = await fetch(location.href, { credentials: 'include', signal: AbortSignal.timeout(2500) });
|
|
const body = await r.text();
|
|
if (maybePlain && !isJson) { try { JSON.parse(body.trim()); } catch (e) { return null; } }
|
|
return { contentType: ct, body: body.slice(0, 15000) };
|
|
} catch (e) { return null; }
|
|
})()`;
|
|
try {
|
|
const res = await evalInPage(wv, code);
|
|
if (res && !res.error && typeof res.body === 'string') return res;
|
|
return null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// Get the card off a raw-data wall: step back to the real page the agent came from, or, if it opened
|
|
// straight onto the data URL, fall back to the site homepage. Fire-and-forget, the agent already has
|
|
// the data; this is purely so a human sees a page instead of JSON. Exported for BrowserCard's own-load path.
|
|
export function recoverCardOffDataWall(wv: BrowserWebview, dataUrl: string): void {
|
|
let origin = '';
|
|
try { origin = new URL(dataUrl).origin; } catch { /* keep '' */ }
|
|
try {
|
|
if (wv.canGoBack()) {
|
|
wv.goBack();
|
|
// A card that opened STRAIGHT onto the data URL has only the webview's blank initial entry
|
|
// behind it, so goBack lands on about:blank. Detect that and fall back to the site homepage
|
|
// so the card shows a real page rather than a blank one.
|
|
window.setTimeout(() => {
|
|
try {
|
|
const u = wv.getURL();
|
|
if ((!u || u === 'about:blank') && origin) void wv.loadURL(origin).catch(() => {});
|
|
} catch { /* leave as-is */ }
|
|
}, 400);
|
|
return;
|
|
}
|
|
if (origin) void wv.loadURL(origin).catch(() => {});
|
|
} catch {
|
|
/* leave the card as-is if recovery fails; the data still reached the agent */
|
|
}
|
|
}
|
|
|
|
async function handleClick(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const selector = params.selector as string;
|
|
if (!selector) return { error: 'selector parameter is required' };
|
|
const safeSelector = JSON.stringify(selector);
|
|
const code = `(()=>{
|
|
const el = document.querySelector(${safeSelector});
|
|
if (!el) return { error: 'Element not found: ' + ${safeSelector} };
|
|
el.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
const rect = el.getBoundingClientRect();
|
|
const x = rect.left + rect.width / 2;
|
|
const y = rect.top + rect.height / 2;
|
|
const opts = { bubbles: true, cancelable: true, clientX: x, clientY: y, button: 0 };
|
|
el.dispatchEvent(new PointerEvent('pointerdown', { ...opts, pointerId: 1 }));
|
|
el.dispatchEvent(new MouseEvent('mousedown', opts));
|
|
el.dispatchEvent(new PointerEvent('pointerup', { ...opts, pointerId: 1 }));
|
|
el.dispatchEvent(new MouseEvent('mouseup', opts));
|
|
el.dispatchEvent(new MouseEvent('click', opts));
|
|
return {
|
|
text: 'Clicked element: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''),
|
|
url: location.href,
|
|
clickX: window.innerWidth > 0 ? x / window.innerWidth : 0.5,
|
|
clickY: window.innerHeight > 0 ? y / window.innerHeight : 0.5,
|
|
};
|
|
})()`;
|
|
const result = await evalInPage(wv, code);
|
|
return result;
|
|
}
|
|
|
|
async function handleType(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const selector = params.selector as string;
|
|
const text = params.text as string;
|
|
if (!selector) return { error: 'selector parameter is required' };
|
|
if (text == null) return { error: 'text parameter is required' };
|
|
const safeSelector = JSON.stringify(selector);
|
|
const safeText = JSON.stringify(text);
|
|
// Try the cheap in-page fill first, and read it back: an editor that rejects synthetic
|
|
// execCommand insertText (Reddit's Lexical, strict React contenteditables) leaves the box
|
|
// empty, and we only learn that by checking the value, not by assuming the call worked.
|
|
const code = `(async ()=>{
|
|
const el = document.querySelector(${safeSelector});
|
|
if (!el) return { error: 'Element not found: ' + ${safeSelector} };
|
|
el.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
el.focus();
|
|
if (el.select) el.select();
|
|
document.execCommand('selectAll', false);
|
|
document.execCommand('delete', false);
|
|
document.execCommand('insertText', false, ${safeText});
|
|
el.dispatchEvent(new InputEvent('input', {
|
|
bubbles: true, cancelable: true, inputType: 'insertText', data: ${safeText},
|
|
}));
|
|
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
const now = (el.value != null ? el.value : (el.textContent || ''));
|
|
return { text: 'Typed into: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''),
|
|
committed: now.includes(${safeText}) };
|
|
})()`;
|
|
const result = await evalInPage(wv, code);
|
|
// Real-keystroke fallback when the synthetic fill did not commit (same lever the finder uses).
|
|
if (result && !result.error && result.committed === false) {
|
|
const ok = await keystrokeFill(wv, selector, text);
|
|
result.text = ok ? `Typed into ${selector} via keystrokes` : `Type may not have committed into ${selector}`;
|
|
result.committed = ok;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Find the page's composer STRUCTURALLY, not by accessible name: the biggest editable
|
|
// region (contenteditable / textarea / text input) that is a real writing surface, not a
|
|
// search box. Ranks by editable-richness + proximity to a Send/Post/Reply control + size,
|
|
// so it picks the comment box out of a page that also has a search field. Marks the winner
|
|
// with data-osw-composer so a follow-up type/click can target it by a stable selector.
|
|
// With {fill}, it also types + reads back in the SAME in-page context (the only reliable
|
|
// commit-check for a React-controlled contenteditable, whose value never reaches the AX tree).
|
|
// With {reveal:true}, when no composer is painted yet it takes ONE reversible reveal action
|
|
// (click a compose-trigger, open the first list item, or scroll) and rescans, up to a small
|
|
// bound. Reveal actions are same-document only: SPA route changes (pushState) keep this JS
|
|
// context alive, so X/LinkedIn/Reddit/YouTube stay drivable in one call; a full navigation
|
|
// just cuts the await short and the backend re-perceives. Reveal NEVER clicks an irreversible
|
|
// control (send/submit/pay/delete...): it only opens a surface, it never commits one.
|
|
async function handleFindComposer(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const fill = params.fill != null ? String(params.fill) : null;
|
|
const safeFill = JSON.stringify(fill);
|
|
const reveal = params.reveal === true ? 'true' : 'false';
|
|
const code = `(async () => {
|
|
const SUBMIT = /\\b(send|post|reply|comment|tweet|publish|share|message)\\b/i;
|
|
const SEARCH = /search|find\\b|filter|query|lookup|explore|jump to/i;
|
|
// Reversible compose openers: reveal a writing surface, never commit one.
|
|
const OPENER = /\\b(start a post|start a thread|create( a)?( new)? (post|thread)|new post|new thread|add a comment|write a comment|leave a comment|post a comment|write a review|create a review|new message|new chat|send a message|compose|reply|comment|tweet|create|write)\\b/i;
|
|
// Never clickable by reveal, even if the label also looks like an opener.
|
|
const HARDBLOCK = /\\b(send|submit|pay|buy|purchase|checkout|order|delete|remove|unfollow|unsubscribe|log ?out|sign ?out|report|block|confirm|deactivate|save)\\b/i;
|
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
const vis = (el) => {
|
|
const r = el.getBoundingClientRect(); const s = getComputedStyle(el);
|
|
return r.width >= 80 && r.height >= 16 && s.visibility !== 'hidden'
|
|
&& s.display !== 'none' && s.opacity !== '0';
|
|
};
|
|
// Pierce shadow DOM: Reddit (shreddit), Telegram, WhatsApp, Slack build their composer
|
|
// inside web-component shadow roots that a light-DOM querySelectorAll can't see. Collect
|
|
// only the SELECTOR matches (a few hundred at most), never every node: a heavy SPA like X
|
|
// has 10k+ nodes and the composer sits late in document order, so an all-node walk with a
|
|
// node cap truncates before ever reaching it (the regression that hid X's own composer).
|
|
const deepMatch = (root, sel, out, depth) => {
|
|
if (depth > 8 || out.length > 400) return out;
|
|
let hits; try { hits = root.querySelectorAll(sel); } catch (e) { hits = []; }
|
|
for (const el of hits) out.push(el);
|
|
let all; try { all = root.querySelectorAll('*'); } catch (e) { return out; }
|
|
for (const el of all) { if (el.shadowRoot) deepMatch(el.shadowRoot, sel, out, depth + 1); }
|
|
return out;
|
|
};
|
|
const EDIT = 'textarea, [contenteditable="true"], [role="textbox"], input[type="text"]';
|
|
const labelOf = (el) => ((el.getAttribute('aria-label')||'') + ' ' + (el.getAttribute('placeholder')||'')
|
|
+ ' ' + (el.getAttribute('data-placeholder')||'') + ' ' + (el.getAttribute('name')||'')).trim();
|
|
|
|
const findBest = () => {
|
|
const all = deepMatch(document, EDIT, [], 0);
|
|
let best = null, bestScore = 0, bestNear = false;
|
|
for (const el of all) {
|
|
if (!vis(el) || el.readOnly || el.disabled) continue;
|
|
const label = labelOf(el);
|
|
if (el.type === 'search' || SEARCH.test(label)) continue;
|
|
const rich = el.tagName === 'TEXTAREA' || el.isContentEditable || el.getAttribute('role') === 'textbox';
|
|
const area = el.closest('form, [role="dialog"], [role="group"], section, main, [role="main"]') || document.body;
|
|
let near = false;
|
|
for (const b of area.querySelectorAll('button, [role="button"], input[type="submit"]')) {
|
|
const bt = ((b.textContent||'') + ' ' + (b.getAttribute('aria-label')||'')).trim();
|
|
if (bt.length < 40 && SUBMIT.test(bt)) { near = true; break; }
|
|
}
|
|
if (!rich && !near) continue; // a bare form input near nothing is not a composer
|
|
const r = el.getBoundingClientRect();
|
|
const score = (el.isContentEditable ? 2 : 0) + (el.tagName === 'TEXTAREA' ? 2 : 0)
|
|
+ (near ? 3 : 0) + Math.min((r.width * r.height) / 40000, 3) + (SUBMIT.test(label) ? 1 : 0);
|
|
if (score > bestScore) { bestScore = score; best = el; bestNear = near; }
|
|
}
|
|
return (best && bestScore >= 2) ? { el: best, score: bestScore, near: bestNear } : null;
|
|
};
|
|
|
|
const pollFind = async (ms) => {
|
|
const t0 = Date.now(); let h = findBest();
|
|
while (!h && Date.now() - t0 < ms) { await sleep(140); h = findBest(); }
|
|
return h;
|
|
};
|
|
|
|
// The reveal actions, tried in yield order. Each returns true if it clicked/scrolled.
|
|
// Match id-based placeholders too (YouTube's #placeholder-area "Add a comment...") not just
|
|
// class-based ones, so a lazy comment box becomes a clickable trigger once it scrolls in.
|
|
const TRIGGER_SEL = 'button, [role="button"], a[href], summary, [tabindex], [id*="placeholder" i], [class*="placeholder" i]';
|
|
// inViewOnly confines the pick to what is CURRENTLY on screen: during the scroll phase a
|
|
// header "Create"/"Reply" (scrolled off the top, so r.top<0) would otherwise outscore the
|
|
// comment box that just scrolled into view, and clicking it opens/navigates the wrong thing.
|
|
const clickTrigger = (inViewOnly) => {
|
|
const all = deepMatch(document, TRIGGER_SEL, [], 0);
|
|
let best = null, bestScore = -1;
|
|
for (const el of all) {
|
|
if (!vis(el)) continue;
|
|
const r = el.getBoundingClientRect();
|
|
if (inViewOnly && (r.bottom < 0 || r.top > window.innerHeight)) continue;
|
|
const txt = ((el.textContent||'') + ' ' + (el.getAttribute('aria-label')||'') + ' ' + (el.getAttribute('placeholder')||'')).trim();
|
|
if (txt.length > 60 || !OPENER.test(txt) || HARDBLOCK.test(txt)) continue;
|
|
// Short-labelled wins; higher-on-page when scanning the whole page, viewport-center when scrolling.
|
|
const posScore = inViewOnly
|
|
? Math.max(0, 300 - Math.abs(r.top - window.innerHeight / 2)) / 100
|
|
: Math.max(0, 600 - r.top) / 100;
|
|
const score = (60 - txt.length) + posScore;
|
|
if (score > bestScore) { bestScore = score; best = el; }
|
|
}
|
|
if (!best) return false;
|
|
best.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
best.click();
|
|
return true;
|
|
};
|
|
const openFirstItem = () => {
|
|
// Chat/message lists (X DM [data-testid=conversation], WhatsApp/Telegram [role=grid]
|
|
// rows, conversation links) plus generic feeds. Pierce shadow DOM so web-component chat
|
|
// lists are reachable. Take the first visible list-like item and open it.
|
|
const ITEM_SEL = '[data-testid="conversation"], [role="listitem"], [role="row"], [role="article"], article, li a[href], a[role="link"], a[href*="/messages/"], a[href*="/chat/"]';
|
|
const items = deepMatch(document, ITEM_SEL, [], 0);
|
|
for (const it of items) {
|
|
if (!vis(it)) continue;
|
|
const r = it.getBoundingClientRect();
|
|
if (r.top < 40 || r.top > window.innerHeight || r.height > window.innerHeight * 0.6) continue; // skip headers / offscreen / the whole pane
|
|
const target = it.matches('a[href], [role="link"]') ? it : (it.querySelector('a[href], [role="link"], [role="button"]') || it);
|
|
target.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
target.click();
|
|
return true;
|
|
}
|
|
return false;
|
|
};
|
|
let hit = findBest();
|
|
const acts = [];
|
|
if (!hit && ${reveal}) {
|
|
// Self-imposed budget. The tiers below sum to ~24s of worst case but the command that
|
|
// carries them is killed at its own timeout, which threw away ALL the work and returned
|
|
// nothing (measured: linkedin died mid-scroll 2 of 3 runs, so retop/open-first could never
|
|
// run). Each tier now only STARTS if the budget can still pay for it, and polls are clipped
|
|
// to what is left, so the routine always returns its own best answer instead of being shot.
|
|
const DEADLINE = Date.now() + 21000;
|
|
const left = () => DEADLINE - Date.now();
|
|
const budget = (want) => Math.max(0, Math.min(want, left()));
|
|
// 1. A compose opener visible up top (Gmail "Compose", LinkedIn "Start a post"). Patient
|
|
// poll: LinkedIn code-splits its share modal, and under a heavy session the editor
|
|
// chunk lands past 2.5s (measured: the 2.5s poll missed ~half the time; poll exits the
|
|
// moment the editable appears, so the patience costs nothing on the happy path).
|
|
try { if (clickTrigger(false)) { acts.push('trigger'); hit = await pollFind(budget(6000)); } } catch (e) { /* keep going */ }
|
|
// 2. Progressive scroll for a below-fold / lazy composer: YouTube comments hydrate on
|
|
// scroll and start as a placeholder that only becomes editable once clicked, so scroll
|
|
// a step, re-scan, and re-click the trigger on whatever just entered the viewport, up to
|
|
// a small bound, stopping at page bottom. This is what a human does to reach comments.
|
|
if (!hit) {
|
|
const sc = document.scrollingElement || document.documentElement;
|
|
let scrolled = false;
|
|
// Reserve time for the retop + open-first tiers below, so the ladder can't eat the budget.
|
|
for (let step = 0; step < 6 && !hit && left() > 7000; step++) {
|
|
const before = sc.scrollTop;
|
|
sc.scrollBy(0, Math.round(window.innerHeight * 0.9));
|
|
scrolled = true;
|
|
await sleep(500);
|
|
hit = findBest();
|
|
if (!hit) { try { if (clickTrigger(true)) hit = await pollFind(budget(1400)); } catch (e) { /* keep going */ } }
|
|
if (sc.scrollTop === before) break;
|
|
}
|
|
if (scrolled) acts.push('scroll');
|
|
}
|
|
// 3. Back to the top opener: the scroll ladder ends at page bottom with the top compose
|
|
// entry off-screen; a modal that opened slowly (or needed a second click) is only
|
|
// winnable by returning and retrying once.
|
|
if (!hit && left() > 2500) {
|
|
try {
|
|
(document.scrollingElement || document.documentElement).scrollTo(0, 0);
|
|
await sleep(400);
|
|
if (clickTrigger(false)) { acts.push('retop'); hit = await pollFind(budget(4000)); }
|
|
} catch (e) { /* keep going */ }
|
|
}
|
|
// 4. Last resort: open the first list item (X DMs / chat lists). Navigational, so it runs
|
|
// only after trigger+scroll fail, which stops it from yanking YouTube to another video.
|
|
if (!hit && left() > 1200) { let did = false; try { did = openFirstItem(); } catch (e) { did = false; } if (did) { acts.push('open-first'); hit = await pollFind(budget(2000)); } }
|
|
}
|
|
if (!hit) return { found: false, reveals: acts };
|
|
|
|
const fillEl = (el) => {
|
|
el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus();
|
|
if (el.select) el.select();
|
|
document.execCommand('selectAll', false); document.execCommand('delete', false);
|
|
document.execCommand('insertText', false, ${safeFill});
|
|
el.dispatchEvent(new InputEvent('input', { bubbles: true, cancelable: true, inputType: 'insertText', data: ${safeFill} }));
|
|
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
const now = (el.value != null ? el.value : (el.textContent || ''));
|
|
return now.includes(${safeFill});
|
|
};
|
|
|
|
let best = hit.el;
|
|
best.setAttribute('data-osw-composer', '1');
|
|
let filled = false;
|
|
if (${safeFill} != null) {
|
|
filled = fillEl(best);
|
|
// Activation-click fallback: YouTube (and many comment widgets) render a placeholder
|
|
// that only spawns the real contenteditable once clicked. If the fill didn't commit,
|
|
// click the found element, let the real editor mount, rescan, and fill THAT.
|
|
if (!filled) {
|
|
try { best.click(); } catch (e) { /* click may be intercepted */ }
|
|
const re = await pollFind(1500);
|
|
if (re) {
|
|
best.removeAttribute('data-osw-composer');
|
|
best = re.el; hit = re; best.setAttribute('data-osw-composer', '1');
|
|
filled = fillEl(best);
|
|
acts.push('activate');
|
|
}
|
|
}
|
|
}
|
|
return { found: true, selector: '[data-osw-composer="1"]', tag: best.tagName.toLowerCase(),
|
|
role: best.isContentEditable ? 'contenteditable' : (best.getAttribute('role') || best.tagName.toLowerCase()),
|
|
score: Math.round(hit.score * 10) / 10, nearSubmit: hit.near, filled, reveals: acts };
|
|
})()`;
|
|
const result = await evalInPage(wv, code);
|
|
// Real-keystroke fill fallback: some editors (Reddit's Lexical, strict React contenteditables)
|
|
// manage their own state through beforeinput and ignore execCommand insertText, so the in-page
|
|
// fill reads back empty. Retype the SAME text as OS-level key events (isTrusted, the way a human
|
|
// types), which those editors DO honor, then re-read to confirm. Only fires when the cheap
|
|
// in-page fill already failed, so the fast path is untouched.
|
|
if (fill != null && result && result.found && !result.filled && result.selector) {
|
|
const ok = await keystrokeFill(wv, String(result.selector), fill);
|
|
result.filled = ok;
|
|
result.fillMode = ok ? 'keystroke' : 'keystroke-failed';
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Type `text` into the element at `selector` as real, trusted key events, for editors that reject
|
|
// synthetic execCommand insertText. Focuses + clears in-page first, types each char, then reads the
|
|
// value back in-page to verify.
|
|
async function keystrokeFill(wv: BrowserWebview, selector: string, text: string): Promise<boolean> {
|
|
const safeSel = JSON.stringify(selector);
|
|
await evalInPage(wv, `(() => { const el = document.querySelector(${safeSel});
|
|
if (!el) return false; el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus();
|
|
if (el.select) el.select(); document.execCommand('selectAll', false); document.execCommand('delete', false);
|
|
return true; })()`);
|
|
const typed = await typeChars((m, p) => sendCdp(wv, m, p), text);
|
|
if (!typed.dispatched) return false;
|
|
// Read back from the marked element OR the active element: editors like Reddit's swap the
|
|
// node on activation, so the original selector can go stale even though the keystrokes landed
|
|
// in whatever now holds focus. Checking both survives that re-render.
|
|
const readBack = `(() => {
|
|
const check = (el) => { if (!el) return false;
|
|
el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
const now = (el.value != null ? el.value : (el.textContent || '')); return now.includes(${JSON.stringify(text)}); };
|
|
return check(document.querySelector(${safeSel})) || check(document.activeElement); })()`;
|
|
return Boolean(await evalInPage(wv, readBack));
|
|
}
|
|
|
|
// Electron sendInputEvent expects names like 'Up', 'Enter', 'Space', not 'ArrowUp'/' '/'Esc'.
|
|
const KEY_NAME_MAP: Record<string, string> = {
|
|
ArrowUp: 'Up',
|
|
ArrowDown: 'Down',
|
|
ArrowLeft: 'Left',
|
|
ArrowRight: 'Right',
|
|
' ': 'Space',
|
|
Spacebar: 'Space',
|
|
Esc: 'Escape',
|
|
Del: 'Delete',
|
|
};
|
|
|
|
interface CdpKeyDescriptor { key: string; code: string; vk: number; text?: string }
|
|
|
|
const CDP_KEYS: Record<string, CdpKeyDescriptor> = {
|
|
Enter: { key: 'Enter', code: 'Enter', vk: 13, text: '\r' },
|
|
Tab: { key: 'Tab', code: 'Tab', vk: 9 },
|
|
Escape: { key: 'Escape', code: 'Escape', vk: 27 },
|
|
Backspace: { key: 'Backspace', code: 'Backspace', vk: 8 },
|
|
Delete: { key: 'Delete', code: 'Delete', vk: 46 },
|
|
ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', vk: 38 },
|
|
ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', vk: 40 },
|
|
ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', vk: 37 },
|
|
ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', vk: 39 },
|
|
Home: { key: 'Home', code: 'Home', vk: 36 },
|
|
End: { key: 'End', code: 'End', vk: 35 },
|
|
PageUp: { key: 'PageUp', code: 'PageUp', vk: 33 },
|
|
PageDown: { key: 'PageDown', code: 'PageDown', vk: 34 },
|
|
' ': { key: ' ', code: 'Space', vk: 32, text: ' ' },
|
|
};
|
|
|
|
// Loose names the model actually sends, folded onto the canonical DOM names above.
|
|
const CDP_KEY_ALIASES: Record<string, string> = {
|
|
Up: 'ArrowUp', Down: 'ArrowDown', Left: 'ArrowLeft', Right: 'ArrowRight',
|
|
Space: ' ', Spacebar: ' ', Esc: 'Escape', Del: 'Delete', Return: 'Enter',
|
|
};
|
|
|
|
function cdpKeyDescriptor(rawKey: string): CdpKeyDescriptor | null {
|
|
const canonical = CDP_KEY_ALIASES[rawKey] || rawKey;
|
|
const named = CDP_KEYS[canonical];
|
|
if (named) return named;
|
|
if (canonical.length === 1) {
|
|
const upper = canonical.toUpperCase();
|
|
const code = /[a-z]/i.test(canonical) ? `Key${upper}` : /[0-9]/.test(canonical) ? `Digit${canonical}` : '';
|
|
return { key: canonical, code, vk: upper.charCodeAt(0), text: canonical };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function handlePressKey(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const rawKey = (params.key as string) || '';
|
|
if (!rawKey) return { error: 'key parameter is required' };
|
|
await evalInPage(wv, 'document.body && document.body.focus && document.body.focus(); true');
|
|
const desc = cdpKeyDescriptor(rawKey);
|
|
if (desc) {
|
|
try {
|
|
// CDP key events are trusted AND scoped to THIS webview no matter where the user's cursor sits; the sendInputEvent path delivered to whatever had focus, which is the "agent typed into my note" bug. keyDown-with-text inserts the char; bare named keys use rawKeyDown so no stray char lands.
|
|
const down: Record<string, any> = { type: desc.text ? 'keyDown' : 'rawKeyDown', key: desc.key, windowsVirtualKeyCode: desc.vk, nativeVirtualKeyCode: desc.vk };
|
|
if (desc.code) down.code = desc.code;
|
|
if (desc.text) down.text = desc.text;
|
|
await sendCdp(wv, 'Input.dispatchKeyEvent', down);
|
|
const up: Record<string, any> = { type: 'keyUp', key: desc.key, windowsVirtualKeyCode: desc.vk, nativeVirtualKeyCode: desc.vk };
|
|
if (desc.code) up.code = desc.code;
|
|
await sendCdp(wv, 'Input.dispatchKeyEvent', up);
|
|
return { text: `Pressed ${rawKey}` };
|
|
} catch { /* fall through to the legacy path so a CDP hiccup never makes a key dead */ }
|
|
}
|
|
// Legacy focus-dependent fallback (exotic keys or CDP unavailable): keeps every key that worked before working.
|
|
const keyCode = KEY_NAME_MAP[rawKey] || rawKey;
|
|
await evalInPage(wv, 'document.body && document.body.focus && document.body.focus(); true');
|
|
// Native OS-level key events have isTrusted=true, so hostile sites' keyboard handlers respect them.
|
|
wv.sendInputEvent({ type: 'keyDown', keyCode });
|
|
wv.sendInputEvent({ type: 'char', keyCode });
|
|
wv.sendInputEvent({ type: 'keyUp', keyCode });
|
|
return { text: `Pressed ${rawKey}` };
|
|
}
|
|
|
|
// Click at a viewport coordinate (percent of the view's width/height) with a
|
|
// real, trusted CDP mouse event, NO DOM element required. This is what lets the
|
|
// app agent operate a bare <canvas> game the way a person taps the screen: the
|
|
// AX-tree click paths (click_index/click_by_name) can't target a canvas because
|
|
// it exposes no nodes, but a coordinate dispatch lands anywhere. Optional
|
|
// hold_ms presses and holds (platformers, charge-up mechanics).
|
|
async function handleClickPoint(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const xPercent = Number(params.xPercent);
|
|
const yPercent = Number(params.yPercent);
|
|
if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) {
|
|
return { error: 'xPercent and yPercent are required (0-100, percent of the view).' };
|
|
}
|
|
const cx = Math.max(0, Math.min(100, xPercent));
|
|
const cy = Math.max(0, Math.min(100, yPercent));
|
|
const button = params.button === 'right' ? 'right' : params.button === 'middle' ? 'middle' : 'left';
|
|
const holdMs = Math.max(0, Math.min(Number(params.hold_ms) || 0, 5000));
|
|
// Read the guest's own viewport so coords are correct under zoom/DPR, not the
|
|
// host element's box. One cheap round-trip; falls back to the element box.
|
|
let vw = wv.clientWidth, vh = wv.clientHeight;
|
|
try {
|
|
const d = await evalInPage(wv, '({w: window.innerWidth, h: window.innerHeight})');
|
|
if (d && d.w > 0 && d.h > 0) { vw = d.w; vh = d.h; }
|
|
} catch { /* use the element box as a fallback */ }
|
|
const x = (cx / 100) * vw;
|
|
const y = (cy / 100) * vh;
|
|
// hoverOnly: real mouse move, no press. Hover-revealed controls (reddit's post kebab only
|
|
// materializes on tile hover) need this; a click there would navigate into the post.
|
|
const hoverOnly = params.hoverOnly === true;
|
|
try {
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y });
|
|
if (!hoverOnly) {
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: 1 });
|
|
if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs));
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: 1 });
|
|
}
|
|
} catch (err: any) {
|
|
return { error: `Click point failed: ${err?.message || String(err)}` };
|
|
}
|
|
return {
|
|
text: `${hoverOnly ? 'Hovered' : 'Clicked'} at (${Math.round(x)}, ${Math.round(y)})${holdMs && !hoverOnly ? ` held ${holdMs}ms` : ''}.`,
|
|
clickX: cx, clickY: cy, url: wv.getURL(),
|
|
};
|
|
}
|
|
|
|
// CDP Accessibility.getFullAXTree sees computed roles/names even on hostile sites with unlabeled DOMs.
|
|
const INTERACTIVE_ROLES = new Set([
|
|
'button', 'link', 'textbox', 'combobox', 'checkbox', 'menuitem',
|
|
'tab', 'switch', 'searchbox', 'slider', 'listbox', 'option',
|
|
'radio', 'menuitemcheckbox', 'menuitemradio', 'spinbutton', 'treeitem',
|
|
]);
|
|
|
|
interface InteractiveElement {
|
|
index: number;
|
|
role: string;
|
|
name: string;
|
|
backendNodeId: number;
|
|
sessionId?: string;
|
|
value?: string;
|
|
}
|
|
|
|
function extractAxValue(prop: any): string {
|
|
if (!prop) return '';
|
|
if (typeof prop === 'string') return prop;
|
|
if (prop.value !== undefined) {
|
|
if (typeof prop.value === 'string') return prop.value;
|
|
if (typeof prop.value === 'object' && prop.value && 'value' in prop.value) {
|
|
return String(prop.value.value || '');
|
|
}
|
|
}
|
|
return '';
|
|
}
|
|
|
|
interface CdpResult { ok: boolean; result?: any; error?: string }
|
|
|
|
// sessionId undefined => root frame; a child-frame sessionId => that OOPIF.
|
|
async function sendCdp(wv: BrowserWebview, method: string, params?: Record<string, any>, sessionId?: string): Promise<any> {
|
|
const wcId = wv.getWebContentsId();
|
|
const bridge = (window as any).openswarm?.sendCdpCommand as
|
|
| ((id: number, m: string, p?: any, s?: string) => Promise<CdpResult>)
|
|
| undefined;
|
|
if (!bridge) throw new Error('CDP bridge not available, restart the app');
|
|
const resp = await bridge(wcId, method, params, sessionId);
|
|
if (!resp || !resp.ok) {
|
|
throw new Error(resp?.error || `CDP ${method} failed`);
|
|
}
|
|
return resp.result;
|
|
}
|
|
|
|
interface ChildSession { sessionId: string; frameId: string; parentSessionId: string | null; url: string }
|
|
|
|
async function getChildSessions(wv: BrowserWebview): Promise<ChildSession[]> {
|
|
const bridge = (window as any).openswarm?.cdpChildSessionsGet as
|
|
| ((id: number) => Promise<ChildSession[]>) | undefined;
|
|
if (!bridge) return [];
|
|
try {
|
|
return (await bridge(wv.getWebContentsId())) || [];
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
// Roles whose name is useful as disambiguating context for a nearby control (the person's name above a "Message" button, the section heading of a form).
|
|
const _CONTEXT_ROLES = new Set(['heading', 'statictext', 'link']);
|
|
const _CONTEXT_MAX_CHARS = 60;
|
|
const _CONTEXT_LOOKBACK = 30;
|
|
|
|
function axNodesToCandidates(nodes: any[], sessionId?: string): RankItem[] {
|
|
const byId = new Map<string, any>();
|
|
const parentOf = new Map<string, string>();
|
|
const orderOf = new Map<string, number>();
|
|
for (let i = 0; i < nodes.length; i++) {
|
|
const node = nodes[i];
|
|
if (node.nodeId != null) {
|
|
byId.set(String(node.nodeId), node);
|
|
orderOf.set(String(node.nodeId), i);
|
|
}
|
|
}
|
|
for (const node of nodes) {
|
|
for (const c of node.childIds || []) parentOf.set(String(c), String(node.nodeId));
|
|
}
|
|
const isCandidate = (n: any): boolean => {
|
|
if (!n || n.ignored || n.backendDOMNodeId == null) return false;
|
|
return INTERACTIVE_ROLES.has(extractAxValue(n.role));
|
|
};
|
|
// Which card/section does this control sit in? Nearest named non-interactive ancestor wins (a listitem's name aggregates its card text); else the nearest preceding heading/text/link in document order (browser-use's trick). This is what tells "Message" for Tyler apart from "Message" for everyone else.
|
|
const contextOf = (node: any, ownName: string): string => {
|
|
let p = parentOf.get(String(node.nodeId));
|
|
for (let hops = 0; p && hops < 12; hops++) {
|
|
const anc = byId.get(p);
|
|
if (anc && !anc.ignored && !isCandidate(anc)) {
|
|
const ancName = extractAxValue(anc.name).trim();
|
|
if (ancName && ancName !== ownName) return ancName.slice(0, _CONTEXT_MAX_CHARS);
|
|
}
|
|
p = parentOf.get(p);
|
|
}
|
|
const pos = orderOf.get(String(node.nodeId));
|
|
if (pos == null) return '';
|
|
for (let i = pos - 1; i >= 0 && i >= pos - _CONTEXT_LOOKBACK; i--) {
|
|
const prev = nodes[i];
|
|
if (!prev || prev.ignored) continue;
|
|
if (!_CONTEXT_ROLES.has(extractAxValue(prev.role).toLowerCase())) continue;
|
|
const prevName = extractAxValue(prev.name).trim();
|
|
if (prevName && prevName !== ownName && prevName.length >= 3) {
|
|
return prevName.slice(0, _CONTEXT_MAX_CHARS);
|
|
}
|
|
}
|
|
return '';
|
|
};
|
|
// A same-named interactive ancestor owns this hit target (a link inside a button, an icon twin inside its labeled wrapper); listing both just gives the model two indexes for one click. Names must match so a menu never swallows its menuitems.
|
|
const twinOfAncestor = (node: any, name: string): boolean => {
|
|
if (!name) return false;
|
|
let p = parentOf.get(String(node.nodeId));
|
|
while (p) {
|
|
const anc = byId.get(p);
|
|
if (isCandidate(anc)) return extractAxValue(anc.name).slice(0, 80) === name;
|
|
p = parentOf.get(p);
|
|
}
|
|
return false;
|
|
};
|
|
const out: RankItem[] = [];
|
|
for (const node of nodes) {
|
|
if (node.ignored) continue;
|
|
const role = extractAxValue(node.role);
|
|
if (!INTERACTIVE_ROLES.has(role)) continue;
|
|
const name = extractAxValue(node.name);
|
|
if (!name && role !== 'textbox' && role !== 'searchbox' && role !== 'combobox') continue;
|
|
const backendNodeId = node.backendDOMNodeId;
|
|
if (backendNodeId == null) continue;
|
|
const shortName = name.slice(0, 80);
|
|
if (twinOfAncestor(node, shortName)) continue;
|
|
let value = '';
|
|
if (role === 'textbox' || role === 'searchbox' || role === 'combobox') {
|
|
const isProtected = (node.properties || []).some(
|
|
(p: any) => p?.name === 'protected' && p?.value?.value === true,
|
|
);
|
|
value = isProtected ? '' : extractAxValue(node.value).slice(0, 60);
|
|
}
|
|
out.push({ role, name: shortName, backendNodeId, sessionId, context: contextOf(node, name), value });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// What the user watching a form get filled sees on the page itself: blue when the agent picks a
|
|
// field, green once the value is VERIFIED in it, red when it never landed. Green and red are the
|
|
// point: a fill that silently failed used to look exactly like one that worked.
|
|
const FLASH_COLORS = {
|
|
acting: 'rgba(77,163,255,0.9)',
|
|
ok: 'rgba(52,199,89,0.95)',
|
|
fail: 'rgba(248,113,113,0.95)',
|
|
} as const;
|
|
// Long enough for a human to catch a field going green as the form fills; the intent flash stays
|
|
// short because the outcome flash is right behind it.
|
|
const FLASH_MS = { acting: 450, ok: 900, fail: 1200 } as const;
|
|
|
|
// The outline dance itself, over a variable `el`. One source, because it is now applied from two
|
|
// places (straight away, and from a listener the page fires later) and they must not drift.
|
|
// Saved-once + timer-reset, because a fill flashes twice in a row (intent, then outcome) and a
|
|
// naive save would capture the FIRST flash's own outline as the "original" and leave it painted.
|
|
function flashBody(el: string, kind: keyof typeof FLASH_COLORS): string {
|
|
return `if (${el}.oswFlashTimer) clearTimeout(${el}.oswFlashTimer);`
|
|
+ ` if (!${el}.oswFlashSaved) ${el}.oswFlashSaved = [${el}.style.outline, ${el}.style.outlineOffset];`
|
|
+ ` ${el}.style.outline = "3px solid ${FLASH_COLORS[kind]}"; ${el}.style.outlineOffset = "2px";`
|
|
+ ` ${el}.oswFlashTimer = setTimeout(function () {`
|
|
+ ` ${el}.style.outline = ${el}.oswFlashSaved[0]; ${el}.style.outlineOffset = ${el}.oswFlashSaved[1];`
|
|
+ ` ${el}.oswFlashSaved = null; ${el}.oswFlashTimer = null;`
|
|
+ ` }, ${FLASH_MS[kind]});`;
|
|
}
|
|
|
|
// Outline only, NEVER an injected node. A little "filled!" label with text in it would land in
|
|
// document.body.innerText, which is exactly what BrowserGetText returns and what the send path
|
|
// diffs to prove a composer cleared. Decorating the page must never change what the page SAYS.
|
|
// (browser-use needed a whole data-browser-use-exclude mechanism because their panel does inject.)
|
|
function flashField(
|
|
wv: BrowserWebview, objectId: string | undefined, sessionId: string | undefined,
|
|
kind: keyof typeof FLASH_COLORS,
|
|
): void {
|
|
if (!objectId) return;
|
|
sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId,
|
|
functionDeclaration: `function() { var el = this; ${flashBody('el', kind)} }`,
|
|
}, sessionId).catch(() => {});
|
|
}
|
|
|
|
// Cumulative top-left offset of a frame within the root viewport: climb the session chain adding each owning <iframe>'s top-left. Used ONLY to place the cosmetic click ripple; the click itself dispatches in the element's own frame, so this is best-effort. Verified getFrameOwner works through Electron.
|
|
async function frameOffset(
|
|
wv: BrowserWebview, sessionId: string | undefined, children: ChildSession[],
|
|
): Promise<{ dx: number; dy: number }> {
|
|
let dx = 0, dy = 0;
|
|
const byId = new Map(children.map((c) => [c.sessionId, c]));
|
|
const seen = new Set<string>();
|
|
let s: string | null | undefined = sessionId;
|
|
while (s && !seen.has(s)) {
|
|
seen.add(s);
|
|
const info = byId.get(s);
|
|
if (!info) break;
|
|
const parent = info.parentSessionId || undefined; // undefined => root
|
|
const owner = await sendCdp(wv, 'DOM.getFrameOwner', { frameId: info.frameId }, parent);
|
|
const ownerBox = await sendCdp(wv, 'DOM.getBoxModel', { backendNodeId: owner.backendNodeId }, parent);
|
|
const oc = ownerBox?.model?.content;
|
|
if (!Array.isArray(oc) || oc.length < 8) break;
|
|
dx += oc[0];
|
|
dy += oc[1];
|
|
s = info.parentSessionId;
|
|
}
|
|
return { dx, dy };
|
|
}
|
|
|
|
// Enumerate interactive elements across the root frame + every attached OOPIF child frame. Shared by list_interactives (numbered list) and click_by_name (stable re-resolution for replay), so both see the exact same surface. Root gets a generous budget (a big real page legitimately takes a few seconds); only a genuinely hung renderer exceeds it. Child frames (usually tracker/ad OOPIFs on heavy sites) must be quick or they're skipped, and we cap how many we walk so an ad-heavy page can't multiply full-tree calls. The page's own content is in the root tree (the about:blank compose iframe is same-process, so it's in the root too).
|
|
const _AX_ROOT_TIMEOUT_MS = 8000;
|
|
const _AX_CHILD_TIMEOUT_MS = 2500;
|
|
const _MAX_AX_CHILD_FRAMES = 6;
|
|
const _PAGE_TREE_TIMEOUT_MS = 1500;
|
|
const _MAX_TOTAL_FRAMES = 12;
|
|
|
|
function withTimeout<T>(p: Promise<T>, ms: number, label: string): Promise<T> {
|
|
p.catch(() => {}); // swallow a late rejection if the timeout wins the race first
|
|
let timer: ReturnType<typeof setTimeout>;
|
|
const timeout = new Promise<never>((_, reject) => {
|
|
timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
|
});
|
|
return Promise.race([p, timeout]).finally(() => clearTimeout(timer));
|
|
}
|
|
|
|
function flattenFrameTree(tree: any, out: string[] = []): string[] {
|
|
if (!tree) return out;
|
|
const id = tree?.frame?.id;
|
|
if (id) out.push(id);
|
|
for (const c of tree.childFrames || []) flattenFrameTree(c, out);
|
|
return out;
|
|
}
|
|
|
|
async function enumerateCandidates(wv: BrowserWebview): Promise<RankItem[]> {
|
|
const candidates: RankItem[] = [];
|
|
let framesWalked = 0;
|
|
let framesDropped = 0;
|
|
|
|
const walkSession = async (
|
|
sessionId: string | undefined, budgetMs: number, label: string,
|
|
): Promise<{ ok: boolean; lastErr?: any }> => {
|
|
const sessionStart = Date.now();
|
|
const remaining = () => Math.max(1, budgetMs - (Date.now() - sessionStart));
|
|
|
|
let frameIds: string[] = [];
|
|
try {
|
|
const tree = await withTimeout(
|
|
sendCdp(wv, 'Page.getFrameTree', {}, sessionId),
|
|
Math.min(_PAGE_TREE_TIMEOUT_MS, remaining()), `${label} frame tree`);
|
|
frameIds = flattenFrameTree(tree?.frameTree);
|
|
} catch { /* fall through to a single AX call below */ }
|
|
|
|
if (frameIds.length === 0) {
|
|
if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; return { ok: true }; }
|
|
try {
|
|
const ax = await withTimeout(
|
|
sendCdp(wv, 'Accessibility.getFullAXTree', {}, sessionId), remaining(), label);
|
|
candidates.push(...axNodesToCandidates(ax?.nodes || [], sessionId));
|
|
framesWalked++;
|
|
return { ok: true };
|
|
} catch (err: any) {
|
|
return { ok: false, lastErr: err };
|
|
}
|
|
}
|
|
|
|
let lastErr: any;
|
|
let anySuccess = false;
|
|
for (const frameId of frameIds) {
|
|
if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; continue; }
|
|
if (remaining() <= 1) { framesDropped++; continue; }
|
|
try {
|
|
const ax = await withTimeout(
|
|
sendCdp(wv, 'Accessibility.getFullAXTree', { frameId }, sessionId), remaining(), label);
|
|
candidates.push(...axNodesToCandidates(ax?.nodes || [], sessionId));
|
|
anySuccess = true;
|
|
} catch (err: any) { lastErr = err; }
|
|
framesWalked++;
|
|
}
|
|
return { ok: anySuccess, lastErr };
|
|
};
|
|
|
|
const root = await walkSession(undefined, _AX_ROOT_TIMEOUT_MS, 'page perception');
|
|
if (!root.ok) {
|
|
// A saturated/hung renderer can't answer; surface a clear, actionable signal instead of silently blocking to the hard command timeout, so the agent can wait a beat and retry (the freeze is often intermittent) rather than abort.
|
|
throw new Error(
|
|
`the page is too busy to read right now (${root.lastErr?.message || 'timed out'}); `
|
|
+ 'wait a moment with BrowserWait and try again, or reload the page.');
|
|
}
|
|
const children = (await getChildSessions(wv)).slice(0, _MAX_AX_CHILD_FRAMES);
|
|
for (const child of children) {
|
|
if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; continue; }
|
|
await walkSession(child.sessionId, _AX_CHILD_TIMEOUT_MS, 'child frame');
|
|
}
|
|
if (framesDropped > 0) {
|
|
console.log(`[cdp] enumerateCandidates capped at ${_MAX_TOTAL_FRAMES} frames; dropped ${framesDropped}`);
|
|
}
|
|
return candidates;
|
|
}
|
|
|
|
// Put a real cursor in an editable by clicking its centre with trusted CDP mouse events. Best
|
|
// effort on purpose: a failure here just means the keystroke tier tries without it, exactly as
|
|
// before. Coordinates are viewport-relative and the caller has already scrolled the node into view.
|
|
async function clickToFocus(
|
|
wv: BrowserWebview, objectId: string, sessionId: string | undefined,
|
|
): Promise<void> {
|
|
try {
|
|
const r = await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId,
|
|
functionDeclaration:
|
|
'function() { this.scrollIntoView({ block: "center", behavior: "instant" }); '
|
|
+ 'var b = this.getBoundingClientRect(); '
|
|
+ 'return b.width > 0 && b.height > 0 ? { x: b.x + b.width / 2, y: b.y + b.height / 2 } : null; }',
|
|
returnByValue: true,
|
|
}, sessionId);
|
|
const box = r?.result?.value as { x: number; y: number } | null;
|
|
if (!box) return;
|
|
for (const type of ['mousePressed', 'mouseReleased']) {
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent',
|
|
{ type, x: box.x, y: box.y, button: 'left', clickCount: 1 });
|
|
}
|
|
} catch { /* the tier below reports honestly whether the text landed */ }
|
|
}
|
|
|
|
// Resolve + click a specific backend node (revalidate, frame-local box model, OS-level dispatch in the element's own frame, cosmetic top-level ripple). Shared by click_index (cache lookup) and click_by_name (fresh resolution).
|
|
async function clickBackendNode(
|
|
wv: BrowserWebview, backendNodeId: number, sessionId: string | undefined, label: string,
|
|
opts: { role?: string; text?: string } = {},
|
|
): Promise<Record<string, any>> {
|
|
let resolvedObjectId: string | undefined;
|
|
try {
|
|
const resolved = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
|
resolvedObjectId = resolved?.object?.objectId;
|
|
} catch (err: any) {
|
|
return { error: `${label} is no longer valid (${err.message || 'node not found'}). The page may have changed.` };
|
|
}
|
|
|
|
// Blue = about to act on this. The outcome colors land later, once we know it.
|
|
flashField(wv, resolvedObjectId, sessionId, 'acting');
|
|
|
|
// A text box is focused DIRECTLY by node id, never by screen coordinates. Inside an about:blank compose iframe (LinkedIn/Gmail messaging) the coordinate path lands on the wrong element (the box model is frame-local but the click dispatches in the root frame), while DOM.focus reaches the node in any frame. With a `text` arg we then insert the whole string at once, no clicking, no character-by-character typing.
|
|
const _role = opts.role || '';
|
|
const _wantsText = typeof opts.text === 'string' && opts.text.length > 0;
|
|
if (/\b(textbox|searchbox)\b/i.test(_role) || (/\bcombobox\b/i.test(_role) && _wantsText)) {
|
|
try {
|
|
await sendCdp(wv, 'DOM.focus', { backendNodeId }, sessionId);
|
|
} catch (err: any) {
|
|
return { error: `${label} could not be focused (${err?.message || 'focus failed'}); it may be disabled or hidden.` };
|
|
}
|
|
if (typeof opts.text === 'string' && opts.text.length > 0) {
|
|
// Read the text back from the node itself; "insert reported OK" is not "the box has the text" (rich-text editors can swallow synthetic input). Returns the box's actual content (or null on miss) so the result can echo the OBSERVED state; a bare "typed it" claim loses to a wrongly pessimistic expect-confirm and provokes a double-fill.
|
|
const readBack = async (): Promise<string | null> => {
|
|
try {
|
|
const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
|
const r = await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: t.object.objectId,
|
|
functionDeclaration:
|
|
'function(s) { const v = (this.value !== undefined ? this.value : this.textContent) || ""; return v.includes(s) ? v.slice(0, 120) : null; }',
|
|
arguments: [{ value: opts.text }],
|
|
returnByValue: true,
|
|
}, sessionId);
|
|
return typeof r?.result?.value === 'string' ? r.result.value : null;
|
|
} catch { return opts.text ?? ''; } // unverifiable beats a false alarm
|
|
};
|
|
// Every one of the four fill tiers exits through here, so the green flash is tied to the
|
|
// read-back rather than to any single mechanism: the field goes green exactly when the value
|
|
// is provably in it, never merely because a tier ran.
|
|
const landedMsg = (got: string, via = '') => {
|
|
flashField(wv, resolvedObjectId, sessionId, 'ok');
|
|
return { text: `Focused ${label} and typed the text in${via}. Verified: the box now contains "${got}". Do NOT type it again.` };
|
|
};
|
|
try {
|
|
await sendCdp(wv, 'Input.insertText', { text: opts.text }, sessionId);
|
|
} catch (err: any) {
|
|
return { error: `Focused ${label} but could not type into it: ${err?.message || String(err)}` };
|
|
}
|
|
let got = await readBack();
|
|
if (got !== null) return landedMsg(got);
|
|
try {
|
|
const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
|
await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: t.object.objectId,
|
|
functionDeclaration:
|
|
'function(s) { this.focus(); document.execCommand("insertText", false, s); }',
|
|
arguments: [{ value: opts.text }],
|
|
}, sessionId);
|
|
} catch { /* verified below; the honest error covers this failing too */ }
|
|
got = await readBack();
|
|
if (got !== null) return landedMsg(got, ' (via editor command)');
|
|
// Third tier: REAL keystrokes. Both attempts above are synthetic, and a whole class of editor
|
|
// (Lexical, ProseMirror, strict Draft.js) only commits on beforeinput from genuine key events
|
|
// and drops both of them on the floor. This is the very advice the error below has been
|
|
// handing out for months, so take it here rather than spending a model turn on it: measured
|
|
// live on twitch, the composer was found and focused and the fill still errored, which cost
|
|
// the site its entire write path. Clearing first, because a partial insert plus keystrokes is
|
|
// how you post the same sentence twice.
|
|
let typed: TypedKeys = { dispatched: false, skipped: 'empty' };
|
|
try {
|
|
const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
|
// Click the box for real before typing into it. DOM.focus sets the focused node but never
|
|
// runs the editor's own mousedown/selection handling, so a Slate/Lexical/ProseMirror editor
|
|
// has no cursor and drops genuine keystrokes on the floor. That is twitch exactly: focus
|
|
// fine, real keys, text ignored. Third rung of the same focus ladder browser-use climbs.
|
|
await clickToFocus(wv, t.object.objectId, sessionId);
|
|
await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: t.object.objectId,
|
|
functionDeclaration:
|
|
'function() { this.focus(); if (this.select) this.select(); document.execCommand("selectAll", false); document.execCommand("delete", false); }',
|
|
}, sessionId);
|
|
// Focus is set in the node's own frame above, but the key events themselves go to the ROOT
|
|
// target with no sessionId: Chromium routes keyboard input to whichever frame holds focus,
|
|
// and the Input domain is not reliably there on an OOPIF session. Sending them at the child
|
|
// would kill exactly the case this cares about, a composer inside an iframe.
|
|
typed = await typeChars((m, p) => sendCdp(wv, m, p), opts.text);
|
|
} catch { /* verified below; the honest error covers this failing too */ }
|
|
got = await readBack();
|
|
if (got !== null) return landedMsg(got, ' (via keystrokes)');
|
|
// Editors like Reddit's swap the node out on activation, so the original backendNodeId can go
|
|
// stale even though the keystrokes landed in whatever now holds focus. Same lesson keystrokeFill
|
|
// already learned; checking both is what survives that re-render.
|
|
try {
|
|
const live = await evalInPage(wv, `(() => { const el = document.activeElement;
|
|
const v = el ? ((el.value != null ? el.value : el.textContent) || '') : '';
|
|
return v.includes(${JSON.stringify(opts.text)}) ? v.slice(0, 120) : null; })()`);
|
|
if (typeof live === 'string') return landedMsg(live, ' (via keystrokes)');
|
|
} catch { /* fall through to the honest error */ }
|
|
flashField(wv, resolvedObjectId, sessionId, 'fail');
|
|
// Saying "not even as real keystrokes" when we deliberately declined to send them is the
|
|
// kind of small lie that costs a debugging session.
|
|
if (typed.skipped === 'multiline') {
|
|
return { error: `Focused ${label} but this editor ignored both synthetic fills, and the text contains a line break, so real keystrokes were NOT attempted: pressing Enter in a composer can send it half-written. Fill it as a single line, or use a different element.` };
|
|
}
|
|
return { error: `Focused ${label} but the text did not register even as real keystrokes; the box may be a custom editor that rejects automation. Try a different element.` };
|
|
}
|
|
return { text: `Focused ${label}; the cursor is in it now (type with BrowserPressKey, or pass a text arg to fill it in one call).` };
|
|
}
|
|
|
|
try {
|
|
await sendCdp(wv, 'DOM.scrollIntoViewIfNeeded', { backendNodeId }, sessionId);
|
|
} catch { /* not scrollable or already visible; the box model below decides */ }
|
|
let boxModel;
|
|
try {
|
|
boxModel = await sendCdp(wv, 'DOM.getBoxModel', { backendNodeId }, sessionId);
|
|
} catch (err: any) {
|
|
return { error: `${label} has no box model (likely off-screen or hidden). Try scrolling first.` };
|
|
}
|
|
const content = boxModel?.model?.content;
|
|
if (!Array.isArray(content) || content.length < 8) {
|
|
return { error: `${label} has no valid bounding rect.` };
|
|
}
|
|
const lx = (content[0] + content[4]) / 2;
|
|
const ly = (content[1] + content[5]) / 2;
|
|
|
|
// Hit-test before dispatching: a sticky banner or header twin can cover the element's center, and a blind coordinate click lands on the overlay instead (the "Reactivate Premium" misfire). If covered, click the chosen node itself.
|
|
// This is also the ground-truth "did my hand land where I aimed" signal: when the element at the click point is NOT the intended node, a naive coordinate click hits the WRONG element (the failure a page-change metric can't see, because the wrong element also changes the page). We surface what was actually hit so the wrong-target RATE is measurable.
|
|
let covered = false;
|
|
let hitDesc = '';
|
|
let targetObjectId: string | undefined;
|
|
try {
|
|
const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
|
targetObjectId = t?.object?.objectId;
|
|
if (targetObjectId) {
|
|
const rel = await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: targetObjectId,
|
|
functionDeclaration:
|
|
'function(x, y) { const r = this.getRootNode(); const h = (r.elementFromPoint ? r : document).elementFromPoint(x, y);'
|
|
+ ' const landed = h ? (this === h || this.contains(h) || h.contains(this)) : true;'
|
|
+ ' const d = h ? (h.tagName.toLowerCase() + (h.getAttribute("aria-label") ? "[" + h.getAttribute("aria-label").slice(0,30) + "]" : (h.textContent ? "[" + h.textContent.trim().slice(0,30) + "]" : ""))) : "none";'
|
|
+ ' return { landed: landed, hit: d }; }',
|
|
arguments: [{ value: lx }, { value: ly }],
|
|
returnByValue: true,
|
|
}, sessionId);
|
|
const hv = rel?.result?.value as { landed?: boolean; hit?: string } | undefined;
|
|
covered = hv?.landed === false;
|
|
hitDesc = hv?.hit || '';
|
|
}
|
|
} catch { /* hit-test is best-effort; fall through to the coordinate click */ }
|
|
// Attach the aim signal to every click result (near-zero cost, already computed): landed=false means the intended element was not under the click point (occluded / stale box / moved), i.e. a wrong-target click.
|
|
const p_aim = { clickLanded: !covered, clickHit: hitDesc };
|
|
|
|
let rx = lx, ry = ly;
|
|
if (sessionId) {
|
|
try {
|
|
const children = await getChildSessions(wv);
|
|
const { dx, dy } = await frameOffset(wv, sessionId, children);
|
|
rx = lx + dx; ry = ly + dy;
|
|
} catch { /* fall back to frame-local for the ripple */ }
|
|
}
|
|
const ripple = { clickX: rx / wv.clientWidth * 100, clickY: ry / wv.clientHeight * 100 };
|
|
|
|
if (covered && targetObjectId) {
|
|
try {
|
|
await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: targetObjectId,
|
|
functionDeclaration:
|
|
'function() { if (this.click) { this.click(); } else { this.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true })); } }',
|
|
}, sessionId);
|
|
return { text: `Clicked ${label} via its element (another element covers its screen position).`, ...ripple, ...p_aim };
|
|
} catch (err: any) {
|
|
return { error: `${label} is covered by another element and could not be clicked (${err?.message || String(err)}). Scroll, or pick a different element.` };
|
|
}
|
|
}
|
|
|
|
// A dropdown, checkbox or radio is the rest of a form, and the click path could not say whether
|
|
// any of them actually took: only a typed field ever went green. So read the control's state
|
|
// either side of the click and let the CHANGE be the proof. Deliberately not "it holds a value
|
|
// now", which would paint a pre-filled field green and claim credit for work nobody did.
|
|
const controlState = async (): Promise<string | null> => {
|
|
if (!targetObjectId) return null;
|
|
try {
|
|
const r = await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: targetObjectId,
|
|
functionDeclaration:
|
|
'function() { var t = this.tagName; '
|
|
+ 'if (t !== "SELECT" && t !== "INPUT" && t !== "TEXTAREA") return null; '
|
|
+ 'return String(this.checked) + "\\u0000" + String(this.value); }',
|
|
returnByValue: true,
|
|
}, sessionId);
|
|
const v = r?.result?.value;
|
|
return typeof v === 'string' ? v : null;
|
|
} catch { return null; }
|
|
};
|
|
const p_before = await controlState();
|
|
|
|
try {
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: lx, y: ly, button: 'left', clickCount: 1 }, sessionId);
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: lx, y: ly, button: 'left', clickCount: 1 }, sessionId);
|
|
} catch (err: any) {
|
|
flashField(wv, resolvedObjectId, sessionId, 'fail');
|
|
return { error: `Click failed: ${err.message || String(err)}` };
|
|
}
|
|
if (p_before !== null) {
|
|
const p_after = await controlState();
|
|
// Only a real change earns the green. No change is left alone rather than painted red: plenty
|
|
// of legitimate clicks on a control change nothing yet (opening a native select is one), and
|
|
// crying failure there would be the same overclaim pointed the other way.
|
|
if (p_after !== null && p_after !== p_before) flashField(wv, resolvedObjectId, sessionId, 'ok');
|
|
}
|
|
return {
|
|
text: `Clicked ${label} at (${Math.round(rx)}, ${Math.round(ry)})`,
|
|
...ripple,
|
|
...p_aim,
|
|
};
|
|
}
|
|
|
|
// Wrap a click so we can measure whether it actually did anything, covering EVERY
|
|
// exit of clickBackendNode (coordinate, covered-element JS click, ...), which is why
|
|
// it lives out here and not in the one coordinate branch. Metric only, flag-gated.
|
|
async function measureClickEffect(
|
|
wv: BrowserWebview, run: () => Promise<Record<string, any>>,
|
|
): Promise<Record<string, any>> {
|
|
let before = '';
|
|
try { before = String(await wv.executeJavaScript(FP_EXPR)); } catch { /* unreadable */ }
|
|
const result = await run();
|
|
if (!result.error) {
|
|
await new Promise((r) => setTimeout(r, 400));
|
|
let after = '';
|
|
try { after = String(await wv.executeJavaScript(FP_EXPR)); } catch { /* unreadable */ }
|
|
result.clickEffect = clickEffect(before, after);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Drop list rows the user literally cannot click: zero-size nodes and ones whose center hits a DIFFERENT element (modal backdrop, sticky header, cookie banner). Ground truth via elementFromPoint, the same predicate the click path trusts. Offscreen-but-scrollable elements are kept; the page-wide list is deliberately wider than the viewport. Chunked with a hard budget so a heavy page degrades to an unfiltered list, never a stall.
|
|
const _OCCLUSION_BUDGET_MS = 1500;
|
|
const _OCCLUSION_CHUNK = 10;
|
|
async function dropCoveredElements(
|
|
wv: BrowserWebview, items: RankItem[],
|
|
): Promise<{ kept: RankItem[]; dropped: number }> {
|
|
const deadline = Date.now() + _OCCLUSION_BUDGET_MS;
|
|
const kept: RankItem[] = [];
|
|
let dropped = 0;
|
|
for (let i = 0; i < items.length; i += _OCCLUSION_CHUNK) {
|
|
const chunk = items.slice(i, i + _OCCLUSION_CHUNK);
|
|
if (Date.now() > deadline) {
|
|
kept.push(...items.slice(i));
|
|
break;
|
|
}
|
|
const verdicts = await Promise.all(chunk.map(async (el) => {
|
|
try {
|
|
const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId: el.backendNodeId }, el.sessionId);
|
|
const objectId = t?.object?.objectId;
|
|
if (!objectId) return 'clear';
|
|
const r = await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId,
|
|
functionDeclaration:
|
|
'function() {'
|
|
+ ' const rect = this.getBoundingClientRect();'
|
|
+ ' if (rect.width === 0 || rect.height === 0) return "hidden";'
|
|
+ ' const cx = rect.left + rect.width / 2, cy = rect.top + rect.height / 2;'
|
|
+ ' if (cx < 0 || cy < 0 || cx > window.innerWidth || cy > window.innerHeight) return "offscreen";'
|
|
+ ' const root = this.getRootNode();'
|
|
+ ' const h = (root.elementFromPoint ? root : document).elementFromPoint(cx, cy);'
|
|
+ ' if (!h) return "clear";'
|
|
+ ' return (this === h || this.contains(h) || h.contains(this)) ? "clear" : "covered";'
|
|
+ ' }',
|
|
returnByValue: true,
|
|
}, el.sessionId);
|
|
return r?.result?.value || 'clear';
|
|
} catch {
|
|
return 'clear';
|
|
}
|
|
}));
|
|
chunk.forEach((el, j) => {
|
|
if (verdicts[j] === 'covered' || verdicts[j] === 'hidden') dropped += 1;
|
|
else kept.push(el);
|
|
});
|
|
}
|
|
return { kept, dropped };
|
|
}
|
|
|
|
async function handleListInteractives(wv: BrowserWebview, params: Record<string, any> = {}): Promise<Record<string, any>> {
|
|
let candidates: RankItem[];
|
|
try {
|
|
candidates = await enumerateCandidates(wv);
|
|
} catch (err: any) {
|
|
return { error: `getFullAXTree failed: ${err.message || String(err)}` };
|
|
}
|
|
|
|
// Dedupe twins, rank what a human acts on first (and the current goal highest), cap the long tail.
|
|
const goal = typeof params?.goal === 'string' ? params.goal : '';
|
|
const { shown: ranked, truncated } = rankAndCapInteractives(candidates, { goal, docOrder: params?.docOrder !== false });
|
|
const { kept: shown, dropped: covered } = await dropCoveredElements(wv, ranked);
|
|
|
|
// The previous look's cache feeds two things: * markers for brand-new elements, and STABLE indices so the same element keeps the same number across looks (the model can act on a remembered index without re-reading the whole list, browser-use's stable-hash trick on our node ids).
|
|
let prevIds: Set<string> | null = null;
|
|
const prevIndexByKey = new Map<string, number>();
|
|
try {
|
|
const cacheBridge = (window as any).openswarm?.cdpCacheGet;
|
|
const prev = cacheBridge ? await cacheBridge(wv.getWebContentsId()) : null;
|
|
if (prev && typeof prev === 'object') {
|
|
for (const [idxStr, e] of Object.entries(prev)) {
|
|
const key = `${(typeof e === 'object' && (e as any)?.sessionId) || ''}:${typeof e === 'number' ? e : (e as any)?.backendNodeId}`;
|
|
prevIndexByKey.set(key, Number(idxStr));
|
|
}
|
|
prevIds = new Set(prevIndexByKey.keys());
|
|
}
|
|
} catch { /* no previous list; fresh numbering, no markers */ }
|
|
const keyOf = (el: RankItem) => `${el.sessionId || ''}:${el.backendNodeId}`;
|
|
const isNew = (el: RankItem) => !!prevIds && prevIds.size > 0 && !prevIds.has(keyOf(el));
|
|
|
|
// Sticky only while some elements carried over; full turnover (a navigation, node ids all changed) or runaway numbering restarts cleanly at 1.
|
|
const matchedPrev = shown.filter((el) => prevIndexByKey.has(keyOf(el))).length;
|
|
let nextFree = prevIndexByKey.size > 0 ? Math.max(...prevIndexByKey.values()) + 1 : 1;
|
|
const sticky = matchedPrev > 0 && nextFree + shown.length < 1000;
|
|
const used = new Set<number>();
|
|
const interactives: (InteractiveElement & { isNew: boolean; context?: string })[] = shown.map((el, i) => {
|
|
let index = sticky ? prevIndexByKey.get(keyOf(el)) : undefined;
|
|
if (index == null || used.has(index)) index = sticky ? nextFree++ : i + 1;
|
|
used.add(index);
|
|
return {
|
|
index,
|
|
role: el.role,
|
|
name: el.name,
|
|
backendNodeId: el.backendNodeId,
|
|
sessionId: el.sessionId,
|
|
value: el.value,
|
|
isNew: isNew(el),
|
|
context: el.context,
|
|
};
|
|
});
|
|
|
|
// Cache in main-process so click_index can resolve across separate WS commands. role+name ride along so click_index can report WHAT it clicked (the agent loop records that as a stable, replayable click-by-name step).
|
|
const indexMap: Record<number, { backendNodeId: number; sessionId?: string; role?: string; name?: string }> = {};
|
|
for (const el of interactives) {
|
|
indexMap[el.index] = { backendNodeId: el.backendNodeId, sessionId: el.sessionId, role: el.role, name: el.name };
|
|
}
|
|
try {
|
|
const cacheBridge = (window as any).openswarm?.cdpCacheSet;
|
|
if (cacheBridge) await cacheBridge(wv.getWebContentsId(), indexMap);
|
|
} catch {
|
|
// best-effort; click_index falls back to re-listing.
|
|
}
|
|
|
|
// ctx only where it disambiguates: rows whose role+name appear more than once (eight "Message" buttons). Unique rows skip it to keep tokens lean.
|
|
const nameCounts = new Map<string, number>();
|
|
for (const el of interactives) {
|
|
const k = `${el.role}|${el.name}`;
|
|
nameCounts.set(k, (nameCounts.get(k) || 0) + 1);
|
|
}
|
|
const lines = interactives.map((el) => {
|
|
const dup = (nameCounts.get(`${el.role}|${el.name}`) || 0) > 1;
|
|
const ctx = dup && el.context ? ` ctx="${el.context}"` : '';
|
|
const val = el.value ? ` value="${el.value}"` : '';
|
|
return `[${el.index}]${el.isNew ? '*' : ''}<${el.role} "${el.name}"${ctx}${val}>`;
|
|
});
|
|
let text: string;
|
|
if (lines.length === 0) {
|
|
text = 'No interactive elements found on this page.';
|
|
} else {
|
|
text = `${lines.length} interactive elements (* = new since your last look; same number = same element as before):\n${lines.join('\n')}`;
|
|
if (truncated > 0) {
|
|
text += `\n... ${truncated} more not shown; scroll or scope with BrowserGetElements to reach them.`;
|
|
}
|
|
if (covered > 0) {
|
|
text += `\n(${covered} elements hidden behind overlays were omitted; close the overlay to reach them.)`;
|
|
}
|
|
}
|
|
|
|
return {
|
|
text,
|
|
elements: interactives.map((el) => ({ index: el.index, role: el.role, name: el.name })),
|
|
url: wv.getURL(),
|
|
};
|
|
}
|
|
|
|
async function handleClickIndex(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const idx = Number(params.index);
|
|
if (!Number.isFinite(idx) || idx < 1) {
|
|
return { error: 'index parameter is required and must be a positive integer' };
|
|
}
|
|
|
|
let backendNodeId: number | undefined;
|
|
let sessionId: string | undefined;
|
|
let role: string | undefined;
|
|
let name: string | undefined;
|
|
try {
|
|
const cacheBridge = (window as any).openswarm?.cdpCacheGet;
|
|
if (cacheBridge) {
|
|
const cached = await cacheBridge(wv.getWebContentsId());
|
|
const entry = cached && cached[idx];
|
|
if (typeof entry === 'number') {
|
|
backendNodeId = entry; // legacy cache shape
|
|
} else if (entry && typeof entry === 'object' && entry.backendNodeId != null) {
|
|
backendNodeId = Number(entry.backendNodeId);
|
|
sessionId = entry.sessionId || undefined;
|
|
role = entry.role; name = entry.name;
|
|
}
|
|
}
|
|
} catch {
|
|
// fall through to error path below
|
|
}
|
|
|
|
if (backendNodeId == null) {
|
|
return {
|
|
error: `Index ${idx} is not in the cached element map. Call BrowserListInteractives first to refresh the index, then try again.`,
|
|
};
|
|
}
|
|
|
|
const wantsText = typeof params.text === 'string' && params.text.length > 0;
|
|
const doClick = () => clickBackendNode(wv, backendNodeId as number, sessionId, `index ${idx}`,
|
|
{ role, text: wantsText ? params.text : undefined });
|
|
// Effect metric on plain clicks only (a fill verifies via readback); covers every clickBackendNode exit.
|
|
const result = (params.effectProbe === true && !wantsText)
|
|
? await measureClickEffect(wv, doClick)
|
|
: await doClick();
|
|
// Self-healing escalation: a cached index goes stale the instant the page mutates, which is HALF of all runs' tool-errors. An explicit clickBackendNode error PROVES the click never landed (so re-trying the same target can't double-act), so before we hand a ~3s re-strategize turn back to the model, resolve the SAME element fresh from the full DOM by its name+role, the exact rung the model would have climbed to itself. Plain clicks only (a text-fill has its own readback path); gated so an A/B can turn it off.
|
|
if (shouldSelfHealClick(!!result.error, wantsText, name, params.selfheal)) {
|
|
const healed = await handleClickByName(wv, { name: name as string, role: role || '' });
|
|
if (!healed.error) {
|
|
console.log(`[selfheal] index ${idx} stale -> recovered via name "${String(name).slice(0, 40)}"`);
|
|
healed.clickedRole = role || '';
|
|
healed.clickedName = name || '';
|
|
healed.selfHealed = 'by-name';
|
|
return healed;
|
|
}
|
|
}
|
|
// Surface what was clicked so the agent loop can record a stable, replayable click-by-name step (indices are ephemeral; names aren't).
|
|
if (!result.error) {
|
|
result.clickedRole = role || '';
|
|
result.clickedName = name || '';
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Attach a local file to an <input type="file"> without ever opening the OS picker. A native dialog
|
|
// lives outside the page, so an agent that clicks an Upload button can neither see nor dismiss it and
|
|
// the run dead-ends (ENG-47); CDP fills the field directly and the page's change handlers fire as if
|
|
// a human had picked it. Sites almost always hide the real input behind a styled button, so we find it
|
|
// ourselves rather than trusting a visible-element index.
|
|
async function handleUploadFile(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const path = String(params.path || '');
|
|
if (!path) return { error: 'path parameter is required' };
|
|
|
|
let backendNodeId: number | undefined;
|
|
let sessionId: string | undefined;
|
|
const idx = Number(params.index);
|
|
if (Number.isFinite(idx) && idx >= 1) {
|
|
try {
|
|
const cacheBridge = (window as any).openswarm?.cdpCacheGet;
|
|
const cached = cacheBridge ? await cacheBridge(wv.getWebContentsId()) : null;
|
|
const entry = cached && cached[idx];
|
|
if (typeof entry === 'number') backendNodeId = entry;
|
|
else if (entry && typeof entry === 'object' && entry.backendNodeId != null) {
|
|
backendNodeId = Number(entry.backendNodeId);
|
|
sessionId = entry.sessionId || undefined;
|
|
}
|
|
} catch { /* fall through to the page-wide search below */ }
|
|
}
|
|
|
|
let found = 0;
|
|
if (backendNodeId == null) {
|
|
try {
|
|
// Tier 1, cheap: a plain selector over the light DOM, which is where most upload inputs live.
|
|
const doc = await sendCdp(wv, 'DOM.getDocument', { depth: 0 });
|
|
const hits = await sendCdp(wv, 'DOM.querySelectorAll',
|
|
{ nodeId: doc?.root?.nodeId, selector: 'input[type=file]' });
|
|
const nodeIds: number[] = hits?.nodeIds || [];
|
|
found = nodeIds.length;
|
|
if (found) {
|
|
const described = await sendCdp(wv, 'DOM.describeNode', { nodeId: nodeIds[0] });
|
|
backendNodeId = described?.node?.backendNodeId;
|
|
}
|
|
} catch { /* tier 2 below is the real answer anyway */ }
|
|
}
|
|
if (backendNodeId == null) {
|
|
// Tier 2: querySelectorAll cannot see into a shadow root or a cross-origin iframe, and design
|
|
// systems put upload widgets in both. `pierce` walks through them. It returns the whole tree, so
|
|
// it stays behind the cheap tier rather than being the default.
|
|
try {
|
|
const pierced = await sendCdp(wv, 'DOM.getDocument', { depth: -1, pierce: true });
|
|
const inputs: number[] = [];
|
|
const walk = (n: any): void => {
|
|
if (!n || inputs.length > 20) return;
|
|
if (n.nodeName === 'INPUT') {
|
|
const a: string[] = n.attributes || [];
|
|
for (let i = 0; i < a.length - 1; i += 2) {
|
|
if (a[i] === 'type' && String(a[i + 1]).toLowerCase() === 'file') { inputs.push(n.backendNodeId); break; }
|
|
}
|
|
}
|
|
(n.children || []).forEach(walk);
|
|
(n.shadowRoots || []).forEach(walk);
|
|
if (n.contentDocument) walk(n.contentDocument);
|
|
};
|
|
walk(pierced?.root);
|
|
found = inputs.length;
|
|
if (!found) {
|
|
return { error: 'No file-upload field on this page, including inside its embedded frames. Open the page or dialog that has the upload control first, then retry.' };
|
|
}
|
|
backendNodeId = inputs[0];
|
|
} catch (err: any) {
|
|
return { error: `Could not search the page for an upload field (${err?.message || 'DOM query failed'}).` };
|
|
}
|
|
}
|
|
if (backendNodeId == null) return { error: 'Could not resolve the upload field on this page.' };
|
|
|
|
try {
|
|
await sendCdp(wv, 'DOM.setFileInputFiles', { files: [path], backendNodeId }, sessionId);
|
|
} catch (err: any) {
|
|
return { error: `Attaching the file failed (${err?.message || 'setFileInputFiles failed'}).` };
|
|
}
|
|
|
|
// Read the filename back off the input. "The command returned OK" is not "the page has the file".
|
|
try {
|
|
const resolved = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
|
const r = await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: resolved?.object?.objectId,
|
|
functionDeclaration: 'function() { const f = this.files; return f && f.length ? f[0].name + "|" + f[0].size : ""; }',
|
|
returnByValue: true,
|
|
}, sessionId);
|
|
const receipt = String(r?.result?.value || '');
|
|
if (!receipt) {
|
|
return { error: 'The upload field did not accept the file (it reports no file attached). The site may restrict the file type.' };
|
|
}
|
|
const [name, size] = receipt.split('|');
|
|
return {
|
|
text: `Attached "${name}" (${size} bytes) to the upload field${found > 1 ? ` (page has ${found} upload fields; used the first)` : ''}. The page's change handlers have fired. Any Submit/Save step is still yours to do.`,
|
|
uploadedName: name,
|
|
};
|
|
} catch {
|
|
return { text: `Sent "${path.split('/').pop()}" to the upload field, but could not read the field back to confirm it took. Check the page before submitting.` };
|
|
}
|
|
}
|
|
|
|
// Robust click for REPLAY: re-resolve the target fresh by (role, name) instead of a stale index, so a recorded skill survives index shifts between runs.
|
|
async function handleClickByName(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const wantName = String(params.name || '').trim();
|
|
const wantRole = String(params.role || '').trim();
|
|
if (!wantName && !wantRole) return { error: 'click_by_name needs a name and/or role' };
|
|
let candidates: RankItem[];
|
|
try {
|
|
candidates = await enumerateCandidates(wv);
|
|
} catch (err: any) {
|
|
return { error: `enumerate failed: ${err.message || String(err)}` };
|
|
}
|
|
const norm = (s: string) => s.trim().toLowerCase();
|
|
// Exact (role,name) first, then name-only, so we click the most specific match. Long names are card blobs whose SUFFIX mutates between visits (feed snippets, counters) while the prefix stays stable; fall back to a 40-char prefix match so a replayed click survives the churn.
|
|
const wantPrefix = norm(wantName).slice(0, 40);
|
|
const match =
|
|
candidates.find((c) => (!wantRole || norm(c.role) === norm(wantRole)) && norm(c.name) === norm(wantName)) ||
|
|
candidates.find((c) => norm(c.name) === norm(wantName)) ||
|
|
(wantName.length > 40
|
|
? candidates.find((c) => (!wantRole || norm(c.role) === norm(wantRole)) && norm(c.name).startsWith(wantPrefix))
|
|
: undefined);
|
|
if (!match) {
|
|
return { error: `No element matching role="${wantRole}" name="${wantName}" on this page.` };
|
|
}
|
|
return clickBackendNode(wv, match.backendNodeId, match.sessionId, `${match.role} "${match.name}"`,
|
|
{ role: match.role });
|
|
}
|
|
|
|
// Sequential sub-actions; aborts mid-batch if URL changes (indices/selectors go stale on navigation).
|
|
const MAX_BATCH_ACTIONS = 5;
|
|
|
|
type SubActionType =
|
|
| 'click_index' | 'press_key' | 'type' | 'wait'
|
|
| 'scroll' | 'navigate' | 'click' | 'click_point' | 'list_interactives';
|
|
|
|
const BATCH_DISPATCH: Record<SubActionType, (wv: BrowserWebview, p: Record<string, any>) => Promise<Record<string, any>>> = {
|
|
click_index: handleClickIndex,
|
|
press_key: handlePressKey,
|
|
click_point: handleClickPoint,
|
|
type: handleType,
|
|
wait: handleWait,
|
|
scroll: handleScroll,
|
|
navigate: handleNavigate,
|
|
click: handleClick,
|
|
// Allowed as the LAST sub-action so a click->wait->read folds into one turn.
|
|
list_interactives: handleListInteractives,
|
|
};
|
|
|
|
async function handleBatch(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const actions: any[] = Array.isArray(params.actions) ? params.actions : [];
|
|
if (actions.length === 0) {
|
|
return { error: 'actions parameter must be a non-empty array' };
|
|
}
|
|
if (actions.length > MAX_BATCH_ACTIONS) {
|
|
return {
|
|
error: `Batch too large: ${actions.length} actions (max ${MAX_BATCH_ACTIONS}). Split into smaller batches.`,
|
|
};
|
|
}
|
|
|
|
const results: Array<Record<string, any>> = [];
|
|
let aborted_at: number | null = null;
|
|
let abort_reason: string | null = null;
|
|
|
|
for (let i = 0; i < actions.length; i++) {
|
|
const action = actions[i];
|
|
const subType = action?.type as SubActionType;
|
|
const subParams = action?.params || {};
|
|
|
|
if (!subType || !(subType in BATCH_DISPATCH)) {
|
|
results.push({ index: i, type: subType, error: `Unknown sub-action type: ${subType}` });
|
|
// per-action failures don't abort the batch
|
|
continue;
|
|
}
|
|
|
|
const urlBefore = wv.getURL();
|
|
// Carry the self-heal + click-effect toggles into click sub-actions (most clicks are batched, so gating only the top-level click_index misses them).
|
|
if (subType === 'click_index') {
|
|
if (params.selfheal !== undefined && subParams.selfheal === undefined) subParams.selfheal = params.selfheal;
|
|
if (params.effectProbe && subParams.effectProbe === undefined) subParams.effectProbe = true;
|
|
}
|
|
let subResult: Record<string, any>;
|
|
try {
|
|
subResult = await BATCH_DISPATCH[subType](wv, subParams);
|
|
} catch (err: any) {
|
|
subResult = { error: `Sub-action failed: ${err?.message || String(err)}` };
|
|
}
|
|
results.push({ index: i, type: subType, ...subResult });
|
|
|
|
// A failed sub-action means every later one is operating on a page that isn't in the state it assumed, so stop instead of compounding the error (browser-use's multi_act breaks the same way). The terminal read is the last action, so a read failure never trips this.
|
|
if (subResult.error && i < actions.length - 1) {
|
|
aborted_at = i + 1;
|
|
abort_reason = `Sub-action ${i + 1} (${subType}) failed: ${subResult.error}; remaining ${actions.length - i - 1} action(s) skipped`;
|
|
break;
|
|
}
|
|
|
|
// URL changed: selectors and indices are stale on the half-loaded page; abort.
|
|
const urlAfter = wv.getURL();
|
|
if (urlAfter !== urlBefore && i < actions.length - 1) {
|
|
aborted_at = i + 1;
|
|
abort_reason = `URL changed mid-batch from ${urlBefore} to ${urlAfter}; remaining ${actions.length - i - 1} action(s) skipped`;
|
|
break;
|
|
}
|
|
}
|
|
|
|
const summary_lines = results.map((r, i) => {
|
|
const status = r.error ? `FAIL (${r.error})` : 'OK';
|
|
return ` ${i + 1}. ${r.type}: ${status}`;
|
|
});
|
|
const text = [
|
|
`Batch executed ${results.length}/${actions.length} actions`,
|
|
...summary_lines,
|
|
aborted_at !== null ? `\nABORTED at action ${aborted_at}: ${abort_reason}` : '',
|
|
].filter(Boolean).join('\n');
|
|
|
|
return {
|
|
text,
|
|
results,
|
|
aborted_at,
|
|
abort_reason,
|
|
url: wv.getURL(),
|
|
};
|
|
}
|
|
|
|
async function handleScroll(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const direction = (params.direction as string) || 'down';
|
|
const amount = (params.amount as number) || 500;
|
|
const code = `(() => {
|
|
function findScrollable() {
|
|
const candidates = document.querySelectorAll(
|
|
'[class*="scroller"], [class*="scroll-container"], [class*="content"], '
|
|
+ 'main, [role="main"], article, .notion-scroller, .notion-frame'
|
|
);
|
|
for (const el of candidates) {
|
|
const s = window.getComputedStyle(el);
|
|
const isScrollable = (s.overflow === 'auto' || s.overflow === 'scroll'
|
|
|| s.overflowY === 'auto' || s.overflowY === 'scroll');
|
|
if (isScrollable && el.scrollHeight > el.clientHeight + 10) return el;
|
|
}
|
|
const all = document.querySelectorAll('*');
|
|
for (const el of all) {
|
|
if (el === document.body || el === document.documentElement) continue;
|
|
const s = window.getComputedStyle(el);
|
|
const isScrollable = (s.overflow === 'auto' || s.overflow === 'scroll'
|
|
|| s.overflowY === 'auto' || s.overflowY === 'scroll');
|
|
if (isScrollable && el.scrollHeight > el.clientHeight + 50
|
|
&& el.clientHeight > 200) return el;
|
|
}
|
|
return null;
|
|
}
|
|
const dy = ${JSON.stringify(direction)} === 'up' ? -${amount} : ${amount};
|
|
const container = findScrollable();
|
|
if (container) {
|
|
const before = container.scrollTop;
|
|
container.scrollBy({ top: dy, behavior: 'instant' });
|
|
const after = container.scrollTop;
|
|
return {
|
|
scrolled: Math.abs(after - before),
|
|
scrollTop: after,
|
|
scrollHeight: container.scrollHeight,
|
|
clientHeight: container.clientHeight,
|
|
atTop: after <= 0,
|
|
atBottom: after + container.clientHeight >= container.scrollHeight - 5,
|
|
target: 'container',
|
|
};
|
|
}
|
|
const before = window.scrollY;
|
|
window.scrollBy({ top: dy, behavior: 'instant' });
|
|
const after = window.scrollY;
|
|
return {
|
|
scrolled: Math.abs(after - before),
|
|
scrollTop: after,
|
|
scrollHeight: document.documentElement.scrollHeight,
|
|
clientHeight: window.innerHeight,
|
|
atTop: after <= 0,
|
|
atBottom: after + window.innerHeight >= document.documentElement.scrollHeight - 5,
|
|
target: 'window',
|
|
};
|
|
})()`;
|
|
try {
|
|
const result = await evalInPage(wv, code);
|
|
const status = result.atBottom ? ' (reached bottom)' : result.atTop ? ' (reached top)' : '';
|
|
return {
|
|
text: `Scrolled ${direction} by ${result.scrolled}px${status}. Position: ${result.scrollTop}/${result.scrollHeight - result.clientHeight}px`,
|
|
...result,
|
|
url: wv.getURL(),
|
|
};
|
|
} catch (err: any) {
|
|
return { error: `Scroll failed: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
async function handleWait(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const ms = Math.min(Math.max((params.milliseconds as number) || 1000, 100), 10000);
|
|
const until = typeof params.until === 'string' ? params.until : '';
|
|
const probeJs = settleProbeJs(until);
|
|
const start = Date.now();
|
|
let settled = false;
|
|
let found = false;
|
|
let probeErrors = 0;
|
|
let lastElems: number | null = null;
|
|
let elemsChangedAt = start; // DOM-settle clock
|
|
while (Date.now() - start < ms) {
|
|
const remaining = ms - (Date.now() - start);
|
|
await new Promise((resolve) => setTimeout(resolve, Math.min(SETTLE_POLL_MS, Math.max(0, remaining))));
|
|
const elapsed = Date.now() - start;
|
|
if (elapsed >= ms) break;
|
|
try {
|
|
const probe = JSON.parse(await evalInPage(wv, probeJs));
|
|
probeErrors = 0;
|
|
if (probe.elems !== lastElems) { lastElems = probe.elems; elemsChangedAt = Date.now(); }
|
|
const domStable = Date.now() - elemsChangedAt;
|
|
if (shouldStopWaiting(probe.ready, probe.quiet || 0, domStable, !!probe.found, elapsed)) {
|
|
settled = true; found = !!probe.found; break;
|
|
}
|
|
} catch {
|
|
// Mid-navigation pages aren't evaluable yet; a few misses is normal, but a wedged tab shouldn't make us burn the whole cap, so bail after a short streak.
|
|
if (++probeErrors >= 3) break;
|
|
}
|
|
}
|
|
const waited = Date.now() - start;
|
|
const state = found ? 'found target' : settled ? 'page settled' : 'reached cap';
|
|
return {
|
|
text: `Waited ${waited}ms (${state}). Current URL: ${wv.getURL()}`,
|
|
url: wv.getURL(),
|
|
title: wv.getTitle(),
|
|
};
|
|
}
|
|
|
|
async function handleGetElements(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const scope = (params.selector as string) || 'body';
|
|
const safeScope = JSON.stringify(scope);
|
|
const code = `(() => {
|
|
const scope = document.querySelector(${safeScope}) || document.body;
|
|
const interactive = scope.querySelectorAll(
|
|
'a[href], button, input, textarea, select, [role="button"], [role="link"], '
|
|
+ '[role="textbox"], [role="searchbox"], [role="menuitem"], [role="tab"], '
|
|
+ '[role="checkbox"], [role="switch"], [role="option"], '
|
|
+ '[onclick], [tabindex]:not([tabindex="-1"]), '
|
|
+ '[data-block-id], [contenteditable="true"]'
|
|
);
|
|
const seen = new Set();
|
|
const results = [];
|
|
for (const el of interactive) {
|
|
if (results.length >= 80) break;
|
|
const rect = el.getBoundingClientRect();
|
|
if (rect.width === 0 && rect.height === 0) continue;
|
|
const style = window.getComputedStyle(el);
|
|
if (style.visibility === 'hidden' || style.display === 'none') continue;
|
|
if (style.opacity === '0') continue;
|
|
|
|
let selector = el.tagName.toLowerCase();
|
|
if (el.id) {
|
|
selector = '#' + CSS.escape(el.id);
|
|
} else if (el.getAttribute('data-block-id')) {
|
|
selector = '[data-block-id="' + el.getAttribute('data-block-id') + '"]';
|
|
} else if (el.getAttribute('name')) {
|
|
selector = el.tagName.toLowerCase() + '[name="' + CSS.escape(el.getAttribute('name')) + '"]';
|
|
} else if (el.getAttribute('aria-label')) {
|
|
selector = el.tagName.toLowerCase() + '[aria-label="' + CSS.escape(el.getAttribute('aria-label')) + '"]';
|
|
} else if (el.getAttribute('type') && el.tagName === 'INPUT') {
|
|
selector = 'input[type="' + el.getAttribute('type') + '"]';
|
|
if (el.getAttribute('placeholder'))
|
|
selector += '[placeholder="' + CSS.escape(el.getAttribute('placeholder')) + '"]';
|
|
} else if (el.className && typeof el.className === 'string') {
|
|
const cls = el.className.trim().split(/\\s+/)[0];
|
|
if (cls && cls.length < 60)
|
|
selector = el.tagName.toLowerCase() + '.' + CSS.escape(cls);
|
|
}
|
|
|
|
if (seen.has(selector)) {
|
|
const parent = el.parentElement;
|
|
if (parent && parent.id) {
|
|
selector = '#' + CSS.escape(parent.id) + ' > ' + selector;
|
|
} else {
|
|
const siblings = parent ? Array.from(parent.children) : [];
|
|
const idx = siblings.indexOf(el);
|
|
if (idx >= 0) selector += ':nth-child(' + (idx + 1) + ')';
|
|
}
|
|
}
|
|
seen.add(selector);
|
|
|
|
results.push({
|
|
selector,
|
|
tag: el.tagName.toLowerCase(),
|
|
type: el.type || null,
|
|
text: (el.textContent || '').trim().substring(0, 120) || null,
|
|
placeholder: el.placeholder || null,
|
|
ariaLabel: el.getAttribute('aria-label') || null,
|
|
role: el.getAttribute('role') || null,
|
|
href: el.href && el.href !== location.href ? el.href : null,
|
|
});
|
|
}
|
|
return { elements: results, total: interactive.length, url: location.href, title: document.title };
|
|
})()`;
|
|
try {
|
|
const result = await evalInPage(wv, code);
|
|
return { text: JSON.stringify(result, null, 2), url: wv.getURL() };
|
|
} catch (err: any) {
|
|
return { error: `Failed to get elements: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
// Tier 1: detect a site's declared WebMCP tools (navigator.modelContext). When a site exposes its own tools the agent can prefer them over scraping the UI. The API is a Chrome 149 origin-trial standard; this Electron's Chromium predates it so real pages return "not present" today, this is forward-compatible probing, also covers the MCP-B convention (getRegisteredTools/listTools/tools array).
|
|
async function handleDetectWebMCP(wv: BrowserWebview): Promise<Record<string, any>> {
|
|
const code = `(() => {
|
|
const mc = navigator.modelContext;
|
|
if (!mc) return { present: false, tools: [] };
|
|
let raw = [];
|
|
try {
|
|
if (typeof mc.getRegisteredTools === 'function') raw = mc.getRegisteredTools() || [];
|
|
else if (typeof mc.listTools === 'function') raw = mc.listTools() || [];
|
|
else if (Array.isArray(mc.tools)) raw = mc.tools;
|
|
} catch (e) {}
|
|
const tools = (raw || []).map(t => ({
|
|
name: String((t && t.name) || ''),
|
|
description: String((t && t.description) || '').slice(0, 200),
|
|
})).filter(t => t.name);
|
|
return { present: true, tools };
|
|
})()`;
|
|
try {
|
|
const r = await evalInPage(wv, code);
|
|
if (!r || !r.present) {
|
|
return { text: 'No WebMCP on this page (navigator.modelContext not present). Use the normal browser tools.', url: wv.getURL() };
|
|
}
|
|
if (!r.tools.length) {
|
|
return { text: 'WebMCP is present but exposes no callable tools. Use the normal browser tools.', url: wv.getURL() };
|
|
}
|
|
const lines = r.tools.map((t: any) => `- ${t.name}: ${t.description}`).join('\n');
|
|
return { text: `WebMCP tools declared by this page:\n${lines}`, tools: r.tools, url: wv.getURL() };
|
|
} catch (err: any) {
|
|
return { error: `WebMCP detection failed: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
// Tier 2: the API routes captured for the current site, so the agent can act directly instead of
|
|
// re-scraping/clicking the UI. Default lists the safe GET/HEAD routes (all replay_route will run).
|
|
// With { writes: true } it lists the MUTATING routes (POST/PUT/PATCH/DELETE) the site's own UI
|
|
// fired, for BrowserApiWrite's general 'route' path; the write itself is same-origin + captured +
|
|
// session-borrowed + flag-gated in the backend, this only SURFACES the endpoint shape.
|
|
async function handleListRoutes(wv: BrowserWebview, params?: Record<string, any>): Promise<Record<string, any>> {
|
|
const bridge = (window as any).openswarm?.cdpRoutesGet as
|
|
| ((id: number, origin?: string) => Promise<any[]>) | undefined;
|
|
if (!bridge) return { error: 'Route capture not available, restart the app.' };
|
|
let origin = '';
|
|
try { origin = new URL(wv.getURL()).origin; } catch {}
|
|
let routes: any[] = [];
|
|
try { routes = (await bridge(wv.getWebContentsId(), origin)) || []; } catch {}
|
|
|
|
if (params?.writes) {
|
|
const writes = routes.filter((r) => r && r.safe === false);
|
|
if (!writes.length) {
|
|
return { text: 'No write (POST/PUT/PATCH/DELETE) API routes captured for this site yet. Do the write once through the UI so it gets recorded, then the route path can replay it.', url: wv.getURL() };
|
|
}
|
|
const wlines = writes.slice(0, 40).map((r) => `${r.method} ${r.template} body-shape: ${JSON.stringify(r.bodyShape)} (seen ${r.hits}x)`);
|
|
return {
|
|
text: `Write endpoints this site's UI uses (for BrowserApiWrite action='route'). Pass the `
|
|
+ `method + url + a body matching the shape, with your content in the text field:\n${wlines.join('\n')}`,
|
|
routes: writes.slice(0, 40),
|
|
url: wv.getURL(),
|
|
};
|
|
}
|
|
|
|
const safe = routes.filter((r) => r && r.safe);
|
|
if (!safe.length) {
|
|
return { text: 'No replayable (GET) API routes captured for this site yet. Use the page first so they get recorded, then try again.', url: wv.getURL() };
|
|
}
|
|
const lines = safe.slice(0, 40).map((r) => `${r.method} ${r.example || r.template} (seen ${r.hits}x)`);
|
|
return {
|
|
text: `Replayable API routes for this site (safe GETs). To READ the same kind of `
|
|
+ `data for many inputs fast: swap the varying value in the URL with {{value}} `
|
|
+ `and use a replay_route step in BrowserRepeatFlow (or call BrowserReplayRoute `
|
|
+ `per item). Far cheaper than navigating + scraping each page:\n${lines.join('\n')}`,
|
|
routes: safe.slice(0, 40),
|
|
url: wv.getURL(),
|
|
};
|
|
}
|
|
|
|
// Tier 2: replay a captured endpoint directly. GET/HEAD only (idempotent) and same-origin only; the fetch runs IN the page so cookies/CSRF come for free. Mutating methods are intentionally refused, those must go through the UI.
|
|
async function handleReplayRoute(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const rawUrl = params.url as string;
|
|
const method = String(params.method || 'GET').toUpperCase();
|
|
if (!rawUrl) return { error: 'url parameter is required' };
|
|
if (method !== 'GET' && method !== 'HEAD') {
|
|
return { error: `BrowserReplayRoute only runs safe GET/HEAD requests. ${method} changes data, do that through the UI (click the button) instead.` };
|
|
}
|
|
let absUrl: string;
|
|
let pageOrigin: string;
|
|
try {
|
|
pageOrigin = new URL(wv.getURL()).origin;
|
|
absUrl = new URL(rawUrl, wv.getURL()).href;
|
|
} catch {
|
|
return { error: 'invalid url' };
|
|
}
|
|
if (new URL(absUrl).origin !== pageOrigin) {
|
|
return { error: "BrowserReplayRoute can only call the current site's own API (same origin)." };
|
|
}
|
|
const code = `(async () => {
|
|
try {
|
|
const r = await fetch(${JSON.stringify(absUrl)}, { method: ${JSON.stringify(method)}, credentials: 'include' });
|
|
const body = await r.text();
|
|
return { status: r.status, body: body.slice(0, 15000) };
|
|
} catch (e) { return { error: String((e && e.message) || e) }; }
|
|
})()`;
|
|
try {
|
|
const res = await evalInPage(wv, code);
|
|
if (res.error) return { error: `Replay failed: ${res.error}` };
|
|
return { text: `${method} ${absUrl} -> HTTP ${res.status}\n${res.body}`, status: res.status, url: wv.getURL() };
|
|
} catch (err: any) {
|
|
return { error: `Replay failed: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
async function handleEvaluate(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const expression = params.expression as string;
|
|
if (!expression) return { error: 'expression parameter is required' };
|
|
try {
|
|
const result = await evalInPage(wv, expression);
|
|
const text = typeof result === 'string' ? result : JSON.stringify(result, null, 2);
|
|
// evaluate is the agent's main read path; sample routes here too (XHRs have fired by now) so the backend can surface the fast network tier once.
|
|
const routes_available = await countSafeRoutes(wv);
|
|
return { text: text ?? 'undefined', url: wv.getURL(), routes_available };
|
|
} catch (err: any) {
|
|
return { error: `JS evaluation error: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
// The registry is renderer-local and a card briefly unregisters on remount / tab-switch; a command landing in that gap shouldn't hard-fail. Wait a bounded window for (re)registration before giving up, so the error stays a real "card is gone" signal rather than a transient race.
|
|
async function awaitWebview(browserId: string, tabId?: string, action?: string): Promise<BrowserWebview | undefined> {
|
|
// A suspended (snapshot-swapped) card has no webview at all; wake it and wait out the remount + page reload before the command touches it.
|
|
const wasSuspended = !!store.getState().dashboardLayout.suspendedBrowserCards[browserId];
|
|
if (wasSuspended) store.dispatch(resumeBrowserCard(browserId));
|
|
const deadline = Date.now() + (wasSuspended ? 12000 : 2000);
|
|
let wv = getWebview(browserId, tabId);
|
|
while (!wv && Date.now() < deadline) {
|
|
await new Promise((r) => setTimeout(r, 100));
|
|
wv = getWebview(browserId, tabId);
|
|
}
|
|
if (wasSuspended && wv) {
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
if (!wv.isLoading() && wv.getURL() !== 'about:blank') break;
|
|
} catch {
|
|
// mid-mount hiccup; keep waiting
|
|
}
|
|
await new Promise((r) => setTimeout(r, 150));
|
|
}
|
|
}
|
|
// A lazy background tab mounts at about:blank with its real page deferred; an agent command needs
|
|
// the real page, so wake it and wait out the load, same as a resumed suspended card. A navigate
|
|
// is about to load its own url, so just drop the deferred load instead of loading the old one first.
|
|
if (wv && isPendingLoad(wv)) {
|
|
if (action === 'navigate') {
|
|
clearPendingLoad(wv);
|
|
} else if (wakePendingLoad(wv)) {
|
|
const loadDeadline = Date.now() + 12000;
|
|
while (Date.now() < loadDeadline) {
|
|
try {
|
|
if (!wv.isLoading() && wv.getURL() !== 'about:blank') break;
|
|
} catch {
|
|
// mid-load hiccup; keep waiting
|
|
}
|
|
await new Promise((r) => setTimeout(r, 150));
|
|
}
|
|
}
|
|
}
|
|
return wv;
|
|
}
|
|
|
|
// The backend re-broadcasts unanswered commands to heal a dead-socket gap, so a duplicate request_id must never run twice (a re-sent click would double-click).
|
|
const inflightCommands = new Set<string>();
|
|
const completedCommands = new Map<string, Record<string, any>>();
|
|
const _COMPLETED_CACHE_MAX = 50;
|
|
|
|
// Agent commands focus elements INSIDE the guest page, and a guest focus pulls the window's focus off whatever the user is typing in, so every agent action yanked the cursor (ENG-226). Capture the user's host focus target before the command; put it back ONLY if the command moved focus to the agent's webview or nowhere, never if the user deliberately clicked elsewhere mid-command.
|
|
function captureHostFocus(): () => void {
|
|
const before = document.activeElement as HTMLElement | null;
|
|
const isUserSurface = !!before && before.tagName !== 'WEBVIEW' && before.tagName !== 'BODY'
|
|
&& (before.tagName === 'INPUT' || before.tagName === 'TEXTAREA' || before.isContentEditable);
|
|
if (!isUserSurface) return () => {};
|
|
return () => {
|
|
const now = document.activeElement as HTMLElement | null;
|
|
const stolen = !now || now === document.body || now.tagName === 'WEBVIEW';
|
|
if (stolen && before && before.isConnected) {
|
|
try { before.focus({ preventScroll: true }); } catch { /* surface unmounted mid-command */ }
|
|
}
|
|
};
|
|
}
|
|
|
|
async function handleBrowserCommand(data: Record<string, any>) {
|
|
const { request_id, action, browser_id, tab_id, params = {} } = data;
|
|
if (!request_id) return;
|
|
if (inflightCommands.has(request_id)) return;
|
|
const cached = completedCommands.get(request_id);
|
|
if (cached) {
|
|
dashboardWs.send('browser:result', { request_id, ...cached });
|
|
return;
|
|
}
|
|
inflightCommands.add(request_id);
|
|
const restoreFocus = captureHostFocus();
|
|
try {
|
|
await runBrowserCommand(request_id, action, browser_id, tab_id, params);
|
|
} finally {
|
|
inflightCommands.delete(request_id);
|
|
restoreFocus();
|
|
}
|
|
}
|
|
|
|
// Hand a vetted social platform's partition cookies to its session-backed MCP shim. No webview needed: it reads the main-process cookie store directly, so it runs before the webview lookup.
|
|
async function handleSessionCookies(params: Record<string, any>): Promise<Record<string, any>> {
|
|
const bridge = (window as any).openswarm?.getPartitionCookies as
|
|
| ((domain: string) => Promise<{ cookies: { name: string; value: string }[]; userAgent: string; error?: string }>)
|
|
| undefined;
|
|
if (!bridge) return { error: 'Cookie bridge unavailable (desktop app only)', cookies: [] };
|
|
try {
|
|
return await bridge(String(params.domain || ''));
|
|
} catch (err: any) {
|
|
return { error: `Cookie bridge failed: ${err?.message || String(err)}`, cookies: [] };
|
|
}
|
|
}
|
|
|
|
// Load the user's own existing sign-in for a site into the browser partition, so an agent stuck at
|
|
// a login wall can carry on as them instead of interrupting to ask for a password. Like the cookie
|
|
// bridge above this needs no webview: it writes straight to the main-process cookie store.
|
|
async function handleImportSession(params: Record<string, any>): Promise<Record<string, any>> {
|
|
const bridge = (window as any).openswarm?.setPartitionCookies as
|
|
| ((domain: string, cookies: Record<string, any>[]) => Promise<{ ok: boolean; set: number; error?: string }>)
|
|
| undefined;
|
|
if (!bridge) return { ok: false, set: 0, error: 'Session import unavailable (desktop app only)' };
|
|
const cookies = Array.isArray(params.cookies) ? params.cookies : [];
|
|
try {
|
|
return await bridge(String(params.domain || ''), cookies);
|
|
} catch (err: any) {
|
|
return { ok: false, set: 0, error: `Session import failed: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
// Drive a session-borrow site's own already-open card: resolve the webview by its live domain
|
|
// (no browser_id, like the cookie bridge), then run a small navigate/evaluate step sequence.
|
|
// The shims use this for writes on sites that sign every HTTP request (TikTok).
|
|
async function handlePerformAction(params: Record<string, any>): Promise<Record<string, any>> {
|
|
const domain = String(params.domain || '').toLowerCase().replace(/^\./, '');
|
|
const wv = findWebviewByDomain(domain);
|
|
if (!wv) {
|
|
return { error: `No ${domain} browser card is open. Open ${domain} in an OpenSwarm browser card and sign in, then retry.` };
|
|
}
|
|
// findWebviewByDomain can resolve a deferred background tab by its intended url; wake it and wait out the load before driving it, so the session-borrow shims never act on an about:blank tab.
|
|
if (isPendingLoad(wv) && wakePendingLoad(wv)) {
|
|
const loadDeadline = Date.now() + 12000;
|
|
while (Date.now() < loadDeadline) {
|
|
try {
|
|
if (!wv.isLoading() && wv.getURL() !== 'about:blank') break;
|
|
} catch {
|
|
// mid-load hiccup; keep waiting
|
|
}
|
|
await new Promise((res) => setTimeout(res, 150));
|
|
}
|
|
}
|
|
const steps = Array.isArray(params.steps) ? params.steps : [];
|
|
const results: Record<string, any>[] = [];
|
|
for (const step of steps) {
|
|
const op = String(step?.op || '');
|
|
let r: Record<string, any>;
|
|
if (op === 'navigate') r = await handleNavigate(wv, { url: step.url });
|
|
else if (op === 'evaluate') r = await handleEvaluate(wv, { expression: step.expression });
|
|
else if (op === 'wait') { await new Promise((res) => setTimeout(res, Math.min(Number(step.ms) || 0, 8000))); r = { text: 'waited' }; }
|
|
else r = { error: `Unknown perform_action op: ${op}` };
|
|
results.push(r);
|
|
if (r && r.error) return { error: r.error, results };
|
|
}
|
|
return { ok: true, results };
|
|
}
|
|
|
|
async function runBrowserCommand(
|
|
request_id: string, action: string, browser_id: string, tab_id: string | undefined,
|
|
params: Record<string, any>,
|
|
) {
|
|
if (action === 'get_session_cookies') {
|
|
const result = await handleSessionCookies(params);
|
|
dashboardWs.send('browser:result', { request_id, ...result });
|
|
return;
|
|
}
|
|
if (action === 'perform_action') {
|
|
const result = await handlePerformAction(params);
|
|
dashboardWs.send('browser:result', { request_id, ...result });
|
|
return;
|
|
}
|
|
if (action === 'import_session') {
|
|
const result = await handleImportSession(params);
|
|
dashboardWs.send('browser:result', { request_id, ...result });
|
|
return;
|
|
}
|
|
const p_gateT0 = Date.now();
|
|
const wv = await awaitWebview(browser_id, tab_id || undefined, action);
|
|
// A command that spends seconds before its handler even starts looks identical, from the backend,
|
|
// to a slow handler. Splitting the two is the whole diagnosis for the 15s screenshot wedge, so
|
|
// say which half ate the time. Only fires when it is genuinely slow, so a healthy run stays quiet.
|
|
const p_gateMs = Date.now() - p_gateT0;
|
|
if (!wv) {
|
|
dashboardWs.send('browser:result', {
|
|
request_id,
|
|
error: `Browser card '${browser_id}'${tab_id ? ` tab '${tab_id}'` : ''} not found or not an Electron webview`,
|
|
});
|
|
return;
|
|
}
|
|
|
|
const detail = params.url || params.selector || params.expression || undefined;
|
|
setActivity(browser_id, { action: action as BrowserAction, detail });
|
|
|
|
let result: Record<string, any>;
|
|
try {
|
|
switch (action) {
|
|
case 'screenshot':
|
|
result = await handleScreenshot(wv, browser_id, params);
|
|
break;
|
|
case 'get_text':
|
|
result = await handleGetText(wv, params);
|
|
break;
|
|
case 'get_console':
|
|
result = await handleGetConsole(wv);
|
|
break;
|
|
case 'navigate':
|
|
result = await handleNavigate(wv, params);
|
|
break;
|
|
case 'click':
|
|
result = await handleClick(wv, params);
|
|
if (result.clickX != null && result.clickY != null) {
|
|
setActivity(browser_id, {
|
|
action: 'click',
|
|
detail,
|
|
coords: { xPercent: result.clickX, yPercent: result.clickY },
|
|
});
|
|
}
|
|
break;
|
|
case 'type':
|
|
result = await handleType(wv, params);
|
|
break;
|
|
case 'find_composer':
|
|
result = await handleFindComposer(wv, params);
|
|
break;
|
|
case 'evaluate':
|
|
result = await handleEvaluate(wv, params);
|
|
break;
|
|
case 'get_elements':
|
|
result = await handleGetElements(wv, params);
|
|
break;
|
|
case 'scroll':
|
|
result = await handleScroll(wv, params);
|
|
break;
|
|
case 'wait':
|
|
result = await handleWait(wv, params);
|
|
break;
|
|
case 'press_key':
|
|
result = await handlePressKey(wv, params);
|
|
break;
|
|
case 'list_interactives':
|
|
result = await handleListInteractives(wv, params);
|
|
break;
|
|
case 'upload_file':
|
|
result = await handleUploadFile(wv, params);
|
|
break;
|
|
case 'click_index':
|
|
result = await handleClickIndex(wv, params);
|
|
if (result.clickX != null && result.clickY != null) {
|
|
setActivity(browser_id, {
|
|
action: 'click_index',
|
|
detail,
|
|
coords: { xPercent: result.clickX, yPercent: result.clickY },
|
|
});
|
|
}
|
|
break;
|
|
case 'click_point':
|
|
result = await handleClickPoint(wv, params);
|
|
if (result.clickX != null && result.clickY != null) {
|
|
setActivity(browser_id, {
|
|
action: 'click_point',
|
|
detail,
|
|
coords: { xPercent: result.clickX, yPercent: result.clickY },
|
|
});
|
|
}
|
|
break;
|
|
case 'batch':
|
|
result = await handleBatch(wv, params);
|
|
break;
|
|
case 'detect_webmcp':
|
|
result = await handleDetectWebMCP(wv);
|
|
break;
|
|
case 'list_routes':
|
|
result = await handleListRoutes(wv, params);
|
|
break;
|
|
case 'click_by_name':
|
|
result = await handleClickByName(wv, params);
|
|
if (result.clickX != null && result.clickY != null) {
|
|
setActivity(browser_id, { action: 'click_by_name', detail, coords: { xPercent: result.clickX, yPercent: result.clickY } });
|
|
}
|
|
break;
|
|
case 'replay_route':
|
|
result = await handleReplayRoute(wv, params);
|
|
break;
|
|
default:
|
|
result = { error: `Unknown browser action: ${action}` };
|
|
}
|
|
} catch (err: any) {
|
|
result = { error: `Browser command failed: ${err?.message || String(err)}` };
|
|
}
|
|
|
|
setActivity(browser_id, null);
|
|
completedCommands.set(request_id, result);
|
|
if (completedCommands.size > _COMPLETED_CACHE_MAX) {
|
|
completedCommands.delete(completedCommands.keys().next().value as string);
|
|
}
|
|
// Ride the pre-handler wait back with the result: a renderer console.log never reaches the main
|
|
// process, and from the backend a slow GATE and a slow HANDLER look identical.
|
|
dashboardWs.send('browser:result', { request_id, ...result, gate_ms: p_gateMs, total_ms: Date.now() - p_gateT0 });
|
|
}
|
|
|
|
export function initBrowserCommandHandler(): () => void {
|
|
if (initialized) return () => {};
|
|
initialized = true;
|
|
const unsub = dashboardWs.on('browser:command', handleBrowserCommand);
|
|
return () => {
|
|
unsub();
|
|
initialized = false;
|
|
};
|
|
}
|