mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
2146 lines
105 KiB
TypeScript
2146 lines
105 KiB
TypeScript
import { getWebview, findWebviewByDomain, hasDomReady, markDomReady, isPendingLoad, wakePendingLoad, clearPendingLoad, type BrowserWebview } from './browserRegistry';
|
|
import { shouldSelfHealClick } from './selfHealClick';
|
|
import { FP_EXPR, clickEffect } from './clickEffect';
|
|
import { store } from './state/store';
|
|
import { resumeBrowserCard } from './state/dashboardLayoutSlice';
|
|
import { dashboardWs } from './ws/WebSocketManager';
|
|
import { resolveInput } from './resolveUrl';
|
|
import { rankAndCapInteractives, type RankItem } from './interactiveRanking';
|
|
import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettle';
|
|
import { unwrapCdpEval } from './cdpEval';
|
|
|
|
let initialized = false;
|
|
|
|
export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'click_point' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name' | 'find_composer';
|
|
|
|
export interface BrowserActivity {
|
|
action: BrowserAction;
|
|
detail?: string;
|
|
coords?: { xPercent: number; yPercent: number };
|
|
}
|
|
|
|
type ActivityListener = (browserId: string, activity: BrowserActivity | null) => void;
|
|
|
|
const activityMap = new Map<string, BrowserActivity>();
|
|
const listeners = new Set<ActivityListener>();
|
|
|
|
// A webview keeps churning for a beat after an action lands; capturing it into the dashboard snapshot during that churn is what crashes the renderer (SharedImage 'non-existent mailbox' -> V8 ToLocalChecked), so we hold "busy" this long past the last command before letting the thumbnail capture run again.
|
|
const BUSY_COOLDOWN_MS = 1500;
|
|
let lastActivityAt = 0;
|
|
|
|
function setActivity(browserId: string, activity: BrowserActivity | null) {
|
|
lastActivityAt = Date.now();
|
|
if (activity) {
|
|
activityMap.set(browserId, activity);
|
|
} else {
|
|
activityMap.delete(browserId);
|
|
}
|
|
listeners.forEach((fn) => fn(browserId, activity));
|
|
}
|
|
|
|
export function getActivity(browserId: string): BrowserActivity | null {
|
|
return activityMap.get(browserId) ?? null;
|
|
}
|
|
|
|
// True while an agent is actively driving any browser webview (a command is in flight, or one finished within the cooldown). The dashboard thumbnail capture checks this and skips rather than screenshot a live, churning webview.
|
|
export function isAnyBrowserBusy(): boolean {
|
|
if (activityMap.size > 0) return true;
|
|
return Date.now() - lastActivityAt < BUSY_COOLDOWN_MS;
|
|
}
|
|
|
|
export function subscribeActivity(fn: ActivityListener): () => void {
|
|
listeners.add(fn);
|
|
return () => { listeners.delete(fn); };
|
|
}
|
|
|
|
const ACTION_LABELS: Record<string, string> = {
|
|
screenshot: 'Capturing...',
|
|
get_text: 'Reading...',
|
|
navigate: 'Navigating...',
|
|
click: 'Clicking...',
|
|
type: 'Typing...',
|
|
evaluate: 'Evaluating...',
|
|
get_elements: 'Inspecting...',
|
|
scroll: 'Scrolling...',
|
|
wait: 'Waiting...',
|
|
press_key: 'Pressing key...',
|
|
list_interactives: 'Reading page structure...',
|
|
click_index: 'Clicking element...',
|
|
click_point: 'Tapping screen...',
|
|
click_by_name: 'Clicking element...',
|
|
batch: 'Running batch...',
|
|
};
|
|
|
|
export function getActionLabel(action: string): string {
|
|
return ACTION_LABELS[action] ?? 'Working...';
|
|
}
|
|
|
|
// Draw each cached element's index as a colored chip on the live page right before capture (browser-use's trick): the screenshot then speaks the same numbers as BrowserListInteractives, so the vision side can act by index.
|
|
const _ANNOTATION_COLORS = ['#e5484d', '#0091ff', '#30a46c', '#f76b15', '#8e4ec6', '#00a2c7'];
|
|
const _ANNOTATE_BUDGET_MS = 1500;
|
|
|
|
// One unsettled CDP bridge promise must never hang the screenshot; race each call.
|
|
function _cdpTimeout<T>(p: Promise<T>, ms: number): Promise<T> {
|
|
return Promise.race([p, new Promise<T>((_, rej) => setTimeout(() => rej(new Error('cdp call timed out')), ms))]);
|
|
}
|
|
|
|
async function annotateElements(wv: BrowserWebview): Promise<number> {
|
|
const cacheBridge = (window as any).openswarm?.cdpCacheGet;
|
|
const cached = cacheBridge ? await _cdpTimeout(cacheBridge(wv.getWebContentsId()), 500) : null;
|
|
if (!cached || typeof cached !== 'object') return 0;
|
|
const deadline = Date.now() + _ANNOTATE_BUDGET_MS;
|
|
const drawOne = async (idxStr: string, entry: any): Promise<boolean> => {
|
|
const backendNodeId = typeof entry === 'number' ? entry : entry?.backendNodeId;
|
|
// v1 is root-frame only: OOPIF rects are frame-local and would land wrong
|
|
if (!backendNodeId || entry?.sessionId) return false;
|
|
try {
|
|
const t = await _cdpTimeout(sendCdp(wv, 'DOM.resolveNode', { backendNodeId }), 300);
|
|
const r = await _cdpTimeout(sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: t.object.objectId,
|
|
functionDeclaration:
|
|
'function(idx, color) {'
|
|
+ ' const r = this.getBoundingClientRect();'
|
|
+ ' if (r.width <= 0 || r.height <= 0) return false;'
|
|
+ ' if (r.bottom < 0 || r.top > innerHeight || r.right < 0 || r.left > innerWidth) return false;'
|
|
+ ' let c = document.getElementById("__osw_annotations__");'
|
|
+ ' if (!c) { c = document.createElement("div"); c.id = "__osw_annotations__";'
|
|
+ ' c.style.cssText = "position:fixed;inset:0;z-index:2147483647;pointer-events:none;";'
|
|
+ ' document.documentElement.appendChild(c); }'
|
|
+ ' const box = document.createElement("div");'
|
|
+ ' box.style.cssText = "position:fixed;left:" + r.left + "px;top:" + r.top + "px;width:" + r.width + "px;height:" + r.height + "px;border:2px solid " + color + ";box-sizing:border-box;";'
|
|
+ ' const tag = document.createElement("span");'
|
|
+ ' tag.textContent = String(idx);'
|
|
+ ' tag.style.cssText = "position:absolute;left:-2px;top:-16px;background:" + color + ";color:#fff;font:bold 11px/14px monospace;padding:0 4px;border-radius:2px;";'
|
|
+ ' if (r.top < 18) { tag.style.top = "-2px"; }'
|
|
+ ' box.appendChild(tag); c.appendChild(box); return true;'
|
|
+ ' }',
|
|
arguments: [{ value: Number(idxStr) }, { value: _ANNOTATION_COLORS[Number(idxStr) % _ANNOTATION_COLORS.length] }],
|
|
returnByValue: true,
|
|
}), 300);
|
|
return r?.result?.value === true;
|
|
} catch { return false; } // node gone or call timed out; skip
|
|
};
|
|
let drawn = 0;
|
|
const entries = Object.entries(cached);
|
|
const CHUNK = 10;
|
|
for (let i = 0; i < entries.length && Date.now() < deadline; i += CHUNK) {
|
|
const results = await Promise.allSettled(
|
|
entries.slice(i, i + CHUNK).map(([idxStr, e]) => drawOne(idxStr, e)),
|
|
);
|
|
drawn += results.filter((r) => r.status === 'fulfilled' && r.value === true).length;
|
|
}
|
|
return drawn;
|
|
}
|
|
|
|
async function removeAnnotations(wv: BrowserWebview): Promise<void> {
|
|
try {
|
|
await _cdpTimeout(sendCdp(wv, 'Runtime.evaluate', {
|
|
expression: 'document.getElementById("__osw_annotations__")?.remove()',
|
|
}), 800);
|
|
} catch { /* page navigated mid-capture; the overlay died with it */ }
|
|
}
|
|
|
|
async function handleScreenshot(wv: BrowserWebview, params?: Record<string, any>): Promise<Record<string, any>> {
|
|
if (params?.annotate !== false) {
|
|
let drawn = 0;
|
|
try {
|
|
drawn = await annotateElements(wv);
|
|
if (drawn > 0) {
|
|
const shot = await captureRetry(wv);
|
|
if (shot.image) shot.text = `Screenshot with ${drawn} numbered boxes matching your element list (pass annotate:false for a clean shot).`;
|
|
return shot;
|
|
}
|
|
} catch { /* annotation is decoration; a plain shot always beats an error */
|
|
} finally {
|
|
if (drawn > 0) await removeAnnotations(wv);
|
|
}
|
|
}
|
|
return captureRetry(wv);
|
|
}
|
|
|
|
async function captureRetry(wv: BrowserWebview): Promise<Record<string, any>> {
|
|
// capturePage throws UnknownVizError if the webview hasn't composited a frame yet (the Viz compositor races the first paint, reliably bit turn-0 captures). Retry a few times with a short backoff so a cold first screenshot succeeds instead of burning a whole agent turn on a transient error.
|
|
let lastErr: any;
|
|
for (let attempt = 0; attempt < 4; attempt++) {
|
|
try {
|
|
const nativeImage = await wv.capturePage();
|
|
if (!nativeImage.isEmpty()) {
|
|
// Stable PNG capture. The resize()+toJPEG() variant was reverted: it's the prime suspect for the renderer "V8 Empty MaybeLocal" crash, NativeImage's JPEG codec returns an empty image on some retina captures, which is the shape of that native fault. A stable app beats a faster screenshot.
|
|
const dataUrl = nativeImage.toDataURL();
|
|
const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, '');
|
|
return { image: base64, url: wv.getURL(), title: wv.getTitle() };
|
|
}
|
|
lastErr = new Error('capturePage returned an empty image (frame not painted yet)');
|
|
} catch (err: any) {
|
|
lastErr = err;
|
|
}
|
|
await new Promise((r) => setTimeout(r, 250 * (attempt + 1)));
|
|
}
|
|
return { error: `Screenshot failed after retries: ${lastErr?.message || String(lastErr)}` };
|
|
}
|
|
|
|
// Count the safe (GET) API endpoints captured for this site so the backend can nudge the agent toward the fast network path. Best-effort, never throws.
|
|
async function countSafeRoutes(wv: BrowserWebview): Promise<number> {
|
|
try {
|
|
const bridge = (window as any).openswarm?.cdpRoutesGet as
|
|
| ((id: number, origin?: string) => Promise<any[]>) | undefined;
|
|
if (!bridge) return 0;
|
|
let origin = '';
|
|
try { origin = new URL(wv.getURL()).origin; } catch {}
|
|
const routes = (await bridge(wv.getWebContentsId(), origin)) || [];
|
|
return routes.filter((r) => r && r.safe).length;
|
|
} catch { return 0; }
|
|
}
|
|
|
|
// Electron queues executeJavaScript until the page "stops loading", and pages with straggler subresources (recaptcha/tracker iframes) can stay isLoading for minutes, starving EVERY command into its backend timeout (the wedged-webview tail). Once the document itself is ready, wv.stop() cancels only the stragglers and fires did-stop-loading, which flushes the queue; a genuinely-still-loading document (no dom-ready yet) is left alone.
|
|
const STUCK_EVAL_GRACE_MS = 2500;
|
|
const STUCK_EVAL_LIMIT_MS = 9000;
|
|
|
|
// Run `code` in the guest page. In Electron we prefer CDP Runtime.evaluate: it runs in the
|
|
// browser process, so it is NOT suspended while the page is still loading, the way
|
|
// webContents.executeJavaScript is (that suspend, on a page whose trackers never let it "stop
|
|
// loading", is the 15s command wedge). When the CDP bridge isn't there (dev Chrome, or a forced
|
|
// A/B via window.__OSW_CDP_EVAL__ = false) we fall back to the executeJavaScript path unchanged,
|
|
// so behavior never regresses where CDP can't run. Both paths keep the same contract: return the
|
|
// value, throw on a page-side error, mark dom-ready on success.
|
|
async function evalInPage(wv: BrowserWebview, code: string): Promise<any> {
|
|
const cdpBridge = (window as any).openswarm?.sendCdpCommand;
|
|
if (cdpBridge && (window as any).__OSW_CDP_EVAL__ !== false) {
|
|
let cdp: any;
|
|
try {
|
|
cdp = await sendCdp(wv, 'Runtime.evaluate',
|
|
{ expression: code, returnByValue: true, awaitPromise: true });
|
|
} catch {
|
|
// CDP INFRA failure (the debugger can't attach because DevTools or a remote-debugging
|
|
// port already holds this webContents, or the bridge errored). Never worse than today:
|
|
// fall through to the executeJavaScript path. A real PAGE exception is NOT an infra
|
|
// failure, it rides exceptionDetails below, so it still surfaces as a throw.
|
|
cdp = undefined;
|
|
}
|
|
if (cdp !== undefined) {
|
|
const value = unwrapCdpEval(cdp); // throws on a real page-side exception
|
|
markDomReady(wv);
|
|
return value;
|
|
}
|
|
}
|
|
return await evalViaExecuteJs(wv, code);
|
|
}
|
|
|
|
// The original webContents.executeJavaScript path, kept as the dev-Chrome / bridge-absent
|
|
// fallback: it suspends until the page stops loading, so a grace/limit race cancels stragglers
|
|
// with wv.stop() once the document is ready and flushes the queue.
|
|
async function evalViaExecuteJs(wv: BrowserWebview, code: string): Promise<any> {
|
|
const run = wv.executeJavaScript(code).then((v) => {
|
|
markDomReady(wv);
|
|
return { done: true as const, value: v };
|
|
});
|
|
const grace = new Promise<{ done: false }>((r) => setTimeout(() => r({ done: false }), STUCK_EVAL_GRACE_MS));
|
|
let first = await Promise.race([run, grace]);
|
|
if (!first.done) {
|
|
let stopped = false;
|
|
try {
|
|
if (wv.isLoading() && hasDomReady(wv)) {
|
|
wv.stop();
|
|
stopped = true;
|
|
}
|
|
} catch {
|
|
// torn-down webview; the limit below surfaces it
|
|
}
|
|
const limit = new Promise<{ done: false }>((r) => setTimeout(() => r({ done: false }), STUCK_EVAL_LIMIT_MS));
|
|
first = await Promise.race([run, limit]);
|
|
if (!first.done) {
|
|
throw new Error(stopped
|
|
? 'page never finished loading even after cancelling stragglers'
|
|
: 'page is still loading; retry shortly');
|
|
}
|
|
}
|
|
return first.value;
|
|
}
|
|
|
|
// What the MAIN model loop gets. Every read lands in its context, so this stays modest on purpose.
|
|
const GET_TEXT_DEFAULT_CHARS = 15000;
|
|
// Ceiling for a caller that asks for more. The read script (one cheap aux call with its own context)
|
|
// asks for its full budget: measured, 9 of 18 of its reads came back at EXACTLY 15000 chars, meaning
|
|
// cut off, and on a reddit thread the comment scores sit after the post body, so they were never in
|
|
// the text it was handed. It then declined, and a 100-220s model loop went scrolling for what we had
|
|
// truncated ourselves.
|
|
const GET_TEXT_MAX_CHARS = 30000;
|
|
|
|
async function handleGetText(wv: BrowserWebview, params: Record<string, any> = {}): Promise<Record<string, any>> {
|
|
const asked = Number(params.max_chars) || GET_TEXT_DEFAULT_CHARS;
|
|
const cap = Math.min(Math.max(asked, 1000), GET_TEXT_MAX_CHARS);
|
|
const text: string = await evalInPage(wv,
|
|
`document.body.innerText.substring(0, ${cap})`
|
|
);
|
|
// Sampled HERE (on a read), not on navigate: by the time the agent reads the page, the SPA's XHR/fetch have fired, so routes are actually captured.
|
|
const routes_available = await countSafeRoutes(wv);
|
|
return { text, url: wv.getURL(), title: wv.getTitle(), routes_available };
|
|
}
|
|
|
|
// Recent warn+error console output for this webview (captured in main.js). Lets a stuck agent see the page's OWN errors (JS exceptions, failed loads) instead of guessing. Read-only, fail-safe: any miss returns an empty, honest result.
|
|
async function handleGetConsole(wv: BrowserWebview): Promise<Record<string, any>> {
|
|
try {
|
|
const bridge = (window as any).openswarm?.getWebviewConsole as
|
|
| ((id: number) => Promise<Array<{ level: string; message: string; source?: string; line?: number }>>)
|
|
| undefined;
|
|
if (!bridge) return { text: 'Console capture is unavailable here.', errors: [] };
|
|
const errors = (await bridge(wv.getWebContentsId())) || [];
|
|
if (errors.length === 0) {
|
|
return { text: 'No console warnings or errors recorded on this page.', errors: [], url: wv.getURL() };
|
|
}
|
|
const lines = errors.map(
|
|
(e) => `[${e.level}] ${e.message}${e.source ? ` (${e.source}:${e.line ?? '?'})` : ''}`,
|
|
);
|
|
return {
|
|
text: `Page console, ${errors.length} recent warning(s)/error(s), newest last:\n${lines.join('\n')}`,
|
|
errors,
|
|
url: wv.getURL(),
|
|
};
|
|
} catch (err: any) {
|
|
return { text: `Could not read console: ${err?.message || String(err)}`, errors: [] };
|
|
}
|
|
}
|
|
|
|
async function handleNavigate(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const raw = params.url as string;
|
|
if (!raw) return { error: 'url parameter is required' };
|
|
const url = resolveInput(raw);
|
|
// loadURL resolves only on the full 'load' event, which heavy SPAs (LinkedIn, Gmail) hold open with persistent connections long past our timeout even though the page is usable in a second. Return the moment the DOM is ready and let the agent's next wait settle the rest, the way a person clicks before every background request has finished.
|
|
let removeReady = () => {};
|
|
const domReady = new Promise<void>((resolve) => {
|
|
const onReady = () => resolve();
|
|
wv.addEventListener('dom-ready', onReady, { once: true });
|
|
removeReady = () => wv.removeEventListener('dom-ready', onReady);
|
|
});
|
|
const fullyLoaded = wv.loadURL(url).catch((err: any) => {
|
|
// A superseded navigation aborts the old load; that's normal, not a failure.
|
|
if (err?.message?.includes('ERR_ABORTED')) return;
|
|
throw err;
|
|
});
|
|
fullyLoaded.catch(() => {}); // a late load failure shouldn't throw once dom-ready returned
|
|
try {
|
|
await Promise.race([fullyLoaded, domReady]);
|
|
} finally {
|
|
removeReady();
|
|
}
|
|
// A navigate that lands on a raw JSON/API document (Instagram's topsearch, any /api/... GET)
|
|
// paints an unreadable wall in the card and reads as a crash to the user. Hand the data to the
|
|
// agent as the result instead, and quietly get the card off the wall, so a person never sees
|
|
// raw JSON where a page should be.
|
|
const data = await readDataDocument(wv);
|
|
if (data) {
|
|
recoverCardOffDataWall(wv, url);
|
|
return {
|
|
text: `Fetched ${data.contentType} data from ${url} (this URL is a raw API endpoint, not a page):\n${data.body}`,
|
|
url: wv.getURL(),
|
|
data_document: true,
|
|
};
|
|
}
|
|
// Route-count is sampled on the next READ (handleGetText), not here: at navigate-return the SPA's XHRs haven't fired yet, so this would always be ~0.
|
|
return { text: `Navigated to ${url}`, url };
|
|
}
|
|
|
|
// Chromium renders any JSON (or JSON-shaped text) response with its built-in viewer, so a navigate
|
|
// to an API endpoint leaves the card showing a wall of raw data. contentType is the reliable tell;
|
|
// re-fetch same-origin (the just-loaded GET, idempotent) to hand the agent the exact bytes.
|
|
// Exported so BrowserCard can reuse the exact same detection for the card's own initial load.
|
|
export async function readDataDocument(wv: BrowserWebview): Promise<{ contentType: string; body: string } | null> {
|
|
const code = `(async () => {
|
|
const ct = String(document.contentType || '').toLowerCase();
|
|
const isJson = ct.startsWith('application/json');
|
|
const maybePlain = ct.startsWith('text/plain');
|
|
if (!isJson && !maybePlain) return null;
|
|
try {
|
|
const r = await fetch(location.href, { credentials: 'include', signal: AbortSignal.timeout(2500) });
|
|
const body = await r.text();
|
|
if (maybePlain && !isJson) { try { JSON.parse(body.trim()); } catch (e) { return null; } }
|
|
return { contentType: ct, body: body.slice(0, 15000) };
|
|
} catch (e) { return null; }
|
|
})()`;
|
|
try {
|
|
const res = await evalInPage(wv, code);
|
|
if (res && !res.error && typeof res.body === 'string') return res;
|
|
return null;
|
|
} catch {
|
|
return null;
|
|
}
|
|
}
|
|
|
|
// Get the card off a raw-data wall: step back to the real page the agent came from, or, if it opened
|
|
// straight onto the data URL, fall back to the site homepage. Fire-and-forget, the agent already has
|
|
// the data; this is purely so a human sees a page instead of JSON. Exported for BrowserCard's own-load path.
|
|
export function recoverCardOffDataWall(wv: BrowserWebview, dataUrl: string): void {
|
|
let origin = '';
|
|
try { origin = new URL(dataUrl).origin; } catch { /* keep '' */ }
|
|
try {
|
|
if (wv.canGoBack()) {
|
|
wv.goBack();
|
|
// A card that opened STRAIGHT onto the data URL has only the webview's blank initial entry
|
|
// behind it, so goBack lands on about:blank. Detect that and fall back to the site homepage
|
|
// so the card shows a real page rather than a blank one.
|
|
window.setTimeout(() => {
|
|
try {
|
|
const u = wv.getURL();
|
|
if ((!u || u === 'about:blank') && origin) void wv.loadURL(origin).catch(() => {});
|
|
} catch { /* leave as-is */ }
|
|
}, 400);
|
|
return;
|
|
}
|
|
if (origin) void wv.loadURL(origin).catch(() => {});
|
|
} catch {
|
|
/* leave the card as-is if recovery fails; the data still reached the agent */
|
|
}
|
|
}
|
|
|
|
async function handleClick(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const selector = params.selector as string;
|
|
if (!selector) return { error: 'selector parameter is required' };
|
|
const safeSelector = JSON.stringify(selector);
|
|
const code = `(()=>{
|
|
const el = document.querySelector(${safeSelector});
|
|
if (!el) return { error: 'Element not found: ' + ${safeSelector} };
|
|
el.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
const rect = el.getBoundingClientRect();
|
|
const x = rect.left + rect.width / 2;
|
|
const y = rect.top + rect.height / 2;
|
|
const opts = { bubbles: true, cancelable: true, clientX: x, clientY: y, button: 0 };
|
|
el.dispatchEvent(new PointerEvent('pointerdown', { ...opts, pointerId: 1 }));
|
|
el.dispatchEvent(new MouseEvent('mousedown', opts));
|
|
el.dispatchEvent(new PointerEvent('pointerup', { ...opts, pointerId: 1 }));
|
|
el.dispatchEvent(new MouseEvent('mouseup', opts));
|
|
el.dispatchEvent(new MouseEvent('click', opts));
|
|
return {
|
|
text: 'Clicked element: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''),
|
|
url: location.href,
|
|
clickX: window.innerWidth > 0 ? x / window.innerWidth : 0.5,
|
|
clickY: window.innerHeight > 0 ? y / window.innerHeight : 0.5,
|
|
};
|
|
})()`;
|
|
const result = await evalInPage(wv, code);
|
|
return result;
|
|
}
|
|
|
|
async function handleType(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const selector = params.selector as string;
|
|
const text = params.text as string;
|
|
if (!selector) return { error: 'selector parameter is required' };
|
|
if (text == null) return { error: 'text parameter is required' };
|
|
const safeSelector = JSON.stringify(selector);
|
|
const safeText = JSON.stringify(text);
|
|
// Try the cheap in-page fill first, and read it back: an editor that rejects synthetic
|
|
// execCommand insertText (Reddit's Lexical, strict React contenteditables) leaves the box
|
|
// empty, and we only learn that by checking the value, not by assuming the call worked.
|
|
const code = `(async ()=>{
|
|
const el = document.querySelector(${safeSelector});
|
|
if (!el) return { error: 'Element not found: ' + ${safeSelector} };
|
|
el.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
el.focus();
|
|
if (el.select) el.select();
|
|
document.execCommand('selectAll', false);
|
|
document.execCommand('delete', false);
|
|
document.execCommand('insertText', false, ${safeText});
|
|
el.dispatchEvent(new InputEvent('input', {
|
|
bubbles: true, cancelable: true, inputType: 'insertText', data: ${safeText},
|
|
}));
|
|
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
const now = (el.value != null ? el.value : (el.textContent || ''));
|
|
return { text: 'Typed into: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''),
|
|
committed: now.includes(${safeText}) };
|
|
})()`;
|
|
const result = await evalInPage(wv, code);
|
|
// Real-keystroke fallback when the synthetic fill did not commit (same lever the finder uses).
|
|
if (result && !result.error && result.committed === false) {
|
|
const ok = await keystrokeFill(wv, selector, text);
|
|
result.text = ok ? `Typed into ${selector} via keystrokes` : `Type may not have committed into ${selector}`;
|
|
result.committed = ok;
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Find the page's composer STRUCTURALLY, not by accessible name: the biggest editable
|
|
// region (contenteditable / textarea / text input) that is a real writing surface, not a
|
|
// search box. Ranks by editable-richness + proximity to a Send/Post/Reply control + size,
|
|
// so it picks the comment box out of a page that also has a search field. Marks the winner
|
|
// with data-osw-composer so a follow-up type/click can target it by a stable selector.
|
|
// With {fill}, it also types + reads back in the SAME in-page context (the only reliable
|
|
// commit-check for a React-controlled contenteditable, whose value never reaches the AX tree).
|
|
// With {reveal:true}, when no composer is painted yet it takes ONE reversible reveal action
|
|
// (click a compose-trigger, open the first list item, or scroll) and rescans, up to a small
|
|
// bound. Reveal actions are same-document only: SPA route changes (pushState) keep this JS
|
|
// context alive, so X/LinkedIn/Reddit/YouTube stay drivable in one call; a full navigation
|
|
// just cuts the await short and the backend re-perceives. Reveal NEVER clicks an irreversible
|
|
// control (send/submit/pay/delete...): it only opens a surface, it never commits one.
|
|
async function handleFindComposer(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const fill = params.fill != null ? String(params.fill) : null;
|
|
const safeFill = JSON.stringify(fill);
|
|
const reveal = params.reveal === true ? 'true' : 'false';
|
|
const code = `(async () => {
|
|
const SUBMIT = /\\b(send|post|reply|comment|tweet|publish|share|message)\\b/i;
|
|
const SEARCH = /search|find\\b|filter|query|lookup|explore|jump to/i;
|
|
// Reversible compose openers: reveal a writing surface, never commit one.
|
|
const OPENER = /\\b(start a post|start a thread|create( a)?( new)? (post|thread)|new post|new thread|add a comment|write a comment|leave a comment|post a comment|write a review|create a review|new message|new chat|send a message|compose|reply|comment|tweet|create|write)\\b/i;
|
|
// Never clickable by reveal, even if the label also looks like an opener.
|
|
const HARDBLOCK = /\\b(send|submit|pay|buy|purchase|checkout|order|delete|remove|unfollow|unsubscribe|log ?out|sign ?out|report|block|confirm|deactivate|save)\\b/i;
|
|
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
const vis = (el) => {
|
|
const r = el.getBoundingClientRect(); const s = getComputedStyle(el);
|
|
return r.width >= 80 && r.height >= 16 && s.visibility !== 'hidden'
|
|
&& s.display !== 'none' && s.opacity !== '0';
|
|
};
|
|
// Pierce shadow DOM: Reddit (shreddit), Telegram, WhatsApp, Slack build their composer
|
|
// inside web-component shadow roots that a light-DOM querySelectorAll can't see. Collect
|
|
// only the SELECTOR matches (a few hundred at most), never every node: a heavy SPA like X
|
|
// has 10k+ nodes and the composer sits late in document order, so an all-node walk with a
|
|
// node cap truncates before ever reaching it (the regression that hid X's own composer).
|
|
const deepMatch = (root, sel, out, depth) => {
|
|
if (depth > 8 || out.length > 400) return out;
|
|
let hits; try { hits = root.querySelectorAll(sel); } catch (e) { hits = []; }
|
|
for (const el of hits) out.push(el);
|
|
let all; try { all = root.querySelectorAll('*'); } catch (e) { return out; }
|
|
for (const el of all) { if (el.shadowRoot) deepMatch(el.shadowRoot, sel, out, depth + 1); }
|
|
return out;
|
|
};
|
|
const EDIT = 'textarea, [contenteditable="true"], [role="textbox"], input[type="text"]';
|
|
const labelOf = (el) => ((el.getAttribute('aria-label')||'') + ' ' + (el.getAttribute('placeholder')||'')
|
|
+ ' ' + (el.getAttribute('data-placeholder')||'') + ' ' + (el.getAttribute('name')||'')).trim();
|
|
|
|
const findBest = () => {
|
|
const all = deepMatch(document, EDIT, [], 0);
|
|
let best = null, bestScore = 0, bestNear = false;
|
|
for (const el of all) {
|
|
if (!vis(el) || el.readOnly || el.disabled) continue;
|
|
const label = labelOf(el);
|
|
if (el.type === 'search' || SEARCH.test(label)) continue;
|
|
const rich = el.tagName === 'TEXTAREA' || el.isContentEditable || el.getAttribute('role') === 'textbox';
|
|
const area = el.closest('form, [role="dialog"], [role="group"], section, main, [role="main"]') || document.body;
|
|
let near = false;
|
|
for (const b of area.querySelectorAll('button, [role="button"], input[type="submit"]')) {
|
|
const bt = ((b.textContent||'') + ' ' + (b.getAttribute('aria-label')||'')).trim();
|
|
if (bt.length < 40 && SUBMIT.test(bt)) { near = true; break; }
|
|
}
|
|
if (!rich && !near) continue; // a bare form input near nothing is not a composer
|
|
const r = el.getBoundingClientRect();
|
|
const score = (el.isContentEditable ? 2 : 0) + (el.tagName === 'TEXTAREA' ? 2 : 0)
|
|
+ (near ? 3 : 0) + Math.min((r.width * r.height) / 40000, 3) + (SUBMIT.test(label) ? 1 : 0);
|
|
if (score > bestScore) { bestScore = score; best = el; bestNear = near; }
|
|
}
|
|
return (best && bestScore >= 2) ? { el: best, score: bestScore, near: bestNear } : null;
|
|
};
|
|
|
|
const pollFind = async (ms) => {
|
|
const t0 = Date.now(); let h = findBest();
|
|
while (!h && Date.now() - t0 < ms) { await sleep(140); h = findBest(); }
|
|
return h;
|
|
};
|
|
|
|
// The reveal actions, tried in yield order. Each returns true if it clicked/scrolled.
|
|
// Match id-based placeholders too (YouTube's #placeholder-area "Add a comment...") not just
|
|
// class-based ones, so a lazy comment box becomes a clickable trigger once it scrolls in.
|
|
const TRIGGER_SEL = 'button, [role="button"], a[href], summary, [tabindex], [id*="placeholder" i], [class*="placeholder" i]';
|
|
// inViewOnly confines the pick to what is CURRENTLY on screen: during the scroll phase a
|
|
// header "Create"/"Reply" (scrolled off the top, so r.top<0) would otherwise outscore the
|
|
// comment box that just scrolled into view, and clicking it opens/navigates the wrong thing.
|
|
const clickTrigger = (inViewOnly) => {
|
|
const all = deepMatch(document, TRIGGER_SEL, [], 0);
|
|
let best = null, bestScore = -1;
|
|
for (const el of all) {
|
|
if (!vis(el)) continue;
|
|
const r = el.getBoundingClientRect();
|
|
if (inViewOnly && (r.bottom < 0 || r.top > window.innerHeight)) continue;
|
|
const txt = ((el.textContent||'') + ' ' + (el.getAttribute('aria-label')||'') + ' ' + (el.getAttribute('placeholder')||'')).trim();
|
|
if (txt.length > 60 || !OPENER.test(txt) || HARDBLOCK.test(txt)) continue;
|
|
// Short-labelled wins; higher-on-page when scanning the whole page, viewport-center when scrolling.
|
|
const posScore = inViewOnly
|
|
? Math.max(0, 300 - Math.abs(r.top - window.innerHeight / 2)) / 100
|
|
: Math.max(0, 600 - r.top) / 100;
|
|
const score = (60 - txt.length) + posScore;
|
|
if (score > bestScore) { bestScore = score; best = el; }
|
|
}
|
|
if (!best) return false;
|
|
best.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
best.click();
|
|
return true;
|
|
};
|
|
const openFirstItem = () => {
|
|
// Chat/message lists (X DM [data-testid=conversation], WhatsApp/Telegram [role=grid]
|
|
// rows, conversation links) plus generic feeds. Pierce shadow DOM so web-component chat
|
|
// lists are reachable. Take the first visible list-like item and open it.
|
|
const ITEM_SEL = '[data-testid="conversation"], [role="listitem"], [role="row"], [role="article"], article, li a[href], a[role="link"], a[href*="/messages/"], a[href*="/chat/"]';
|
|
const items = deepMatch(document, ITEM_SEL, [], 0);
|
|
for (const it of items) {
|
|
if (!vis(it)) continue;
|
|
const r = it.getBoundingClientRect();
|
|
if (r.top < 40 || r.top > window.innerHeight || r.height > window.innerHeight * 0.6) continue; // skip headers / offscreen / the whole pane
|
|
const target = it.matches('a[href], [role="link"]') ? it : (it.querySelector('a[href], [role="link"], [role="button"]') || it);
|
|
target.scrollIntoView({ block: 'center', behavior: 'instant' });
|
|
target.click();
|
|
return true;
|
|
}
|
|
return false;
|
|
};
|
|
let hit = findBest();
|
|
const acts = [];
|
|
if (!hit && ${reveal}) {
|
|
// Self-imposed budget. The tiers below sum to ~24s of worst case but the command that
|
|
// carries them is killed at its own timeout, which threw away ALL the work and returned
|
|
// nothing (measured: linkedin died mid-scroll 2 of 3 runs, so retop/open-first could never
|
|
// run). Each tier now only STARTS if the budget can still pay for it, and polls are clipped
|
|
// to what is left, so the routine always returns its own best answer instead of being shot.
|
|
const DEADLINE = Date.now() + 21000;
|
|
const left = () => DEADLINE - Date.now();
|
|
const budget = (want) => Math.max(0, Math.min(want, left()));
|
|
// 1. A compose opener visible up top (Gmail "Compose", LinkedIn "Start a post"). Patient
|
|
// poll: LinkedIn code-splits its share modal, and under a heavy session the editor
|
|
// chunk lands past 2.5s (measured: the 2.5s poll missed ~half the time; poll exits the
|
|
// moment the editable appears, so the patience costs nothing on the happy path).
|
|
try { if (clickTrigger(false)) { acts.push('trigger'); hit = await pollFind(budget(6000)); } } catch (e) { /* keep going */ }
|
|
// 2. Progressive scroll for a below-fold / lazy composer: YouTube comments hydrate on
|
|
// scroll and start as a placeholder that only becomes editable once clicked, so scroll
|
|
// a step, re-scan, and re-click the trigger on whatever just entered the viewport, up to
|
|
// a small bound, stopping at page bottom. This is what a human does to reach comments.
|
|
if (!hit) {
|
|
const sc = document.scrollingElement || document.documentElement;
|
|
let scrolled = false;
|
|
// Reserve time for the retop + open-first tiers below, so the ladder can't eat the budget.
|
|
for (let step = 0; step < 6 && !hit && left() > 7000; step++) {
|
|
const before = sc.scrollTop;
|
|
sc.scrollBy(0, Math.round(window.innerHeight * 0.9));
|
|
scrolled = true;
|
|
await sleep(500);
|
|
hit = findBest();
|
|
if (!hit) { try { if (clickTrigger(true)) hit = await pollFind(budget(1400)); } catch (e) { /* keep going */ } }
|
|
if (sc.scrollTop === before) break;
|
|
}
|
|
if (scrolled) acts.push('scroll');
|
|
}
|
|
// 3. Back to the top opener: the scroll ladder ends at page bottom with the top compose
|
|
// entry off-screen; a modal that opened slowly (or needed a second click) is only
|
|
// winnable by returning and retrying once.
|
|
if (!hit && left() > 2500) {
|
|
try {
|
|
(document.scrollingElement || document.documentElement).scrollTo(0, 0);
|
|
await sleep(400);
|
|
if (clickTrigger(false)) { acts.push('retop'); hit = await pollFind(budget(4000)); }
|
|
} catch (e) { /* keep going */ }
|
|
}
|
|
// 4. Last resort: open the first list item (X DMs / chat lists). Navigational, so it runs
|
|
// only after trigger+scroll fail, which stops it from yanking YouTube to another video.
|
|
if (!hit && left() > 1200) { let did = false; try { did = openFirstItem(); } catch (e) { did = false; } if (did) { acts.push('open-first'); hit = await pollFind(budget(2000)); } }
|
|
}
|
|
if (!hit) return { found: false, reveals: acts };
|
|
|
|
const fillEl = (el) => {
|
|
el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus();
|
|
if (el.select) el.select();
|
|
document.execCommand('selectAll', false); document.execCommand('delete', false);
|
|
document.execCommand('insertText', false, ${safeFill});
|
|
el.dispatchEvent(new InputEvent('input', { bubbles: true, cancelable: true, inputType: 'insertText', data: ${safeFill} }));
|
|
el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
const now = (el.value != null ? el.value : (el.textContent || ''));
|
|
return now.includes(${safeFill});
|
|
};
|
|
|
|
let best = hit.el;
|
|
best.setAttribute('data-osw-composer', '1');
|
|
let filled = false;
|
|
if (${safeFill} != null) {
|
|
filled = fillEl(best);
|
|
// Activation-click fallback: YouTube (and many comment widgets) render a placeholder
|
|
// that only spawns the real contenteditable once clicked. If the fill didn't commit,
|
|
// click the found element, let the real editor mount, rescan, and fill THAT.
|
|
if (!filled) {
|
|
try { best.click(); } catch (e) { /* click may be intercepted */ }
|
|
const re = await pollFind(1500);
|
|
if (re) {
|
|
best.removeAttribute('data-osw-composer');
|
|
best = re.el; hit = re; best.setAttribute('data-osw-composer', '1');
|
|
filled = fillEl(best);
|
|
acts.push('activate');
|
|
}
|
|
}
|
|
}
|
|
return { found: true, selector: '[data-osw-composer="1"]', tag: best.tagName.toLowerCase(),
|
|
role: best.isContentEditable ? 'contenteditable' : (best.getAttribute('role') || best.tagName.toLowerCase()),
|
|
score: Math.round(hit.score * 10) / 10, nearSubmit: hit.near, filled, reveals: acts };
|
|
})()`;
|
|
const result = await evalInPage(wv, code);
|
|
// Real-keystroke fill fallback: some editors (Reddit's Lexical, strict React contenteditables)
|
|
// manage their own state through beforeinput and ignore execCommand insertText, so the in-page
|
|
// fill reads back empty. Retype the SAME text as OS-level key events (isTrusted, the way a human
|
|
// types), which those editors DO honor, then re-read to confirm. Only fires when the cheap
|
|
// in-page fill already failed, so the fast path is untouched.
|
|
if (fill != null && result && result.found && !result.filled && result.selector) {
|
|
const ok = await keystrokeFill(wv, String(result.selector), fill);
|
|
result.filled = ok;
|
|
result.fillMode = ok ? 'keystroke' : 'keystroke-failed';
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Type `text` into the element at `selector` as real OS-level key events (Electron
|
|
// sendInputEvent 'char'), for editors that reject synthetic execCommand insertText. Focuses +
|
|
// clears in-page first, types each char natively, then reads the value back in-page to verify.
|
|
async function keystrokeFill(wv: BrowserWebview, selector: string, text: string): Promise<boolean> {
|
|
const safeSel = JSON.stringify(selector);
|
|
await evalInPage(wv, `(() => { const el = document.querySelector(${safeSel});
|
|
if (!el) return false; el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus();
|
|
if (el.select) el.select(); document.execCommand('selectAll', false); document.execCommand('delete', false);
|
|
return true; })()`);
|
|
for (const ch of text) {
|
|
wv.sendInputEvent({ type: 'char', keyCode: ch });
|
|
}
|
|
// Read back from the marked element OR the active element: editors like Reddit's swap the
|
|
// node on activation, so the original selector can go stale even though the keystrokes landed
|
|
// in whatever now holds focus. Checking both survives that re-render.
|
|
const readBack = `(() => {
|
|
const check = (el) => { if (!el) return false;
|
|
el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true }));
|
|
const now = (el.value != null ? el.value : (el.textContent || '')); return now.includes(${JSON.stringify(text)}); };
|
|
return check(document.querySelector(${safeSel})) || check(document.activeElement); })()`;
|
|
return Boolean(await evalInPage(wv, readBack));
|
|
}
|
|
|
|
// Electron sendInputEvent expects names like 'Up', 'Enter', 'Space', not 'ArrowUp'/' '/'Esc'.
|
|
const KEY_NAME_MAP: Record<string, string> = {
|
|
ArrowUp: 'Up',
|
|
ArrowDown: 'Down',
|
|
ArrowLeft: 'Left',
|
|
ArrowRight: 'Right',
|
|
' ': 'Space',
|
|
Spacebar: 'Space',
|
|
Esc: 'Escape',
|
|
Del: 'Delete',
|
|
};
|
|
|
|
interface CdpKeyDescriptor { key: string; code: string; vk: number; text?: string }
|
|
|
|
const CDP_KEYS: Record<string, CdpKeyDescriptor> = {
|
|
Enter: { key: 'Enter', code: 'Enter', vk: 13, text: '\r' },
|
|
Tab: { key: 'Tab', code: 'Tab', vk: 9 },
|
|
Escape: { key: 'Escape', code: 'Escape', vk: 27 },
|
|
Backspace: { key: 'Backspace', code: 'Backspace', vk: 8 },
|
|
Delete: { key: 'Delete', code: 'Delete', vk: 46 },
|
|
ArrowUp: { key: 'ArrowUp', code: 'ArrowUp', vk: 38 },
|
|
ArrowDown: { key: 'ArrowDown', code: 'ArrowDown', vk: 40 },
|
|
ArrowLeft: { key: 'ArrowLeft', code: 'ArrowLeft', vk: 37 },
|
|
ArrowRight: { key: 'ArrowRight', code: 'ArrowRight', vk: 39 },
|
|
Home: { key: 'Home', code: 'Home', vk: 36 },
|
|
End: { key: 'End', code: 'End', vk: 35 },
|
|
PageUp: { key: 'PageUp', code: 'PageUp', vk: 33 },
|
|
PageDown: { key: 'PageDown', code: 'PageDown', vk: 34 },
|
|
' ': { key: ' ', code: 'Space', vk: 32, text: ' ' },
|
|
};
|
|
|
|
// Loose names the model actually sends, folded onto the canonical DOM names above.
|
|
const CDP_KEY_ALIASES: Record<string, string> = {
|
|
Up: 'ArrowUp', Down: 'ArrowDown', Left: 'ArrowLeft', Right: 'ArrowRight',
|
|
Space: ' ', Spacebar: ' ', Esc: 'Escape', Del: 'Delete', Return: 'Enter',
|
|
};
|
|
|
|
function cdpKeyDescriptor(rawKey: string): CdpKeyDescriptor | null {
|
|
const canonical = CDP_KEY_ALIASES[rawKey] || rawKey;
|
|
const named = CDP_KEYS[canonical];
|
|
if (named) return named;
|
|
if (canonical.length === 1) {
|
|
const upper = canonical.toUpperCase();
|
|
const code = /[a-z]/i.test(canonical) ? `Key${upper}` : /[0-9]/.test(canonical) ? `Digit${canonical}` : '';
|
|
return { key: canonical, code, vk: upper.charCodeAt(0), text: canonical };
|
|
}
|
|
return null;
|
|
}
|
|
|
|
async function handlePressKey(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const rawKey = (params.key as string) || '';
|
|
if (!rawKey) return { error: 'key parameter is required' };
|
|
await evalInPage(wv, 'document.body && document.body.focus && document.body.focus(); true');
|
|
const desc = cdpKeyDescriptor(rawKey);
|
|
if (desc) {
|
|
try {
|
|
// CDP key events are trusted AND scoped to THIS webview no matter where the user's cursor sits; the sendInputEvent path delivered to whatever had focus, which is the "agent typed into my note" bug. keyDown-with-text inserts the char; bare named keys use rawKeyDown so no stray char lands.
|
|
const down: Record<string, any> = { type: desc.text ? 'keyDown' : 'rawKeyDown', key: desc.key, windowsVirtualKeyCode: desc.vk, nativeVirtualKeyCode: desc.vk };
|
|
if (desc.code) down.code = desc.code;
|
|
if (desc.text) down.text = desc.text;
|
|
await sendCdp(wv, 'Input.dispatchKeyEvent', down);
|
|
const up: Record<string, any> = { type: 'keyUp', key: desc.key, windowsVirtualKeyCode: desc.vk, nativeVirtualKeyCode: desc.vk };
|
|
if (desc.code) up.code = desc.code;
|
|
await sendCdp(wv, 'Input.dispatchKeyEvent', up);
|
|
return { text: `Pressed ${rawKey}` };
|
|
} catch { /* fall through to the legacy path so a CDP hiccup never makes a key dead */ }
|
|
}
|
|
// Legacy focus-dependent fallback (exotic keys or CDP unavailable): keeps every key that worked before working.
|
|
const keyCode = KEY_NAME_MAP[rawKey] || rawKey;
|
|
await evalInPage(wv, 'document.body && document.body.focus && document.body.focus(); true');
|
|
// Native OS-level key events have isTrusted=true, so hostile sites' keyboard handlers respect them.
|
|
wv.sendInputEvent({ type: 'keyDown', keyCode });
|
|
wv.sendInputEvent({ type: 'char', keyCode });
|
|
wv.sendInputEvent({ type: 'keyUp', keyCode });
|
|
return { text: `Pressed ${rawKey}` };
|
|
}
|
|
|
|
// Click at a viewport coordinate (percent of the view's width/height) with a
|
|
// real, trusted CDP mouse event, NO DOM element required. This is what lets the
|
|
// app agent operate a bare <canvas> game the way a person taps the screen: the
|
|
// AX-tree click paths (click_index/click_by_name) can't target a canvas because
|
|
// it exposes no nodes, but a coordinate dispatch lands anywhere. Optional
|
|
// hold_ms presses and holds (platformers, charge-up mechanics).
|
|
async function handleClickPoint(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const xPercent = Number(params.xPercent);
|
|
const yPercent = Number(params.yPercent);
|
|
if (!Number.isFinite(xPercent) || !Number.isFinite(yPercent)) {
|
|
return { error: 'xPercent and yPercent are required (0-100, percent of the view).' };
|
|
}
|
|
const cx = Math.max(0, Math.min(100, xPercent));
|
|
const cy = Math.max(0, Math.min(100, yPercent));
|
|
const button = params.button === 'right' ? 'right' : params.button === 'middle' ? 'middle' : 'left';
|
|
const holdMs = Math.max(0, Math.min(Number(params.hold_ms) || 0, 5000));
|
|
// Read the guest's own viewport so coords are correct under zoom/DPR, not the
|
|
// host element's box. One cheap round-trip; falls back to the element box.
|
|
let vw = wv.clientWidth, vh = wv.clientHeight;
|
|
try {
|
|
const d = await evalInPage(wv, '({w: window.innerWidth, h: window.innerHeight})');
|
|
if (d && d.w > 0 && d.h > 0) { vw = d.w; vh = d.h; }
|
|
} catch { /* use the element box as a fallback */ }
|
|
const x = (cx / 100) * vw;
|
|
const y = (cy / 100) * vh;
|
|
// hoverOnly: real mouse move, no press. Hover-revealed controls (reddit's post kebab only
|
|
// materializes on tile hover) need this; a click there would navigate into the post.
|
|
const hoverOnly = params.hoverOnly === true;
|
|
try {
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y });
|
|
if (!hoverOnly) {
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: 1 });
|
|
if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs));
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: 1 });
|
|
}
|
|
} catch (err: any) {
|
|
return { error: `Click point failed: ${err?.message || String(err)}` };
|
|
}
|
|
return {
|
|
text: `${hoverOnly ? 'Hovered' : 'Clicked'} at (${Math.round(x)}, ${Math.round(y)})${holdMs && !hoverOnly ? ` held ${holdMs}ms` : ''}.`,
|
|
clickX: cx, clickY: cy, url: wv.getURL(),
|
|
};
|
|
}
|
|
|
|
// CDP Accessibility.getFullAXTree sees computed roles/names even on hostile sites with unlabeled DOMs.
|
|
const INTERACTIVE_ROLES = new Set([
|
|
'button', 'link', 'textbox', 'combobox', 'checkbox', 'menuitem',
|
|
'tab', 'switch', 'searchbox', 'slider', 'listbox', 'option',
|
|
'radio', 'menuitemcheckbox', 'menuitemradio', 'spinbutton', 'treeitem',
|
|
]);
|
|
|
|
interface InteractiveElement {
|
|
index: number;
|
|
role: string;
|
|
name: string;
|
|
backendNodeId: number;
|
|
sessionId?: string;
|
|
value?: string;
|
|
}
|
|
|
|
function extractAxValue(prop: any): string {
|
|
if (!prop) return '';
|
|
if (typeof prop === 'string') return prop;
|
|
if (prop.value !== undefined) {
|
|
if (typeof prop.value === 'string') return prop.value;
|
|
if (typeof prop.value === 'object' && prop.value && 'value' in prop.value) {
|
|
return String(prop.value.value || '');
|
|
}
|
|
}
|
|
return '';
|
|
}
|
|
|
|
interface CdpResult { ok: boolean; result?: any; error?: string }
|
|
|
|
// sessionId undefined => root frame; a child-frame sessionId => that OOPIF.
|
|
async function sendCdp(wv: BrowserWebview, method: string, params?: Record<string, any>, sessionId?: string): Promise<any> {
|
|
const wcId = wv.getWebContentsId();
|
|
const bridge = (window as any).openswarm?.sendCdpCommand as
|
|
| ((id: number, m: string, p?: any, s?: string) => Promise<CdpResult>)
|
|
| undefined;
|
|
if (!bridge) throw new Error('CDP bridge not available, restart the app');
|
|
const resp = await bridge(wcId, method, params, sessionId);
|
|
if (!resp || !resp.ok) {
|
|
throw new Error(resp?.error || `CDP ${method} failed`);
|
|
}
|
|
return resp.result;
|
|
}
|
|
|
|
interface ChildSession { sessionId: string; frameId: string; parentSessionId: string | null; url: string }
|
|
|
|
async function getChildSessions(wv: BrowserWebview): Promise<ChildSession[]> {
|
|
const bridge = (window as any).openswarm?.cdpChildSessionsGet as
|
|
| ((id: number) => Promise<ChildSession[]>) | undefined;
|
|
if (!bridge) return [];
|
|
try {
|
|
return (await bridge(wv.getWebContentsId())) || [];
|
|
} catch {
|
|
return [];
|
|
}
|
|
}
|
|
|
|
// Roles whose name is useful as disambiguating context for a nearby control (the person's name above a "Message" button, the section heading of a form).
|
|
const _CONTEXT_ROLES = new Set(['heading', 'statictext', 'link']);
|
|
const _CONTEXT_MAX_CHARS = 60;
|
|
const _CONTEXT_LOOKBACK = 30;
|
|
|
|
function axNodesToCandidates(nodes: any[], sessionId?: string): RankItem[] {
|
|
const byId = new Map<string, any>();
|
|
const parentOf = new Map<string, string>();
|
|
const orderOf = new Map<string, number>();
|
|
for (let i = 0; i < nodes.length; i++) {
|
|
const node = nodes[i];
|
|
if (node.nodeId != null) {
|
|
byId.set(String(node.nodeId), node);
|
|
orderOf.set(String(node.nodeId), i);
|
|
}
|
|
}
|
|
for (const node of nodes) {
|
|
for (const c of node.childIds || []) parentOf.set(String(c), String(node.nodeId));
|
|
}
|
|
const isCandidate = (n: any): boolean => {
|
|
if (!n || n.ignored || n.backendDOMNodeId == null) return false;
|
|
return INTERACTIVE_ROLES.has(extractAxValue(n.role));
|
|
};
|
|
// Which card/section does this control sit in? Nearest named non-interactive ancestor wins (a listitem's name aggregates its card text); else the nearest preceding heading/text/link in document order (browser-use's trick). This is what tells "Message" for Tyler apart from "Message" for everyone else.
|
|
const contextOf = (node: any, ownName: string): string => {
|
|
let p = parentOf.get(String(node.nodeId));
|
|
for (let hops = 0; p && hops < 12; hops++) {
|
|
const anc = byId.get(p);
|
|
if (anc && !anc.ignored && !isCandidate(anc)) {
|
|
const ancName = extractAxValue(anc.name).trim();
|
|
if (ancName && ancName !== ownName) return ancName.slice(0, _CONTEXT_MAX_CHARS);
|
|
}
|
|
p = parentOf.get(p);
|
|
}
|
|
const pos = orderOf.get(String(node.nodeId));
|
|
if (pos == null) return '';
|
|
for (let i = pos - 1; i >= 0 && i >= pos - _CONTEXT_LOOKBACK; i--) {
|
|
const prev = nodes[i];
|
|
if (!prev || prev.ignored) continue;
|
|
if (!_CONTEXT_ROLES.has(extractAxValue(prev.role).toLowerCase())) continue;
|
|
const prevName = extractAxValue(prev.name).trim();
|
|
if (prevName && prevName !== ownName && prevName.length >= 3) {
|
|
return prevName.slice(0, _CONTEXT_MAX_CHARS);
|
|
}
|
|
}
|
|
return '';
|
|
};
|
|
// A same-named interactive ancestor owns this hit target (a link inside a button, an icon twin inside its labeled wrapper); listing both just gives the model two indexes for one click. Names must match so a menu never swallows its menuitems.
|
|
const twinOfAncestor = (node: any, name: string): boolean => {
|
|
if (!name) return false;
|
|
let p = parentOf.get(String(node.nodeId));
|
|
while (p) {
|
|
const anc = byId.get(p);
|
|
if (isCandidate(anc)) return extractAxValue(anc.name).slice(0, 80) === name;
|
|
p = parentOf.get(p);
|
|
}
|
|
return false;
|
|
};
|
|
const out: RankItem[] = [];
|
|
for (const node of nodes) {
|
|
if (node.ignored) continue;
|
|
const role = extractAxValue(node.role);
|
|
if (!INTERACTIVE_ROLES.has(role)) continue;
|
|
const name = extractAxValue(node.name);
|
|
if (!name && role !== 'textbox' && role !== 'searchbox' && role !== 'combobox') continue;
|
|
const backendNodeId = node.backendDOMNodeId;
|
|
if (backendNodeId == null) continue;
|
|
const shortName = name.slice(0, 80);
|
|
if (twinOfAncestor(node, shortName)) continue;
|
|
let value = '';
|
|
if (role === 'textbox' || role === 'searchbox' || role === 'combobox') {
|
|
const isProtected = (node.properties || []).some(
|
|
(p: any) => p?.name === 'protected' && p?.value?.value === true,
|
|
);
|
|
value = isProtected ? '' : extractAxValue(node.value).slice(0, 60);
|
|
}
|
|
out.push({ role, name: shortName, backendNodeId, sessionId, context: contextOf(node, name), value });
|
|
}
|
|
return out;
|
|
}
|
|
|
|
// Cumulative top-left offset of a frame within the root viewport: climb the session chain adding each owning <iframe>'s top-left. Used ONLY to place the cosmetic click ripple; the click itself dispatches in the element's own frame, so this is best-effort. Verified getFrameOwner works through Electron.
|
|
async function frameOffset(
|
|
wv: BrowserWebview, sessionId: string | undefined, children: ChildSession[],
|
|
): Promise<{ dx: number; dy: number }> {
|
|
let dx = 0, dy = 0;
|
|
const byId = new Map(children.map((c) => [c.sessionId, c]));
|
|
const seen = new Set<string>();
|
|
let s: string | null | undefined = sessionId;
|
|
while (s && !seen.has(s)) {
|
|
seen.add(s);
|
|
const info = byId.get(s);
|
|
if (!info) break;
|
|
const parent = info.parentSessionId || undefined; // undefined => root
|
|
const owner = await sendCdp(wv, 'DOM.getFrameOwner', { frameId: info.frameId }, parent);
|
|
const ownerBox = await sendCdp(wv, 'DOM.getBoxModel', { backendNodeId: owner.backendNodeId }, parent);
|
|
const oc = ownerBox?.model?.content;
|
|
if (!Array.isArray(oc) || oc.length < 8) break;
|
|
dx += oc[0];
|
|
dy += oc[1];
|
|
s = info.parentSessionId;
|
|
}
|
|
return { dx, dy };
|
|
}
|
|
|
|
// Enumerate interactive elements across the root frame + every attached OOPIF child frame. Shared by list_interactives (numbered list) and click_by_name (stable re-resolution for replay), so both see the exact same surface. Root gets a generous budget (a big real page legitimately takes a few seconds); only a genuinely hung renderer exceeds it. Child frames (usually tracker/ad OOPIFs on heavy sites) must be quick or they're skipped, and we cap how many we walk so an ad-heavy page can't multiply full-tree calls. The page's own content is in the root tree (the about:blank compose iframe is same-process, so it's in the root too).
|
|
const _AX_ROOT_TIMEOUT_MS = 8000;
|
|
const _AX_CHILD_TIMEOUT_MS = 2500;
|
|
const _MAX_AX_CHILD_FRAMES = 6;
|
|
const _PAGE_TREE_TIMEOUT_MS = 1500;
|
|
const _MAX_TOTAL_FRAMES = 12;
|
|
|
|
function withTimeout<T>(p: Promise<T>, ms: number, label: string): Promise<T> {
|
|
p.catch(() => {}); // swallow a late rejection if the timeout wins the race first
|
|
let timer: ReturnType<typeof setTimeout>;
|
|
const timeout = new Promise<never>((_, reject) => {
|
|
timer = setTimeout(() => reject(new Error(`${label} timed out after ${ms}ms`)), ms);
|
|
});
|
|
return Promise.race([p, timeout]).finally(() => clearTimeout(timer));
|
|
}
|
|
|
|
function flattenFrameTree(tree: any, out: string[] = []): string[] {
|
|
if (!tree) return out;
|
|
const id = tree?.frame?.id;
|
|
if (id) out.push(id);
|
|
for (const c of tree.childFrames || []) flattenFrameTree(c, out);
|
|
return out;
|
|
}
|
|
|
|
async function enumerateCandidates(wv: BrowserWebview): Promise<RankItem[]> {
|
|
const candidates: RankItem[] = [];
|
|
let framesWalked = 0;
|
|
let framesDropped = 0;
|
|
|
|
const walkSession = async (
|
|
sessionId: string | undefined, budgetMs: number, label: string,
|
|
): Promise<{ ok: boolean; lastErr?: any }> => {
|
|
const sessionStart = Date.now();
|
|
const remaining = () => Math.max(1, budgetMs - (Date.now() - sessionStart));
|
|
|
|
let frameIds: string[] = [];
|
|
try {
|
|
const tree = await withTimeout(
|
|
sendCdp(wv, 'Page.getFrameTree', {}, sessionId),
|
|
Math.min(_PAGE_TREE_TIMEOUT_MS, remaining()), `${label} frame tree`);
|
|
frameIds = flattenFrameTree(tree?.frameTree);
|
|
} catch { /* fall through to a single AX call below */ }
|
|
|
|
if (frameIds.length === 0) {
|
|
if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; return { ok: true }; }
|
|
try {
|
|
const ax = await withTimeout(
|
|
sendCdp(wv, 'Accessibility.getFullAXTree', {}, sessionId), remaining(), label);
|
|
candidates.push(...axNodesToCandidates(ax?.nodes || [], sessionId));
|
|
framesWalked++;
|
|
return { ok: true };
|
|
} catch (err: any) {
|
|
return { ok: false, lastErr: err };
|
|
}
|
|
}
|
|
|
|
let lastErr: any;
|
|
let anySuccess = false;
|
|
for (const frameId of frameIds) {
|
|
if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; continue; }
|
|
if (remaining() <= 1) { framesDropped++; continue; }
|
|
try {
|
|
const ax = await withTimeout(
|
|
sendCdp(wv, 'Accessibility.getFullAXTree', { frameId }, sessionId), remaining(), label);
|
|
candidates.push(...axNodesToCandidates(ax?.nodes || [], sessionId));
|
|
anySuccess = true;
|
|
} catch (err: any) { lastErr = err; }
|
|
framesWalked++;
|
|
}
|
|
return { ok: anySuccess, lastErr };
|
|
};
|
|
|
|
const root = await walkSession(undefined, _AX_ROOT_TIMEOUT_MS, 'page perception');
|
|
if (!root.ok) {
|
|
// A saturated/hung renderer can't answer; surface a clear, actionable signal instead of silently blocking to the hard command timeout, so the agent can wait a beat and retry (the freeze is often intermittent) rather than abort.
|
|
throw new Error(
|
|
`the page is too busy to read right now (${root.lastErr?.message || 'timed out'}); `
|
|
+ 'wait a moment with BrowserWait and try again, or reload the page.');
|
|
}
|
|
const children = (await getChildSessions(wv)).slice(0, _MAX_AX_CHILD_FRAMES);
|
|
for (const child of children) {
|
|
if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; continue; }
|
|
await walkSession(child.sessionId, _AX_CHILD_TIMEOUT_MS, 'child frame');
|
|
}
|
|
if (framesDropped > 0) {
|
|
console.log(`[cdp] enumerateCandidates capped at ${_MAX_TOTAL_FRAMES} frames; dropped ${framesDropped}`);
|
|
}
|
|
return candidates;
|
|
}
|
|
|
|
// Resolve + click a specific backend node (revalidate, frame-local box model, OS-level dispatch in the element's own frame, cosmetic top-level ripple). Shared by click_index (cache lookup) and click_by_name (fresh resolution).
|
|
async function clickBackendNode(
|
|
wv: BrowserWebview, backendNodeId: number, sessionId: string | undefined, label: string,
|
|
opts: { role?: string; text?: string } = {},
|
|
): Promise<Record<string, any>> {
|
|
let resolvedObjectId: string | undefined;
|
|
try {
|
|
const resolved = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
|
resolvedObjectId = resolved?.object?.objectId;
|
|
} catch (err: any) {
|
|
return { error: `${label} is no longer valid (${err.message || 'node not found'}). The page may have changed.` };
|
|
}
|
|
|
|
// Brief outline pulse on the element the agent chose, so a watching user sees WHAT is being acted on, not just a ripple somewhere on the page.
|
|
if (resolvedObjectId) {
|
|
sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: resolvedObjectId,
|
|
functionDeclaration:
|
|
'function() {'
|
|
+ ' const o = this.style.outline, f = this.style.outlineOffset;'
|
|
+ ' this.style.outline = "3px solid rgba(77,163,255,0.9)"; this.style.outlineOffset = "2px";'
|
|
+ ' setTimeout(() => { this.style.outline = o; this.style.outlineOffset = f; }, 450);'
|
|
+ ' }',
|
|
}, sessionId).catch(() => {});
|
|
}
|
|
|
|
// A text box is focused DIRECTLY by node id, never by screen coordinates. Inside an about:blank compose iframe (LinkedIn/Gmail messaging) the coordinate path lands on the wrong element (the box model is frame-local but the click dispatches in the root frame), while DOM.focus reaches the node in any frame. With a `text` arg we then insert the whole string at once, no clicking, no character-by-character typing.
|
|
const _role = opts.role || '';
|
|
const _wantsText = typeof opts.text === 'string' && opts.text.length > 0;
|
|
if (/\b(textbox|searchbox)\b/i.test(_role) || (/\bcombobox\b/i.test(_role) && _wantsText)) {
|
|
try {
|
|
await sendCdp(wv, 'DOM.focus', { backendNodeId }, sessionId);
|
|
} catch (err: any) {
|
|
return { error: `${label} could not be focused (${err?.message || 'focus failed'}); it may be disabled or hidden.` };
|
|
}
|
|
if (typeof opts.text === 'string' && opts.text.length > 0) {
|
|
// Read the text back from the node itself; "insert reported OK" is not "the box has the text" (rich-text editors can swallow synthetic input). Returns the box's actual content (or null on miss) so the result can echo the OBSERVED state; a bare "typed it" claim loses to a wrongly pessimistic expect-confirm and provokes a double-fill.
|
|
const readBack = async (): Promise<string | null> => {
|
|
try {
|
|
const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
|
const r = await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: t.object.objectId,
|
|
functionDeclaration:
|
|
'function(s) { const v = (this.value !== undefined ? this.value : this.textContent) || ""; return v.includes(s) ? v.slice(0, 120) : null; }',
|
|
arguments: [{ value: opts.text }],
|
|
returnByValue: true,
|
|
}, sessionId);
|
|
return typeof r?.result?.value === 'string' ? r.result.value : null;
|
|
} catch { return opts.text ?? ''; } // unverifiable beats a false alarm
|
|
};
|
|
const landedMsg = (got: string, via = '') =>
|
|
({ text: `Focused ${label} and typed the text in${via}. Verified: the box now contains "${got}". Do NOT type it again.` });
|
|
try {
|
|
await sendCdp(wv, 'Input.insertText', { text: opts.text }, sessionId);
|
|
} catch (err: any) {
|
|
return { error: `Focused ${label} but could not type into it: ${err?.message || String(err)}` };
|
|
}
|
|
let got = await readBack();
|
|
if (got !== null) return landedMsg(got);
|
|
try {
|
|
const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
|
await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: t.object.objectId,
|
|
functionDeclaration:
|
|
'function(s) { this.focus(); document.execCommand("insertText", false, s); }',
|
|
arguments: [{ value: opts.text }],
|
|
}, sessionId);
|
|
} catch { /* verified below; the honest error covers this failing too */ }
|
|
got = await readBack();
|
|
if (got !== null) return landedMsg(got, ' (via editor command)');
|
|
return { error: `Focused ${label} but the text did not register; the box may be a custom editor. Try BrowserPressKey per character or a different element.` };
|
|
}
|
|
return { text: `Focused ${label}; the cursor is in it now (type with BrowserPressKey, or pass a text arg to fill it in one call).` };
|
|
}
|
|
|
|
try {
|
|
await sendCdp(wv, 'DOM.scrollIntoViewIfNeeded', { backendNodeId }, sessionId);
|
|
} catch { /* not scrollable or already visible; the box model below decides */ }
|
|
let boxModel;
|
|
try {
|
|
boxModel = await sendCdp(wv, 'DOM.getBoxModel', { backendNodeId }, sessionId);
|
|
} catch (err: any) {
|
|
return { error: `${label} has no box model (likely off-screen or hidden). Try scrolling first.` };
|
|
}
|
|
const content = boxModel?.model?.content;
|
|
if (!Array.isArray(content) || content.length < 8) {
|
|
return { error: `${label} has no valid bounding rect.` };
|
|
}
|
|
const lx = (content[0] + content[4]) / 2;
|
|
const ly = (content[1] + content[5]) / 2;
|
|
|
|
// Hit-test before dispatching: a sticky banner or header twin can cover the element's center, and a blind coordinate click lands on the overlay instead (the "Reactivate Premium" misfire). If covered, click the chosen node itself.
|
|
// This is also the ground-truth "did my hand land where I aimed" signal: when the element at the click point is NOT the intended node, a naive coordinate click hits the WRONG element (the failure a page-change metric can't see, because the wrong element also changes the page). We surface what was actually hit so the wrong-target RATE is measurable.
|
|
let covered = false;
|
|
let hitDesc = '';
|
|
let targetObjectId: string | undefined;
|
|
try {
|
|
const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
|
targetObjectId = t?.object?.objectId;
|
|
if (targetObjectId) {
|
|
const rel = await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: targetObjectId,
|
|
functionDeclaration:
|
|
'function(x, y) { const r = this.getRootNode(); const h = (r.elementFromPoint ? r : document).elementFromPoint(x, y);'
|
|
+ ' const landed = h ? (this === h || this.contains(h) || h.contains(this)) : true;'
|
|
+ ' const d = h ? (h.tagName.toLowerCase() + (h.getAttribute("aria-label") ? "[" + h.getAttribute("aria-label").slice(0,30) + "]" : (h.textContent ? "[" + h.textContent.trim().slice(0,30) + "]" : ""))) : "none";'
|
|
+ ' return { landed: landed, hit: d }; }',
|
|
arguments: [{ value: lx }, { value: ly }],
|
|
returnByValue: true,
|
|
}, sessionId);
|
|
const hv = rel?.result?.value as { landed?: boolean; hit?: string } | undefined;
|
|
covered = hv?.landed === false;
|
|
hitDesc = hv?.hit || '';
|
|
}
|
|
} catch { /* hit-test is best-effort; fall through to the coordinate click */ }
|
|
// Attach the aim signal to every click result (near-zero cost, already computed): landed=false means the intended element was not under the click point (occluded / stale box / moved), i.e. a wrong-target click.
|
|
const p_aim = { clickLanded: !covered, clickHit: hitDesc };
|
|
|
|
let rx = lx, ry = ly;
|
|
if (sessionId) {
|
|
try {
|
|
const children = await getChildSessions(wv);
|
|
const { dx, dy } = await frameOffset(wv, sessionId, children);
|
|
rx = lx + dx; ry = ly + dy;
|
|
} catch { /* fall back to frame-local for the ripple */ }
|
|
}
|
|
const ripple = { clickX: rx / wv.clientWidth * 100, clickY: ry / wv.clientHeight * 100 };
|
|
|
|
if (covered && targetObjectId) {
|
|
try {
|
|
await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId: targetObjectId,
|
|
functionDeclaration:
|
|
'function() { if (this.click) { this.click(); } else { this.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true })); } }',
|
|
}, sessionId);
|
|
return { text: `Clicked ${label} via its element (another element covers its screen position).`, ...ripple, ...p_aim };
|
|
} catch (err: any) {
|
|
return { error: `${label} is covered by another element and could not be clicked (${err?.message || String(err)}). Scroll, or pick a different element.` };
|
|
}
|
|
}
|
|
|
|
try {
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x: lx, y: ly, button: 'left', clickCount: 1 }, sessionId);
|
|
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x: lx, y: ly, button: 'left', clickCount: 1 }, sessionId);
|
|
} catch (err: any) {
|
|
return { error: `Click failed: ${err.message || String(err)}` };
|
|
}
|
|
return {
|
|
text: `Clicked ${label} at (${Math.round(rx)}, ${Math.round(ry)})`,
|
|
...ripple,
|
|
...p_aim,
|
|
};
|
|
}
|
|
|
|
// Wrap a click so we can measure whether it actually did anything, covering EVERY
|
|
// exit of clickBackendNode (coordinate, covered-element JS click, ...), which is why
|
|
// it lives out here and not in the one coordinate branch. Metric only, flag-gated.
|
|
async function measureClickEffect(
|
|
wv: BrowserWebview, run: () => Promise<Record<string, any>>,
|
|
): Promise<Record<string, any>> {
|
|
let before = '';
|
|
try { before = String(await wv.executeJavaScript(FP_EXPR)); } catch { /* unreadable */ }
|
|
const result = await run();
|
|
if (!result.error) {
|
|
await new Promise((r) => setTimeout(r, 400));
|
|
let after = '';
|
|
try { after = String(await wv.executeJavaScript(FP_EXPR)); } catch { /* unreadable */ }
|
|
result.clickEffect = clickEffect(before, after);
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Drop list rows the user literally cannot click: zero-size nodes and ones whose center hits a DIFFERENT element (modal backdrop, sticky header, cookie banner). Ground truth via elementFromPoint, the same predicate the click path trusts. Offscreen-but-scrollable elements are kept; the page-wide list is deliberately wider than the viewport. Chunked with a hard budget so a heavy page degrades to an unfiltered list, never a stall.
|
|
const _OCCLUSION_BUDGET_MS = 1500;
|
|
const _OCCLUSION_CHUNK = 10;
|
|
async function dropCoveredElements(
|
|
wv: BrowserWebview, items: RankItem[],
|
|
): Promise<{ kept: RankItem[]; dropped: number }> {
|
|
const deadline = Date.now() + _OCCLUSION_BUDGET_MS;
|
|
const kept: RankItem[] = [];
|
|
let dropped = 0;
|
|
for (let i = 0; i < items.length; i += _OCCLUSION_CHUNK) {
|
|
const chunk = items.slice(i, i + _OCCLUSION_CHUNK);
|
|
if (Date.now() > deadline) {
|
|
kept.push(...items.slice(i));
|
|
break;
|
|
}
|
|
const verdicts = await Promise.all(chunk.map(async (el) => {
|
|
try {
|
|
const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId: el.backendNodeId }, el.sessionId);
|
|
const objectId = t?.object?.objectId;
|
|
if (!objectId) return 'clear';
|
|
const r = await sendCdp(wv, 'Runtime.callFunctionOn', {
|
|
objectId,
|
|
functionDeclaration:
|
|
'function() {'
|
|
+ ' const rect = this.getBoundingClientRect();'
|
|
+ ' if (rect.width === 0 || rect.height === 0) return "hidden";'
|
|
+ ' const cx = rect.left + rect.width / 2, cy = rect.top + rect.height / 2;'
|
|
+ ' if (cx < 0 || cy < 0 || cx > window.innerWidth || cy > window.innerHeight) return "offscreen";'
|
|
+ ' const root = this.getRootNode();'
|
|
+ ' const h = (root.elementFromPoint ? root : document).elementFromPoint(cx, cy);'
|
|
+ ' if (!h) return "clear";'
|
|
+ ' return (this === h || this.contains(h) || h.contains(this)) ? "clear" : "covered";'
|
|
+ ' }',
|
|
returnByValue: true,
|
|
}, el.sessionId);
|
|
return r?.result?.value || 'clear';
|
|
} catch {
|
|
return 'clear';
|
|
}
|
|
}));
|
|
chunk.forEach((el, j) => {
|
|
if (verdicts[j] === 'covered' || verdicts[j] === 'hidden') dropped += 1;
|
|
else kept.push(el);
|
|
});
|
|
}
|
|
return { kept, dropped };
|
|
}
|
|
|
|
async function handleListInteractives(wv: BrowserWebview, params: Record<string, any> = {}): Promise<Record<string, any>> {
|
|
let candidates: RankItem[];
|
|
try {
|
|
candidates = await enumerateCandidates(wv);
|
|
} catch (err: any) {
|
|
return { error: `getFullAXTree failed: ${err.message || String(err)}` };
|
|
}
|
|
|
|
// Dedupe twins, rank what a human acts on first (and the current goal highest), cap the long tail.
|
|
const goal = typeof params?.goal === 'string' ? params.goal : '';
|
|
const { shown: ranked, truncated } = rankAndCapInteractives(candidates, { goal, docOrder: params?.docOrder !== false });
|
|
const { kept: shown, dropped: covered } = await dropCoveredElements(wv, ranked);
|
|
|
|
// The previous look's cache feeds two things: * markers for brand-new elements, and STABLE indices so the same element keeps the same number across looks (the model can act on a remembered index without re-reading the whole list, browser-use's stable-hash trick on our node ids).
|
|
let prevIds: Set<string> | null = null;
|
|
const prevIndexByKey = new Map<string, number>();
|
|
try {
|
|
const cacheBridge = (window as any).openswarm?.cdpCacheGet;
|
|
const prev = cacheBridge ? await cacheBridge(wv.getWebContentsId()) : null;
|
|
if (prev && typeof prev === 'object') {
|
|
for (const [idxStr, e] of Object.entries(prev)) {
|
|
const key = `${(typeof e === 'object' && (e as any)?.sessionId) || ''}:${typeof e === 'number' ? e : (e as any)?.backendNodeId}`;
|
|
prevIndexByKey.set(key, Number(idxStr));
|
|
}
|
|
prevIds = new Set(prevIndexByKey.keys());
|
|
}
|
|
} catch { /* no previous list; fresh numbering, no markers */ }
|
|
const keyOf = (el: RankItem) => `${el.sessionId || ''}:${el.backendNodeId}`;
|
|
const isNew = (el: RankItem) => !!prevIds && prevIds.size > 0 && !prevIds.has(keyOf(el));
|
|
|
|
// Sticky only while some elements carried over; full turnover (a navigation, node ids all changed) or runaway numbering restarts cleanly at 1.
|
|
const matchedPrev = shown.filter((el) => prevIndexByKey.has(keyOf(el))).length;
|
|
let nextFree = prevIndexByKey.size > 0 ? Math.max(...prevIndexByKey.values()) + 1 : 1;
|
|
const sticky = matchedPrev > 0 && nextFree + shown.length < 1000;
|
|
const used = new Set<number>();
|
|
const interactives: (InteractiveElement & { isNew: boolean; context?: string })[] = shown.map((el, i) => {
|
|
let index = sticky ? prevIndexByKey.get(keyOf(el)) : undefined;
|
|
if (index == null || used.has(index)) index = sticky ? nextFree++ : i + 1;
|
|
used.add(index);
|
|
return {
|
|
index,
|
|
role: el.role,
|
|
name: el.name,
|
|
backendNodeId: el.backendNodeId,
|
|
sessionId: el.sessionId,
|
|
value: el.value,
|
|
isNew: isNew(el),
|
|
context: el.context,
|
|
};
|
|
});
|
|
|
|
// Cache in main-process so click_index can resolve across separate WS commands. role+name ride along so click_index can report WHAT it clicked (the agent loop records that as a stable, replayable click-by-name step).
|
|
const indexMap: Record<number, { backendNodeId: number; sessionId?: string; role?: string; name?: string }> = {};
|
|
for (const el of interactives) {
|
|
indexMap[el.index] = { backendNodeId: el.backendNodeId, sessionId: el.sessionId, role: el.role, name: el.name };
|
|
}
|
|
try {
|
|
const cacheBridge = (window as any).openswarm?.cdpCacheSet;
|
|
if (cacheBridge) await cacheBridge(wv.getWebContentsId(), indexMap);
|
|
} catch {
|
|
// best-effort; click_index falls back to re-listing.
|
|
}
|
|
|
|
// ctx only where it disambiguates: rows whose role+name appear more than once (eight "Message" buttons). Unique rows skip it to keep tokens lean.
|
|
const nameCounts = new Map<string, number>();
|
|
for (const el of interactives) {
|
|
const k = `${el.role}|${el.name}`;
|
|
nameCounts.set(k, (nameCounts.get(k) || 0) + 1);
|
|
}
|
|
const lines = interactives.map((el) => {
|
|
const dup = (nameCounts.get(`${el.role}|${el.name}`) || 0) > 1;
|
|
const ctx = dup && el.context ? ` ctx="${el.context}"` : '';
|
|
const val = el.value ? ` value="${el.value}"` : '';
|
|
return `[${el.index}]${el.isNew ? '*' : ''}<${el.role} "${el.name}"${ctx}${val}>`;
|
|
});
|
|
let text: string;
|
|
if (lines.length === 0) {
|
|
text = 'No interactive elements found on this page.';
|
|
} else {
|
|
text = `${lines.length} interactive elements (* = new since your last look; same number = same element as before):\n${lines.join('\n')}`;
|
|
if (truncated > 0) {
|
|
text += `\n... ${truncated} more not shown; scroll or scope with BrowserGetElements to reach them.`;
|
|
}
|
|
if (covered > 0) {
|
|
text += `\n(${covered} elements hidden behind overlays were omitted; close the overlay to reach them.)`;
|
|
}
|
|
}
|
|
|
|
return {
|
|
text,
|
|
elements: interactives.map((el) => ({ index: el.index, role: el.role, name: el.name })),
|
|
url: wv.getURL(),
|
|
};
|
|
}
|
|
|
|
async function handleClickIndex(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const idx = Number(params.index);
|
|
if (!Number.isFinite(idx) || idx < 1) {
|
|
return { error: 'index parameter is required and must be a positive integer' };
|
|
}
|
|
|
|
let backendNodeId: number | undefined;
|
|
let sessionId: string | undefined;
|
|
let role: string | undefined;
|
|
let name: string | undefined;
|
|
try {
|
|
const cacheBridge = (window as any).openswarm?.cdpCacheGet;
|
|
if (cacheBridge) {
|
|
const cached = await cacheBridge(wv.getWebContentsId());
|
|
const entry = cached && cached[idx];
|
|
if (typeof entry === 'number') {
|
|
backendNodeId = entry; // legacy cache shape
|
|
} else if (entry && typeof entry === 'object' && entry.backendNodeId != null) {
|
|
backendNodeId = Number(entry.backendNodeId);
|
|
sessionId = entry.sessionId || undefined;
|
|
role = entry.role; name = entry.name;
|
|
}
|
|
}
|
|
} catch {
|
|
// fall through to error path below
|
|
}
|
|
|
|
if (backendNodeId == null) {
|
|
return {
|
|
error: `Index ${idx} is not in the cached element map. Call BrowserListInteractives first to refresh the index, then try again.`,
|
|
};
|
|
}
|
|
|
|
const wantsText = typeof params.text === 'string' && params.text.length > 0;
|
|
const doClick = () => clickBackendNode(wv, backendNodeId as number, sessionId, `index ${idx}`,
|
|
{ role, text: wantsText ? params.text : undefined });
|
|
// Effect metric on plain clicks only (a fill verifies via readback); covers every clickBackendNode exit.
|
|
const result = (params.effectProbe === true && !wantsText)
|
|
? await measureClickEffect(wv, doClick)
|
|
: await doClick();
|
|
// Self-healing escalation: a cached index goes stale the instant the page mutates, which is HALF of all runs' tool-errors. An explicit clickBackendNode error PROVES the click never landed (so re-trying the same target can't double-act), so before we hand a ~3s re-strategize turn back to the model, resolve the SAME element fresh from the full DOM by its name+role, the exact rung the model would have climbed to itself. Plain clicks only (a text-fill has its own readback path); gated so an A/B can turn it off.
|
|
if (shouldSelfHealClick(!!result.error, wantsText, name, params.selfheal)) {
|
|
const healed = await handleClickByName(wv, { name: name as string, role: role || '' });
|
|
if (!healed.error) {
|
|
console.log(`[selfheal] index ${idx} stale -> recovered via name "${String(name).slice(0, 40)}"`);
|
|
healed.clickedRole = role || '';
|
|
healed.clickedName = name || '';
|
|
healed.selfHealed = 'by-name';
|
|
return healed;
|
|
}
|
|
}
|
|
// Surface what was clicked so the agent loop can record a stable, replayable click-by-name step (indices are ephemeral; names aren't).
|
|
if (!result.error) {
|
|
result.clickedRole = role || '';
|
|
result.clickedName = name || '';
|
|
}
|
|
return result;
|
|
}
|
|
|
|
// Robust click for REPLAY: re-resolve the target fresh by (role, name) instead of a stale index, so a recorded skill survives index shifts between runs.
|
|
async function handleClickByName(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const wantName = String(params.name || '').trim();
|
|
const wantRole = String(params.role || '').trim();
|
|
if (!wantName && !wantRole) return { error: 'click_by_name needs a name and/or role' };
|
|
let candidates: RankItem[];
|
|
try {
|
|
candidates = await enumerateCandidates(wv);
|
|
} catch (err: any) {
|
|
return { error: `enumerate failed: ${err.message || String(err)}` };
|
|
}
|
|
const norm = (s: string) => s.trim().toLowerCase();
|
|
// Exact (role,name) first, then name-only, so we click the most specific match. Long names are card blobs whose SUFFIX mutates between visits (feed snippets, counters) while the prefix stays stable; fall back to a 40-char prefix match so a replayed click survives the churn.
|
|
const wantPrefix = norm(wantName).slice(0, 40);
|
|
const match =
|
|
candidates.find((c) => (!wantRole || norm(c.role) === norm(wantRole)) && norm(c.name) === norm(wantName)) ||
|
|
candidates.find((c) => norm(c.name) === norm(wantName)) ||
|
|
(wantName.length > 40
|
|
? candidates.find((c) => (!wantRole || norm(c.role) === norm(wantRole)) && norm(c.name).startsWith(wantPrefix))
|
|
: undefined);
|
|
if (!match) {
|
|
return { error: `No element matching role="${wantRole}" name="${wantName}" on this page.` };
|
|
}
|
|
return clickBackendNode(wv, match.backendNodeId, match.sessionId, `${match.role} "${match.name}"`,
|
|
{ role: match.role });
|
|
}
|
|
|
|
// Sequential sub-actions; aborts mid-batch if URL changes (indices/selectors go stale on navigation).
|
|
const MAX_BATCH_ACTIONS = 5;
|
|
|
|
type SubActionType =
|
|
| 'click_index' | 'press_key' | 'type' | 'wait'
|
|
| 'scroll' | 'navigate' | 'click' | 'click_point' | 'list_interactives';
|
|
|
|
const BATCH_DISPATCH: Record<SubActionType, (wv: BrowserWebview, p: Record<string, any>) => Promise<Record<string, any>>> = {
|
|
click_index: handleClickIndex,
|
|
press_key: handlePressKey,
|
|
click_point: handleClickPoint,
|
|
type: handleType,
|
|
wait: handleWait,
|
|
scroll: handleScroll,
|
|
navigate: handleNavigate,
|
|
click: handleClick,
|
|
// Allowed as the LAST sub-action so a click->wait->read folds into one turn.
|
|
list_interactives: handleListInteractives,
|
|
};
|
|
|
|
async function handleBatch(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const actions: any[] = Array.isArray(params.actions) ? params.actions : [];
|
|
if (actions.length === 0) {
|
|
return { error: 'actions parameter must be a non-empty array' };
|
|
}
|
|
if (actions.length > MAX_BATCH_ACTIONS) {
|
|
return {
|
|
error: `Batch too large: ${actions.length} actions (max ${MAX_BATCH_ACTIONS}). Split into smaller batches.`,
|
|
};
|
|
}
|
|
|
|
const results: Array<Record<string, any>> = [];
|
|
let aborted_at: number | null = null;
|
|
let abort_reason: string | null = null;
|
|
|
|
for (let i = 0; i < actions.length; i++) {
|
|
const action = actions[i];
|
|
const subType = action?.type as SubActionType;
|
|
const subParams = action?.params || {};
|
|
|
|
if (!subType || !(subType in BATCH_DISPATCH)) {
|
|
results.push({ index: i, type: subType, error: `Unknown sub-action type: ${subType}` });
|
|
// per-action failures don't abort the batch
|
|
continue;
|
|
}
|
|
|
|
const urlBefore = wv.getURL();
|
|
// Carry the self-heal + click-effect toggles into click sub-actions (most clicks are batched, so gating only the top-level click_index misses them).
|
|
if (subType === 'click_index') {
|
|
if (params.selfheal !== undefined && subParams.selfheal === undefined) subParams.selfheal = params.selfheal;
|
|
if (params.effectProbe && subParams.effectProbe === undefined) subParams.effectProbe = true;
|
|
}
|
|
let subResult: Record<string, any>;
|
|
try {
|
|
subResult = await BATCH_DISPATCH[subType](wv, subParams);
|
|
} catch (err: any) {
|
|
subResult = { error: `Sub-action failed: ${err?.message || String(err)}` };
|
|
}
|
|
results.push({ index: i, type: subType, ...subResult });
|
|
|
|
// A failed sub-action means every later one is operating on a page that isn't in the state it assumed, so stop instead of compounding the error (browser-use's multi_act breaks the same way). The terminal read is the last action, so a read failure never trips this.
|
|
if (subResult.error && i < actions.length - 1) {
|
|
aborted_at = i + 1;
|
|
abort_reason = `Sub-action ${i + 1} (${subType}) failed: ${subResult.error}; remaining ${actions.length - i - 1} action(s) skipped`;
|
|
break;
|
|
}
|
|
|
|
// URL changed: selectors and indices are stale on the half-loaded page; abort.
|
|
const urlAfter = wv.getURL();
|
|
if (urlAfter !== urlBefore && i < actions.length - 1) {
|
|
aborted_at = i + 1;
|
|
abort_reason = `URL changed mid-batch from ${urlBefore} to ${urlAfter}; remaining ${actions.length - i - 1} action(s) skipped`;
|
|
break;
|
|
}
|
|
}
|
|
|
|
const summary_lines = results.map((r, i) => {
|
|
const status = r.error ? `FAIL (${r.error})` : 'OK';
|
|
return ` ${i + 1}. ${r.type}: ${status}`;
|
|
});
|
|
const text = [
|
|
`Batch executed ${results.length}/${actions.length} actions`,
|
|
...summary_lines,
|
|
aborted_at !== null ? `\nABORTED at action ${aborted_at}: ${abort_reason}` : '',
|
|
].filter(Boolean).join('\n');
|
|
|
|
return {
|
|
text,
|
|
results,
|
|
aborted_at,
|
|
abort_reason,
|
|
url: wv.getURL(),
|
|
};
|
|
}
|
|
|
|
async function handleScroll(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const direction = (params.direction as string) || 'down';
|
|
const amount = (params.amount as number) || 500;
|
|
const code = `(() => {
|
|
function findScrollable() {
|
|
const candidates = document.querySelectorAll(
|
|
'[class*="scroller"], [class*="scroll-container"], [class*="content"], '
|
|
+ 'main, [role="main"], article, .notion-scroller, .notion-frame'
|
|
);
|
|
for (const el of candidates) {
|
|
const s = window.getComputedStyle(el);
|
|
const isScrollable = (s.overflow === 'auto' || s.overflow === 'scroll'
|
|
|| s.overflowY === 'auto' || s.overflowY === 'scroll');
|
|
if (isScrollable && el.scrollHeight > el.clientHeight + 10) return el;
|
|
}
|
|
const all = document.querySelectorAll('*');
|
|
for (const el of all) {
|
|
if (el === document.body || el === document.documentElement) continue;
|
|
const s = window.getComputedStyle(el);
|
|
const isScrollable = (s.overflow === 'auto' || s.overflow === 'scroll'
|
|
|| s.overflowY === 'auto' || s.overflowY === 'scroll');
|
|
if (isScrollable && el.scrollHeight > el.clientHeight + 50
|
|
&& el.clientHeight > 200) return el;
|
|
}
|
|
return null;
|
|
}
|
|
const dy = ${JSON.stringify(direction)} === 'up' ? -${amount} : ${amount};
|
|
const container = findScrollable();
|
|
if (container) {
|
|
const before = container.scrollTop;
|
|
container.scrollBy({ top: dy, behavior: 'instant' });
|
|
const after = container.scrollTop;
|
|
return {
|
|
scrolled: Math.abs(after - before),
|
|
scrollTop: after,
|
|
scrollHeight: container.scrollHeight,
|
|
clientHeight: container.clientHeight,
|
|
atTop: after <= 0,
|
|
atBottom: after + container.clientHeight >= container.scrollHeight - 5,
|
|
target: 'container',
|
|
};
|
|
}
|
|
const before = window.scrollY;
|
|
window.scrollBy({ top: dy, behavior: 'instant' });
|
|
const after = window.scrollY;
|
|
return {
|
|
scrolled: Math.abs(after - before),
|
|
scrollTop: after,
|
|
scrollHeight: document.documentElement.scrollHeight,
|
|
clientHeight: window.innerHeight,
|
|
atTop: after <= 0,
|
|
atBottom: after + window.innerHeight >= document.documentElement.scrollHeight - 5,
|
|
target: 'window',
|
|
};
|
|
})()`;
|
|
try {
|
|
const result = await evalInPage(wv, code);
|
|
const status = result.atBottom ? ' (reached bottom)' : result.atTop ? ' (reached top)' : '';
|
|
return {
|
|
text: `Scrolled ${direction} by ${result.scrolled}px${status}. Position: ${result.scrollTop}/${result.scrollHeight - result.clientHeight}px`,
|
|
...result,
|
|
url: wv.getURL(),
|
|
};
|
|
} catch (err: any) {
|
|
return { error: `Scroll failed: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
async function handleWait(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const ms = Math.min(Math.max((params.milliseconds as number) || 1000, 100), 10000);
|
|
const until = typeof params.until === 'string' ? params.until : '';
|
|
const probeJs = settleProbeJs(until);
|
|
const start = Date.now();
|
|
let settled = false;
|
|
let found = false;
|
|
let probeErrors = 0;
|
|
let lastElems: number | null = null;
|
|
let elemsChangedAt = start; // DOM-settle clock
|
|
while (Date.now() - start < ms) {
|
|
const remaining = ms - (Date.now() - start);
|
|
await new Promise((resolve) => setTimeout(resolve, Math.min(SETTLE_POLL_MS, Math.max(0, remaining))));
|
|
const elapsed = Date.now() - start;
|
|
if (elapsed >= ms) break;
|
|
try {
|
|
const probe = JSON.parse(await evalInPage(wv, probeJs));
|
|
probeErrors = 0;
|
|
if (probe.elems !== lastElems) { lastElems = probe.elems; elemsChangedAt = Date.now(); }
|
|
const domStable = Date.now() - elemsChangedAt;
|
|
if (shouldStopWaiting(probe.ready, probe.quiet || 0, domStable, !!probe.found, elapsed)) {
|
|
settled = true; found = !!probe.found; break;
|
|
}
|
|
} catch {
|
|
// Mid-navigation pages aren't evaluable yet; a few misses is normal, but a wedged tab shouldn't make us burn the whole cap, so bail after a short streak.
|
|
if (++probeErrors >= 3) break;
|
|
}
|
|
}
|
|
const waited = Date.now() - start;
|
|
const state = found ? 'found target' : settled ? 'page settled' : 'reached cap';
|
|
return {
|
|
text: `Waited ${waited}ms (${state}). Current URL: ${wv.getURL()}`,
|
|
url: wv.getURL(),
|
|
title: wv.getTitle(),
|
|
};
|
|
}
|
|
|
|
async function handleGetElements(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const scope = (params.selector as string) || 'body';
|
|
const safeScope = JSON.stringify(scope);
|
|
const code = `(() => {
|
|
const scope = document.querySelector(${safeScope}) || document.body;
|
|
const interactive = scope.querySelectorAll(
|
|
'a[href], button, input, textarea, select, [role="button"], [role="link"], '
|
|
+ '[role="textbox"], [role="searchbox"], [role="menuitem"], [role="tab"], '
|
|
+ '[role="checkbox"], [role="switch"], [role="option"], '
|
|
+ '[onclick], [tabindex]:not([tabindex="-1"]), '
|
|
+ '[data-block-id], [contenteditable="true"]'
|
|
);
|
|
const seen = new Set();
|
|
const results = [];
|
|
for (const el of interactive) {
|
|
if (results.length >= 80) break;
|
|
const rect = el.getBoundingClientRect();
|
|
if (rect.width === 0 && rect.height === 0) continue;
|
|
const style = window.getComputedStyle(el);
|
|
if (style.visibility === 'hidden' || style.display === 'none') continue;
|
|
if (style.opacity === '0') continue;
|
|
|
|
let selector = el.tagName.toLowerCase();
|
|
if (el.id) {
|
|
selector = '#' + CSS.escape(el.id);
|
|
} else if (el.getAttribute('data-block-id')) {
|
|
selector = '[data-block-id="' + el.getAttribute('data-block-id') + '"]';
|
|
} else if (el.getAttribute('name')) {
|
|
selector = el.tagName.toLowerCase() + '[name="' + CSS.escape(el.getAttribute('name')) + '"]';
|
|
} else if (el.getAttribute('aria-label')) {
|
|
selector = el.tagName.toLowerCase() + '[aria-label="' + CSS.escape(el.getAttribute('aria-label')) + '"]';
|
|
} else if (el.getAttribute('type') && el.tagName === 'INPUT') {
|
|
selector = 'input[type="' + el.getAttribute('type') + '"]';
|
|
if (el.getAttribute('placeholder'))
|
|
selector += '[placeholder="' + CSS.escape(el.getAttribute('placeholder')) + '"]';
|
|
} else if (el.className && typeof el.className === 'string') {
|
|
const cls = el.className.trim().split(/\\s+/)[0];
|
|
if (cls && cls.length < 60)
|
|
selector = el.tagName.toLowerCase() + '.' + CSS.escape(cls);
|
|
}
|
|
|
|
if (seen.has(selector)) {
|
|
const parent = el.parentElement;
|
|
if (parent && parent.id) {
|
|
selector = '#' + CSS.escape(parent.id) + ' > ' + selector;
|
|
} else {
|
|
const siblings = parent ? Array.from(parent.children) : [];
|
|
const idx = siblings.indexOf(el);
|
|
if (idx >= 0) selector += ':nth-child(' + (idx + 1) + ')';
|
|
}
|
|
}
|
|
seen.add(selector);
|
|
|
|
results.push({
|
|
selector,
|
|
tag: el.tagName.toLowerCase(),
|
|
type: el.type || null,
|
|
text: (el.textContent || '').trim().substring(0, 120) || null,
|
|
placeholder: el.placeholder || null,
|
|
ariaLabel: el.getAttribute('aria-label') || null,
|
|
role: el.getAttribute('role') || null,
|
|
href: el.href && el.href !== location.href ? el.href : null,
|
|
});
|
|
}
|
|
return { elements: results, total: interactive.length, url: location.href, title: document.title };
|
|
})()`;
|
|
try {
|
|
const result = await evalInPage(wv, code);
|
|
return { text: JSON.stringify(result, null, 2), url: wv.getURL() };
|
|
} catch (err: any) {
|
|
return { error: `Failed to get elements: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
// Tier 1: detect a site's declared WebMCP tools (navigator.modelContext). When a site exposes its own tools the agent can prefer them over scraping the UI. The API is a Chrome 149 origin-trial standard; this Electron's Chromium predates it so real pages return "not present" today, this is forward-compatible probing, also covers the MCP-B convention (getRegisteredTools/listTools/tools array).
|
|
async function handleDetectWebMCP(wv: BrowserWebview): Promise<Record<string, any>> {
|
|
const code = `(() => {
|
|
const mc = navigator.modelContext;
|
|
if (!mc) return { present: false, tools: [] };
|
|
let raw = [];
|
|
try {
|
|
if (typeof mc.getRegisteredTools === 'function') raw = mc.getRegisteredTools() || [];
|
|
else if (typeof mc.listTools === 'function') raw = mc.listTools() || [];
|
|
else if (Array.isArray(mc.tools)) raw = mc.tools;
|
|
} catch (e) {}
|
|
const tools = (raw || []).map(t => ({
|
|
name: String((t && t.name) || ''),
|
|
description: String((t && t.description) || '').slice(0, 200),
|
|
})).filter(t => t.name);
|
|
return { present: true, tools };
|
|
})()`;
|
|
try {
|
|
const r = await evalInPage(wv, code);
|
|
if (!r || !r.present) {
|
|
return { text: 'No WebMCP on this page (navigator.modelContext not present). Use the normal browser tools.', url: wv.getURL() };
|
|
}
|
|
if (!r.tools.length) {
|
|
return { text: 'WebMCP is present but exposes no callable tools. Use the normal browser tools.', url: wv.getURL() };
|
|
}
|
|
const lines = r.tools.map((t: any) => `- ${t.name}: ${t.description}`).join('\n');
|
|
return { text: `WebMCP tools declared by this page:\n${lines}`, tools: r.tools, url: wv.getURL() };
|
|
} catch (err: any) {
|
|
return { error: `WebMCP detection failed: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
// Tier 2: the API routes captured for the current site, so the agent can act directly instead of
|
|
// re-scraping/clicking the UI. Default lists the safe GET/HEAD routes (all replay_route will run).
|
|
// With { writes: true } it lists the MUTATING routes (POST/PUT/PATCH/DELETE) the site's own UI
|
|
// fired, for BrowserApiWrite's general 'route' path; the write itself is same-origin + captured +
|
|
// session-borrowed + flag-gated in the backend, this only SURFACES the endpoint shape.
|
|
async function handleListRoutes(wv: BrowserWebview, params?: Record<string, any>): Promise<Record<string, any>> {
|
|
const bridge = (window as any).openswarm?.cdpRoutesGet as
|
|
| ((id: number, origin?: string) => Promise<any[]>) | undefined;
|
|
if (!bridge) return { error: 'Route capture not available, restart the app.' };
|
|
let origin = '';
|
|
try { origin = new URL(wv.getURL()).origin; } catch {}
|
|
let routes: any[] = [];
|
|
try { routes = (await bridge(wv.getWebContentsId(), origin)) || []; } catch {}
|
|
|
|
if (params?.writes) {
|
|
const writes = routes.filter((r) => r && r.safe === false);
|
|
if (!writes.length) {
|
|
return { text: 'No write (POST/PUT/PATCH/DELETE) API routes captured for this site yet. Do the write once through the UI so it gets recorded, then the route path can replay it.', url: wv.getURL() };
|
|
}
|
|
const wlines = writes.slice(0, 40).map((r) => `${r.method} ${r.template} body-shape: ${JSON.stringify(r.bodyShape)} (seen ${r.hits}x)`);
|
|
return {
|
|
text: `Write endpoints this site's UI uses (for BrowserApiWrite action='route'). Pass the `
|
|
+ `method + url + a body matching the shape, with your content in the text field:\n${wlines.join('\n')}`,
|
|
routes: writes.slice(0, 40),
|
|
url: wv.getURL(),
|
|
};
|
|
}
|
|
|
|
const safe = routes.filter((r) => r && r.safe);
|
|
if (!safe.length) {
|
|
return { text: 'No replayable (GET) API routes captured for this site yet. Use the page first so they get recorded, then try again.', url: wv.getURL() };
|
|
}
|
|
const lines = safe.slice(0, 40).map((r) => `${r.method} ${r.example || r.template} (seen ${r.hits}x)`);
|
|
return {
|
|
text: `Replayable API routes for this site (safe GETs). To READ the same kind of `
|
|
+ `data for many inputs fast: swap the varying value in the URL with {{value}} `
|
|
+ `and use a replay_route step in BrowserRepeatFlow (or call BrowserReplayRoute `
|
|
+ `per item). Far cheaper than navigating + scraping each page:\n${lines.join('\n')}`,
|
|
routes: safe.slice(0, 40),
|
|
url: wv.getURL(),
|
|
};
|
|
}
|
|
|
|
// Tier 2: replay a captured endpoint directly. GET/HEAD only (idempotent) and same-origin only; the fetch runs IN the page so cookies/CSRF come for free. Mutating methods are intentionally refused, those must go through the UI.
|
|
async function handleReplayRoute(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const rawUrl = params.url as string;
|
|
const method = String(params.method || 'GET').toUpperCase();
|
|
if (!rawUrl) return { error: 'url parameter is required' };
|
|
if (method !== 'GET' && method !== 'HEAD') {
|
|
return { error: `BrowserReplayRoute only runs safe GET/HEAD requests. ${method} changes data, do that through the UI (click the button) instead.` };
|
|
}
|
|
let absUrl: string;
|
|
let pageOrigin: string;
|
|
try {
|
|
pageOrigin = new URL(wv.getURL()).origin;
|
|
absUrl = new URL(rawUrl, wv.getURL()).href;
|
|
} catch {
|
|
return { error: 'invalid url' };
|
|
}
|
|
if (new URL(absUrl).origin !== pageOrigin) {
|
|
return { error: "BrowserReplayRoute can only call the current site's own API (same origin)." };
|
|
}
|
|
const code = `(async () => {
|
|
try {
|
|
const r = await fetch(${JSON.stringify(absUrl)}, { method: ${JSON.stringify(method)}, credentials: 'include' });
|
|
const body = await r.text();
|
|
return { status: r.status, body: body.slice(0, 15000) };
|
|
} catch (e) { return { error: String((e && e.message) || e) }; }
|
|
})()`;
|
|
try {
|
|
const res = await evalInPage(wv, code);
|
|
if (res.error) return { error: `Replay failed: ${res.error}` };
|
|
return { text: `${method} ${absUrl} -> HTTP ${res.status}\n${res.body}`, status: res.status, url: wv.getURL() };
|
|
} catch (err: any) {
|
|
return { error: `Replay failed: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
async function handleEvaluate(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
|
const expression = params.expression as string;
|
|
if (!expression) return { error: 'expression parameter is required' };
|
|
try {
|
|
const result = await evalInPage(wv, expression);
|
|
const text = typeof result === 'string' ? result : JSON.stringify(result, null, 2);
|
|
// evaluate is the agent's main read path; sample routes here too (XHRs have fired by now) so the backend can surface the fast network tier once.
|
|
const routes_available = await countSafeRoutes(wv);
|
|
return { text: text ?? 'undefined', url: wv.getURL(), routes_available };
|
|
} catch (err: any) {
|
|
return { error: `JS evaluation error: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
// The registry is renderer-local and a card briefly unregisters on remount / tab-switch; a command landing in that gap shouldn't hard-fail. Wait a bounded window for (re)registration before giving up, so the error stays a real "card is gone" signal rather than a transient race.
|
|
async function awaitWebview(browserId: string, tabId?: string, action?: string): Promise<BrowserWebview | undefined> {
|
|
// A suspended (snapshot-swapped) card has no webview at all; wake it and wait out the remount + page reload before the command touches it.
|
|
const wasSuspended = !!store.getState().dashboardLayout.suspendedBrowserCards[browserId];
|
|
if (wasSuspended) store.dispatch(resumeBrowserCard(browserId));
|
|
const deadline = Date.now() + (wasSuspended ? 12000 : 2000);
|
|
let wv = getWebview(browserId, tabId);
|
|
while (!wv && Date.now() < deadline) {
|
|
await new Promise((r) => setTimeout(r, 100));
|
|
wv = getWebview(browserId, tabId);
|
|
}
|
|
if (wasSuspended && wv) {
|
|
while (Date.now() < deadline) {
|
|
try {
|
|
if (!wv.isLoading() && wv.getURL() !== 'about:blank') break;
|
|
} catch {
|
|
// mid-mount hiccup; keep waiting
|
|
}
|
|
await new Promise((r) => setTimeout(r, 150));
|
|
}
|
|
}
|
|
// A lazy background tab mounts at about:blank with its real page deferred; an agent command needs
|
|
// the real page, so wake it and wait out the load, same as a resumed suspended card. A navigate
|
|
// is about to load its own url, so just drop the deferred load instead of loading the old one first.
|
|
if (wv && isPendingLoad(wv)) {
|
|
if (action === 'navigate') {
|
|
clearPendingLoad(wv);
|
|
} else if (wakePendingLoad(wv)) {
|
|
const loadDeadline = Date.now() + 12000;
|
|
while (Date.now() < loadDeadline) {
|
|
try {
|
|
if (!wv.isLoading() && wv.getURL() !== 'about:blank') break;
|
|
} catch {
|
|
// mid-load hiccup; keep waiting
|
|
}
|
|
await new Promise((r) => setTimeout(r, 150));
|
|
}
|
|
}
|
|
}
|
|
return wv;
|
|
}
|
|
|
|
// The backend re-broadcasts unanswered commands to heal a dead-socket gap, so a duplicate request_id must never run twice (a re-sent click would double-click).
|
|
const inflightCommands = new Set<string>();
|
|
const completedCommands = new Map<string, Record<string, any>>();
|
|
const _COMPLETED_CACHE_MAX = 50;
|
|
|
|
async function handleBrowserCommand(data: Record<string, any>) {
|
|
const { request_id, action, browser_id, tab_id, params = {} } = data;
|
|
if (!request_id) return;
|
|
if (inflightCommands.has(request_id)) return;
|
|
const cached = completedCommands.get(request_id);
|
|
if (cached) {
|
|
dashboardWs.send('browser:result', { request_id, ...cached });
|
|
return;
|
|
}
|
|
inflightCommands.add(request_id);
|
|
try {
|
|
await runBrowserCommand(request_id, action, browser_id, tab_id, params);
|
|
} finally {
|
|
inflightCommands.delete(request_id);
|
|
}
|
|
}
|
|
|
|
// Hand a vetted social platform's partition cookies to its session-backed MCP shim. No webview needed: it reads the main-process cookie store directly, so it runs before the webview lookup.
|
|
async function handleSessionCookies(params: Record<string, any>): Promise<Record<string, any>> {
|
|
const bridge = (window as any).openswarm?.getPartitionCookies as
|
|
| ((domain: string) => Promise<{ cookies: { name: string; value: string }[]; userAgent: string; error?: string }>)
|
|
| undefined;
|
|
if (!bridge) return { error: 'Cookie bridge unavailable (desktop app only)', cookies: [] };
|
|
try {
|
|
return await bridge(String(params.domain || ''));
|
|
} catch (err: any) {
|
|
return { error: `Cookie bridge failed: ${err?.message || String(err)}`, cookies: [] };
|
|
}
|
|
}
|
|
|
|
// Load the user's own existing sign-in for a site into the browser partition, so an agent stuck at
|
|
// a login wall can carry on as them instead of interrupting to ask for a password. Like the cookie
|
|
// bridge above this needs no webview: it writes straight to the main-process cookie store.
|
|
async function handleImportSession(params: Record<string, any>): Promise<Record<string, any>> {
|
|
const bridge = (window as any).openswarm?.setPartitionCookies as
|
|
| ((domain: string, cookies: Record<string, any>[]) => Promise<{ ok: boolean; set: number; error?: string }>)
|
|
| undefined;
|
|
if (!bridge) return { ok: false, set: 0, error: 'Session import unavailable (desktop app only)' };
|
|
const cookies = Array.isArray(params.cookies) ? params.cookies : [];
|
|
try {
|
|
return await bridge(String(params.domain || ''), cookies);
|
|
} catch (err: any) {
|
|
return { ok: false, set: 0, error: `Session import failed: ${err?.message || String(err)}` };
|
|
}
|
|
}
|
|
|
|
// Drive a session-borrow site's own already-open card: resolve the webview by its live domain
|
|
// (no browser_id, like the cookie bridge), then run a small navigate/evaluate step sequence.
|
|
// The shims use this for writes on sites that sign every HTTP request (TikTok).
|
|
async function handlePerformAction(params: Record<string, any>): Promise<Record<string, any>> {
|
|
const domain = String(params.domain || '').toLowerCase().replace(/^\./, '');
|
|
const wv = findWebviewByDomain(domain);
|
|
if (!wv) {
|
|
return { error: `No ${domain} browser card is open. Open ${domain} in an OpenSwarm browser card and sign in, then retry.` };
|
|
}
|
|
// findWebviewByDomain can resolve a deferred background tab by its intended url; wake it and wait out the load before driving it, so the session-borrow shims never act on an about:blank tab.
|
|
if (isPendingLoad(wv) && wakePendingLoad(wv)) {
|
|
const loadDeadline = Date.now() + 12000;
|
|
while (Date.now() < loadDeadline) {
|
|
try {
|
|
if (!wv.isLoading() && wv.getURL() !== 'about:blank') break;
|
|
} catch {
|
|
// mid-load hiccup; keep waiting
|
|
}
|
|
await new Promise((res) => setTimeout(res, 150));
|
|
}
|
|
}
|
|
const steps = Array.isArray(params.steps) ? params.steps : [];
|
|
const results: Record<string, any>[] = [];
|
|
for (const step of steps) {
|
|
const op = String(step?.op || '');
|
|
let r: Record<string, any>;
|
|
if (op === 'navigate') r = await handleNavigate(wv, { url: step.url });
|
|
else if (op === 'evaluate') r = await handleEvaluate(wv, { expression: step.expression });
|
|
else if (op === 'wait') { await new Promise((res) => setTimeout(res, Math.min(Number(step.ms) || 0, 8000))); r = { text: 'waited' }; }
|
|
else r = { error: `Unknown perform_action op: ${op}` };
|
|
results.push(r);
|
|
if (r && r.error) return { error: r.error, results };
|
|
}
|
|
return { ok: true, results };
|
|
}
|
|
|
|
async function runBrowserCommand(
|
|
request_id: string, action: string, browser_id: string, tab_id: string | undefined,
|
|
params: Record<string, any>,
|
|
) {
|
|
if (action === 'get_session_cookies') {
|
|
const result = await handleSessionCookies(params);
|
|
dashboardWs.send('browser:result', { request_id, ...result });
|
|
return;
|
|
}
|
|
if (action === 'perform_action') {
|
|
const result = await handlePerformAction(params);
|
|
dashboardWs.send('browser:result', { request_id, ...result });
|
|
return;
|
|
}
|
|
if (action === 'import_session') {
|
|
const result = await handleImportSession(params);
|
|
dashboardWs.send('browser:result', { request_id, ...result });
|
|
return;
|
|
}
|
|
const wv = await awaitWebview(browser_id, tab_id || undefined, action);
|
|
if (!wv) {
|
|
dashboardWs.send('browser:result', {
|
|
request_id,
|
|
error: `Browser card '${browser_id}'${tab_id ? ` tab '${tab_id}'` : ''} not found or not an Electron webview`,
|
|
});
|
|
return;
|
|
}
|
|
|
|
const detail = params.url || params.selector || params.expression || undefined;
|
|
setActivity(browser_id, { action: action as BrowserAction, detail });
|
|
|
|
let result: Record<string, any>;
|
|
try {
|
|
switch (action) {
|
|
case 'screenshot':
|
|
result = await handleScreenshot(wv, params);
|
|
break;
|
|
case 'get_text':
|
|
result = await handleGetText(wv, params);
|
|
break;
|
|
case 'get_console':
|
|
result = await handleGetConsole(wv);
|
|
break;
|
|
case 'navigate':
|
|
result = await handleNavigate(wv, params);
|
|
break;
|
|
case 'click':
|
|
result = await handleClick(wv, params);
|
|
if (result.clickX != null && result.clickY != null) {
|
|
setActivity(browser_id, {
|
|
action: 'click',
|
|
detail,
|
|
coords: { xPercent: result.clickX, yPercent: result.clickY },
|
|
});
|
|
}
|
|
break;
|
|
case 'type':
|
|
result = await handleType(wv, params);
|
|
break;
|
|
case 'find_composer':
|
|
result = await handleFindComposer(wv, params);
|
|
break;
|
|
case 'evaluate':
|
|
result = await handleEvaluate(wv, params);
|
|
break;
|
|
case 'get_elements':
|
|
result = await handleGetElements(wv, params);
|
|
break;
|
|
case 'scroll':
|
|
result = await handleScroll(wv, params);
|
|
break;
|
|
case 'wait':
|
|
result = await handleWait(wv, params);
|
|
break;
|
|
case 'press_key':
|
|
result = await handlePressKey(wv, params);
|
|
break;
|
|
case 'list_interactives':
|
|
result = await handleListInteractives(wv, params);
|
|
break;
|
|
case 'click_index':
|
|
result = await handleClickIndex(wv, params);
|
|
if (result.clickX != null && result.clickY != null) {
|
|
setActivity(browser_id, {
|
|
action: 'click_index',
|
|
detail,
|
|
coords: { xPercent: result.clickX, yPercent: result.clickY },
|
|
});
|
|
}
|
|
break;
|
|
case 'click_point':
|
|
result = await handleClickPoint(wv, params);
|
|
if (result.clickX != null && result.clickY != null) {
|
|
setActivity(browser_id, {
|
|
action: 'click_point',
|
|
detail,
|
|
coords: { xPercent: result.clickX, yPercent: result.clickY },
|
|
});
|
|
}
|
|
break;
|
|
case 'batch':
|
|
result = await handleBatch(wv, params);
|
|
break;
|
|
case 'detect_webmcp':
|
|
result = await handleDetectWebMCP(wv);
|
|
break;
|
|
case 'list_routes':
|
|
result = await handleListRoutes(wv, params);
|
|
break;
|
|
case 'click_by_name':
|
|
result = await handleClickByName(wv, params);
|
|
if (result.clickX != null && result.clickY != null) {
|
|
setActivity(browser_id, { action: 'click_by_name', detail, coords: { xPercent: result.clickX, yPercent: result.clickY } });
|
|
}
|
|
break;
|
|
case 'replay_route':
|
|
result = await handleReplayRoute(wv, params);
|
|
break;
|
|
default:
|
|
result = { error: `Unknown browser action: ${action}` };
|
|
}
|
|
} catch (err: any) {
|
|
result = { error: `Browser command failed: ${err?.message || String(err)}` };
|
|
}
|
|
|
|
setActivity(browser_id, null);
|
|
completedCommands.set(request_id, result);
|
|
if (completedCommands.size > _COMPLETED_CACHE_MAX) {
|
|
completedCommands.delete(completedCommands.keys().next().value as string);
|
|
}
|
|
dashboardWs.send('browser:result', { request_id, ...result });
|
|
}
|
|
|
|
export function initBrowserCommandHandler(): () => void {
|
|
if (initialized) return () => {};
|
|
initialized = true;
|
|
const unsub = dashboardWs.on('browser:command', handleBrowserCommand);
|
|
return () => {
|
|
unsub();
|
|
initialized = false;
|
|
};
|
|
}
|