import { getWebview, type BrowserWebview } from './browserRegistry'; import { dashboardWs } from './ws/WebSocketManager'; import { resolveInput } from './resolveUrl'; let initialized = false; export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'batch'; export interface BrowserActivity { action: BrowserAction; detail?: string; coords?: { xPercent: number; yPercent: number }; } type ActivityListener = (browserId: string, activity: BrowserActivity | null) => void; const activityMap = new Map(); const listeners = new Set(); function setActivity(browserId: string, activity: BrowserActivity | null) { if (activity) { activityMap.set(browserId, activity); } else { activityMap.delete(browserId); } listeners.forEach((fn) => fn(browserId, activity)); } export function getActivity(browserId: string): BrowserActivity | null { return activityMap.get(browserId) ?? null; } export function subscribeActivity(fn: ActivityListener): () => void { listeners.add(fn); return () => { listeners.delete(fn); }; } const ACTION_LABELS: Record = { screenshot: 'Capturing...', get_text: 'Reading...', navigate: 'Navigating...', click: 'Clicking...', type: 'Typing...', evaluate: 'Evaluating...', get_elements: 'Inspecting...', scroll: 'Scrolling...', wait: 'Waiting...', press_key: 'Pressing key...', list_interactives: 'Reading page structure...', click_index: 'Clicking element...', batch: 'Running batch...', }; export function getActionLabel(action: string): string { return ACTION_LABELS[action] ?? 'Working...'; } async function handleScreenshot(wv: BrowserWebview): Promise> { const nativeImage = await wv.capturePage(); const dataUrl = nativeImage.toDataURL(); const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, ''); return { image: base64, url: wv.getURL(), title: wv.getTitle() }; } async function handleGetText(wv: BrowserWebview): Promise> { const text: string = await wv.executeJavaScript( 'document.body.innerText.substring(0, 15000)' ); return { text, url: wv.getURL(), title: wv.getTitle() }; } async function handleNavigate(wv: BrowserWebview, params: Record): Promise> { const raw = params.url as string; if (!raw) return { error: 'url parameter is required' }; const url = resolveInput(raw); try { await wv.loadURL(url); } catch (err: any) { if (!err?.message?.includes('ERR_ABORTED')) throw err; } return { text: `Navigated to ${url}`, url }; } async function handleClick(wv: BrowserWebview, params: Record): Promise> { const selector = params.selector as string; if (!selector) return { error: 'selector parameter is required' }; const safeSelector = JSON.stringify(selector); const code = `(()=>{ const el = document.querySelector(${safeSelector}); if (!el) return { error: 'Element not found: ' + ${safeSelector} }; el.scrollIntoView({ block: 'center', behavior: 'instant' }); const rect = el.getBoundingClientRect(); const x = rect.left + rect.width / 2; const y = rect.top + rect.height / 2; const opts = { bubbles: true, cancelable: true, clientX: x, clientY: y, button: 0 }; el.dispatchEvent(new PointerEvent('pointerdown', { ...opts, pointerId: 1 })); el.dispatchEvent(new MouseEvent('mousedown', opts)); el.dispatchEvent(new PointerEvent('pointerup', { ...opts, pointerId: 1 })); el.dispatchEvent(new MouseEvent('mouseup', opts)); el.dispatchEvent(new MouseEvent('click', opts)); return { text: 'Clicked element: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''), url: location.href, clickX: window.innerWidth > 0 ? x / window.innerWidth : 0.5, clickY: window.innerHeight > 0 ? y / window.innerHeight : 0.5, }; })()`; const result = await wv.executeJavaScript(code); return result; } async function handleType(wv: BrowserWebview, params: Record): Promise> { const selector = params.selector as string; const text = params.text as string; if (!selector) return { error: 'selector parameter is required' }; if (text == null) return { error: 'text parameter is required' }; const safeSelector = JSON.stringify(selector); const safeText = JSON.stringify(text); const code = `(async ()=>{ const el = document.querySelector(${safeSelector}); if (!el) return { error: 'Element not found: ' + ${safeSelector} }; el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus(); if (el.select) el.select(); document.execCommand('selectAll', false); document.execCommand('delete', false); document.execCommand('insertText', false, ${safeText}); el.dispatchEvent(new InputEvent('input', { bubbles: true, cancelable: true, inputType: 'insertText', data: ${safeText}, })); el.dispatchEvent(new Event('change', { bubbles: true })); return { text: 'Typed into: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''), }; })()`; const result = await wv.executeJavaScript(code); return result; } // Map common JS KeyboardEvent.key values to Electron's accelerator keyCodes. // Electron's sendInputEvent expects: 'Up', 'Down', 'Left', 'Right', 'Enter', // 'Escape', 'Tab', 'Backspace', 'Delete', 'Space', or single char letters. const KEY_NAME_MAP: Record = { ArrowUp: 'Up', ArrowDown: 'Down', ArrowLeft: 'Left', ArrowRight: 'Right', ' ': 'Space', Spacebar: 'Space', Esc: 'Escape', Del: 'Delete', }; async function handlePressKey(wv: BrowserWebview, params: Record): Promise> { const rawKey = (params.key as string) || ''; if (!rawKey) return { error: 'key parameter is required' }; const keyCode = KEY_NAME_MAP[rawKey] || rawKey; // Focus the page first so the key event has a sensible target. await wv.executeJavaScript('document.body && document.body.focus && document.body.focus(); true'); // Native OS-level key events — these have event.isTrusted === true so site // keyboard handlers (Tinder, Slack, Notion, etc.) actually respect them. wv.sendInputEvent({ type: 'keyDown', keyCode }); wv.sendInputEvent({ type: 'char', keyCode }); wv.sendInputEvent({ type: 'keyUp', keyCode }); return { text: `Pressed ${rawKey}` }; } // --------------------------------------------------------------------------- // CDP accessibility-tree element indexing // --------------------------------------------------------------------------- // list_interactives uses Chrome DevTools Protocol's Accessibility.getFullAXTree // to get the *computed* accessibility tree, not the raw DOM. This sees roles, // names, and labels even on hostile sites (Tinder, Instagram) where the raw // HTML is just unlabeled
s with click handlers — because Chromium computes // accessible names for screen readers from icons, surrounding text, etc. // // Each interactive element is assigned a numeric index. The index → backendNodeId // map is cached server-side per webContents and used by click_index. This is // orders of magnitude more reliable than CSS-selector-based clicking on sites // that don't expose semantic markup. const INTERACTIVE_ROLES = new Set([ 'button', 'link', 'textbox', 'combobox', 'checkbox', 'menuitem', 'tab', 'switch', 'searchbox', 'slider', 'listbox', 'option', 'radio', 'menuitemcheckbox', 'menuitemradio', 'spinbutton', 'treeitem', ]); interface InteractiveElement { index: number; role: string; name: string; backendNodeId: number; } function extractAxValue(prop: any): string { if (!prop) return ''; if (typeof prop === 'string') return prop; if (prop.value !== undefined) { if (typeof prop.value === 'string') return prop.value; if (typeof prop.value === 'object' && prop.value && 'value' in prop.value) { return String(prop.value.value || ''); } } return ''; } interface CdpResult { ok: boolean; result?: any; error?: string } async function sendCdp(wv: BrowserWebview, method: string, params?: Record): Promise { const wcId = wv.getWebContentsId(); const bridge = (window as any).openswarm?.sendCdpCommand as | ((id: number, m: string, p?: any) => Promise) | undefined; if (!bridge) throw new Error('CDP bridge not available — restart the app'); const resp = await bridge(wcId, method, params); if (!resp || !resp.ok) { throw new Error(resp?.error || `CDP ${method} failed`); } return resp.result; } async function handleListInteractives(wv: BrowserWebview): Promise> { let axResult; try { axResult = await sendCdp(wv, 'Accessibility.getFullAXTree', {}); } catch (err: any) { return { error: `getFullAXTree failed: ${err.message || String(err)}` }; } const nodes: any[] = axResult?.nodes || []; const interactives: InteractiveElement[] = []; let index = 1; for (const node of nodes) { if (node.ignored) continue; const role = extractAxValue(node.role); if (!INTERACTIVE_ROLES.has(role)) continue; const name = extractAxValue(node.name); if (!name && role !== 'textbox' && role !== 'searchbox' && role !== 'combobox') { // Skip nameless elements unless they're inputs (which can be empty) continue; } const backendNodeId = node.backendDOMNodeId; if (backendNodeId == null) continue; interactives.push({ index, role, name: name.slice(0, 80), backendNodeId }); index++; } // Cache the index map in main-process storage so click_index can resolve it // even across separate WebSocket commands. const indexMap: Record = {}; for (const el of interactives) { indexMap[el.index] = el.backendNodeId; } try { const cacheBridge = (window as any).openswarm?.cdpCacheSet; if (cacheBridge) await cacheBridge(wv.getWebContentsId(), indexMap); } catch { // Cache is best-effort; click_index will fall back to re-listing. } // Build the model-friendly text representation: [1]