import { getWebview, type BrowserWebview } from './browserRegistry'; import { dashboardWs } from './ws/WebSocketManager'; import { resolveInput } from './resolveUrl'; let initialized = false; export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait'; export interface BrowserActivity { action: BrowserAction; detail?: string; coords?: { xPercent: number; yPercent: number }; } type ActivityListener = (browserId: string, activity: BrowserActivity | null) => void; const activityMap = new Map(); const listeners = new Set(); function setActivity(browserId: string, activity: BrowserActivity | null) { if (activity) { activityMap.set(browserId, activity); } else { activityMap.delete(browserId); } listeners.forEach((fn) => fn(browserId, activity)); } export function getActivity(browserId: string): BrowserActivity | null { return activityMap.get(browserId) ?? null; } export function subscribeActivity(fn: ActivityListener): () => void { listeners.add(fn); return () => { listeners.delete(fn); }; } const ACTION_LABELS: Record = { screenshot: 'Capturing...', get_text: 'Reading...', navigate: 'Navigating...', click: 'Clicking...', type: 'Typing...', evaluate: 'Evaluating...', get_elements: 'Inspecting...', scroll: 'Scrolling...', wait: 'Waiting...', }; export function getActionLabel(action: string): string { return ACTION_LABELS[action] ?? 'Working...'; } async function handleScreenshot(wv: BrowserWebview): Promise> { const nativeImage = await wv.capturePage(); const dataUrl = nativeImage.toDataURL(); const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, ''); return { image: base64, url: wv.getURL(), title: wv.getTitle() }; } async function handleGetText(wv: BrowserWebview): Promise> { const text: string = await wv.executeJavaScript( 'document.body.innerText.substring(0, 15000)' ); return { text, url: wv.getURL(), title: wv.getTitle() }; } async function handleNavigate(wv: BrowserWebview, params: Record): Promise> { const raw = params.url as string; if (!raw) return { error: 'url parameter is required' }; const url = resolveInput(raw); try { await wv.loadURL(url); } catch (err: any) { if (!err?.message?.includes('ERR_ABORTED')) throw err; } return { text: `Navigated to ${url}`, url }; } async function handleClick(wv: BrowserWebview, params: Record): Promise> { const selector = params.selector as string; if (!selector) return { error: 'selector parameter is required' }; const safeSelector = JSON.stringify(selector); const code = `(()=>{ const el = document.querySelector(${safeSelector}); if (!el) return { error: 'Element not found: ' + ${safeSelector} }; el.scrollIntoView({ block: 'center', behavior: 'instant' }); const rect = el.getBoundingClientRect(); const x = rect.left + rect.width / 2; const y = rect.top + rect.height / 2; const opts = { bubbles: true, cancelable: true, clientX: x, clientY: y, button: 0 }; el.dispatchEvent(new PointerEvent('pointerdown', { ...opts, pointerId: 1 })); el.dispatchEvent(new MouseEvent('mousedown', opts)); el.dispatchEvent(new PointerEvent('pointerup', { ...opts, pointerId: 1 })); el.dispatchEvent(new MouseEvent('mouseup', opts)); el.dispatchEvent(new MouseEvent('click', opts)); return { text: 'Clicked element: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''), url: location.href, clickX: window.innerWidth > 0 ? x / window.innerWidth : 0.5, clickY: window.innerHeight > 0 ? y / window.innerHeight : 0.5, }; })()`; const result = await wv.executeJavaScript(code); return result; } async function handleType(wv: BrowserWebview, params: Record): Promise> { const selector = params.selector as string; const text = params.text as string; if (!selector) return { error: 'selector parameter is required' }; if (text == null) return { error: 'text parameter is required' }; const safeSelector = JSON.stringify(selector); const safeText = JSON.stringify(text); const code = `(async ()=>{ const el = document.querySelector(${safeSelector}); if (!el) return { error: 'Element not found: ' + ${safeSelector} }; el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus(); if (el.select) el.select(); document.execCommand('selectAll', false); document.execCommand('delete', false); document.execCommand('insertText', false, ${safeText}); el.dispatchEvent(new InputEvent('input', { bubbles: true, cancelable: true, inputType: 'insertText', data: ${safeText}, })); el.dispatchEvent(new Event('change', { bubbles: true })); return { text: 'Typed into: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''), }; })()`; const result = await wv.executeJavaScript(code); return result; } async function handleScroll(wv: BrowserWebview, params: Record): Promise> { const direction = (params.direction as string) || 'down'; const amount = (params.amount as number) || 500; const code = `(() => { function findScrollable() { const candidates = document.querySelectorAll( '[class*="scroller"], [class*="scroll-container"], [class*="content"], ' + 'main, [role="main"], article, .notion-scroller, .notion-frame' ); for (const el of candidates) { const s = window.getComputedStyle(el); const isScrollable = (s.overflow === 'auto' || s.overflow === 'scroll' || s.overflowY === 'auto' || s.overflowY === 'scroll'); if (isScrollable && el.scrollHeight > el.clientHeight + 10) return el; } const all = document.querySelectorAll('*'); for (const el of all) { if (el === document.body || el === document.documentElement) continue; const s = window.getComputedStyle(el); const isScrollable = (s.overflow === 'auto' || s.overflow === 'scroll' || s.overflowY === 'auto' || s.overflowY === 'scroll'); if (isScrollable && el.scrollHeight > el.clientHeight + 50 && el.clientHeight > 200) return el; } return null; } const dy = ${JSON.stringify(direction)} === 'up' ? -${amount} : ${amount}; const container = findScrollable(); if (container) { const before = container.scrollTop; container.scrollBy({ top: dy, behavior: 'instant' }); const after = container.scrollTop; return { scrolled: Math.abs(after - before), scrollTop: after, scrollHeight: container.scrollHeight, clientHeight: container.clientHeight, atTop: after <= 0, atBottom: after + container.clientHeight >= container.scrollHeight - 5, target: 'container', }; } const before = window.scrollY; window.scrollBy({ top: dy, behavior: 'instant' }); const after = window.scrollY; return { scrolled: Math.abs(after - before), scrollTop: after, scrollHeight: document.documentElement.scrollHeight, clientHeight: window.innerHeight, atTop: after <= 0, atBottom: after + window.innerHeight >= document.documentElement.scrollHeight - 5, target: 'window', }; })()`; try { const result = await wv.executeJavaScript(code); const status = result.atBottom ? ' (reached bottom)' : result.atTop ? ' (reached top)' : ''; return { text: `Scrolled ${direction} by ${result.scrolled}px${status}. Position: ${result.scrollTop}/${result.scrollHeight - result.clientHeight}px`, ...result, url: wv.getURL(), }; } catch (err: any) { return { error: `Scroll failed: ${err?.message || String(err)}` }; } } async function handleWait(wv: BrowserWebview, params: Record): Promise> { const ms = Math.min(Math.max((params.milliseconds as number) || 1000, 100), 10000); await new Promise((resolve) => setTimeout(resolve, ms)); return { text: `Waited ${ms}ms. Current URL: ${wv.getURL()}`, url: wv.getURL(), title: wv.getTitle(), }; } async function handleGetElements(wv: BrowserWebview, params: Record): Promise> { const scope = (params.selector as string) || 'body'; const safeScope = JSON.stringify(scope); const code = `(() => { const scope = document.querySelector(${safeScope}) || document.body; const interactive = scope.querySelectorAll( 'a[href], button, input, textarea, select, [role="button"], [role="link"], ' + '[role="textbox"], [role="searchbox"], [role="menuitem"], [role="tab"], ' + '[role="checkbox"], [role="switch"], [role="option"], ' + '[onclick], [tabindex]:not([tabindex="-1"]), ' + '[data-block-id], [contenteditable="true"]' ); const seen = new Set(); const results = []; for (const el of interactive) { if (results.length >= 80) break; const rect = el.getBoundingClientRect(); if (rect.width === 0 && rect.height === 0) continue; const style = window.getComputedStyle(el); if (style.visibility === 'hidden' || style.display === 'none') continue; if (style.opacity === '0') continue; let selector = el.tagName.toLowerCase(); if (el.id) { selector = '#' + CSS.escape(el.id); } else if (el.getAttribute('data-block-id')) { selector = '[data-block-id="' + el.getAttribute('data-block-id') + '"]'; } else if (el.getAttribute('name')) { selector = el.tagName.toLowerCase() + '[name="' + CSS.escape(el.getAttribute('name')) + '"]'; } else if (el.getAttribute('aria-label')) { selector = el.tagName.toLowerCase() + '[aria-label="' + CSS.escape(el.getAttribute('aria-label')) + '"]'; } else if (el.getAttribute('type') && el.tagName === 'INPUT') { selector = 'input[type="' + el.getAttribute('type') + '"]'; if (el.getAttribute('placeholder')) selector += '[placeholder="' + CSS.escape(el.getAttribute('placeholder')) + '"]'; } else if (el.className && typeof el.className === 'string') { const cls = el.className.trim().split(/\\s+/)[0]; if (cls && cls.length < 60) selector = el.tagName.toLowerCase() + '.' + CSS.escape(cls); } if (seen.has(selector)) { const parent = el.parentElement; if (parent && parent.id) { selector = '#' + CSS.escape(parent.id) + ' > ' + selector; } else { const siblings = parent ? Array.from(parent.children) : []; const idx = siblings.indexOf(el); if (idx >= 0) selector += ':nth-child(' + (idx + 1) + ')'; } } seen.add(selector); results.push({ selector, tag: el.tagName.toLowerCase(), type: el.type || null, text: (el.textContent || '').trim().substring(0, 120) || null, placeholder: el.placeholder || null, ariaLabel: el.getAttribute('aria-label') || null, role: el.getAttribute('role') || null, href: el.href && el.href !== location.href ? el.href : null, }); } return { elements: results, total: interactive.length, url: location.href, title: document.title }; })()`; try { const result = await wv.executeJavaScript(code); return { text: JSON.stringify(result, null, 2), url: wv.getURL() }; } catch (err: any) { return { error: `Failed to get elements: ${err?.message || String(err)}` }; } } async function handleEvaluate(wv: BrowserWebview, params: Record): Promise> { const expression = params.expression as string; if (!expression) return { error: 'expression parameter is required' }; try { const result = await wv.executeJavaScript(expression); const text = typeof result === 'string' ? result : JSON.stringify(result, null, 2); return { text: text ?? 'undefined', url: wv.getURL() }; } catch (err: any) { return { error: `JS evaluation error: ${err?.message || String(err)}` }; } } async function handleBrowserCommand(data: Record) { const { request_id, action, browser_id, tab_id, params = {} } = data; if (!request_id) return; const wv = getWebview(browser_id, tab_id || undefined); if (!wv) { dashboardWs.send('browser:result', { request_id, error: `Browser card '${browser_id}'${tab_id ? ` tab '${tab_id}'` : ''} not found or not an Electron webview`, }); return; } const detail = params.url || params.selector || params.expression || undefined; setActivity(browser_id, { action: action as BrowserAction, detail }); let result: Record; try { switch (action) { case 'screenshot': result = await handleScreenshot(wv); break; case 'get_text': result = await handleGetText(wv); break; case 'navigate': result = await handleNavigate(wv, params); break; case 'click': result = await handleClick(wv, params); if (result.clickX != null && result.clickY != null) { setActivity(browser_id, { action: 'click', detail, coords: { xPercent: result.clickX, yPercent: result.clickY }, }); } break; case 'type': result = await handleType(wv, params); break; case 'evaluate': result = await handleEvaluate(wv, params); break; case 'get_elements': result = await handleGetElements(wv, params); break; case 'scroll': result = await handleScroll(wv, params); break; case 'wait': result = await handleWait(wv, params); break; default: result = { error: `Unknown browser action: ${action}` }; } } catch (err: any) { result = { error: `Browser command failed: ${err?.message || String(err)}` }; } setActivity(browser_id, null); dashboardWs.send('browser:result', { request_id, ...result }); } export function initBrowserCommandHandler(): () => void { if (initialized) return () => {}; initialized = true; const unsub = dashboardWs.on('browser:command', handleBrowserCommand); return () => { unsub(); initialized = false; }; }