diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 631daffd..c3d7f01a 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -1,6 +1,7 @@ import { getWebview, type BrowserWebview } from './browserRegistry'; import { dashboardWs } from './ws/WebSocketManager'; import { resolveInput } from './resolveUrl'; +import { rankAndCapInteractives, type RankItem } from './interactiveRanking'; let initialized = false; @@ -173,6 +174,7 @@ interface InteractiveElement { role: string; name: string; backendNodeId: number; + sessionId?: string; } function extractAxValue(prop: any): string { @@ -189,49 +191,113 @@ function extractAxValue(prop: any): string { interface CdpResult { ok: boolean; result?: any; error?: string } -async function sendCdp(wv: BrowserWebview, method: string, params?: Record): Promise { +// sessionId undefined => root frame; a child-frame sessionId => that OOPIF. +async function sendCdp(wv: BrowserWebview, method: string, params?: Record, sessionId?: string): Promise { const wcId = wv.getWebContentsId(); const bridge = (window as any).openswarm?.sendCdpCommand as - | ((id: number, m: string, p?: any) => Promise) + | ((id: number, m: string, p?: any, s?: string) => Promise) | undefined; if (!bridge) throw new Error('CDP bridge not available, restart the app'); - const resp = await bridge(wcId, method, params); + const resp = await bridge(wcId, method, params, sessionId); if (!resp || !resp.ok) { throw new Error(resp?.error || `CDP ${method} failed`); } return resp.result; } -async function handleListInteractives(wv: BrowserWebview): Promise> { - let axResult; +interface ChildSession { sessionId: string; frameId: string; parentSessionId: string | null; url: string } + +async function getChildSessions(wv: BrowserWebview): Promise { + const bridge = (window as any).openswarm?.cdpChildSessionsGet as + | ((id: number) => Promise) | undefined; + if (!bridge) return []; try { - axResult = await sendCdp(wv, 'Accessibility.getFullAXTree', {}); - } catch (err: any) { - return { error: `getFullAXTree failed: ${err.message || String(err)}` }; + return (await bridge(wv.getWebContentsId())) || []; + } catch { + return []; } +} - const nodes: any[] = axResult?.nodes || []; - const interactives: InteractiveElement[] = []; - let index = 1; - +function axNodesToCandidates(nodes: any[], sessionId?: string): RankItem[] { + const out: RankItem[] = []; for (const node of nodes) { if (node.ignored) continue; const role = extractAxValue(node.role); if (!INTERACTIVE_ROLES.has(role)) continue; const name = extractAxValue(node.name); - if (!name && role !== 'textbox' && role !== 'searchbox' && role !== 'combobox') { - continue; - } + if (!name && role !== 'textbox' && role !== 'searchbox' && role !== 'combobox') continue; const backendNodeId = node.backendDOMNodeId; if (backendNodeId == null) continue; - interactives.push({ index, role, name: name.slice(0, 80), backendNodeId }); - index++; + out.push({ role, name: name.slice(0, 80), backendNodeId, sessionId }); + } + return out; +} + +// Cumulative top-left offset of a frame within the root viewport: climb the +// session chain adding each owning