[eric] browser: rank/cap/goal-boost the element list and click into cross-origin iframes

frontend/src/shared/browserCommandHandler.ts

@@ -1,6 +1,7 @@
 import { getWebview, type BrowserWebview } from './browserRegistry';
 import { dashboardWs } from './ws/WebSocketManager';
 import { resolveInput } from './resolveUrl';
+import { rankAndCapInteractives, type RankItem } from './interactiveRanking';
 
 let initialized = false;
 
@@ -173,6 +174,7 @@ interface InteractiveElement {
   role: string;
   name: string;
   backendNodeId: number;
+  sessionId?: string;
 }
 
 function extractAxValue(prop: any): string {
@@ -189,49 +191,113 @@ function extractAxValue(prop: any): string {
 
 interface CdpResult { ok: boolean; result?: any; error?: string }
 
-async function sendCdp(wv: BrowserWebview, method: string, params?: Record<string, any>): Promise<any> {
+// sessionId undefined => root frame; a child-frame sessionId => that OOPIF.
+async function sendCdp(wv: BrowserWebview, method: string, params?: Record<string, any>, sessionId?: string): Promise<any> {
   const wcId = wv.getWebContentsId();
   const bridge = (window as any).openswarm?.sendCdpCommand as
-    | ((id: number, m: string, p?: any) => Promise<CdpResult>)
+    | ((id: number, m: string, p?: any, s?: string) => Promise<CdpResult>)
     | undefined;
   if (!bridge) throw new Error('CDP bridge not available, restart the app');
-  const resp = await bridge(wcId, method, params);
+  const resp = await bridge(wcId, method, params, sessionId);
   if (!resp || !resp.ok) {
     throw new Error(resp?.error || `CDP ${method} failed`);
   }
   return resp.result;
 }
 
-async function handleListInteractives(wv: BrowserWebview): Promise<Record<string, any>> {
-  let axResult;
+interface ChildSession { sessionId: string; frameId: string; parentSessionId: string | null; url: string }
+
+async function getChildSessions(wv: BrowserWebview): Promise<ChildSession[]> {
+  const bridge = (window as any).openswarm?.cdpChildSessionsGet as
+    | ((id: number) => Promise<ChildSession[]>) | undefined;
+  if (!bridge) return [];
   try {
-    axResult = await sendCdp(wv, 'Accessibility.getFullAXTree', {});
-  } catch (err: any) {
-    return { error: `getFullAXTree failed: ${err.message || String(err)}` };
+    return (await bridge(wv.getWebContentsId())) || [];
+  } catch {
+    return [];
   }
+}
 
-  const nodes: any[] = axResult?.nodes || [];
-  const interactives: InteractiveElement[] = [];
-  let index = 1;
-
+function axNodesToCandidates(nodes: any[], sessionId?: string): RankItem[] {
+  const out: RankItem[] = [];
   for (const node of nodes) {
     if (node.ignored) continue;
     const role = extractAxValue(node.role);
     if (!INTERACTIVE_ROLES.has(role)) continue;
     const name = extractAxValue(node.name);
-    if (!name && role !== 'textbox' && role !== 'searchbox' && role !== 'combobox') {
-      continue;
-    }
+    if (!name && role !== 'textbox' && role !== 'searchbox' && role !== 'combobox') continue;
     const backendNodeId = node.backendDOMNodeId;
     if (backendNodeId == null) continue;
-    interactives.push({ index, role, name: name.slice(0, 80), backendNodeId });
-    index++;
+    out.push({ role, name: name.slice(0, 80), backendNodeId, sessionId });
+  }
+  return out;
+}
+
+// Cumulative top-left offset of a frame within the root viewport: climb the
+// session chain adding each owning <iframe>'s top-left. Used ONLY to place the
+// cosmetic click ripple; the click itself dispatches in the element's own
+// frame, so this is best-effort. Verified getFrameOwner works through Electron.
+async function frameOffset(
+  wv: BrowserWebview, sessionId: string | undefined, children: ChildSession[],
+): Promise<{ dx: number; dy: number }> {
+  let dx = 0, dy = 0;
+  const byId = new Map(children.map((c) => [c.sessionId, c]));
+  const seen = new Set<string>();
+  let s: string | null | undefined = sessionId;
+  while (s && !seen.has(s)) {
+    seen.add(s);
+    const info = byId.get(s);
+    if (!info) break;
+    const parent = info.parentSessionId || undefined; // undefined => root
+    const owner = await sendCdp(wv, 'DOM.getFrameOwner', { frameId: info.frameId }, parent);
+    const ownerBox = await sendCdp(wv, 'DOM.getBoxModel', { backendNodeId: owner.backendNodeId }, parent);
+    const oc = ownerBox?.model?.content;
+    if (!Array.isArray(oc) || oc.length < 8) break;
+    dx += oc[0];
+    dy += oc[1];
+    s = info.parentSessionId;
+  }
+  return { dx, dy };
+}
+
+async function handleListInteractives(wv: BrowserWebview, params: Record<string, any> = {}): Promise<Record<string, any>> {
+  const candidates: RankItem[] = [];
+  try {
+    const rootTree = await sendCdp(wv, 'Accessibility.getFullAXTree', {});
+    candidates.push(...axNodesToCandidates(rootTree?.nodes || []));
+  } catch (err: any) {
+    return { error: `getFullAXTree failed: ${err.message || String(err)}` };
   }
 
+  // Merge cross-origin OOPIF child frames (e.g. the Google Docs share dialog),
+  // whose nodes never appear in the root tree. A frame that won't answer
+  // (closed / navigating) is skipped, not fatal.
+  const children = await getChildSessions(wv);
+  for (const child of children) {
+    try {
+      const childTree = await sendCdp(wv, 'Accessibility.getFullAXTree', {}, child.sessionId);
+      candidates.push(...axNodesToCandidates(childTree?.nodes || [], child.sessionId));
+    } catch {
+      // skip unresponsive frame
+    }
+  }
+
+  // Dedupe twins, rank what a human acts on first (and the current goal
+  // highest), cap the long tail.
+  const goal = typeof params?.goal === 'string' ? params.goal : '';
+  const { shown, truncated } = rankAndCapInteractives(candidates, { goal });
+  const interactives: InteractiveElement[] = shown.map((el, i) => ({
+    index: i + 1,
+    role: el.role,
+    name: el.name,
+    backendNodeId: el.backendNodeId,
+    sessionId: el.sessionId,
+  }));
+
   // Cache in main-process so click_index can resolve across separate WS commands.
-  const indexMap: Record<number, number> = {};
+  const indexMap: Record<number, { backendNodeId: number; sessionId?: string }> = {};
   for (const el of interactives) {
-    indexMap[el.index] = el.backendNodeId;
+    indexMap[el.index] = { backendNodeId: el.backendNodeId, sessionId: el.sessionId };
   }
   try {
     const cacheBridge = (window as any).openswarm?.cdpCacheSet;
@@ -243,9 +309,15 @@ async function handleListInteractives(wv: BrowserWebview): Promise<Record<string
   const lines = interactives.map(
     (el) => `[${el.index}]<${el.role} "${el.name}">`,
   );
-  const text = lines.length
-    ? `${lines.length} interactive elements:\n${lines.join('\n')}`
-    : 'No interactive elements found on this page.';
+  let text: string;
+  if (lines.length === 0) {
+    text = 'No interactive elements found on this page.';
+  } else {
+    text = `${lines.length} interactive elements:\n${lines.join('\n')}`;
+    if (truncated > 0) {
+      text += `\n... ${truncated} more not shown; scroll or scope with BrowserGetElements to reach them.`;
+    }
+  }
 
   return {
     text,
@@ -261,12 +333,17 @@ async function handleClickIndex(wv: BrowserWebview, params: Record<string, any>)
   }
 
   let backendNodeId: number | undefined;
+  let sessionId: string | undefined;
   try {
     const cacheBridge = (window as any).openswarm?.cdpCacheGet;
     if (cacheBridge) {
       const cached = await cacheBridge(wv.getWebContentsId());
-      if (cached && cached[idx] != null) {
-        backendNodeId = Number(cached[idx]);
+      const entry = cached && cached[idx];
+      if (typeof entry === 'number') {
+        backendNodeId = entry; // legacy cache shape
+      } else if (entry && typeof entry === 'object' && entry.backendNodeId != null) {
+        backendNodeId = Number(entry.backendNodeId);
+        sessionId = entry.sessionId || undefined;
       }
     }
   } catch {
@@ -279,54 +356,63 @@ async function handleClickIndex(wv: BrowserWebview, params: Record<string, any>)
     };
   }
 
-  // Revalidate: fails fast if the page mutated and the node is gone (vs. clicking the wrong element).
+  // Revalidate in the element's own frame: fails fast if the page mutated and
+  // the node is gone (vs. clicking the wrong element).
   try {
-    await sendCdp(wv, 'DOM.resolveNode', { backendNodeId });
+    await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
   } catch (err: any) {
     return {
       error: `Index ${idx} is no longer valid (${err.message || 'node not found'}). The page may have changed. Call BrowserListInteractives again.`,
     };
   }
 
-  // Input.dispatchMouseEvent (OS-level) bypasses synthetic-event filtering on hostile sites.
+  // Box model in the element's OWN frame -> frame-local center.
   let boxModel;
   try {
-    boxModel = await sendCdp(wv, 'DOM.getBoxModel', { backendNodeId });
+    boxModel = await sendCdp(wv, 'DOM.getBoxModel', { backendNodeId }, sessionId);
   } catch (err: any) {
     return {
       error: `Index ${idx} has no box model (likely off-screen or hidden). Try scrolling first or call BrowserListInteractives again.`,
     };
   }
-
   const content = boxModel?.model?.content;
   if (!Array.isArray(content) || content.length < 8) {
     return { error: `Index ${idx} has no valid bounding rect.` };
   }
-  // content is [x1,y1, x2,y2, x3,y3, x4,y4]; compute center
-  const x = (content[0] + content[4]) / 2;
-  const y = (content[1] + content[5]) / 2;
+  const lx = (content[0] + content[4]) / 2;
+  const ly = (content[1] + content[5]) / 2;
 
+  // Dispatch the OS-level click in the element's OWN frame (sessionId routes
+  // into the OOPIF). This is compositor-independent, so it lands even when the
+  // card is offscreen, and still passes the isTrusted check on hostile sites.
   try {
     await sendCdp(wv, 'Input.dispatchMouseEvent', {
-      type: 'mousePressed',
-      x, y,
-      button: 'left',
-      clickCount: 1,
-    });
+      type: 'mousePressed', x: lx, y: ly, button: 'left', clickCount: 1,
+    }, sessionId);
     await sendCdp(wv, 'Input.dispatchMouseEvent', {
-      type: 'mouseReleased',
-      x, y,
-      button: 'left',
-      clickCount: 1,
-    });
+      type: 'mouseReleased', x: lx, y: ly, button: 'left', clickCount: 1,
+    }, sessionId);
   } catch (err: any) {
     return { error: `Click failed: ${err.message || String(err)}` };
   }
 
+  // Cosmetic ripple: place it in top-level coords. Best-effort for OOPIF.
+  let rx = lx, ry = ly;
+  if (sessionId) {
+    try {
+      const children = await getChildSessions(wv);
+      const { dx, dy } = await frameOffset(wv, sessionId, children);
+      rx = lx + dx;
+      ry = ly + dy;
+    } catch {
+      // fall back to frame-local coords for the ripple
+    }
+  }
+
   return {
-    text: `Clicked index ${idx} at (${Math.round(x)}, ${Math.round(y)})`,
-    clickX: x / wv.clientWidth * 100,
-    clickY: y / wv.clientHeight * 100,
+    text: `Clicked index ${idx} at (${Math.round(rx)}, ${Math.round(ry)})`,
+    clickX: rx / wv.clientWidth * 100,
+    clickY: ry / wv.clientHeight * 100,
   };
 }
 
@@ -645,7 +731,7 @@ async function handleBrowserCommand(data: Record<string, any>) {
         result = await handlePressKey(wv, params);
         break;
       case 'list_interactives':
-        result = await handleListInteractives(wv);
+        result = await handleListInteractives(wv, params);
         break;
       case 'click_index':
         result = await handleClickIndex(wv, params);

frontend/src/shared/interactiveRanking.test.ts

@@ -0,0 +1,120 @@
+// Run: node --test frontend/src/shared/interactiveRanking.test.ts
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { rankAndCapInteractives, goalKeywords, type RankItem } from './interactiveRanking.ts';
+
+const mk = (role: string, name: string, id = 0): RankItem => ({ role, name, backendNodeId: id });
+
+test('consecutive twins with same role+name collapse', () => {
+  const { shown } = rankAndCapInteractives([
+    mk('button', 'Like', 1),
+    mk('button', 'Like', 2),
+    mk('button', 'Like', 3),
+    mk('button', 'Share', 4),
+  ]);
+  assert.deepEqual(shown.map((x) => x.backendNodeId), [1, 4]);
+});
+
+test('same role+name in different frames (sessionId) is NOT collapsed at the seam', () => {
+  const root: RankItem = { role: 'button', name: 'Close', backendNodeId: 1 };
+  const child: RankItem = { role: 'button', name: 'Close', backendNodeId: 2, sessionId: 'frameA' };
+  const { shown } = rankAndCapInteractives([root, child]);
+  // both survive: they are genuinely different elements across a frame boundary
+  assert.equal(shown.length, 2);
+  assert.ok(shown.some((x) => x.sessionId === 'frameA'));
+});
+
+test('non-consecutive same name is preserved (real list items)', () => {
+  const { shown } = rankAndCapInteractives([
+    mk('button', 'Add to cart', 1),
+    mk('link', 'Widget A', 2),
+    mk('button', 'Add to cart', 3),
+    mk('link', 'Widget B', 4),
+    mk('button', 'Add to cart', 5),
+  ]);
+  // all three "Add to cart" survive because they are not back-to-back
+  const carts = shown.filter((x) => x.name === 'Add to cart');
+  assert.equal(carts.length, 3);
+});
+
+test('ranks by role priority: input > button/link > toggle > option', () => {
+  const { shown } = rankAndCapInteractives([
+    mk('option', 'opt', 1),
+    mk('checkbox', 'agree', 2),
+    mk('button', 'Go', 3),
+    mk('textbox', 'email', 4),
+  ]);
+  assert.deepEqual(shown.map((x) => x.role), ['textbox', 'button', 'checkbox', 'option']);
+});
+
+test('preserves document order within the same priority tier', () => {
+  const { shown } = rankAndCapInteractives([
+    mk('button', 'First', 1),
+    mk('link', 'Second', 2),
+    mk('button', 'Third', 3),
+  ]);
+  // button and link share tier 1; original order First, Second, Third holds
+  assert.deepEqual(shown.map((x) => x.backendNodeId), [1, 2, 3]);
+});
+
+test('caps to N and reports the truncated remainder', () => {
+  const items = Array.from({ length: 150 }, (_, i) => mk('link', `L${i}`, i));
+  const { shown, truncated } = rankAndCapInteractives(items, { cap: 60 });
+  assert.equal(shown.length, 60);
+  assert.equal(truncated, 90);
+});
+
+test('cap of 0 means no cap', () => {
+  const items = Array.from({ length: 5 }, (_, i) => mk('link', `L${i}`, i));
+  const { shown, truncated } = rankAndCapInteractives(items, { cap: 0 });
+  assert.equal(shown.length, 5);
+  assert.equal(truncated, 0);
+});
+
+test('goal-matched elements float to the top, above role priority', () => {
+  const { shown } = rankAndCapInteractives([
+    mk('textbox', 'Search', 1),
+    mk('link', 'Account settings', 2),
+    mk('button', 'Save', 3),
+  ], { goal: 'open the settings page' });
+  // "Account settings" matches "settings" and jumps ahead of the textbox
+  assert.equal(shown[0].backendNodeId, 2);
+});
+
+test('goal match survives the cap even when buried deep', () => {
+  const items = Array.from({ length: 100 }, (_, i) => mk('link', `Item ${i}`, i));
+  items.push(mk('button', 'Checkout now', 999));
+  const { shown } = rankAndCapInteractives(items, { cap: 30, goal: 'click checkout' });
+  assert.ok(shown.some((x) => x.backendNodeId === 999), 'checkout should be retained');
+  assert.equal(shown[0].backendNodeId, 999);
+});
+
+test('no goal leaves pure role-priority ordering', () => {
+  const { shown } = rankAndCapInteractives([
+    mk('option', 'opt', 1),
+    mk('button', 'Go', 2),
+    mk('textbox', 'email', 3),
+  ]);
+  assert.deepEqual(shown.map((x) => x.role), ['textbox', 'button', 'option']);
+});
+
+test('goalKeywords strips stopwords, action verbs, and short tokens', () => {
+  assert.deepEqual(goalKeywords('Click the Submit button to send'), ['submit', 'send']);
+  assert.deepEqual(goalKeywords('type into the search box'), ['search']);
+  assert.deepEqual(goalKeywords(''), []);
+});
+
+test('empty input yields empty result', () => {
+  const { shown, truncated } = rankAndCapInteractives([]);
+  assert.equal(shown.length, 0);
+  assert.equal(truncated, 0);
+});
+
+test('unknown role falls into the middle tier, not dropped', () => {
+  const { shown } = rankAndCapInteractives([
+    mk('option', 'opt', 1),
+    mk('weirdrole', 'mystery', 2),
+    mk('textbox', 'field', 3),
+  ]);
+  assert.deepEqual(shown.map((x) => x.role), ['textbox', 'weirdrole', 'option']);
+});

frontend/src/shared/interactiveRanking.ts

@@ -0,0 +1,104 @@
+// Pure ranking + capping for the interactive-element list the browser agent
+// sees from the accessibility tree. No DOM/CDP deps so it stays unit-testable.
+//
+// Why: BrowserListInteractives used to dump EVERY interactive node with no cap.
+// On heavy pages that is 200+ rows, which dilutes the model's attention and
+// burns tokens. We dedupe twins, rank the things a human acts on first, and
+// cap, so the model gets a short, high-signal menu.
+
+export interface RankItem {
+  role: string;
+  name: string;
+  backendNodeId: number;
+  // Present when the element lives in a cross-origin (OOPIF) child frame; the
+  // CDP session to address it through. Ranking ignores it, just carries it.
+  sessionId?: string;
+}
+
+// Lower number = higher priority. Inputs the user types into first, then
+// navigation/buttons, then toggles, then the long tail of list/option roles.
+const ROLE_PRIORITY: Record<string, number> = {
+  textbox: 0, searchbox: 0, combobox: 0,
+  button: 1, link: 1, menuitem: 1, tab: 1,
+  checkbox: 2, radio: 2, switch: 2, menuitemcheckbox: 2, menuitemradio: 2,
+  option: 3, treeitem: 3, listbox: 3, slider: 3, spinbutton: 3,
+};
+
+const DEFAULT_PRIORITY = 2;
+export const DEFAULT_INTERACTIVE_CAP = 60;
+
+function rolePriority(role: string): number {
+  return role in ROLE_PRIORITY ? ROLE_PRIORITY[role] : DEFAULT_PRIORITY;
+}
+
+// Drop back-to-back duplicates with the same role+name. The AX tree often
+// emits an icon node and its label as twins, and sticky headers repeat the
+// same control. Consecutive-only so a genuine list (5 distinct "Add to cart"
+// buttons interleaved with product text) is never collapsed. The sessionId is
+// part of the key so a same-named element in a cross-origin child frame is
+// never mistaken for a twin of the root frame's last element at the seam.
+function dedupeConsecutive(items: RankItem[]): RankItem[] {
+  const out: RankItem[] = [];
+  for (const it of items) {
+    const prev = out[out.length - 1];
+    if (prev && prev.role === it.role && prev.name === it.name && prev.sessionId === it.sessionId) continue;
+    out.push(it);
+  }
+  return out;
+}
+
+export interface RankResult {
+  shown: RankItem[];
+  truncated: number;
+}
+
+export interface RankOptions {
+  cap?: number;
+  // The agent's current goal; elements whose name matches it float to the top
+  // so the thing the model is actually looking for survives the cap.
+  goal?: string;
+}
+
+// Words too generic to be useful signal, including the browser-action verbs
+// and UI nouns that would otherwise match half the page ("click the button").
+const STOPWORDS = new Set([
+  'the', 'and', 'for', 'with', 'click', 'type', 'into', 'button', 'link',
+  'press', 'select', 'open', 'goto', 'navigate', 'find', 'tap', 'this',
+  'that', 'your', 'from', 'page', 'then', 'enter', 'input', 'field', 'box',
+  'icon', 'menu', 'option', 'item', 'element',
+]);
+
+export function goalKeywords(goal: string): string[] {
+  const words = goal.toLowerCase().split(/[^a-z0-9]+/).filter(Boolean);
+  const kept = words.filter((w) => w.length >= 3 && !STOPWORDS.has(w));
+  return Array.from(new Set(kept)).slice(0, 8);
+}
+
+function matchesGoal(name: string, keywords: string[]): boolean {
+  if (keywords.length === 0) return false;
+  const lower = name.toLowerCase();
+  return keywords.some((k) => lower.includes(k));
+}
+
+export function rankAndCapInteractives(
+  items: RankItem[],
+  opts: RankOptions = {},
+): RankResult {
+  const cap = opts.cap ?? DEFAULT_INTERACTIVE_CAP;
+  const keywords = opts.goal ? goalKeywords(opts.goal) : [];
+  const deduped = dedupeConsecutive(items);
+  // Sort: goal-matched first, then role priority, tiebroken on original
+  // document order so the result is deterministic regardless of engine sort.
+  const ranked = deduped
+    .map((it, i) => ({ it, i, m: matchesGoal(it.name, keywords) ? 0 : 1 }))
+    .sort((a, b) => {
+      if (a.m !== b.m) return a.m - b.m;
+      const pa = rolePriority(a.it.role);
+      const pb = rolePriority(b.it.role);
+      if (pa !== pb) return pa - pb;
+      return a.i - b.i;
+    })
+    .map((x) => x.it);
+  const shown = cap > 0 ? ranked.slice(0, cap) : ranked;
+  return { shown, truncated: Math.max(0, ranked.length - shown.length) };
+}

frontend/tsconfig.json

@@ -23,5 +23,5 @@
     }
   },
   "include": ["next-env.d.ts", "**/*.ts", "**/*.tsx", ".next/types/**/*.ts"],
-  "exclude": ["node_modules"]
+  "exclude": ["node_modules", "**/*.test.ts"]
 }