[aidan] bug: cross-origin iframe perception + spawned card end UX (#78)

Browser sub-agents can now drive Google Drive’s share-doc flow end to end: same-process cross-origin iframes (the share dialog) are visible through a frame-tree-aware AX walk, and combobox autocomplete fields fill in one click_index call via DOM.focus + Input.insertText. When a spawned browser card’s agent finishes, the card now fades through a “Task done” pill with a Keep button for ~3s instead of snapping out, so the user sees what happened.

frontend/src/shared/browserCommandHandler.ts

@@ -532,6 +532,8 @@ async function frameOffset(
 const _AX_ROOT_TIMEOUT_MS = 8000;
 const _AX_CHILD_TIMEOUT_MS = 2500;
 const _MAX_AX_CHILD_FRAMES = 6;
+const _PAGE_TREE_TIMEOUT_MS = 1500;
+const _MAX_TOTAL_FRAMES = 12;
 
 function withTimeout<T>(p: Promise<T>, ms: number, label: string): Promise<T> {
   p.catch(() => {}); // swallow a late rejection if the timeout wins the race first
@@ -542,30 +544,78 @@ function withTimeout<T>(p: Promise<T>, ms: number, label: string): Promise<T> {
   return Promise.race([p, timeout]).finally(() => clearTimeout(timer));
 }
 
+function flattenFrameTree(tree: any, out: string[] = []): string[] {
+  if (!tree) return out;
+  const id = tree?.frame?.id;
+  if (id) out.push(id);
+  for (const c of tree.childFrames || []) flattenFrameTree(c, out);
+  return out;
+}
+
 async function enumerateCandidates(wv: BrowserWebview): Promise<RankItem[]> {
   const candidates: RankItem[] = [];
-  let rootTree;
-  try {
-    rootTree = await withTimeout(
-      sendCdp(wv, 'Accessibility.getFullAXTree', {}), _AX_ROOT_TIMEOUT_MS, 'page perception');
-  } catch (err: any) {
+  let framesWalked = 0;
+  let framesDropped = 0;
+
+  const walkSession = async (
+    sessionId: string | undefined, budgetMs: number, label: string,
+  ): Promise<{ ok: boolean; lastErr?: any }> => {
+    const sessionStart = Date.now();
+    const remaining = () => Math.max(1, budgetMs - (Date.now() - sessionStart));
+
+    let frameIds: string[] = [];
+    try {
+      const tree = await withTimeout(
+        sendCdp(wv, 'Page.getFrameTree', {}, sessionId),
+        Math.min(_PAGE_TREE_TIMEOUT_MS, remaining()), `${label} frame tree`);
+      frameIds = flattenFrameTree(tree?.frameTree);
+    } catch { /* fall through to a single AX call below */ }
+
+    if (frameIds.length === 0) {
+      if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; return { ok: true }; }
+      try {
+        const ax = await withTimeout(
+          sendCdp(wv, 'Accessibility.getFullAXTree', {}, sessionId), remaining(), label);
+        candidates.push(...axNodesToCandidates(ax?.nodes || [], sessionId));
+        framesWalked++;
+        return { ok: true };
+      } catch (err: any) {
+        return { ok: false, lastErr: err };
+      }
+    }
+
+    let lastErr: any;
+    let anySuccess = false;
+    for (const frameId of frameIds) {
+      if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; continue; }
+      if (remaining() <= 1) { framesDropped++; continue; }
+      try {
+        const ax = await withTimeout(
+          sendCdp(wv, 'Accessibility.getFullAXTree', { frameId }, sessionId), remaining(), label);
+        candidates.push(...axNodesToCandidates(ax?.nodes || [], sessionId));
+        anySuccess = true;
+      } catch (err: any) { lastErr = err; }
+      framesWalked++;
+    }
+    return { ok: anySuccess, lastErr };
+  };
+
+  const root = await walkSession(undefined, _AX_ROOT_TIMEOUT_MS, 'page perception');
+  if (!root.ok) {
     // A saturated/hung renderer can't answer; surface a clear, actionable signal
     // instead of silently blocking to the hard command timeout, so the agent can
     // wait a beat and retry (the freeze is often intermittent) rather than abort.
     throw new Error(
-      `the page is too busy to read right now (${err?.message || 'timed out'}); `
+      `the page is too busy to read right now (${root.lastErr?.message || 'timed out'}); `
       + 'wait a moment with BrowserWait and try again, or reload the page.');
   }
-  candidates.push(...axNodesToCandidates(rootTree?.nodes || []));
   const children = (await getChildSessions(wv)).slice(0, _MAX_AX_CHILD_FRAMES);
   for (const child of children) {
-    try {
-      const childTree = await withTimeout(
-        sendCdp(wv, 'Accessibility.getFullAXTree', {}, child.sessionId), _AX_CHILD_TIMEOUT_MS, 'child frame');
-      candidates.push(...axNodesToCandidates(childTree?.nodes || [], child.sessionId));
-    } catch {
-      // skip a slow/unresponsive frame rather than stalling the whole list
-    }
+    if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; continue; }
+    await walkSession(child.sessionId, _AX_CHILD_TIMEOUT_MS, 'child frame');
+  }
+  if (framesDropped > 0) {
+    console.log(`[cdp] enumerateCandidates capped at ${_MAX_TOTAL_FRAMES} frames; dropped ${framesDropped}`);
   }
   return candidates;
 }
@@ -604,7 +654,9 @@ async function clickBackendNode(
   // the wrong element (the box model is frame-local but the click dispatches in the root
   // frame), while DOM.focus reaches the node in any frame. With a `text` arg we then
   // insert the whole string at once, no clicking, no character-by-character typing.
-  if (/\b(textbox|searchbox)\b/i.test(opts.role || '')) {
+  const _role = opts.role || '';
+  const _wantsText = typeof opts.text === 'string' && opts.text.length > 0;
+  if (/\b(textbox|searchbox)\b/i.test(_role) || (/\bcombobox\b/i.test(_role) && _wantsText)) {
     try {
       await sendCdp(wv, 'DOM.focus', { backendNodeId }, sessionId);
     } catch (err: any) {

frontend/src/shared/state/dashboardLayoutSlice.ts

@@ -96,6 +96,8 @@ export interface DashboardLayoutState {
   pendingFocusNoteId: string | null;
   /** Transient: snapshot stand-ins for off-screen webviews; never rides the layout PUT. */
   suspendedBrowserCards: Record<string, { dataUrl: string; capturedAt: number }>;
+  /** Transient: spawned cards that are about to be removed; surfaces the fade + Keep pill. */
+  endingBrowserCards: Record<string, { status: 'completed' | 'error'; at: number }>;
 }
 
 const initialState: DashboardLayoutState = {
@@ -113,6 +115,7 @@ const initialState: DashboardLayoutState = {
   pendingFocusBrowserId: null,
   pendingFocusNoteId: null,
   suspendedBrowserCards: {},
+  endingBrowserCards: {},
 };
 
 interface LayoutPayload {
@@ -635,6 +638,21 @@ const dashboardLayoutSlice = createSlice({
     removeBrowserCard(state, action: PayloadAction<string>) {
       delete state.browserCards[action.payload];
       delete state.suspendedBrowserCards[action.payload];
+      delete state.endingBrowserCards[action.payload];
+    },
+
+    markBrowserCardEnding(
+      state, action: PayloadAction<{ browserId: string; status: 'completed' | 'error' }>,
+    ) {
+      if (!state.browserCards[action.payload.browserId]) return;
+      state.endingBrowserCards[action.payload.browserId] = {
+        status: action.payload.status,
+        at: Date.now(),
+      };
+    },
+
+    cancelBrowserCardEnding(state, action: PayloadAction<string>) {
+      delete state.endingBrowserCards[action.payload];
     },
 
     suspendBrowserCard(state, action: PayloadAction<{ browserId: string; dataUrl: string }>) {
@@ -959,6 +977,7 @@ const dashboardLayoutSlice = createSlice({
       state.initialized = false;
       state.pendingFocusNoteId = null;
       state.suspendedBrowserCards = {};
+      state.endingBrowserCards = {};
     },
 
   },
@@ -1068,6 +1087,8 @@ export const {
   removeBrowserCard,
   suspendBrowserCard,
   resumeBrowserCard,
+  markBrowserCardEnding,
+  cancelBrowserCardEnding,
   pasteBrowserCard,
   updateBrowserCardUrl,
   addBrowserTab,

frontend/src/shared/ws/WebSocketManager.ts

@@ -23,7 +23,7 @@ import {
   clearTurnLabel,
 } from '../state/agentsSlice';
 import { streamStart, streamDelta, streamEnd, clearStreamingForSession } from '../state/streamingSlice';
-import { addBrowserCardFromBackend, removeBrowserCard, setBrowserCardPosition, setGlowingBrowserCards, GRID_GAP } from '../state/dashboardLayoutSlice';
+import { addBrowserCardFromBackend, markBrowserCardEnding, setBrowserCardPosition, setGlowingBrowserCards, GRID_GAP } from '../state/dashboardLayoutSlice';
 import { upsertOutput } from '../state/outputsSlice';
 import { getAuthToken } from '../config';
 import { notifyAgentCompletion } from '../notifications';
@@ -500,6 +500,8 @@ class WebSocketManager {
         // sub-agent: the parent reuses the same browser_id for its next step,
         // so deleting on sub-agent completion strands BrowserAgent(browser_id)
         // on a dead card. 'stopped' skipped to allow inspect-after-manual-stop.
+        // Mark the card as ending instead of removing immediately so the card
+        // shows a fade + Keep pill; BrowserCard owns the 3s timer to remove.
         if (
           session_id &&
           (data.status === 'completed' || data.status === 'error') &&
@@ -508,7 +510,9 @@ class WebSocketManager {
           const browserCards = store.getState().dashboardLayout.browserCards;
           for (const card of Object.values(browserCards)) {
             if (card.spawned_by === session_id) {
-              store.dispatch(removeBrowserCard(card.browser_id));
+              store.dispatch(markBrowserCardEnding({
+                browserId: card.browser_id, status: data.status,
+              }));
             }
           }
         }
@@ -723,11 +727,15 @@ class WebSocketManager {
           // Auto-delete browsers spawned by this agent when it finishes
           // normally or errors out. We intentionally skip 'stopped' , the
           // user may want to inspect the browser after manually stopping.
+          // Mark for removal so BrowserCard renders a fade + Keep pill;
+          // the card itself runs the 3s timer to dispatch removeBrowserCard.
           if (closedStatus === 'completed' || closedStatus === 'error') {
             const browserCards = store.getState().dashboardLayout.browserCards;
             for (const card of Object.values(browserCards)) {
               if (card.spawned_by === session_id) {
-                store.dispatch(removeBrowserCard(card.browser_id));
+                store.dispatch(markBrowserCardEnding({
+                  browserId: card.browser_id, status: closedStatus,
+                }));
               }
             }
           }