[aidan] bug: cross-origin iframe perception + spawned card end UX (#78)

Browser sub-agents can now drive Google Drive’s share-doc flow end to end: same-process cross-origin iframes (the share dialog) are visible through a frame-tree-aware AX walk, and combobox autocomplete fields fill in one click_index call via DOM.focus + Input.insertText. When a spawned browser card’s agent finishes, the card now fades through a “Task done” pill with a Keep button for ~3s instead of snapping out, so the user sees what happened.

backend/apps/agents/browser/browser_agent.py

@@ -1586,10 +1586,12 @@ async def run_browser_agent(
                 # send click is proof enough; drive to the OUTCOME.
                 if task_is_send and not send_confirmed and "error" not in result and tu.name in _CONFIRM_TOOLS:
                     _cn = result.get("clickedName") or ""
-                    _send_click = browser_batch_replay.is_replay_boundary(
-                        {"action": "click", "name": _cn}) or any(
-                        browser_batch_replay.is_replay_boundary(
-                            {"action": "click", "name": r.get("clickedName") or ""})
+                    _cr = result.get("clickedRole") or ""
+                    _send_click = browser_batch_replay.is_send_completed(
+                        {"action": "click", "name": _cn, "role": _cr}) or any(
+                        browser_batch_replay.is_send_completed(
+                            {"action": "click", "name": r.get("clickedName") or "",
+                             "role": r.get("clickedRole") or ""})
                         for r in (result.get("results") or []))
                     if _send_click:
                         send_confirmed = True

backend/apps/agents/browser/browser_batch_replay.py

@@ -130,6 +130,26 @@ def is_replay_boundary(step: dict) -> bool:
     return False
 
 
+_SEND_COMPLETED_RE = re.compile(
+    r"\b(send|submit|pay|place\s*order|complete\s*(order|purchase|checkout|payment))\b",
+    re.I,
+)
+_OPENER_ROLES = frozenset({"menuitem", "menuitemcheckbox", "menuitemradio", "link", "tab"})
+
+
+def is_send_completed(step: dict) -> bool:
+    """True only when the click was a non-opener role AND the label matches an
+    unambiguous send-completion verb. Menuitems, links, and tabs label proximate
+    UI rather than the action itself, so they never count even if their name
+    matches (Drive's 'Share' menuitem was the false positive that prompted this)."""
+    if step.get("action") != "click":
+        return False
+    role = str(step.get("role") or "").lower()
+    if role in _OPENER_ROLES:
+        return False
+    return bool(_SEND_COMPLETED_RE.search(str(step.get("name") or "")))
+
+
 def live_batch_guard(actions, seen_lines, composer_pending: bool = False) -> str:
     """Reason string if a live BrowserBatch carries an irreversible step, else ''.
 

frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx

@@ -26,6 +26,7 @@ import {
   setBrowserCardSize,
   removeBrowserCard,
   resumeBrowserCard,
+  cancelBrowserCardEnding,
   addBrowserTab,
   removeBrowserTab,
   setActiveBrowserTab,
@@ -219,6 +220,7 @@ const BrowserCard: React.FC<Props> = ({
   const browserAgentSession = useAppSelector(selectBrowserAgentSession);
 
   const suspendedSnap = useAppSelector((state) => state.dashboardLayout.suspendedBrowserCards[browserId]);
+  const endingState = useAppSelector((state) => state.dashboardLayout.endingBrowserCards[browserId]);
 
   // Arm the Windows webview crash-safety marker synchronously, before React commits
   // the <webview> below. Cleared on dom-ready; a leftover marker next launch tells
@@ -236,6 +238,7 @@ const BrowserCard: React.FC<Props> = ({
   const [tabLocalStates, setTabLocalStates] = useState<Record<string, TabLocalState>>({});
   // Electron webviews can't trigger OS platform auth; preload sends "passkey-detected" and we explain via modal.
   const [passkeyDialogOpen, setPasskeyDialogOpen] = useState(false);
+  const [crashedTabs, setCrashedTabs] = useState<Set<string>>(new Set());
   const updateTabLocal = useCallback((tabId: string, update: Partial<TabLocalState>) => {
     setTabLocalStates((prev) => {
       const existing = prev[tabId] ?? { loading: false, canGoBack: false, canGoForward: false };
@@ -269,6 +272,17 @@ const BrowserCard: React.FC<Props> = ({
     if (suspendedSnap) initializedTabs.current.clear();
   }, [suspendedSnap]);
 
+  // Spawned cards get marked "ending" by WebSocketManager when the parent agent
+  // finishes; show the fade pill for ~3s, then dispatch the real remove. Keep
+  // clears the flag and the cleanup below cancels the pending remove.
+  useEffect(() => {
+    if (!endingState) return;
+    const timer = setTimeout(() => {
+      dispatch(removeBrowserCard(browserId));
+    }, 3000);
+    return () => clearTimeout(timer);
+  }, [endingState, browserId, dispatch]);
+
   const tabIdKey = tabs.map((t) => t.id).join(',');
   useEffect(() => {
     if (!isElectron) return;
@@ -342,6 +356,20 @@ const BrowserCard: React.FC<Props> = ({
         updateTabLocal(tabId, { loading: false });
         onNavigate();
         onTitleUpdate();
+        setCrashedTabs((prev) => {
+          if (!prev.has(tabId)) return prev;
+          const next = new Set(prev);
+          next.delete(tabId);
+          return next;
+        });
+      };
+      const onProcessGone = () => {
+        setCrashedTabs((prev) => {
+          if (prev.has(tabId)) return prev;
+          const next = new Set(prev);
+          next.add(tabId);
+          return next;
+        });
       };
 
       const onFaviconUpdate = (e: any) => {
@@ -366,6 +394,8 @@ const BrowserCard: React.FC<Props> = ({
       wv.addEventListener('page-favicon-updated', onFaviconUpdate);
       wv.addEventListener('ipc-message', onIpcMessage as any);
       wv.addEventListener('new-window', onNewWindow as any);
+      wv.addEventListener('render-process-gone', onProcessGone as any);
+      wv.addEventListener('crashed', onProcessGone as any);
 
       cleanups.push(() => {
         unregisterWebview(browserId, tabId);
@@ -377,6 +407,8 @@ const BrowserCard: React.FC<Props> = ({
         wv.removeEventListener('page-favicon-updated', onFaviconUpdate);
         wv.removeEventListener('ipc-message', onIpcMessage as any);
         wv.removeEventListener('new-window', onNewWindow as any);
+        wv.removeEventListener('render-process-gone', onProcessGone as any);
+        wv.removeEventListener('crashed', onProcessGone as any);
       });
     }
 
@@ -1119,31 +1151,104 @@ const BrowserCard: React.FC<Props> = ({
               </Box>
             )
           ) : (
-          tabs.map((tab) => (
-            <webview
-              key={tab.id}
-              ref={(el: any) => {
-                if (el) webviewMap.current.set(tab.id, el as unknown as WebviewElement);
-                else webviewMap.current.delete(tab.id);
-              }}
-              data-tab-id={tab.id}
-              src="about:blank"
-              {...({ allowpopups: 'true' } as any) /* React drops boolean-valued unknown attrs, so string it stays; @types/react wrongly says boolean */}
-              useragent={chromeUserAgent}
-              {...(webviewPreloadPath ? { preload: webviewPreloadPath } : {})}
-              webpreferences="plugins=yes, autoplayPolicy=no-user-gesture-required, backgroundThrottling=no" /* throttling: guests get occlusion-suspended on their own even with the host's disable-renderer-backgrounding, freezing agent JS when the window is covered */
-              style={{
-                position: 'absolute',
-                top: 0,
-                left: 0,
-                width: '100%',
-                height: '100%',
-                border: 'none',
-                visibility: tab.id === activeTabId ? 'visible' : 'hidden',
-                zIndex: tab.id === activeTabId ? 1 : 0,
-              }}
-            />
-          ))
+          <>
+            {tabs.map((tab) => (
+              <webview
+                key={tab.id}
+                ref={(el: any) => {
+                  if (el) webviewMap.current.set(tab.id, el as unknown as WebviewElement);
+                  else webviewMap.current.delete(tab.id);
+                }}
+                data-tab-id={tab.id}
+                src="about:blank"
+                {...({ allowpopups: 'true' } as any) /* React drops boolean-valued unknown attrs, so string it stays; @types/react wrongly says boolean */}
+                useragent={chromeUserAgent}
+                {...(webviewPreloadPath ? { preload: webviewPreloadPath } : {})}
+                webpreferences="plugins=yes, autoplayPolicy=no-user-gesture-required, backgroundThrottling=no" /* throttling: guests get occlusion-suspended on their own even with the host's disable-renderer-backgrounding, freezing agent JS when the window is covered */
+                style={{
+                  position: 'absolute',
+                  top: 0,
+                  left: 0,
+                  width: '100%',
+                  height: '100%',
+                  border: 'none',
+                  visibility: tab.id === activeTabId ? 'visible' : 'hidden',
+                  zIndex: tab.id === activeTabId ? 1 : 0,
+                }}
+              />
+            ))}
+            <Fade in={!!endingState && !crashedTabs.has(activeTabId)} timeout={{ enter: 200, exit: 220 }} unmountOnExit>
+              <Box
+                sx={{
+                  position: 'absolute',
+                  inset: 0,
+                  zIndex: 5,
+                  display: 'flex',
+                  flexDirection: 'column',
+                  alignItems: 'center',
+                  justifyContent: 'center',
+                  gap: 1.25,
+                  bgcolor: c.bg.surface,
+                }}
+              >
+                <Typography sx={{ color: c.text.primary, fontSize: '0.95rem', fontWeight: 500 }}>
+                  {endingState?.status === 'error' ? 'Task ended with an error.' : 'Task done.'}
+                </Typography>
+                <Button
+                  onClick={() => dispatch(cancelBrowserCardEnding(browserId))}
+                  sx={{
+                    textTransform: 'none',
+                    fontSize: '0.82rem',
+                    fontWeight: 600,
+                    bgcolor: c.accent.primary,
+                    color: '#fff',
+                    borderRadius: `${c.radius.md}px`,
+                    px: 2.25,
+                    py: 0.6,
+                    '&:hover': { bgcolor: c.accent.hover || c.accent.primary },
+                  }}
+                >
+                  Keep
+                </Button>
+              </Box>
+            </Fade>
+            <Fade in={crashedTabs.has(activeTabId)} timeout={{ enter: 200, exit: 220 }} unmountOnExit>
+              <Box
+                sx={{
+                  position: 'absolute',
+                  inset: 0,
+                  zIndex: 6,
+                  display: 'flex',
+                  flexDirection: 'column',
+                  alignItems: 'center',
+                  justifyContent: 'center',
+                  gap: 1.25,
+                  bgcolor: c.bg.surface,
+                }}
+              >
+                <Typography sx={{ color: c.text.primary, fontSize: '0.95rem', fontWeight: 500 }}>
+                  This page stopped responding.
+                </Typography>
+                <Button
+                  onClick={() => webviewMap.current.get(activeTabId)?.reload()}
+                  startIcon={<RefreshIcon sx={{ fontSize: '1rem' }} />}
+                  sx={{
+                    textTransform: 'none',
+                    fontSize: '0.82rem',
+                    fontWeight: 600,
+                    bgcolor: c.accent.primary,
+                    color: '#fff',
+                    borderRadius: `${c.radius.md}px`,
+                    px: 2.25,
+                    py: 0.6,
+                    '&:hover': { bgcolor: c.accent.hover || c.accent.primary },
+                  }}
+                >
+                  Reload
+                </Button>
+              </Box>
+            </Fade>
+          </>
           )
         ) : null}
         <Dialog

frontend/src/shared/browserCommandHandler.ts

@@ -532,6 +532,8 @@ async function frameOffset(
 const _AX_ROOT_TIMEOUT_MS = 8000;
 const _AX_CHILD_TIMEOUT_MS = 2500;
 const _MAX_AX_CHILD_FRAMES = 6;
+const _PAGE_TREE_TIMEOUT_MS = 1500;
+const _MAX_TOTAL_FRAMES = 12;
 
 function withTimeout<T>(p: Promise<T>, ms: number, label: string): Promise<T> {
   p.catch(() => {}); // swallow a late rejection if the timeout wins the race first
@@ -542,30 +544,78 @@ function withTimeout<T>(p: Promise<T>, ms: number, label: string): Promise<T> {
   return Promise.race([p, timeout]).finally(() => clearTimeout(timer));
 }
 
+function flattenFrameTree(tree: any, out: string[] = []): string[] {
+  if (!tree) return out;
+  const id = tree?.frame?.id;
+  if (id) out.push(id);
+  for (const c of tree.childFrames || []) flattenFrameTree(c, out);
+  return out;
+}
+
 async function enumerateCandidates(wv: BrowserWebview): Promise<RankItem[]> {
   const candidates: RankItem[] = [];
-  let rootTree;
-  try {
-    rootTree = await withTimeout(
-      sendCdp(wv, 'Accessibility.getFullAXTree', {}), _AX_ROOT_TIMEOUT_MS, 'page perception');
-  } catch (err: any) {
+  let framesWalked = 0;
+  let framesDropped = 0;
+
+  const walkSession = async (
+    sessionId: string | undefined, budgetMs: number, label: string,
+  ): Promise<{ ok: boolean; lastErr?: any }> => {
+    const sessionStart = Date.now();
+    const remaining = () => Math.max(1, budgetMs - (Date.now() - sessionStart));
+
+    let frameIds: string[] = [];
+    try {
+      const tree = await withTimeout(
+        sendCdp(wv, 'Page.getFrameTree', {}, sessionId),
+        Math.min(_PAGE_TREE_TIMEOUT_MS, remaining()), `${label} frame tree`);
+      frameIds = flattenFrameTree(tree?.frameTree);
+    } catch { /* fall through to a single AX call below */ }
+
+    if (frameIds.length === 0) {
+      if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; return { ok: true }; }
+      try {
+        const ax = await withTimeout(
+          sendCdp(wv, 'Accessibility.getFullAXTree', {}, sessionId), remaining(), label);
+        candidates.push(...axNodesToCandidates(ax?.nodes || [], sessionId));
+        framesWalked++;
+        return { ok: true };
+      } catch (err: any) {
+        return { ok: false, lastErr: err };
+      }
+    }
+
+    let lastErr: any;
+    let anySuccess = false;
+    for (const frameId of frameIds) {
+      if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; continue; }
+      if (remaining() <= 1) { framesDropped++; continue; }
+      try {
+        const ax = await withTimeout(
+          sendCdp(wv, 'Accessibility.getFullAXTree', { frameId }, sessionId), remaining(), label);
+        candidates.push(...axNodesToCandidates(ax?.nodes || [], sessionId));
+        anySuccess = true;
+      } catch (err: any) { lastErr = err; }
+      framesWalked++;
+    }
+    return { ok: anySuccess, lastErr };
+  };
+
+  const root = await walkSession(undefined, _AX_ROOT_TIMEOUT_MS, 'page perception');
+  if (!root.ok) {
     // A saturated/hung renderer can't answer; surface a clear, actionable signal
     // instead of silently blocking to the hard command timeout, so the agent can
     // wait a beat and retry (the freeze is often intermittent) rather than abort.
     throw new Error(
-      `the page is too busy to read right now (${err?.message || 'timed out'}); `
+      `the page is too busy to read right now (${root.lastErr?.message || 'timed out'}); `
       + 'wait a moment with BrowserWait and try again, or reload the page.');
   }
-  candidates.push(...axNodesToCandidates(rootTree?.nodes || []));
   const children = (await getChildSessions(wv)).slice(0, _MAX_AX_CHILD_FRAMES);
   for (const child of children) {
-    try {
-      const childTree = await withTimeout(
-        sendCdp(wv, 'Accessibility.getFullAXTree', {}, child.sessionId), _AX_CHILD_TIMEOUT_MS, 'child frame');
-      candidates.push(...axNodesToCandidates(childTree?.nodes || [], child.sessionId));
-    } catch {
-      // skip a slow/unresponsive frame rather than stalling the whole list
-    }
+    if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; continue; }
+    await walkSession(child.sessionId, _AX_CHILD_TIMEOUT_MS, 'child frame');
+  }
+  if (framesDropped > 0) {
+    console.log(`[cdp] enumerateCandidates capped at ${_MAX_TOTAL_FRAMES} frames; dropped ${framesDropped}`);
   }
   return candidates;
 }
@@ -604,7 +654,9 @@ async function clickBackendNode(
   // the wrong element (the box model is frame-local but the click dispatches in the root
   // frame), while DOM.focus reaches the node in any frame. With a `text` arg we then
   // insert the whole string at once, no clicking, no character-by-character typing.
-  if (/\b(textbox|searchbox)\b/i.test(opts.role || '')) {
+  const _role = opts.role || '';
+  const _wantsText = typeof opts.text === 'string' && opts.text.length > 0;
+  if (/\b(textbox|searchbox)\b/i.test(_role) || (/\bcombobox\b/i.test(_role) && _wantsText)) {
     try {
       await sendCdp(wv, 'DOM.focus', { backendNodeId }, sessionId);
     } catch (err: any) {

frontend/src/shared/state/dashboardLayoutSlice.ts

@@ -96,6 +96,8 @@ export interface DashboardLayoutState {
   pendingFocusNoteId: string | null;
   /** Transient: snapshot stand-ins for off-screen webviews; never rides the layout PUT. */
   suspendedBrowserCards: Record<string, { dataUrl: string; capturedAt: number }>;
+  /** Transient: spawned cards that are about to be removed; surfaces the fade + Keep pill. */
+  endingBrowserCards: Record<string, { status: 'completed' | 'error'; at: number }>;
 }
 
 const initialState: DashboardLayoutState = {
@@ -113,6 +115,7 @@ const initialState: DashboardLayoutState = {
   pendingFocusBrowserId: null,
   pendingFocusNoteId: null,
   suspendedBrowserCards: {},
+  endingBrowserCards: {},
 };
 
 interface LayoutPayload {
@@ -635,6 +638,21 @@ const dashboardLayoutSlice = createSlice({
     removeBrowserCard(state, action: PayloadAction<string>) {
       delete state.browserCards[action.payload];
       delete state.suspendedBrowserCards[action.payload];
+      delete state.endingBrowserCards[action.payload];
+    },
+
+    markBrowserCardEnding(
+      state, action: PayloadAction<{ browserId: string; status: 'completed' | 'error' }>,
+    ) {
+      if (!state.browserCards[action.payload.browserId]) return;
+      state.endingBrowserCards[action.payload.browserId] = {
+        status: action.payload.status,
+        at: Date.now(),
+      };
+    },
+
+    cancelBrowserCardEnding(state, action: PayloadAction<string>) {
+      delete state.endingBrowserCards[action.payload];
     },
 
     suspendBrowserCard(state, action: PayloadAction<{ browserId: string; dataUrl: string }>) {
@@ -959,6 +977,7 @@ const dashboardLayoutSlice = createSlice({
       state.initialized = false;
       state.pendingFocusNoteId = null;
       state.suspendedBrowserCards = {};
+      state.endingBrowserCards = {};
     },
 
   },
@@ -1068,6 +1087,8 @@ export const {
   removeBrowserCard,
   suspendBrowserCard,
   resumeBrowserCard,
+  markBrowserCardEnding,
+  cancelBrowserCardEnding,
   pasteBrowserCard,
   updateBrowserCardUrl,
   addBrowserTab,

frontend/src/shared/ws/WebSocketManager.ts

@@ -23,7 +23,7 @@ import {
   clearTurnLabel,
 } from '../state/agentsSlice';
 import { streamStart, streamDelta, streamEnd, clearStreamingForSession } from '../state/streamingSlice';
-import { addBrowserCardFromBackend, removeBrowserCard, setBrowserCardPosition, setGlowingBrowserCards, GRID_GAP } from '../state/dashboardLayoutSlice';
+import { addBrowserCardFromBackend, markBrowserCardEnding, setBrowserCardPosition, setGlowingBrowserCards, GRID_GAP } from '../state/dashboardLayoutSlice';
 import { upsertOutput } from '../state/outputsSlice';
 import { getAuthToken } from '../config';
 import { notifyAgentCompletion } from '../notifications';
@@ -500,6 +500,8 @@ class WebSocketManager {
         // sub-agent: the parent reuses the same browser_id for its next step,
         // so deleting on sub-agent completion strands BrowserAgent(browser_id)
         // on a dead card. 'stopped' skipped to allow inspect-after-manual-stop.
+        // Mark the card as ending instead of removing immediately so the card
+        // shows a fade + Keep pill; BrowserCard owns the 3s timer to remove.
         if (
           session_id &&
           (data.status === 'completed' || data.status === 'error') &&
@@ -508,7 +510,9 @@ class WebSocketManager {
           const browserCards = store.getState().dashboardLayout.browserCards;
           for (const card of Object.values(browserCards)) {
             if (card.spawned_by === session_id) {
-              store.dispatch(removeBrowserCard(card.browser_id));
+              store.dispatch(markBrowserCardEnding({
+                browserId: card.browser_id, status: data.status,
+              }));
             }
           }
         }
@@ -723,11 +727,15 @@ class WebSocketManager {
           // Auto-delete browsers spawned by this agent when it finishes
           // normally or errors out. We intentionally skip 'stopped' , the
           // user may want to inspect the browser after manually stopping.
+          // Mark for removal so BrowserCard renders a fade + Keep pill;
+          // the card itself runs the 3s timer to dispatch removeBrowserCard.
           if (closedStatus === 'completed' || closedStatus === 'error') {
             const browserCards = store.getState().dashboardLayout.browserCards;
             for (const card of Object.values(browserCards)) {
               if (card.spawned_by === session_id) {
-                store.dispatch(removeBrowserCard(card.browser_id));
+                store.dispatch(markBrowserCardEnding({
+                  browserId: card.browser_id, status: closedStatus,
+                }));
               }
             }
           }