diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index fafb1a9a..69f52f4f 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -1586,10 +1586,12 @@ async def run_browser_agent( # send click is proof enough; drive to the OUTCOME. if task_is_send and not send_confirmed and "error" not in result and tu.name in _CONFIRM_TOOLS: _cn = result.get("clickedName") or "" - _send_click = browser_batch_replay.is_replay_boundary( - {"action": "click", "name": _cn}) or any( - browser_batch_replay.is_replay_boundary( - {"action": "click", "name": r.get("clickedName") or ""}) + _cr = result.get("clickedRole") or "" + _send_click = browser_batch_replay.is_send_completed( + {"action": "click", "name": _cn, "role": _cr}) or any( + browser_batch_replay.is_send_completed( + {"action": "click", "name": r.get("clickedName") or "", + "role": r.get("clickedRole") or ""}) for r in (result.get("results") or [])) if _send_click: send_confirmed = True diff --git a/backend/apps/agents/browser/browser_batch_replay.py b/backend/apps/agents/browser/browser_batch_replay.py index 7ec54662..438caa20 100644 --- a/backend/apps/agents/browser/browser_batch_replay.py +++ b/backend/apps/agents/browser/browser_batch_replay.py @@ -130,6 +130,26 @@ def is_replay_boundary(step: dict) -> bool: return False +_SEND_COMPLETED_RE = re.compile( + r"\b(send|submit|pay|place\s*order|complete\s*(order|purchase|checkout|payment))\b", + re.I, +) +_OPENER_ROLES = frozenset({"menuitem", "menuitemcheckbox", "menuitemradio", "link", "tab"}) + + +def is_send_completed(step: dict) -> bool: + """True only when the click was a non-opener role AND the label matches an + unambiguous send-completion verb. Menuitems, links, and tabs label proximate + UI rather than the action itself, so they never count even if their name + matches (Drive's 'Share' menuitem was the false positive that prompted this).""" + if step.get("action") != "click": + return False + role = str(step.get("role") or "").lower() + if role in _OPENER_ROLES: + return False + return bool(_SEND_COMPLETED_RE.search(str(step.get("name") or ""))) + + def live_batch_guard(actions, seen_lines, composer_pending: bool = False) -> str: """Reason string if a live BrowserBatch carries an irreversible step, else ''. diff --git a/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx b/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx index 78c2c734..457139ea 100644 --- a/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx +++ b/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx @@ -26,6 +26,7 @@ import { setBrowserCardSize, removeBrowserCard, resumeBrowserCard, + cancelBrowserCardEnding, addBrowserTab, removeBrowserTab, setActiveBrowserTab, @@ -219,6 +220,7 @@ const BrowserCard: React.FC = ({ const browserAgentSession = useAppSelector(selectBrowserAgentSession); const suspendedSnap = useAppSelector((state) => state.dashboardLayout.suspendedBrowserCards[browserId]); + const endingState = useAppSelector((state) => state.dashboardLayout.endingBrowserCards[browserId]); // Arm the Windows webview crash-safety marker synchronously, before React commits // the below. Cleared on dom-ready; a leftover marker next launch tells @@ -236,6 +238,7 @@ const BrowserCard: React.FC = ({ const [tabLocalStates, setTabLocalStates] = useState>({}); // Electron webviews can't trigger OS platform auth; preload sends "passkey-detected" and we explain via modal. const [passkeyDialogOpen, setPasskeyDialogOpen] = useState(false); + const [crashedTabs, setCrashedTabs] = useState>(new Set()); const updateTabLocal = useCallback((tabId: string, update: Partial) => { setTabLocalStates((prev) => { const existing = prev[tabId] ?? { loading: false, canGoBack: false, canGoForward: false }; @@ -269,6 +272,17 @@ const BrowserCard: React.FC = ({ if (suspendedSnap) initializedTabs.current.clear(); }, [suspendedSnap]); + // Spawned cards get marked "ending" by WebSocketManager when the parent agent + // finishes; show the fade pill for ~3s, then dispatch the real remove. Keep + // clears the flag and the cleanup below cancels the pending remove. + useEffect(() => { + if (!endingState) return; + const timer = setTimeout(() => { + dispatch(removeBrowserCard(browserId)); + }, 3000); + return () => clearTimeout(timer); + }, [endingState, browserId, dispatch]); + const tabIdKey = tabs.map((t) => t.id).join(','); useEffect(() => { if (!isElectron) return; @@ -342,6 +356,20 @@ const BrowserCard: React.FC = ({ updateTabLocal(tabId, { loading: false }); onNavigate(); onTitleUpdate(); + setCrashedTabs((prev) => { + if (!prev.has(tabId)) return prev; + const next = new Set(prev); + next.delete(tabId); + return next; + }); + }; + const onProcessGone = () => { + setCrashedTabs((prev) => { + if (prev.has(tabId)) return prev; + const next = new Set(prev); + next.add(tabId); + return next; + }); }; const onFaviconUpdate = (e: any) => { @@ -366,6 +394,8 @@ const BrowserCard: React.FC = ({ wv.addEventListener('page-favicon-updated', onFaviconUpdate); wv.addEventListener('ipc-message', onIpcMessage as any); wv.addEventListener('new-window', onNewWindow as any); + wv.addEventListener('render-process-gone', onProcessGone as any); + wv.addEventListener('crashed', onProcessGone as any); cleanups.push(() => { unregisterWebview(browserId, tabId); @@ -377,6 +407,8 @@ const BrowserCard: React.FC = ({ wv.removeEventListener('page-favicon-updated', onFaviconUpdate); wv.removeEventListener('ipc-message', onIpcMessage as any); wv.removeEventListener('new-window', onNewWindow as any); + wv.removeEventListener('render-process-gone', onProcessGone as any); + wv.removeEventListener('crashed', onProcessGone as any); }); } @@ -1119,31 +1151,104 @@ const BrowserCard: React.FC = ({ ) ) : ( - tabs.map((tab) => ( - { - if (el) webviewMap.current.set(tab.id, el as unknown as WebviewElement); - else webviewMap.current.delete(tab.id); - }} - data-tab-id={tab.id} - src="about:blank" - {...({ allowpopups: 'true' } as any) /* React drops boolean-valued unknown attrs, so string it stays; @types/react wrongly says boolean */} - useragent={chromeUserAgent} - {...(webviewPreloadPath ? { preload: webviewPreloadPath } : {})} - webpreferences="plugins=yes, autoplayPolicy=no-user-gesture-required, backgroundThrottling=no" /* throttling: guests get occlusion-suspended on their own even with the host's disable-renderer-backgrounding, freezing agent JS when the window is covered */ - style={{ - position: 'absolute', - top: 0, - left: 0, - width: '100%', - height: '100%', - border: 'none', - visibility: tab.id === activeTabId ? 'visible' : 'hidden', - zIndex: tab.id === activeTabId ? 1 : 0, - }} - /> - )) + <> + {tabs.map((tab) => ( + { + if (el) webviewMap.current.set(tab.id, el as unknown as WebviewElement); + else webviewMap.current.delete(tab.id); + }} + data-tab-id={tab.id} + src="about:blank" + {...({ allowpopups: 'true' } as any) /* React drops boolean-valued unknown attrs, so string it stays; @types/react wrongly says boolean */} + useragent={chromeUserAgent} + {...(webviewPreloadPath ? { preload: webviewPreloadPath } : {})} + webpreferences="plugins=yes, autoplayPolicy=no-user-gesture-required, backgroundThrottling=no" /* throttling: guests get occlusion-suspended on their own even with the host's disable-renderer-backgrounding, freezing agent JS when the window is covered */ + style={{ + position: 'absolute', + top: 0, + left: 0, + width: '100%', + height: '100%', + border: 'none', + visibility: tab.id === activeTabId ? 'visible' : 'hidden', + zIndex: tab.id === activeTabId ? 1 : 0, + }} + /> + ))} + + + + {endingState?.status === 'error' ? 'Task ended with an error.' : 'Task done.'} + + + + + + + + This page stopped responding. + + + + + ) ) : null} (p: Promise, ms: number, label: string): Promise { p.catch(() => {}); // swallow a late rejection if the timeout wins the race first @@ -542,30 +544,78 @@ function withTimeout(p: Promise, ms: number, label: string): Promise { return Promise.race([p, timeout]).finally(() => clearTimeout(timer)); } +function flattenFrameTree(tree: any, out: string[] = []): string[] { + if (!tree) return out; + const id = tree?.frame?.id; + if (id) out.push(id); + for (const c of tree.childFrames || []) flattenFrameTree(c, out); + return out; +} + async function enumerateCandidates(wv: BrowserWebview): Promise { const candidates: RankItem[] = []; - let rootTree; - try { - rootTree = await withTimeout( - sendCdp(wv, 'Accessibility.getFullAXTree', {}), _AX_ROOT_TIMEOUT_MS, 'page perception'); - } catch (err: any) { + let framesWalked = 0; + let framesDropped = 0; + + const walkSession = async ( + sessionId: string | undefined, budgetMs: number, label: string, + ): Promise<{ ok: boolean; lastErr?: any }> => { + const sessionStart = Date.now(); + const remaining = () => Math.max(1, budgetMs - (Date.now() - sessionStart)); + + let frameIds: string[] = []; + try { + const tree = await withTimeout( + sendCdp(wv, 'Page.getFrameTree', {}, sessionId), + Math.min(_PAGE_TREE_TIMEOUT_MS, remaining()), `${label} frame tree`); + frameIds = flattenFrameTree(tree?.frameTree); + } catch { /* fall through to a single AX call below */ } + + if (frameIds.length === 0) { + if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; return { ok: true }; } + try { + const ax = await withTimeout( + sendCdp(wv, 'Accessibility.getFullAXTree', {}, sessionId), remaining(), label); + candidates.push(...axNodesToCandidates(ax?.nodes || [], sessionId)); + framesWalked++; + return { ok: true }; + } catch (err: any) { + return { ok: false, lastErr: err }; + } + } + + let lastErr: any; + let anySuccess = false; + for (const frameId of frameIds) { + if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; continue; } + if (remaining() <= 1) { framesDropped++; continue; } + try { + const ax = await withTimeout( + sendCdp(wv, 'Accessibility.getFullAXTree', { frameId }, sessionId), remaining(), label); + candidates.push(...axNodesToCandidates(ax?.nodes || [], sessionId)); + anySuccess = true; + } catch (err: any) { lastErr = err; } + framesWalked++; + } + return { ok: anySuccess, lastErr }; + }; + + const root = await walkSession(undefined, _AX_ROOT_TIMEOUT_MS, 'page perception'); + if (!root.ok) { // A saturated/hung renderer can't answer; surface a clear, actionable signal // instead of silently blocking to the hard command timeout, so the agent can // wait a beat and retry (the freeze is often intermittent) rather than abort. throw new Error( - `the page is too busy to read right now (${err?.message || 'timed out'}); ` + `the page is too busy to read right now (${root.lastErr?.message || 'timed out'}); ` + 'wait a moment with BrowserWait and try again, or reload the page.'); } - candidates.push(...axNodesToCandidates(rootTree?.nodes || [])); const children = (await getChildSessions(wv)).slice(0, _MAX_AX_CHILD_FRAMES); for (const child of children) { - try { - const childTree = await withTimeout( - sendCdp(wv, 'Accessibility.getFullAXTree', {}, child.sessionId), _AX_CHILD_TIMEOUT_MS, 'child frame'); - candidates.push(...axNodesToCandidates(childTree?.nodes || [], child.sessionId)); - } catch { - // skip a slow/unresponsive frame rather than stalling the whole list - } + if (framesWalked >= _MAX_TOTAL_FRAMES) { framesDropped++; continue; } + await walkSession(child.sessionId, _AX_CHILD_TIMEOUT_MS, 'child frame'); + } + if (framesDropped > 0) { + console.log(`[cdp] enumerateCandidates capped at ${_MAX_TOTAL_FRAMES} frames; dropped ${framesDropped}`); } return candidates; } @@ -604,7 +654,9 @@ async function clickBackendNode( // the wrong element (the box model is frame-local but the click dispatches in the root // frame), while DOM.focus reaches the node in any frame. With a `text` arg we then // insert the whole string at once, no clicking, no character-by-character typing. - if (/\b(textbox|searchbox)\b/i.test(opts.role || '')) { + const _role = opts.role || ''; + const _wantsText = typeof opts.text === 'string' && opts.text.length > 0; + if (/\b(textbox|searchbox)\b/i.test(_role) || (/\bcombobox\b/i.test(_role) && _wantsText)) { try { await sendCdp(wv, 'DOM.focus', { backendNodeId }, sessionId); } catch (err: any) { diff --git a/frontend/src/shared/state/dashboardLayoutSlice.ts b/frontend/src/shared/state/dashboardLayoutSlice.ts index 1a9d097f..fee90e3e 100644 --- a/frontend/src/shared/state/dashboardLayoutSlice.ts +++ b/frontend/src/shared/state/dashboardLayoutSlice.ts @@ -96,6 +96,8 @@ export interface DashboardLayoutState { pendingFocusNoteId: string | null; /** Transient: snapshot stand-ins for off-screen webviews; never rides the layout PUT. */ suspendedBrowserCards: Record; + /** Transient: spawned cards that are about to be removed; surfaces the fade + Keep pill. */ + endingBrowserCards: Record; } const initialState: DashboardLayoutState = { @@ -113,6 +115,7 @@ const initialState: DashboardLayoutState = { pendingFocusBrowserId: null, pendingFocusNoteId: null, suspendedBrowserCards: {}, + endingBrowserCards: {}, }; interface LayoutPayload { @@ -635,6 +638,21 @@ const dashboardLayoutSlice = createSlice({ removeBrowserCard(state, action: PayloadAction) { delete state.browserCards[action.payload]; delete state.suspendedBrowserCards[action.payload]; + delete state.endingBrowserCards[action.payload]; + }, + + markBrowserCardEnding( + state, action: PayloadAction<{ browserId: string; status: 'completed' | 'error' }>, + ) { + if (!state.browserCards[action.payload.browserId]) return; + state.endingBrowserCards[action.payload.browserId] = { + status: action.payload.status, + at: Date.now(), + }; + }, + + cancelBrowserCardEnding(state, action: PayloadAction) { + delete state.endingBrowserCards[action.payload]; }, suspendBrowserCard(state, action: PayloadAction<{ browserId: string; dataUrl: string }>) { @@ -959,6 +977,7 @@ const dashboardLayoutSlice = createSlice({ state.initialized = false; state.pendingFocusNoteId = null; state.suspendedBrowserCards = {}; + state.endingBrowserCards = {}; }, }, @@ -1068,6 +1087,8 @@ export const { removeBrowserCard, suspendBrowserCard, resumeBrowserCard, + markBrowserCardEnding, + cancelBrowserCardEnding, pasteBrowserCard, updateBrowserCardUrl, addBrowserTab, diff --git a/frontend/src/shared/ws/WebSocketManager.ts b/frontend/src/shared/ws/WebSocketManager.ts index cd228599..f18ba8bf 100644 --- a/frontend/src/shared/ws/WebSocketManager.ts +++ b/frontend/src/shared/ws/WebSocketManager.ts @@ -23,7 +23,7 @@ import { clearTurnLabel, } from '../state/agentsSlice'; import { streamStart, streamDelta, streamEnd, clearStreamingForSession } from '../state/streamingSlice'; -import { addBrowserCardFromBackend, removeBrowserCard, setBrowserCardPosition, setGlowingBrowserCards, GRID_GAP } from '../state/dashboardLayoutSlice'; +import { addBrowserCardFromBackend, markBrowserCardEnding, setBrowserCardPosition, setGlowingBrowserCards, GRID_GAP } from '../state/dashboardLayoutSlice'; import { upsertOutput } from '../state/outputsSlice'; import { getAuthToken } from '../config'; import { notifyAgentCompletion } from '../notifications'; @@ -500,6 +500,8 @@ class WebSocketManager { // sub-agent: the parent reuses the same browser_id for its next step, // so deleting on sub-agent completion strands BrowserAgent(browser_id) // on a dead card. 'stopped' skipped to allow inspect-after-manual-stop. + // Mark the card as ending instead of removing immediately so the card + // shows a fade + Keep pill; BrowserCard owns the 3s timer to remove. if ( session_id && (data.status === 'completed' || data.status === 'error') && @@ -508,7 +510,9 @@ class WebSocketManager { const browserCards = store.getState().dashboardLayout.browserCards; for (const card of Object.values(browserCards)) { if (card.spawned_by === session_id) { - store.dispatch(removeBrowserCard(card.browser_id)); + store.dispatch(markBrowserCardEnding({ + browserId: card.browser_id, status: data.status, + })); } } } @@ -723,11 +727,15 @@ class WebSocketManager { // Auto-delete browsers spawned by this agent when it finishes // normally or errors out. We intentionally skip 'stopped' , the // user may want to inspect the browser after manually stopping. + // Mark for removal so BrowserCard renders a fade + Keep pill; + // the card itself runs the 3s timer to dispatch removeBrowserCard. if (closedStatus === 'completed' || closedStatus === 'error') { const browserCards = store.getState().dashboardLayout.browserCards; for (const card of Object.values(browserCards)) { if (card.spawned_by === session_id) { - store.dispatch(removeBrowserCard(card.browser_id)); + store.dispatch(markBrowserCardEnding({ + browserId: card.browser_id, status: closedStatus, + })); } } }