[eric] browser: show a collapsed agent's browser as a frozen shot, not a capture that never lands

This commit is contained in:
ciregenz
2026-07-30 18:49:03 -07:00
parent 96451de87e
commit 1e2022d1cb
4 changed files with 95 additions and 36 deletions
@@ -43,7 +43,7 @@ import { agentCardMenuRows } from './agentCardMenuRows';
import { extractLatestTodos } from '../desktop/agentTodos';
import { extractLatestShowUi, extractPendingAskUi, freezeIfDone } from '@/app/pages/AgentChat/tool-ui/showUiPayload';
import { useDragEndBackstops } from '../hooks/interaction/useDragEndBackstops';
import { getWebview } from '@/shared/browserRegistry';
import { useBrowserPillShot } from '../desktop/useBrowserPillShot';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import { QuestionForm } from '@/app/pages/AgentChat/shell/ApprovalBar';
import AgentChat from '@/app/pages/AgentChat/AgentChat';
@@ -741,33 +741,7 @@ const AgentCard: React.FC<Props> = ({
}, [pillMode, session.messages, session.id, dispatch]);
// f7's collapsed state: a session's browser (spawned by it or docked into it) shows under the pill.
const spawnedBrowserId = useAppSelector((s) => {
for (const bc of Object.values(s.dashboardLayout.browserCards)) {
if (bc.spawned_by === session.id || bc.docked_to === session.id) return bc.browser_id;
}
return null;
});
const [browserShot, setBrowserShot] = useState<string | null>(null);
useEffect(() => {
if (!pillMode || pillArtifact || !spawnedBrowserId) {
setBrowserShot(null);
return undefined;
}
let cancelled = false;
const capture = (): void => {
const wv = getWebview(spawnedBrowserId);
const p = wv?.capturePage?.();
if (p && typeof (p as Promise<unknown>).then === 'function') {
(p as Promise<{ toDataURL(): string }>)
.then((img) => { if (!cancelled) setBrowserShot(img.toDataURL()); })
.catch(() => undefined);
}
};
capture();
// Refresh while the agent is driving so the shot tracks the page; parked cards keep the last frame.
const timer = pillRunning ? window.setInterval(capture, 5000) : null;
return () => { cancelled = true; if (timer) window.clearInterval(timer); };
}, [pillMode, pillArtifact, spawnedBrowserId, pillRunning]);
const browserShot = useBrowserPillShot(session.id, pillMode && !pillArtifact);
const noTransition = isDragging || isResizing || (isSelected && !!multiDragDelta);
@@ -46,7 +46,7 @@ import {
} from '@/shared/state/dashboardLayoutSlice';
import WindowControls from './WindowControls';
import { useTiledStyle, computeTiledStyle } from './tileZones';
import { saveMinimizedShot } from '../desktop/minimizedShots';
import { getMinimizedShot, saveMinimizedShot } from '../desktop/minimizedShots';
import { removeBrowserCardCleanly } from '@/shared/browserTeardown';
import { createSelector } from '@reduxjs/toolkit';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
@@ -58,6 +58,7 @@ import {
setActiveTab as setRegistryActiveTab,
registerPendingLoad,
wakePendingLoad,
hasDomReady,
type BrowserWebview,
} from '@/shared/browserRegistry';
import { setLastInteractedBrowser } from '@/shared/browserFocus';
@@ -66,7 +67,7 @@ import BrowserFindBar from './BrowserFindBar';
import { openCardContextMenu, isNativeMenuTarget } from '../desktop/openCardContextMenu';
import { browserCardMenuRows, browserTabMenuRows } from './browserCardMenuRows';
import { useBrowserActivity } from '@/shared/useBrowserActivity';
import { getActionLabel } from '@/shared/browserCommandHandler';
import { getActionLabel, isAnyBrowserBusy } from '@/shared/browserCommandHandler';
import { resolveInput, isGoogleSearch } from '@/shared/resolveUrl';
import BrowserAgentOverlay from './BrowserAgentOverlay';
import { useOverlayScrollPassthrough } from '../hooks/interaction/useOverlayScrollPassthrough';
@@ -83,6 +84,10 @@ import { useElementSelection } from '@/app/components/editor/ElementSelectionCon
type ResizeDir = 'n' | 's' | 'e' | 'w' | 'ne' | 'nw' | 'se' | 'sw';
// Pill-preview capture cadence: fast until the card has handed the pill a frame, slow upkeep after.
const PILL_SHOT_WARMUP_MS = 800;
const PILL_SHOT_REFRESH_MS = 5000;
const PILL_SHOT_WARMUP_MAX_MS = 8000;
const EDGE_THICKNESS = 6;
const CORNER_SIZE = 14;
const MIN_W = 400;
@@ -288,6 +293,13 @@ const BrowserCard: React.FC<Props> = ({
const suspendedSnap = useAppSelector((state) => state.dashboardLayout.suspendedBrowserCards[browserId]);
const endingState = useAppSelector((state) => state.dashboardLayout.endingBrowserCards[browserId]);
// An agent's browser is what its collapsed chat shows as a pill preview, so this card owes the pill a frozen frame.
const pillShotOwner = useAppSelector((state) => {
const card = state.dashboardLayout.browserCards[browserId];
return card?.docked_to ?? card?.spawned_by ?? null;
});
const [pillShotSettled, setPillShotSettled] = useState(() => !!getMinimizedShot(browserId));
// Arm the Windows webview crash-safety marker synchronously, before React commits the <webview> below. Cleared on dom-ready; a leftover marker next launch tells windowsWebviewEnabled() the mount crashed, so it falls back to the iframe. MUST skip parked cards: they render no webview, so dom-ready never fires and a stale marker reads as a phantom crash that locks Windows out of webviews.
if (isElectron && isWindows && !suspendedSnap) armWindowsWebviewPending();
@@ -417,7 +429,14 @@ const BrowserCard: React.FC<Props> = ({
cleanups.push(() => wv.removeEventListener('dom-ready', onReady));
}
const mirrorUrl = () => dispatch(updateBrowserTabUrl({ browserId, tabId, url: wv.getURL() }));
// Every guest sits at about:blank before its real load (lazy tabs never leave it); mirroring
// that would overwrite the tab's actual url, and a tab dragged into a fresh card then loads
// blank and stops looking like the page it is.
const mirrorUrl = () => {
const live = wv.getURL();
if (!live || live === 'about:blank') return;
dispatch(updateBrowserTabUrl({ browserId, tabId, url: live }));
};
const onNavigate = () => {
updateTabLocal(tabId, {
canGoBack: wv.canGoBack(),
@@ -1004,9 +1023,39 @@ const BrowserCard: React.FC<Props> = ({
: c.shadow.md;
const dockActive = !!dockRect && !dragging && !localResize && !tiledStyle && !keepAliveHidden && !isMinimized;
// Chat collapsed: its docked browser parks off-screen and lives on as the pill's live shot,
// instead of teleporting back to wherever it sat before docking.
const dockParked = !!dockedTo && !!dockParentCard && !dockParentExpanded && !dragging && !tiledStyle && !isMinimized && !keepAliveHidden;
// Chat collapsed: its docked browser parks off-screen and lives on as the pill's frozen shot,
// instead of teleporting back to wherever it sat before docking. The park waits for that shot:
// an off-screen guest never paints again, and capturePage on one never settles (Electron 42).
const wantsDockPark = !!dockedTo && !!dockParentCard && !dockParentExpanded && !dragging && !tiledStyle && !isMinimized && !keepAliveHidden;
const dockParked = wantsDockPark && pillShotSettled;
const pillShotPaintable = !!pillShotOwner && !dockParked && !isMinimized && !keepAliveHidden && !suspendedSnap;
useEffect(() => {
if (!pillShotPaintable) return undefined;
let cancelled = false;
let inFlight = false;
const freeze = (): void => {
// Capturing a webview an agent is mid-command on is the SharedImage-mailbox renderer crash.
if (inFlight || isAnyBrowserBusy()) return;
const wv = webviewMap.current.get(activeTabId);
// capturePage THROWS on a guest that hasn't reached dom-ready yet, and an uncaught one here kills the whole card tree.
if (!wv || !hasDomReady(wv)) return;
let shot: Promise<{ isEmpty: () => boolean; toDataURL: () => string }> | undefined;
try { shot = wv.capturePage(); } catch { return; }
if (!shot) return;
inFlight = true;
shot.then((img) => {
inFlight = false;
if (cancelled || img.isEmpty()) return;
saveMinimizedShot(browserId, img.toDataURL());
setPillShotSettled(true);
}, () => { inFlight = false; });
};
freeze();
const timer = window.setInterval(freeze, pillShotSettled ? PILL_SHOT_REFRESH_MS : PILL_SHOT_WARMUP_MS);
// A page that can never paint (dead guest, about:blank) must not camp on the canvas forever.
const giveUp = window.setTimeout(() => setPillShotSettled(true), PILL_SHOT_WARMUP_MAX_MS);
return () => { cancelled = true; window.clearInterval(timer); window.clearTimeout(giveUp); };
}, [pillShotPaintable, pillShotSettled, browserId, activeTabId]);
return (
<Box
@@ -1,6 +1,13 @@
/** Last visual of a card captured at minimize time; in-memory only, keyed by card id. */
/** Last visual of a card, frozen while it could still paint; in-memory only, keyed by card id. */
const shots = new Map<string, string>();
const CAP = 40;
const listeners = new Set<() => void>();
/** Readers that render a frozen shot subscribe here; a saved frame is otherwise invisible to them. */
export function subscribeMinimizedShots(fn: () => void): () => void {
listeners.add(fn);
return () => { listeners.delete(fn); };
}
export function saveMinimizedShot(cardId: string, dataUrl: string): void {
if (shots.size >= CAP && !shots.has(cardId)) {
@@ -8,6 +15,7 @@ export function saveMinimizedShot(cardId: string, dataUrl: string): void {
if (oldest) shots.delete(oldest);
}
shots.set(cardId, dataUrl);
listeners.forEach((fn) => fn());
}
export function getMinimizedShot(cardId: string): string | undefined {
@@ -15,5 +23,6 @@ export function getMinimizedShot(cardId: string): string | undefined {
}
export function dropMinimizedShot(cardId: string): void {
shots.delete(cardId);
if (!shots.delete(cardId)) return;
listeners.forEach((fn) => fn());
}
@@ -0,0 +1,27 @@
import { useSyncExternalStore } from 'react';
import { useAppSelector } from '@/shared/hooks';
import { getMinimizedShot, subscribeMinimizedShots } from './minimizedShots';
/**
* The browser preview under a collapsed agent pill: the session's browser (spawned by it or docked
* into it) as the frame BrowserCard froze while that card could still paint. Never captures here:
* a docked card parks off-screen the moment its chat collapses, and capturePage on an unpainted
* guest never settles (Electron 42), which is what used to leave the pill permanently blank.
*/
export function useBrowserPillShot(sessionId: string, active: boolean): string | null {
const browserId = useAppSelector((s) => {
for (const bc of Object.values(s.dashboardLayout.browserCards)) {
if (bc.spawned_by === sessionId || bc.docked_to === sessionId) return bc.browser_id;
}
return null;
});
const suspendedShot = useAppSelector(
(s) => (browserId ? s.dashboardLayout.suspendedBrowserCards[browserId]?.dataUrl || null : null),
);
const frozenShot = useSyncExternalStore(
subscribeMinimizedShots,
() => (browserId ? getMinimizedShot(browserId) ?? null : null),
);
if (!active || !browserId) return null;
return frozenShot ?? suspendedShot;
}