mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-06 01:37:43 +02:00
[eric] browser: show a collapsed agent's browser as a frozen shot, not a capture that never lands
This commit is contained in:
@@ -43,7 +43,7 @@ import { agentCardMenuRows } from './agentCardMenuRows';
|
||||
import { extractLatestTodos } from '../desktop/agentTodos';
|
||||
import { extractLatestShowUi, extractPendingAskUi, freezeIfDone } from '@/app/pages/AgentChat/tool-ui/showUiPayload';
|
||||
import { useDragEndBackstops } from '../hooks/interaction/useDragEndBackstops';
|
||||
import { getWebview } from '@/shared/browserRegistry';
|
||||
import { useBrowserPillShot } from '../desktop/useBrowserPillShot';
|
||||
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
|
||||
import { QuestionForm } from '@/app/pages/AgentChat/shell/ApprovalBar';
|
||||
import AgentChat from '@/app/pages/AgentChat/AgentChat';
|
||||
@@ -741,33 +741,7 @@ const AgentCard: React.FC<Props> = ({
|
||||
}, [pillMode, session.messages, session.id, dispatch]);
|
||||
|
||||
// f7's collapsed state: a session's browser (spawned by it or docked into it) shows under the pill.
|
||||
const spawnedBrowserId = useAppSelector((s) => {
|
||||
for (const bc of Object.values(s.dashboardLayout.browserCards)) {
|
||||
if (bc.spawned_by === session.id || bc.docked_to === session.id) return bc.browser_id;
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const [browserShot, setBrowserShot] = useState<string | null>(null);
|
||||
useEffect(() => {
|
||||
if (!pillMode || pillArtifact || !spawnedBrowserId) {
|
||||
setBrowserShot(null);
|
||||
return undefined;
|
||||
}
|
||||
let cancelled = false;
|
||||
const capture = (): void => {
|
||||
const wv = getWebview(spawnedBrowserId);
|
||||
const p = wv?.capturePage?.();
|
||||
if (p && typeof (p as Promise<unknown>).then === 'function') {
|
||||
(p as Promise<{ toDataURL(): string }>)
|
||||
.then((img) => { if (!cancelled) setBrowserShot(img.toDataURL()); })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
};
|
||||
capture();
|
||||
// Refresh while the agent is driving so the shot tracks the page; parked cards keep the last frame.
|
||||
const timer = pillRunning ? window.setInterval(capture, 5000) : null;
|
||||
return () => { cancelled = true; if (timer) window.clearInterval(timer); };
|
||||
}, [pillMode, pillArtifact, spawnedBrowserId, pillRunning]);
|
||||
const browserShot = useBrowserPillShot(session.id, pillMode && !pillArtifact);
|
||||
|
||||
const noTransition = isDragging || isResizing || (isSelected && !!multiDragDelta);
|
||||
|
||||
|
||||
@@ -46,7 +46,7 @@ import {
|
||||
} from '@/shared/state/dashboardLayoutSlice';
|
||||
import WindowControls from './WindowControls';
|
||||
import { useTiledStyle, computeTiledStyle } from './tileZones';
|
||||
import { saveMinimizedShot } from '../desktop/minimizedShots';
|
||||
import { getMinimizedShot, saveMinimizedShot } from '../desktop/minimizedShots';
|
||||
import { removeBrowserCardCleanly } from '@/shared/browserTeardown';
|
||||
import { createSelector } from '@reduxjs/toolkit';
|
||||
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
|
||||
@@ -58,6 +58,7 @@ import {
|
||||
setActiveTab as setRegistryActiveTab,
|
||||
registerPendingLoad,
|
||||
wakePendingLoad,
|
||||
hasDomReady,
|
||||
type BrowserWebview,
|
||||
} from '@/shared/browserRegistry';
|
||||
import { setLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
@@ -66,7 +67,7 @@ import BrowserFindBar from './BrowserFindBar';
|
||||
import { openCardContextMenu, isNativeMenuTarget } from '../desktop/openCardContextMenu';
|
||||
import { browserCardMenuRows, browserTabMenuRows } from './browserCardMenuRows';
|
||||
import { useBrowserActivity } from '@/shared/useBrowserActivity';
|
||||
import { getActionLabel } from '@/shared/browserCommandHandler';
|
||||
import { getActionLabel, isAnyBrowserBusy } from '@/shared/browserCommandHandler';
|
||||
import { resolveInput, isGoogleSearch } from '@/shared/resolveUrl';
|
||||
import BrowserAgentOverlay from './BrowserAgentOverlay';
|
||||
import { useOverlayScrollPassthrough } from '../hooks/interaction/useOverlayScrollPassthrough';
|
||||
@@ -83,6 +84,10 @@ import { useElementSelection } from '@/app/components/editor/ElementSelectionCon
|
||||
|
||||
type ResizeDir = 'n' | 's' | 'e' | 'w' | 'ne' | 'nw' | 'se' | 'sw';
|
||||
|
||||
// Pill-preview capture cadence: fast until the card has handed the pill a frame, slow upkeep after.
|
||||
const PILL_SHOT_WARMUP_MS = 800;
|
||||
const PILL_SHOT_REFRESH_MS = 5000;
|
||||
const PILL_SHOT_WARMUP_MAX_MS = 8000;
|
||||
const EDGE_THICKNESS = 6;
|
||||
const CORNER_SIZE = 14;
|
||||
const MIN_W = 400;
|
||||
@@ -288,6 +293,13 @@ const BrowserCard: React.FC<Props> = ({
|
||||
const suspendedSnap = useAppSelector((state) => state.dashboardLayout.suspendedBrowserCards[browserId]);
|
||||
const endingState = useAppSelector((state) => state.dashboardLayout.endingBrowserCards[browserId]);
|
||||
|
||||
// An agent's browser is what its collapsed chat shows as a pill preview, so this card owes the pill a frozen frame.
|
||||
const pillShotOwner = useAppSelector((state) => {
|
||||
const card = state.dashboardLayout.browserCards[browserId];
|
||||
return card?.docked_to ?? card?.spawned_by ?? null;
|
||||
});
|
||||
const [pillShotSettled, setPillShotSettled] = useState(() => !!getMinimizedShot(browserId));
|
||||
|
||||
// Arm the Windows webview crash-safety marker synchronously, before React commits the <webview> below. Cleared on dom-ready; a leftover marker next launch tells windowsWebviewEnabled() the mount crashed, so it falls back to the iframe. MUST skip parked cards: they render no webview, so dom-ready never fires and a stale marker reads as a phantom crash that locks Windows out of webviews.
|
||||
if (isElectron && isWindows && !suspendedSnap) armWindowsWebviewPending();
|
||||
|
||||
@@ -417,7 +429,14 @@ const BrowserCard: React.FC<Props> = ({
|
||||
cleanups.push(() => wv.removeEventListener('dom-ready', onReady));
|
||||
}
|
||||
|
||||
const mirrorUrl = () => dispatch(updateBrowserTabUrl({ browserId, tabId, url: wv.getURL() }));
|
||||
// Every guest sits at about:blank before its real load (lazy tabs never leave it); mirroring
|
||||
// that would overwrite the tab's actual url, and a tab dragged into a fresh card then loads
|
||||
// blank and stops looking like the page it is.
|
||||
const mirrorUrl = () => {
|
||||
const live = wv.getURL();
|
||||
if (!live || live === 'about:blank') return;
|
||||
dispatch(updateBrowserTabUrl({ browserId, tabId, url: live }));
|
||||
};
|
||||
const onNavigate = () => {
|
||||
updateTabLocal(tabId, {
|
||||
canGoBack: wv.canGoBack(),
|
||||
@@ -1004,9 +1023,39 @@ const BrowserCard: React.FC<Props> = ({
|
||||
: c.shadow.md;
|
||||
|
||||
const dockActive = !!dockRect && !dragging && !localResize && !tiledStyle && !keepAliveHidden && !isMinimized;
|
||||
// Chat collapsed: its docked browser parks off-screen and lives on as the pill's live shot,
|
||||
// instead of teleporting back to wherever it sat before docking.
|
||||
const dockParked = !!dockedTo && !!dockParentCard && !dockParentExpanded && !dragging && !tiledStyle && !isMinimized && !keepAliveHidden;
|
||||
// Chat collapsed: its docked browser parks off-screen and lives on as the pill's frozen shot,
|
||||
// instead of teleporting back to wherever it sat before docking. The park waits for that shot:
|
||||
// an off-screen guest never paints again, and capturePage on one never settles (Electron 42).
|
||||
const wantsDockPark = !!dockedTo && !!dockParentCard && !dockParentExpanded && !dragging && !tiledStyle && !isMinimized && !keepAliveHidden;
|
||||
const dockParked = wantsDockPark && pillShotSettled;
|
||||
const pillShotPaintable = !!pillShotOwner && !dockParked && !isMinimized && !keepAliveHidden && !suspendedSnap;
|
||||
useEffect(() => {
|
||||
if (!pillShotPaintable) return undefined;
|
||||
let cancelled = false;
|
||||
let inFlight = false;
|
||||
const freeze = (): void => {
|
||||
// Capturing a webview an agent is mid-command on is the SharedImage-mailbox renderer crash.
|
||||
if (inFlight || isAnyBrowserBusy()) return;
|
||||
const wv = webviewMap.current.get(activeTabId);
|
||||
// capturePage THROWS on a guest that hasn't reached dom-ready yet, and an uncaught one here kills the whole card tree.
|
||||
if (!wv || !hasDomReady(wv)) return;
|
||||
let shot: Promise<{ isEmpty: () => boolean; toDataURL: () => string }> | undefined;
|
||||
try { shot = wv.capturePage(); } catch { return; }
|
||||
if (!shot) return;
|
||||
inFlight = true;
|
||||
shot.then((img) => {
|
||||
inFlight = false;
|
||||
if (cancelled || img.isEmpty()) return;
|
||||
saveMinimizedShot(browserId, img.toDataURL());
|
||||
setPillShotSettled(true);
|
||||
}, () => { inFlight = false; });
|
||||
};
|
||||
freeze();
|
||||
const timer = window.setInterval(freeze, pillShotSettled ? PILL_SHOT_REFRESH_MS : PILL_SHOT_WARMUP_MS);
|
||||
// A page that can never paint (dead guest, about:blank) must not camp on the canvas forever.
|
||||
const giveUp = window.setTimeout(() => setPillShotSettled(true), PILL_SHOT_WARMUP_MAX_MS);
|
||||
return () => { cancelled = true; window.clearInterval(timer); window.clearTimeout(giveUp); };
|
||||
}, [pillShotPaintable, pillShotSettled, browserId, activeTabId]);
|
||||
|
||||
return (
|
||||
<Box
|
||||
|
||||
@@ -1,6 +1,13 @@
|
||||
/** Last visual of a card captured at minimize time; in-memory only, keyed by card id. */
|
||||
/** Last visual of a card, frozen while it could still paint; in-memory only, keyed by card id. */
|
||||
const shots = new Map<string, string>();
|
||||
const CAP = 40;
|
||||
const listeners = new Set<() => void>();
|
||||
|
||||
/** Readers that render a frozen shot subscribe here; a saved frame is otherwise invisible to them. */
|
||||
export function subscribeMinimizedShots(fn: () => void): () => void {
|
||||
listeners.add(fn);
|
||||
return () => { listeners.delete(fn); };
|
||||
}
|
||||
|
||||
export function saveMinimizedShot(cardId: string, dataUrl: string): void {
|
||||
if (shots.size >= CAP && !shots.has(cardId)) {
|
||||
@@ -8,6 +15,7 @@ export function saveMinimizedShot(cardId: string, dataUrl: string): void {
|
||||
if (oldest) shots.delete(oldest);
|
||||
}
|
||||
shots.set(cardId, dataUrl);
|
||||
listeners.forEach((fn) => fn());
|
||||
}
|
||||
|
||||
export function getMinimizedShot(cardId: string): string | undefined {
|
||||
@@ -15,5 +23,6 @@ export function getMinimizedShot(cardId: string): string | undefined {
|
||||
}
|
||||
|
||||
export function dropMinimizedShot(cardId: string): void {
|
||||
shots.delete(cardId);
|
||||
if (!shots.delete(cardId)) return;
|
||||
listeners.forEach((fn) => fn());
|
||||
}
|
||||
|
||||
@@ -0,0 +1,27 @@
|
||||
import { useSyncExternalStore } from 'react';
|
||||
import { useAppSelector } from '@/shared/hooks';
|
||||
import { getMinimizedShot, subscribeMinimizedShots } from './minimizedShots';
|
||||
|
||||
/**
|
||||
* The browser preview under a collapsed agent pill: the session's browser (spawned by it or docked
|
||||
* into it) as the frame BrowserCard froze while that card could still paint. Never captures here:
|
||||
* a docked card parks off-screen the moment its chat collapses, and capturePage on an unpainted
|
||||
* guest never settles (Electron 42), which is what used to leave the pill permanently blank.
|
||||
*/
|
||||
export function useBrowserPillShot(sessionId: string, active: boolean): string | null {
|
||||
const browserId = useAppSelector((s) => {
|
||||
for (const bc of Object.values(s.dashboardLayout.browserCards)) {
|
||||
if (bc.spawned_by === sessionId || bc.docked_to === sessionId) return bc.browser_id;
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const suspendedShot = useAppSelector(
|
||||
(s) => (browserId ? s.dashboardLayout.suspendedBrowserCards[browserId]?.dataUrl || null : null),
|
||||
);
|
||||
const frozenShot = useSyncExternalStore(
|
||||
subscribeMinimizedShots,
|
||||
() => (browserId ? getMinimizedShot(browserId) ?? null : null),
|
||||
);
|
||||
if (!active || !browserId) return null;
|
||||
return frozenShot ?? suspendedShot;
|
||||
}
|
||||
Reference in New Issue
Block a user