mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-08 10:47:44 +02:00
[eric] browser: Chrome-style suspend capsules (sessionStorage+scroll survive), last-visible-frame snapshots, 20s working grace
This commit is contained in:
@@ -51,6 +51,7 @@ import {
|
||||
type BrowserWebview,
|
||||
} from '@/shared/browserRegistry';
|
||||
import { setLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
import { registerCapsuleForRestore } from '@/shared/browserStateCapsule';
|
||||
import BrowserFindBar from './BrowserFindBar';
|
||||
import { useBrowserActivity } from '@/shared/useBrowserActivity';
|
||||
import { getActionLabel } from '@/shared/browserCommandHandler';
|
||||
@@ -313,6 +314,8 @@ const BrowserCard: React.FC<Props> = ({
|
||||
const doLoad = () => {
|
||||
// Reaching dom-ready proves the webview survived Chromium's commit phase (the historical Windows mount segfault). Clear the crash-safety marker.
|
||||
if (isWindows) markWindowsWebviewSurvived();
|
||||
// Registered BEFORE loadURL so the guest preload can sync-take it at document-start: a resumed tab gets its sessionStorage back Chrome-style instead of a logged-out reload. No-op when no capsule exists.
|
||||
registerCapsuleForRestore(wv, tabId);
|
||||
wv.loadURL(targetUrl).catch(() => {});
|
||||
try {
|
||||
(wv as any).setVisualZoomLevelLimits?.(1, 1);
|
||||
|
||||
@@ -7,8 +7,9 @@ import {
|
||||
type BrowserCardPosition,
|
||||
} from '@/shared/state/dashboardLayoutSlice';
|
||||
import { getWebview } from '@/shared/browserRegistry';
|
||||
import { getActivity } from '@/shared/browserCommandHandler';
|
||||
import { getActivity, isAnyBrowserBusy } from '@/shared/browserCommandHandler';
|
||||
import { isKeepAliveBrowser } from '@/shared/browserFocus';
|
||||
import { captureTabCapsule } from '@/shared/browserStateCapsule';
|
||||
|
||||
const isElectron = typeof navigator !== 'undefined' && navigator.userAgent.includes('Electron');
|
||||
|
||||
@@ -39,8 +40,21 @@ function cardIntersectsViewport(card: BrowserCardPosition, vp: Viewport, marginP
|
||||
return card.x < vx + vw && card.x + card.width > vx && card.y < vy + vh && card.y + card.height > vy;
|
||||
}
|
||||
|
||||
function sessionIsWorking(s: { status?: string } | undefined): boolean {
|
||||
return !!s && (s.status === 'running' || s.status === 'waiting_approval');
|
||||
// Grace after terminal so an agent whose status blips completed->running between back-to-back turns can't lose its browser in the gap.
|
||||
const WORKING_GRACE_MS = 20_000;
|
||||
const lastWorkingAt = new Map<string, number>();
|
||||
|
||||
function sessionIsWorking(s: { id?: string; status?: string } | undefined): boolean {
|
||||
if (!s) return false;
|
||||
if (s.status === 'running' || s.status === 'waiting_approval') {
|
||||
if (s.id) lastWorkingAt.set(s.id, Date.now());
|
||||
return true;
|
||||
}
|
||||
const t = s.id ? lastWorkingAt.get(s.id) : undefined;
|
||||
if (lastWorkingAt.size > 300) {
|
||||
for (const [k, v] of lastWorkingAt) if (Date.now() - v > WORKING_GRACE_MS) lastWorkingAt.delete(k);
|
||||
}
|
||||
return t !== undefined && Date.now() - t < WORKING_GRACE_MS;
|
||||
}
|
||||
|
||||
function agentNeedsLive(browserId: string, card: BrowserCardPosition): boolean {
|
||||
@@ -126,12 +140,13 @@ export function useWebviewSuspend(
|
||||
|
||||
const timer = setTimeout(async () => {
|
||||
const isSuspended = (id: string) => !!store.getState().dashboardLayout.suspendedBrowserCards[id];
|
||||
await refreshVisibleFrames(browserCards, isSuspended, vpRef.current);
|
||||
for (const [id, card] of Object.entries(browserCards)) {
|
||||
if (isSuspended(id)) continue;
|
||||
if (cardIntersectsViewport(card, vpRef.current, SUSPEND_MARGIN_PX)) continue;
|
||||
if (mustStayLive(id, card)) continue;
|
||||
// An empty dataUrl still suspends (placeholder renders): a card whose capture hangs/fails must not keep its renderer alive forever.
|
||||
const dataUrl = await captureCard(id, card);
|
||||
const dataUrl = await captureForSuspend(id, card);
|
||||
// The capture await yielded; conditions may have changed under us.
|
||||
if (cardIntersectsViewport(card, vpRef.current, SUSPEND_MARGIN_PX) || mustStayLive(id, card)) continue;
|
||||
dispatch(suspendBrowserCard({ browserId: id, dataUrl }));
|
||||
@@ -144,7 +159,7 @@ export function useWebviewSuspend(
|
||||
.sort((a, b) => distFromCenter(b[1], vpRef.current) - distFromCenter(a[1], vpRef.current));
|
||||
for (const [id, card] of candidates) {
|
||||
if (countLive() <= MAX_LIVE_WEBVIEWS) break;
|
||||
const dataUrl = await captureCard(id, card);
|
||||
const dataUrl = await captureForSuspend(id, card);
|
||||
if (mustStayLive(id, card)) continue;
|
||||
dispatch(suspendBrowserCard({ browserId: id, dataUrl }));
|
||||
}
|
||||
@@ -166,6 +181,49 @@ function distFromCenter(card: BrowserCardPosition, vp: Viewport): number {
|
||||
// capturePage on an already-off-screen webview can HANG forever (Electron 42/Viz stops producing frames for unpainted guests), and one hung await used to wedge the whole suspend pass, silently disabling suspension for every card. Bound it hard.
|
||||
const CAPTURE_TIMEOUT_MS = 1500;
|
||||
|
||||
// Last frame grabbed while each card was still VISIBLE: off-screen webviews can't produce frames, so this cache is what makes suspended cards show a real screenshot instead of the bare title placeholder.
|
||||
const lastFrames = new Map<string, { dataUrl: string; at: number }>();
|
||||
const FRAME_TTL_MS = 45_000;
|
||||
const FRAME_CACHE_CAP = 30;
|
||||
|
||||
function rememberFrame(id: string, dataUrl: string): void {
|
||||
lastFrames.set(id, { dataUrl, at: Date.now() });
|
||||
if (lastFrames.size > FRAME_CACHE_CAP) {
|
||||
const oldest = [...lastFrames.entries()].sort((a, b) => a[1].at - b[1].at)[0];
|
||||
if (oldest) lastFrames.delete(oldest[0]);
|
||||
}
|
||||
}
|
||||
|
||||
async function refreshVisibleFrames(
|
||||
cards: Record<string, BrowserCardPosition>,
|
||||
isSuspended: (id: string) => boolean,
|
||||
vp: Viewport,
|
||||
): Promise<void> {
|
||||
// Capturing while an agent drives a webview is the SharedImage-mailbox crash class; skip the whole pass.
|
||||
if (isAnyBrowserBusy()) return;
|
||||
for (const [id, card] of Object.entries(cards)) {
|
||||
if (isSuspended(id)) continue;
|
||||
if (!cardIntersectsViewport(card, vp, 0)) continue;
|
||||
const prev = lastFrames.get(id);
|
||||
if (prev && Date.now() - prev.at < FRAME_TTL_MS) continue;
|
||||
const dataUrl = await captureCard(id, card);
|
||||
if (dataUrl) rememberFrame(id, dataUrl);
|
||||
}
|
||||
}
|
||||
|
||||
async function captureForSuspend(id: string, card: BrowserCardPosition): Promise<string> {
|
||||
// Chrome-style state capsules first (sessionStorage + scroll per tab), so resume restores logins instead of wiping them; JS still runs off-screen even when frames don't.
|
||||
for (const tab of card.tabs ?? []) {
|
||||
await captureTabCapsule(getWebview(id, tab.id), tab.id);
|
||||
}
|
||||
const live = await captureCard(id, card);
|
||||
if (live) {
|
||||
rememberFrame(id, live);
|
||||
return live;
|
||||
}
|
||||
return lastFrames.get(id)?.dataUrl ?? '';
|
||||
}
|
||||
|
||||
async function captureCard(id: string, card: BrowserCardPosition): Promise<string> {
|
||||
const wv = getWebview(id, card.activeTabId);
|
||||
if (!wv) return '';
|
||||
|
||||
@@ -0,0 +1,58 @@
|
||||
import type { BrowserWebview } from './browserRegistry';
|
||||
|
||||
// Capsules carry site session tokens: in-memory ONLY, never redux, never disk, never logged.
|
||||
export interface TabCapsule {
|
||||
ss: Record<string, string>;
|
||||
sx: number;
|
||||
sy: number;
|
||||
origin: string;
|
||||
capturedAt: number;
|
||||
}
|
||||
|
||||
const CAPSULE_CAP = 100;
|
||||
const CAPTURE_TIMEOUT_MS = 800;
|
||||
const capsules = new Map<string, TabCapsule>();
|
||||
|
||||
interface OpenswarmCapsuleBridge {
|
||||
setSessionCapsule?: (wcId: number, capsule: TabCapsule) => void;
|
||||
}
|
||||
|
||||
/** Snapshot a tab's sessionStorage + scroll before its webview unmounts, so resume can restore it Chrome-style instead of logging the user out. */
|
||||
export async function captureTabCapsule(wv: BrowserWebview | null | undefined, tabId: string): Promise<void> {
|
||||
if (!wv) return;
|
||||
try {
|
||||
const raw = await Promise.race([
|
||||
wv.executeJavaScript(
|
||||
`(() => { const ss = {}; for (let i = 0; i < sessionStorage.length; i++) { const k = sessionStorage.key(i); if (k !== null) ss[k] = sessionStorage.getItem(k); } return JSON.stringify({ ss, sx: window.scrollX, sy: window.scrollY, origin: location.origin }); })()`,
|
||||
),
|
||||
new Promise<null>((resolve) => setTimeout(() => resolve(null), CAPTURE_TIMEOUT_MS)),
|
||||
]);
|
||||
if (typeof raw !== 'string') return;
|
||||
const parsed = JSON.parse(raw) as { ss: Record<string, string>; sx: number; sy: number; origin: string };
|
||||
if (!parsed.origin || !parsed.origin.startsWith('http')) return;
|
||||
capsules.set(tabId, { ss: parsed.ss, sx: parsed.sx, sy: parsed.sy, origin: parsed.origin, capturedAt: Date.now() });
|
||||
if (capsules.size > CAPSULE_CAP) {
|
||||
const oldest = [...capsules.entries()].sort((a, b) => a[1].capturedAt - b[1].capturedAt)[0];
|
||||
if (oldest) capsules.delete(oldest[0]);
|
||||
}
|
||||
} catch {
|
||||
// A dead/navigating webview just means no capsule; resume falls back to a plain reload.
|
||||
}
|
||||
}
|
||||
|
||||
/** Hand a resumed tab's capsule to the main process, keyed by the fresh webContents id, BEFORE loadURL fires; the guest preload sync-takes it at document-start so page scripts see restored state. */
|
||||
export function registerCapsuleForRestore(wv: BrowserWebview, tabId: string): void {
|
||||
const capsule = capsules.get(tabId);
|
||||
if (!capsule) return;
|
||||
const bridge = (window as unknown as { openswarm?: OpenswarmCapsuleBridge }).openswarm;
|
||||
if (!bridge?.setSessionCapsule) return;
|
||||
try {
|
||||
bridge.setSessionCapsule(wv.getWebContentsId(), capsule);
|
||||
} catch {
|
||||
// Bridge unavailable (iframe fallback / non-Electron): plain reload, same as before capsules existed.
|
||||
}
|
||||
}
|
||||
|
||||
export function hasCapsule(tabId: string): boolean {
|
||||
return capsules.has(tabId);
|
||||
}
|
||||
Reference in New Issue
Block a user