diff --git a/electron/main.js b/electron/main.js index 957221da..d99d0ec4 100644 --- a/electron/main.js +++ b/electron/main.js @@ -2799,6 +2799,31 @@ async function readPartitionCookies(domain) { } ipcMain.handle('get-partition-cookies', (_e, domain) => readPartitionCookies(domain)); +// Suspend/resume state capsules: the app renderer stages a resumed webview's sessionStorage snapshot here (keyed by that guest's webContents id) right before loadURL; the guest preload sync-takes it at document-start with an origin match, so page scripts see restored state and logins survive suspension. In-memory only, single-shot, short TTL; a guest can only ever take its OWN capsule. +const pendingSessionCapsules = new Map(); +const SESSION_CAPSULE_TTL_MS = 2 * 60 * 1000; +ipcMain.on('browser-capsule-set', (event, wcId, capsule) => { + // Only the app window may stage capsules; a compromised guest must not be able to seed storage into another guest. + if (!mainWindow || event.sender !== mainWindow.webContents) return; + if (typeof wcId !== 'number' || !capsule || typeof capsule.origin !== 'string' || typeof capsule.ss !== 'object') return; + pendingSessionCapsules.set(wcId, { capsule, expiresAt: Date.now() + SESSION_CAPSULE_TTL_MS }); +}); +ipcMain.on('browser-capsule-take', (event, origin) => { + const entry = pendingSessionCapsules.get(event.sender.id); + if (!entry || entry.expiresAt < Date.now()) { + if (entry) pendingSessionCapsules.delete(event.sender.id); + event.returnValue = null; + return; + } + // Origin-gated take: about:blank and cross-origin redirects leave the capsule staged for the real page (until TTL). + if (entry.capsule.origin !== origin) { + event.returnValue = null; + return; + } + pendingSessionCapsules.delete(event.sender.id); + event.returnValue = entry.capsule; +}); + // The renderer relays cookie reads for the session-borrow bridge, but macOS throttles it when the // window is backgrounded, so those reads intermittently time out. Main never throttles: hold our own // socket to the backend and answer get_session_cookies here. Cookie reads only; the renderer still diff --git a/electron/preload.js b/electron/preload.js index 9c3854fb..93a1e8b2 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -62,6 +62,8 @@ contextBridge.exposeInMainWorld('openswarm', { connectSlack: () => ipcRenderer.invoke('connect-slack'), // Hands a vetted social platform's partition cookies to its session-backed MCP shim (allowlisted domains only, gated again in the main process). getPartitionCookies: (domain) => ipcRenderer.invoke('get-partition-cookies', domain), + // Suspend/resume state capsule: stages a resumed webview's sessionStorage snapshot in main (keyed by webContents id, short TTL) so the guest preload can sync-take it at document-start. Fire-and-forget; main validates the sender. + setSessionCapsule: (wcId, capsule) => ipcRenderer.send('browser-capsule-set', wcId, capsule), sendCdpCommand: (wcId, method, params, sessionId) => ipcRenderer.invoke('send-cdp-command', wcId, method, params, sessionId), cdpDetachClean: (wcId) => ipcRenderer.invoke('cdp-detach-clean', wcId), cdpCacheSet: (wcId, indexMap) => ipcRenderer.invoke('cdp-cache-set', wcId, indexMap), diff --git a/electron/webview-preload.js b/electron/webview-preload.js index bef6bd47..cbb51b7d 100644 --- a/electron/webview-preload.js +++ b/electron/webview-preload.js @@ -10,6 +10,23 @@ // this webview. Surfaces via main.js's console-message listener. try { console.warn('[openswarm:webview-preload] loaded for', window.location.href); } catch (_) {} +// Chrome-style suspend/resume: sync-take a pending sessionStorage capsule (staged by the host right before this load) at document-start, so page scripts wake to their old state and logins survive suspension. Origin-matched in main; null for every ordinary navigation. +(function restoreSessionCapsule() { + try { + const { ipcRenderer } = require('electron'); + const cap = ipcRenderer.sendSync('browser-capsule-take', window.location.origin); + if (!cap || !cap.ss) return; + for (const k of Object.keys(cap.ss)) { + try { sessionStorage.setItem(k, cap.ss[k]); } catch (_) {} + } + if (cap.sx || cap.sy) { + window.addEventListener('load', () => { + setTimeout(() => { try { window.scrollTo(cap.sx, cap.sy); } catch (_) {} }, 80); + }); + } + } catch (_) {} +})(); + // Hide webdriver flag Object.defineProperty(navigator, 'webdriver', { get: () => false, diff --git a/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx b/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx index f8d8dd3f..7833aaf2 100644 --- a/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx +++ b/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx @@ -51,6 +51,7 @@ import { type BrowserWebview, } from '@/shared/browserRegistry'; import { setLastInteractedBrowser } from '@/shared/browserFocus'; +import { registerCapsuleForRestore } from '@/shared/browserStateCapsule'; import BrowserFindBar from './BrowserFindBar'; import { useBrowserActivity } from '@/shared/useBrowserActivity'; import { getActionLabel } from '@/shared/browserCommandHandler'; @@ -313,6 +314,8 @@ const BrowserCard: React.FC = ({ const doLoad = () => { // Reaching dom-ready proves the webview survived Chromium's commit phase (the historical Windows mount segfault). Clear the crash-safety marker. if (isWindows) markWindowsWebviewSurvived(); + // Registered BEFORE loadURL so the guest preload can sync-take it at document-start: a resumed tab gets its sessionStorage back Chrome-style instead of a logged-out reload. No-op when no capsule exists. + registerCapsuleForRestore(wv, tabId); wv.loadURL(targetUrl).catch(() => {}); try { (wv as any).setVisualZoomLevelLimits?.(1, 1); diff --git a/frontend/src/app/pages/Dashboard/hooks/interaction/useWebviewSuspend.ts b/frontend/src/app/pages/Dashboard/hooks/interaction/useWebviewSuspend.ts index a9ed83cb..ebeba010 100644 --- a/frontend/src/app/pages/Dashboard/hooks/interaction/useWebviewSuspend.ts +++ b/frontend/src/app/pages/Dashboard/hooks/interaction/useWebviewSuspend.ts @@ -7,8 +7,9 @@ import { type BrowserCardPosition, } from '@/shared/state/dashboardLayoutSlice'; import { getWebview } from '@/shared/browserRegistry'; -import { getActivity } from '@/shared/browserCommandHandler'; +import { getActivity, isAnyBrowserBusy } from '@/shared/browserCommandHandler'; import { isKeepAliveBrowser } from '@/shared/browserFocus'; +import { captureTabCapsule } from '@/shared/browserStateCapsule'; const isElectron = typeof navigator !== 'undefined' && navigator.userAgent.includes('Electron'); @@ -39,8 +40,21 @@ function cardIntersectsViewport(card: BrowserCardPosition, vp: Viewport, marginP return card.x < vx + vw && card.x + card.width > vx && card.y < vy + vh && card.y + card.height > vy; } -function sessionIsWorking(s: { status?: string } | undefined): boolean { - return !!s && (s.status === 'running' || s.status === 'waiting_approval'); +// Grace after terminal so an agent whose status blips completed->running between back-to-back turns can't lose its browser in the gap. +const WORKING_GRACE_MS = 20_000; +const lastWorkingAt = new Map(); + +function sessionIsWorking(s: { id?: string; status?: string } | undefined): boolean { + if (!s) return false; + if (s.status === 'running' || s.status === 'waiting_approval') { + if (s.id) lastWorkingAt.set(s.id, Date.now()); + return true; + } + const t = s.id ? lastWorkingAt.get(s.id) : undefined; + if (lastWorkingAt.size > 300) { + for (const [k, v] of lastWorkingAt) if (Date.now() - v > WORKING_GRACE_MS) lastWorkingAt.delete(k); + } + return t !== undefined && Date.now() - t < WORKING_GRACE_MS; } function agentNeedsLive(browserId: string, card: BrowserCardPosition): boolean { @@ -126,12 +140,13 @@ export function useWebviewSuspend( const timer = setTimeout(async () => { const isSuspended = (id: string) => !!store.getState().dashboardLayout.suspendedBrowserCards[id]; + await refreshVisibleFrames(browserCards, isSuspended, vpRef.current); for (const [id, card] of Object.entries(browserCards)) { if (isSuspended(id)) continue; if (cardIntersectsViewport(card, vpRef.current, SUSPEND_MARGIN_PX)) continue; if (mustStayLive(id, card)) continue; // An empty dataUrl still suspends (placeholder renders): a card whose capture hangs/fails must not keep its renderer alive forever. - const dataUrl = await captureCard(id, card); + const dataUrl = await captureForSuspend(id, card); // The capture await yielded; conditions may have changed under us. if (cardIntersectsViewport(card, vpRef.current, SUSPEND_MARGIN_PX) || mustStayLive(id, card)) continue; dispatch(suspendBrowserCard({ browserId: id, dataUrl })); @@ -144,7 +159,7 @@ export function useWebviewSuspend( .sort((a, b) => distFromCenter(b[1], vpRef.current) - distFromCenter(a[1], vpRef.current)); for (const [id, card] of candidates) { if (countLive() <= MAX_LIVE_WEBVIEWS) break; - const dataUrl = await captureCard(id, card); + const dataUrl = await captureForSuspend(id, card); if (mustStayLive(id, card)) continue; dispatch(suspendBrowserCard({ browserId: id, dataUrl })); } @@ -166,6 +181,49 @@ function distFromCenter(card: BrowserCardPosition, vp: Viewport): number { // capturePage on an already-off-screen webview can HANG forever (Electron 42/Viz stops producing frames for unpainted guests), and one hung await used to wedge the whole suspend pass, silently disabling suspension for every card. Bound it hard. const CAPTURE_TIMEOUT_MS = 1500; +// Last frame grabbed while each card was still VISIBLE: off-screen webviews can't produce frames, so this cache is what makes suspended cards show a real screenshot instead of the bare title placeholder. +const lastFrames = new Map(); +const FRAME_TTL_MS = 45_000; +const FRAME_CACHE_CAP = 30; + +function rememberFrame(id: string, dataUrl: string): void { + lastFrames.set(id, { dataUrl, at: Date.now() }); + if (lastFrames.size > FRAME_CACHE_CAP) { + const oldest = [...lastFrames.entries()].sort((a, b) => a[1].at - b[1].at)[0]; + if (oldest) lastFrames.delete(oldest[0]); + } +} + +async function refreshVisibleFrames( + cards: Record, + isSuspended: (id: string) => boolean, + vp: Viewport, +): Promise { + // Capturing while an agent drives a webview is the SharedImage-mailbox crash class; skip the whole pass. + if (isAnyBrowserBusy()) return; + for (const [id, card] of Object.entries(cards)) { + if (isSuspended(id)) continue; + if (!cardIntersectsViewport(card, vp, 0)) continue; + const prev = lastFrames.get(id); + if (prev && Date.now() - prev.at < FRAME_TTL_MS) continue; + const dataUrl = await captureCard(id, card); + if (dataUrl) rememberFrame(id, dataUrl); + } +} + +async function captureForSuspend(id: string, card: BrowserCardPosition): Promise { + // Chrome-style state capsules first (sessionStorage + scroll per tab), so resume restores logins instead of wiping them; JS still runs off-screen even when frames don't. + for (const tab of card.tabs ?? []) { + await captureTabCapsule(getWebview(id, tab.id), tab.id); + } + const live = await captureCard(id, card); + if (live) { + rememberFrame(id, live); + return live; + } + return lastFrames.get(id)?.dataUrl ?? ''; +} + async function captureCard(id: string, card: BrowserCardPosition): Promise { const wv = getWebview(id, card.activeTabId); if (!wv) return ''; diff --git a/frontend/src/shared/browserStateCapsule.ts b/frontend/src/shared/browserStateCapsule.ts new file mode 100644 index 00000000..c2ac4323 --- /dev/null +++ b/frontend/src/shared/browserStateCapsule.ts @@ -0,0 +1,58 @@ +import type { BrowserWebview } from './browserRegistry'; + +// Capsules carry site session tokens: in-memory ONLY, never redux, never disk, never logged. +export interface TabCapsule { + ss: Record; + sx: number; + sy: number; + origin: string; + capturedAt: number; +} + +const CAPSULE_CAP = 100; +const CAPTURE_TIMEOUT_MS = 800; +const capsules = new Map(); + +interface OpenswarmCapsuleBridge { + setSessionCapsule?: (wcId: number, capsule: TabCapsule) => void; +} + +/** Snapshot a tab's sessionStorage + scroll before its webview unmounts, so resume can restore it Chrome-style instead of logging the user out. */ +export async function captureTabCapsule(wv: BrowserWebview | null | undefined, tabId: string): Promise { + if (!wv) return; + try { + const raw = await Promise.race([ + wv.executeJavaScript( + `(() => { const ss = {}; for (let i = 0; i < sessionStorage.length; i++) { const k = sessionStorage.key(i); if (k !== null) ss[k] = sessionStorage.getItem(k); } return JSON.stringify({ ss, sx: window.scrollX, sy: window.scrollY, origin: location.origin }); })()`, + ), + new Promise((resolve) => setTimeout(() => resolve(null), CAPTURE_TIMEOUT_MS)), + ]); + if (typeof raw !== 'string') return; + const parsed = JSON.parse(raw) as { ss: Record; sx: number; sy: number; origin: string }; + if (!parsed.origin || !parsed.origin.startsWith('http')) return; + capsules.set(tabId, { ss: parsed.ss, sx: parsed.sx, sy: parsed.sy, origin: parsed.origin, capturedAt: Date.now() }); + if (capsules.size > CAPSULE_CAP) { + const oldest = [...capsules.entries()].sort((a, b) => a[1].capturedAt - b[1].capturedAt)[0]; + if (oldest) capsules.delete(oldest[0]); + } + } catch { + // A dead/navigating webview just means no capsule; resume falls back to a plain reload. + } +} + +/** Hand a resumed tab's capsule to the main process, keyed by the fresh webContents id, BEFORE loadURL fires; the guest preload sync-takes it at document-start so page scripts see restored state. */ +export function registerCapsuleForRestore(wv: BrowserWebview, tabId: string): void { + const capsule = capsules.get(tabId); + if (!capsule) return; + const bridge = (window as unknown as { openswarm?: OpenswarmCapsuleBridge }).openswarm; + if (!bridge?.setSessionCapsule) return; + try { + bridge.setSessionCapsule(wv.getWebContentsId(), capsule); + } catch { + // Bridge unavailable (iframe fallback / non-Electron): plain reload, same as before capsules existed. + } +} + +export function hasCapsule(tabId: string): boolean { + return capsules.has(tabId); +}