[eric] browser: reap finished-agent browsers, close an agent's browsers with it; dictation confirms the guest target before claiming success; agent-driven focus steal restores the user's caret; window size persists across a crash-recovery recreate

This commit is contained in:
ciregenz
2026-08-11 14:53:16 -07:00
parent 5516c8d808
commit f598becfd2
12 changed files with 170 additions and 19 deletions
+44 -3
View File
@@ -1429,12 +1429,47 @@ async function loadAuthToken() {
console.warn(`[auth] FAILED to load auth token from ${tokenPath} after 2s — WS/HTTP will be rejected`);
}
function windowBoundsPath() {
try { return path.join(app.getPath('userData'), 'window-bounds.json'); } catch { return null; }
}
// The user's manual window size is persisted and restored, so a crash-recovery recreateMainWindow no
// longer rebuilds at the default 1400x900 and drops the size they dragged out to (ENG-253). Only
// honored when the saved rect still fits on some display, so unplugging a monitor can't hide the app.
function loadSavedBounds() {
const p = windowBoundsPath();
if (!p) return null;
try {
const b = JSON.parse(fs.readFileSync(p, 'utf8'));
if (!b || !Number.isFinite(b.width) || !Number.isFinite(b.height) || b.width < 800 || b.height < 600) return null;
const displays = require('electron').screen.getAllDisplays();
const onScreen = displays.some((d) => {
const wa = d.workArea;
return b.x != null && b.y != null && b.x < wa.x + wa.width && b.x + 200 > wa.x && b.y < wa.y + wa.height && b.y + 100 > wa.y;
});
return onScreen ? b : { width: b.width, height: b.height };
} catch { return null; }
}
let boundsSaveTimer = null;
function persistWindowBounds(win) {
if (!win || win.isDestroyed() || win.isFullScreen() || win.isMinimized()) return;
const p = windowBoundsPath();
if (!p) return;
if (boundsSaveTimer) clearTimeout(boundsSaveTimer);
boundsSaveTimer = setTimeout(() => {
try { fs.writeFileSync(p, JSON.stringify(win.getBounds())); } catch (_) {}
}, 400);
}
function createWindow() {
isCreatingMainWindow = true;
console.log('[diag][main] createWindow start');
const saved = loadSavedBounds();
mainWindow = new BrowserWindow({
width: 1400,
height: 900,
width: (saved && saved.width) || 1400,
height: (saved && saved.height) || 900,
...(saved && saved.x != null ? { x: saved.x, y: saved.y } : {}),
minWidth: 800,
minHeight: 600,
title: 'OpenSwarm',
@@ -1684,6 +1719,10 @@ function createWindow() {
mainWindow.on('blur', () => sendFocusEvent('blur'));
mainWindow.on('focus', () => sendFocusEvent('focus'));
// Remember the size/position the user set, so nothing (a crash-recovery recreate especially) drops it (ENG-253).
mainWindow.on('resize', () => persistWindowBounds(mainWindow));
mainWindow.on('move', () => persistWindowBounds(mainWindow));
// Forward renderer console output to main stderr so packaged-build diagnostics survive without DevTools open.
mainWindow.webContents.on('console-message', (_e, level, message, line, sourceId) => {
const tag = ['LOG', 'INFO', 'WARN', 'ERROR'][level] || 'LOG';
@@ -1717,7 +1756,9 @@ function createWindow() {
// Why setImmediate for the destroy:
// - We're INSIDE the old window's render-process-gone handler. Destroying its BrowserWindow from inside its own event callback works in current Electron but is fragile across version bumps; deferring one tick is free insurance.
function recreateMainWindow() {
console.log('[diag][main] recreateMainWindow START, crashesInWindow=', rendererCrashTimes.length);
// Named on purpose: this is the prime suspect for "the window resized itself" (ENG-253); the diag
// line lets a report say whether a silent recreate is what dropped the user's window size.
console.log('[diag][main] recreateMainWindow START (window will restore saved bounds), crashesInWindow=', rendererCrashTimes.length);
const oldWindow = mainWindow;
mainWindowReady = false;
try {
@@ -4,6 +4,7 @@ import { ElementSelectionProvider } from '@/app/components/editor/ElementSelecti
import { useDomElementSelector } from '@/app/components/editor/useDomElementSelector';
import { useDashboardActive } from '@/shared/hooks/useDashboardActive';
import { useDashboardController } from './hooks/state/useDashboardController';
import { useOrphanBrowserReaper } from './hooks/useOrphanBrowserReaper';
import DashboardCanvas from './canvas/DashboardCanvas';
const DashboardSelectionOverlay: React.FC = () => {
@@ -20,6 +21,7 @@ interface DashboardProps {
const DashboardInner: React.FC<DashboardProps> = ({ dashboardId, isActive = true }) => {
const controller = useDashboardController(dashboardId, isActive);
useOrphanBrowserReaper();
return (
<>
<DashboardSelectionOverlay />
@@ -32,10 +32,9 @@ import {
clearGlowingAgentCard,
removeCard,
recordClosedCard,
setBrowserDocked,
keepBrowserCardOpen,
} from '@/shared/state/dashboardLayoutSlice';
import { store } from '@/shared/state/store';
import { removeBrowserCardCleanly } from '@/shared/browserTeardown';
import WindowControls, { ARC_CHIP_SX } from './WindowControls';
import { useTiledCard } from './useTiledCard';
import { useCardTiling } from './useCardTiling';
@@ -672,12 +671,14 @@ const AgentCard: React.FC<Props> = ({
if (linkedWorkflowSidecarId) {
dispatch(setCardSidecar({ workflowId: linkedWorkflowSidecarId, sessionId: null, kind: null }));
}
// Closing the CHAT must not take the browser with it: its browser undocks to the canvas and is pinned open (keep_open exempts it from the finished-agent auto-reap). The user closes it separately.
// Closing the chat takes its browsers with it (Haik's ENG-249: the old undock-and-pin behavior
// read as the browser "popping open on its own", forcing a second cleanup every time). Each one
// lands in recently-closed first, so Cmd+Shift+T brings it back if it was wanted.
if (!glowEntry) {
for (const bc of Object.values(store.getState().dashboardLayout.browserCards)) {
if (bc.docked_to !== session.id && bc.spawned_by !== session.id) continue;
if (bc.docked_to === session.id) dispatch(setBrowserDocked({ browserId: bc.browser_id, dockedTo: null }));
dispatch(keepBrowserCardOpen(bc.browser_id));
dispatch(recordClosedCard({ kind: 'browser', id: bc.browser_id }));
removeBrowserCardCleanly(bc.browser_id, dispatch);
}
}
// Record for Cmd+Shift+T BEFORE removeCard wipes the position, but only on a real close (the glow branch just clears a tether, it doesn't close the session).
@@ -66,6 +66,7 @@ import {
import { captureBrowserShot } from '@/shared/captureBrowserShot';
import { setLastInteractedBrowser } from '@/shared/browserFocus';
import { isAgentDrivenBrowser } from '@/shared/isAgentDrivenBrowser';
import { restoreLastUserFocus } from '@/shared/lastUserFocus';
import { registerCapsuleForRestore } from '@/shared/browserStateCapsule';
import BrowserFindBar from './BrowserFindBar';
import { openCardContextMenu, isNativeMenuTarget } from '../desktop/openCardContextMenu';
@@ -681,8 +682,15 @@ const BrowserCard: React.FC<Props> = ({
wv.addEventListener('render-process-gone', onProcessGone as any);
wv.addEventListener('crashed', onProcessGone as any);
wv.addEventListener('did-fail-load', onDidFailLoad as any);
// While an agent drives this browser, the guest focusing an input hands the host <webview>
// element focus, yanking the user out of whatever they were typing (ENG-252, 4x-reported). The
// agent drives via the webContents transport, not host DOM focus, so restoring the user's caret
// here does not disturb the agent. Only fires for agent-driven cards; a user's own click is untouched.
const onWvFocus = () => { if (isAgentDrivenBrowser(browserId)) restoreLastUserFocus(); };
wv.addEventListener('focus', onWvFocus);
cleanups.push(() => {
wv.removeEventListener('focus', onWvFocus);
unregisterWebview(browserId, tabId);
wv.removeEventListener('did-navigate', onNavigate);
wv.removeEventListener('did-navigate-in-page', onNavigate);
@@ -30,8 +30,12 @@ export async function pasteClipboardCards({ dispatch, dashboardId, expandedSessi
selection.deselectAll();
const newSelection = new Map<string, CardType>();
// Old agent id -> new pasted id, so a browser copied alongside its agent re-docks under the copy (ENG-250).
const agentRemap = new Map<string, string>();
// Agents first so their remap exists before their browsers are pasted, whatever the copy order.
const ordered = [...copied].sort((a, b) => (a.type === 'agent' ? -1 : 0) - (b.type === 'agent' ? -1 : 0));
for (const card of copied) {
for (const card of ordered) {
const px = at ? at.x + (card.x - anchorX) : card.x + PASTE_OFFSET;
const py = at ? at.y + (card.y - anchorY) : card.y - PASTE_OFFSET;
@@ -39,6 +43,7 @@ export async function pasteClipboardCards({ dispatch, dashboardId, expandedSessi
const action = await dispatch(duplicateSession({ sessionId: card.id, dashboardId }));
if (duplicateSession.fulfilled.match(action)) {
const newId = action.payload.id;
agentRemap.set(card.id, newId);
dispatch(placeCard({ sessionId: newId, x: px, y: py, width: card.width, height: card.height, expandedSessionIds }));
if (card.expanded) dispatch(expandSession(newId));
newSelection.set(newId, 'agent');
@@ -55,9 +60,11 @@ export async function pasteClipboardCards({ dispatch, dashboardId, expandedSessi
newSelection.set(pastedKey, 'view');
} else if (card.type === 'browser') {
const browserId = `browser-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
const originalOwner = card.meta.spawnedBy as string | undefined;
const dockTo = originalOwner ? agentRemap.get(originalOwner) ?? null : null;
dispatch(pasteBrowserCard({
id: browserId, tabs: card.meta.tabs || [], url: card.meta.url || '',
x: px, y: py, width: card.width, height: card.height,
x: px, y: py, width: card.width, height: card.height, dockTo,
}));
newSelection.set(browserId, 'browser');
}
@@ -83,7 +83,8 @@ export function useDashboardClipboard({
const title = activeTab?.title || 'Browser';
copied.push({
type, id, name: title,
meta: { name: title, url: activeTab?.url || bc.url, tabs: bc.tabs },
// Carry the owning session so a group paste can re-dock the browser under the NEW agent (ENG-250).
meta: { name: title, url: activeTab?.url || bc.url, tabs: bc.tabs, spawnedBy: bc.docked_to ?? bc.spawned_by ?? null },
x: bc.x, y: bc.y, width: bc.width, height: bc.height,
});
names.push(title);
@@ -0,0 +1,36 @@
import { useEffect } from 'react';
import { useAppDispatch } from '@/shared/hooks';
import { store } from '@/shared/state/store';
import { markBrowserCardEnding } from '@/shared/state/dashboardLayoutSlice';
// The event-driven despawn (WebSocketManager marks a finished agent's browsers ending) misses any
// terminal event the renderer never saw: a reload mid-run, a workflow finishing while the app was
// closed. Those cards then live forever, which is Haik's 20-browser pile-up (ENG-248). This sweep is
// the belt: anything owned by a session the STORE can see is terminal gets the same fade + Keep pill
// the event path uses. A session missing from the store entirely is left alone on purpose; before
// sessions load, "missing" means "not fetched yet", and axing on that would kill live cards at boot.
const SWEEP_MS = 45_000;
const FIRST_SWEEP_MS = 7_000;
export function useOrphanBrowserReaper(): void {
const dispatch = useAppDispatch();
useEffect(() => {
const sweep = (): void => {
const st = store.getState();
const sessions = st.agents.sessions;
for (const card of Object.values(st.dashboardLayout.browserCards)) {
if (!card.spawned_by || card.keep_open) continue;
if (st.dashboardLayout.endingBrowserCards[card.browser_id]) continue;
const owner = sessions[card.spawned_by];
if (!owner) continue;
// 'stopped' is skipped, same as the event path: a manual stop keeps the browser for inspection.
if (owner.status === 'completed' || owner.status === 'error') {
dispatch(markBrowserCardEnding({ browserId: card.browser_id, status: owner.status }));
}
}
};
const first = window.setTimeout(sweep, FIRST_SWEEP_MS);
const timer = window.setInterval(sweep, SWEEP_MS);
return () => { window.clearTimeout(first); window.clearInterval(timer); };
}, [dispatch]);
}
+27
View File
@@ -0,0 +1,27 @@
// Remembers the editable the USER last put their caret in, so when an agent-driven webview steals
// host focus (Chromium hands the <webview> element focus when the guest focuses an input) we can put
// the caret back where the user was instead of leaving them typing into the agent's page (ENG-252).
let lastEl: HTMLElement | null = null;
function isEditable(el: Element | null): el is HTMLElement {
if (!el) return false;
const tag = el.tagName;
return tag === 'INPUT' || tag === 'TEXTAREA' || (el as HTMLElement).isContentEditable === true;
}
if (typeof document !== 'undefined') {
document.addEventListener('focusin', (e) => {
const t = e.target as Element | null;
// A webview grabbing focus is exactly what we guard against, never a target to remember.
if (t && t.tagName !== 'WEBVIEW' && isEditable(t)) lastEl = t as HTMLElement;
}, true);
}
/** Put the caret back in the user's last editable, if it is still attached. Returns true if it stuck. */
export function restoreLastUserFocus(): boolean {
if (lastEl && lastEl.isConnected) {
try { lastEl.focus(); return document.activeElement === lastEl; } catch { return false; }
}
return false;
}
@@ -1430,6 +1430,7 @@ const dashboardLayoutSlice = createSlice({
action: PayloadAction<{
tabs: BrowserTab[]; url: string; expandedSessionIds?: string[];
id?: string; x?: number; y?: number; width?: number; height?: number;
dockTo?: string | null;
}>
) {
const { x, y, width, height } = action.payload;
@@ -1463,6 +1464,10 @@ const dashboardLayoutSlice = createSlice({
zOrder: state.nextZOrder++,
// Pasted onto the dashboard the user is looking at, else it bleeds onto every dashboard.
dashboard_id: getLastDashboardId() ?? undefined,
// When the copied browser belonged to a copied agent, dock it inline under the NEW agent so
// the paste looks like the original instead of a stray full-size canvas browser (ENG-250).
spawned_by: action.payload.dockTo || null,
docked_to: (action.payload.dockTo && (clearOtherDocks(state, action.payload.dockTo), action.payload.dockTo)) || null,
};
},
@@ -0,0 +1,18 @@
// webview.insertText silently NO-OPS when the guest page has no focused editable, so dictation
// "landed" in a browser and vanished (ENG-254). Ask the guest first; only a confirmed editable
// earns the insert, everything else falls through to the visible composer fallback.
export async function guestHasEditableFocus(wv: { executeJavaScript?: (code: string) => Promise<unknown> }): Promise<boolean> {
if (!wv.executeJavaScript) return false;
try {
const ok = await Promise.race([
wv.executeJavaScript(
'(() => { const a = document.activeElement; return !!(a && (a.tagName === "INPUT" || a.tagName === "TEXTAREA" || a.isContentEditable)); })()',
),
// A suspended or wedged guest never answers; treat silence as "no target" instead of hanging the paste.
new Promise<boolean>((resolve) => { setTimeout(() => resolve(false), 800); }),
]);
return ok === true;
} catch {
return false;
}
}
+12 -7
View File
@@ -1,13 +1,14 @@
import { getLastInteractedBrowser } from '@/shared/browserFocus';
import { getWebview } from '@/shared/browserRegistry';
import { takeInjectSnapshot, setInjectSnapshot, isUsableTarget } from './injectTargetSnapshot';
import { guestHasEditableFocus } from './guestHasEditableFocus';
// Dictation lands where the user's cursor actually is, like every real dictation tool: a focused
// in-app field gets the text typed in (undo-friendly, fires React input events), a focused browser
// card forwards into the guest page's field, anything else falls back to the OS-level paste.
export type InjectTarget = 'field' | 'webview' | 'composer' | null;
export function injectAtFocus(text: string): InjectTarget {
export async function injectAtFocus(text: string): Promise<InjectTarget> {
const snap = takeInjectSnapshot();
// The cursor wins, not where you started. Wispr's grammar, and Eric's call: you dictate, you click
// where you want it, it lands there. This deliberately reverts the snapshot-first version, which
@@ -36,16 +37,20 @@ export function injectAtFocus(text: string): InjectTarget {
}
}
// A webview steals focus when the user clicks into a page, so activeElement IS the webview tag.
const focusedTag = active && active.tagName === 'WEBVIEW' ? (active as unknown as { insertText?: (t: string) => Promise<void> }) : null;
if (focusedTag?.insertText) {
try { void focusedTag.insertText(text); return 'webview'; } catch { /* fall through */ }
// insertText silently no-ops when the guest has no focused editable, which read as the dictation
// vanishing (ENG-254): confirm the guest target BEFORE claiming success, and await the insert so a
// suspended/crashed guest falls through to the visible composer fallback instead of eating words.
const focusedTag = active && active.tagName === 'WEBVIEW'
? (active as unknown as { insertText?: (t: string) => Promise<void>; executeJavaScript?: (c: string) => Promise<unknown> }) : null;
if (focusedTag?.insertText && await guestHasEditableFocus(focusedTag)) {
try { await focusedTag.insertText(text); return 'webview'; } catch { /* fall through */ }
}
// Last-interacted browser card: the user clicked a page field, then hit the hotkey.
const browserId = snap.browserId || getLastInteractedBrowser();
if (browserId) {
const wv = getWebview(browserId) as unknown as { insertText?: (t: string) => Promise<void>; focus?: () => void } | undefined;
if (wv?.insertText) {
try { wv.focus?.(); void wv.insertText(text); return 'webview'; } catch { /* fall through */ }
const wv = getWebview(browserId) as unknown as { insertText?: (t: string) => Promise<void>; executeJavaScript?: (c: string) => Promise<unknown>; focus?: () => void } | undefined;
if (wv?.insertText && await guestHasEditableFocus(wv)) {
try { wv.focus?.(); await wv.insertText(text); return 'webview'; } catch { /* fall through */ }
}
}
// No cursor anywhere: open the dashboard composer with the transcript typed in. Words are never dropped.
@@ -369,7 +369,7 @@ export function useVoiceDictation() {
// the OS paste fallback (other apps). The floating bubble is just confirmation, not the output.
// Success is silent: the text landing at the cursor IS the feedback. Only the clipboard
// fallback still speaks, because the user has to act (paste) to get the text.
const target = injectAtFocus(text);
const target = await injectAtFocus(text);
pushDictation(text, target || 'clipboard');
learnFromTranscript(text);
if (!target) {