mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-07 02:07:45 +02:00
[eric] browser: reap finished-agent browsers, close an agent's browsers with it; dictation confirms the guest target before claiming success; agent-driven focus steal restores the user's caret; window size persists across a crash-recovery recreate
This commit is contained in:
+44
-3
@@ -1429,12 +1429,47 @@ async function loadAuthToken() {
|
||||
console.warn(`[auth] FAILED to load auth token from ${tokenPath} after 2s — WS/HTTP will be rejected`);
|
||||
}
|
||||
|
||||
function windowBoundsPath() {
|
||||
try { return path.join(app.getPath('userData'), 'window-bounds.json'); } catch { return null; }
|
||||
}
|
||||
|
||||
// The user's manual window size is persisted and restored, so a crash-recovery recreateMainWindow no
|
||||
// longer rebuilds at the default 1400x900 and drops the size they dragged out to (ENG-253). Only
|
||||
// honored when the saved rect still fits on some display, so unplugging a monitor can't hide the app.
|
||||
function loadSavedBounds() {
|
||||
const p = windowBoundsPath();
|
||||
if (!p) return null;
|
||||
try {
|
||||
const b = JSON.parse(fs.readFileSync(p, 'utf8'));
|
||||
if (!b || !Number.isFinite(b.width) || !Number.isFinite(b.height) || b.width < 800 || b.height < 600) return null;
|
||||
const displays = require('electron').screen.getAllDisplays();
|
||||
const onScreen = displays.some((d) => {
|
||||
const wa = d.workArea;
|
||||
return b.x != null && b.y != null && b.x < wa.x + wa.width && b.x + 200 > wa.x && b.y < wa.y + wa.height && b.y + 100 > wa.y;
|
||||
});
|
||||
return onScreen ? b : { width: b.width, height: b.height };
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
let boundsSaveTimer = null;
|
||||
function persistWindowBounds(win) {
|
||||
if (!win || win.isDestroyed() || win.isFullScreen() || win.isMinimized()) return;
|
||||
const p = windowBoundsPath();
|
||||
if (!p) return;
|
||||
if (boundsSaveTimer) clearTimeout(boundsSaveTimer);
|
||||
boundsSaveTimer = setTimeout(() => {
|
||||
try { fs.writeFileSync(p, JSON.stringify(win.getBounds())); } catch (_) {}
|
||||
}, 400);
|
||||
}
|
||||
|
||||
function createWindow() {
|
||||
isCreatingMainWindow = true;
|
||||
console.log('[diag][main] createWindow start');
|
||||
const saved = loadSavedBounds();
|
||||
mainWindow = new BrowserWindow({
|
||||
width: 1400,
|
||||
height: 900,
|
||||
width: (saved && saved.width) || 1400,
|
||||
height: (saved && saved.height) || 900,
|
||||
...(saved && saved.x != null ? { x: saved.x, y: saved.y } : {}),
|
||||
minWidth: 800,
|
||||
minHeight: 600,
|
||||
title: 'OpenSwarm',
|
||||
@@ -1684,6 +1719,10 @@ function createWindow() {
|
||||
mainWindow.on('blur', () => sendFocusEvent('blur'));
|
||||
mainWindow.on('focus', () => sendFocusEvent('focus'));
|
||||
|
||||
// Remember the size/position the user set, so nothing (a crash-recovery recreate especially) drops it (ENG-253).
|
||||
mainWindow.on('resize', () => persistWindowBounds(mainWindow));
|
||||
mainWindow.on('move', () => persistWindowBounds(mainWindow));
|
||||
|
||||
// Forward renderer console output to main stderr so packaged-build diagnostics survive without DevTools open.
|
||||
mainWindow.webContents.on('console-message', (_e, level, message, line, sourceId) => {
|
||||
const tag = ['LOG', 'INFO', 'WARN', 'ERROR'][level] || 'LOG';
|
||||
@@ -1717,7 +1756,9 @@ function createWindow() {
|
||||
// Why setImmediate for the destroy:
|
||||
// - We're INSIDE the old window's render-process-gone handler. Destroying its BrowserWindow from inside its own event callback works in current Electron but is fragile across version bumps; deferring one tick is free insurance.
|
||||
function recreateMainWindow() {
|
||||
console.log('[diag][main] recreateMainWindow START, crashesInWindow=', rendererCrashTimes.length);
|
||||
// Named on purpose: this is the prime suspect for "the window resized itself" (ENG-253); the diag
|
||||
// line lets a report say whether a silent recreate is what dropped the user's window size.
|
||||
console.log('[diag][main] recreateMainWindow START (window will restore saved bounds), crashesInWindow=', rendererCrashTimes.length);
|
||||
const oldWindow = mainWindow;
|
||||
mainWindowReady = false;
|
||||
try {
|
||||
|
||||
@@ -4,6 +4,7 @@ import { ElementSelectionProvider } from '@/app/components/editor/ElementSelecti
|
||||
import { useDomElementSelector } from '@/app/components/editor/useDomElementSelector';
|
||||
import { useDashboardActive } from '@/shared/hooks/useDashboardActive';
|
||||
import { useDashboardController } from './hooks/state/useDashboardController';
|
||||
import { useOrphanBrowserReaper } from './hooks/useOrphanBrowserReaper';
|
||||
import DashboardCanvas from './canvas/DashboardCanvas';
|
||||
|
||||
const DashboardSelectionOverlay: React.FC = () => {
|
||||
@@ -20,6 +21,7 @@ interface DashboardProps {
|
||||
|
||||
const DashboardInner: React.FC<DashboardProps> = ({ dashboardId, isActive = true }) => {
|
||||
const controller = useDashboardController(dashboardId, isActive);
|
||||
useOrphanBrowserReaper();
|
||||
return (
|
||||
<>
|
||||
<DashboardSelectionOverlay />
|
||||
|
||||
@@ -32,10 +32,9 @@ import {
|
||||
clearGlowingAgentCard,
|
||||
removeCard,
|
||||
recordClosedCard,
|
||||
setBrowserDocked,
|
||||
keepBrowserCardOpen,
|
||||
} from '@/shared/state/dashboardLayoutSlice';
|
||||
import { store } from '@/shared/state/store';
|
||||
import { removeBrowserCardCleanly } from '@/shared/browserTeardown';
|
||||
import WindowControls, { ARC_CHIP_SX } from './WindowControls';
|
||||
import { useTiledCard } from './useTiledCard';
|
||||
import { useCardTiling } from './useCardTiling';
|
||||
@@ -672,12 +671,14 @@ const AgentCard: React.FC<Props> = ({
|
||||
if (linkedWorkflowSidecarId) {
|
||||
dispatch(setCardSidecar({ workflowId: linkedWorkflowSidecarId, sessionId: null, kind: null }));
|
||||
}
|
||||
// Closing the CHAT must not take the browser with it: its browser undocks to the canvas and is pinned open (keep_open exempts it from the finished-agent auto-reap). The user closes it separately.
|
||||
// Closing the chat takes its browsers with it (Haik's ENG-249: the old undock-and-pin behavior
|
||||
// read as the browser "popping open on its own", forcing a second cleanup every time). Each one
|
||||
// lands in recently-closed first, so Cmd+Shift+T brings it back if it was wanted.
|
||||
if (!glowEntry) {
|
||||
for (const bc of Object.values(store.getState().dashboardLayout.browserCards)) {
|
||||
if (bc.docked_to !== session.id && bc.spawned_by !== session.id) continue;
|
||||
if (bc.docked_to === session.id) dispatch(setBrowserDocked({ browserId: bc.browser_id, dockedTo: null }));
|
||||
dispatch(keepBrowserCardOpen(bc.browser_id));
|
||||
dispatch(recordClosedCard({ kind: 'browser', id: bc.browser_id }));
|
||||
removeBrowserCardCleanly(bc.browser_id, dispatch);
|
||||
}
|
||||
}
|
||||
// Record for Cmd+Shift+T BEFORE removeCard wipes the position, but only on a real close (the glow branch just clears a tether, it doesn't close the session).
|
||||
|
||||
@@ -66,6 +66,7 @@ import {
|
||||
import { captureBrowserShot } from '@/shared/captureBrowserShot';
|
||||
import { setLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
import { isAgentDrivenBrowser } from '@/shared/isAgentDrivenBrowser';
|
||||
import { restoreLastUserFocus } from '@/shared/lastUserFocus';
|
||||
import { registerCapsuleForRestore } from '@/shared/browserStateCapsule';
|
||||
import BrowserFindBar from './BrowserFindBar';
|
||||
import { openCardContextMenu, isNativeMenuTarget } from '../desktop/openCardContextMenu';
|
||||
@@ -681,8 +682,15 @@ const BrowserCard: React.FC<Props> = ({
|
||||
wv.addEventListener('render-process-gone', onProcessGone as any);
|
||||
wv.addEventListener('crashed', onProcessGone as any);
|
||||
wv.addEventListener('did-fail-load', onDidFailLoad as any);
|
||||
// While an agent drives this browser, the guest focusing an input hands the host <webview>
|
||||
// element focus, yanking the user out of whatever they were typing (ENG-252, 4x-reported). The
|
||||
// agent drives via the webContents transport, not host DOM focus, so restoring the user's caret
|
||||
// here does not disturb the agent. Only fires for agent-driven cards; a user's own click is untouched.
|
||||
const onWvFocus = () => { if (isAgentDrivenBrowser(browserId)) restoreLastUserFocus(); };
|
||||
wv.addEventListener('focus', onWvFocus);
|
||||
|
||||
cleanups.push(() => {
|
||||
wv.removeEventListener('focus', onWvFocus);
|
||||
unregisterWebview(browserId, tabId);
|
||||
wv.removeEventListener('did-navigate', onNavigate);
|
||||
wv.removeEventListener('did-navigate-in-page', onNavigate);
|
||||
|
||||
@@ -30,8 +30,12 @@ export async function pasteClipboardCards({ dispatch, dashboardId, expandedSessi
|
||||
|
||||
selection.deselectAll();
|
||||
const newSelection = new Map<string, CardType>();
|
||||
// Old agent id -> new pasted id, so a browser copied alongside its agent re-docks under the copy (ENG-250).
|
||||
const agentRemap = new Map<string, string>();
|
||||
// Agents first so their remap exists before their browsers are pasted, whatever the copy order.
|
||||
const ordered = [...copied].sort((a, b) => (a.type === 'agent' ? -1 : 0) - (b.type === 'agent' ? -1 : 0));
|
||||
|
||||
for (const card of copied) {
|
||||
for (const card of ordered) {
|
||||
const px = at ? at.x + (card.x - anchorX) : card.x + PASTE_OFFSET;
|
||||
const py = at ? at.y + (card.y - anchorY) : card.y - PASTE_OFFSET;
|
||||
|
||||
@@ -39,6 +43,7 @@ export async function pasteClipboardCards({ dispatch, dashboardId, expandedSessi
|
||||
const action = await dispatch(duplicateSession({ sessionId: card.id, dashboardId }));
|
||||
if (duplicateSession.fulfilled.match(action)) {
|
||||
const newId = action.payload.id;
|
||||
agentRemap.set(card.id, newId);
|
||||
dispatch(placeCard({ sessionId: newId, x: px, y: py, width: card.width, height: card.height, expandedSessionIds }));
|
||||
if (card.expanded) dispatch(expandSession(newId));
|
||||
newSelection.set(newId, 'agent');
|
||||
@@ -55,9 +60,11 @@ export async function pasteClipboardCards({ dispatch, dashboardId, expandedSessi
|
||||
newSelection.set(pastedKey, 'view');
|
||||
} else if (card.type === 'browser') {
|
||||
const browserId = `browser-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`;
|
||||
const originalOwner = card.meta.spawnedBy as string | undefined;
|
||||
const dockTo = originalOwner ? agentRemap.get(originalOwner) ?? null : null;
|
||||
dispatch(pasteBrowserCard({
|
||||
id: browserId, tabs: card.meta.tabs || [], url: card.meta.url || '',
|
||||
x: px, y: py, width: card.width, height: card.height,
|
||||
x: px, y: py, width: card.width, height: card.height, dockTo,
|
||||
}));
|
||||
newSelection.set(browserId, 'browser');
|
||||
}
|
||||
|
||||
@@ -83,7 +83,8 @@ export function useDashboardClipboard({
|
||||
const title = activeTab?.title || 'Browser';
|
||||
copied.push({
|
||||
type, id, name: title,
|
||||
meta: { name: title, url: activeTab?.url || bc.url, tabs: bc.tabs },
|
||||
// Carry the owning session so a group paste can re-dock the browser under the NEW agent (ENG-250).
|
||||
meta: { name: title, url: activeTab?.url || bc.url, tabs: bc.tabs, spawnedBy: bc.docked_to ?? bc.spawned_by ?? null },
|
||||
x: bc.x, y: bc.y, width: bc.width, height: bc.height,
|
||||
});
|
||||
names.push(title);
|
||||
|
||||
@@ -0,0 +1,36 @@
|
||||
import { useEffect } from 'react';
|
||||
import { useAppDispatch } from '@/shared/hooks';
|
||||
import { store } from '@/shared/state/store';
|
||||
import { markBrowserCardEnding } from '@/shared/state/dashboardLayoutSlice';
|
||||
|
||||
// The event-driven despawn (WebSocketManager marks a finished agent's browsers ending) misses any
|
||||
// terminal event the renderer never saw: a reload mid-run, a workflow finishing while the app was
|
||||
// closed. Those cards then live forever, which is Haik's 20-browser pile-up (ENG-248). This sweep is
|
||||
// the belt: anything owned by a session the STORE can see is terminal gets the same fade + Keep pill
|
||||
// the event path uses. A session missing from the store entirely is left alone on purpose; before
|
||||
// sessions load, "missing" means "not fetched yet", and axing on that would kill live cards at boot.
|
||||
const SWEEP_MS = 45_000;
|
||||
const FIRST_SWEEP_MS = 7_000;
|
||||
|
||||
export function useOrphanBrowserReaper(): void {
|
||||
const dispatch = useAppDispatch();
|
||||
useEffect(() => {
|
||||
const sweep = (): void => {
|
||||
const st = store.getState();
|
||||
const sessions = st.agents.sessions;
|
||||
for (const card of Object.values(st.dashboardLayout.browserCards)) {
|
||||
if (!card.spawned_by || card.keep_open) continue;
|
||||
if (st.dashboardLayout.endingBrowserCards[card.browser_id]) continue;
|
||||
const owner = sessions[card.spawned_by];
|
||||
if (!owner) continue;
|
||||
// 'stopped' is skipped, same as the event path: a manual stop keeps the browser for inspection.
|
||||
if (owner.status === 'completed' || owner.status === 'error') {
|
||||
dispatch(markBrowserCardEnding({ browserId: card.browser_id, status: owner.status }));
|
||||
}
|
||||
}
|
||||
};
|
||||
const first = window.setTimeout(sweep, FIRST_SWEEP_MS);
|
||||
const timer = window.setInterval(sweep, SWEEP_MS);
|
||||
return () => { window.clearTimeout(first); window.clearInterval(timer); };
|
||||
}, [dispatch]);
|
||||
}
|
||||
@@ -0,0 +1,27 @@
|
||||
// Remembers the editable the USER last put their caret in, so when an agent-driven webview steals
|
||||
// host focus (Chromium hands the <webview> element focus when the guest focuses an input) we can put
|
||||
// the caret back where the user was instead of leaving them typing into the agent's page (ENG-252).
|
||||
|
||||
let lastEl: HTMLElement | null = null;
|
||||
|
||||
function isEditable(el: Element | null): el is HTMLElement {
|
||||
if (!el) return false;
|
||||
const tag = el.tagName;
|
||||
return tag === 'INPUT' || tag === 'TEXTAREA' || (el as HTMLElement).isContentEditable === true;
|
||||
}
|
||||
|
||||
if (typeof document !== 'undefined') {
|
||||
document.addEventListener('focusin', (e) => {
|
||||
const t = e.target as Element | null;
|
||||
// A webview grabbing focus is exactly what we guard against, never a target to remember.
|
||||
if (t && t.tagName !== 'WEBVIEW' && isEditable(t)) lastEl = t as HTMLElement;
|
||||
}, true);
|
||||
}
|
||||
|
||||
/** Put the caret back in the user's last editable, if it is still attached. Returns true if it stuck. */
|
||||
export function restoreLastUserFocus(): boolean {
|
||||
if (lastEl && lastEl.isConnected) {
|
||||
try { lastEl.focus(); return document.activeElement === lastEl; } catch { return false; }
|
||||
}
|
||||
return false;
|
||||
}
|
||||
@@ -1430,6 +1430,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
action: PayloadAction<{
|
||||
tabs: BrowserTab[]; url: string; expandedSessionIds?: string[];
|
||||
id?: string; x?: number; y?: number; width?: number; height?: number;
|
||||
dockTo?: string | null;
|
||||
}>
|
||||
) {
|
||||
const { x, y, width, height } = action.payload;
|
||||
@@ -1463,6 +1464,10 @@ const dashboardLayoutSlice = createSlice({
|
||||
zOrder: state.nextZOrder++,
|
||||
// Pasted onto the dashboard the user is looking at, else it bleeds onto every dashboard.
|
||||
dashboard_id: getLastDashboardId() ?? undefined,
|
||||
// When the copied browser belonged to a copied agent, dock it inline under the NEW agent so
|
||||
// the paste looks like the original instead of a stray full-size canvas browser (ENG-250).
|
||||
spawned_by: action.payload.dockTo || null,
|
||||
docked_to: (action.payload.dockTo && (clearOtherDocks(state, action.payload.dockTo), action.payload.dockTo)) || null,
|
||||
};
|
||||
},
|
||||
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
// webview.insertText silently NO-OPS when the guest page has no focused editable, so dictation
|
||||
// "landed" in a browser and vanished (ENG-254). Ask the guest first; only a confirmed editable
|
||||
// earns the insert, everything else falls through to the visible composer fallback.
|
||||
export async function guestHasEditableFocus(wv: { executeJavaScript?: (code: string) => Promise<unknown> }): Promise<boolean> {
|
||||
if (!wv.executeJavaScript) return false;
|
||||
try {
|
||||
const ok = await Promise.race([
|
||||
wv.executeJavaScript(
|
||||
'(() => { const a = document.activeElement; return !!(a && (a.tagName === "INPUT" || a.tagName === "TEXTAREA" || a.isContentEditable)); })()',
|
||||
),
|
||||
// A suspended or wedged guest never answers; treat silence as "no target" instead of hanging the paste.
|
||||
new Promise<boolean>((resolve) => { setTimeout(() => resolve(false), 800); }),
|
||||
]);
|
||||
return ok === true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
@@ -1,13 +1,14 @@
|
||||
import { getLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
import { getWebview } from '@/shared/browserRegistry';
|
||||
import { takeInjectSnapshot, setInjectSnapshot, isUsableTarget } from './injectTargetSnapshot';
|
||||
import { guestHasEditableFocus } from './guestHasEditableFocus';
|
||||
|
||||
// Dictation lands where the user's cursor actually is, like every real dictation tool: a focused
|
||||
// in-app field gets the text typed in (undo-friendly, fires React input events), a focused browser
|
||||
// card forwards into the guest page's field, anything else falls back to the OS-level paste.
|
||||
export type InjectTarget = 'field' | 'webview' | 'composer' | null;
|
||||
|
||||
export function injectAtFocus(text: string): InjectTarget {
|
||||
export async function injectAtFocus(text: string): Promise<InjectTarget> {
|
||||
const snap = takeInjectSnapshot();
|
||||
// The cursor wins, not where you started. Wispr's grammar, and Eric's call: you dictate, you click
|
||||
// where you want it, it lands there. This deliberately reverts the snapshot-first version, which
|
||||
@@ -36,16 +37,20 @@ export function injectAtFocus(text: string): InjectTarget {
|
||||
}
|
||||
}
|
||||
// A webview steals focus when the user clicks into a page, so activeElement IS the webview tag.
|
||||
const focusedTag = active && active.tagName === 'WEBVIEW' ? (active as unknown as { insertText?: (t: string) => Promise<void> }) : null;
|
||||
if (focusedTag?.insertText) {
|
||||
try { void focusedTag.insertText(text); return 'webview'; } catch { /* fall through */ }
|
||||
// insertText silently no-ops when the guest has no focused editable, which read as the dictation
|
||||
// vanishing (ENG-254): confirm the guest target BEFORE claiming success, and await the insert so a
|
||||
// suspended/crashed guest falls through to the visible composer fallback instead of eating words.
|
||||
const focusedTag = active && active.tagName === 'WEBVIEW'
|
||||
? (active as unknown as { insertText?: (t: string) => Promise<void>; executeJavaScript?: (c: string) => Promise<unknown> }) : null;
|
||||
if (focusedTag?.insertText && await guestHasEditableFocus(focusedTag)) {
|
||||
try { await focusedTag.insertText(text); return 'webview'; } catch { /* fall through */ }
|
||||
}
|
||||
// Last-interacted browser card: the user clicked a page field, then hit the hotkey.
|
||||
const browserId = snap.browserId || getLastInteractedBrowser();
|
||||
if (browserId) {
|
||||
const wv = getWebview(browserId) as unknown as { insertText?: (t: string) => Promise<void>; focus?: () => void } | undefined;
|
||||
if (wv?.insertText) {
|
||||
try { wv.focus?.(); void wv.insertText(text); return 'webview'; } catch { /* fall through */ }
|
||||
const wv = getWebview(browserId) as unknown as { insertText?: (t: string) => Promise<void>; executeJavaScript?: (c: string) => Promise<unknown>; focus?: () => void } | undefined;
|
||||
if (wv?.insertText && await guestHasEditableFocus(wv)) {
|
||||
try { wv.focus?.(); await wv.insertText(text); return 'webview'; } catch { /* fall through */ }
|
||||
}
|
||||
}
|
||||
// No cursor anywhere: open the dashboard composer with the transcript typed in. Words are never dropped.
|
||||
|
||||
@@ -369,7 +369,7 @@ export function useVoiceDictation() {
|
||||
// the OS paste fallback (other apps). The floating bubble is just confirmation, not the output.
|
||||
// Success is silent: the text landing at the cursor IS the feedback. Only the clipboard
|
||||
// fallback still speaks, because the user has to act (paste) to get the text.
|
||||
const target = injectAtFocus(text);
|
||||
const target = await injectAtFocus(text);
|
||||
pushDictation(text, target || 'clipboard');
|
||||
learnFromTranscript(text);
|
||||
if (!target) {
|
||||
|
||||
Reference in New Issue
Block a user