diff --git a/electron/main.js b/electron/main.js index 795bb874..02a2ad77 100644 --- a/electron/main.js +++ b/electron/main.js @@ -3025,6 +3025,26 @@ app.on('web-contents-created', (_event, contents) => { // place; without this a site's "Continue with Google" was unreachable inside a card (ENG-238). if (contents.getType() === 'webview' && !childWindow.isDestroyed()) { try { childWindow.webContents.setUserAgent(contents.getUserAgent()); } catch (_) { /* popup already gone */ } + // ENG-279: agents were blind to auth popups (a native window has no , so no + // browser_id). Hand the renderer the child's webContents id keyed by the opener's, so the + // command layer can drive the popup over CDP while it lives. The popup stays a REAL popup, + // preserving window.opener (how OAuth returns its code), which is why it is not a tab. + try { + const popupWcId = childWindow.webContents.id; + const openerWcId = contents.id; + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send('browser-popup-created', { + openerWebContentsId: openerWcId, + childWebContentsId: popupWcId, + url: childWindow.webContents.getURL() || '', + }); + } + childWindow.on('closed', () => { + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send('browser-popup-closed', { childWebContentsId: popupWcId }); + } + }); + } catch (_) { /* popup raced its own close */ } } }); diff --git a/electron/preload.js b/electron/preload.js index 879d81e3..15d252ee 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -202,6 +202,18 @@ contextBridge.exposeInMainWorld('openswarm', { return () => ipcRenderer.removeListener('webview-new-window', listener); }, + // ENG-279: native popups (window.open from a browser card) announce themselves so agent tools can drive them by webContents id. + onBrowserPopupCreated: (cb) => { + const listener = (_event, payload) => cb(payload); + ipcRenderer.on('browser-popup-created', listener); + return () => ipcRenderer.removeListener('browser-popup-created', listener); + }, + onBrowserPopupClosed: (cb) => { + const listener = (_event, payload) => cb(payload); + ipcRenderer.on('browser-popup-closed', listener); + return () => ipcRenderer.removeListener('browser-popup-closed', listener); + }, + // Cmd/Ctrl+R, intercepted in main (kills the default-menu reload), so the renderer can reload the focused browser instead of the whole app. onReloadShortcut: (cb) => { const listener = () => cb(); diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index fb6caeab..b0126c0f 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -1,4 +1,5 @@ -import { getWebview, findWebviewByDomain, hasDomReady, markDomReady, isPendingLoad, wakePendingLoad, clearPendingLoad, type BrowserWebview } from './browserRegistry'; +import { getWebview, findWebviewByDomain, findBrowserIdByWebContentsId, hasDomReady, markDomReady, isPendingLoad, wakePendingLoad, clearPendingLoad, type BrowserWebview } from './browserRegistry'; +import { registerPopup, unregisterPopupByWcId, activePopupShim } from './browserPopupRegistry'; import { scheduleCaretHandback } from './caretHandback'; import { shouldSelfHealClick } from './selfHealClick'; import { focusGuestForKeys } from './focusGuestForKeys'; @@ -2174,6 +2175,10 @@ async function handleEvaluate(wv: BrowserWebview, params: Record): // The registry is renderer-local and a card briefly unregisters on remount / tab-switch; a command landing in that gap shouldn't hard-fail. Wait a bounded window for (re)registration before giving up, so the error stays a real "card is gone" signal rather than a transient race. async function awaitWebview(browserId: string, tabId?: string, action?: string): Promise { + // A live auth popup OWNS its card's commands (ENG-279): the user-visible action is in the popup, + // so tools drive it there until it closes, then fall back to the card's webview automatically. + const popup = activePopupShim(browserId); + if (popup) return popup; // A suspended (snapshot-swapped) card has no webview at all; wake it and wait out the remount + page reload before the command touches it. const wasSuspended = !!store.getState().dashboardLayout.suspendedBrowserCards[browserId]; if (wasSuspended) store.dispatch(resumeBrowserCard(browserId)); @@ -2468,8 +2473,20 @@ export function initBrowserCommandHandler(): () => void { if (initialized) return () => {}; initialized = true; const unsub = dashboardWs.on('browser:command', handleBrowserCommand); + // ENG-279: native auth popups announce themselves from main; map them to the opener's card so + // agent tools can see and drive the sign-in window instead of going blind at it. + const bridge = (window as unknown as { openswarm?: { onBrowserPopupCreated?: (cb: (p: { openerWebContentsId: number; childWebContentsId: number; url: string }) => void) => () => void; onBrowserPopupClosed?: (cb: (p: { childWebContentsId: number }) => void) => () => void } }).openswarm; + const offCreated = bridge?.onBrowserPopupCreated?.((p) => { + const browserId = findBrowserIdByWebContentsId(p.openerWebContentsId); + if (browserId) registerPopup(browserId, p.childWebContentsId, p.url); + }); + const offClosed = bridge?.onBrowserPopupClosed?.((p) => { + unregisterPopupByWcId(p.childWebContentsId); + }); return () => { unsub(); + offCreated?.(); + offClosed?.(); initialized = false; }; } diff --git a/frontend/src/shared/browserPopupRegistry.test.ts b/frontend/src/shared/browserPopupRegistry.test.ts new file mode 100644 index 00000000..a835a48a --- /dev/null +++ b/frontend/src/shared/browserPopupRegistry.test.ts @@ -0,0 +1,64 @@ +// ENG-279: a native auth popup must become drivable the moment main announces it, and the card's +// commands must fall back to the real webview the moment it closes. The shim speaks CDP by +// webContents id, so these pin the method->CDP mapping with a recording fake bridge. +import { test, beforeEach } from 'node:test'; +import assert from 'node:assert/strict'; +import { registerPopup, unregisterPopupByWcId, activePopupShim, activePopupCount } from './browserPopupRegistry'; + +const calls: Array<{ wcId: number; method: string; params: any }> = []; +(globalThis as any).window = { + openswarm: { + sendCdpCommand: async (wcId: number, method: string, params: any) => { + calls.push({ wcId, method, params }); + if (method === 'Runtime.evaluate') return { result: { value: '{"u":"https://accounts.google.com/o/oauth2","t":"Sign in"}' } }; + if (method === 'Page.captureScreenshot') return { data: 'aGk=' }; + if (method === 'Page.getLayoutMetrics') return { cssLayoutViewport: { clientWidth: 520, clientHeight: 680 } }; + return {}; + }, + }, +}; + +beforeEach(() => { + calls.length = 0; + unregisterPopupByWcId(901); + unregisterPopupByWcId(902); +}); + +test('a registered popup owns the card until closed, then the card falls back', () => { + assert.equal(activePopupShim('card-1'), undefined); + registerPopup('card-1', 901, 'https://accounts.google.com/'); + const shim = activePopupShim('card-1'); + assert.ok(shim, 'popup must be resolvable by the owning card'); + assert.equal(shim!.getWebContentsId(), 901, 'every CDP tool keys on this id'); + assert.equal(unregisterPopupByWcId(901), 'card-1'); + assert.equal(activePopupShim('card-1'), undefined, 'commands must return to the real webview'); + assert.equal(activePopupCount(), 0); +}); + +test('executeJavaScript rides Runtime.evaluate with returnByValue', async () => { + registerPopup('card-1', 901, 'https://x/'); + await activePopupShim('card-1')!.executeJavaScript('1+1'); + const ev = calls.find((c) => c.method === 'Runtime.evaluate' && c.params.expression === '1+1'); + assert.ok(ev && ev.wcId === 901 && ev.params.returnByValue === true); +}); + +test('capturePage returns a data-url-able image sized from layout metrics', async () => { + registerPopup('card-1', 901, 'https://x/'); + const img = await activePopupShim('card-1')!.capturePage(); + assert.equal(img.toDataURL(), 'data:image/png;base64,aGk='); + assert.deepEqual(img.getSize(), { width: 520, height: 680 }); +}); + +test('the electron key trio maps to the CDP trio', () => { + registerPopup('card-1', 901, 'https://x/'); + const shim = activePopupShim('card-1')!; + shim.sendInputEvent({ type: 'keyDown', keyCode: 'a' }); + shim.sendInputEvent({ type: 'char', keyCode: 'a' }); + shim.sendInputEvent({ type: 'keyUp', keyCode: 'a' }); + const kinds = calls.filter((c) => c.method === 'Input.dispatchKeyEvent').map((c) => c.params.type); + assert.deepEqual(kinds, ['rawKeyDown', 'char', 'keyUp']); +}); + +test('unregistering an unknown popup is a safe no-op', () => { + assert.equal(unregisterPopupByWcId(777), null); +}); diff --git a/frontend/src/shared/browserPopupRegistry.ts b/frontend/src/shared/browserPopupRegistry.ts new file mode 100644 index 00000000..3c5521d1 --- /dev/null +++ b/frontend/src/shared/browserPopupRegistry.ts @@ -0,0 +1,123 @@ +import type { BrowserWebview, ElectronNativeImage } from '@/shared/browserRegistry'; + +// ENG-279: a window.open from a browser card becomes a NATIVE window with no element, +// so every tool that resolves a target via the DOM registry went blind exactly at the auth popup. +// The CDP layer is target-agnostic (main addresses webContents by id), so this registry hands the +// command chokepoint a shim that speaks the BrowserWebview subset the agent path actually uses, +// proxied over sendCdpCommand by webContents id. The popup stays a REAL popup, so window.opener +// (how OAuth returns its code) is fully preserved; that is why fix A (reopen as a tab) was rejected. + +interface PopupRecord { + browserId: string; + wcId: number; + url: string; + shim: BrowserWebview; +} + +const p_byBrowserId = new Map(); + +type CdpSend = (wcId: number, method: string, params?: object) => Promise; + +function p_cdp(): CdpSend { + const bridge = (window as any).openswarm; + if (!bridge?.sendCdpCommand) throw new Error('CDP bridge unavailable'); + return (wcId, method, params) => bridge.sendCdpCommand(wcId, method, params ?? {}); +} + +function p_makeImage(base64: string, width: number, height: number): ElectronNativeImage { + const img: ElectronNativeImage = { + toDataURL: () => `data:image/png;base64,${base64}`, + toPNG: () => Buffer.from(base64, 'base64'), + toJPEG: () => Buffer.from(base64, 'base64'), + isEmpty: () => base64.length === 0, + getSize: () => ({ width, height }), + resize: () => img, + }; + return img; +} + +// The agent path calls: getWebContentsId (every CDP tool), executeJavaScript, getURL/getTitle, +// isLoading, loadURL, capturePage, sendInputEvent (keyDown/char/keyUp), stop/reload, focus. +// Everything else is a safe no-op; the shim is never handed to layout/UI code. +function p_makeShim(rec: { browserId: string; wcId: number; url: string; title: string }): BrowserWebview { + const send = p_cdp(); + const shim: any = { + src: rec.url, + getWebContentsId: () => rec.wcId, + executeJavaScript: async (code: string) => { + const r = await send(rec.wcId, 'Runtime.evaluate', { expression: code, returnByValue: true, awaitPromise: true, userGesture: true }); + p_refresh(rec, send); + return r?.result?.value; + }, + loadURL: async (url: string) => { await send(rec.wcId, 'Page.navigate', { url }); rec.url = url; }, + getURL: () => rec.url, + getTitle: () => rec.title, + isLoading: () => false, + isCurrentlyAudible: () => false, + reload: () => { void send(rec.wcId, 'Page.reload', {}); }, + stop: () => { void send(rec.wcId, 'Page.stopLoading', {}); }, + goBack: () => { void send(rec.wcId, 'Runtime.evaluate', { expression: 'history.back()' }); }, + goForward: () => { void send(rec.wcId, 'Runtime.evaluate', { expression: 'history.forward()' }); }, + canGoBack: () => false, + canGoForward: () => false, + capturePage: async () => { + const r = await send(rec.wcId, 'Page.captureScreenshot', { format: 'png' }); + const metrics = await send(rec.wcId, 'Page.getLayoutMetrics').catch(() => null); + const vp = metrics?.cssLayoutViewport; + return p_makeImage(String(r?.data || ''), vp?.clientWidth || 520, vp?.clientHeight || 680); + }, + sendInputEvent: (event: { type: string; keyCode?: string }) => { + // The handler's key path sends Electron keyDown/char/keyUp with a keyCode string; CDP's + // equivalent trio is rawKeyDown/char/keyUp keyed on text for printable input. + const k = String(event.keyCode ?? ''); + if (event.type === 'keyDown') void send(rec.wcId, 'Input.dispatchKeyEvent', { type: 'rawKeyDown', key: k, text: k.length === 1 ? k : undefined }); + else if (event.type === 'char') void send(rec.wcId, 'Input.dispatchKeyEvent', { type: 'char', text: k, key: k }); + else if (event.type === 'keyUp') void send(rec.wcId, 'Input.dispatchKeyEvent', { type: 'keyUp', key: k }); + }, + getZoomLevel: () => 0, + setZoomLevel: () => undefined, + findInPage: () => 0, + stopFindInPage: () => undefined, + focus: () => { void send(rec.wcId, 'Page.bringToFront', {}); }, + getBoundingClientRect: () => ({ x: 0, y: 0, left: 0, top: 0, right: 520, bottom: 680, width: 520, height: 680, toJSON: () => ({}) }), + addEventListener: () => undefined, + removeEventListener: () => undefined, + }; + return shim as BrowserWebview; +} + +function p_refresh(rec: { wcId: number; url: string; title: string }, send: CdpSend): void { + void send(rec.wcId, 'Runtime.evaluate', { expression: 'JSON.stringify({u: location.href, t: document.title})', returnByValue: true }) + .then((r) => { + try { + const v = JSON.parse(r?.result?.value || '{}'); + if (v.u) rec.url = v.u; + if (typeof v.t === 'string') rec.title = v.t; + } catch { /* popup mid-navigation; keep the cached values */ } + }) + .catch(() => undefined); +} + +export function registerPopup(browserId: string, wcId: number, url: string): void { + const rec = { browserId, wcId, url, title: '' }; + p_byBrowserId.set(browserId, { browserId, wcId, url, shim: p_makeShim(rec) }); +} + +export function unregisterPopupByWcId(wcId: number): string | null { + for (const [bid, rec] of p_byBrowserId) { + if (rec.wcId === wcId) { + p_byBrowserId.delete(bid); + return bid; + } + } + return null; +} + +/** The live popup shim for a card, or undefined; while one is open it OWNS the card's commands. */ +export function activePopupShim(browserId: string): BrowserWebview | undefined { + return p_byBrowserId.get(browserId)?.shim; +} + +export function activePopupCount(): number { + return p_byBrowserId.size; +} diff --git a/frontend/src/shared/browserRegistry.ts b/frontend/src/shared/browserRegistry.ts index 69299581..c550ff71 100644 --- a/frontend/src/shared/browserRegistry.ts +++ b/frontend/src/shared/browserRegistry.ts @@ -140,6 +140,16 @@ export function findBrowserByWebContentsId(wcId: number): string | undefined { // Find the live webview currently on `domain` (e.g. tiktok.com). The session-borrow shims use // this to drive the user's own already-open, logged-in card for that site, resolving by the // LIVE url (not a stale persisted card.url) so the action lands on the real tab. +/** The card that owns a live webContents id, for mapping a native popup back to its opener (ENG-279). */ +export function findBrowserIdByWebContentsId(wcId: number): string | null { + for (const [key, wv] of registry) { + try { + if (wv.getWebContentsId() === wcId) return key.split(':')[0]; + } catch { /* a mid-teardown webview throws; skip it */ } + } + return null; +} + export function findWebviewByDomain(domain: string): BrowserWebview | undefined { const d = domain.toLowerCase().replace(/^\./, ''); const matchesHost = (u: string): boolean => {