diff --git a/.gitignore b/.gitignore index 47256d70..d546f98d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ backend/data/** electron/dist/ electron/python-env/ electron/node_modules/ +electron/package-lock.json # Frontend build output frontend/dist/ \ No newline at end of file diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index c92b6b10..5913cb15 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -621,6 +621,7 @@ class AgentManager: "OPENSWARM_AGENT_MODEL": session.model, "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), + "OPENSWARM_PARENT_SESSION_ID": session.id, }, "type": "stdio", } @@ -1401,4 +1402,22 @@ class AgentManager: def get_session(self, session_id: str) -> Optional[AgentSession]: return self.sessions.get(session_id) + def get_browser_agent_children(self, parent_session_id: str) -> list[dict]: + """Return browser-agent sessions for a parent, from memory or disk.""" + results: list[dict] = [] + seen: set[str] = set() + + for s in self.sessions.values(): + if s.mode == "browser-agent" and s.parent_session_id == parent_session_id: + results.append(s.model_dump(mode="json")) + seen.add(s.id) + + for sid, data in _load_all_session_data(): + if sid in seen: + continue + if data.get("mode") == "browser-agent" and data.get("parent_session_id") == parent_session_id: + results.append(data) + + return results + agent_manager = AgentManager() diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 462b8fde..c4cd2df1 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -151,6 +151,11 @@ async def get_history(q: str = "", limit: int = 20, offset: int = 0, dashboard_i dashboard_id=dashboard_id or None, ) +@agents.router.get("/sessions/{session_id}/browser-agents") +async def get_browser_agent_children(session_id: str): + children = agent_manager.get_browser_agent_children(session_id) + return {"sessions": children} + @agents.router.post("/sessions/{session_id}/resume") async def resume_session(session_id: str): try: diff --git a/backend/apps/agents/browser_agent.py b/backend/apps/agents/browser_agent.py index 11244acf..ea2064e6 100644 --- a/backend/apps/agents/browser_agent.py +++ b/backend/apps/agents/browser_agent.py @@ -188,6 +188,7 @@ async def run_browser_agent( tab_id: str = "", pre_selected: bool = False, initial_url: str | None = None, + parent_session_id: str | None = None, ) -> dict: """Run a browser sub-agent loop for a single browser card. @@ -206,6 +207,7 @@ async def run_browser_agent( dashboard_id=dashboard_id, browser_id=browser_id, system_prompt=SYSTEM_PROMPT, + parent_session_id=parent_session_id, ) agent_manager.sessions[session_id] = session @@ -425,6 +427,7 @@ async def run_browser_agents( api_key: str, dashboard_id: str | None = None, pre_selected_browser_ids: list[str] | None = None, + parent_session_id: str | None = None, ) -> list[dict]: """Run multiple browser sub-agents in parallel. @@ -451,6 +454,7 @@ async def run_browser_agents( dashboard_id=dashboard_id, pre_selected=is_pre_selected, initial_url=url if url and browser_id not in pre_selected else None, + parent_session_id=parent_session_id, ) results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True) diff --git a/backend/apps/agents/browser_agent_mcp_server.py b/backend/apps/agents/browser_agent_mcp_server.py index 2a46a139..a229b852 100644 --- a/backend/apps/agents/browser_agent_mcp_server.py +++ b/backend/apps/agents/browser_agent_mcp_server.py @@ -25,6 +25,7 @@ BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/browser-agent/run" MODEL = os.environ.get("OPENSWARM_AGENT_MODEL", "sonnet") DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "") PRE_SELECTED_BROWSER_IDS = os.environ.get("OPENSWARM_PRE_SELECTED_BROWSER_IDS", "") +PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") TOOLS = [ { @@ -119,6 +120,7 @@ def call_backend(tasks: list[dict]) -> dict: "model": MODEL, "dashboard_id": DASHBOARD_ID, "pre_selected_browser_ids": pre_selected, + "parent_session_id": PARENT_SESSION_ID, }).encode() req = urllib.request.Request( BACKEND_URL, diff --git a/backend/apps/agents/models.py b/backend/apps/agents/models.py index a5f03df3..41480761 100644 --- a/backend/apps/agents/models.py +++ b/backend/apps/agents/models.py @@ -72,3 +72,4 @@ class AgentSession(BaseModel): tool_group_meta: dict[str, "ToolGroupMeta"] = Field(default_factory=dict) dashboard_id: Optional[str] = None browser_id: Optional[str] = None + parent_session_id: Optional[str] = None diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index 343b245e..f2ef9988 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -21,3 +21,4 @@ class AppSettings(BaseModel): new_agent_shortcut: str = "Meta+l" anthropic_api_key: Optional[str] = None browser_homepage: str = "https://www.google.com" + auto_select_mode_on_new_agent: bool = False diff --git a/backend/apps/tools_lib/tools_lib.py b/backend/apps/tools_lib/tools_lib.py index f19add53..d440bda0 100644 --- a/backend/apps/tools_lib/tools_lib.py +++ b/backend/apps/tools_lib/tools_lib.py @@ -18,9 +18,11 @@ from backend.apps.tools_lib.models import ToolDefinition, ToolCreate, ToolUpdate logger = logging.getLogger(__name__) -from backend.config.paths import BACKEND_DIR, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH +from backend.config.paths import BACKEND_DIR, DATA_ROOT, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH load_dotenv(os.path.join(BACKEND_DIR, ".env")) +if os.environ.get("OPENSWARM_PACKAGED") == "1": + load_dotenv(os.path.join(os.path.dirname(DATA_ROOT), ".env"), override=True) @asynccontextmanager diff --git a/backend/main.py b/backend/main.py index d7f30cee..81c15119 100644 --- a/backend/main.py +++ b/backend/main.py @@ -126,6 +126,7 @@ async def browser_agent_run(request: Request): model = body.get("model", "sonnet") dashboard_id = body.get("dashboard_id", "") pre_selected_browser_ids = body.get("pre_selected_browser_ids", []) + parent_session_id = body.get("parent_session_id", "") if not tasks: return JSONResponse({"error": "tasks array is required"}, status_code=400) @@ -140,6 +141,7 @@ async def browser_agent_run(request: Request): api_key=settings.anthropic_api_key, dashboard_id=dashboard_id or None, pre_selected_browser_ids=pre_selected_browser_ids, + parent_session_id=parent_session_id or None, ) return JSONResponse({"results": results}) diff --git a/electron/main.js b/electron/main.js index d5f79a99..90b909c1 100644 --- a/electron/main.js +++ b/electron/main.js @@ -1,5 +1,6 @@ -const { app, BrowserWindow, ipcMain, shell } = require('electron'); -const { autoUpdater } = require('electron-updater'); +const { app, components, BrowserWindow, ipcMain, shell, session } = require('electron'); +let autoUpdater; +try { autoUpdater = require('electron-updater').autoUpdater; } catch (_) {} const path = require('path'); const { spawn, execFileSync } = require('child_process'); const os = require('os'); @@ -7,6 +8,12 @@ const fs = require('fs'); const getPort = require('get-port'); const http = require('http'); +app.commandLine.appendSwitch('disable-features', 'HardwareMediaKeyHandling'); +app.commandLine.appendSwitch('ignore-gpu-blocklist'); +app.commandLine.appendSwitch('enable-gpu-rasterization'); +app.commandLine.appendSwitch('enable-zero-copy'); +app.commandLine.appendSwitch('autoplay-policy', 'no-user-gesture-required'); + let mainWindow = null; let backendProcess = null; let backendPort = null; @@ -24,9 +31,10 @@ const iconPath = path.join(__dirname, 'build', 'icon.png'); function getShellPath() { if (process.platform !== 'darwin' || isDev) return process.env.PATH || ''; + // Strategy 1: ask the user's login shell for its PATH try { - const shell = process.env.SHELL || '/bin/zsh'; - const result = execFileSync(shell, ['-ilc', 'echo $PATH'], { + const userShell = process.env.SHELL || '/bin/zsh'; + const result = execFileSync(userShell, ['-ilc', 'echo $PATH'], { encoding: 'utf8', timeout: 5000, env: { ...process.env, HOME: os.homedir() }, @@ -35,19 +43,40 @@ function getShellPath() { if (resolved) return resolved; } catch (_) { /* fall through */ } + // Strategy 2: read macOS system PATH config (/etc/paths + /etc/paths.d/*) + const systemPaths = []; + try { + const base = fs.readFileSync('/etc/paths', 'utf8'); + for (const line of base.split('\n')) { + const p = line.trim(); + if (p) systemPaths.push(p); + } + } catch (_) { /* ignore */ } + try { + const pathsD = '/etc/paths.d'; + if (fs.existsSync(pathsD)) { + for (const file of fs.readdirSync(pathsD).sort()) { + const content = fs.readFileSync(path.join(pathsD, file), 'utf8'); + for (const line of content.split('\n')) { + const p = line.trim(); + if (p) systemPaths.push(p); + } + } + } + } catch (_) { /* ignore */ } + + // Strategy 3: well-known user-local bin directories const home = os.homedir(); const fallbackDirs = [ - path.join(home, '.nvm/versions/node'), + path.join(home, '.local/bin'), path.join(home, '.volta/bin'), path.join(home, '.fnm/aliases/default/bin'), path.join(home, '.bun/bin'), path.join(home, '.cargo/bin'), - path.join(home, '.local/bin'), '/opt/homebrew/bin', '/usr/local/bin', ]; - // For nvm, resolve the current default version dynamically const nvmDir = path.join(home, '.nvm/versions/node'); try { if (fs.existsSync(nvmDir)) { @@ -58,10 +87,14 @@ function getShellPath() { } } catch (_) { /* ignore */ } - const existing = fallbackDirs.filter((d) => { - try { return fs.statSync(d).isDirectory(); } catch { return false; } - }); - return [...existing, process.env.PATH || ''].join(':'); + const seen = new Set(); + const dirs = []; + for (const d of [...fallbackDirs, ...systemPaths, ...(process.env.PATH || '').split(':')]) { + if (!d || seen.has(d)) continue; + seen.add(d); + try { if (fs.statSync(d).isDirectory()) dirs.push(d); } catch { /* skip */ } + } + return dirs.join(':'); } function getResourcePath(...segments) { @@ -190,6 +223,11 @@ function createWindow() { mainWindow.loadFile(frontendPath); } + mainWindow.webContents.on('will-attach-webview', (_event, webPreferences, _params) => { + webPreferences.plugins = true; + webPreferences.enableBlinkFeatures = 'EncryptedMedia'; + }); + mainWindow.on('closed', () => { mainWindow = null; }); @@ -202,6 +240,7 @@ function sendToRenderer(channel, ...args) { } function setupAutoUpdater() { + if (!autoUpdater) return; autoUpdater.autoDownload = false; autoUpdater.autoInstallOnAppQuit = false; @@ -252,6 +291,68 @@ app.whenReady().then(async () => { try { app.dock.setIcon(iconPath); } catch (_) {} } + session.defaultSession.setPermissionRequestHandler((_wc, permission, callback) => { + const allowed = [ + 'media', 'mediaKeySystem', 'protected-media-identifier', + 'geolocation', 'notifications', 'midi', 'midiSysex', + 'clipboard-read', 'clipboard-sanitized-write', + 'pointerLock', 'fullscreen', 'idle-detection', + ]; + console.log('Permission request:', permission, '->', allowed.includes(permission) ? 'granted' : 'denied'); + callback(allowed.includes(permission)); + }); + session.defaultSession.setPermissionCheckHandler((_wc, permission) => { + const allowed = [ + 'media', 'mediaKeySystem', 'protected-media-identifier', + 'clipboard-read', 'clipboard-sanitized-write', + 'pointerLock', 'fullscreen', 'idle-detection', + ]; + return allowed.includes(permission); + }); + + // Read-only logging for DRM license requests — no modifying interceptors + // so the network stack can set Content-Type and other headers normally. + session.defaultSession.webRequest.onSendHeaders( + { urls: ['*://*/*widevine*license*'] }, + (details) => { + console.log(`[drm-req] ${details.method} ${details.url}`); + for (const [k, v] of Object.entries(details.requestHeaders || {})) { + if (/content-type|origin|referer|auth|accept/i.test(k)) { + console.log(`[drm-req] ${k}: ${v}`); + } + } + }, + ); + session.defaultSession.webRequest.onCompleted( + { urls: ['*://*/*widevine*', '*://*/*license*'] }, + (details) => { + console.log(`[drm-net] ${details.method} ${details.url} → ${details.statusCode}`); + }, + ); + session.defaultSession.webRequest.onErrorOccurred( + { urls: ['*://*/*widevine*', '*://*/*license*'] }, + (details) => { + console.log(`[drm-net] FAILED ${details.method} ${details.url} → ${details.error}`); + }, + ); + + // Wait for the Widevine CDM to be downloaded/ready (CastLabs Component + // Updater Service). On first launch this downloads the CDM; subsequent + // launches use the cached version. + if (components && typeof components.whenReady === 'function') { + try { + await components.whenReady(); + console.log('Widevine CDM ready'); + if (typeof components.status === 'function') { + console.log('CDM component status:', JSON.stringify(components.status())); + } + } catch (err) { + console.warn('Widevine CDM not available:', err.message); + } + } else { + console.log('CastLabs components API not available — using standard Electron (no DRM)'); + } + try { if (isDev) { backendPort = parseInt(process.env.OPENSWARM_PORT || '8324', 10); @@ -269,6 +370,79 @@ app.whenReady().then(async () => { } }); +app.on('web-contents-created', (_event, contents) => { + contents.setWindowOpenHandler(({ url, disposition }) => { + if (disposition === 'foreground-tab' || disposition === 'background-tab') { + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send('webview-new-window', url, contents.id); + } + return { action: 'deny' }; + } + + return { + action: 'allow', + overrideBrowserWindowOptions: { + parent: mainWindow || undefined, + }, + }; + }); + + contents.on('did-create-window', (childWindow) => { + if (mainWindow && !mainWindow.isDestroyed() && !childWindow.isDestroyed()) { + childWindow.setParentWindow(mainWindow); + } + }); + + if (contents.getType() === 'webview') { + contents.on('console-message', (_e, level, message, line, sourceId) => { + if (message.includes('widevine') || message.includes('drm') || + message.includes('license') || message.includes('MediaKeySession') || + message.includes('EME') || message.includes('[drm-diag]') || level >= 2) { + const tag = ['LOG', 'INFO', 'WARN', 'ERROR'][level] || 'LOG'; + const src = sourceId ? sourceId.split('/').pop() : ''; + console.log(`[webview:${tag}] ${message}${src ? ` (${src}:${line})` : ''}`); + } + }); + + contents.on('dom-ready', () => { + const url = contents.getURL(); + if (url.includes('spotify')) { + contents.executeJavaScript(` + (function() { + const origFetch = window.fetch; + window.fetch = async function(...args) { + const resp = await origFetch.apply(this, args); + const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || ''; + if (url.includes('widevine-license') && !resp.ok) { + const clone = resp.clone(); + try { + const text = await clone.text(); + console.log('[drm-diag] License response ' + resp.status + ': ' + text.substring(0, 500)); + } catch(e) {} + } + return resp; + }; + + // Check EME availability + if (navigator.requestMediaKeySystemAccess) { + navigator.requestMediaKeySystemAccess('com.widevine.alpha', [{ + initDataTypes: ['cenc'], + audioCapabilities: [{contentType: 'audio/mp4; codecs="mp4a.40.2"'}], + }]).then(function(access) { + console.log('[drm-diag] Widevine EME access: ' + access.keySystem); + }).catch(function(err) { + console.log('[drm-diag] Widevine EME FAILED: ' + err.message); + }); + } else { + console.log('[drm-diag] EME API not available'); + } + })(); + `).catch(() => {}); + } + }); + } +}); + app.on('window-all-closed', () => { if (!isDev) killBackend(); app.quit(); @@ -286,9 +460,12 @@ app.on('activate', () => { ipcMain.handle('get-backend-port', () => backendPort); ipcMain.handle('get-app-version', () => app.getVersion()); +ipcMain.handle('get-webview-preload-path', () => { + return `file://${path.join(__dirname, 'webview-preload.js')}`; +}); ipcMain.handle('check-for-updates', async () => { - if (!isPackaged) { + if (!autoUpdater || !isPackaged) { sendToRenderer('update-error', 'Update check is only available in the packaged app.'); return { success: false, error: 'Not packaged' }; } @@ -305,6 +482,7 @@ ipcMain.handle('check-for-updates', async () => { }); ipcMain.handle('download-update', async () => { + if (!autoUpdater) return { success: false, error: 'Updater not available' }; try { await autoUpdater.downloadUpdate(); return { success: true }; @@ -314,6 +492,7 @@ ipcMain.handle('download-update', async () => { }); ipcMain.handle('install-update', () => { + if (!autoUpdater) return; autoUpdater.quitAndInstall(false, true); }); diff --git a/electron/package-lock.json b/electron/package-lock.json index 3358b1a3..769e8896 100644 --- a/electron/package-lock.json +++ b/electron/package-lock.json @@ -1,19 +1,20 @@ { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.4", + "hasInstallScript": true, "dependencies": { "electron-updater": "^6.3.0", "get-port": "^5.1.1" }, "devDependencies": { "@electron/notarize": "^3.1.1", - "electron": "^33.0.0", + "electron": "castlabs/electron-releases#v33.4.11+wvcus", "electron-builder": "^25.1.0" } }, @@ -2206,9 +2207,8 @@ } }, "node_modules/electron": { - "version": "33.4.11", - "resolved": "https://registry.npmjs.org/electron/-/electron-33.4.11.tgz", - "integrity": "sha512-xmdAs5QWRkInC7TpXGNvzo/7exojubk+72jn1oJL7keNeIlw7xNglf8TGtJtkR4rWC5FJq0oXiIXPS9BcK2Irg==", + "version": "33.4.11+wvcus", + "resolved": "git+ssh://git@github.com/castlabs/electron-releases.git#d1cf58c11ec0a8a04f307ed362d7efde2816778d", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3953,9 +3953,9 @@ } }, "node_modules/node-abi": { - "version": "3.88.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.88.0.tgz", - "integrity": "sha512-At6b4UqIEVudaqPsXjmUO1r/N5BUr4yhDGs5PkBE8/oG5+TfLPhFechiskFsnT6Ql0VfUXbalUUCbfXxtj7K+w==", + "version": "3.89.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", + "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", "dev": true, "license": "MIT", "dependencies": { @@ -4605,9 +4605,9 @@ } }, "node_modules/sax": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.5.0.tgz", - "integrity": "sha512-21IYA3Q5cQf089Z6tgaUTr7lDAyzoTPx5HRtbhsME8Udispad8dC/+sziTNugOEx54ilvatQ9YCzl4KQLPcRHA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", + "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", "license": "BlueOak-1.0.0", "engines": { "node": ">=11.0.0" diff --git a/electron/package.json b/electron/package.json index 48ee49a7..a3320e17 100644 --- a/electron/package.json +++ b/electron/package.json @@ -1,11 +1,13 @@ { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.4", "description": "OpenSwarm — AI Agent Orchestrator", "main": "main.js", "scripts": { "start": "electron .", "dev": "ELECTRON_DEV=1 electron .", + "postinstall": "bash scripts/sign-vmp.sh", + "sign-vmp": "bash scripts/sign-vmp.sh", "dist": "electron-builder --mac --publish never", "dist:publish": "electron-builder --mac --publish always", "dist:all": "electron-builder --mac --win --linux" @@ -16,12 +18,15 @@ }, "devDependencies": { "@electron/notarize": "^3.1.1", - "electron": "^33.0.0", + "electron": "castlabs/electron-releases#v33.4.11+wvcus", "electron-builder": "^25.1.0" }, "build": { "appId": "com.clusterlabs.openswarm", "productName": "OpenSwarm", + "electronDownload": { + "mirror": "https://github.com/castlabs/electron-releases/releases/download/v" + }, "directories": { "output": "dist" }, diff --git a/electron/preload.js b/electron/preload.js index 2f9037be..134e64c8 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -2,11 +2,13 @@ const { contextBridge, ipcRenderer } = require('electron'); (async () => { const port = await ipcRenderer.invoke('get-backend-port'); + const webviewPreloadPath = await ipcRenderer.invoke('get-webview-preload-path'); contextBridge.exposeInMainWorld('__OPENSWARM_PORT__', port); contextBridge.exposeInMainWorld('openswarm', { getBackendPort: () => port, + getWebviewPreloadPath: () => webviewPreloadPath, getAppVersion: () => ipcRenderer.invoke('get-app-version'), openExternal: (url) => ipcRenderer.invoke('open-external', url), @@ -40,5 +42,11 @@ const { contextBridge, ipcRenderer } = require('electron'); ipcRenderer.on('update-error', listener); return () => ipcRenderer.removeListener('update-error', listener); }, + + onWebviewNewWindow: (cb) => { + const listener = (_event, url, webContentsId) => cb(url, webContentsId); + ipcRenderer.on('webview-new-window', listener); + return () => ipcRenderer.removeListener('webview-new-window', listener); + }, }); })(); diff --git a/electron/scripts/sign-vmp.sh b/electron/scripts/sign-vmp.sh new file mode 100755 index 00000000..ba5eb833 --- /dev/null +++ b/electron/scripts/sign-vmp.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Signs the CastLabs Electron binary with a production VMP certificate via EVS, +# then repairs macOS framework symlinks that npm/signing may strip. +# +# First-time setup (one-time): +# pip3 install --user castlabs-evs +# python3 -m castlabs_evs.account signup +# +# After signup, this script runs automatically. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ELECTRON_DIR="$SCRIPT_DIR/../node_modules/electron/dist" +FW_BASE="$ELECTRON_DIR/Electron.app/Contents/Frameworks" + +fix_framework_symlinks() { + [ -d "$FW_BASE" ] || return 0 + for fw in "$FW_BASE"/*.framework; do + [ -d "$fw/Versions/A" ] || continue + local name + name=$(basename "$fw" .framework) + cd "$fw" + (cd Versions && ln -sf A Current 2>/dev/null) + ln -sf "Versions/Current/$name" "$name" 2>/dev/null + [ -d "Versions/A/Resources" ] && ln -sf Versions/Current/Resources Resources 2>/dev/null + [ -d "Versions/A/Libraries" ] && ln -sf Versions/Current/Libraries Libraries 2>/dev/null + [ -d "Versions/A/Helpers" ] && ln -sf Versions/Current/Helpers Helpers 2>/dev/null + done +} + +if [ ! -d "$ELECTRON_DIR" ]; then + echo "[vmp] Electron dist not found at $ELECTRON_DIR — skipping VMP signing" + fix_framework_symlinks + exit 0 +fi + +# Always fix symlinks first (npm git installs strip them) +fix_framework_symlinks + +if ! python3 -c "import castlabs_evs" 2>/dev/null; then + echo "[vmp] castlabs-evs not installed. Install with: pip3 install --user castlabs-evs" + echo "[vmp] Skipping VMP signing — DRM playback will be limited" + exit 0 +fi + +VERIFY_OUTPUT=$(python3 -m castlabs_evs.vmp verify-pkg "$ELECTRON_DIR" 2>&1) +if echo "$VERIFY_OUTPUT" | grep -q "Signature is valid" && ! echo "$VERIFY_OUTPUT" | grep -q "development only"; then + echo "[vmp] Electron already has a valid production VMP signature" + exit 0 +fi + +echo "[vmp] Signing Electron with production VMP certificate..." +if python3 -m castlabs_evs.vmp sign-pkg "$ELECTRON_DIR" 2>&1; then + echo "[vmp] VMP signing successful — full DRM playback enabled" + # Re-fix symlinks in case signing modified the bundle + fix_framework_symlinks +else + echo "[vmp] VMP signing failed — you may need to run: python3 -m castlabs_evs.account signup" + echo "[vmp] DRM playback will be limited to previews until signed" +fi + +exit 0 diff --git a/electron/webview-preload.js b/electron/webview-preload.js new file mode 100644 index 00000000..95a93d66 --- /dev/null +++ b/electron/webview-preload.js @@ -0,0 +1,87 @@ +/** + * Webview preload script — patches browser fingerprinting so sites like + * Spotify/Netflix don't detect an Electron shell and disable features. + * Loaded via the webview's `preload` attribute before any page script runs. + */ + +'use strict'; + +// Hide webdriver flag +Object.defineProperty(navigator, 'webdriver', { + get: () => false, + configurable: true, +}); + +// Spoof navigator.plugins (Chrome has a few built-in ones) +const fakePlugins = { + 0: { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, + 1: { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' }, + 2: { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }, + length: 3, + item: (i) => fakePlugins[i] || null, + namedItem: (name) => { + for (let i = 0; i < fakePlugins.length; i++) { + if (fakePlugins[i].name === name) return fakePlugins[i]; + } + return null; + }, + refresh: () => {}, + [Symbol.iterator]: function* () { + for (let i = 0; i < this.length; i++) yield this[i]; + }, +}; +try { + Object.defineProperty(navigator, 'plugins', { + get: () => fakePlugins, + configurable: true, + }); +} catch (_) {} + +// Ensure window.chrome exists (sites test for it) +if (!window.chrome) { + window.chrome = {}; +} +if (!window.chrome.runtime) { + window.chrome.runtime = { + connect: () => {}, + sendMessage: () => {}, + onMessage: { addListener: () => {}, removeListener: () => {} }, + }; +} + +// Ensure navigator.languages has sensible values +try { + Object.defineProperty(navigator, 'languages', { + get: () => ['en-US', 'en'], + configurable: true, + }); +} catch (_) {} + +// Patch permissions.query to report 'granted' for common permissions +const originalQuery = navigator.permissions?.query?.bind(navigator.permissions); +if (originalQuery) { + navigator.permissions.query = (params) => { + if (params.name === 'notifications') { + return Promise.resolve({ state: 'granted', onchange: null }); + } + return originalQuery(params).catch(() => + Promise.resolve({ state: 'prompt', onchange: null }) + ); + }; +} + +// Prevent iframe detection heuristics +try { + Object.defineProperty(document, 'hidden', { + get: () => false, + configurable: true, + }); + Object.defineProperty(document, 'visibilityState', { + get: () => 'visible', + configurable: true, + }); +} catch (_) {} + +// Fix console.debug detection (some sites use it as a breakpoint detector) +const noop = () => {}; +if (!window.console.debug) window.console.debug = noop; diff --git a/frontend/src/app/pages/AgentChat/AgentChat.tsx b/frontend/src/app/pages/AgentChat/AgentChat.tsx index 0c5c43c5..e10d80ca 100644 --- a/frontend/src/app/pages/AgentChat/AgentChat.tsx +++ b/frontend/src/app/pages/AgentChat/AgentChat.tsx @@ -527,11 +527,11 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose {renderItems.map((item) => { if (isToolGroup(item)) { const groupMeta = session.tool_group_meta?.[item.id]; - return ; + return ; } if (isToolPair(item)) { const isPending = item.result === null && sessionRunning; - return ; + return ; } const msg = item; const siblings = getSiblingBranches(msg.id); @@ -566,6 +566,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose key={`streaming-${session.streamingMessage.id}`} isStreaming isPending + sessionId={session.id} call={{ id: session.streamingMessage.id, role: 'tool_call', diff --git a/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx b/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx new file mode 100644 index 00000000..77130063 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx @@ -0,0 +1,392 @@ +import React, { useEffect, useRef, useMemo } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined'; +import CheckCircleOutlineIcon from '@mui/icons-material/CheckCircleOutline'; +import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; +import LanguageIcon from '@mui/icons-material/Language'; +import OpenInNewIcon from '@mui/icons-material/OpenInNew'; +import TouchAppOutlinedIcon from '@mui/icons-material/TouchAppOutlined'; +import KeyboardOutlinedIcon from '@mui/icons-material/KeyboardOutlined'; +import CameraAltOutlinedIcon from '@mui/icons-material/CameraAltOutlined'; +import ArticleOutlinedIcon from '@mui/icons-material/ArticleOutlined'; +import AccountTreeOutlinedIcon from '@mui/icons-material/AccountTreeOutlined'; +import CodeOutlinedIcon from '@mui/icons-material/CodeOutlined'; +import BuildOutlinedIcon from '@mui/icons-material/BuildOutlined'; +import { useAppSelector, useAppDispatch } from '@/shared/hooks'; +import { AgentMessage, AgentSession, fetchBrowserAgentChildren } from '@/shared/state/agentsSlice'; +import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; + +interface Props { + parentSessionId: string; + browserId?: string; +} + +interface FeedEntry { + type: 'thought' | 'action' | 'result' | 'system'; + text: string; + actionTool?: string; + sessionLabel?: string; +} + +function formatMessage(msg: AgentMessage): FeedEntry | null { + if (msg.role === 'user') return null; + + if (msg.role === 'assistant' && typeof msg.content === 'string') { + const trimmed = msg.content.trim(); + if (!trimmed) return null; + return { type: 'thought', text: trimmed }; + } + + if (msg.role === 'tool_call') { + const content = + typeof msg.content === 'string' + ? (() => { try { return JSON.parse(msg.content); } catch { return {}; } })() + : msg.content; + const tool = content?.tool || content?.name || '?'; + const input = content?.input || {}; + let brief = ''; + switch (tool) { + case 'BrowserNavigate': + brief = `Navigate → ${input.url || '...'}`; + break; + case 'BrowserClick': + brief = `Click ${input.selector || '...'}`; + break; + case 'BrowserType': { + const txt = (input.text || '').slice(0, 40); + const ellipsis = (input.text || '').length > 40 ? '…' : ''; + brief = `Type "${txt}${ellipsis}" into ${input.selector || '...'}`; + break; + } + case 'BrowserScreenshot': + brief = 'Screenshot'; + break; + case 'BrowserGetText': + brief = 'Read page text'; + break; + case 'BrowserGetElements': + brief = `Inspect elements${input.selector ? ` (${input.selector})` : ''}`; + break; + case 'BrowserEvaluate': + brief = `Evaluate JS`; + break; + default: + brief = `${tool}(${JSON.stringify(input).slice(0, 60)})`; + } + return { type: 'action', text: brief, actionTool: tool }; + } + + if (msg.role === 'tool_result') { + const content = + typeof msg.content === 'string' + ? (() => { try { return JSON.parse(msg.content); } catch { return { text: msg.content }; } })() + : msg.content; + const toolName = content?.tool_name || ''; + const elapsed = content?.elapsed_ms; + const text = content?.text || ''; + + if (toolName === 'BrowserScreenshot') { + return { type: 'result', text: `Screenshot captured${elapsed ? ` (${elapsed}ms)` : ''}` }; + } + const preview = text.length > 120 ? text.slice(0, 120) + '…' : text; + return { type: 'result', text: `${preview}${elapsed ? ` (${elapsed}ms)` : ''}` }; + } + + if (msg.role === 'system') { + return { type: 'system', text: typeof msg.content === 'string' ? msg.content : '' }; + } + + return null; +} + +type SvgIconComponent = typeof OpenInNewIcon; + +function getActionIcon(tool?: string): SvgIconComponent { + switch (tool) { + case 'BrowserNavigate': return OpenInNewIcon; + case 'BrowserClick': return TouchAppOutlinedIcon; + case 'BrowserType': return KeyboardOutlinedIcon; + case 'BrowserScreenshot': return CameraAltOutlinedIcon; + case 'BrowserGetText': return ArticleOutlinedIcon; + case 'BrowserGetElements': return AccountTreeOutlinedIcon; + case 'BrowserEvaluate': return CodeOutlinedIcon; + default: return BuildOutlinedIcon; + } +} + +interface FeedColors { + thought: string; + thoughtIcon: string; + result: string; + error: string; + errorIcon: string; + scrollThumb: string; +} + +const darkFeedColors: FeedColors = { + thought: '#a0aab8', + thoughtIcon: '#555b6e', + result: '#555b6e', + error: '#ff8787', + errorIcon: '#ff8787', + scrollThumb: '#2a2d3e', +}; + +const lightFeedColors: FeedColors = { + thought: '#555550', + thoughtIcon: '#9e9c95', + result: '#9e9c95', + error: '#c03030', + errorIcon: '#c03030', + scrollThumb: '#ccc9c0', +}; + +const BrowserAgentInlineFeed: React.FC = ({ parentSessionId, browserId }) => { + const c = useClaudeTokens(); + const dispatch = useAppDispatch(); + const { mode } = useThemeMode(); + const fc = mode === 'dark' ? darkFeedColors : lightFeedColors; + const scrollRef = useRef(null); + const fetchedRef = useRef(false); + + const browserSessions = useAppSelector((state) => { + const all = state.agents.sessions; + return Object.values(all).filter( + (s): s is AgentSession => + s.mode === 'browser-agent' && + s.parent_session_id === parentSessionId && + (!browserId || s.browser_id === browserId), + ); + }); + + useEffect(() => { + if (browserSessions.length === 0 && !fetchedRef.current) { + fetchedRef.current = true; + dispatch(fetchBrowserAgentChildren(parentSessionId)); + } + }, [browserSessions.length, parentSessionId, dispatch]); + + const sessionsWithEntries = useMemo(() => { + return browserSessions.map((session) => { + const entries: FeedEntry[] = []; + for (const msg of session.messages) { + const entry = formatMessage(msg); + if (entry) entries.push(entry); + } + if (session.streamingMessage?.role === 'assistant' && session.streamingMessage.content) { + entries.push({ type: 'thought', text: session.streamingMessage.content }); + } + return { session, entries }; + }); + }, [browserSessions]); + + const totalMessages = browserSessions.reduce( + (n, s) => n + s.messages.length + (s.streamingMessage ? 1 : 0), + 0, + ); + + useEffect(() => { + if (scrollRef.current) { + scrollRef.current.scrollTop = scrollRef.current.scrollHeight; + } + }, [totalMessages]); + + if (browserSessions.length === 0) return null; + + const showLabels = sessionsWithEntries.length > 1; + const accentColor = c.accent.primary; + + return ( + + {sessionsWithEntries.map(({ session, entries }, si) => ( + + {showLabels && ( + 0 ? 1 : 0, mb: 0.25 }}> + + + {session.browser_id || `Browser ${si + 1}`} + + + + )} + + {!showLabels && entries.length === 0 && session.status === 'running' && ( + + Starting browser agent... + + )} + + {entries.map((entry, i) => ( + + ))} + + {!showLabels && session.status === 'running' && entries.length > 0 && ( + + + + )} + + ))} + + ); +}; + +const EntryRow: React.FC<{ entry: FeedEntry; accentColor: string; fc: FeedColors }> = ({ entry, accentColor, fc }) => { + const c = useClaudeTokens(); + + if (entry.type === 'thought') { + return ( + + + + {entry.text} + + + ); + } + + if (entry.type === 'action') { + const ActionIcon = getActionIcon(entry.actionTool); + return ( + + + + {entry.text} + + + ); + } + + if (entry.type === 'result') { + return ( + + + ↳ {entry.text} + + + ); + } + + if (entry.type === 'system') { + return ( + + + + {entry.text} + + + ); + } + + return null; +}; + +const SessionStatusChip: React.FC<{ status: string }> = ({ status }) => { + const c = useClaudeTokens(); + if (status === 'running') { + return ( + + ); + } + if (status === 'completed') { + return ; + } + if (status === 'error') { + return ; + } + return null; +}; + +export default React.memo(BrowserAgentInlineFeed); diff --git a/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx b/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx index f058b69e..0fb31fb5 100644 --- a/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx +++ b/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx @@ -19,6 +19,7 @@ import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; import { AgentMessage } from '@/shared/state/agentsSlice'; import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; +import BrowserAgentInlineFeed from './BrowserAgentInlineFeed'; const GoogleServiceIcon: React.FC<{ service: string; size?: number }> = ({ service, size = 14 }) => { if (service === 'gmail') { @@ -466,6 +467,7 @@ interface ToolCallBubbleProps { isPending?: boolean; isStreaming?: boolean; mcpCompact?: boolean; + sessionId?: string; } interface TermColors { @@ -1169,8 +1171,14 @@ const McpResultCard: React.FC<{ parsed: ParsedMcpResult; compact?: boolean }> = return ; }; +function isBrowserAgentTool(name: string): boolean { + if (name === 'BrowserAgent' || name === 'BrowserAgents') return true; + const mcp = parseMcpToolName(name); + return mcp.isMcp && mcp.serverSlug === 'openswarm-browser-agent'; +} + const ToolCallBubble: React.FC = React.memo( - ({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false }) => { + ({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false, sessionId }) => { const c = useClaudeTokens(); const tc = useTermColors(); const [expanded, setExpanded] = useState(false); @@ -1180,7 +1188,10 @@ const ToolCallBubble: React.FC = React.memo( const inputSummary = getInputSummary(toolName, input); const formattedInput = useMemo(() => formatInputDisplay(toolName, input), [toolName, input]); const showTimer = isPending && !isDenied && !isStreaming; - const showBody = expanded || isStreaming; + + const isBrowserAgent = isBrowserAgentTool(toolName); + const browserAgentAutoExpand = isBrowserAgent && isPending && !isStreaming; + const showBody = expanded || isStreaming || browserAgentAutoExpand; const resultContent = result?.content; const hasStructuredResult = @@ -1320,6 +1331,12 @@ const ToolCallBubble: React.FC = React.memo( '&::-webkit-scrollbar-thumb': { background: tc.SCROLLBAR_THUMB, borderRadius: 3 }, }} > + {isBrowserAgent && sessionId && ( + + )} {parsedResult && parsedResult.type === 'mcp' ? ( ) : parsedResult ? ( @@ -1330,7 +1347,7 @@ const ToolCallBubble: React.FC = React.memo( {parsedResult.type === 'text' ? parsedResult.content : ''} ) : null} - {!parsedResult && isPending && !isStreaming && ( + {!parsedResult && isPending && !isStreaming && !isBrowserAgent && ( @@ -1525,6 +1542,14 @@ const ToolCallBubble: React.FC = React.memo( )} + {/* Browser agent inline feed */} + {isBrowserAgent && sessionId && ( + + )} + {/* Output */} {parsedResult && parsedResult.type === 'mcp' ? ( @@ -1566,8 +1591,8 @@ const ToolCallBubble: React.FC = React.memo( ) : null} - {/* Pending indicator when waiting for result */} - {!parsedResult && isPending && !isStreaming && ( + {/* Pending indicator when waiting for result (skip for browser agent — feed replaces it) */} + {!parsedResult && isPending && !isStreaming && !isBrowserAgent && ( = React.memo(({ group, isSessionRunning = false, meta }) => { +const ToolGroupBubble: React.FC = React.memo(({ group, isSessionRunning = false, meta, sessionId }) => { const c = useClaudeTokens(); const isMcp = !!group.mcpServer; const [expanded, setExpanded] = useState(isMcp); @@ -186,6 +187,7 @@ const ToolGroupBubble: React.FC = React.memo(({ group, isSessionRunning = result={pair.result} isPending={pair.result === null && isSessionRunning} mcpCompact + sessionId={sessionId} /> ))} diff --git a/frontend/src/app/pages/Dashboard/BrowserCard.tsx b/frontend/src/app/pages/Dashboard/BrowserCard.tsx index d7772294..50ecd827 100644 --- a/frontend/src/app/pages/Dashboard/BrowserCard.tsx +++ b/frontend/src/app/pages/Dashboard/BrowserCard.tsx @@ -67,6 +67,14 @@ const HANDLE_DEFS: { dir: ResizeDir; sx: Record }[] = [ const isElectron = navigator.userAgent.includes('Electron'); +const chromeUserAgent = navigator.userAgent + .replace(/\s*Electron\/\S+/, '') + .replace(/\s*OpenSwarm\/\S+/, ''); + +const webviewPreloadPath: string | undefined = isElectron + ? (window as any).openswarm?.getWebviewPreloadPath?.() + : undefined; + type WebviewElement = BrowserWebview; interface TabLocalState { @@ -192,10 +200,6 @@ const BrowserCard: React.FC = ({ onTitleUpdate(); }; - const onNewWindow = (e: any) => { - if (e.url) dispatch(addBrowserTab({ browserId, url: e.url, makeActive: true })); - }; - const onFaviconUpdate = (e: any) => { const favicons = e.favicons || (e.detail && e.detail.favicons); if (favicons?.[0]) { @@ -208,7 +212,6 @@ const BrowserCard: React.FC = ({ wv.addEventListener('page-title-updated', onTitleUpdate); wv.addEventListener('did-start-loading', onLoadStart); wv.addEventListener('did-stop-loading', onLoadStop); - wv.addEventListener('new-window', onNewWindow); wv.addEventListener('page-favicon-updated', onFaviconUpdate); cleanups.push(() => { @@ -218,7 +221,6 @@ const BrowserCard: React.FC = ({ wv.removeEventListener('page-title-updated', onTitleUpdate); wv.removeEventListener('did-start-loading', onLoadStart); wv.removeEventListener('did-stop-loading', onLoadStop); - wv.removeEventListener('new-window', onNewWindow); wv.removeEventListener('page-favicon-updated', onFaviconUpdate); }); } @@ -988,6 +990,9 @@ const BrowserCard: React.FC = ({ data-tab-id={tab.id} src="about:blank" allowpopups="true" + useragent={chromeUserAgent} + {...(webviewPreloadPath ? { preload: webviewPreloadPath } : {})} + webpreferences="plugins=yes, autoplayPolicy=no-user-gesture-required" style={{ position: 'absolute', top: 0, diff --git a/frontend/src/app/pages/Dashboard/Dashboard.tsx b/frontend/src/app/pages/Dashboard/Dashboard.tsx index 2aa9393b..c6ce78ca 100644 --- a/frontend/src/app/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/app/pages/Dashboard/Dashboard.tsx @@ -22,6 +22,7 @@ import { tidyLayout, addViewCard, addBrowserCard, + addBrowserTab, moveCards, resetLayout, setGlowingBrowserCards, @@ -31,6 +32,7 @@ import { fetchOutputs } from '@/shared/state/outputsSlice'; import { generateDashboardName, updateDashboardThumbnail } from '@/shared/state/dashboardsSlice'; import { dashboardWs } from '@/shared/ws/WebSocketManager'; import { initBrowserCommandHandler } from '@/shared/browserCommandHandler'; +import { findBrowserByWebContentsId } from '@/shared/browserRegistry'; import AgentCard from './AgentCard'; import DashboardViewCard from './DashboardViewCard'; import BrowserCard from './BrowserCard'; @@ -200,6 +202,26 @@ const DashboardInner: React.FC = () => { return () => { cleanupBrowserHandler(); dashboardWs.disconnect(); }; }, [dispatch, dashboardId]); + useEffect(() => { + const w = window as any; + if (!w.openswarm?.onWebviewNewWindow) return; + let lastUrl = ''; + let lastTime = 0; + return w.openswarm.onWebviewNewWindow((url: string, webContentsId: number) => { + const now = Date.now(); + if (url === lastUrl && now - lastTime < 1000) return; + lastUrl = url; + lastTime = now; + + const browserId = findBrowserByWebContentsId(webContentsId); + if (browserId) { + dispatch(addBrowserTab({ browserId, url, makeActive: true })); + } else { + dispatch(addBrowserCard({ url, expandedSessionIds })); + } + }); + }, [dispatch, expandedSessionIds]); + // Capture a thumbnail screenshot of the dashboard. // Uses Electron's native capturePage for pixel-perfect results. // Captures current viewport as-is (no DOM mutation) to avoid visual flashes. diff --git a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx index 38b000b0..d44f4fb6 100644 --- a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx @@ -204,13 +204,17 @@ const DashboardToolbar = React.forwardRef( const isExpanded = inputOpen || viewPickerOpen || historyOpen; + const autoSelectOnNew = useAppSelector((s) => s.settings.data.auto_select_mode_on_new_agent); const prevInputOpenRef = useRef(inputOpen); useEffect(() => { if (prevInputOpenRef.current && !inputOpen && elementSelection?.selectMode) { elementSelection.setSelectMode(false); } + if (!prevInputOpenRef.current && inputOpen && autoSelectOnNew) { + elementSelection?.setSelectMode(true); + } prevInputOpenRef.current = inputOpen; - }, [inputOpen, elementSelection]); + }, [inputOpen, elementSelection, autoSelectOnNew]); useEffect(() => { if (!isExpanded) return; diff --git a/frontend/src/app/pages/Settings/Settings.tsx b/frontend/src/app/pages/Settings/Settings.tsx index 70a14a8d..58d71b10 100644 --- a/frontend/src/app/pages/Settings/Settings.tsx +++ b/frontend/src/app/pages/Settings/Settings.tsx @@ -11,6 +11,7 @@ import InputAdornment from '@mui/material/InputAdornment'; import ToggleButton from '@mui/material/ToggleButton'; import ToggleButtonGroup from '@mui/material/ToggleButtonGroup'; import Slider from '@mui/material/Slider'; +import Switch from '@mui/material/Switch'; import Snackbar from '@mui/material/Snackbar'; import Alert from '@mui/material/Alert'; import Tab from '@mui/material/Tab'; @@ -495,7 +496,7 @@ const Settings: React.FC = () => { - + New agent shortcut Keyboard shortcut to create an agent. @@ -553,6 +554,21 @@ const Settings: React.FC = () => { + + + Auto-enable element selection + Automatically enter element selection mode when creating a new agent. + + setForm({ ...form, auto_select_mode_on_new_agent: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {/* ── Browser ── */} Browser diff --git a/frontend/src/app/pages/Tools/Tools.tsx b/frontend/src/app/pages/Tools/Tools.tsx index a6e05224..22c7bdad 100644 --- a/frontend/src/app/pages/Tools/Tools.tsx +++ b/frontend/src/app/pages/Tools/Tools.tsx @@ -488,7 +488,8 @@ const Tools: React.FC = () => { if (discoverTools.fulfilled.match(result)) { setSnackbar({ open: true, message: 'Actions discovered successfully' }); } else { - setSnackbar({ open: true, message: 'Discovery failed — is the MCP server running?', severity: 'error' }); + const detail = (result as any).error?.message || 'Discovery failed — is the MCP server running?'; + setSnackbar({ open: true, message: detail, severity: 'error' }); } } finally { setDiscovering(false); diff --git a/frontend/src/shared/browserRegistry.ts b/frontend/src/shared/browserRegistry.ts index 09e79f52..a8eba9a5 100644 --- a/frontend/src/shared/browserRegistry.ts +++ b/frontend/src/shared/browserRegistry.ts @@ -51,6 +51,15 @@ export function getAllWebviews(): Map { return new Map(registry); } +export function findBrowserByWebContentsId(wcId: number): string | undefined { + for (const [key, wv] of registry.entries()) { + if ((wv as any).getWebContentsId?.() === wcId) { + return key.split(':')[0]; + } + } + return undefined; +} + export function unregisterAllForBrowser(browserId: string): void { const prefix = `${browserId}:`; for (const key of registry.keys()) { diff --git a/frontend/src/shared/state/agentsSlice.ts b/frontend/src/shared/state/agentsSlice.ts index 4fb21107..2be0f62c 100644 --- a/frontend/src/shared/state/agentsSlice.ts +++ b/frontend/src/shared/state/agentsSlice.ts @@ -69,6 +69,7 @@ export interface AgentSession { tool_group_meta: Record; dashboard_id?: string; browser_id?: string | null; + parent_session_id?: string | null; } export interface AgentConfig { @@ -380,6 +381,15 @@ export const resumeSession = createAsyncThunk( } ); +export const fetchBrowserAgentChildren = createAsyncThunk( + 'agents/fetchBrowserAgentChildren', + async (parentSessionId: string) => { + const res = await fetch(`${AGENTS_API}/sessions/${parentSessionId}/browser-agents`); + const data = await res.json(); + return data.sessions as AgentSession[]; + } +); + const agentsSlice = createSlice({ name: 'agents', initialState, @@ -629,7 +639,19 @@ const agentsSlice = createSlice({ closeSessionFromWs(state, action: PayloadAction) { const entry = action.payload; state.history[entry.id] = entry; - delete state.sessions[entry.id]; + + const session = state.sessions[entry.id]; + if (session?.mode === 'browser-agent' && session.parent_session_id) { + session.status = (entry.status as AgentSession['status']) || 'completed'; + } else { + delete state.sessions[entry.id]; + for (const [id, s] of Object.entries(state.sessions)) { + if (s.mode === 'browser-agent' && s.parent_session_id === entry.id) { + delete state.sessions[id]; + } + } + } + if (state.activeSessionId === entry.id) { state.activeSessionId = null; } @@ -816,6 +838,17 @@ const agentsSlice = createSlice({ }; } }) + .addCase(fetchBrowserAgentChildren.fulfilled, (state, action) => { + for (const session of action.payload) { + if (!state.sessions[session.id]) { + state.sessions[session.id] = { + ...session, + streamingMessage: null, + tool_group_meta: session.tool_group_meta ?? {}, + }; + } + } + }) .addCase(searchHistory.pending, (state) => { state.historySearch.loading = true; }) diff --git a/frontend/src/shared/state/settingsSlice.ts b/frontend/src/shared/state/settingsSlice.ts index 58961793..4530871c 100644 --- a/frontend/src/shared/state/settingsSlice.ts +++ b/frontend/src/shared/state/settingsSlice.ts @@ -22,6 +22,7 @@ export interface AppSettings { new_agent_shortcut: string; anthropic_api_key: string | null; browser_homepage: string; + auto_select_mode_on_new_agent: boolean; } export interface BrowseResult { @@ -50,6 +51,7 @@ const initialState: SettingsState = { new_agent_shortcut: 'Meta+l', anthropic_api_key: null, browser_homepage: 'https://www.google.com', + auto_select_mode_on_new_agent: false, }, loading: false, loaded: false, diff --git a/frontend/src/types/electron.d.ts b/frontend/src/types/electron.d.ts index a9f3557b..1fe719e4 100644 --- a/frontend/src/types/electron.d.ts +++ b/frontend/src/types/electron.d.ts @@ -10,6 +10,8 @@ declare global { partition?: string; allowpopups?: string; nodeintegration?: string; + webpreferences?: string; + useragent?: string; }, HTMLElement >; @@ -31,6 +33,7 @@ declare global { interface OpenSwarmAPI { getBackendPort: () => number; + getWebviewPreloadPath: () => string; getAppVersion: () => Promise; checkForUpdates: () => Promise<{ success: boolean; version?: string; error?: string }>; downloadUpdate: () => Promise<{ success: boolean; error?: string }>; @@ -40,6 +43,7 @@ declare global { onDownloadProgress: (cb: (progress: OpenSwarmDownloadProgress) => void) => () => void; onUpdateDownloaded: (cb: (info: OpenSwarmUpdateInfo) => void) => () => void; onUpdateError: (cb: (message: string) => void) => () => void; + onWebviewNewWindow: (cb: (url: string, webContentsId: number) => void) => () => void; } interface Window { diff --git a/run.sh b/run.sh index 68a48501..494a6a7e 100755 --- a/run.sh +++ b/run.sh @@ -127,10 +127,18 @@ if (( frontend_elapsed >= FRONTEND_MAX_WAIT )); then exit 1 fi +# --- Sign Electron VMP for DRM (if EVS account exists) --- +if [ -f "$PROJECT_ROOT/electron/scripts/sign-vmp.sh" ]; then + echo -e "${YELLOW}${BOLD}[vmp]${RESET} Checking VMP signature..." + bash "$PROJECT_ROOT/electron/scripts/sign-vmp.sh" 2>&1 | while IFS= read -r line; do + printf "${YELLOW}${BOLD}%s${RESET}\n" "$line" + done +fi + # --- Start Electron in dev mode --- MAGENTA='\033[0;35m' echo -e "${MAGENTA}${BOLD}[electron]${RESET} Launching Electron dev shell..." -(cd "$PROJECT_ROOT/electron" && ELECTRON_DEV=1 npx electron .) > >( +(cd "$PROJECT_ROOT/electron" && unset ELECTRON_RUN_AS_NODE && ELECTRON_DEV=1 npx electron .) > >( while IFS= read -r line; do printf "${MAGENTA}${BOLD}[electron]${RESET} %s\n" "$line" done