From 6f02daff2d738262de9196cf153e8cf6dda26a54 Mon Sep 17 00:00:00 2001 From: haikdc Date: Tue, 17 Mar 2026 01:48:50 -0700 Subject: [PATCH 01/11] [Haik]: (Browser actions shown in agent card output) (User can have item selection turned on by default in settings) (most browser related functionality works - altho spotify is tricky) (mcp discovery in prod shld worke now?) --- .gitignore | 1 + backend/apps/agents/agent_manager.py | 19 + backend/apps/agents/agents.py | 5 + backend/apps/agents/browser_agent.py | 4 + .../apps/agents/browser_agent_mcp_server.py | 2 + backend/apps/agents/models.py | 1 + backend/apps/settings/models.py | 1 + backend/apps/tools_lib/tools_lib.py | 4 +- backend/main.py | 2 + electron/main.js | 203 ++++++++- electron/package-lock.json | 24 +- electron/package.json | 9 +- electron/preload.js | 8 + electron/scripts/sign-vmp.sh | 61 +++ electron/webview-preload.js | 87 ++++ .../src/app/pages/AgentChat/AgentChat.tsx | 5 +- .../AgentChat/BrowserAgentInlineFeed.tsx | 392 ++++++++++++++++++ .../app/pages/AgentChat/ToolCallBubble.tsx | 35 +- .../app/pages/AgentChat/ToolGroupBubble.tsx | 4 +- .../src/app/pages/Dashboard/BrowserCard.tsx | 17 +- .../src/app/pages/Dashboard/Dashboard.tsx | 22 + .../app/pages/Dashboard/DashboardToolbar.tsx | 6 +- frontend/src/app/pages/Settings/Settings.tsx | 18 +- frontend/src/app/pages/Tools/Tools.tsx | 3 +- frontend/src/shared/browserRegistry.ts | 9 + frontend/src/shared/state/agentsSlice.ts | 35 +- frontend/src/shared/state/settingsSlice.ts | 2 + frontend/src/types/electron.d.ts | 4 + run.sh | 10 +- 29 files changed, 947 insertions(+), 46 deletions(-) create mode 100755 electron/scripts/sign-vmp.sh create mode 100644 electron/webview-preload.js create mode 100644 frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx diff --git a/.gitignore b/.gitignore index 47256d70..d546f98d 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ backend/data/** electron/dist/ electron/python-env/ electron/node_modules/ +electron/package-lock.json # Frontend build output frontend/dist/ \ No newline at end of file diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index c92b6b10..5913cb15 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -621,6 +621,7 @@ class AgentManager: "OPENSWARM_AGENT_MODEL": session.model, "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), + "OPENSWARM_PARENT_SESSION_ID": session.id, }, "type": "stdio", } @@ -1401,4 +1402,22 @@ class AgentManager: def get_session(self, session_id: str) -> Optional[AgentSession]: return self.sessions.get(session_id) + def get_browser_agent_children(self, parent_session_id: str) -> list[dict]: + """Return browser-agent sessions for a parent, from memory or disk.""" + results: list[dict] = [] + seen: set[str] = set() + + for s in self.sessions.values(): + if s.mode == "browser-agent" and s.parent_session_id == parent_session_id: + results.append(s.model_dump(mode="json")) + seen.add(s.id) + + for sid, data in _load_all_session_data(): + if sid in seen: + continue + if data.get("mode") == "browser-agent" and data.get("parent_session_id") == parent_session_id: + results.append(data) + + return results + agent_manager = AgentManager() diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 462b8fde..c4cd2df1 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -151,6 +151,11 @@ async def get_history(q: str = "", limit: int = 20, offset: int = 0, dashboard_i dashboard_id=dashboard_id or None, ) +@agents.router.get("/sessions/{session_id}/browser-agents") +async def get_browser_agent_children(session_id: str): + children = agent_manager.get_browser_agent_children(session_id) + return {"sessions": children} + @agents.router.post("/sessions/{session_id}/resume") async def resume_session(session_id: str): try: diff --git a/backend/apps/agents/browser_agent.py b/backend/apps/agents/browser_agent.py index 11244acf..ea2064e6 100644 --- a/backend/apps/agents/browser_agent.py +++ b/backend/apps/agents/browser_agent.py @@ -188,6 +188,7 @@ async def run_browser_agent( tab_id: str = "", pre_selected: bool = False, initial_url: str | None = None, + parent_session_id: str | None = None, ) -> dict: """Run a browser sub-agent loop for a single browser card. @@ -206,6 +207,7 @@ async def run_browser_agent( dashboard_id=dashboard_id, browser_id=browser_id, system_prompt=SYSTEM_PROMPT, + parent_session_id=parent_session_id, ) agent_manager.sessions[session_id] = session @@ -425,6 +427,7 @@ async def run_browser_agents( api_key: str, dashboard_id: str | None = None, pre_selected_browser_ids: list[str] | None = None, + parent_session_id: str | None = None, ) -> list[dict]: """Run multiple browser sub-agents in parallel. @@ -451,6 +454,7 @@ async def run_browser_agents( dashboard_id=dashboard_id, pre_selected=is_pre_selected, initial_url=url if url and browser_id not in pre_selected else None, + parent_session_id=parent_session_id, ) results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True) diff --git a/backend/apps/agents/browser_agent_mcp_server.py b/backend/apps/agents/browser_agent_mcp_server.py index 2a46a139..a229b852 100644 --- a/backend/apps/agents/browser_agent_mcp_server.py +++ b/backend/apps/agents/browser_agent_mcp_server.py @@ -25,6 +25,7 @@ BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/browser-agent/run" MODEL = os.environ.get("OPENSWARM_AGENT_MODEL", "sonnet") DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "") PRE_SELECTED_BROWSER_IDS = os.environ.get("OPENSWARM_PRE_SELECTED_BROWSER_IDS", "") +PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") TOOLS = [ { @@ -119,6 +120,7 @@ def call_backend(tasks: list[dict]) -> dict: "model": MODEL, "dashboard_id": DASHBOARD_ID, "pre_selected_browser_ids": pre_selected, + "parent_session_id": PARENT_SESSION_ID, }).encode() req = urllib.request.Request( BACKEND_URL, diff --git a/backend/apps/agents/models.py b/backend/apps/agents/models.py index a5f03df3..41480761 100644 --- a/backend/apps/agents/models.py +++ b/backend/apps/agents/models.py @@ -72,3 +72,4 @@ class AgentSession(BaseModel): tool_group_meta: dict[str, "ToolGroupMeta"] = Field(default_factory=dict) dashboard_id: Optional[str] = None browser_id: Optional[str] = None + parent_session_id: Optional[str] = None diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index 343b245e..f2ef9988 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -21,3 +21,4 @@ class AppSettings(BaseModel): new_agent_shortcut: str = "Meta+l" anthropic_api_key: Optional[str] = None browser_homepage: str = "https://www.google.com" + auto_select_mode_on_new_agent: bool = False diff --git a/backend/apps/tools_lib/tools_lib.py b/backend/apps/tools_lib/tools_lib.py index f19add53..d440bda0 100644 --- a/backend/apps/tools_lib/tools_lib.py +++ b/backend/apps/tools_lib/tools_lib.py @@ -18,9 +18,11 @@ from backend.apps.tools_lib.models import ToolDefinition, ToolCreate, ToolUpdate logger = logging.getLogger(__name__) -from backend.config.paths import BACKEND_DIR, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH +from backend.config.paths import BACKEND_DIR, DATA_ROOT, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH load_dotenv(os.path.join(BACKEND_DIR, ".env")) +if os.environ.get("OPENSWARM_PACKAGED") == "1": + load_dotenv(os.path.join(os.path.dirname(DATA_ROOT), ".env"), override=True) @asynccontextmanager diff --git a/backend/main.py b/backend/main.py index d7f30cee..81c15119 100644 --- a/backend/main.py +++ b/backend/main.py @@ -126,6 +126,7 @@ async def browser_agent_run(request: Request): model = body.get("model", "sonnet") dashboard_id = body.get("dashboard_id", "") pre_selected_browser_ids = body.get("pre_selected_browser_ids", []) + parent_session_id = body.get("parent_session_id", "") if not tasks: return JSONResponse({"error": "tasks array is required"}, status_code=400) @@ -140,6 +141,7 @@ async def browser_agent_run(request: Request): api_key=settings.anthropic_api_key, dashboard_id=dashboard_id or None, pre_selected_browser_ids=pre_selected_browser_ids, + parent_session_id=parent_session_id or None, ) return JSONResponse({"results": results}) diff --git a/electron/main.js b/electron/main.js index d5f79a99..90b909c1 100644 --- a/electron/main.js +++ b/electron/main.js @@ -1,5 +1,6 @@ -const { app, BrowserWindow, ipcMain, shell } = require('electron'); -const { autoUpdater } = require('electron-updater'); +const { app, components, BrowserWindow, ipcMain, shell, session } = require('electron'); +let autoUpdater; +try { autoUpdater = require('electron-updater').autoUpdater; } catch (_) {} const path = require('path'); const { spawn, execFileSync } = require('child_process'); const os = require('os'); @@ -7,6 +8,12 @@ const fs = require('fs'); const getPort = require('get-port'); const http = require('http'); +app.commandLine.appendSwitch('disable-features', 'HardwareMediaKeyHandling'); +app.commandLine.appendSwitch('ignore-gpu-blocklist'); +app.commandLine.appendSwitch('enable-gpu-rasterization'); +app.commandLine.appendSwitch('enable-zero-copy'); +app.commandLine.appendSwitch('autoplay-policy', 'no-user-gesture-required'); + let mainWindow = null; let backendProcess = null; let backendPort = null; @@ -24,9 +31,10 @@ const iconPath = path.join(__dirname, 'build', 'icon.png'); function getShellPath() { if (process.platform !== 'darwin' || isDev) return process.env.PATH || ''; + // Strategy 1: ask the user's login shell for its PATH try { - const shell = process.env.SHELL || '/bin/zsh'; - const result = execFileSync(shell, ['-ilc', 'echo $PATH'], { + const userShell = process.env.SHELL || '/bin/zsh'; + const result = execFileSync(userShell, ['-ilc', 'echo $PATH'], { encoding: 'utf8', timeout: 5000, env: { ...process.env, HOME: os.homedir() }, @@ -35,19 +43,40 @@ function getShellPath() { if (resolved) return resolved; } catch (_) { /* fall through */ } + // Strategy 2: read macOS system PATH config (/etc/paths + /etc/paths.d/*) + const systemPaths = []; + try { + const base = fs.readFileSync('/etc/paths', 'utf8'); + for (const line of base.split('\n')) { + const p = line.trim(); + if (p) systemPaths.push(p); + } + } catch (_) { /* ignore */ } + try { + const pathsD = '/etc/paths.d'; + if (fs.existsSync(pathsD)) { + for (const file of fs.readdirSync(pathsD).sort()) { + const content = fs.readFileSync(path.join(pathsD, file), 'utf8'); + for (const line of content.split('\n')) { + const p = line.trim(); + if (p) systemPaths.push(p); + } + } + } + } catch (_) { /* ignore */ } + + // Strategy 3: well-known user-local bin directories const home = os.homedir(); const fallbackDirs = [ - path.join(home, '.nvm/versions/node'), + path.join(home, '.local/bin'), path.join(home, '.volta/bin'), path.join(home, '.fnm/aliases/default/bin'), path.join(home, '.bun/bin'), path.join(home, '.cargo/bin'), - path.join(home, '.local/bin'), '/opt/homebrew/bin', '/usr/local/bin', ]; - // For nvm, resolve the current default version dynamically const nvmDir = path.join(home, '.nvm/versions/node'); try { if (fs.existsSync(nvmDir)) { @@ -58,10 +87,14 @@ function getShellPath() { } } catch (_) { /* ignore */ } - const existing = fallbackDirs.filter((d) => { - try { return fs.statSync(d).isDirectory(); } catch { return false; } - }); - return [...existing, process.env.PATH || ''].join(':'); + const seen = new Set(); + const dirs = []; + for (const d of [...fallbackDirs, ...systemPaths, ...(process.env.PATH || '').split(':')]) { + if (!d || seen.has(d)) continue; + seen.add(d); + try { if (fs.statSync(d).isDirectory()) dirs.push(d); } catch { /* skip */ } + } + return dirs.join(':'); } function getResourcePath(...segments) { @@ -190,6 +223,11 @@ function createWindow() { mainWindow.loadFile(frontendPath); } + mainWindow.webContents.on('will-attach-webview', (_event, webPreferences, _params) => { + webPreferences.plugins = true; + webPreferences.enableBlinkFeatures = 'EncryptedMedia'; + }); + mainWindow.on('closed', () => { mainWindow = null; }); @@ -202,6 +240,7 @@ function sendToRenderer(channel, ...args) { } function setupAutoUpdater() { + if (!autoUpdater) return; autoUpdater.autoDownload = false; autoUpdater.autoInstallOnAppQuit = false; @@ -252,6 +291,68 @@ app.whenReady().then(async () => { try { app.dock.setIcon(iconPath); } catch (_) {} } + session.defaultSession.setPermissionRequestHandler((_wc, permission, callback) => { + const allowed = [ + 'media', 'mediaKeySystem', 'protected-media-identifier', + 'geolocation', 'notifications', 'midi', 'midiSysex', + 'clipboard-read', 'clipboard-sanitized-write', + 'pointerLock', 'fullscreen', 'idle-detection', + ]; + console.log('Permission request:', permission, '->', allowed.includes(permission) ? 'granted' : 'denied'); + callback(allowed.includes(permission)); + }); + session.defaultSession.setPermissionCheckHandler((_wc, permission) => { + const allowed = [ + 'media', 'mediaKeySystem', 'protected-media-identifier', + 'clipboard-read', 'clipboard-sanitized-write', + 'pointerLock', 'fullscreen', 'idle-detection', + ]; + return allowed.includes(permission); + }); + + // Read-only logging for DRM license requests — no modifying interceptors + // so the network stack can set Content-Type and other headers normally. + session.defaultSession.webRequest.onSendHeaders( + { urls: ['*://*/*widevine*license*'] }, + (details) => { + console.log(`[drm-req] ${details.method} ${details.url}`); + for (const [k, v] of Object.entries(details.requestHeaders || {})) { + if (/content-type|origin|referer|auth|accept/i.test(k)) { + console.log(`[drm-req] ${k}: ${v}`); + } + } + }, + ); + session.defaultSession.webRequest.onCompleted( + { urls: ['*://*/*widevine*', '*://*/*license*'] }, + (details) => { + console.log(`[drm-net] ${details.method} ${details.url} → ${details.statusCode}`); + }, + ); + session.defaultSession.webRequest.onErrorOccurred( + { urls: ['*://*/*widevine*', '*://*/*license*'] }, + (details) => { + console.log(`[drm-net] FAILED ${details.method} ${details.url} → ${details.error}`); + }, + ); + + // Wait for the Widevine CDM to be downloaded/ready (CastLabs Component + // Updater Service). On first launch this downloads the CDM; subsequent + // launches use the cached version. + if (components && typeof components.whenReady === 'function') { + try { + await components.whenReady(); + console.log('Widevine CDM ready'); + if (typeof components.status === 'function') { + console.log('CDM component status:', JSON.stringify(components.status())); + } + } catch (err) { + console.warn('Widevine CDM not available:', err.message); + } + } else { + console.log('CastLabs components API not available — using standard Electron (no DRM)'); + } + try { if (isDev) { backendPort = parseInt(process.env.OPENSWARM_PORT || '8324', 10); @@ -269,6 +370,79 @@ app.whenReady().then(async () => { } }); +app.on('web-contents-created', (_event, contents) => { + contents.setWindowOpenHandler(({ url, disposition }) => { + if (disposition === 'foreground-tab' || disposition === 'background-tab') { + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send('webview-new-window', url, contents.id); + } + return { action: 'deny' }; + } + + return { + action: 'allow', + overrideBrowserWindowOptions: { + parent: mainWindow || undefined, + }, + }; + }); + + contents.on('did-create-window', (childWindow) => { + if (mainWindow && !mainWindow.isDestroyed() && !childWindow.isDestroyed()) { + childWindow.setParentWindow(mainWindow); + } + }); + + if (contents.getType() === 'webview') { + contents.on('console-message', (_e, level, message, line, sourceId) => { + if (message.includes('widevine') || message.includes('drm') || + message.includes('license') || message.includes('MediaKeySession') || + message.includes('EME') || message.includes('[drm-diag]') || level >= 2) { + const tag = ['LOG', 'INFO', 'WARN', 'ERROR'][level] || 'LOG'; + const src = sourceId ? sourceId.split('/').pop() : ''; + console.log(`[webview:${tag}] ${message}${src ? ` (${src}:${line})` : ''}`); + } + }); + + contents.on('dom-ready', () => { + const url = contents.getURL(); + if (url.includes('spotify')) { + contents.executeJavaScript(` + (function() { + const origFetch = window.fetch; + window.fetch = async function(...args) { + const resp = await origFetch.apply(this, args); + const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || ''; + if (url.includes('widevine-license') && !resp.ok) { + const clone = resp.clone(); + try { + const text = await clone.text(); + console.log('[drm-diag] License response ' + resp.status + ': ' + text.substring(0, 500)); + } catch(e) {} + } + return resp; + }; + + // Check EME availability + if (navigator.requestMediaKeySystemAccess) { + navigator.requestMediaKeySystemAccess('com.widevine.alpha', [{ + initDataTypes: ['cenc'], + audioCapabilities: [{contentType: 'audio/mp4; codecs="mp4a.40.2"'}], + }]).then(function(access) { + console.log('[drm-diag] Widevine EME access: ' + access.keySystem); + }).catch(function(err) { + console.log('[drm-diag] Widevine EME FAILED: ' + err.message); + }); + } else { + console.log('[drm-diag] EME API not available'); + } + })(); + `).catch(() => {}); + } + }); + } +}); + app.on('window-all-closed', () => { if (!isDev) killBackend(); app.quit(); @@ -286,9 +460,12 @@ app.on('activate', () => { ipcMain.handle('get-backend-port', () => backendPort); ipcMain.handle('get-app-version', () => app.getVersion()); +ipcMain.handle('get-webview-preload-path', () => { + return `file://${path.join(__dirname, 'webview-preload.js')}`; +}); ipcMain.handle('check-for-updates', async () => { - if (!isPackaged) { + if (!autoUpdater || !isPackaged) { sendToRenderer('update-error', 'Update check is only available in the packaged app.'); return { success: false, error: 'Not packaged' }; } @@ -305,6 +482,7 @@ ipcMain.handle('check-for-updates', async () => { }); ipcMain.handle('download-update', async () => { + if (!autoUpdater) return { success: false, error: 'Updater not available' }; try { await autoUpdater.downloadUpdate(); return { success: true }; @@ -314,6 +492,7 @@ ipcMain.handle('download-update', async () => { }); ipcMain.handle('install-update', () => { + if (!autoUpdater) return; autoUpdater.quitAndInstall(false, true); }); diff --git a/electron/package-lock.json b/electron/package-lock.json index 3358b1a3..769e8896 100644 --- a/electron/package-lock.json +++ b/electron/package-lock.json @@ -1,19 +1,20 @@ { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.4", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.4", + "hasInstallScript": true, "dependencies": { "electron-updater": "^6.3.0", "get-port": "^5.1.1" }, "devDependencies": { "@electron/notarize": "^3.1.1", - "electron": "^33.0.0", + "electron": "castlabs/electron-releases#v33.4.11+wvcus", "electron-builder": "^25.1.0" } }, @@ -2206,9 +2207,8 @@ } }, "node_modules/electron": { - "version": "33.4.11", - "resolved": "https://registry.npmjs.org/electron/-/electron-33.4.11.tgz", - "integrity": "sha512-xmdAs5QWRkInC7TpXGNvzo/7exojubk+72jn1oJL7keNeIlw7xNglf8TGtJtkR4rWC5FJq0oXiIXPS9BcK2Irg==", + "version": "33.4.11+wvcus", + "resolved": "git+ssh://git@github.com/castlabs/electron-releases.git#d1cf58c11ec0a8a04f307ed362d7efde2816778d", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3953,9 +3953,9 @@ } }, "node_modules/node-abi": { - "version": "3.88.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.88.0.tgz", - "integrity": "sha512-At6b4UqIEVudaqPsXjmUO1r/N5BUr4yhDGs5PkBE8/oG5+TfLPhFechiskFsnT6Ql0VfUXbalUUCbfXxtj7K+w==", + "version": "3.89.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", + "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", "dev": true, "license": "MIT", "dependencies": { @@ -4605,9 +4605,9 @@ } }, "node_modules/sax": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.5.0.tgz", - "integrity": "sha512-21IYA3Q5cQf089Z6tgaUTr7lDAyzoTPx5HRtbhsME8Udispad8dC/+sziTNugOEx54ilvatQ9YCzl4KQLPcRHA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", + "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", "license": "BlueOak-1.0.0", "engines": { "node": ">=11.0.0" diff --git a/electron/package.json b/electron/package.json index 48ee49a7..a3320e17 100644 --- a/electron/package.json +++ b/electron/package.json @@ -1,11 +1,13 @@ { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.4", "description": "OpenSwarm — AI Agent Orchestrator", "main": "main.js", "scripts": { "start": "electron .", "dev": "ELECTRON_DEV=1 electron .", + "postinstall": "bash scripts/sign-vmp.sh", + "sign-vmp": "bash scripts/sign-vmp.sh", "dist": "electron-builder --mac --publish never", "dist:publish": "electron-builder --mac --publish always", "dist:all": "electron-builder --mac --win --linux" @@ -16,12 +18,15 @@ }, "devDependencies": { "@electron/notarize": "^3.1.1", - "electron": "^33.0.0", + "electron": "castlabs/electron-releases#v33.4.11+wvcus", "electron-builder": "^25.1.0" }, "build": { "appId": "com.clusterlabs.openswarm", "productName": "OpenSwarm", + "electronDownload": { + "mirror": "https://github.com/castlabs/electron-releases/releases/download/v" + }, "directories": { "output": "dist" }, diff --git a/electron/preload.js b/electron/preload.js index 2f9037be..134e64c8 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -2,11 +2,13 @@ const { contextBridge, ipcRenderer } = require('electron'); (async () => { const port = await ipcRenderer.invoke('get-backend-port'); + const webviewPreloadPath = await ipcRenderer.invoke('get-webview-preload-path'); contextBridge.exposeInMainWorld('__OPENSWARM_PORT__', port); contextBridge.exposeInMainWorld('openswarm', { getBackendPort: () => port, + getWebviewPreloadPath: () => webviewPreloadPath, getAppVersion: () => ipcRenderer.invoke('get-app-version'), openExternal: (url) => ipcRenderer.invoke('open-external', url), @@ -40,5 +42,11 @@ const { contextBridge, ipcRenderer } = require('electron'); ipcRenderer.on('update-error', listener); return () => ipcRenderer.removeListener('update-error', listener); }, + + onWebviewNewWindow: (cb) => { + const listener = (_event, url, webContentsId) => cb(url, webContentsId); + ipcRenderer.on('webview-new-window', listener); + return () => ipcRenderer.removeListener('webview-new-window', listener); + }, }); })(); diff --git a/electron/scripts/sign-vmp.sh b/electron/scripts/sign-vmp.sh new file mode 100755 index 00000000..ba5eb833 --- /dev/null +++ b/electron/scripts/sign-vmp.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Signs the CastLabs Electron binary with a production VMP certificate via EVS, +# then repairs macOS framework symlinks that npm/signing may strip. +# +# First-time setup (one-time): +# pip3 install --user castlabs-evs +# python3 -m castlabs_evs.account signup +# +# After signup, this script runs automatically. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ELECTRON_DIR="$SCRIPT_DIR/../node_modules/electron/dist" +FW_BASE="$ELECTRON_DIR/Electron.app/Contents/Frameworks" + +fix_framework_symlinks() { + [ -d "$FW_BASE" ] || return 0 + for fw in "$FW_BASE"/*.framework; do + [ -d "$fw/Versions/A" ] || continue + local name + name=$(basename "$fw" .framework) + cd "$fw" + (cd Versions && ln -sf A Current 2>/dev/null) + ln -sf "Versions/Current/$name" "$name" 2>/dev/null + [ -d "Versions/A/Resources" ] && ln -sf Versions/Current/Resources Resources 2>/dev/null + [ -d "Versions/A/Libraries" ] && ln -sf Versions/Current/Libraries Libraries 2>/dev/null + [ -d "Versions/A/Helpers" ] && ln -sf Versions/Current/Helpers Helpers 2>/dev/null + done +} + +if [ ! -d "$ELECTRON_DIR" ]; then + echo "[vmp] Electron dist not found at $ELECTRON_DIR — skipping VMP signing" + fix_framework_symlinks + exit 0 +fi + +# Always fix symlinks first (npm git installs strip them) +fix_framework_symlinks + +if ! python3 -c "import castlabs_evs" 2>/dev/null; then + echo "[vmp] castlabs-evs not installed. Install with: pip3 install --user castlabs-evs" + echo "[vmp] Skipping VMP signing — DRM playback will be limited" + exit 0 +fi + +VERIFY_OUTPUT=$(python3 -m castlabs_evs.vmp verify-pkg "$ELECTRON_DIR" 2>&1) +if echo "$VERIFY_OUTPUT" | grep -q "Signature is valid" && ! echo "$VERIFY_OUTPUT" | grep -q "development only"; then + echo "[vmp] Electron already has a valid production VMP signature" + exit 0 +fi + +echo "[vmp] Signing Electron with production VMP certificate..." +if python3 -m castlabs_evs.vmp sign-pkg "$ELECTRON_DIR" 2>&1; then + echo "[vmp] VMP signing successful — full DRM playback enabled" + # Re-fix symlinks in case signing modified the bundle + fix_framework_symlinks +else + echo "[vmp] VMP signing failed — you may need to run: python3 -m castlabs_evs.account signup" + echo "[vmp] DRM playback will be limited to previews until signed" +fi + +exit 0 diff --git a/electron/webview-preload.js b/electron/webview-preload.js new file mode 100644 index 00000000..95a93d66 --- /dev/null +++ b/electron/webview-preload.js @@ -0,0 +1,87 @@ +/** + * Webview preload script — patches browser fingerprinting so sites like + * Spotify/Netflix don't detect an Electron shell and disable features. + * Loaded via the webview's `preload` attribute before any page script runs. + */ + +'use strict'; + +// Hide webdriver flag +Object.defineProperty(navigator, 'webdriver', { + get: () => false, + configurable: true, +}); + +// Spoof navigator.plugins (Chrome has a few built-in ones) +const fakePlugins = { + 0: { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, + 1: { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' }, + 2: { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }, + length: 3, + item: (i) => fakePlugins[i] || null, + namedItem: (name) => { + for (let i = 0; i < fakePlugins.length; i++) { + if (fakePlugins[i].name === name) return fakePlugins[i]; + } + return null; + }, + refresh: () => {}, + [Symbol.iterator]: function* () { + for (let i = 0; i < this.length; i++) yield this[i]; + }, +}; +try { + Object.defineProperty(navigator, 'plugins', { + get: () => fakePlugins, + configurable: true, + }); +} catch (_) {} + +// Ensure window.chrome exists (sites test for it) +if (!window.chrome) { + window.chrome = {}; +} +if (!window.chrome.runtime) { + window.chrome.runtime = { + connect: () => {}, + sendMessage: () => {}, + onMessage: { addListener: () => {}, removeListener: () => {} }, + }; +} + +// Ensure navigator.languages has sensible values +try { + Object.defineProperty(navigator, 'languages', { + get: () => ['en-US', 'en'], + configurable: true, + }); +} catch (_) {} + +// Patch permissions.query to report 'granted' for common permissions +const originalQuery = navigator.permissions?.query?.bind(navigator.permissions); +if (originalQuery) { + navigator.permissions.query = (params) => { + if (params.name === 'notifications') { + return Promise.resolve({ state: 'granted', onchange: null }); + } + return originalQuery(params).catch(() => + Promise.resolve({ state: 'prompt', onchange: null }) + ); + }; +} + +// Prevent iframe detection heuristics +try { + Object.defineProperty(document, 'hidden', { + get: () => false, + configurable: true, + }); + Object.defineProperty(document, 'visibilityState', { + get: () => 'visible', + configurable: true, + }); +} catch (_) {} + +// Fix console.debug detection (some sites use it as a breakpoint detector) +const noop = () => {}; +if (!window.console.debug) window.console.debug = noop; diff --git a/frontend/src/app/pages/AgentChat/AgentChat.tsx b/frontend/src/app/pages/AgentChat/AgentChat.tsx index 0c5c43c5..e10d80ca 100644 --- a/frontend/src/app/pages/AgentChat/AgentChat.tsx +++ b/frontend/src/app/pages/AgentChat/AgentChat.tsx @@ -527,11 +527,11 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose {renderItems.map((item) => { if (isToolGroup(item)) { const groupMeta = session.tool_group_meta?.[item.id]; - return ; + return ; } if (isToolPair(item)) { const isPending = item.result === null && sessionRunning; - return ; + return ; } const msg = item; const siblings = getSiblingBranches(msg.id); @@ -566,6 +566,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose key={`streaming-${session.streamingMessage.id}`} isStreaming isPending + sessionId={session.id} call={{ id: session.streamingMessage.id, role: 'tool_call', diff --git a/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx b/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx new file mode 100644 index 00000000..77130063 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx @@ -0,0 +1,392 @@ +import React, { useEffect, useRef, useMemo } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined'; +import CheckCircleOutlineIcon from '@mui/icons-material/CheckCircleOutline'; +import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; +import LanguageIcon from '@mui/icons-material/Language'; +import OpenInNewIcon from '@mui/icons-material/OpenInNew'; +import TouchAppOutlinedIcon from '@mui/icons-material/TouchAppOutlined'; +import KeyboardOutlinedIcon from '@mui/icons-material/KeyboardOutlined'; +import CameraAltOutlinedIcon from '@mui/icons-material/CameraAltOutlined'; +import ArticleOutlinedIcon from '@mui/icons-material/ArticleOutlined'; +import AccountTreeOutlinedIcon from '@mui/icons-material/AccountTreeOutlined'; +import CodeOutlinedIcon from '@mui/icons-material/CodeOutlined'; +import BuildOutlinedIcon from '@mui/icons-material/BuildOutlined'; +import { useAppSelector, useAppDispatch } from '@/shared/hooks'; +import { AgentMessage, AgentSession, fetchBrowserAgentChildren } from '@/shared/state/agentsSlice'; +import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; + +interface Props { + parentSessionId: string; + browserId?: string; +} + +interface FeedEntry { + type: 'thought' | 'action' | 'result' | 'system'; + text: string; + actionTool?: string; + sessionLabel?: string; +} + +function formatMessage(msg: AgentMessage): FeedEntry | null { + if (msg.role === 'user') return null; + + if (msg.role === 'assistant' && typeof msg.content === 'string') { + const trimmed = msg.content.trim(); + if (!trimmed) return null; + return { type: 'thought', text: trimmed }; + } + + if (msg.role === 'tool_call') { + const content = + typeof msg.content === 'string' + ? (() => { try { return JSON.parse(msg.content); } catch { return {}; } })() + : msg.content; + const tool = content?.tool || content?.name || '?'; + const input = content?.input || {}; + let brief = ''; + switch (tool) { + case 'BrowserNavigate': + brief = `Navigate → ${input.url || '...'}`; + break; + case 'BrowserClick': + brief = `Click ${input.selector || '...'}`; + break; + case 'BrowserType': { + const txt = (input.text || '').slice(0, 40); + const ellipsis = (input.text || '').length > 40 ? '…' : ''; + brief = `Type "${txt}${ellipsis}" into ${input.selector || '...'}`; + break; + } + case 'BrowserScreenshot': + brief = 'Screenshot'; + break; + case 'BrowserGetText': + brief = 'Read page text'; + break; + case 'BrowserGetElements': + brief = `Inspect elements${input.selector ? ` (${input.selector})` : ''}`; + break; + case 'BrowserEvaluate': + brief = `Evaluate JS`; + break; + default: + brief = `${tool}(${JSON.stringify(input).slice(0, 60)})`; + } + return { type: 'action', text: brief, actionTool: tool }; + } + + if (msg.role === 'tool_result') { + const content = + typeof msg.content === 'string' + ? (() => { try { return JSON.parse(msg.content); } catch { return { text: msg.content }; } })() + : msg.content; + const toolName = content?.tool_name || ''; + const elapsed = content?.elapsed_ms; + const text = content?.text || ''; + + if (toolName === 'BrowserScreenshot') { + return { type: 'result', text: `Screenshot captured${elapsed ? ` (${elapsed}ms)` : ''}` }; + } + const preview = text.length > 120 ? text.slice(0, 120) + '…' : text; + return { type: 'result', text: `${preview}${elapsed ? ` (${elapsed}ms)` : ''}` }; + } + + if (msg.role === 'system') { + return { type: 'system', text: typeof msg.content === 'string' ? msg.content : '' }; + } + + return null; +} + +type SvgIconComponent = typeof OpenInNewIcon; + +function getActionIcon(tool?: string): SvgIconComponent { + switch (tool) { + case 'BrowserNavigate': return OpenInNewIcon; + case 'BrowserClick': return TouchAppOutlinedIcon; + case 'BrowserType': return KeyboardOutlinedIcon; + case 'BrowserScreenshot': return CameraAltOutlinedIcon; + case 'BrowserGetText': return ArticleOutlinedIcon; + case 'BrowserGetElements': return AccountTreeOutlinedIcon; + case 'BrowserEvaluate': return CodeOutlinedIcon; + default: return BuildOutlinedIcon; + } +} + +interface FeedColors { + thought: string; + thoughtIcon: string; + result: string; + error: string; + errorIcon: string; + scrollThumb: string; +} + +const darkFeedColors: FeedColors = { + thought: '#a0aab8', + thoughtIcon: '#555b6e', + result: '#555b6e', + error: '#ff8787', + errorIcon: '#ff8787', + scrollThumb: '#2a2d3e', +}; + +const lightFeedColors: FeedColors = { + thought: '#555550', + thoughtIcon: '#9e9c95', + result: '#9e9c95', + error: '#c03030', + errorIcon: '#c03030', + scrollThumb: '#ccc9c0', +}; + +const BrowserAgentInlineFeed: React.FC = ({ parentSessionId, browserId }) => { + const c = useClaudeTokens(); + const dispatch = useAppDispatch(); + const { mode } = useThemeMode(); + const fc = mode === 'dark' ? darkFeedColors : lightFeedColors; + const scrollRef = useRef(null); + const fetchedRef = useRef(false); + + const browserSessions = useAppSelector((state) => { + const all = state.agents.sessions; + return Object.values(all).filter( + (s): s is AgentSession => + s.mode === 'browser-agent' && + s.parent_session_id === parentSessionId && + (!browserId || s.browser_id === browserId), + ); + }); + + useEffect(() => { + if (browserSessions.length === 0 && !fetchedRef.current) { + fetchedRef.current = true; + dispatch(fetchBrowserAgentChildren(parentSessionId)); + } + }, [browserSessions.length, parentSessionId, dispatch]); + + const sessionsWithEntries = useMemo(() => { + return browserSessions.map((session) => { + const entries: FeedEntry[] = []; + for (const msg of session.messages) { + const entry = formatMessage(msg); + if (entry) entries.push(entry); + } + if (session.streamingMessage?.role === 'assistant' && session.streamingMessage.content) { + entries.push({ type: 'thought', text: session.streamingMessage.content }); + } + return { session, entries }; + }); + }, [browserSessions]); + + const totalMessages = browserSessions.reduce( + (n, s) => n + s.messages.length + (s.streamingMessage ? 1 : 0), + 0, + ); + + useEffect(() => { + if (scrollRef.current) { + scrollRef.current.scrollTop = scrollRef.current.scrollHeight; + } + }, [totalMessages]); + + if (browserSessions.length === 0) return null; + + const showLabels = sessionsWithEntries.length > 1; + const accentColor = c.accent.primary; + + return ( + + {sessionsWithEntries.map(({ session, entries }, si) => ( + + {showLabels && ( + 0 ? 1 : 0, mb: 0.25 }}> + + + {session.browser_id || `Browser ${si + 1}`} + + + + )} + + {!showLabels && entries.length === 0 && session.status === 'running' && ( + + Starting browser agent... + + )} + + {entries.map((entry, i) => ( + + ))} + + {!showLabels && session.status === 'running' && entries.length > 0 && ( + + + + )} + + ))} + + ); +}; + +const EntryRow: React.FC<{ entry: FeedEntry; accentColor: string; fc: FeedColors }> = ({ entry, accentColor, fc }) => { + const c = useClaudeTokens(); + + if (entry.type === 'thought') { + return ( + + + + {entry.text} + + + ); + } + + if (entry.type === 'action') { + const ActionIcon = getActionIcon(entry.actionTool); + return ( + + + + {entry.text} + + + ); + } + + if (entry.type === 'result') { + return ( + + + ↳ {entry.text} + + + ); + } + + if (entry.type === 'system') { + return ( + + + + {entry.text} + + + ); + } + + return null; +}; + +const SessionStatusChip: React.FC<{ status: string }> = ({ status }) => { + const c = useClaudeTokens(); + if (status === 'running') { + return ( + + ); + } + if (status === 'completed') { + return ; + } + if (status === 'error') { + return ; + } + return null; +}; + +export default React.memo(BrowserAgentInlineFeed); diff --git a/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx b/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx index f058b69e..0fb31fb5 100644 --- a/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx +++ b/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx @@ -19,6 +19,7 @@ import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; import { AgentMessage } from '@/shared/state/agentsSlice'; import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; +import BrowserAgentInlineFeed from './BrowserAgentInlineFeed'; const GoogleServiceIcon: React.FC<{ service: string; size?: number }> = ({ service, size = 14 }) => { if (service === 'gmail') { @@ -466,6 +467,7 @@ interface ToolCallBubbleProps { isPending?: boolean; isStreaming?: boolean; mcpCompact?: boolean; + sessionId?: string; } interface TermColors { @@ -1169,8 +1171,14 @@ const McpResultCard: React.FC<{ parsed: ParsedMcpResult; compact?: boolean }> = return ; }; +function isBrowserAgentTool(name: string): boolean { + if (name === 'BrowserAgent' || name === 'BrowserAgents') return true; + const mcp = parseMcpToolName(name); + return mcp.isMcp && mcp.serverSlug === 'openswarm-browser-agent'; +} + const ToolCallBubble: React.FC = React.memo( - ({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false }) => { + ({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false, sessionId }) => { const c = useClaudeTokens(); const tc = useTermColors(); const [expanded, setExpanded] = useState(false); @@ -1180,7 +1188,10 @@ const ToolCallBubble: React.FC = React.memo( const inputSummary = getInputSummary(toolName, input); const formattedInput = useMemo(() => formatInputDisplay(toolName, input), [toolName, input]); const showTimer = isPending && !isDenied && !isStreaming; - const showBody = expanded || isStreaming; + + const isBrowserAgent = isBrowserAgentTool(toolName); + const browserAgentAutoExpand = isBrowserAgent && isPending && !isStreaming; + const showBody = expanded || isStreaming || browserAgentAutoExpand; const resultContent = result?.content; const hasStructuredResult = @@ -1320,6 +1331,12 @@ const ToolCallBubble: React.FC = React.memo( '&::-webkit-scrollbar-thumb': { background: tc.SCROLLBAR_THUMB, borderRadius: 3 }, }} > + {isBrowserAgent && sessionId && ( + + )} {parsedResult && parsedResult.type === 'mcp' ? ( ) : parsedResult ? ( @@ -1330,7 +1347,7 @@ const ToolCallBubble: React.FC = React.memo( {parsedResult.type === 'text' ? parsedResult.content : ''} ) : null} - {!parsedResult && isPending && !isStreaming && ( + {!parsedResult && isPending && !isStreaming && !isBrowserAgent && ( @@ -1525,6 +1542,14 @@ const ToolCallBubble: React.FC = React.memo( )} + {/* Browser agent inline feed */} + {isBrowserAgent && sessionId && ( + + )} + {/* Output */} {parsedResult && parsedResult.type === 'mcp' ? ( @@ -1566,8 +1591,8 @@ const ToolCallBubble: React.FC = React.memo( ) : null} - {/* Pending indicator when waiting for result */} - {!parsedResult && isPending && !isStreaming && ( + {/* Pending indicator when waiting for result (skip for browser agent — feed replaces it) */} + {!parsedResult && isPending && !isStreaming && !isBrowserAgent && ( = React.memo(({ group, isSessionRunning = false, meta }) => { +const ToolGroupBubble: React.FC = React.memo(({ group, isSessionRunning = false, meta, sessionId }) => { const c = useClaudeTokens(); const isMcp = !!group.mcpServer; const [expanded, setExpanded] = useState(isMcp); @@ -186,6 +187,7 @@ const ToolGroupBubble: React.FC = React.memo(({ group, isSessionRunning = result={pair.result} isPending={pair.result === null && isSessionRunning} mcpCompact + sessionId={sessionId} /> ))} diff --git a/frontend/src/app/pages/Dashboard/BrowserCard.tsx b/frontend/src/app/pages/Dashboard/BrowserCard.tsx index d7772294..50ecd827 100644 --- a/frontend/src/app/pages/Dashboard/BrowserCard.tsx +++ b/frontend/src/app/pages/Dashboard/BrowserCard.tsx @@ -67,6 +67,14 @@ const HANDLE_DEFS: { dir: ResizeDir; sx: Record }[] = [ const isElectron = navigator.userAgent.includes('Electron'); +const chromeUserAgent = navigator.userAgent + .replace(/\s*Electron\/\S+/, '') + .replace(/\s*OpenSwarm\/\S+/, ''); + +const webviewPreloadPath: string | undefined = isElectron + ? (window as any).openswarm?.getWebviewPreloadPath?.() + : undefined; + type WebviewElement = BrowserWebview; interface TabLocalState { @@ -192,10 +200,6 @@ const BrowserCard: React.FC = ({ onTitleUpdate(); }; - const onNewWindow = (e: any) => { - if (e.url) dispatch(addBrowserTab({ browserId, url: e.url, makeActive: true })); - }; - const onFaviconUpdate = (e: any) => { const favicons = e.favicons || (e.detail && e.detail.favicons); if (favicons?.[0]) { @@ -208,7 +212,6 @@ const BrowserCard: React.FC = ({ wv.addEventListener('page-title-updated', onTitleUpdate); wv.addEventListener('did-start-loading', onLoadStart); wv.addEventListener('did-stop-loading', onLoadStop); - wv.addEventListener('new-window', onNewWindow); wv.addEventListener('page-favicon-updated', onFaviconUpdate); cleanups.push(() => { @@ -218,7 +221,6 @@ const BrowserCard: React.FC = ({ wv.removeEventListener('page-title-updated', onTitleUpdate); wv.removeEventListener('did-start-loading', onLoadStart); wv.removeEventListener('did-stop-loading', onLoadStop); - wv.removeEventListener('new-window', onNewWindow); wv.removeEventListener('page-favicon-updated', onFaviconUpdate); }); } @@ -988,6 +990,9 @@ const BrowserCard: React.FC = ({ data-tab-id={tab.id} src="about:blank" allowpopups="true" + useragent={chromeUserAgent} + {...(webviewPreloadPath ? { preload: webviewPreloadPath } : {})} + webpreferences="plugins=yes, autoplayPolicy=no-user-gesture-required" style={{ position: 'absolute', top: 0, diff --git a/frontend/src/app/pages/Dashboard/Dashboard.tsx b/frontend/src/app/pages/Dashboard/Dashboard.tsx index 2aa9393b..c6ce78ca 100644 --- a/frontend/src/app/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/app/pages/Dashboard/Dashboard.tsx @@ -22,6 +22,7 @@ import { tidyLayout, addViewCard, addBrowserCard, + addBrowserTab, moveCards, resetLayout, setGlowingBrowserCards, @@ -31,6 +32,7 @@ import { fetchOutputs } from '@/shared/state/outputsSlice'; import { generateDashboardName, updateDashboardThumbnail } from '@/shared/state/dashboardsSlice'; import { dashboardWs } from '@/shared/ws/WebSocketManager'; import { initBrowserCommandHandler } from '@/shared/browserCommandHandler'; +import { findBrowserByWebContentsId } from '@/shared/browserRegistry'; import AgentCard from './AgentCard'; import DashboardViewCard from './DashboardViewCard'; import BrowserCard from './BrowserCard'; @@ -200,6 +202,26 @@ const DashboardInner: React.FC = () => { return () => { cleanupBrowserHandler(); dashboardWs.disconnect(); }; }, [dispatch, dashboardId]); + useEffect(() => { + const w = window as any; + if (!w.openswarm?.onWebviewNewWindow) return; + let lastUrl = ''; + let lastTime = 0; + return w.openswarm.onWebviewNewWindow((url: string, webContentsId: number) => { + const now = Date.now(); + if (url === lastUrl && now - lastTime < 1000) return; + lastUrl = url; + lastTime = now; + + const browserId = findBrowserByWebContentsId(webContentsId); + if (browserId) { + dispatch(addBrowserTab({ browserId, url, makeActive: true })); + } else { + dispatch(addBrowserCard({ url, expandedSessionIds })); + } + }); + }, [dispatch, expandedSessionIds]); + // Capture a thumbnail screenshot of the dashboard. // Uses Electron's native capturePage for pixel-perfect results. // Captures current viewport as-is (no DOM mutation) to avoid visual flashes. diff --git a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx index 38b000b0..d44f4fb6 100644 --- a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx @@ -204,13 +204,17 @@ const DashboardToolbar = React.forwardRef( const isExpanded = inputOpen || viewPickerOpen || historyOpen; + const autoSelectOnNew = useAppSelector((s) => s.settings.data.auto_select_mode_on_new_agent); const prevInputOpenRef = useRef(inputOpen); useEffect(() => { if (prevInputOpenRef.current && !inputOpen && elementSelection?.selectMode) { elementSelection.setSelectMode(false); } + if (!prevInputOpenRef.current && inputOpen && autoSelectOnNew) { + elementSelection?.setSelectMode(true); + } prevInputOpenRef.current = inputOpen; - }, [inputOpen, elementSelection]); + }, [inputOpen, elementSelection, autoSelectOnNew]); useEffect(() => { if (!isExpanded) return; diff --git a/frontend/src/app/pages/Settings/Settings.tsx b/frontend/src/app/pages/Settings/Settings.tsx index 70a14a8d..58d71b10 100644 --- a/frontend/src/app/pages/Settings/Settings.tsx +++ b/frontend/src/app/pages/Settings/Settings.tsx @@ -11,6 +11,7 @@ import InputAdornment from '@mui/material/InputAdornment'; import ToggleButton from '@mui/material/ToggleButton'; import ToggleButtonGroup from '@mui/material/ToggleButtonGroup'; import Slider from '@mui/material/Slider'; +import Switch from '@mui/material/Switch'; import Snackbar from '@mui/material/Snackbar'; import Alert from '@mui/material/Alert'; import Tab from '@mui/material/Tab'; @@ -495,7 +496,7 @@ const Settings: React.FC = () => { - + New agent shortcut Keyboard shortcut to create an agent. @@ -553,6 +554,21 @@ const Settings: React.FC = () => { + + + Auto-enable element selection + Automatically enter element selection mode when creating a new agent. + + setForm({ ...form, auto_select_mode_on_new_agent: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {/* ── Browser ── */} Browser diff --git a/frontend/src/app/pages/Tools/Tools.tsx b/frontend/src/app/pages/Tools/Tools.tsx index a6e05224..22c7bdad 100644 --- a/frontend/src/app/pages/Tools/Tools.tsx +++ b/frontend/src/app/pages/Tools/Tools.tsx @@ -488,7 +488,8 @@ const Tools: React.FC = () => { if (discoverTools.fulfilled.match(result)) { setSnackbar({ open: true, message: 'Actions discovered successfully' }); } else { - setSnackbar({ open: true, message: 'Discovery failed — is the MCP server running?', severity: 'error' }); + const detail = (result as any).error?.message || 'Discovery failed — is the MCP server running?'; + setSnackbar({ open: true, message: detail, severity: 'error' }); } } finally { setDiscovering(false); diff --git a/frontend/src/shared/browserRegistry.ts b/frontend/src/shared/browserRegistry.ts index 09e79f52..a8eba9a5 100644 --- a/frontend/src/shared/browserRegistry.ts +++ b/frontend/src/shared/browserRegistry.ts @@ -51,6 +51,15 @@ export function getAllWebviews(): Map { return new Map(registry); } +export function findBrowserByWebContentsId(wcId: number): string | undefined { + for (const [key, wv] of registry.entries()) { + if ((wv as any).getWebContentsId?.() === wcId) { + return key.split(':')[0]; + } + } + return undefined; +} + export function unregisterAllForBrowser(browserId: string): void { const prefix = `${browserId}:`; for (const key of registry.keys()) { diff --git a/frontend/src/shared/state/agentsSlice.ts b/frontend/src/shared/state/agentsSlice.ts index 4fb21107..2be0f62c 100644 --- a/frontend/src/shared/state/agentsSlice.ts +++ b/frontend/src/shared/state/agentsSlice.ts @@ -69,6 +69,7 @@ export interface AgentSession { tool_group_meta: Record; dashboard_id?: string; browser_id?: string | null; + parent_session_id?: string | null; } export interface AgentConfig { @@ -380,6 +381,15 @@ export const resumeSession = createAsyncThunk( } ); +export const fetchBrowserAgentChildren = createAsyncThunk( + 'agents/fetchBrowserAgentChildren', + async (parentSessionId: string) => { + const res = await fetch(`${AGENTS_API}/sessions/${parentSessionId}/browser-agents`); + const data = await res.json(); + return data.sessions as AgentSession[]; + } +); + const agentsSlice = createSlice({ name: 'agents', initialState, @@ -629,7 +639,19 @@ const agentsSlice = createSlice({ closeSessionFromWs(state, action: PayloadAction) { const entry = action.payload; state.history[entry.id] = entry; - delete state.sessions[entry.id]; + + const session = state.sessions[entry.id]; + if (session?.mode === 'browser-agent' && session.parent_session_id) { + session.status = (entry.status as AgentSession['status']) || 'completed'; + } else { + delete state.sessions[entry.id]; + for (const [id, s] of Object.entries(state.sessions)) { + if (s.mode === 'browser-agent' && s.parent_session_id === entry.id) { + delete state.sessions[id]; + } + } + } + if (state.activeSessionId === entry.id) { state.activeSessionId = null; } @@ -816,6 +838,17 @@ const agentsSlice = createSlice({ }; } }) + .addCase(fetchBrowserAgentChildren.fulfilled, (state, action) => { + for (const session of action.payload) { + if (!state.sessions[session.id]) { + state.sessions[session.id] = { + ...session, + streamingMessage: null, + tool_group_meta: session.tool_group_meta ?? {}, + }; + } + } + }) .addCase(searchHistory.pending, (state) => { state.historySearch.loading = true; }) diff --git a/frontend/src/shared/state/settingsSlice.ts b/frontend/src/shared/state/settingsSlice.ts index 58961793..4530871c 100644 --- a/frontend/src/shared/state/settingsSlice.ts +++ b/frontend/src/shared/state/settingsSlice.ts @@ -22,6 +22,7 @@ export interface AppSettings { new_agent_shortcut: string; anthropic_api_key: string | null; browser_homepage: string; + auto_select_mode_on_new_agent: boolean; } export interface BrowseResult { @@ -50,6 +51,7 @@ const initialState: SettingsState = { new_agent_shortcut: 'Meta+l', anthropic_api_key: null, browser_homepage: 'https://www.google.com', + auto_select_mode_on_new_agent: false, }, loading: false, loaded: false, diff --git a/frontend/src/types/electron.d.ts b/frontend/src/types/electron.d.ts index a9f3557b..1fe719e4 100644 --- a/frontend/src/types/electron.d.ts +++ b/frontend/src/types/electron.d.ts @@ -10,6 +10,8 @@ declare global { partition?: string; allowpopups?: string; nodeintegration?: string; + webpreferences?: string; + useragent?: string; }, HTMLElement >; @@ -31,6 +33,7 @@ declare global { interface OpenSwarmAPI { getBackendPort: () => number; + getWebviewPreloadPath: () => string; getAppVersion: () => Promise; checkForUpdates: () => Promise<{ success: boolean; version?: string; error?: string }>; downloadUpdate: () => Promise<{ success: boolean; error?: string }>; @@ -40,6 +43,7 @@ declare global { onDownloadProgress: (cb: (progress: OpenSwarmDownloadProgress) => void) => () => void; onUpdateDownloaded: (cb: (info: OpenSwarmUpdateInfo) => void) => () => void; onUpdateError: (cb: (message: string) => void) => () => void; + onWebviewNewWindow: (cb: (url: string, webContentsId: number) => void) => () => void; } interface Window { diff --git a/run.sh b/run.sh index 68a48501..494a6a7e 100755 --- a/run.sh +++ b/run.sh @@ -127,10 +127,18 @@ if (( frontend_elapsed >= FRONTEND_MAX_WAIT )); then exit 1 fi +# --- Sign Electron VMP for DRM (if EVS account exists) --- +if [ -f "$PROJECT_ROOT/electron/scripts/sign-vmp.sh" ]; then + echo -e "${YELLOW}${BOLD}[vmp]${RESET} Checking VMP signature..." + bash "$PROJECT_ROOT/electron/scripts/sign-vmp.sh" 2>&1 | while IFS= read -r line; do + printf "${YELLOW}${BOLD}%s${RESET}\n" "$line" + done +fi + # --- Start Electron in dev mode --- MAGENTA='\033[0;35m' echo -e "${MAGENTA}${BOLD}[electron]${RESET} Launching Electron dev shell..." -(cd "$PROJECT_ROOT/electron" && ELECTRON_DEV=1 npx electron .) > >( +(cd "$PROJECT_ROOT/electron" && unset ELECTRON_RUN_AS_NODE && ELECTRON_DEV=1 npx electron .) > >( while IFS= read -r line; do printf "${MAGENTA}${BOLD}[electron]${RESET} %s\n" "$line" done From 6e1fc1dee20994ddb6feffc199a3873079239a5d Mon Sep 17 00:00:00 2001 From: haikdc Date: Tue, 17 Mar 2026 23:52:20 -0700 Subject: [PATCH 02/11] [Haik]: ckpt (Round 1 of fixing mcp server issues in prod - still has bugs) (Fixed some browser related issues like popup windows rendering out of app and weird js framework related errors when u open sites like spotify) (Added option in settings to toggle item selection to be on by default when selecting a new chat) (Browser agent actions now show up in chat output) (Browser agent actions each have their own custom icon) --- backend/apps/agents/browser_agent.py | 64 +++++++++- backend/apps/agents/browser_mcp_server.py | 54 ++++++++ frontend/src/shared/browserCommandHandler.ts | 126 ++++++++++++++++--- 3 files changed, 222 insertions(+), 22 deletions(-) diff --git a/backend/apps/agents/browser_agent.py b/backend/apps/agents/browser_agent.py index ea2064e6..09ba789d 100644 --- a/backend/apps/agents/browser_agent.py +++ b/backend/apps/agents/browser_agent.py @@ -112,6 +112,48 @@ BROWSER_TOOLS_SCHEMA = [ "required": [], }, }, + { + "name": "BrowserScroll", + "description": ( + "Scroll the page up or down. Automatically finds the correct scrollable " + "container (works on SPAs like Notion, Gmail, etc. that use nested scroll " + "containers instead of window-level scrolling). Returns scroll position info " + "including whether top/bottom has been reached." + ), + "input_schema": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "enum": ["up", "down"], + "description": "Scroll direction. Defaults to 'down'.", + }, + "amount": { + "type": "number", + "description": "Pixels to scroll. Defaults to 500.", + }, + }, + "required": [], + }, + }, + { + "name": "BrowserWait", + "description": ( + "Wait for a specified duration. Useful after navigation or actions that " + "trigger page loads, animations, or async content rendering. " + "Min 100ms, max 10000ms." + ), + "input_schema": { + "type": "object", + "properties": { + "milliseconds": { + "type": "number", + "description": "Duration to wait in milliseconds. Defaults to 1000.", + }, + }, + "required": [], + }, + }, ] ACTION_MAP = { @@ -122,17 +164,29 @@ ACTION_MAP = { "BrowserType": "type", "BrowserEvaluate": "evaluate", "BrowserGetElements": "get_elements", + "BrowserScroll": "scroll", + "BrowserWait": "wait", } SYSTEM_PROMPT = ( "You are a browser automation agent. You control a single browser tab and " "execute the task you are given.\n\n" "Strategy:\n" - "1. Start by taking a screenshot or calling BrowserGetElements to understand the page.\n" - "2. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n" - "3. After performing actions, take a screenshot to verify the result.\n" - "4. If an action fails, try alternative selectors or approaches.\n" - "5. When the task is complete, provide a clear summary of what you accomplished.\n\n" + "1. Start by taking a screenshot to understand the page.\n" + "2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n" + "3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with " + "window.scrollBy() as many sites use nested scroll containers that BrowserScroll " + "handles automatically.\n" + "4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n" + "5. After performing actions, take a screenshot to verify the result.\n" + "6. If an action fails, try alternative selectors or approaches.\n" + "7. When the task is complete, provide a clear summary of what you accomplished.\n\n" + "Important notes:\n" + "- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n" + "- BrowserScroll returns position info including atTop/atBottom — use this to know when " + "you've reached the end of the page.\n" + "- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n" + "- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n" "You have access ONLY to browser tools. Do not ask the user questions — " "complete the task autonomously to the best of your ability." ) diff --git a/backend/apps/agents/browser_mcp_server.py b/backend/apps/agents/browser_mcp_server.py index f32054c2..86f1ad5e 100644 --- a/backend/apps/agents/browser_mcp_server.py +++ b/backend/apps/agents/browser_mcp_server.py @@ -181,6 +181,58 @@ TOOLS = [ "required": ["browser_id"], }, }, + { + "name": "BrowserScroll", + "description": ( + "Scroll the page up or down. Automatically finds the correct scrollable " + "container (works on SPAs like Notion, Gmail, etc. that use nested scroll " + "containers instead of window-level scrolling). Returns scroll position info " + "including whether top/bottom has been reached." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "tab_id": TAB_ID_PROP, + "direction": { + "type": "string", + "enum": ["up", "down"], + "description": "Scroll direction. Defaults to 'down'.", + }, + "amount": { + "type": "number", + "description": "Pixels to scroll. Defaults to 500.", + }, + }, + "required": ["browser_id"], + }, + }, + { + "name": "BrowserWait", + "description": ( + "Wait for a specified duration. Useful after navigation or actions that " + "trigger page loads, animations, or async content rendering. " + "Min 100ms, max 10000ms." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "tab_id": TAB_ID_PROP, + "milliseconds": { + "type": "number", + "description": "Duration to wait in milliseconds. Defaults to 1000.", + }, + }, + "required": ["browser_id"], + }, + }, ] @@ -260,6 +312,8 @@ def handle_tool_call(tool_name: str, arguments: dict) -> dict: "BrowserType": "type", "BrowserEvaluate": "evaluate", "BrowserGetElements": "get_elements", + "BrowserScroll": "scroll", + "BrowserWait": "wait", } action = action_map.get(tool_name) if not action: diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index b9ddd233..fa0446fc 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -4,7 +4,7 @@ import { resolveInput } from './resolveUrl'; let initialized = false; -export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements'; +export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait'; export interface BrowserActivity { action: BrowserAction; @@ -42,6 +42,8 @@ const ACTION_LABELS: Record = { type: 'Typing...', evaluate: 'Evaluating...', get_elements: 'Inspecting...', + scroll: 'Scrolling...', + wait: 'Waiting...', }; export function getActionLabel(action: string): string { @@ -128,6 +130,84 @@ async function handleType(wv: BrowserWebview, params: Record): Prom return result; } +async function handleScroll(wv: BrowserWebview, params: Record): Promise> { + const direction = (params.direction as string) || 'down'; + const amount = (params.amount as number) || 500; + const code = `(() => { + function findScrollable() { + const candidates = document.querySelectorAll( + '[class*="scroller"], [class*="scroll-container"], [class*="content"], ' + + 'main, [role="main"], article, .notion-scroller, .notion-frame' + ); + for (const el of candidates) { + const s = window.getComputedStyle(el); + const isScrollable = (s.overflow === 'auto' || s.overflow === 'scroll' + || s.overflowY === 'auto' || s.overflowY === 'scroll'); + if (isScrollable && el.scrollHeight > el.clientHeight + 10) return el; + } + const all = document.querySelectorAll('*'); + for (const el of all) { + if (el === document.body || el === document.documentElement) continue; + const s = window.getComputedStyle(el); + const isScrollable = (s.overflow === 'auto' || s.overflow === 'scroll' + || s.overflowY === 'auto' || s.overflowY === 'scroll'); + if (isScrollable && el.scrollHeight > el.clientHeight + 50 + && el.clientHeight > 200) return el; + } + return null; + } + const dy = ${JSON.stringify(direction)} === 'up' ? -${amount} : ${amount}; + const container = findScrollable(); + if (container) { + const before = container.scrollTop; + container.scrollBy({ top: dy, behavior: 'instant' }); + const after = container.scrollTop; + return { + scrolled: Math.abs(after - before), + scrollTop: after, + scrollHeight: container.scrollHeight, + clientHeight: container.clientHeight, + atTop: after <= 0, + atBottom: after + container.clientHeight >= container.scrollHeight - 5, + target: 'container', + }; + } + const before = window.scrollY; + window.scrollBy({ top: dy, behavior: 'instant' }); + const after = window.scrollY; + return { + scrolled: Math.abs(after - before), + scrollTop: after, + scrollHeight: document.documentElement.scrollHeight, + clientHeight: window.innerHeight, + atTop: after <= 0, + atBottom: after + window.innerHeight >= document.documentElement.scrollHeight - 5, + target: 'window', + }; + })()`; + try { + const result = await wv.executeJavaScript(code); + const status = result.atBottom ? ' (reached bottom)' : result.atTop ? ' (reached top)' : ''; + return { + text: `Scrolled ${direction} by ${result.scrolled}px${status}. Position: ${result.scrollTop}/${result.scrollHeight - result.clientHeight}px`, + ...result, + url: wv.getURL(), + }; + } catch (err: any) { + return { error: `Scroll failed: ${err?.message || String(err)}` }; + } +} + +async function handleWait(wv: BrowserWebview, params: Record): Promise> { + const ms = Math.min(Math.max((params.milliseconds as number) || 1000, 100), 10000); + await new Promise((resolve) => setTimeout(resolve, ms)); + return { + text: `Waited ${ms}ms. Current URL: ${wv.getURL()}`, + url: wv.getURL(), + title: wv.getTitle(), + }; +} + async function handleGetElements(wv: BrowserWebview, params: Record): Promise> { const scope = (params.selector as string) || 'body'; const safeScope = JSON.stringify(scope); @@ -135,51 +215,57 @@ async function handleGetElements(wv: BrowserWebview, params: Record const scope = document.querySelector(${safeScope}) || document.body; const interactive = scope.querySelectorAll( 'a[href], button, input, textarea, select, [role="button"], [role="link"], ' - + '[role="textbox"], [role="searchbox"], [onclick], [tabindex]:not([tabindex="-1"])' + + '[role="textbox"], [role="searchbox"], [role="menuitem"], [role="tab"], ' + + '[role="checkbox"], [role="switch"], [role="option"], ' + + '[onclick], [tabindex]:not([tabindex="-1"]), ' + + '[data-block-id], [contenteditable="true"]' ); + const seen = new Set(); const results = []; for (const el of interactive) { - if (results.length >= 60) break; + if (results.length >= 80) break; const rect = el.getBoundingClientRect(); if (rect.width === 0 && rect.height === 0) continue; - if (window.getComputedStyle(el).visibility === 'hidden') continue; + const style = window.getComputedStyle(el); + if (style.visibility === 'hidden' || style.display === 'none') continue; + if (style.opacity === '0') continue; let selector = el.tagName.toLowerCase(); if (el.id) { - selector = '#' + el.id; + selector = '#' + CSS.escape(el.id); + } else if (el.getAttribute('data-block-id')) { + selector = '[data-block-id="' + el.getAttribute('data-block-id') + '"]'; } else if (el.getAttribute('name')) { - selector = el.tagName.toLowerCase() + '[name="' + el.getAttribute('name') + '"]'; + selector = el.tagName.toLowerCase() + '[name="' + CSS.escape(el.getAttribute('name')) + '"]'; } else if (el.getAttribute('aria-label')) { - selector = el.tagName.toLowerCase() + '[aria-label="' + el.getAttribute('aria-label') + '"]'; + selector = el.tagName.toLowerCase() + '[aria-label="' + CSS.escape(el.getAttribute('aria-label')) + '"]'; } else if (el.getAttribute('type') && el.tagName === 'INPUT') { selector = 'input[type="' + el.getAttribute('type') + '"]'; if (el.getAttribute('placeholder')) - selector += '[placeholder="' + el.getAttribute('placeholder') + '"]'; + selector += '[placeholder="' + CSS.escape(el.getAttribute('placeholder')) + '"]'; } else if (el.className && typeof el.className === 'string') { const cls = el.className.trim().split(/\\s+/)[0]; - if (cls && cls.length < 40) + if (cls && cls.length < 60) selector = el.tagName.toLowerCase() + '.' + CSS.escape(cls); } - const verify = document.querySelectorAll(selector); - if (verify.length > 1) { + if (seen.has(selector)) { const parent = el.parentElement; if (parent && parent.id) { - selector = '#' + parent.id + ' > ' + selector; + selector = '#' + CSS.escape(parent.id) + ' > ' + selector; } else { - const siblings = parent ? Array.from(parent.querySelectorAll(':scope > ' + el.tagName.toLowerCase())) : []; + const siblings = parent ? Array.from(parent.children) : []; const idx = siblings.indexOf(el); - if (idx >= 0 && parent) - selector = (parent.tagName.toLowerCase() + (parent.className ? '.' + CSS.escape(parent.className.trim().split(/\\s+/)[0]) : '')) - + ' > ' + el.tagName.toLowerCase() + ':nth-child(' + (idx + 1) + ')'; + if (idx >= 0) selector += ':nth-child(' + (idx + 1) + ')'; } } + seen.add(selector); results.push({ selector, tag: el.tagName.toLowerCase(), type: el.type || null, - text: (el.textContent || '').trim().substring(0, 80) || null, + text: (el.textContent || '').trim().substring(0, 120) || null, placeholder: el.placeholder || null, ariaLabel: el.getAttribute('aria-label') || null, role: el.getAttribute('role') || null, @@ -248,6 +334,12 @@ async function handleBrowserCommand(data: Record) { case 'get_elements': result = await handleGetElements(wv, params); break; + case 'scroll': + result = await handleScroll(wv, params); + break; + case 'wait': + result = await handleWait(wv, params); + break; default: result = { error: `Unknown browser action: ${action}` }; } From b937003a24c1e45369a6056ae805acee3a2f8484 Mon Sep 17 00:00:00 2001 From: haikdc Date: Wed, 18 Mar 2026 01:48:53 -0700 Subject: [PATCH 03/11] [Haik]: ckpt (builtin tool call permissions are now functional) (when card is selected in dash hover scroll still works) (notification ui now shows actively running agents too - with the ability to stop them too) (agent card with is persistent post auto layout) (gworkspace discovery func works in prod now - to be tested in prod) (stable release naming for files so most recent version has fixed release link - to be tested in prod) --- backend/apps/agents/agent_manager.py | 121 ++++-- backend/apps/tools_lib/tools_lib.py | 26 +- electron/main.js | 2 + electron/package-lock.json | 4 +- electron/package.json | 3 +- frontend/src/app/Main.tsx | 8 +- .../app/components/GlobalApprovalOverlay.tsx | 360 +++++++++++++++--- .../src/app/pages/Dashboard/AgentCard.tsx | 5 +- .../src/app/pages/Dashboard/BrowserCard.tsx | 5 +- .../app/pages/Dashboard/DashboardToolbar.tsx | 14 +- .../app/pages/Dashboard/DashboardViewCard.tsx | 5 +- .../Dashboard/useOverlayScrollPassthrough.ts | 80 ++++ frontend/src/app/pages/Tools/Tools.tsx | 2 +- frontend/src/shared/state/agentsSlice.ts | 29 ++ .../src/shared/state/dashboardLayoutSlice.ts | 21 +- frontend/src/shared/ws/WebSocketManager.ts | 4 + 16 files changed, 573 insertions(+), 116 deletions(-) create mode 100644 frontend/src/app/pages/Dashboard/useOverlayScrollPassthrough.ts diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index 5913cb15..3f705df0 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -481,58 +481,44 @@ class AgentManager: _builtin_perms = load_builtin_permissions() - def _check_tool_permission(tool_name: str) -> str | None: - """Check tool permissions for both builtin and MCP tools. - Returns 'always_allow', 'deny', or None (ask).""" + def _get_effective_policy(tool_name: str) -> str: + """Return 'always_allow', 'deny', or 'ask' for any tool.""" if tool_name in _builtin_perms: - policy = _builtin_perms[tool_name] - if policy in ("always_allow", "deny"): - return policy - return None + return _builtin_perms[tool_name] import re as _re m = _re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name) - if not m: - return None - server_slug, mcp_tool_name = m.group(1), m.group(2) - for t in load_all_tools(): - if not t.mcp_config or not t.enabled: - continue - if _sanitize_server_name(t.name) == server_slug: - policy = t.tool_permissions.get(mcp_tool_name, "ask") - if policy in ("always_allow", "deny"): - return policy - return None - return None - - async def can_use_tool(tool_name, input_data, context): - if tool_name != "AskUserQuestion": - policy = _check_tool_permission(tool_name) - if policy == "always_allow": - return PermissionResultAllow(updated_input=input_data) - if policy == "deny": - return PermissionResultDeny(message="Tool denied by permission policy") + if m: + server_slug, mcp_tool_name = m.group(1), m.group(2) + for t in load_all_tools(): + if not t.mcp_config or not t.enabled: + continue + if _sanitize_server_name(t.name) == server_slug: + return t.tool_permissions.get(mcp_tool_name, "ask") + return "always_allow" + async def _request_user_approval(tool_name: str, tool_input) -> dict: + """Send an approval request via WebSocket and wait for the user's decision.""" + safe_input = tool_input if isinstance(tool_input, dict) else {} request_id = uuid4().hex approval_req = ApprovalRequest( id=request_id, session_id=session_id, tool_name=tool_name, - tool_input=input_data if isinstance(input_data, dict) else {}, + tool_input=safe_input, ) session.pending_approvals.append(approval_req) session.status = "waiting_approval" - + await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, "status": "waiting_approval", }) - + decision = await ws_manager.send_approval_request( - session_id, request_id, tool_name, - input_data if isinstance(input_data, dict) else {} + session_id, request_id, tool_name, safe_input ) - + session.pending_approvals = [ a for a in session.pending_approvals if a.id != request_id ] @@ -541,22 +527,67 @@ class AgentManager: "session_id": session_id, "status": "running", }) - + return decision + + async def can_use_tool(tool_name, input_data, context): + if tool_name != "AskUserQuestion": + policy = _get_effective_policy(tool_name) + if policy == "always_allow": + return PermissionResultAllow(updated_input=input_data) + if policy == "deny": + return PermissionResultDeny(message="Tool denied by permission policy") + + decision = await _request_user_approval(tool_name, input_data) if decision.get("behavior") == "allow": return PermissionResultAllow( updated_input=decision.get("updated_input", input_data) ) - else: - return PermissionResultDeny( - message=decision.get("message", "User denied this action") - ) + return PermissionResultDeny( + message=decision.get("message", "User denied this action") + ) tool_start_times: dict[str, float] = {} async def pre_tool_hook(input_data, tool_use_id, context): + tool_name = input_data.get("tool_name", "") + hook_event = input_data.get("hook_event_name", "PreToolUse") + + if tool_name and tool_name != "AskUserQuestion": + policy = _get_effective_policy(tool_name) + + if policy == "deny": + return { + "hookSpecificOutput": { + "hookEventName": hook_event, + "permissionDecision": "deny", + "permissionDecisionReason": "Tool denied by permission policy", + } + } + + if policy == "ask": + tool_input = input_data.get("tool_input", {}) + decision = await _request_user_approval(tool_name, tool_input) + + if decision.get("behavior") == "allow": + if tool_use_id: + tool_start_times[tool_use_id] = time.time() + return { + "hookSpecificOutput": { + "hookEventName": hook_event, + "permissionDecision": "allow", + } + } + return { + "hookSpecificOutput": { + "hookEventName": hook_event, + "permissionDecision": "deny", + "permissionDecisionReason": decision.get("message", "User denied this action"), + } + } + if tool_use_id: tool_start_times[tool_use_id] = time.time() - return {"continue_": True} + return {} async def post_tool_hook(input_data, tool_use_id, context): elapsed_ms = None @@ -628,8 +659,14 @@ class AgentManager: effective_allowed = [ t for t in session.allowed_tools - if _builtin_perms.get(t, "always_allow") == "always_allow" + if t in FULL_TOOLS and _builtin_perms.get(t, "always_allow") == "always_allow" ] + + effective_disallowed = [ + t for t in FULL_TOOLS + if _builtin_perms.get(t, "always_allow") == "deny" + ] + if mcp_servers: all_tools_list = load_all_tools() for name in mcp_servers: @@ -645,6 +682,8 @@ class AgentManager: policy = tool_def.tool_permissions.get(tn, "ask") if policy == "always_allow": effective_allowed.append(f"mcp__{name}__{tn}") + for tn in denied: + effective_disallowed.append(f"mcp__{name}__{tn}") else: effective_allowed.append(f"mcp__{name}__*") @@ -653,12 +692,14 @@ class AgentManager: options_kwargs = { "model": session.model, "max_buffer_size": 5 * 1024 * 1024, + "permission_mode": "default", "can_use_tool": can_use_tool, "hooks": { "PreToolUse": [HookMatcher(matcher=None, hooks=[pre_tool_hook])], "PostToolUse": [HookMatcher(matcher=None, hooks=[post_tool_hook])], }, "allowed_tools": effective_allowed, + "disallowed_tools": effective_disallowed, "include_partial_messages": True, } if not global_settings.anthropic_api_key: diff --git a/backend/apps/tools_lib/tools_lib.py b/backend/apps/tools_lib/tools_lib.py index d440bda0..b7710a9f 100644 --- a/backend/apps/tools_lib/tools_lib.py +++ b/backend/apps/tools_lib/tools_lib.py @@ -624,6 +624,25 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, @tools_lib.router.post("/{tool_id}/discover") async def discover_tools(tool_id: str): tool = _load(tool_id) + + if tool.auth_type == "oauth2" and tool.auth_status == "connected": + refreshed = await refresh_google_token(tool) + if not refreshed and tool.oauth_tokens.get("access_token"): + expiry = tool.oauth_tokens.get("token_expiry", 0) + if time.time() >= expiry - 60: + client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "") + if not client_id: + raise HTTPException( + status_code=400, + detail="OAuth token expired and GOOGLE_OAUTH_CLIENT_ID is not set. " + "In the packaged app, create ~/.openswarm.env or " + "~/Library/Application Support/OpenSwarm/.env with your Google OAuth credentials.", + ) + raise HTTPException( + status_code=502, + detail="OAuth token expired and refresh failed. Try reconnecting Google.", + ) + config = derive_mcp_config(tool) if not config: raise HTTPException(status_code=400, detail="Cannot derive MCP config for tool") @@ -657,8 +676,11 @@ async def discover_tools(tool_id: str): except HTTPException: raise except Exception as e: - logger.warning(f"MCP tool discovery failed for {tool.name}: {e}") - raise HTTPException(status_code=502, detail=f"Discovery failed: {e}") + msg = str(e).strip() + if not msg: + msg = type(e).__name__ + logger.warning(f"MCP tool discovery failed for {tool.name}: {msg}", exc_info=True) + raise HTTPException(status_code=502, detail=f"Discovery failed: {msg}") services: dict[str, dict[str, list[str]]] = {} service_groups: dict[str, list[str]] = {} diff --git a/electron/main.js b/electron/main.js index 90b909c1..8078bdbb 100644 --- a/electron/main.js +++ b/electron/main.js @@ -152,6 +152,8 @@ async function startBackend() { OPENSWARM_PACKAGED: isPackaged ? '1' : '0', OPENSWARM_PORT: String(backendPort), PYTHONDONTWRITEBYTECODE: '1', + GOOGLE_OAUTH_CLIENT_ID: process.env.GOOGLE_OAUTH_CLIENT_ID || 'REPLACE_WITH_YOUR_CLIENT_ID.apps.googleusercontent.com', + GOOGLE_OAUTH_CLIENT_SECRET: process.env.GOOGLE_OAUTH_CLIENT_SECRET || 'REPLACE_WITH_YOUR_CLIENT_SECRET', }; if (isPackaged) { diff --git a/electron/package-lock.json b/electron/package-lock.json index 769e8896..7fbc9ffd 100644 --- a/electron/package-lock.json +++ b/electron/package-lock.json @@ -1,12 +1,12 @@ { "name": "openswarm", - "version": "1.0.4", + "version": "1.0.5", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openswarm", - "version": "1.0.4", + "version": "1.0.5", "hasInstallScript": true, "dependencies": { "electron-updater": "^6.3.0", diff --git a/electron/package.json b/electron/package.json index a3320e17..812e5983 100644 --- a/electron/package.json +++ b/electron/package.json @@ -1,6 +1,6 @@ { "name": "openswarm", - "version": "1.0.4", + "version": "1.0.5", "description": "OpenSwarm — AI Agent Orchestrator", "main": "main.js", "scripts": { @@ -43,6 +43,7 @@ "entitlementsInherit": "build/entitlements.mac.plist" }, "dmg": { + "artifactName": "OpenSwarm-${arch}.${ext}", "title": "OpenSwarm", "contents": [ { diff --git a/frontend/src/app/Main.tsx b/frontend/src/app/Main.tsx index 7628f5f0..67748ca8 100644 --- a/frontend/src/app/Main.tsx +++ b/frontend/src/app/Main.tsx @@ -3,7 +3,7 @@ import { Provider } from 'react-redux'; import { HashRouter, Routes, Route } from 'react-router-dom'; import { ThemeProvider as MuiThemeProvider, createTheme, CssBaseline } from '@mui/material'; import { store } from '../shared/state/store'; -import { useAppDispatch } from '@/shared/hooks'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { fetchSettings } from '@/shared/state/settingsSlice'; import { setAppVersion, @@ -155,9 +155,15 @@ const ShortcutsProvider: React.FC<{ children: React.ReactNode }> = ({ children } const SettingsLoader: React.FC<{ children: React.ReactNode }> = ({ children }) => { const dispatch = useAppDispatch(); + const { setMode: setThemeMode } = useThemeMode(); + const theme = useAppSelector((s) => s.settings.data.theme); + const loaded = useAppSelector((s) => s.settings.loaded); useEffect(() => { dispatch(fetchSettings()); }, [dispatch]); + useEffect(() => { + if (loaded) setThemeMode(theme as 'light' | 'dark'); + }, [loaded, theme, setThemeMode]); return <>{children}; }; diff --git a/frontend/src/app/components/GlobalApprovalOverlay.tsx b/frontend/src/app/components/GlobalApprovalOverlay.tsx index 4405e7bf..e4200ee4 100644 --- a/frontend/src/app/components/GlobalApprovalOverlay.tsx +++ b/frontend/src/app/components/GlobalApprovalOverlay.tsx @@ -3,11 +3,22 @@ import Box from '@mui/material/Box'; import Typography from '@mui/material/Typography'; import Chip from '@mui/material/Chip'; import IconButton from '@mui/material/IconButton'; +import Tooltip from '@mui/material/Tooltip'; import ExpandLessIcon from '@mui/icons-material/ExpandLess'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import NotificationsActiveIcon from '@mui/icons-material/NotificationsActive'; +import StopCircleOutlinedIcon from '@mui/icons-material/StopCircleOutlined'; +import CloseIcon from '@mui/icons-material/Close'; +import { useNavigate } from 'react-router-dom'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; -import { handleApproval, ApprovalRequest } from '@/shared/state/agentsSlice'; +import { + handleApproval, + stopAgent, + dismissAgentNotification, + ApprovalRequest, + AgentSession, + HistorySession, +} from '@/shared/state/agentsSlice'; import ApprovalBar, { BatchApprovalBar } from '@/app/pages/AgentChat/ApprovalBar'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; @@ -17,10 +28,127 @@ interface SessionApprovalGroup { approvals: ApprovalRequest[]; } +type TrackedAgent = { + id: string; + name: string; + status: AgentSession['status'] | string; + dashboardId?: string; +}; + +const STATUS_CONFIG: Record = { + running: { color: '', label: 'Running', tokenKey: 'success' }, + waiting_approval: { color: '', label: 'Waiting', tokenKey: 'warning' }, + completed: { color: '', label: 'Done', tokenKey: 'success' }, + error: { color: '', label: 'Error', tokenKey: 'error' }, + stopped: { color: '', label: 'Stopped', tokenKey: 'info' }, +}; + +const StatusDot: React.FC<{ status: string; c: ReturnType }> = ({ status, c }) => { + const cfg = STATUS_CONFIG[status]; + const color = cfg?.tokenKey ? (c.status as any)[cfg.tokenKey] : c.text.ghost; + const isActive = status === 'running'; + return ( + + ); +}; + +const AgentStatusRow: React.FC<{ + agent: TrackedAgent; + c: ReturnType; + onStop: (id: string) => void; + onDismiss: (id: string) => void; + onNavigate: (dashboardId: string) => void; +}> = ({ agent, c, onStop, onDismiss, onNavigate }) => { + const isActive = agent.status === 'running' || agent.status === 'waiting_approval'; + const cfg = STATUS_CONFIG[agent.status] ?? { label: agent.status }; + + return ( + agent.dashboardId && onNavigate(agent.dashboardId)} + sx={{ + display: 'flex', + alignItems: 'center', + gap: 1, + px: 2, + py: 0.75, + cursor: agent.dashboardId ? 'pointer' : 'default', + '&:hover': { bgcolor: `${c.text.ghost}10` }, + transition: 'background-color 0.15s', + minHeight: 36, + }} + > + + + {agent.name} + + + {cfg.label} + + {isActive ? ( + + { e.stopPropagation(); onStop(agent.id); }} + sx={{ p: 0.25, color: c.status.error, '&:hover': { bgcolor: `${c.status.error}15` } }} + > + + + + ) : ( + + { e.stopPropagation(); onDismiss(agent.id); }} + sx={{ p: 0.25, color: c.text.ghost, '&:hover': { bgcolor: `${c.text.ghost}15` } }} + > + + + + )} + + ); +}; + const GlobalApprovalOverlay: React.FC = () => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const navigate = useNavigate(); const sessions = useAppSelector((state) => state.agents.sessions); + const history = useAppSelector((state) => state.agents.history); + const trackedIds = useAppSelector((state) => state.agents.trackedNotificationIds); const [collapsed, setCollapsed] = useState(false); const groups: SessionApprovalGroup[] = useMemo(() => { @@ -42,11 +170,38 @@ const GlobalApprovalOverlay: React.FC = () => { [groups], ); + const trackedAgents: TrackedAgent[] = useMemo(() => { + return trackedIds + .map((id): TrackedAgent | null => { + const session = sessions[id]; + if (session && session.status !== 'draft') { + return { id, name: session.name, status: session.status, dashboardId: session.dashboard_id }; + } + const hist: HistorySession | undefined = history[id]; + if (hist) { + return { id, name: hist.name, status: hist.status, dashboardId: hist.dashboard_id }; + } + return null; + }) + .filter((a): a is TrackedAgent => a !== null); + }, [trackedIds, sessions, history]); + + const activeAgents = useMemo( + () => trackedAgents.filter((a) => a.status === 'running' || a.status === 'waiting_approval'), + [trackedAgents], + ); + const finishedAgents = useMemo( + () => trackedAgents.filter((a) => a.status !== 'running' && a.status !== 'waiting_approval'), + [trackedAgents], + ); + + const totalBadge = totalApprovals + activeAgents.length; + useEffect(() => { - if (totalApprovals > 0) { + if (totalApprovals > 0 || activeAgents.length > 0) { setCollapsed(false); } - }, [totalApprovals]); + }, [totalApprovals, activeAgents.length]); const onApprove = useCallback( (requestId: string, updatedInput?: Record) => { @@ -62,7 +217,38 @@ const GlobalApprovalOverlay: React.FC = () => { [dispatch], ); - if (totalApprovals === 0) return null; + const onStopAgent = useCallback( + (sessionId: string) => { + dispatch(stopAgent({ sessionId })); + }, + [dispatch], + ); + + const onDismissAgent = useCallback( + (sessionId: string) => { + dispatch(dismissAgentNotification(sessionId)); + }, + [dispatch], + ); + + const onNavigateToDashboard = useCallback( + (dashboardId: string) => { + navigate(`/dashboard/${dashboardId}`); + }, + [navigate], + ); + + if (totalApprovals === 0 && trackedAgents.length === 0) return null; + + const hasApprovals = totalApprovals > 0; + const hasAgents = trackedAgents.length > 0; + const headerTitle = hasApprovals && !hasAgents + ? 'Approval Required' + : hasAgents && !hasApprovals + ? 'Agents' + : 'Notifications'; + const headerColor = hasApprovals ? c.status.warning : c.status.info; + const headerBg = hasApprovals ? c.status.warningBg : c.status.infoBg; return ( { flexDirection: 'column', borderRadius: `${c.radius.xl}px`, bgcolor: c.bg.surface, - border: `1px solid ${c.status.warning}40`, - boxShadow: `0 8px 32px rgba(0,0,0,0.25), 0 0 0 1px ${c.status.warning}20`, + border: `1px solid ${headerColor}40`, + boxShadow: `0 8px 32px rgba(0,0,0,0.25), 0 0 0 1px ${headerColor}20`, overflow: 'hidden', animation: 'approvalSlideIn 0.25s ease-out', '@keyframes approvalSlideIn': { @@ -97,19 +283,19 @@ const GlobalApprovalOverlay: React.FC = () => { gap: 1, px: 2, py: 1.25, - bgcolor: c.status.warningBg, - borderBottom: collapsed ? 'none' : `1px solid ${c.status.warning}20`, + bgcolor: headerBg, + borderBottom: collapsed ? 'none' : `1px solid ${headerColor}20`, cursor: 'pointer', userSelect: 'none', - '&:hover': { bgcolor: `${c.status.warning}18` }, + '&:hover': { bgcolor: `${headerColor}18` }, transition: 'background-color 0.15s', }} > { }, }} /> - - Approval Required + + {headerTitle} - + {totalBadge > 0 && ( + + )} {collapsed ? : } @@ -146,7 +334,6 @@ const GlobalApprovalOverlay: React.FC = () => { { scrollbarColor: `${c.border.medium} transparent`, }} > - {groups.map((group) => ( - - {groups.length > 1 && ( + {/* Approvals section */} + {hasApprovals && ( + + {hasAgents && ( - {group.sessionName} + Approvals )} - {group.approvals.length > 1 ? ( - - ) : ( - group.approvals.map((req) => ( - - )) - )} + {groups.map((group) => ( + + {groups.length > 1 && ( + + {group.sessionName} + + )} + {group.approvals.length > 1 ? ( + + ) : ( + group.approvals.map((req) => ( + + )) + )} + + ))} - ))} + )} + + {/* Divider between sections */} + {hasApprovals && hasAgents && ( + + )} + + {/* Agent status section */} + {hasAgents && ( + + {hasApprovals && ( + + Agents + + )} + {activeAgents.map((agent) => ( + + ))} + {finishedAgents.map((agent) => ( + + ))} + + )} )} diff --git a/frontend/src/app/pages/Dashboard/AgentCard.tsx b/frontend/src/app/pages/Dashboard/AgentCard.tsx index 31c2e07f..2bd69d43 100644 --- a/frontend/src/app/pages/Dashboard/AgentCard.tsx +++ b/frontend/src/app/pages/Dashboard/AgentCard.tsx @@ -28,6 +28,7 @@ import { QuestionForm } from '@/app/pages/AgentChat/ApprovalBar'; import AgentChat from '@/app/pages/AgentChat/AgentChat'; import { parseMcpToolName, getMcpShortAction } from '@/app/pages/AgentChat/ToolCallBubble'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { useOverlayScrollPassthrough } from './useOverlayScrollPassthrough'; // --------------------------------------------------------------------------- // Helper components & functions (unchanged) @@ -190,6 +191,7 @@ const AgentCard: React.FC = ({ }) => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); const STATUS_COLORS: Record = { running: { color: c.status.success, bg: c.status.successBg }, @@ -481,9 +483,10 @@ const AgentCard: React.FC = ({ /> ))} - {/* Selection overlay – blocks content interaction while selected, enabling drag from anywhere */} + {/* Selection overlay – blocks click interaction while selected, enabling drag from anywhere */} {isSelected && ( = ({ }) => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage); const elementSelectionCtx = useElementSelection(); const isElementSelectMode = elementSelectionCtx?.selectMode ?? false; @@ -590,9 +592,10 @@ const BrowserCard: React.FC = ({ }), }} > - {/* Selection overlay – blocks content interaction while selected, enabling drag from anywhere */} + {/* Selection overlay – blocks click interaction while selected, enabling drag from anywhere */} {isSelected && ( ( const searchInputRef = useRef(null); const historyInputRef = useRef(null); const historyListRef = useRef(null); - const [mode, setMode] = useState('agent'); - const [model, setModel] = useState('sonnet'); + const defaultMode = useAppSelector((s) => s.settings.data.default_mode); + const defaultModel = useAppSelector((s) => s.settings.data.default_model); + const [mode, setMode] = useState(defaultMode || 'agent'); + const [model, setModel] = useState(defaultModel || 'sonnet'); + const settingsApplied = useRef(false); + useEffect(() => { + if (!settingsApplied.current) { + setMode(defaultMode || 'agent'); + setModel(defaultModel || 'sonnet'); + settingsApplied.current = true; + } + }, [defaultMode, defaultModel]); const [viewPickerOpen, setViewPickerOpen] = useState(false); const [viewSearch, setViewSearch] = useState(''); const [historyOpen, setHistoryOpen] = useState(false); diff --git a/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx b/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx index 19c7fd0b..ecdde20d 100644 --- a/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx @@ -14,6 +14,7 @@ import { useAppDispatch } from '@/shared/hooks'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; import ViewPreview, { ViewPreviewHandle } from '@/app/pages/Views/ViewPreview'; import { getDefault } from '@/app/pages/Views/InputSchemaForm'; +import { useOverlayScrollPassthrough } from './useOverlayScrollPassthrough'; type ResizeDir = 'n' | 's' | 'e' | 'w' | 'ne' | 'nw' | 'se' | 'sw'; @@ -60,6 +61,7 @@ const DashboardViewCard: React.FC = ({ }) => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); const previewRef = useRef(null); const [inputData, setInputData] = useState>(() => getDefault(output.input_schema)); @@ -304,9 +306,10 @@ const DashboardViewCard: React.FC = ({ }), }} > - {/* Selection overlay – blocks content interaction while selected, enabling drag from anywhere */} + {/* Selection overlay – blocks click interaction while selected, enabling drag from anywhere */} {isSelected && ( ` elements — executes JS inside the webview to scroll + * the element at the cursor position. + */ +export function useOverlayScrollPassthrough(active: boolean) { + const ref = useRef(null); + + useEffect(() => { + const el = ref.current; + if (!active || !el) return; + + const handleWheel = (e: WheelEvent) => { + if (e.ctrlKey || e.metaKey) return; + + el.style.pointerEvents = 'none'; + const underneath = document.elementFromPoint(e.clientX, e.clientY); + el.style.pointerEvents = ''; + + let dx = e.deltaX; + let dy = e.deltaY; + if (e.deltaMode === 1) { + dx *= 20; + dy *= 20; + } + + let node = underneath as HTMLElement | null; + while (node) { + if (node.tagName === 'WEBVIEW') { + e.stopPropagation(); + e.preventDefault(); + const rect = node.getBoundingClientRect(); + const relX = Math.round(e.clientX - rect.left); + const relY = Math.round(e.clientY - rect.top); + (node as any).executeJavaScript?.( + `(function(){` + + `var el=document.elementFromPoint(${relX},${relY});` + + `while(el){` + + `var s=getComputedStyle(el);` + + `if((s.overflowY==='auto'||s.overflowY==='scroll')&&el.scrollHeight>el.clientHeight){el.scrollBy(${dx},${dy});return}` + + `if((s.overflowX==='auto'||s.overflowX==='scroll')&&el.scrollWidth>el.clientWidth){el.scrollBy(${dx},${dy});return}` + + `el=el.parentElement}` + + `window.scrollBy(${dx},${dy})` + + `})()` + ).catch(() => {}); + return; + } + + const cs = getComputedStyle(node); + const canScrollY = + node.scrollHeight > node.clientHeight && + (cs.overflowY === 'auto' || cs.overflowY === 'scroll'); + const canScrollX = + node.scrollWidth > node.clientWidth && + (cs.overflowX === 'auto' || cs.overflowX === 'scroll'); + + if (canScrollY || canScrollX) { + e.stopPropagation(); + e.preventDefault(); + node.scrollBy(dx, dy); + return; + } + node = node.parentElement; + } + }; + + el.addEventListener('wheel', handleWheel, { passive: false }); + return () => el.removeEventListener('wheel', handleWheel); + }, [active]); + + return ref; +} diff --git a/frontend/src/app/pages/Tools/Tools.tsx b/frontend/src/app/pages/Tools/Tools.tsx index 22c7bdad..9a9dff31 100644 --- a/frontend/src/app/pages/Tools/Tools.tsx +++ b/frontend/src/app/pages/Tools/Tools.tsx @@ -582,7 +582,7 @@ const Tools: React.FC = () => { const handleSectionEnabledChange = async (tools: BuiltinTool[], enabled: boolean) => { const perms: Record = {}; - for (const t of tools) perms[t.name] = enabled ? 'ask' : 'deny'; + for (const t of tools) perms[t.name] = enabled ? 'always_allow' : 'deny'; await dispatch(updateBuiltinPermissions(perms)); }; diff --git a/frontend/src/shared/state/agentsSlice.ts b/frontend/src/shared/state/agentsSlice.ts index 2be0f62c..c52c13cf 100644 --- a/frontend/src/shared/state/agentsSlice.ts +++ b/frontend/src/shared/state/agentsSlice.ts @@ -110,6 +110,7 @@ interface AgentsState { expandedSessionIds: string[]; loading: boolean; historySearch: HistorySearchState; + trackedNotificationIds: string[]; } const initialState: AgentsState = { @@ -119,6 +120,7 @@ const initialState: AgentsState = { expandedSessionIds: [], loading: false, historySearch: { results: [], total: 0, hasMore: false, query: '', loading: false }, + trackedNotificationIds: [], }; export const fetchSessions = createAsyncThunk( @@ -507,6 +509,9 @@ const agentsSlice = createSlice({ streamingMessage: existing?.streamingMessage ?? action.payload.streamingMessage ?? null, tool_group_meta: { ...existing?.tool_group_meta, ...action.payload.tool_group_meta }, }; + if (action.payload.status === 'running' && !state.trackedNotificationIds.includes(action.payload.id)) { + state.trackedNotificationIds.push(action.payload.id); + } }, updateSessionStatus( @@ -517,6 +522,9 @@ const agentsSlice = createSlice({ if (session) { session.status = action.payload.status; } + if (action.payload.status === 'running' && !state.trackedNotificationIds.includes(action.payload.sessionId)) { + state.trackedNotificationIds.push(action.payload.sessionId); + } }, addMessage(state, action: PayloadAction<{ sessionId: string; message: AgentMessage }>) { @@ -673,6 +681,18 @@ const agentsSlice = createSlice({ clearHistorySearch(state) { state.historySearch = { results: [], total: 0, hasMore: false, query: '', loading: false }; }, + + trackAgentNotification(state, action: PayloadAction) { + if (!state.trackedNotificationIds.includes(action.payload)) { + state.trackedNotificationIds.push(action.payload); + } + }, + + dismissAgentNotification(state, action: PayloadAction) { + state.trackedNotificationIds = state.trackedNotificationIds.filter( + (id) => id !== action.payload, + ); + }, }, extraReducers: (builder) => { builder @@ -704,6 +724,9 @@ const agentsSlice = createSlice({ if (!state.expandedSessionIds.includes(action.payload.id)) { state.expandedSessionIds.push(action.payload.id); } + if (!state.trackedNotificationIds.includes(action.payload.id)) { + state.trackedNotificationIds.push(action.payload.id); + } }) .addCase(launchAndSendFirstMessage.fulfilled, (state, action) => { const { draftId, session } = action.payload; @@ -715,6 +738,9 @@ const agentsSlice = createSlice({ if (shouldExpand && !state.expandedSessionIds.includes(session.id)) { state.expandedSessionIds.push(session.id); } + if (!state.trackedNotificationIds.includes(session.id)) { + state.trackedNotificationIds.push(session.id); + } }) .addCase(generateTitle.fulfilled, (state, action) => { const session = state.sessions[action.payload.sessionId]; @@ -810,6 +836,7 @@ const agentsSlice = createSlice({ state.activeSessionId = null; } state.expandedSessionIds = state.expandedSessionIds.filter((id) => id !== sessionId); + state.trackedNotificationIds = state.trackedNotificationIds.filter((id) => id !== sessionId); }) .addCase(fetchHistory.fulfilled, (state, action) => { const history: Record = {}; @@ -897,6 +924,8 @@ export const { closeSessionFromWs, removeDraftSession, clearHistorySearch, + trackAgentNotification, + dismissAgentNotification, } = agentsSlice.actions; export default agentsSlice.reducer; diff --git a/frontend/src/shared/state/dashboardLayoutSlice.ts b/frontend/src/shared/state/dashboardLayoutSlice.ts index 721aef6c..8a51e148 100644 --- a/frontend/src/shared/state/dashboardLayoutSlice.ts +++ b/frontend/src/shared/state/dashboardLayoutSlice.ts @@ -259,9 +259,9 @@ const dashboardLayoutSlice = createSlice({ if (total === 0) return; const allItems = [ - ...agentCards.map((c) => ({ kind: 'agent' as const, id: c.session_id, x: c.x, y: c.y, storedH: c.height })), - ...viewCards.map((c) => ({ kind: 'view' as const, id: c.output_id, x: c.x, y: c.y, storedH: c.height })), - ...bCards.map((c) => ({ kind: 'browser' as const, id: c.browser_id, x: c.x, y: c.y, storedH: c.height })), + ...agentCards.map((c) => ({ kind: 'agent' as const, id: c.session_id, x: c.x, y: c.y, storedW: c.width, storedH: c.height })), + ...viewCards.map((c) => ({ kind: 'view' as const, id: c.output_id, x: c.x, y: c.y, storedW: c.width, storedH: c.height })), + ...bCards.map((c) => ({ kind: 'browser' as const, id: c.browser_id, x: c.x, y: c.y, storedW: c.width, storedH: c.height })), ]; allItems.sort((a, b) => a.y - b.y || a.x - b.x); @@ -270,12 +270,11 @@ const dashboardLayoutSlice = createSlice({ for (const item of allItems) { let w: number, h: number; if (item.kind === 'agent') { - w = DEFAULT_CARD_W; - h = expanded.has(item.id) ? Math.max(EXPANDED_CARD_MIN_H, item.storedH) : DEFAULT_CARD_H; - } else if (item.kind === 'view') { - w = DEFAULT_VIEW_CARD_W; h = DEFAULT_VIEW_CARD_H; + w = item.storedW; + h = expanded.has(item.id) ? Math.max(EXPANDED_CARD_MIN_H, item.storedH) : item.storedH; } else { - w = DEFAULT_BROWSER_CARD_W; h = DEFAULT_BROWSER_CARD_H; + w = item.storedW; + h = item.storedH; } const pos = findOpenGridCell(placedRects, w, h); @@ -283,13 +282,13 @@ const dashboardLayoutSlice = createSlice({ if (item.kind === 'agent') { const card = state.cards[item.id]; - if (card) { card.x = pos.x; card.y = pos.y; card.width = w; card.height = h; } + if (card) { card.x = pos.x; card.y = pos.y; } } else if (item.kind === 'view') { const card = state.viewCards[item.id]; - if (card) { card.x = pos.x; card.y = pos.y; card.width = w; card.height = h; } + if (card) { card.x = pos.x; card.y = pos.y; } } else { const card = state.browserCards[item.id]; - if (card) { card.x = pos.x; card.y = pos.y; card.width = w; card.height = h; } + if (card) { card.x = pos.x; card.y = pos.y; } } } }, diff --git a/frontend/src/shared/ws/WebSocketManager.ts b/frontend/src/shared/ws/WebSocketManager.ts index 71b5dce4..e9bd13d6 100644 --- a/frontend/src/shared/ws/WebSocketManager.ts +++ b/frontend/src/shared/ws/WebSocketManager.ts @@ -14,6 +14,7 @@ import { addBranch, setActiveBranch, closeSessionFromWs, + trackAgentNotification, } from '../state/agentsSlice'; import { addBrowserCardFromBackend } from '../state/dashboardLayoutSlice'; @@ -125,6 +126,9 @@ class WebSocketManager { } else if (session_id) { store.dispatch(updateSessionStatus({ sessionId: session_id, status: data.status })); } + if (data.status === 'running' && session_id) { + store.dispatch(trackAgentNotification(session_id)); + } break; case 'agent:message': From dfa87389ba423f2ce1c5c89b540e18338f000a5c Mon Sep 17 00:00:00 2001 From: haikdc Date: Wed, 18 Mar 2026 02:25:31 -0700 Subject: [PATCH 04/11] [Haik]: (cmd+scroll zoom works over browsers) (single clicking a selected item deselects it) (All browser actions have user permissions that r functional blocks) --- backend/apps/agents/agent_manager.py | 55 ++++-- backend/apps/agents/browser_agent.py | 81 ++++++++- backend/apps/tools_lib/models.py | 13 ++ .../src/app/pages/Dashboard/AgentCard.tsx | 5 +- .../src/app/pages/Dashboard/BrowserCard.tsx | 6 +- .../src/app/pages/Dashboard/Dashboard.tsx | 2 + .../app/pages/Dashboard/DashboardViewCard.tsx | 6 +- .../pages/Dashboard/useDashboardSelection.ts | 5 +- frontend/src/app/pages/Tools/Tools.tsx | 166 +++++++++++++++++- 9 files changed, 312 insertions(+), 27 deletions(-) diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index 3f705df0..0d916495 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -487,6 +487,11 @@ class AgentManager: return _builtin_perms[tool_name] import re as _re + + bm = _re.match(r"mcp__openswarm-browser-agent__(.+)", tool_name) + if bm: + return _builtin_perms.get(bm.group(1), "always_allow") + m = _re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name) if m: server_slug, mcp_tool_name = m.group(1), m.group(2) @@ -639,23 +644,30 @@ class AgentManager: mcp_servers = await self._build_mcp_servers(session.allowed_tools) - browser_agent_server_path = os.path.join( - os.path.dirname(__file__), "browser_agent_mcp_server.py" + _browser_delegation_tools = ["BrowserAgent", "BrowserAgents"] + _browser_all_denied = all( + _builtin_perms.get(t, "always_allow") == "deny" + for t in _browser_delegation_tools ) - backend_port = os.environ.get("OPENSWARM_PORT", "8324") - pre_selected_bids = self._get_pre_selected_browser_ids(session.dashboard_id) - mcp_servers["openswarm-browser-agent"] = { - "command": sys.executable, - "args": [browser_agent_server_path], - "env": { - "OPENSWARM_PORT": backend_port, - "OPENSWARM_AGENT_MODEL": session.model, - "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", - "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), - "OPENSWARM_PARENT_SESSION_ID": session.id, - }, - "type": "stdio", - } + + if not _browser_all_denied: + browser_agent_server_path = os.path.join( + os.path.dirname(__file__), "browser_agent_mcp_server.py" + ) + backend_port = os.environ.get("OPENSWARM_PORT", "8324") + pre_selected_bids = self._get_pre_selected_browser_ids(session.dashboard_id) + mcp_servers["openswarm-browser-agent"] = { + "command": sys.executable, + "args": [browser_agent_server_path], + "env": { + "OPENSWARM_PORT": backend_port, + "OPENSWARM_AGENT_MODEL": session.model, + "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", + "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), + "OPENSWARM_PARENT_SESSION_ID": session.id, + }, + "type": "stdio", + } effective_allowed = [ t for t in session.allowed_tools @@ -670,6 +682,15 @@ class AgentManager: if mcp_servers: all_tools_list = load_all_tools() for name in mcp_servers: + if name == "openswarm-browser-agent": + for bt in _browser_delegation_tools: + policy = _builtin_perms.get(bt, "always_allow") + if policy == "always_allow": + effective_allowed.append(f"mcp__openswarm-browser-agent__{bt}") + elif policy == "deny": + effective_disallowed.append(f"mcp__openswarm-browser-agent__{bt}") + continue + tool_def = next( (t for t in all_tools_list if t.mcp_config and t.enabled and _sanitize_server_name(t.name) == name), @@ -687,8 +708,6 @@ class AgentManager: else: effective_allowed.append(f"mcp__{name}__*") - effective_allowed.append("mcp__openswarm-browser-agent__*") - options_kwargs = { "model": session.model, "max_buffer_size": 5 * 1024 * 1024, diff --git a/backend/apps/agents/browser_agent.py b/backend/apps/agents/browser_agent.py index 09ba789d..51d8b0ab 100644 --- a/backend/apps/agents/browser_agent.py +++ b/backend/apps/agents/browser_agent.py @@ -15,8 +15,9 @@ from uuid import uuid4 import anthropic -from backend.apps.agents.models import AgentSession, Message +from backend.apps.agents.models import AgentSession, ApprovalRequest, Message from backend.apps.agents.ws_manager import ws_manager +from backend.apps.tools_lib.tools_lib import load_builtin_permissions logger = logging.getLogger(__name__) @@ -233,6 +234,40 @@ def _format_tool_result(result: dict, tool_name: str) -> list[dict]: return [{"type": "text", "text": str(text)}] +async def _request_browser_approval( + session: AgentSession, tool_name: str, tool_input: dict, +) -> dict: + """Send an approval request for a browser sub-agent tool and wait for the decision.""" + request_id = uuid4().hex + approval_req = ApprovalRequest( + id=request_id, + session_id=session.id, + tool_name=tool_name, + tool_input=tool_input, + ) + session.pending_approvals.append(approval_req) + session.status = "waiting_approval" + + await ws_manager.send_to_session(session.id, "agent:status", { + "session_id": session.id, + "status": "waiting_approval", + }) + + decision = await ws_manager.send_approval_request( + session.id, request_id, tool_name, tool_input, + ) + + session.pending_approvals = [ + a for a in session.pending_approvals if a.id != request_id + ] + session.status = "running" + await ws_manager.send_to_session(session.id, "agent:status", { + "session_id": session.id, + "status": "running", + }) + return decision + + async def run_browser_agent( task: str, browser_id: str, @@ -251,6 +286,8 @@ async def run_browser_agent( """ from backend.apps.agents.agent_manager import agent_manager + _browser_perms = load_builtin_permissions() + session_id = uuid4().hex session = AgentSession( id=session_id, @@ -347,6 +384,48 @@ async def run_browser_agent( tool_results = [] for tu in tool_uses: + policy = _browser_perms.get(tu.name, "always_allow") + + if policy == "deny": + denied_text = f"Tool {tu.name} is denied by permission policy." + tool_results.append({ + "type": "tool_result", + "tool_use_id": tu.id, + "content": [{"type": "text", "text": denied_text}], + }) + result_msg = Message( + role="tool_result", + content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0}, + ) + session.messages.append(result_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": result_msg.model_dump(mode="json"), + }) + continue + + if policy == "ask": + decision = await _request_browser_approval( + session, tu.name, tu.input, + ) + if decision.get("behavior") == "deny": + denied_text = decision.get("message") or f"Tool {tu.name} denied by user." + tool_results.append({ + "type": "tool_result", + "tool_use_id": tu.id, + "content": [{"type": "text", "text": denied_text}], + }) + result_msg = Message( + role="tool_result", + content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0}, + ) + session.messages.append(result_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": result_msg.model_dump(mode="json"), + }) + continue + start = time.time() result = await execute_browser_tool( tu.name, tu.input, browser_id, tab_id, diff --git a/backend/apps/tools_lib/models.py b/backend/apps/tools_lib/models.py index 3c0cc24e..e00d4be6 100644 --- a/backend/apps/tools_lib/models.py +++ b/backend/apps/tools_lib/models.py @@ -33,6 +33,19 @@ BUILTIN_TOOLS: list[BuiltinTool] = [ BuiltinTool(name="CronList", description="List all scheduled tasks", category="scheduling", deferred=True), BuiltinTool(name="CronDelete", description="Delete a scheduled task", category="scheduling", deferred=True), BuiltinTool(name="RenderOutput", description="Render a reusable View artifact with structured input data", category="views", deferred=True), + # Browser delegation tools (Layer 1 — what the main agent calls) + BuiltinTool(name="BrowserAgent", description="Delegate a browser task to a dedicated browser agent", category="browser_delegation"), + BuiltinTool(name="BrowserAgents", description="Run multiple browser tasks in parallel", category="browser_delegation"), + # Browser action tools (Layer 2 — what the sub-agent executes) + BuiltinTool(name="BrowserScreenshot", description="Capture a screenshot of the browser page", category="browser_action"), + BuiltinTool(name="BrowserNavigate", description="Navigate the browser to a URL", category="browser_action"), + BuiltinTool(name="BrowserClick", description="Click an element by CSS selector", category="browser_action"), + BuiltinTool(name="BrowserType", description="Type text into an input element", category="browser_action"), + BuiltinTool(name="BrowserEvaluate", description="Execute JavaScript in the browser", category="browser_action"), + BuiltinTool(name="BrowserGetText", description="Get visible text content of the page", category="browser_action"), + BuiltinTool(name="BrowserGetElements", description="List interactive elements with CSS selectors", category="browser_action"), + BuiltinTool(name="BrowserScroll", description="Scroll the page up or down", category="browser_action"), + BuiltinTool(name="BrowserWait", description="Wait for page loads or animations", category="browser_action"), ] diff --git a/frontend/src/app/pages/Dashboard/AgentCard.tsx b/frontend/src/app/pages/Dashboard/AgentCard.tsx index 2bd69d43..7e02b366 100644 --- a/frontend/src/app/pages/Dashboard/AgentCard.tsx +++ b/frontend/src/app/pages/Dashboard/AgentCard.tsx @@ -402,9 +402,11 @@ const AgentCard: React.FC = ({ data-select-meta={JSON.stringify({ name: session.name || session.id, status: session.status, model: session.model, mode: session.mode })} onClick={(e: React.MouseEvent) => { if (justDraggedRef.current) return; + if (!isSelected && !e.shiftKey) { + dispatch(toggleExpandSession(session.id)); + } onCardSelect?.(session.id, 'agent', e.shiftKey); }} - onDoubleClick={() => dispatch(toggleExpandSession(session.id))} sx={{ position: 'relative', width: localResize ? activeW : Math.max(cardWidth, MIN_W), @@ -494,7 +496,6 @@ const AgentCard: React.FC = ({ if (justDraggedRef.current) return; onCardSelect?.(session.id, 'agent', e.shiftKey); }} - onDoubleClick={() => dispatch(toggleExpandSession(session.id))} sx={{ position: 'absolute', inset: 0, diff --git a/frontend/src/app/pages/Dashboard/BrowserCard.tsx b/frontend/src/app/pages/Dashboard/BrowserCard.tsx index 28691491..bf5eb8f8 100644 --- a/frontend/src/app/pages/Dashboard/BrowserCard.tsx +++ b/frontend/src/app/pages/Dashboard/BrowserCard.tsx @@ -93,6 +93,7 @@ interface Props { cardWidth: number; cardHeight: number; zoom?: number; + cmdHeld?: boolean; isSelected?: boolean; isHighlighted?: boolean; multiDragDelta?: { dx: number; dy: number } | null; @@ -104,7 +105,7 @@ interface Props { const BrowserCard: React.FC = ({ - browserId, tabs, activeTabId, cardX, cardY, cardWidth, cardHeight, zoom = 1, + browserId, tabs, activeTabId, cardX, cardY, cardWidth, cardHeight, zoom = 1, cmdHeld = false, isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd, }) => { const c = useClaudeTokens(); @@ -982,6 +983,9 @@ const BrowserCard: React.FC = ({ {isElementSelectMode && ( )} + {cmdHeld && !isSelected && ( + + )} {isElectron ? ( tabs.map((tab) => ( { cardWidth={vc.width} cardHeight={vc.height} zoom={canvas.zoom} + cmdHeld={canvas.cmdHeld} isSelected={selection.isSelected(vc.output_id)} isHighlighted={highlightedCardId === vc.output_id} multiDragDelta={multiDragDelta} @@ -621,6 +622,7 @@ const DashboardInner: React.FC = () => { cardWidth={bc.width} cardHeight={bc.height} zoom={canvas.zoom} + cmdHeld={canvas.cmdHeld} isSelected={selection.isSelected(bc.browser_id)} isHighlighted={highlightedCardId === bc.browser_id} multiDragDelta={multiDragDelta} diff --git a/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx b/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx index ecdde20d..7b6437c2 100644 --- a/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx @@ -46,6 +46,7 @@ interface Props { cardWidth: number; cardHeight: number; zoom?: number; + cmdHeld?: boolean; isSelected?: boolean; isHighlighted?: boolean; multiDragDelta?: { dx: number; dy: number } | null; @@ -56,7 +57,7 @@ interface Props { } const DashboardViewCard: React.FC = ({ - output, cardX, cardY, cardWidth, cardHeight, zoom = 1, + output, cardX, cardY, cardWidth, cardHeight, zoom = 1, cmdHeld = false, isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd, }) => { const c = useClaudeTokens(); @@ -404,6 +405,9 @@ const DashboardViewCard: React.FC = ({ {/* Preview body */} + {cmdHeld && !isSelected && ( + + )} { const [expandedServices, setExpandedServices] = useState>({}); const [viewsSectionOpen, setViewsSectionOpen] = useState(false); + const [browserSectionOpen, setBrowserSectionOpen] = useState(false); + const [browserCollapsed, setBrowserCollapsed] = useState>({ browser_delegation: true, browser_action: true }); const [builtinSectionOpen, setBuiltinSectionOpen] = useState(true); useEffect(() => { @@ -557,8 +559,12 @@ const Tools: React.FC = () => { }; // Built-in tool grouping - const coreTools = useMemo(() => builtinTools.filter((bt) => !bt.deferred), [builtinTools]); - const deferredTools = useMemo(() => builtinTools.filter((bt) => bt.deferred), [builtinTools]); + const BROWSER_CATEGORIES = new Set(['browser_delegation', 'browser_action']); + const coreTools = useMemo(() => builtinTools.filter((bt) => !bt.deferred && !BROWSER_CATEGORIES.has(bt.category)), [builtinTools]); + const deferredTools = useMemo(() => builtinTools.filter((bt) => bt.deferred && !BROWSER_CATEGORIES.has(bt.category)), [builtinTools]); + const browserTools = useMemo(() => builtinTools.filter((bt) => BROWSER_CATEGORIES.has(bt.category)), [builtinTools]); + const browserDelegationTools = useMemo(() => browserTools.filter((bt) => bt.category === 'browser_delegation'), [browserTools]); + const browserActionTools = useMemo(() => browserTools.filter((bt) => bt.category === 'browser_action'), [browserTools]); const groupTools = (list: BuiltinTool[]) => { const g: Record = {}; for (const bt of list) { if (!g[bt.category]) g[bt.category] = []; g[bt.category].push(bt); } @@ -579,6 +585,10 @@ const Tools: React.FC = () => { () => !outputs.every((o) => o.permission === 'deny'), [outputs], ); + const browserSectionEnabled = useMemo( + () => browserTools.length > 0 && !browserTools.every((t) => builtinPermissions[t.name] === 'deny'), + [browserTools, builtinPermissions], + ); const handleSectionEnabledChange = async (tools: BuiltinTool[], enabled: boolean) => { const perms: Record = {}; @@ -895,7 +905,7 @@ const Tools: React.FC = () => { {builtinSectionOpen ? : } Built-in Action Sets - + @@ -993,6 +1003,156 @@ const Tools: React.FC = () => { )} + {/* Browser */} + {browserTools.length > 0 && ( + + + browserSectionEnabled && setBrowserSectionOpen((v) => !v)} + sx={{ display: 'flex', alignItems: 'center', gap: 2, cursor: browserSectionEnabled ? 'pointer' : 'default' }} + > + + + + + + Browser + + + Browser automation delegation and individual browser actions + + e.stopPropagation()}> + handleSectionEnabledChange(browserTools, checked)} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {browserSectionEnabled && ( + + + + )} + + + + + + + + Action Permissions + + + + + {/* Delegation group */} + {browserDelegationTools.length > 0 && (() => { + const delegationPolicies = browserDelegationTools.map((t) => builtinPermissions[t.name] || 'always_allow'); + const groupPolicy = delegationPolicies.every((p) => p === 'always_allow') ? 'always_allow' + : delegationPolicies.every((p) => p === 'deny') ? 'deny' + : delegationPolicies.every((p) => p === 'ask') ? 'ask' : 'ask'; + const isOpen = !browserCollapsed.browser_delegation; + return ( + + setBrowserCollapsed((p) => ({ ...p, browser_delegation: !p.browser_delegation }))} + > + + + Delegation + + + e.stopPropagation()}> + handleBuiltinCategoryPermissionChange(browserDelegationTools.map((t) => t.name), 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: groupPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinCategoryPermissionChange(browserDelegationTools.map((t) => t.name), 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: groupPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinCategoryPermissionChange(browserDelegationTools.map((t) => t.name), 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: groupPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + + + {browserDelegationTools.map((bt) => { + const toolPolicy = builtinPermissions[bt.name] || 'always_allow'; + return ( + + + {bt.name} + {bt.description && {bt.description}} + + e.stopPropagation()}> + handleBuiltinPermissionChange(bt.name, 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: toolPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinPermissionChange(bt.name, 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: toolPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinPermissionChange(bt.name, 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: toolPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + ); + })} + + + + ); + })()} + + {/* Browser Actions group */} + {browserActionTools.length > 0 && (() => { + const actionPolicies = browserActionTools.map((t) => builtinPermissions[t.name] || 'always_allow'); + const groupPolicy = actionPolicies.every((p) => p === 'always_allow') ? 'always_allow' + : actionPolicies.every((p) => p === 'deny') ? 'deny' + : actionPolicies.every((p) => p === 'ask') ? 'ask' : 'ask'; + const isOpen = !browserCollapsed.browser_action; + return ( + + setBrowserCollapsed((p) => ({ ...p, browser_action: !p.browser_action }))} + > + + + Browser Actions + + + e.stopPropagation()}> + handleBuiltinCategoryPermissionChange(browserActionTools.map((t) => t.name), 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: groupPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinCategoryPermissionChange(browserActionTools.map((t) => t.name), 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: groupPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinCategoryPermissionChange(browserActionTools.map((t) => t.name), 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: groupPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + + + {browserActionTools.map((bt) => { + const toolPolicy = builtinPermissions[bt.name] || 'always_allow'; + return ( + + + {bt.name} + {bt.description && {bt.description}} + + e.stopPropagation()}> + handleBuiltinPermissionChange(bt.name, 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: toolPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinPermissionChange(bt.name, 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: toolPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinPermissionChange(bt.name, 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: toolPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + ); + })} + + + + ); + })()} + + + + + )} + From f8c2fc38b8f93a4c9bee42ef7abf50862c91bb96 Mon Sep 17 00:00:00 2001 From: haikdc Date: Wed, 18 Mar 2026 02:55:21 -0700 Subject: [PATCH 05/11] [Haik]: ckpt (right click in dash triggers marquee drag) (if click enter while a single agent card is selected in dash -> triggers expand/collapse) (also btw that g workspace issue still exists in prod) --- backend/apps/tools_lib/tools_lib.py | 5 +++- electron/package-lock.json | 4 +-- electron/package.json | 2 +- .../src/app/pages/Dashboard/AgentCard.tsx | 2 ++ .../src/app/pages/Dashboard/BrowserCard.tsx | 2 ++ .../src/app/pages/Dashboard/Dashboard.tsx | 25 ++++++++++++++++++- .../app/pages/Dashboard/DashboardViewCard.tsx | 2 ++ .../pages/Dashboard/useDashboardSelection.ts | 2 +- 8 files changed, 38 insertions(+), 6 deletions(-) diff --git a/backend/apps/tools_lib/tools_lib.py b/backend/apps/tools_lib/tools_lib.py index b7710a9f..274f077c 100644 --- a/backend/apps/tools_lib/tools_lib.py +++ b/backend/apps/tools_lib/tools_lib.py @@ -618,7 +618,10 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, proc.terminate() await asyncio.wait_for(proc.wait(), timeout=5.0) except Exception: - proc.kill() + try: + proc.kill() + except Exception: + pass @tools_lib.router.post("/{tool_id}/discover") diff --git a/electron/package-lock.json b/electron/package-lock.json index 7fbc9ffd..bb3fa6aa 100644 --- a/electron/package-lock.json +++ b/electron/package-lock.json @@ -1,12 +1,12 @@ { "name": "openswarm", - "version": "1.0.5", + "version": "1.0.6", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openswarm", - "version": "1.0.5", + "version": "1.0.6", "hasInstallScript": true, "dependencies": { "electron-updater": "^6.3.0", diff --git a/electron/package.json b/electron/package.json index 812e5983..3be7cad4 100644 --- a/electron/package.json +++ b/electron/package.json @@ -1,6 +1,6 @@ { "name": "openswarm", - "version": "1.0.5", + "version": "1.0.6", "description": "OpenSwarm — AI Agent Orchestrator", "main": "main.js", "scripts": { diff --git a/frontend/src/app/pages/Dashboard/AgentCard.tsx b/frontend/src/app/pages/Dashboard/AgentCard.tsx index 7e02b366..12bfb9dc 100644 --- a/frontend/src/app/pages/Dashboard/AgentCard.tsx +++ b/frontend/src/app/pages/Dashboard/AgentCard.tsx @@ -214,6 +214,7 @@ const AgentCard: React.FC = ({ const justDraggedRef = useRef(false); const handleDragPointerDown = useCallback((e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY }; @@ -274,6 +275,7 @@ const AgentCard: React.FC = ({ const handleResizeDown = useCallback( (dir: ResizeDir) => (e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); const effectiveW = Math.max(cardWidth, MIN_W); diff --git a/frontend/src/app/pages/Dashboard/BrowserCard.tsx b/frontend/src/app/pages/Dashboard/BrowserCard.tsx index bf5eb8f8..58c2e47d 100644 --- a/frontend/src/app/pages/Dashboard/BrowserCard.tsx +++ b/frontend/src/app/pages/Dashboard/BrowserCard.tsx @@ -376,6 +376,7 @@ const BrowserCard: React.FC = ({ const justDraggedRef = useRef(false); const handleDragPointerDown = useCallback((e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY }; @@ -431,6 +432,7 @@ const BrowserCard: React.FC = ({ const handleResizeDown = useCallback( (dir: ResizeDir) => (e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); resizeRef.current = { diff --git a/frontend/src/app/pages/Dashboard/Dashboard.tsx b/frontend/src/app/pages/Dashboard/Dashboard.tsx index d25093a4..79c62974 100644 --- a/frontend/src/app/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/app/pages/Dashboard/Dashboard.tsx @@ -13,6 +13,7 @@ import { generateTitle, resumeSession, setExpandedSessionIds, + toggleExpandSession, } from '@/shared/state/agentsSlice'; import type { AgentConfig } from '@/shared/state/agentsSlice'; import { @@ -159,11 +160,17 @@ const DashboardInner: React.FC = () => { canvas.handlers.onMouseDown(e); return; } + + if (e.button === 2) { + e.preventDefault(); + selection.handleCanvasMouseDown(e.nativeEvent); + return; + } + if (e.button !== 0) return; if (isCardTarget(e.target, e.currentTarget)) return; if (isElementSelectMode) { - // Cmd/Ctrl held → allow panning even in element select mode if (e.metaKey || e.ctrlKey) { canvas.handlers.onMouseDown(e); } @@ -348,6 +355,21 @@ const DashboardInner: React.FC = () => { return () => window.removeEventListener('keydown', handleShortcut); }, [newAgentShortcut]); + useEffect(() => { + const handleEnter = (e: KeyboardEvent) => { + if (e.key !== 'Enter') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + if (selection.selectedIds.size !== 1) return; + const [id, type] = selection.selectedIds.entries().next().value!; + if (type !== 'agent') return; + e.preventDefault(); + dispatch(toggleExpandSession(id)); + }; + window.addEventListener('keydown', handleEnter); + return () => window.removeEventListener('keydown', handleEnter); + }, [selection.selectedIds, dispatch]); + const handleNewAgent = useCallback(() => { setToolbarOpen(true); }, []); @@ -508,6 +530,7 @@ const DashboardInner: React.FC = () => { onMouseDown={handleViewportMouseDown} onMouseMove={handleViewportMouseMove} onMouseUp={handleViewportMouseUp} + onContextMenu={(e) => e.preventDefault()} sx={{ position: 'absolute', inset: 0, diff --git a/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx b/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx index 7b6437c2..135214ee 100644 --- a/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardViewCard.tsx @@ -80,6 +80,7 @@ const DashboardViewCard: React.FC = ({ const justDraggedRef = useRef(false); const handleDragPointerDown = useCallback((e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY }; @@ -135,6 +136,7 @@ const DashboardViewCard: React.FC = ({ const handleResizeDown = useCallback( (dir: ResizeDir) => (e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); resizeRef.current = { diff --git a/frontend/src/app/pages/Dashboard/useDashboardSelection.ts b/frontend/src/app/pages/Dashboard/useDashboardSelection.ts index 02640950..aa84f6e2 100644 --- a/frontend/src/app/pages/Dashboard/useDashboardSelection.ts +++ b/frontend/src/app/pages/Dashboard/useDashboardSelection.ts @@ -155,7 +155,7 @@ export function useDashboardSelection( const handleCanvasMouseDown = useCallback( (e: MouseEvent) => { - if (e.button !== 0) return; + if (e.button !== 0 && e.button !== 2) return; marqueeOriginRef.current = { screenX: e.clientX, screenY: e.clientY }; isDraggingMarqueeRef.current = false; From 78552982d83b2c0a1831b46d44883da2239bb14c Mon Sep 17 00:00:00 2001 From: haikdc Date: Wed, 18 Mar 2026 04:09:39 -0700 Subject: [PATCH 06/11] [Haik]: ckpt (input focus persistance in select mode) (can set default spawn in state of a card from expanded to collapsed in settings) (Settings update notification is more in ur face now - to be tested in prod) --- backend/apps/settings/models.py | 1 + backend/apps/tools_lib/tools_lib.py | 2 + .../src/app/components/Layout/AppShell.tsx | 191 +++++++++++++++--- .../app/components/useDomElementSelector.ts | 15 +- .../src/app/pages/AgentChat/ChatInput.tsx | 1 + .../src/app/pages/Dashboard/Dashboard.tsx | 5 +- frontend/src/app/pages/Settings/Settings.tsx | 17 +- frontend/src/shared/state/settingsSlice.ts | 2 + 8 files changed, 206 insertions(+), 28 deletions(-) diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index f2ef9988..14d633d4 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -22,3 +22,4 @@ class AppSettings(BaseModel): anthropic_api_key: Optional[str] = None browser_homepage: str = "https://www.google.com" auto_select_mode_on_new_agent: bool = False + expand_new_chats_in_dashboard: bool = False diff --git a/backend/apps/tools_lib/tools_lib.py b/backend/apps/tools_lib/tools_lib.py index 274f077c..2d7c8bba 100644 --- a/backend/apps/tools_lib/tools_lib.py +++ b/backend/apps/tools_lib/tools_lib.py @@ -382,6 +382,7 @@ def derive_mcp_config(tool: ToolDefinition) -> Optional[dict]: config["command"] = resolved env = config.setdefault("env", {}) env.setdefault("PATH", _augmented_path()) + env.setdefault("PYTHONPATH", "") return config @@ -548,6 +549,7 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, raise HTTPException(status_code=400, detail=f"Command '{command}' not found on PATH or common install locations") proc_env = {**os.environ, **(env or {}), "PATH": _augmented_path()} + proc_env.pop("PYTHONPATH", None) proc = await asyncio.create_subprocess_exec( cmd_path, *(args or []), diff --git a/frontend/src/app/components/Layout/AppShell.tsx b/frontend/src/app/components/Layout/AppShell.tsx index 2d4edc02..2e1ccb79 100644 --- a/frontend/src/app/components/Layout/AppShell.tsx +++ b/frontend/src/app/components/Layout/AppShell.tsx @@ -27,6 +27,9 @@ import ViewSidebarOutlinedIcon from '@mui/icons-material/ViewSidebarOutlined'; import ArrowBackOutlinedIcon from '@mui/icons-material/ArrowBackOutlined'; import ArrowForwardOutlinedIcon from '@mui/icons-material/ArrowForwardOutlined'; import RestartAltIcon from '@mui/icons-material/RestartAlt'; +import SystemUpdateAltIcon from '@mui/icons-material/SystemUpdateAlt'; +import CloseIcon from '@mui/icons-material/Close'; +import LinearProgress from '@mui/material/LinearProgress'; import Settings from '@/app/pages/Settings/Settings'; import GlobalApprovalOverlay from '@/app/components/GlobalApprovalOverlay'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; @@ -38,6 +41,7 @@ const SIDEBAR_MIN = 160; const SIDEBAR_MAX = 400; const SIDEBAR_DEFAULT = 220; const SIDEBAR_WIDTH_KEY = 'openswarm-sidebar-width'; +const UPDATE_DISMISS_KEY = 'openswarm-update-dismissed'; const CUSTOMIZATION_ITEMS = [ { label: 'Prompts', path: '/templates', icon: }, @@ -75,10 +79,34 @@ const AppShell: React.FC = () => { const updateStatus = useAppSelector((state) => state.update.status); const availableVersion = useAppSelector((state) => state.update.availableVersion); - const [updateBannerDismissed, setUpdateBannerDismissed] = useState(false); + const downloadPercent = useAppSelector((state) => state.update.downloadPercent); - const showUpdateDot = updateStatus === 'available' || updateStatus === 'downloaded'; - const showUpdateBanner = updateStatus === 'downloaded' && !updateBannerDismissed; + const [dismissedVersion, setDismissedVersion] = useState(() => { + try { return localStorage.getItem(UPDATE_DISMISS_KEY); } catch { return null; } + }); + const [snackbarDismissed, setSnackbarDismissed] = useState(false); + + const bannerDismissedForVersion = availableVersion != null && dismissedVersion === availableVersion; + const isUpdateActionable = updateStatus === 'available' || updateStatus === 'downloaded' || updateStatus === 'downloading'; + + const showUpdateDot = (updateStatus === 'available' || updateStatus === 'downloaded') && !bannerDismissedForVersion; + const showUpdateBanner = isUpdateActionable && !bannerDismissedForVersion; + const showUpdateSnackbar = (updateStatus === 'available' || updateStatus === 'downloaded') && !bannerDismissedForVersion && !snackbarDismissed; + + const handleDismissBanner = useCallback(() => { + if (availableVersion) { + try { localStorage.setItem(UPDATE_DISMISS_KEY, availableVersion); } catch {} + setDismissedVersion(availableVersion); + } + }, [availableVersion]); + + const handleDownloadUpdate = useCallback(async () => { + try { await (window as any).openswarm?.downloadUpdate(); } catch {} + }, []); + + const handleInstallUpdate = useCallback(() => { + (window as any).openswarm?.installUpdate(); + }, []); const dashboardItems = useAppSelector((state) => state.dashboards.items); const dashboardList = Object.values(dashboardItems).sort( @@ -279,6 +307,98 @@ const AppShell: React.FC = () => { + {showUpdateBanner && ( + + + + {updateStatus === 'available' && `OpenSwarm ${availableVersion} is available`} + {updateStatus === 'downloading' && `Downloading OpenSwarm ${availableVersion}…`} + {updateStatus === 'downloaded' && `OpenSwarm ${availableVersion} is ready to install`} + + {updateStatus === 'downloading' && ( + + )} + {updateStatus === 'downloading' && ( + + {Math.round(downloadPercent)}% + + )} + {updateStatus === 'available' && ( + + )} + {updateStatus === 'downloaded' && ( + + )} + + + + + )} + {!sidebarCollapsed && ( <> @@ -735,36 +855,60 @@ const AppShell: React.FC = () => { setSnackbarDismissed(true)} anchorOrigin={{ vertical: 'bottom', horizontal: 'center' }} > } + icon={updateStatus === 'downloaded' + ? + : + } action={ - + {updateStatus === 'available' && ( + + )} + {updateStatus === 'downloaded' && ( + + )} } sx={{ @@ -775,7 +919,8 @@ const AppShell: React.FC = () => { '& .MuiAlert-icon': { color: c.accent.primary }, }} > - OpenSwarm {availableVersion} downloaded — restart to update + {updateStatus === 'available' && `OpenSwarm ${availableVersion} is available`} + {updateStatus === 'downloaded' && `OpenSwarm ${availableVersion} downloaded — restart to update`} diff --git a/frontend/src/app/components/useDomElementSelector.ts b/frontend/src/app/components/useDomElementSelector.ts index b4535e19..8115769b 100644 --- a/frontend/src/app/components/useDomElementSelector.ts +++ b/frontend/src/app/components/useDomElementSelector.ts @@ -120,6 +120,7 @@ export function useDomElementSelector(): DomSelectorState { const dragOriginRef = useRef<{ x: number; y: number } | null>(null); const isDraggingRef = useRef(false); const dragBoundsRef = useRef<{ left: number; top: number; right: number; bottom: number } | null>(null); + const preDragFocusRef = useRef(null); const excludeIdRef = useRef(null); useEffect(() => { @@ -245,8 +246,11 @@ export function useDomElementSelector(): DomSelectorState { if (e.button !== 0) return; if (e.metaKey || e.ctrlKey) return; const target = e.target as Element; - // Only start drag on "empty" canvas areas (not on selectable elements) - if (target && findSelectableAncestor(target, excludeIdRef.current)) return; + if (target && findSelectableAncestor(target, excludeIdRef.current)) { + e.preventDefault(); + return; + } + preDragFocusRef.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; dragOriginRef.current = { x: e.clientX, y: e.clientY }; isDraggingRef.current = false; }, []); @@ -291,12 +295,17 @@ export function useDomElementSelector(): DomSelectorState { }); } + const wasDragging = isDraggingRef.current; dragOriginRef.current = null; isDraggingRef.current = false; dragBoundsRef.current = null; setDragRect(EMPTY_DRAG); setDragPreview([]); if (dragPreviewRafRef.current) cancelAnimationFrame(dragPreviewRafRef.current); + if (wasDragging && preDragFocusRef.current) { + preDragFocusRef.current.focus(); + } + preDragFocusRef.current = null; }, [ctx]); const handleClick = useCallback((e: MouseEvent) => { @@ -327,6 +336,7 @@ export function useDomElementSelector(): DomSelectorState { dragOriginRef.current = null; dragBoundsRef.current = null; isDraggingRef.current = false; + preDragFocusRef.current = null; return; } @@ -353,6 +363,7 @@ export function useDomElementSelector(): DomSelectorState { dragOriginRef.current = null; dragBoundsRef.current = null; isDraggingRef.current = false; + preDragFocusRef.current = null; }; }, [ctx?.selectMode, handleMouseMove, handleMouseDown, handleMouseUp, handleClick]); diff --git a/frontend/src/app/pages/AgentChat/ChatInput.tsx b/frontend/src/app/pages/AgentChat/ChatInput.tsx index ff028b5d..525bd254 100644 --- a/frontend/src/app/pages/AgentChat/ChatInput.tsx +++ b/frontend/src/app/pages/AgentChat/ChatInput.tsx @@ -981,6 +981,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, e.preventDefault()} onClick={() => { if (!elementSelection.selectMode && sessionId) { elementSelection.setExcludeSelectId(sessionId); diff --git a/frontend/src/app/pages/Dashboard/Dashboard.tsx b/frontend/src/app/pages/Dashboard/Dashboard.tsx index 79c62974..9d597b05 100644 --- a/frontend/src/app/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/app/pages/Dashboard/Dashboard.tsx @@ -84,6 +84,7 @@ const DashboardInner: React.FC = () => { const zoomSensitivity = useAppSelector((state) => state.settings.data.zoom_sensitivity); const newAgentShortcut = useAppSelector((state) => state.settings.data.new_agent_shortcut); const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage); + const expandNewChats = useAppSelector((state) => state.settings.data.expand_new_chats_in_dashboard); const outputs = useAppSelector((state) => state.outputs.items); const sessionList = Object.values(sessions); @@ -420,7 +421,7 @@ const DashboardInner: React.FC = () => { contextPaths: contextPaths?.map((cp) => ({ path: cp.path, type: cp.type })), forcedTools, attachedSkills, - expand: false, + expand: expandNewChats, }), ).then((action) => { if (launchAndSendFirstMessage.fulfilled.match(action)) { @@ -452,7 +453,7 @@ const DashboardInner: React.FC = () => { } }); }, - [canvas.viewportRef, dispatch, dashboardId], + [canvas.viewportRef, dispatch, dashboardId, expandNewChats], ); const handleAddView = useCallback((outputId: string) => { diff --git a/frontend/src/app/pages/Settings/Settings.tsx b/frontend/src/app/pages/Settings/Settings.tsx index 58d71b10..bae49d38 100644 --- a/frontend/src/app/pages/Settings/Settings.tsx +++ b/frontend/src/app/pages/Settings/Settings.tsx @@ -554,7 +554,7 @@ const Settings: React.FC = () => { - + Auto-enable element selection Automatically enter element selection mode when creating a new agent. @@ -569,6 +569,21 @@ const Settings: React.FC = () => { /> + + + Default agent spawn state in dashboard + When enabled, new agents spawn expanded instead of collapsed. + + setForm({ ...form, expand_new_chats_in_dashboard: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {/* ── Browser ── */} Browser diff --git a/frontend/src/shared/state/settingsSlice.ts b/frontend/src/shared/state/settingsSlice.ts index 4530871c..0d7acce8 100644 --- a/frontend/src/shared/state/settingsSlice.ts +++ b/frontend/src/shared/state/settingsSlice.ts @@ -23,6 +23,7 @@ export interface AppSettings { anthropic_api_key: string | null; browser_homepage: string; auto_select_mode_on_new_agent: boolean; + expand_new_chats_in_dashboard: boolean; } export interface BrowseResult { @@ -52,6 +53,7 @@ const initialState: SettingsState = { anthropic_api_key: null, browser_homepage: 'https://www.google.com', auto_select_mode_on_new_agent: false, + expand_new_chats_in_dashboard: false, }, loading: false, loaded: false, From a6a2c68115cb4ad3458e42a0dd34a20f80b551d6 Mon Sep 17 00:00:00 2001 From: haikdc Date: Wed, 18 Mar 2026 07:40:02 -0700 Subject: [PATCH 07/11] [Haik]: ckpt, (view sidebar clicks fixed) (view back button removed) (backspace/delete to remove items from dash) (copy/paste items in dash) (Swapping between branches in chat) (Per message tool tips in chat) (chat branching support and some cool ahhh anims) (auto deselect items on toolbar collapse) (if exit selection mode, items get deselected) (item selections only populate focused chat input as context instead of all chat inputs) (swapped right and left mouse clicks to match that of other softwares) --- backend/apps/agents/agent_manager.py | 118 ++++++- backend/apps/agents/agents.py | 11 + electron/package.json | 2 +- .../components/ElementSelectionContext.tsx | 80 ++++- .../src/app/pages/AgentChat/AgentChat.tsx | 162 ++++++++-- .../app/pages/AgentChat/BranchNavigator.tsx | 51 +-- .../src/app/pages/AgentChat/ChatInput.tsx | 132 +++++--- .../app/pages/AgentChat/MessageActionBar.tsx | 153 +++++++++ .../src/app/pages/AgentChat/MessageBubble.tsx | 41 +-- .../src/app/pages/Dashboard/AgentCard.tsx | 185 +++++++++-- .../src/app/pages/Dashboard/Dashboard.tsx | 295 +++++++++++++++++- .../app/pages/Dashboard/DashboardToolbar.tsx | 57 +++- frontend/src/app/pages/Views/ViewEditor.tsx | 26 -- frontend/src/app/pages/Views/Views.tsx | 5 +- frontend/src/shared/dashboardClipboard.ts | 34 ++ frontend/src/shared/state/agentsSlice.ts | 18 ++ .../src/shared/state/dashboardLayoutSlice.ts | 97 +++++- 17 files changed, 1245 insertions(+), 222 deletions(-) create mode 100644 frontend/src/app/pages/AgentChat/MessageActionBar.tsx create mode 100644 frontend/src/shared/dashboardClipboard.ts diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index 0d916495..ce548740 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -626,7 +626,7 @@ class AgentManager: if elapsed_ms is not None: result_payload["elapsed_ms"] = elapsed_ms - result_msg = Message(role="tool_result", content=result_payload) + result_msg = Message(role="tool_result", content=result_payload, branch_id=session.active_branch_id) session.messages.append(result_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -836,6 +836,7 @@ class AgentManager: id=stream_text_msg_id or uuid4().hex, role="assistant", content="\n".join(content_parts), + branch_id=session.active_branch_id, ) session.messages.append(asst_msg) await ws_manager.send_to_session(session_id, "agent:message", { @@ -845,7 +846,7 @@ class AgentManager: for i, tu in enumerate(tool_uses): msg_id = stream_tool_msg_ids_ordered[i] if i < len(stream_tool_msg_ids_ordered) else uuid4().hex - tool_msg = Message(id=msg_id, role="tool_call", content=tu) + tool_msg = Message(id=msg_id, role="tool_call", content=tu, branch_id=session.active_branch_id) session.messages.append(tool_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -872,7 +873,7 @@ class AgentManager: except Exception as e: logger.exception(f"Agent {session_id} error: {e}") session.status = "error" - error_msg = Message(role="system", content=f"Error: {str(e)}") + error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id) session.messages.append(error_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -973,7 +974,7 @@ class AgentManager: session_id, tool_msg_id, "Bash", _json.dumps(tool_input_content["input"], indent=2), ) - tool_msg = Message(id=tool_msg_id, role="tool_call", content=tool_input_content) + tool_msg = Message(id=tool_msg_id, role="tool_call", content=tool_input_content, branch_id=session.active_branch_id) session.messages.append(tool_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -983,7 +984,7 @@ class AgentManager: await asyncio.sleep(1) if decision.get("behavior") == "allow": - tool_result = Message(role="tool_result", content=f"Processing: {prompt}") + tool_result = Message(role="tool_result", content=f"Processing: {prompt}", branch_id=session.active_branch_id) session.messages.append(tool_result) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -1001,7 +1002,7 @@ class AgentManager: asst_msg_id = uuid4().hex await self._stream_text(session_id, asst_msg_id, asst_text) - asst_msg = Message(id=asst_msg_id, role="assistant", content=asst_text) + asst_msg = Message(id=asst_msg_id, role="assistant", content=asst_text, branch_id=session.active_branch_id) session.messages.append(asst_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -1061,6 +1062,7 @@ class AgentManager: user_msg = Message( role="user", content=prompt, + branch_id=session.active_branch_id, context_paths=context_paths if context_paths else None, attached_skills=skill_meta, forced_tools=forced_tools if forced_tools else None, @@ -1104,6 +1106,14 @@ class AgentManager: if not session: raise ValueError(f"Session {session_id} not found") + existing = self.tasks.get(session_id) + if existing and not existing.done(): + existing.cancel() + try: + await existing + except asyncio.CancelledError: + pass + target_msg = None for i, msg in enumerate(session.messages): if msg.id == message_id: @@ -1113,11 +1123,24 @@ class AgentManager: if not target_msg or target_msg.role != "user": raise ValueError("Can only edit user messages") - new_branch_id = uuid4().hex[:8] + fork_point_id = message_id + fork_parent_branch = target_msg.branch_id + + msg_branch = session.branches.get(target_msg.branch_id) + if msg_branch and msg_branch.fork_point_message_id: + branch_user_msgs = [ + m for m in session.messages + if m.branch_id == target_msg.branch_id and m.role == "user" + ] + if branch_user_msgs and branch_user_msgs[0].id == message_id: + fork_point_id = msg_branch.fork_point_message_id + fork_parent_branch = msg_branch.parent_branch_id or "main" + + new_branch_id = uuid4().hex new_branch = MessageBranch( id=new_branch_id, - parent_branch_id=target_msg.branch_id, - fork_point_message_id=message_id, + parent_branch_id=fork_parent_branch, + fork_point_message_id=fork_point_id, ) session.branches[new_branch_id] = new_branch session.active_branch_id = new_branch_id @@ -1127,6 +1150,10 @@ class AgentManager: content=new_content, branch_id=new_branch_id, parent_id=target_msg.parent_id, + images=target_msg.images, + context_paths=target_msg.context_paths, + forced_tools=target_msg.forced_tools, + attached_skills=target_msg.attached_skills, ) session.messages.append(edited_msg) @@ -1140,7 +1167,14 @@ class AgentManager: "active_branch_id": new_branch_id, }) - task = asyncio.create_task(self._run_agent_loop(session_id, new_content)) + session.sdk_session_id = None + task = asyncio.create_task(self._run_agent_loop( + session_id, new_content, + images=target_msg.images, + context_paths=target_msg.context_paths, + forced_tools=target_msg.forced_tools, + attached_skills=target_msg.attached_skills, + )) self.tasks[session_id] = task async def switch_branch(self, session_id: str, branch_id: str): @@ -1454,6 +1488,70 @@ class AgentManager: _delete_session_file(sid) logger.info(f"Restored session {session.id}") + async def duplicate_session(self, session_id: str, dashboard_id: str | None = None) -> AgentSession: + """Create an independent copy of a session with the same chat history.""" + source = self.sessions.get(session_id) + if not source: + data = _load_session_data(session_id) + if data is None: + raise ValueError(f"Session {session_id} not found") + source = AgentSession(**data) + + old_to_new_msg: dict[str, str] = {} + new_messages: list[Message] = [] + for msg in source.messages: + new_id = uuid4().hex + old_to_new_msg[msg.id] = new_id + new_messages.append(Message( + id=new_id, + role=msg.role, + content=msg.content, + timestamp=msg.timestamp, + branch_id=msg.branch_id, + parent_id=old_to_new_msg.get(msg.parent_id) if msg.parent_id else None, + context_paths=msg.context_paths, + attached_skills=msg.attached_skills, + forced_tools=msg.forced_tools, + images=msg.images, + )) + + new_branches: dict[str, MessageBranch] = {} + for bid, branch in source.branches.items(): + new_branches[bid] = MessageBranch( + id=bid, + parent_branch_id=branch.parent_branch_id, + fork_point_message_id=old_to_new_msg.get(branch.fork_point_message_id) if branch.fork_point_message_id else None, + created_at=branch.created_at, + ) + + new_session = AgentSession( + id=uuid4().hex, + name=f"{source.name} (copy)", + status="stopped", + model=source.model, + mode=source.mode, + system_prompt=source.system_prompt, + allowed_tools=list(source.allowed_tools), + max_turns=source.max_turns, + cwd=source.cwd, + created_at=datetime.now(), + messages=new_messages, + branches=new_branches, + active_branch_id=source.active_branch_id, + tool_group_meta=dict(source.tool_group_meta), + dashboard_id=dashboard_id or source.dashboard_id, + ) + + self.sessions[new_session.id] = new_session + + await ws_manager.send_to_session(new_session.id, "agent:status", { + "session_id": new_session.id, + "status": new_session.status, + "session": new_session.model_dump(mode="json"), + }) + + return new_session + def get_all_sessions(self, dashboard_id: str | None = None) -> list[AgentSession]: if dashboard_id: return [s for s in self.sessions.values() if s.dashboard_id == dashboard_id] diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index c4cd2df1..8bf33a2b 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -131,6 +131,17 @@ async def get_branches(session_id: str): "active_branch_id": session.active_branch_id, } +@agents.router.post("/sessions/{session_id}/duplicate") +async def duplicate_session(session_id: str, body: dict = {}): + try: + session = await agent_manager.duplicate_session( + session_id, + dashboard_id=body.get("dashboard_id"), + ) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + return {"session": session.model_dump(mode="json")} + @agents.router.post("/sessions/{session_id}/close") async def close_session(session_id: str): try: diff --git a/electron/package.json b/electron/package.json index 3be7cad4..61ca2ec8 100644 --- a/electron/package.json +++ b/electron/package.json @@ -1,6 +1,6 @@ { "name": "openswarm", - "version": "1.0.6", + "version": "1.0.7", "description": "OpenSwarm — AI Agent Orchestrator", "main": "main.js", "scripts": { diff --git a/frontend/src/app/components/ElementSelectionContext.tsx b/frontend/src/app/components/ElementSelectionContext.tsx index d42e5d1f..5db25ba1 100644 --- a/frontend/src/app/components/ElementSelectionContext.tsx +++ b/frontend/src/app/components/ElementSelectionContext.tsx @@ -1,4 +1,4 @@ -import React, { createContext, useContext, useState, useRef, useCallback, RefObject } from 'react'; +import React, { createContext, useContext, useState, useRef, useCallback, useMemo, RefObject } from 'react'; export interface SelectedElement { id: string; @@ -20,11 +20,17 @@ interface ElementSelectionContextValue { setSelectMode: (active: boolean) => void; excludeSelectId: string | null; setExcludeSelectId: (id: string | null) => void; + activeOwnerId: string | null; + setActiveOwnerId: (id: string | null) => void; selectedElements: SelectedElement[]; addSelectedElement: (el: SelectedElement) => void; updateSelectedElement: (id: string, patch: Partial) => void; removeSelectedElement: (id: string) => void; clearSelectedElements: () => void; + elementsByOwner: Record; + addElementForOwner: (ownerId: string, el: SelectedElement) => void; + removeOwnerElement: (ownerId: string, elementId: string) => void; + clearOwnerElements: (ownerId: string) => void; iframeRef: RefObject; } @@ -37,9 +43,18 @@ export function useElementSelection() { export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { const [selectMode, setSelectMode] = useState(false); const [excludeSelectId, setExcludeSelectId] = useState(null); - const [selectedElements, setSelectedElements] = useState([]); + const [activeOwnerId, setActiveOwnerId] = useState(null); + const [elementsByOwner, setElementsByOwner] = useState>({}); const iframeRef = useRef(null); + const activeOwnerIdRef = useRef(activeOwnerId); + activeOwnerIdRef.current = activeOwnerId; + + const selectedElements = useMemo( + () => (activeOwnerId ? elementsByOwner[activeOwnerId] ?? [] : []), + [activeOwnerId, elementsByOwner], + ); + const toggleSelectMode = useCallback(() => { setSelectMode((prev) => { if (prev) setExcludeSelectId(null); @@ -48,22 +63,65 @@ export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = }, []); const addSelectedElement = useCallback((el: SelectedElement) => { - setSelectedElements((prev) => { - if (prev.some((e) => e.id === el.id)) return prev; - return [...prev, el]; + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + const existing = prev[ownerId] ?? []; + if (existing.some((e) => e.id === el.id)) return prev; + return { ...prev, [ownerId]: [...existing, el] }; }); }, []); const updateSelectedElement = useCallback((id: string, patch: Partial) => { - setSelectedElements((prev) => prev.map((e) => e.id === id ? { ...e, ...patch } : e)); + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + const existing = prev[ownerId]; + if (!existing) return prev; + return { ...prev, [ownerId]: existing.map((e) => (e.id === id ? { ...e, ...patch } : e)) }; + }); }, []); const removeSelectedElement = useCallback((id: string) => { - setSelectedElements((prev) => prev.filter((e) => e.id !== id)); + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + const existing = prev[ownerId]; + if (!existing) return prev; + return { ...prev, [ownerId]: existing.filter((e) => e.id !== id) }; + }); }, []); const clearSelectedElements = useCallback(() => { - setSelectedElements([]); + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + if (!prev[ownerId]?.length) return prev; + return { ...prev, [ownerId]: [] }; + }); + }, []); + + const addElementForOwner = useCallback((ownerId: string, el: SelectedElement) => { + setElementsByOwner((prev) => { + const existing = prev[ownerId] ?? []; + if (existing.some((e) => e.semanticData?.selectId === el.semanticData?.selectId)) return prev; + return { ...prev, [ownerId]: [...existing, el] }; + }); + }, []); + + const removeOwnerElement = useCallback((ownerId: string, elementId: string) => { + setElementsByOwner((prev) => { + const existing = prev[ownerId]; + if (!existing) return prev; + return { ...prev, [ownerId]: existing.filter((e) => e.id !== elementId) }; + }); + }, []); + + const clearOwnerElements = useCallback((ownerId: string) => { + setElementsByOwner((prev) => { + if (!prev[ownerId]?.length) return prev; + return { ...prev, [ownerId]: [] }; + }); }, []); return ( @@ -74,11 +132,17 @@ export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = setSelectMode, excludeSelectId, setExcludeSelectId, + activeOwnerId, + setActiveOwnerId, selectedElements, addSelectedElement, updateSelectedElement, removeSelectedElement, clearSelectedElements, + elementsByOwner, + addElementForOwner, + removeOwnerElement, + clearOwnerElements, iframeRef, }} > diff --git a/frontend/src/app/pages/AgentChat/AgentChat.tsx b/frontend/src/app/pages/AgentChat/AgentChat.tsx index e10d80ca..0f81a75a 100644 --- a/frontend/src/app/pages/AgentChat/AgentChat.tsx +++ b/frontend/src/app/pages/AgentChat/AgentChat.tsx @@ -17,6 +17,8 @@ import { handleApproval, editMessage, switchBranch, + duplicateSession, + setActiveSession, updateSessionModel, updateSessionMode, fetchSession, @@ -25,12 +27,12 @@ import { import { fetchModes } from '@/shared/state/modesSlice'; import { createSessionWs } from '@/shared/ws/WebSocketManager'; import MessageBubble from './MessageBubble'; +import MessageActionBar from './MessageActionBar'; import ToolCallBubble, { ToolPair } from './ToolCallBubble'; import ToolGroupBubble, { RenderItem, ToolGroup, isToolGroup, isToolPair } from './ToolGroupBubble'; import ApprovalBar, { BatchApprovalBar } from './ApprovalBar'; import ChatInput, { ChatInputHandle } from './ChatInput'; import { ContextPath } from '@/app/components/DirectoryBrowser'; -import BranchNavigator from './BranchNavigator'; import DiffViewer from './DiffViewer'; import { setGlowingBrowserCards, clearGlowingBrowserCards } from '@/shared/state/dashboardLayoutSlice'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; @@ -96,9 +98,10 @@ interface AgentChatProps { onClose?: () => void; embedded?: boolean; initialContextPaths?: ContextPath[]; + onBranch?: (newSessionId: string) => void; } -const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose, embedded, initialContextPaths }) => { +const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose, embedded, initialContextPaths, onBranch }) => { const c = useClaudeTokens(); const STATUS_STYLES: Record = { running: { color: c.status.success, bg: c.status.successBg }, @@ -247,14 +250,21 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose dispatch(stopAgent({ sessionId: id })); }; - const handleEdit = useCallback( + const [editingMessageId, setEditingMessageId] = useState(null); + + const handleSaveEdit = useCallback( (messageId: string, newContent: string) => { if (!id) return; dispatch(editMessage({ sessionId: id, messageId, content: newContent })); + setEditingMessageId(null); }, [id, dispatch] ); + const handleCancelEdit = useCallback(() => { + setEditingMessageId(null); + }, []); + const activeBranchMessages = useMemo(() => { if (!session) return []; const branchId = session.active_branch_id || 'main'; @@ -264,14 +274,74 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose return session.messages.filter((m) => m.branch_id === 'main' || m.branch_id === branchId); } - const forkIdx = session.messages.findIndex((m) => m.id === branch.fork_point_message_id); - const preMessages = session.messages - .slice(0, forkIdx) - .filter((m) => m.branch_id === (branch.parent_branch_id || 'main')); - const branchMessages = session.messages.filter((m) => m.branch_id === branchId); - return [...preMessages, ...branchMessages]; + const segments: Array<{ branchId: string; upToMessageId?: string }> = []; + let cur = branch; + let curId = branchId; + while (cur && cur.fork_point_message_id) { + segments.unshift({ branchId: curId, upToMessageId: cur.fork_point_message_id }); + curId = cur.parent_branch_id || 'main'; + cur = session.branches?.[curId]; + } + segments.unshift({ branchId: curId }); + + const result: typeof session.messages = []; + for (let i = 0; i < segments.length; i++) { + const seg = segments[i]; + const nextForkMsgId = seg.upToMessageId; + if (nextForkMsgId) { + const forkIdx = session.messages.findIndex((m) => m.id === nextForkMsgId); + const pre = session.messages + .slice(0, forkIdx) + .filter((m) => m.branch_id === seg.branchId); + result.push(...pre); + } else if (i < segments.length - 1) { + const nextFork = segments[i + 1].upToMessageId; + const forkIdx = nextFork + ? session.messages.findIndex((m) => m.id === nextFork) + : session.messages.length; + result.push( + ...session.messages.slice(0, forkIdx).filter((m) => m.branch_id === seg.branchId) + ); + } else { + result.push(...session.messages.filter((m) => m.branch_id === seg.branchId)); + } + } + const leafMsgs = session.messages.filter((m) => m.branch_id === branchId); + if (!result.some((m) => m.branch_id === branchId)) { + result.push(...leafMsgs); + } + return result; }, [session?.messages, session?.active_branch_id, session?.branches]); + const handleRegenerate = useCallback( + (assistantMsg: AgentMessage) => { + if (!id) return; + const idx = activeBranchMessages.findIndex((m) => m.id === assistantMsg.id); + for (let i = idx - 1; i >= 0; i--) { + if (activeBranchMessages[i].role === 'user') { + const userMsg = activeBranchMessages[i]; + const content = typeof userMsg.content === 'string' ? userMsg.content : JSON.stringify(userMsg.content); + dispatch(editMessage({ sessionId: id, messageId: userMsg.id, content })); + break; + } + } + }, + [id, activeBranchMessages, dispatch] + ); + + const handleBranchChat = useCallback(async () => { + if (!id) return; + const dashId = session?.dashboard_id; + const action = await dispatch(duplicateSession({ sessionId: id, dashboardId: dashId })); + if (duplicateSession.fulfilled.match(action)) { + if (onBranch) { + onBranch(action.payload.id); + } else { + dispatch(setActiveSession(action.payload.id)); + } + } + }, [id, dispatch, onBranch, session?.dashboard_id]); + const contextEstimate = useMemo(() => { const limit = CONTEXT_WINDOWS[model] || 200_000; let totalChars = 0; @@ -427,11 +497,33 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const getSiblingBranches = useCallback( (messageId: string): string[] => { if (!session?.branches) return []; - return Object.values(session.branches) + + const directForks = Object.values(session.branches) .filter((b) => b.fork_point_message_id === messageId) .map((b) => b.id); + if (directForks.length > 0) { + const originalMsg = session.messages.find((m) => m.id === messageId); + const parentBranchId = originalMsg?.branch_id || 'main'; + return [parentBranchId, ...directForks]; + } + + const msg = session.messages.find((m) => m.id === messageId); + if (!msg || msg.role !== 'user') return []; + const msgBranch = session.branches[msg.branch_id]; + if (!msgBranch?.fork_point_message_id) return []; + const branchUserMsgs = session.messages.filter( + (m) => m.branch_id === msg.branch_id && m.role === 'user' + ); + if (branchUserMsgs.length === 0 || branchUserMsgs[0].id !== messageId) return []; + + const forkPointId = msgBranch.fork_point_message_id; + const siblingBranches = Object.values(session.branches) + .filter((b) => b.fork_point_message_id === forkPointId) + .map((b) => b.id); + const parentBranchId = msgBranch.parent_branch_id || 'main'; + return [parentBranchId, ...siblingBranches]; }, - [session?.branches] + [session?.branches, session?.messages] ); if (!session) { @@ -534,30 +626,48 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose return ; } const msg = item; + const isEditing = editingMessageId === msg.id; const siblings = getSiblingBranches(msg.id); const hasBranches = siblings.length > 0; const currentBranchIdx = hasBranches ? siblings.indexOf(session.active_branch_id || 'main') : 0; + const rawText = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content); return ( - - - {hasBranches && ( - { - const prevBranch = siblings[Math.max(0, currentBranchIdx - 1)]; - if (prevBranch && id) dispatch(switchBranch({ sessionId: id, branchId: prevBranch })); - }} - onNext={() => { - const nextBranch = siblings[Math.min(siblings.length - 1, currentBranchIdx + 1)]; - if (nextBranch && id) dispatch(switchBranch({ sessionId: id, branchId: nextBranch })); - }} + + + {!isEditing && (msg.role === 'user' || msg.role === 'assistant') && ( + navigator.clipboard.writeText(rawText)} + onEdit={msg.role === 'user' ? () => setEditingMessageId(msg.id) : undefined} + onRegenerate={msg.role === 'assistant' ? () => handleRegenerate(msg) : undefined} + onBranch={msg.role === 'assistant' ? handleBranchChat : undefined} + branchNav={ + hasBranches + ? { + currentIndex: Math.max(0, currentBranchIdx), + totalBranches: siblings.length, + onPrevious: () => { + const prevBranch = siblings[Math.max(0, currentBranchIdx - 1)]; + if (prevBranch && id) dispatch(switchBranch({ sessionId: id, branchId: prevBranch })); + }, + onNext: () => { + const nextBranch = siblings[Math.min(siblings.length - 1, currentBranchIdx + 1)]; + if (nextBranch && id) dispatch(switchBranch({ sessionId: id, branchId: nextBranch })); + }, + } + : undefined + } /> )} - + ); })} {session.streamingMessage && ( diff --git a/frontend/src/app/pages/AgentChat/BranchNavigator.tsx b/frontend/src/app/pages/AgentChat/BranchNavigator.tsx index 4dbabb95..e506f8c5 100644 --- a/frontend/src/app/pages/AgentChat/BranchNavigator.tsx +++ b/frontend/src/app/pages/AgentChat/BranchNavigator.tsx @@ -21,31 +21,38 @@ const BranchNavigator: React.FC = ({ currentIndex, totalBranches, onPrevi - - - - - {currentIndex + 1}/{totalBranches} - - - - + + + + + {currentIndex + 1} / {totalBranches} + + + + + ); }; diff --git a/frontend/src/app/pages/AgentChat/ChatInput.tsx b/frontend/src/app/pages/AgentChat/ChatInput.tsx index 525bd254..837bed12 100644 --- a/frontend/src/app/pages/AgentChat/ChatInput.tsx +++ b/frontend/src/app/pages/AgentChat/ChatInput.tsx @@ -26,6 +26,7 @@ import AttachFileIcon from '@mui/icons-material/AttachFile'; import AdsClickIcon from '@mui/icons-material/AdsClick'; import CommandPicker, { CommandPickerItem, getToolGroupIcon } from '@/app/components/CommandPicker'; import { useElementSelection, SelectedElement } from '@/app/components/ElementSelectionContext'; +import { getClipboardCards, clearClipboard } from '@/shared/dashboardClipboard'; import { getWebview } from '@/shared/browserRegistry'; import { API_BASE } from '@/shared/config'; import { ContextPath } from '@/app/components/DirectoryBrowser'; @@ -138,6 +139,9 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, const dispatch = useAppDispatch(); const elementSelection = useElementSelection(); + const fallbackOwnerIdRef = useRef(`input-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`); + const ownerId = sessionId || fallbackOwnerIdRef.current; + useEffect(() => { if (autoFocus) editorRef.current?.focus(); }, [autoFocus]); @@ -281,7 +285,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, let trimmed = serialized.trim(); if (!trimmed) return; - const selectedEls = elementSelection?.selectedElements ?? []; + const selectedEls = elementSelection?.elementsByOwner?.[ownerId] ?? []; let allImages = images.length > 0 ? images.map(({ data, media_type }) => ({ data, media_type })) : []; @@ -359,8 +363,8 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, setForcedTools([]); setAttachedSkills({}); setHasContent(false); - elementSelection?.clearSelectedElements(); - }, [disabled, images, contextPaths, forcedTools, onSend, elementSelection]); + elementSelection?.clearOwnerElements(ownerId); + }, [disabled, images, contextPaths, forcedTools, onSend, elementSelection, ownerId]); const detectTrigger = useCallback(() => { const result = detectEditorTrigger(); @@ -469,6 +473,41 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, }; const handlePaste = useCallback((e: React.ClipboardEvent) => { + const copied = getClipboardCards(); + if (copied.length > 0 && elementSelection) { + e.preventDefault(); + for (const card of copied) { + const semanticTypeMap: Record = { + agent: 'agent-card', + view: 'view-card', + browser: 'browser-card', + }; + const semanticType = semanticTypeMap[card.type]; + if (!semanticType) continue; + const labelMap: Record = { + 'agent-card': 'Agent', + 'view-card': 'View', + 'browser-card': 'Browser', + }; + const semanticLabel = (labelMap[semanticType] || semanticType) + ': ' + card.name; + const el: SelectedElement = { + id: `sel-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + selectorPath: `[data-select-type="${semanticType}"][data-select-id="${card.id}"]`, + tagName: 'DIV', + className: '', + outerHTML: '', + computedStyles: {}, + boundingRect: { x: 0, y: 0, width: 0, height: 0 }, + semanticType, + semanticLabel, + semanticData: { ...card.meta, selectId: card.id }, + }; + elementSelection.addElementForOwner(ownerId, el); + } + clearClipboard(); + return; + } + const items = e.clipboardData?.items; if (!items) return; const imageFiles: File[] = []; @@ -486,7 +525,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, e.preventDefault(); const plain = e.clipboardData.getData('text/plain'); if (plain) document.execCommand('insertText', false, plain); - }, [addImageFiles]); + }, [addImageFiles, elementSelection, ownerId]); const handleDragOver = useCallback((e: React.DragEvent) => { e.preventDefault(); @@ -535,7 +574,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, }, }; - const selectedElements = elementSelection?.selectedElements ?? []; + const selectedElements = elementSelection?.elementsByOwner?.[ownerId] ?? []; const hasAttachments = images.length > 0 || contextPaths.length > 0 || forcedTools.length > 0 || selectedElements.length > 0; return ( @@ -783,7 +822,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, icon={} label={chipLabel} size="small" - onDelete={() => elementSelection?.removeSelectedElement(el.id)} + onDelete={() => elementSelection?.removeOwnerElement(ownerId, el.id)} sx={{ bgcolor: 'rgba(59, 130, 246, 0.1)', color: '#3b82f6', @@ -977,41 +1016,54 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, /> )} - {elementSelection && !autoRunMode && ( - - e.preventDefault()} - onClick={() => { - if (!elementSelection.selectMode && sessionId) { - elementSelection.setExcludeSelectId(sessionId); - } - elementSelection.toggleSelectMode(); - }} - sx={{ - p: 0.5, - ...(elementSelection.selectMode - ? { - bgcolor: '#3b82f6', - color: '#fff', - '&:hover': { bgcolor: '#2563eb' }, - animation: 'selectBtnPulse 2s ease-in-out infinite', - '@keyframes selectBtnPulse': { - '0%, 100%': { boxShadow: '0 0 0 0 rgba(59,130,246,0.4)' }, - '50%': { boxShadow: '0 0 0 4px rgba(59,130,246,0.1)' }, - }, + {elementSelection && !autoRunMode && (() => { + const isMySelectMode = elementSelection.selectMode && elementSelection.activeOwnerId === ownerId; + return ( + + e.preventDefault()} + onClick={() => { + if (isMySelectMode) { + elementSelection.setSelectMode(false); + } else { + if (elementSelection.activeOwnerId !== ownerId) { + elementSelection.clearOwnerElements(ownerId); } - : { - color: c.text.tertiary, - '&:hover': { color: c.text.secondary, bgcolor: 'rgba(0,0,0,0.04)' }, - }), - transition: 'background-color 0.15s, color 0.15s', - }} - > - - - - )} + elementSelection.setActiveOwnerId(ownerId); + if (sessionId) { + elementSelection.setExcludeSelectId(sessionId); + } else { + elementSelection.setExcludeSelectId(null); + } + elementSelection.setSelectMode(true); + } + }} + sx={{ + p: 0.5, + ...(isMySelectMode + ? { + bgcolor: '#3b82f6', + color: '#fff', + '&:hover': { bgcolor: '#2563eb' }, + animation: 'selectBtnPulse 2s ease-in-out infinite', + '@keyframes selectBtnPulse': { + '0%, 100%': { boxShadow: '0 0 0 0 rgba(59,130,246,0.4)' }, + '50%': { boxShadow: '0 0 0 4px rgba(59,130,246,0.1)' }, + }, + } + : { + color: c.text.tertiary, + '&:hover': { color: c.text.secondary, bgcolor: 'rgba(0,0,0,0.04)' }, + }), + transition: 'background-color 0.15s, color 0.15s', + }} + > + + + + ); + })()} void; + onNext: () => void; +} + +interface Props { + role: 'user' | 'assistant'; + onCopy: () => void; + onEdit?: () => void; + onRegenerate?: () => void; + onBranch?: () => void; + branchNav?: BranchNavProps; +} + +const btnSx = (c: ReturnType) => ({ + color: c.text.tertiary, + p: 0.4, + '&:hover': { color: c.text.secondary, bgcolor: 'transparent' }, + '&.Mui-disabled': { color: c.border.medium }, +}); + +const MessageActionBar: React.FC = ({ + role, + onCopy, + onEdit, + onRegenerate, + onBranch, + branchNav, +}) => { + const c = useClaudeTokens(); + const [copied, setCopied] = useState(false); + + const handleCopy = () => { + onCopy(); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + }; + + const isUser = role === 'user'; + + return ( + + {isUser ? ( + <> + + + + + + + + + + {copied ? : } + + + {onEdit && ( + + + + + + )} + {branchNav && branchNav.totalBranches > 1 && ( + + + + + + {branchNav.currentIndex + 1} / {branchNav.totalBranches} + + + + + + )} + + ) : ( + <> + + + {copied ? : } + + + {onRegenerate && ( + + + + + + )} + {onBranch && ( + + + + + + )} + + )} + + ); +}; + +export default MessageActionBar; diff --git a/frontend/src/app/pages/AgentChat/MessageBubble.tsx b/frontend/src/app/pages/AgentChat/MessageBubble.tsx index 5fbd998e..6feef5a6 100644 --- a/frontend/src/app/pages/AgentChat/MessageBubble.tsx +++ b/frontend/src/app/pages/AgentChat/MessageBubble.tsx @@ -8,7 +8,6 @@ import Chip from '@mui/material/Chip'; import Tooltip from '@mui/material/Tooltip'; import Collapse from '@mui/material/Collapse'; import Modal from '@mui/material/Modal'; -import EditIcon from '@mui/icons-material/Edit'; import CloseIcon from '@mui/icons-material/Close'; import AdsClickIcon from '@mui/icons-material/AdsClick'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; @@ -391,13 +390,14 @@ const MessageImageThumbnails: React.FC<{ interface Props { message: AgentMessage; - onEdit?: (messageId: string, newContent: string) => void; + editing?: boolean; + onSaveEdit?: (messageId: string, newContent: string) => void; + onCancelEdit?: () => void; isStreaming?: boolean; } -const MessageBubble: React.FC = React.memo(({ message, onEdit, isStreaming }) => { +const MessageBubble: React.FC = React.memo(({ message, editing = false, onSaveEdit, onCancelEdit, isStreaming }) => { const c = useClaudeTokens(); - const [editing, setEditing] = useState(false); const [editText, setEditText] = useState(''); const { role, content } = message; @@ -440,23 +440,22 @@ const MessageBubble: React.FC = React.memo(({ message, onEdit, isStreamin ? parseElementContext(rawText) : { userMessage: rawText, elements: [] }; - const handleStartEdit = () => { - setEditText(rawText); - setEditing(true); - }; + React.useEffect(() => { + if (editing) setEditText(rawText); + }, [editing, rawText]); const handleCancelEdit = () => { - setEditing(false); setEditText(''); + onCancelEdit?.(); }; const handleSaveEdit = () => { const trimmed = editText.trim(); - if (trimmed && trimmed !== rawText && onEdit) { - onEdit(message.id, trimmed); + if (trimmed && trimmed !== rawText && onSaveEdit) { + onSaveEdit(message.id, trimmed); } - setEditing(false); setEditText(''); + onCancelEdit?.(); }; const truncatedContent = typeof content === 'string' @@ -472,26 +471,8 @@ const MessageBubble: React.FC = React.memo(({ message, onEdit, isStreamin display: 'flex', justifyContent: isUser ? 'flex-end' : 'flex-start', my: 0.75, - '&:hover .edit-btn': { opacity: 1 }, }} > - {isUser && onEdit && !editing && ( - - - - )} void; onDragMove?: (dx: number, dy: number) => void; onDragEnd?: (dx: number, dy: number, didDrag: boolean) => void; + onBranch?: (sourceSessionId: string, newSessionId: string) => void; } const MIN_W = 480; @@ -184,15 +187,39 @@ const MIN_H = 120; const EXPANDED_OVERLAY_H = 620; const SPAWN_SPRING = { type: 'spring' as const, stiffness: 400, damping: 28, mass: 0.6 }; +const BRANCH_SPRING = { type: 'spring' as const, stiffness: 300, damping: 26, mass: 0.8 }; +const GLOW_FADE_MS = 2500; const AgentCard: React.FC = ({ session, expanded, cardX, cardY, cardWidth, cardHeight, zoom = 1, spawnFrom, isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd, + onBranch, }) => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); + // ---- Glow state (for branched cards) ---- + const glowEntry = useAppSelector((s) => s.dashboardLayout.glowingAgentCards[session.id]); + const isGlowingRedux = !!glowEntry; + const glowFading = glowEntry?.fading ?? false; + const glowFadeTimer = useRef | null>(null); + + const dismissGlow = useCallback(() => { + if (!isGlowingRedux || glowFading) return; + dispatch(fadeGlowingAgentCard(session.id)); + glowFadeTimer.current = setTimeout(() => { + dispatch(clearGlowingAgentCard(session.id)); + }, GLOW_FADE_MS + 300); + }, [isGlowingRedux, glowFading, dispatch, session.id]); + + useEffect(() => () => { + if (glowFadeTimer.current) clearTimeout(glowFadeTimer.current); + }, []); + + const accentColor = c.accent.primary; + const accentHover = c.accent.hover; + const STATUS_COLORS: Record = { running: { color: c.status.success, bg: c.status.successBg }, waiting_approval: { color: c.status.warning, bg: c.status.warningBg }, @@ -380,19 +407,23 @@ const AgentCard: React.FC = ({ const activeW = localResize?.w ?? cardWidth; const activeH = localResize?.h ?? cardHeight; + const isBranchSpawn = spawnFrom?.type === 'branch'; + const spawnInitial = spawnFrom + ? isBranchSpawn + ? { opacity: 0.5, scale: 0.92, left: spawnFrom.x, top: spawnFrom.y } + : { opacity: 0, scale: 0.3, left: spawnFrom.x, top: spawnFrom.y } + : false; + const spawnTransition = noTransition || !spawnFrom + ? { duration: 0 } + : isBranchSpawn + ? { left: BRANCH_SPRING, top: BRANCH_SPRING, scale: BRANCH_SPRING, opacity: { duration: 0.25 } } + : { left: SPAWN_SPRING, top: SPAWN_SPRING, scale: SPAWN_SPRING, opacity: { duration: 0.12 } }; + return ( = ({ data-select-type="agent-card" data-select-id={session.id} data-select-meta={JSON.stringify({ name: session.name || session.id, status: session.status, model: session.model, mode: session.mode })} + onPointerDownCapture={dismissGlow} + onWheelCapture={dismissGlow} onClick={(e: React.MouseEvent) => { if (justDraggedRef.current) return; if (!isSelected && !e.shiftKey) { @@ -416,26 +449,34 @@ const AgentCard: React.FC = ({ bgcolor: c.bg.surface, border: isHighlighted ? `2px solid ${c.accent.primary}` - : isSelected - ? '2px solid #3b82f6' - : hasPending && !expanded - ? `1px solid ${c.status.warning}` - : expanded - ? `1px solid ${c.border.strong}` - : `1px solid ${c.border.subtle}`, + : (isGlowingRedux && !glowFading) + ? `2px solid ${accentColor}` + : isSelected + ? '2px solid #3b82f6' + : hasPending && !expanded + ? `1px solid ${c.status.warning}` + : expanded + ? `1px solid ${c.border.strong}` + : `1px solid ${c.border.subtle}`, borderRadius: 3, p: 2, cursor: expanded ? 'default' : 'pointer', - transition: noTransition ? 'none' : c.transition, + transition: noTransition + ? 'none' + : glowFading + ? `border ${GLOW_FADE_MS}ms ease-out, box-shadow ${GLOW_FADE_MS}ms ease-out` + : c.transition, boxShadow: isHighlighted ? `0 0 0 3px ${c.accent.primary}50, 0 0 20px ${c.accent.primary}35, 0 0 40px ${c.accent.primary}15` - : isDragging - ? c.shadow.lg - : isSelected - ? `0 0 0 1px #3b82f6, ${c.shadow.md}` - : expanded - ? c.shadow.md - : c.shadow.sm, + : (isGlowingRedux && !glowFading) + ? `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15` + : isDragging + ? c.shadow.lg + : isSelected + ? `0 0 0 1px #3b82f6, ${c.shadow.md}` + : expanded + ? c.shadow.md + : c.shadow.sm, display: 'flex', flexDirection: 'column', overflow: 'hidden', @@ -460,7 +501,18 @@ const AgentCard: React.FC = ({ }, zIndex: 50, }), - ...(!isHighlighted && !expanded && !isDragging && !isSelected && { + ...(!isHighlighted && isGlowingRedux && !glowFading && { + animation: 'agent-card-glow-pulse 2s ease-in-out infinite', + '@keyframes agent-card-glow-pulse': { + '0%, 100%': { + boxShadow: `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15`, + }, + '50%': { + boxShadow: `0 0 0 3px ${accentColor}60, 0 0 28px ${accentColor}45, 0 0 56px ${accentColor}25`, + }, + }, + }), + ...(!isHighlighted && !(isGlowingRedux && !glowFading) && !expanded && !isDragging && !isSelected && { '&:hover': { boxShadow: c.shadow.md, borderColor: hasPending ? c.status.warning : c.border.strong, @@ -468,6 +520,82 @@ const AgentCard: React.FC = ({ }), }} > + {/* Glow overlays for branched cards */} + {isGlowingRedux && ( + + {/* Rotating conic gradient border */} + + {/* Top edge shimmer */} + + {/* Inner shadow overlay */} + + + )} + {/* Resize handles: 4 edges + 4 corners */} {HANDLE_DEFS.map(({ dir, sx }) => ( = ({ sessionId={session.id} onClose={() => dispatch(collapseSession(session.id))} embedded + onBranch={onBranch ? (newId: string) => onBranch(session.id, newId) : undefined} /> )} diff --git a/frontend/src/app/pages/Dashboard/Dashboard.tsx b/frontend/src/app/pages/Dashboard/Dashboard.tsx index 9d597b05..5bae1868 100644 --- a/frontend/src/app/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/app/pages/Dashboard/Dashboard.tsx @@ -1,4 +1,4 @@ -import React, { useEffect, useCallback, useRef, useState } from 'react'; +import React, { useEffect, useCallback, useRef, useState, useMemo } from 'react'; import { useParams } from 'react-router-dom'; import Box from '@mui/material/Box'; import Typography from '@mui/material/Typography'; @@ -9,6 +9,9 @@ import { fetchSessions, fetchHistory, collapseSession, + closeSession, + duplicateSession, + expandSession, launchAndSendFirstMessage, generateTitle, resumeSession, @@ -27,7 +30,13 @@ import { moveCards, resetLayout, setGlowingBrowserCards, + removeViewCard, + removeBrowserCard, + pasteBrowserCard, + placeCard, + setGlowingAgentCard, EXPANDED_CARD_MIN_H, + GRID_GAP, } from '@/shared/state/dashboardLayoutSlice'; import { fetchOutputs } from '@/shared/state/outputsSlice'; import { generateDashboardName, updateDashboardThumbnail } from '@/shared/state/dashboardsSlice'; @@ -48,6 +57,7 @@ import type { ContextPath } from '@/app/components/DirectoryBrowser'; import { ElementSelectionProvider, useElementSelection } from '@/app/components/ElementSelectionContext'; import { useDomElementSelector } from '@/app/components/useDomElementSelector'; import SelectionOverlay from '@/app/components/SelectionOverlay'; +import { setClipboardCards, getClipboardCards, type ClipboardCard } from '@/shared/dashboardClipboard'; const SELECT_ATTR = 'data-select-type'; @@ -86,6 +96,7 @@ const DashboardInner: React.FC = () => { const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage); const expandNewChats = useAppSelector((state) => state.settings.data.expand_new_chats_in_dashboard); const outputs = useAppSelector((state) => state.outputs.items); + const glowingAgentCards = useAppSelector((state) => state.dashboardLayout.glowingAgentCards); const sessionList = Object.values(sessions); const canvas = useCanvasControls(zoomSensitivity); @@ -110,7 +121,7 @@ const DashboardInner: React.FC = () => { }, 2000); }, []); - const spawnOriginsRef = useRef>({}); + const spawnOriginsRef = useRef>({}); const hasFittedRef = useRef(false); const restoredExpandedRef = useRef(false); const canvasStateRef = useRef({ panX: canvas.panX, panY: canvas.panY, zoom: canvas.zoom }); @@ -118,16 +129,16 @@ const DashboardInner: React.FC = () => { // ---- Multi-drag coordination ---- const [multiDragDelta, setMultiDragDelta] = useState<{ dx: number; dy: number } | null>(null); + const [liveDragInfo, setLiveDragInfo] = useState<{ cardId: string; dx: number; dy: number } | null>(null); const activeDragCardRef = useRef(null); const isMultiDragRef = useRef(false); const handleCardDragStart = useCallback((id: string, _type: CardType) => { + activeDragCardRef.current = id; if (selection.isSelected(id)) { - activeDragCardRef.current = id; isMultiDragRef.current = true; } else { selection.deselectAll(); - activeDragCardRef.current = null; isMultiDragRef.current = false; } }, [selection]); @@ -136,6 +147,9 @@ const DashboardInner: React.FC = () => { if (isMultiDragRef.current) { setMultiDragDelta({ dx, dy }); } + if (activeDragCardRef.current) { + setLiveDragInfo({ cardId: activeDragCardRef.current, dx, dy }); + } }, []); const handleCardDragEnd = useCallback((dx: number, dy: number, didDrag: boolean) => { @@ -149,6 +163,7 @@ const DashboardInner: React.FC = () => { activeDragCardRef.current = null; isMultiDragRef.current = false; setMultiDragDelta(null); + setLiveDragInfo(null); }, [selection, dispatch]); const handleCardSelect = useCallback((id: string, type: CardType, shiftKey: boolean) => { @@ -164,7 +179,7 @@ const DashboardInner: React.FC = () => { if (e.button === 2) { e.preventDefault(); - selection.handleCanvasMouseDown(e.nativeEvent); + canvas.handlers.onMouseDown(e); return; } @@ -179,10 +194,10 @@ const DashboardInner: React.FC = () => { } if (e.metaKey || e.ctrlKey || canvas.spaceHeld) { - selection.handleCanvasMouseDown(e.nativeEvent); - } else { selection.deselectAll(); canvas.handlers.onMouseDown(e); + } else { + selection.handleCanvasMouseDown(e.nativeEvent); } }, [canvas.handlers, canvas.spaceHeld, selection, isElementSelectMode]); @@ -371,6 +386,162 @@ const DashboardInner: React.FC = () => { return () => window.removeEventListener('keydown', handleEnter); }, [selection.selectedIds, dispatch]); + useEffect(() => { + const handleDelete = (e: KeyboardEvent) => { + if (e.key !== 'Backspace' && e.key !== 'Delete') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + if (selection.selectedIds.size === 0) return; + e.preventDefault(); + for (const [id, type] of selection.selectedIds) { + if (type === 'agent') { + dispatch(closeSession({ sessionId: id })); + } else if (type === 'view') { + dispatch(removeViewCard(id)); + } else if (type === 'browser') { + dispatch(removeBrowserCard(id)); + } + } + selection.deselectAll(); + }; + window.addEventListener('keydown', handleDelete); + return () => window.removeEventListener('keydown', handleDelete); + }, [selection, dispatch]); + + useEffect(() => { + const handleCopy = (e: KeyboardEvent) => { + if (!(e.metaKey || e.ctrlKey) || e.key.toLowerCase() !== 'c') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + if (selection.selectedIds.size === 0) return; + + e.preventDefault(); + const copied: ClipboardCard[] = []; + const names: string[] = []; + for (const [id, type] of selection.selectedIds) { + if (type === 'agent') { + const session = sessions[id]; + const card = cards[id]; + if (!session || !card) continue; + copied.push({ + type, id, name: session.name || id, + meta: { name: session.name, status: session.status, model: session.model, mode: session.mode }, + x: card.x, y: card.y, width: card.width, height: card.height, + expanded: expandedSessionIds.includes(id), + }); + names.push(session.name || id); + } else if (type === 'view') { + const output = outputs[id]; + const vc = viewCards[id]; + if (!output || !vc) continue; + copied.push({ + type, id, name: output.name, + meta: { name: output.name, description: output.description }, + x: vc.x, y: vc.y, width: vc.width, height: vc.height, + }); + names.push(output.name); + } else if (type === 'browser') { + const bc = browserCards[id]; + if (!bc) continue; + const activeTab = bc.tabs.find((t) => t.id === bc.activeTabId); + const title = activeTab?.title || 'Browser'; + copied.push({ + type, id, name: title, + meta: { name: title, url: activeTab?.url || bc.url, tabs: bc.tabs }, + x: bc.x, y: bc.y, width: bc.width, height: bc.height, + }); + names.push(title); + } + } + setClipboardCards(copied); + navigator.clipboard.writeText(names.join(', ')).catch(() => {}); + }; + window.addEventListener('keydown', handleCopy); + return () => window.removeEventListener('keydown', handleCopy); + }, [selection.selectedIds, sessions, cards, viewCards, browserCards, outputs, expandedSessionIds]); + + useEffect(() => { + const PASTE_OFFSET = 40; + const handlePaste = async (e: KeyboardEvent) => { + if (!(e.metaKey || e.ctrlKey) || e.key.toLowerCase() !== 'v') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + + const copied = getClipboardCards(); + if (copied.length === 0) return; + e.preventDefault(); + + selection.deselectAll(); + const newSelection = new Map(); + + for (const card of copied) { + const px = card.x + PASTE_OFFSET; + const py = card.y - PASTE_OFFSET; + + if (card.type === 'agent') { + const action = await dispatch(duplicateSession({ sessionId: card.id, dashboardId })); + if (duplicateSession.fulfilled.match(action)) { + const newId = action.payload.id; + dispatch(placeCard({ sessionId: newId, x: px, y: py, width: card.width, height: card.height })); + if (card.expanded) { + dispatch(expandSession(newId)); + } + newSelection.set(newId, 'agent'); + } + } else if (card.type === 'view') { + dispatch(addViewCard({ outputId: card.id, expandedSessionIds, x: px, y: py, width: card.width, height: card.height })); + newSelection.set(card.id, 'view'); + } else if (card.type === 'browser') { + const browserId = `browser-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`; + dispatch(pasteBrowserCard({ + id: browserId, tabs: card.meta.tabs || [], url: card.meta.url || '', + x: px, y: py, width: card.width, height: card.height, + })); + newSelection.set(browserId, 'browser'); + } + } + + if (newSelection.size > 0) { + for (const [id, type] of newSelection) { + selection.selectCard(id, type, true); + } + } + }; + window.addEventListener('keydown', handlePaste); + return () => window.removeEventListener('keydown', handlePaste); + }, [dispatch, dashboardId, expandedSessionIds, selection]); + + const handleBranchFromCard = useCallback( + (sourceSessionId: string, newSessionId: string) => { + const sourceCard = cards[sourceSessionId]; + if (!sourceCard) return; + + const targetX = sourceCard.x + sourceCard.width + GRID_GAP * 3; + const targetY = sourceCard.y; + + spawnOriginsRef.current[newSessionId] = { + x: sourceCard.x, + y: sourceCard.y, + type: 'branch' as const, + }; + + dispatch(placeCard({ + sessionId: newSessionId, + x: targetX, + y: targetY, + width: sourceCard.width, + height: sourceCard.height, + })); + + if (expandedSessionIds.includes(sourceSessionId)) { + dispatch(expandSession(newSessionId)); + } + + dispatch(setGlowingAgentCard({ sessionId: newSessionId, sourceId: sourceSessionId })); + }, + [cards, dispatch, expandedSessionIds], + ); + const handleNewAgent = useCallback(() => { setToolbarOpen(true); }, []); @@ -489,6 +660,37 @@ const DashboardInner: React.FC = () => { canvas.actions.fitToCards(allRects); }, [dispatch, canvas.actions]); + const TETHER_FADE_MS = 2500; + const tethers = useMemo(() => { + return Object.entries(glowingAgentCards).map(([copyId, { sourceId, fading }]) => { + const src = cards[sourceId]; + const dst = cards[copyId]; + if (!src || !dst) return null; + + let srcX = src.x, srcY = src.y; + let dstX = dst.x, dstY = dst.y; + if (liveDragInfo) { + if (liveDragInfo.cardId === sourceId) { srcX += liveDragInfo.dx; srcY += liveDragInfo.dy; } + if (liveDragInfo.cardId === copyId) { dstX += liveDragInfo.dx; dstY += liveDragInfo.dy; } + } + + const srcH = expandedSessionIds.includes(sourceId) + ? Math.max(EXPANDED_CARD_MIN_H, src.height) + : src.height; + const dstH = expandedSessionIds.includes(copyId) + ? Math.max(EXPANDED_CARD_MIN_H, dst.height) + : dst.height; + return { + key: copyId, + x1: srcX + src.width, + y1: srcY + srcH / 2, + x2: dstX, + y2: dstY + dstH / 2, + fading, + }; + }).filter(Boolean) as Array<{ key: string; x1: number; y1: number; x2: number; y2: number; fading: boolean }>; + }, [glowingAgentCards, cards, expandedSessionIds, liveDragInfo]); + const dotSize = Math.max(1, 1.5 * canvas.zoom); const dotSpacing = 24 * canvas.zoom; @@ -539,7 +741,7 @@ const DashboardInner: React.FC = () => { cursor: canvas.isPanning ? 'grabbing' : (canvas.spaceHeld || canvas.cmdHeld) - ? 'crosshair' + ? 'grab' : selection.marquee ? 'crosshair' : 'default', @@ -586,6 +788,82 @@ const DashboardInner: React.FC = () => { position: 'relative', }} > + {/* Tether lines between branched cards */} + {tethers.length > 0 && ( + + + + + + + + + + + + + + + {tethers.map((t) => ( + + + + + + ))} + + )} {Object.values(cards).map((card) => { const session = sessions[card.session_id]; if (!session) return null; @@ -609,6 +887,7 @@ const DashboardInner: React.FC = () => { onDragStart={handleCardDragStart} onDragMove={handleCardDragMove} onDragEnd={handleCardDragEnd} + onBranch={handleBranchFromCard} /> ); })} diff --git a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx index dda3feeb..0cb9eb32 100644 --- a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx @@ -42,6 +42,7 @@ interface Props { dashboardId?: string; } +const TOOLBAR_OWNER_ID = '__toolbar__'; const BTN = 40; const WarmTooltip = styled( @@ -217,11 +218,17 @@ const DashboardToolbar = React.forwardRef( const autoSelectOnNew = useAppSelector((s) => s.settings.data.auto_select_mode_on_new_agent); const prevInputOpenRef = useRef(inputOpen); useEffect(() => { - if (prevInputOpenRef.current && !inputOpen && elementSelection?.selectMode) { - elementSelection.setSelectMode(false); + if (prevInputOpenRef.current && !inputOpen && elementSelection) { + elementSelection.clearOwnerElements(TOOLBAR_OWNER_ID); + if (elementSelection.selectMode && elementSelection.activeOwnerId === TOOLBAR_OWNER_ID) { + elementSelection.setSelectMode(false); + } } - if (!prevInputOpenRef.current && inputOpen && autoSelectOnNew) { - elementSelection?.setSelectMode(true); + if (!prevInputOpenRef.current && inputOpen && autoSelectOnNew && elementSelection) { + elementSelection.clearOwnerElements(TOOLBAR_OWNER_ID); + elementSelection.setActiveOwnerId(TOOLBAR_OWNER_ID); + elementSelection.setExcludeSelectId(null); + elementSelection.setSelectMode(true); } prevInputOpenRef.current = inputOpen; }, [inputOpen, elementSelection, autoSelectOnNew]); @@ -240,21 +247,44 @@ const DashboardToolbar = React.forwardRef( useEffect(() => { if (!isExpanded) return; - const handleClick = (e: MouseEvent) => { - if (elementSelection?.selectMode) return; + let downPos: { x: number; y: number; target: Node } | null = null; + const DRAG_THRESHOLD = 5; + + const handleDown = (e: MouseEvent) => { const target = e.target as Node; if (containerRef.current && !containerRef.current.contains(target)) { - const el = target instanceof Element ? target : target.parentElement; - if (el?.closest('[role="dialog"], [role="presentation"], .MuiModal-root, .MuiPopover-root')) { - return; - } - handleDismiss(); + downPos = { x: e.clientX, y: e.clientY, target }; + } else { + downPos = null; } }; - const t = setTimeout(() => document.addEventListener('mousedown', handleClick), 50); + + const handleUp = (e: MouseEvent) => { + if (!downPos) return; + const dx = e.clientX - downPos.x; + const dy = e.clientY - downPos.y; + const target = downPos.target; + downPos = null; + if (Math.abs(dx) > DRAG_THRESHOLD || Math.abs(dy) > DRAG_THRESHOLD) return; + + const el = target instanceof Element ? target : (target as Node).parentElement; + if (el?.closest('[role="dialog"], [role="presentation"], .MuiModal-root, .MuiPopover-root')) { + return; + } + if (elementSelection?.selectMode && el?.closest('[data-select-type]')) { + return; + } + handleDismiss(); + }; + + const t = setTimeout(() => { + document.addEventListener('mousedown', handleDown, true); + document.addEventListener('mouseup', handleUp, true); + }, 50); return () => { clearTimeout(t); - document.removeEventListener('mousedown', handleClick); + document.removeEventListener('mousedown', handleDown, true); + document.removeEventListener('mouseup', handleUp, true); }; }, [isExpanded, handleDismiss, elementSelection?.selectMode]); @@ -333,6 +363,7 @@ const DashboardToolbar = React.forwardRef( onModelChange={setModel} embedded autoFocus + sessionId={TOOLBAR_OWNER_ID} /> ) : historyOpen ? ( diff --git a/frontend/src/app/pages/Views/ViewEditor.tsx b/frontend/src/app/pages/Views/ViewEditor.tsx index 265de439..8a9c2ac8 100644 --- a/frontend/src/app/pages/Views/ViewEditor.tsx +++ b/frontend/src/app/pages/Views/ViewEditor.tsx @@ -9,7 +9,6 @@ import Tab from '@mui/material/Tab'; import Tooltip from '@mui/material/Tooltip'; import Switch from '@mui/material/Switch'; import CircularProgress from '@mui/material/CircularProgress'; -import ArrowBackIcon from '@mui/icons-material/ArrowBack'; import SaveIcon from '@mui/icons-material/Save'; import PlayArrowIcon from '@mui/icons-material/PlayArrow'; import HtmlIcon from '@mui/icons-material/Code'; @@ -795,27 +794,6 @@ const ViewEditor: React.FC = ({ output, onClose }) => { await performSaveRef.current?.(close); }; - const handleClose = async () => { - if (autoSaveTimerRef.current) clearTimeout(autoSaveTimerRef.current); - if (savedStatusTimerRef.current) clearTimeout(savedStatusTimerRef.current); - const eid = output?.id ?? createdIdRef.current; - if (!savedRef.current && (files['index.html'] ?? '').trim()) { - try { - const body = buildBody(); - let savedId: string; - if (eid) { - await dispatch(updateOutput({ id: eid, ...body })).unwrap(); - savedId = eid; - } else { - const created = await dispatch(createOutput(body)).unwrap(); - savedId = created.id; - } - captureThumbnailAsync(savedId); - } catch {} - } - onClose(); - }; - const handleRunPreview = async () => { const eid = output?.id ?? createdIdRef.current; if (!eid) { @@ -1136,10 +1114,6 @@ const ViewEditor: React.FC = ({ output, onClose }) => { minHeight: 44, }} > - - - - setName(e.target.value)} diff --git a/frontend/src/app/pages/Views/Views.tsx b/frontend/src/app/pages/Views/Views.tsx index 110562d8..7de1b83a 100644 --- a/frontend/src/app/pages/Views/Views.tsx +++ b/frontend/src/app/pages/Views/Views.tsx @@ -39,6 +39,9 @@ const Views: React.FC = () => { setEditorOpen(true); } else if (routeId && routeId !== 'new') { navigate('/apps', { replace: true }); + } else if (!routeId) { + setEditorOpen(false); + setEditingOutput(null); } }, [routeId, loaded, items, navigate]); @@ -62,7 +65,7 @@ const Views: React.FC = () => { }; if (editorOpen) { - return ; + return ; } return ( diff --git a/frontend/src/shared/dashboardClipboard.ts b/frontend/src/shared/dashboardClipboard.ts new file mode 100644 index 00000000..a3b4af88 --- /dev/null +++ b/frontend/src/shared/dashboardClipboard.ts @@ -0,0 +1,34 @@ +import type { CardType } from '@/app/pages/Dashboard/useDashboardSelection'; + +export interface ClipboardCard { + type: CardType; + id: string; + name: string; + meta: Record; + x: number; + y: number; + width: number; + height: number; + expanded?: boolean; +} + +let clipboardCards: ClipboardCard[] = []; +let clipboardTimestamp = 0; + +export function setClipboardCards(cards: ClipboardCard[]): void { + clipboardCards = cards; + clipboardTimestamp = Date.now(); +} + +export function getClipboardCards(): ClipboardCard[] { + return clipboardCards; +} + +export function getClipboardTimestamp(): number { + return clipboardTimestamp; +} + +export function clearClipboard(): void { + clipboardCards = []; + clipboardTimestamp = 0; +} diff --git a/frontend/src/shared/state/agentsSlice.ts b/frontend/src/shared/state/agentsSlice.ts index c52c13cf..d1f98d28 100644 --- a/frontend/src/shared/state/agentsSlice.ts +++ b/frontend/src/shared/state/agentsSlice.ts @@ -331,6 +331,20 @@ export const closeSession = createAsyncThunk( } ); +export const duplicateSession = createAsyncThunk( + 'agents/duplicateSession', + async ({ sessionId, dashboardId }: { sessionId: string; dashboardId?: string }) => { + const res = await fetch(`${AGENTS_API}/sessions/${sessionId}/duplicate`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ dashboard_id: dashboardId }), + }); + if (!res.ok) throw new Error('Failed to duplicate session'); + const data = await res.json(); + return data.session as AgentSession; + } +); + export const deleteSession = createAsyncThunk( 'agents/deleteSession', async ({ sessionId }: { sessionId: string }) => { @@ -784,6 +798,10 @@ const agentsSlice = createSlice({ session.active_branch_id = action.payload.branchId; } }) + .addCase(duplicateSession.fulfilled, (state, action) => { + const session = action.payload; + state.sessions[session.id] = session; + }) .addCase(closeSession.fulfilled, (state, action) => { const sessionId = action.payload; const session = state.sessions[sessionId]; diff --git a/frontend/src/shared/state/dashboardLayoutSlice.ts b/frontend/src/shared/state/dashboardLayoutSlice.ts index 8a51e148..556a805c 100644 --- a/frontend/src/shared/state/dashboardLayoutSlice.ts +++ b/frontend/src/shared/state/dashboardLayoutSlice.ts @@ -11,7 +11,7 @@ export const DEFAULT_VIEW_CARD_H = 800; export const DEFAULT_BROWSER_CARD_W = 1280; export const DEFAULT_BROWSER_CARD_H = 800; export const EXPANDED_CARD_MIN_H = 620; -const GRID_GAP = 24; +export const GRID_GAP = 24; const GRID_ORIGIN = { x: 40, y: 100 }; const GRID_COLS_FALLBACK = 4; @@ -54,6 +54,7 @@ export interface DashboardLayoutState { viewCards: Record; browserCards: Record; glowingBrowserCards: Record; + glowingAgentCards: Record; persistedExpandedSessionIds: string[]; loading: boolean; initialized: boolean; @@ -64,6 +65,7 @@ const initialState: DashboardLayoutState = { viewCards: {}, browserCards: {}, glowingBrowserCards: {}, + glowingAgentCards: {}, persistedExpandedSessionIds: [], loading: false, initialized: false, @@ -214,6 +216,14 @@ const dashboardLayoutSlice = createSlice({ } }, + placeCard( + state, + action: PayloadAction<{ sessionId: string; x: number; y: number; width: number; height: number }> + ) { + const { sessionId, x, y, width, height } = action.payload; + state.cards[sessionId] = { session_id: sessionId, x, y, width, height }; + }, + removeCard(state, action: PayloadAction) { delete state.cards[action.payload]; }, @@ -293,17 +303,28 @@ const dashboardLayoutSlice = createSlice({ } }, - addViewCard(state, action: PayloadAction<{ outputId: string; expandedSessionIds?: string[] }>) { - const { outputId, expandedSessionIds } = action.payload; + addViewCard(state, action: PayloadAction<{ + outputId: string; expandedSessionIds?: string[]; + x?: number; y?: number; width?: number; height?: number; + }>) { + const { outputId, expandedSessionIds, x, y, width, height } = action.payload; if (state.viewCards[outputId]) return; - const rects = collectOccupiedRects(state, expandedSessionIds); - const pos = findOpenGridCell(rects, DEFAULT_VIEW_CARD_W, DEFAULT_VIEW_CARD_H); + let posX: number, posY: number; + if (x != null && y != null) { + posX = x; + posY = y; + } else { + const rects = collectOccupiedRects(state, expandedSessionIds); + const pos = findOpenGridCell(rects, DEFAULT_VIEW_CARD_W, DEFAULT_VIEW_CARD_H); + posX = pos.x; + posY = pos.y; + } state.viewCards[outputId] = { output_id: outputId, - x: pos.x, - y: pos.y, - width: DEFAULT_VIEW_CARD_W, - height: DEFAULT_VIEW_CARD_H, + x: posX, + y: posY, + width: width || DEFAULT_VIEW_CARD_W, + height: height || DEFAULT_VIEW_CARD_H, }; }, @@ -384,6 +405,44 @@ const dashboardLayoutSlice = createSlice({ delete state.browserCards[action.payload]; }, + pasteBrowserCard( + state, + action: PayloadAction<{ + tabs: BrowserTab[]; url: string; expandedSessionIds?: string[]; + id?: string; x?: number; y?: number; width?: number; height?: number; + }> + ) { + const { x, y, width, height } = action.payload; + const id = action.payload.id || `browser-${Date.now().toString(36)}`; + const newTabs = action.payload.tabs.map((t) => ({ + id: generateTabId(), + url: t.url, + title: '', + favicon: undefined, + })); + const activeTab = newTabs[0]; + let posX: number, posY: number; + if (x != null && y != null) { + posX = x; + posY = y; + } else { + const rects = collectOccupiedRects(state, action.payload.expandedSessionIds); + const pos = findOpenGridCell(rects, DEFAULT_BROWSER_CARD_W, DEFAULT_BROWSER_CARD_H); + posX = pos.x; + posY = pos.y; + } + state.browserCards[id] = { + browser_id: id, + url: activeTab?.url || action.payload.url, + tabs: newTabs.length > 0 ? newTabs : [{ id: generateTabId(), url: action.payload.url, title: '' }], + activeTabId: activeTab?.id || generateTabId(), + x: posX, + y: posY, + width: width || DEFAULT_BROWSER_CARD_W, + height: height || DEFAULT_BROWSER_CARD_H, + }; + }, + updateBrowserCardUrl( state, action: PayloadAction<{ browserId: string; url: string }> @@ -555,11 +614,26 @@ const dashboardLayoutSlice = createSlice({ state.glowingBrowserCards = {}; }, + setGlowingAgentCard(state, action: PayloadAction<{ sessionId: string; sourceId: string }>) { + const { sessionId, sourceId } = action.payload; + state.glowingAgentCards[sessionId] = { sourceId, fading: false }; + }, + + fadeGlowingAgentCard(state, action: PayloadAction) { + const entry = state.glowingAgentCards[action.payload]; + if (entry) entry.fading = true; + }, + + clearGlowingAgentCard(state, action: PayloadAction) { + delete state.glowingAgentCards[action.payload]; + }, + resetLayout(state) { state.cards = {}; state.viewCards = {}; state.browserCards = {}; state.glowingBrowserCards = {}; + state.glowingAgentCards = {}; state.persistedExpandedSessionIds = []; state.initialized = false; }, @@ -595,6 +669,7 @@ const dashboardLayoutSlice = createSlice({ export const { setCardPosition, + placeCard, setCardSize, removeCard, reconcileSessions, @@ -609,6 +684,7 @@ export const { setBrowserCardPosition, setBrowserCardSize, removeBrowserCard, + pasteBrowserCard, updateBrowserCardUrl, addBrowserTab, removeBrowserTab, @@ -621,6 +697,9 @@ export const { setGlowingBrowserCards, clearGlowingBrowserCards, clearAllGlowingBrowserCards, + setGlowingAgentCard, + fadeGlowingAgentCard, + clearGlowingAgentCard, resetLayout, } = dashboardLayoutSlice.actions; From a1138a1782263356edcefeeec42dde5bdb301503 Mon Sep 17 00:00:00 2001 From: haikdc Date: Wed, 18 Mar 2026 14:19:42 -0700 Subject: [PATCH 08/11] [Haik]: (ability to resume chats if force end early) (agent selection actually works as advertised) (browser control/creation tool isolation) (in dev mode can see the action input schema) --- .gitignore | 1 + backend/apps/agents/agent_manager.py | 220 +++++++++++-- backend/apps/agents/agents.py | 2 + .../apps/agents/browser_agent_mcp_server.py | 82 +++-- .../apps/agents/invoke_agent_mcp_server.py | 160 ++++++++++ backend/apps/agents/models.py | 1 + backend/apps/settings/models.py | 1 + backend/apps/tools_lib/models.py | 9 +- backend/apps/tools_lib/tools_lib.py | 7 +- backend/main.py | 34 ++ electron/main.js | 2 - electron/package-lock.json | 4 +- electron/package.json | 22 +- .../src/app/pages/AgentChat/AgentChat.tsx | 77 ++++- .../src/app/pages/AgentChat/ChatInput.tsx | 5 +- .../app/pages/AgentChat/ToolCallBubble.tsx | 290 +++++++++++++++++- .../src/app/pages/Dashboard/Dashboard.tsx | 2 +- frontend/src/app/pages/Settings/Settings.tsx | 18 ++ frontend/src/app/pages/Tools/Tools.tsx | 216 +++++++++++-- frontend/src/shared/state/agentsSlice.ts | 14 +- frontend/src/shared/state/settingsSlice.ts | 2 + frontend/src/shared/state/toolsSlice.ts | 1 + scripts/build-app.sh | 36 ++- 23 files changed, 1082 insertions(+), 124 deletions(-) create mode 100644 backend/apps/agents/invoke_agent_mcp_server.py diff --git a/.gitignore b/.gitignore index d546f98d..76fb66c7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,7 @@ backend/data/** # Electron build artifacts electron/dist/ electron/python-env/ +electron/build-staging/ electron/node_modules/ electron/package-lock.json diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index ce548740..c429a7b2 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -66,6 +66,8 @@ FULL_TOOLS = [ "TaskOutput", "TaskStop", "CronCreate", "CronList", "CronDelete", "RenderOutput", + "InvokeAgent", + "Agent", ] def _get_denied_tool_names(tool) -> set[str]: @@ -232,8 +234,12 @@ class AgentManager: + "\n" ) - def _build_browser_context(self, dashboard_id: str | None) -> str | None: - """Build a context block listing browser cards and delegation instructions.""" + def _build_browser_context(self, dashboard_id: str | None, selected_browser_ids: list[str] | None = None) -> str | None: + """Build a context block listing browser cards and delegation instructions. + + Only browser cards explicitly selected by the user are included. + If none are selected, no browser card details are exposed. + """ if not dashboard_id: return None try: @@ -246,34 +252,39 @@ class AgentManager: lines = [ "", - "You have access to browser automation through the BrowserAgent and BrowserAgents tools.", + "You have access to browser automation through the CreateBrowserAgent, BrowserAgent, and BrowserAgents tools.", "", - "- **BrowserAgent(task, browser_id?, url?)**: Delegate a single browser task to a dedicated browser agent. " + "- **CreateBrowserAgent(task, url?)**: Create a new browser card and run a task on it. " + "Use this when you need a fresh browser. Optionally provide a starting URL.", + "- **BrowserAgent(browser_id, task)**: Delegate a task to an existing browser card. " "The browser agent will autonomously navigate, click, type, and interact with the page, then return a summary and screenshot.", - "- **BrowserAgents(tasks)**: Run multiple browser tasks in parallel, each on a different browser.", + "- **BrowserAgents(tasks)**: Run multiple browser tasks in parallel on existing browser cards. " + "Each task requires a browser_id.", "", "You do NOT have direct access to low-level browser tools (click, type, screenshot, etc.). " "Instead, describe what you want accomplished and the browser agent will handle the details.", - "", - "If you omit browser_id, a new browser card will be auto-created. " - "If you provide a url without a browser_id, the new browser navigates there first.", ] - if browser_cards: - lines.append("") - lines.append("Available browser cards on the dashboard:") - for card in browser_cards.values(): - bid = card.get("browser_id", "") - tabs = card.get("tabs", []) - active_tab_id = card.get("activeTabId", "") - active_tab = next((t for t in tabs if t.get("id") == active_tab_id), None) - url = (active_tab or {}).get("url", card.get("url", "")) - title = (active_tab or {}).get("title", "") - lines.append(f"- browser_id: \"{bid}\"") - if title: - lines.append(f" Title: {title}") - if url: - lines.append(f" URL: {url}") + if browser_cards and selected_browser_ids: + visible_cards = [ + card for card in browser_cards.values() + if card.get("browser_id", "") in selected_browser_ids + ] + if visible_cards: + lines.append("") + lines.append("The user selected these browser cards for you to work with:") + for card in visible_cards: + bid = card.get("browser_id", "") + tabs = card.get("tabs", []) + active_tab_id = card.get("activeTabId", "") + active_tab = next((t for t in tabs if t.get("id") == active_tab_id), None) + url = (active_tab or {}).get("url", card.get("url", "")) + title = (active_tab or {}).get("title", "") + lines.append(f"- browser_id: \"{bid}\"") + if title: + lines.append(f" Title: {title}") + if url: + lines.append(f" URL: {url}") lines.append("") return "\n".join(lines) @@ -456,7 +467,7 @@ class AgentManager: }) return content - async def _run_agent_loop(self, session_id: str, prompt: str, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None): + async def _run_agent_loop(self, session_id: str, prompt: str, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None, fork_session: bool = False, selected_browser_ids: list[str] | None = None): """Run the Claude Agent SDK query loop for a session.""" session = self.sessions.get(session_id) if not session: @@ -492,6 +503,10 @@ class AgentManager: if bm: return _builtin_perms.get(bm.group(1), "always_allow") + im = _re.match(r"mcp__openswarm-invoke-agent__(.+)", tool_name) + if im: + return _builtin_perms.get(im.group(1), "always_allow") + m = _re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name) if m: server_slug, mcp_tool_name = m.group(1), m.group(2) @@ -638,13 +653,13 @@ class AgentManager: _, mode_sys_prompt, _ = self._resolve_mode(session.mode) connected_tools_ctx = self._build_connected_tools_context(session.allowed_tools) outputs_ctx = self._build_outputs_context() - browser_ctx = self._build_browser_context(session.dashboard_id) + browser_ctx = self._build_browser_context(session.dashboard_id, selected_browser_ids=selected_browser_ids) global_settings = load_settings() composed_prompt = self._compose_system_prompt(global_settings.default_system_prompt, mode_sys_prompt, session.system_prompt, connected_tools_ctx, outputs_ctx, browser_ctx) mcp_servers = await self._build_mcp_servers(session.allowed_tools) - _browser_delegation_tools = ["BrowserAgent", "BrowserAgents"] + _browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents"] _browser_all_denied = all( _builtin_perms.get(t, "always_allow") == "deny" for t in _browser_delegation_tools @@ -669,6 +684,28 @@ class AgentManager: "type": "stdio", } + _invoke_agent_tools = ["InvokeAgent"] + _invoke_all_denied = all( + _builtin_perms.get(t, "always_allow") == "deny" + for t in _invoke_agent_tools + ) + + if not _invoke_all_denied: + invoke_agent_server_path = os.path.join( + os.path.dirname(__file__), "invoke_agent_mcp_server.py" + ) + backend_port = os.environ.get("OPENSWARM_PORT", "8324") + mcp_servers["openswarm-invoke-agent"] = { + "command": sys.executable, + "args": [invoke_agent_server_path], + "env": { + "OPENSWARM_PORT": backend_port, + "OPENSWARM_PARENT_SESSION_ID": session.id, + "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", + }, + "type": "stdio", + } + effective_allowed = [ t for t in session.allowed_tools if t in FULL_TOOLS and _builtin_perms.get(t, "always_allow") == "always_allow" @@ -691,6 +728,15 @@ class AgentManager: effective_disallowed.append(f"mcp__openswarm-browser-agent__{bt}") continue + if name == "openswarm-invoke-agent": + for it in _invoke_agent_tools: + policy = _builtin_perms.get(it, "always_allow") + if policy == "always_allow": + effective_allowed.append(f"mcp__openswarm-invoke-agent__{it}") + elif policy == "deny": + effective_disallowed.append(f"mcp__openswarm-invoke-agent__{it}") + continue + tool_def = next( (t for t in all_tools_list if t.mcp_config and t.enabled and _sanitize_server_name(t.name) == name), @@ -736,6 +782,8 @@ class AgentManager: if session.sdk_session_id: options_kwargs["resume"] = session.sdk_session_id + if fork_session: + options_kwargs["fork_session"] = True options = ClaudeAgentOptions(**options_kwargs) @@ -1031,6 +1079,8 @@ class AgentManager: context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None, + hidden: bool = False, + selected_browser_ids: list[str] | None = None, ): """Send a follow-up message to an existing session.""" session = self.sessions.get(session_id) @@ -1067,6 +1117,7 @@ class AgentManager: attached_skills=skill_meta, forced_tools=forced_tools if forced_tools else None, images=image_meta, + hidden=hidden, ) session.messages.append(user_msg) await ws_manager.send_to_session(session_id, "agent:message", { @@ -1074,7 +1125,14 @@ class AgentManager: "message": user_msg.model_dump(mode="json"), }) - task = asyncio.create_task(self._run_agent_loop(session_id, prompt, images=images, context_paths=context_paths, forced_tools=forced_tools, attached_skills=attached_skills)) + session.status = "running" + await ws_manager.send_to_session(session_id, "agent:status", { + "session_id": session_id, + "status": "running", + "session": session.model_dump(mode="json"), + }) + + task = asyncio.create_task(self._run_agent_loop(session_id, prompt, images=images, context_paths=context_paths, forced_tools=forced_tools, attached_skills=attached_skills, selected_browser_ids=selected_browser_ids)) self.tasks[session_id] = task async def stop_agent(self, session_id: str): @@ -1552,6 +1610,114 @@ class AgentManager: return new_session + async def invoke_agent( + self, + source_session_id: str, + message: str, + parent_session_id: str | None = None, + dashboard_id: str | None = None, + ) -> dict: + """Fork an existing session and send it a new message, returning the result.""" + source = self.sessions.get(source_session_id) + if not source: + data = _load_session_data(source_session_id) + if data is None: + raise ValueError(f"Session {source_session_id} not found") + source = AgentSession(**data) + + source_name = source.name + + old_to_new_msg: dict[str, str] = {} + new_messages: list[Message] = [] + for msg in source.messages: + new_id = uuid4().hex + old_to_new_msg[msg.id] = new_id + new_messages.append(Message( + id=new_id, + role=msg.role, + content=msg.content, + timestamp=msg.timestamp, + branch_id=msg.branch_id, + parent_id=old_to_new_msg.get(msg.parent_id) if msg.parent_id else None, + context_paths=msg.context_paths, + attached_skills=msg.attached_skills, + forced_tools=msg.forced_tools, + images=msg.images, + )) + + new_branches: dict[str, MessageBranch] = {} + for bid, branch in source.branches.items(): + new_branches[bid] = MessageBranch( + id=bid, + parent_branch_id=branch.parent_branch_id, + fork_point_message_id=( + old_to_new_msg.get(branch.fork_point_message_id) + if branch.fork_point_message_id else None + ), + created_at=branch.created_at, + ) + + fork = AgentSession( + id=uuid4().hex, + name=f"{source_name} (invoked)", + status="running", + model=source.model, + mode="invoked-agent", + sdk_session_id=source.sdk_session_id, + system_prompt=source.system_prompt, + allowed_tools=list(source.allowed_tools), + max_turns=source.max_turns or 25, + cwd=source.cwd, + created_at=datetime.now(), + messages=new_messages, + branches=new_branches, + active_branch_id=source.active_branch_id, + tool_group_meta=dict(source.tool_group_meta), + dashboard_id=dashboard_id or source.dashboard_id, + parent_session_id=parent_session_id, + ) + + self.sessions[fork.id] = fork + + await ws_manager.broadcast_global("agent:status", { + "session_id": fork.id, + "status": fork.status, + "session": fork.model_dump(mode="json"), + }) + + user_msg = Message( + role="user", + content=message, + branch_id=fork.active_branch_id, + ) + fork.messages.append(user_msg) + await ws_manager.send_to_session(fork.id, "agent:message", { + "session_id": fork.id, + "message": user_msg.model_dump(mode="json"), + }) + + await self._run_agent_loop(fork.id, message, fork_session=True) + + last_assistant = None + for msg in reversed(fork.messages): + if msg.role == "assistant": + content = msg.content + if isinstance(content, str): + last_assistant = content + elif isinstance(content, list): + texts = [b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text"] + last_assistant = "\n".join(texts) + else: + last_assistant = str(content) + break + + return { + "forked_session_id": fork.id, + "source_name": source_name, + "response": last_assistant or "No response from invoked agent.", + "cost_usd": fork.cost_usd, + } + def get_all_sessions(self, dashboard_id: str | None = None) -> list[AgentSession]: if dashboard_id: return [s for s in self.sessions.values() if s.dashboard_id == dashboard_id] diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 8bf33a2b..b0052ad5 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -56,6 +56,8 @@ async def send_message(session_id: str, body: dict): context_paths=body.get("context_paths"), forced_tools=body.get("forced_tools"), attached_skills=body.get("attached_skills"), + hidden=body.get("hidden", False), + selected_browser_ids=body.get("selected_browser_ids"), ) return {"ok": True} diff --git a/backend/apps/agents/browser_agent_mcp_server.py b/backend/apps/agents/browser_agent_mcp_server.py index a229b852..0f7f4b6b 100644 --- a/backend/apps/agents/browser_agent_mcp_server.py +++ b/backend/apps/agents/browser_agent_mcp_server.py @@ -29,23 +29,16 @@ PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") TOOLS = [ { - "name": "BrowserAgent", + "name": "CreateBrowserAgent", "description": ( - "Delegate a browser task to a dedicated browser agent. The browser agent " + "Create a new browser card and run a task on it. A dedicated browser agent " "will autonomously perform the task (navigating, clicking, typing, etc.) " "and return a summary of actions taken plus a final screenshot. " - "Use this for any task that requires interacting with a web page." + "Use this when you need a fresh browser for a new task." ), "inputSchema": { "type": "object", "properties": { - "browser_id": { - "type": "string", - "description": ( - "The ID of the browser card to use. If omitted, a new browser " - "card will be automatically created." - ), - }, "task": { "type": "string", "description": ( @@ -56,20 +49,47 @@ TOOLS = [ "url": { "type": "string", "description": ( - "Optional starting URL. If provided and no browser_id is given, " - "the new browser will navigate here first." + "Optional starting URL. The new browser will navigate here " + "before beginning the task." ), }, }, "required": ["task"], }, }, + { + "name": "BrowserAgent", + "description": ( + "Delegate a browser task to a dedicated browser agent on an existing " + "browser card. The browser agent will autonomously perform the task " + "(navigating, clicking, typing, etc.) and return a summary of actions " + "taken plus a final screenshot." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The ID of the existing browser card to use.", + }, + "task": { + "type": "string", + "description": ( + "The task for the browser agent to perform. Be specific and " + "detailed about what you want accomplished." + ), + }, + }, + "required": ["browser_id", "task"], + }, + }, { "name": "BrowserAgents", "description": ( - "Delegate multiple browser tasks to run in parallel, each on a different " - "browser. All tasks execute concurrently and results are returned together. " - "Use this when you need to perform tasks on multiple web pages simultaneously." + "Delegate multiple browser tasks to run in parallel, each on an existing " + "browser card. All tasks execute concurrently and results are returned " + "together. Use this when you need to perform tasks on multiple web pages " + "simultaneously." ), "inputSchema": { "type": "object", @@ -82,18 +102,14 @@ TOOLS = [ "properties": { "browser_id": { "type": "string", - "description": "Optional browser card ID. If omitted, a new browser will be created.", + "description": "The ID of the existing browser card to use.", }, "task": { "type": "string", "description": "The task for this browser agent.", }, - "url": { - "type": "string", - "description": "Optional starting URL.", - }, }, - "required": ["task"], + "required": ["browser_id", "task"], }, }, }, @@ -221,10 +237,10 @@ def format_batch_results(results: list[dict]) -> dict: def handle_tool_call(tool_name: str, arguments: dict) -> dict: - if tool_name == "BrowserAgent": + if tool_name == "CreateBrowserAgent": task_def = { "task": arguments.get("task", ""), - "browser_id": arguments.get("browser_id", ""), + "browser_id": "", "url": arguments.get("url", ""), } result = call_backend([task_def]) @@ -235,10 +251,30 @@ def handle_tool_call(tool_name: str, arguments: dict) -> dict: return format_result(results[0]) return {"content": [{"type": "text", "text": "No result returned."}], "isError": True} + elif tool_name == "BrowserAgent": + browser_id = arguments.get("browser_id", "") + if not browser_id: + return {"content": [{"type": "text", "text": "Error: browser_id is required"}], "isError": True} + task_def = { + "task": arguments.get("task", ""), + "browser_id": browser_id, + "url": "", + } + result = call_backend([task_def]) + if "error" in result: + return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} + results = result.get("results", [result]) + if results: + return format_result(results[0]) + return {"content": [{"type": "text", "text": "No result returned."}], "isError": True} + elif tool_name == "BrowserAgents": tasks = arguments.get("tasks", []) if not tasks: return {"content": [{"type": "text", "text": "Error: tasks array is empty"}], "isError": True} + for t in tasks: + if not t.get("browser_id"): + return {"content": [{"type": "text", "text": "Error: browser_id is required for each task"}], "isError": True} result = call_backend(tasks) if "error" in result: return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} diff --git a/backend/apps/agents/invoke_agent_mcp_server.py b/backend/apps/agents/invoke_agent_mcp_server.py new file mode 100644 index 00000000..cf614b03 --- /dev/null +++ b/backend/apps/agents/invoke_agent_mcp_server.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +""" +Stdio MCP server that exposes the InvokeAgent tool. + +Launched as a subprocess by the Claude Agent SDK. Proxies invocation +requests to the OpenSwarm backend via HTTP, which forks the target +agent session and runs it with the new message. +""" + +import json +import sys +import os +import urllib.request +import urllib.error + +BACKEND_PORT = os.environ.get("OPENSWARM_PORT", "8324") +BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/invoke-agent/run" +PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") +DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "") + +TOOLS = [ + { + "name": "InvokeAgent", + "description": ( + "Invoke a copy of an existing agent session with a new message. " + "The invoked agent will have full context of its prior conversation " + "and will process the new message independently. Use this when you " + "need to query another agent about its prior work or ask it to " + "perform a follow-up task." + ), + "inputSchema": { + "type": "object", + "properties": { + "session_id": { + "type": "string", + "description": ( + "The session ID of the agent to invoke. This is the ID " + "from a selected Agent Card in the context." + ), + }, + "message": { + "type": "string", + "description": ( + "The message to send to the invoked agent. Be specific " + "about what you need from it." + ), + }, + }, + "required": ["session_id", "message"], + }, + }, +] + + +def send_response(id_, result=None, error=None): + msg = {"jsonrpc": "2.0", "id": id_} + if error is not None: + msg["error"] = error + else: + msg["result"] = result + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def call_backend(session_id: str, message: str) -> dict: + payload = json.dumps({ + "session_id": session_id, + "message": message, + "parent_session_id": PARENT_SESSION_ID, + "dashboard_id": DASHBOARD_ID, + }).encode() + req = urllib.request.Request( + BACKEND_URL, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=600) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + body = e.read().decode() if e.fp else str(e) + return {"error": f"HTTP {e.code}: {body}"} + except Exception as e: + return {"error": str(e)} + + +def handle_tool_call(tool_name: str, arguments: dict) -> dict: + if tool_name != "InvokeAgent": + return {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True} + + session_id = arguments.get("session_id", "") + message = arguments.get("message", "") + + if not session_id: + return {"content": [{"type": "text", "text": "Error: session_id is required"}], "isError": True} + if not message: + return {"content": [{"type": "text", "text": "Error: message is required"}], "isError": True} + + result = call_backend(session_id, message) + + if "error" in result: + return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} + + forked_id = result.get("forked_session_id", "") + response = result.get("response", "No response from invoked agent.") + cost = result.get("cost_usd", 0) + source_name = result.get("source_name", "") + + lines = [f"**Invoked Agent Result** (forked session: {forked_id})"] + if source_name: + lines[0] = f"**Invoked Agent Result** — {source_name} (forked session: {forked_id})" + if cost > 0: + lines.append(f"*Cost: ${cost:.4f}*") + lines.append("") + lines.append(response) + + return {"content": [{"type": "text", "text": "\n".join(lines)}]} + + +def main(): + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + + method = msg.get("method") + id_ = msg.get("id") + params = msg.get("params", {}) + + if method == "initialize": + send_response(id_, { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": { + "name": "openswarm-invoke-agent", + "version": "1.0.0", + }, + }) + elif method == "notifications/initialized": + pass + elif method == "tools/list": + send_response(id_, {"tools": TOOLS}) + elif method == "tools/call": + tool_name = params.get("name", "") + arguments = params.get("arguments", {}) + result = handle_tool_call(tool_name, arguments) + send_response(id_, result) + elif method == "ping": + send_response(id_, {}) + elif id_ is not None: + send_response(id_, error={"code": -32601, "message": f"Method not found: {method}"}) + + +if __name__ == "__main__": + main() diff --git a/backend/apps/agents/models.py b/backend/apps/agents/models.py index 41480761..948a007c 100644 --- a/backend/apps/agents/models.py +++ b/backend/apps/agents/models.py @@ -37,6 +37,7 @@ class Message(BaseModel): attached_skills: Optional[list[dict]] = None forced_tools: Optional[list[str]] = None images: Optional[list[dict]] = None + hidden: bool = False class MessageBranch(BaseModel): id: str = Field(default_factory=lambda: uuid4().hex) diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index 14d633d4..bdd41e0c 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -23,3 +23,4 @@ class AppSettings(BaseModel): browser_homepage: str = "https://www.google.com" auto_select_mode_on_new_agent: bool = False expand_new_chats_in_dashboard: bool = False + dev_mode: bool = False diff --git a/backend/apps/tools_lib/models.py b/backend/apps/tools_lib/models.py index e00d4be6..857ccc3a 100644 --- a/backend/apps/tools_lib/models.py +++ b/backend/apps/tools_lib/models.py @@ -5,6 +5,7 @@ from uuid import uuid4 class BuiltinTool(BaseModel): name: str + display_name: Optional[str] = None description: str category: str = "filesystem" deferred: bool = False @@ -33,9 +34,13 @@ BUILTIN_TOOLS: list[BuiltinTool] = [ BuiltinTool(name="CronList", description="List all scheduled tasks", category="scheduling", deferred=True), BuiltinTool(name="CronDelete", description="Delete a scheduled task", category="scheduling", deferred=True), BuiltinTool(name="RenderOutput", description="Render a reusable View artifact with structured input data", category="views", deferred=True), + # Agent tools + BuiltinTool(name="Agent", display_name="CreateAgent", description="Spawn a sub-agent to handle a complex subtask", category="agents"), + BuiltinTool(name="InvokeAgent", description="Invoke a copy of an existing agent with a new message, preserving full conversation context", category="agents"), # Browser delegation tools (Layer 1 — what the main agent calls) - BuiltinTool(name="BrowserAgent", description="Delegate a browser task to a dedicated browser agent", category="browser_delegation"), - BuiltinTool(name="BrowserAgents", description="Run multiple browser tasks in parallel", category="browser_delegation"), + BuiltinTool(name="CreateBrowserAgent", description="Create a new browser and run a task on it", category="browser_delegation"), + BuiltinTool(name="BrowserAgent", description="Delegate a browser task to an existing browser agent", category="browser_delegation"), + BuiltinTool(name="BrowserAgents", description="Run multiple browser tasks in parallel on existing browsers", category="browser_delegation"), # Browser action tools (Layer 2 — what the sub-agent executes) BuiltinTool(name="BrowserScreenshot", description="Capture a screenshot of the browser page", category="browser_action"), BuiltinTool(name="BrowserNavigate", description="Navigate the browser to a URL", category="browser_action"), diff --git a/backend/apps/tools_lib/tools_lib.py b/backend/apps/tools_lib/tools_lib.py index 2d7c8bba..0abb4bf7 100644 --- a/backend/apps/tools_lib/tools_lib.py +++ b/backend/apps/tools_lib/tools_lib.py @@ -513,7 +513,7 @@ async def _discover_mcp_tools_http(url: str, headers: dict | None = None) -> lis raise HTTPException(status_code=502, detail="Empty response from MCP server") tools_list = data.get("result", {}).get("tools", []) - return [{"name": t.get("name", ""), "description": t.get("description", "")} for t in tools_list] + return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list] async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list[dict]: @@ -536,7 +536,7 @@ async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list ) as session: await session.initialize() result = await session.list_tools() - return [{"name": t.name, "description": t.description or ""} for t in result.tools] + return [{"name": t.name, "description": t.description or "", "inputSchema": t.inputSchema if t.inputSchema else None} for t in result.tools] except BaseExceptionGroup as eg: first = eg.exceptions[0] if eg.exceptions else eg raise HTTPException(status_code=502, detail=f"SSE discovery failed: {first}") from first @@ -605,7 +605,7 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, data = await _recv() tools_list = data.get("result", {}).get("tools", []) - return [{"name": t.get("name", ""), "description": t.get("description", "")} for t in tools_list] + return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list] except HTTPException: raise @@ -710,6 +710,7 @@ async def discover_tools(tool_id: str): permissions["_services"] = services permissions["_service_groups"] = service_groups permissions["_tool_descriptions"] = {t["name"]: t["description"] for t in raw_tools} + permissions["_tool_schemas"] = {t["name"]: t.get("inputSchema") for t in raw_tools if t.get("inputSchema")} tool.tool_permissions = permissions _save(tool) diff --git a/backend/main.py b/backend/main.py index 81c15119..c3d8736a 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,6 +1,9 @@ +import logging import os from uuid import uuid4 +logger = logging.getLogger(__name__) + from fastapi.responses import JSONResponse from fastapi import Request from backend.config.Apps import MainApp @@ -146,6 +149,37 @@ async def browser_agent_run(request: Request): return JSONResponse({"results": results}) +@app.post("/api/invoke-agent/run") +async def invoke_agent_run(request: Request): + """Fork an existing agent session and send it a new message. + Called by the invoke_agent_mcp_server stdio subprocess.""" + body = await request.json() + session_id = body.get("session_id", "") + message = body.get("message", "") + parent_session_id = body.get("parent_session_id", "") + dashboard_id = body.get("dashboard_id", "") + + if not session_id: + return JSONResponse({"error": "session_id is required"}, status_code=400) + if not message: + return JSONResponse({"error": "message is required"}, status_code=400) + + try: + from backend.apps.agents.agent_manager import agent_manager + result = await agent_manager.invoke_agent( + source_session_id=session_id, + message=message, + parent_session_id=parent_session_id or None, + dashboard_id=dashboard_id or None, + ) + return JSONResponse(result) + except ValueError as e: + return JSONResponse({"error": str(e)}, status_code=404) + except Exception as e: + logger.exception("invoke_agent_run failed") + return JSONResponse({"error": str(e)}, status_code=500) + + if __name__ == "__main__": import argparse import uvicorn diff --git a/electron/main.js b/electron/main.js index 8078bdbb..90b909c1 100644 --- a/electron/main.js +++ b/electron/main.js @@ -152,8 +152,6 @@ async function startBackend() { OPENSWARM_PACKAGED: isPackaged ? '1' : '0', OPENSWARM_PORT: String(backendPort), PYTHONDONTWRITEBYTECODE: '1', - GOOGLE_OAUTH_CLIENT_ID: process.env.GOOGLE_OAUTH_CLIENT_ID || 'REPLACE_WITH_YOUR_CLIENT_ID.apps.googleusercontent.com', - GOOGLE_OAUTH_CLIENT_SECRET: process.env.GOOGLE_OAUTH_CLIENT_SECRET || 'REPLACE_WITH_YOUR_CLIENT_SECRET', }; if (isPackaged) { diff --git a/electron/package-lock.json b/electron/package-lock.json index bb3fa6aa..f1d1551a 100644 --- a/electron/package-lock.json +++ b/electron/package-lock.json @@ -1,12 +1,12 @@ { "name": "openswarm", - "version": "1.0.6", + "version": "1.0.7", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openswarm", - "version": "1.0.6", + "version": "1.0.7", "hasInstallScript": true, "dependencies": { "electron-updater": "^6.3.0", diff --git a/electron/package.json b/electron/package.json index 61ca2ec8..35450928 100644 --- a/electron/package.json +++ b/electron/package.json @@ -1,6 +1,6 @@ { "name": "openswarm", - "version": "1.0.7", + "version": "1.0.8", "description": "OpenSwarm — AI Agent Orchestrator", "main": "main.js", "scripts": { @@ -60,34 +60,24 @@ }, "extraResources": [ { - "from": "../frontend/dist", + "from": "build-staging/frontend", "to": "frontend", "filter": [ "**/*" ] }, { - "from": "../backend", + "from": "build-staging/backend", "to": "backend", "filter": [ - "**/*", - "!__pycache__/**", - "!**/__pycache__/**", - "!.venv/**", - "!*.pyc" + "**/*" ] }, { - "from": "../debugger", + "from": "build-staging/debugger", "to": "debugger", "filter": [ - "**/*", - "!__pycache__/**", - "!**/__pycache__/**", - "!*.pyc", - "!.venv/**", - "!**/.venv/**", - "!**/node_modules/**" + "**/*" ] }, { diff --git a/frontend/src/app/pages/AgentChat/AgentChat.tsx b/frontend/src/app/pages/AgentChat/AgentChat.tsx index 0f81a75a..cee0281c 100644 --- a/frontend/src/app/pages/AgentChat/AgentChat.tsx +++ b/frontend/src/app/pages/AgentChat/AgentChat.tsx @@ -7,6 +7,7 @@ import IconButton from '@mui/material/IconButton'; import Tooltip from '@mui/material/Tooltip'; import CloseIcon from '@mui/icons-material/Close'; import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'; +import PlayArrowIcon from '@mui/icons-material/PlayArrow'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { sendMessage as sendMessageThunk, @@ -119,6 +120,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const chatInputRef = useRef(null); const isAtBottomRef = useRef(true); const [showScrollButton, setShowScrollButton] = useState(false); + const [showResumeBubble, setShowResumeBubble] = useState(false); const [mode, setMode] = useState('agent'); const [model, setModel] = useState('sonnet'); @@ -167,6 +169,9 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose prevStatusRef.current = curr; if (prev === 'running' && (curr === 'completed' || curr === 'stopped' || curr === 'error')) { if (id) dispatch(clearGlowingBrowserCards(id)); + if (curr === 'stopped') { + setShowResumeBubble(true); + } const currentMode = modesMap[mode]; if (currentMode?.default_next_mode && modesMap[currentMode.default_next_mode]) { setMode(currentMode.default_next_mode); @@ -175,6 +180,9 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose } } } + if (curr === 'running') { + setShowResumeBubble(false); + } }, [session?.status, mode, modesMap, id, isDraft, dispatch]); const SCROLL_THRESHOLD = 50; @@ -204,12 +212,13 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const handleSend = (prompt: string, images?: Array<{ data: string; media_type: string }>, contextPaths?: Array<{ path: string; type: 'file' | 'directory' }>, forcedTools?: string[], attachedSkills?: Array<{ id: string; name: string; content: string }>, selectedBrowserIds?: string[]) => { if (!id) return; + setShowResumeBubble(false); if (isDraft) { const config: Record = { model, mode }; if (session?.system_prompt) config.system_prompt = session.system_prompt; if (session?.target_directory) config.target_directory = session.target_directory; dispatch( - launchAndSendFirstMessage({ draftId: id, config, prompt, mode, model, images, contextPaths, forcedTools, attachedSkills }) + launchAndSendFirstMessage({ draftId: id, config, prompt, mode, model, images, contextPaths, forcedTools, attachedSkills, selectedBrowserIds }) ).then((action) => { if (launchAndSendFirstMessage.fulfilled.match(action)) { const realId = action.payload.session.id; @@ -223,7 +232,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose if (selectedBrowserIds?.length) { dispatch(setGlowingBrowserCards({ browserIds: selectedBrowserIds, sessionId: id })); } - dispatch(sendMessageThunk({ sessionId: id, prompt, mode, model, images, contextPaths, forcedTools, attachedSkills })); + dispatch(sendMessageThunk({ sessionId: id, prompt, mode, model, images, contextPaths, forcedTools, attachedSkills, selectedBrowserIds })); } }; @@ -250,6 +259,18 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose dispatch(stopAgent({ sessionId: id })); }; + const handleResume = useCallback(() => { + if (!id) return; + setShowResumeBubble(false); + dispatch(sendMessageThunk({ + sessionId: id, + prompt: "Continue where you left off. Start you're response EXACTLY with 'Sorry, let me pick up where I left off", + mode, + model, + hidden: true, + })); + }, [id, mode, model, dispatch]); + const [editingMessageId, setEditingMessageId] = useState(null); const handleSaveEdit = useCallback( @@ -443,13 +464,33 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose items.push(...outputItems); } else { - items.push(msg); + if (!msg.hidden) { + items.push(msg); + } i++; } } return items; }, [activeBranchMessages]); + const lastAssistantIdsInTurn = useMemo(() => { + const ids = new Set(); + let lastAssistantId: string | null = null; + for (const item of renderItems) { + if (!isToolGroup(item) && !isToolPair(item)) { + const msg = item as AgentMessage; + if (msg.role === 'assistant') { + lastAssistantId = msg.id; + } else if (msg.role === 'user') { + if (lastAssistantId) ids.add(lastAssistantId); + lastAssistantId = null; + } + } + } + if (lastAssistantId) ids.add(lastAssistantId); + return ids; + }, [renderItems]); + const groupMetaRequestedRef = useRef>(new Set()); const groupMetaRefinedRef = useRef>(new Set()); @@ -642,7 +683,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose onSaveEdit={handleSaveEdit} onCancelEdit={handleCancelEdit} /> - {!isEditing && (msg.role === 'user' || msg.role === 'assistant') && ( + {!isEditing && (msg.role === 'user' || (msg.role === 'assistant' && lastAssistantIdsInTurn.has(msg.id))) && ( navigator.clipboard.writeText(rawText)} @@ -704,6 +745,34 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose {session.status === 'running' && !session.streamingMessage && ( )} + {showResumeBubble && session.status === 'stopped' && ( + + + + + Resume Agent Response + + + + )} {showScrollButton && ( diff --git a/frontend/src/app/pages/AgentChat/ChatInput.tsx b/frontend/src/app/pages/AgentChat/ChatInput.tsx index 837bed12..7835380a 100644 --- a/frontend/src/app/pages/AgentChat/ChatInput.tsx +++ b/frontend/src/app/pages/AgentChat/ChatInput.tsx @@ -302,7 +302,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, lines.push(`${i + 1}. [Browser Card] ${title}`); lines.push(` browser_id: ${el.semanticData.selectId}`); if (url) lines.push(` URL: ${url}`); - lines.push(` (Use BrowserAgent with this browser_id to interact with it)`); + lines.push(` (Use BrowserAgent with this browser_id to interact with it, or CreateBrowserAgent for a new browser)`); } else if (el.semanticType && el.semanticData) { const typeLabel = { 'agent-card': 'Agent Card', @@ -321,6 +321,9 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, .map(([k, v]) => `${k}: ${typeof v === 'string' ? v : JSON.stringify(v)}`) .join(', '); if (metaStr) lines.push(` ${metaStr}`); + if (el.semanticType === 'agent-card' && selectId) { + lines.push(` (Use InvokeAgent with session_id "${selectId}" to query this agent with full conversation context)`); + } } else { const styleStr = Object.entries(el.computedStyles) .map(([k, v]) => `${k}: ${v}`) diff --git a/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx b/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx index 0fb31fb5..c457c942 100644 --- a/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx +++ b/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx @@ -3,6 +3,7 @@ import Box from '@mui/material/Box'; import Typography from '@mui/material/Typography'; import Collapse from '@mui/material/Collapse'; import IconButton from '@mui/material/IconButton'; +import Tooltip from '@mui/material/Tooltip'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import ExpandLessIcon from '@mui/icons-material/ExpandLess'; import TerminalIcon from '@mui/icons-material/Terminal'; @@ -15,9 +16,12 @@ import FolderIcon from '@mui/icons-material/Folder'; import AttachFileIcon from '@mui/icons-material/AttachFile'; import SearchIcon from '@mui/icons-material/Search'; import SendIcon from '@mui/icons-material/Send'; +import CallSplitIcon from '@mui/icons-material/CallSplit'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; -import { AgentMessage } from '@/shared/state/agentsSlice'; +import { AgentMessage, expandSession } from '@/shared/state/agentsSlice'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; +import { placeCard, setGlowingAgentCard, DEFAULT_CARD_W, DEFAULT_CARD_H, GRID_GAP } from '@/shared/state/dashboardLayoutSlice'; import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; import BrowserAgentInlineFeed from './BrowserAgentInlineFeed'; @@ -1172,15 +1176,57 @@ const McpResultCard: React.FC<{ parsed: ParsedMcpResult; compact?: boolean }> = }; function isBrowserAgentTool(name: string): boolean { - if (name === 'BrowserAgent' || name === 'BrowserAgents') return true; + if (name === 'CreateBrowserAgent' || name === 'BrowserAgent' || name === 'BrowserAgents') return true; const mcp = parseMcpToolName(name); return mcp.isMcp && mcp.serverSlug === 'openswarm-browser-agent'; } +function isInvokeAgentTool(name: string): boolean { + if (name === 'InvokeAgent') return true; + const mcp = parseMcpToolName(name); + return mcp.isMcp && mcp.serverSlug === 'openswarm-invoke-agent'; +} + +function parseInvokedSessionId(rawText: string): string | null { + const match = rawText.match(/\(forked session:\s*([a-f0-9]+)\)/); + return match ? match[1] : null; +} + +interface InvokeAgentParsed { + agentName: string; + sessionId: string | null; + cost: string | null; + response: string; +} + +function parseInvokeAgentResult(rawText: string): InvokeAgentParsed | null { + const headerMatch = rawText.match( + /\*\*Invoked Agent Result\*\*(?:\s*—\s*(.+?))?\s*\(forked session:\s*([a-f0-9]+)\)/, + ); + if (!headerMatch) return null; + + const agentName = headerMatch[1]?.trim() || 'Agent'; + const sessionId = headerMatch[2]; + + const costMatch = rawText.match(/\*Cost:\s*\$([0-9.]+)\*/); + const cost = costMatch ? costMatch[1] : null; + + const bodyStart = rawText.indexOf('\n\n'); + let response = bodyStart >= 0 ? rawText.slice(bodyStart + 2).trim() : ''; + if (response.startsWith('*Cost:')) { + const afterCost = response.indexOf('\n'); + response = afterCost >= 0 ? response.slice(afterCost + 1).trim() : ''; + } + + return { agentName, sessionId, cost, response }; +} + const ToolCallBubble: React.FC = React.memo( ({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false, sessionId }) => { const c = useClaudeTokens(); const tc = useTermColors(); + const dispatch = useAppDispatch(); + const cards = useAppSelector((s) => s.dashboardLayout.cards); const [expanded, setExpanded] = useState(false); const { toolName, input, isDenied } = getToolData(call); @@ -1190,6 +1236,7 @@ const ToolCallBubble: React.FC = React.memo( const showTimer = isPending && !isDenied && !isStreaming; const isBrowserAgent = isBrowserAgentTool(toolName); + const isInvokeAgent = isInvokeAgentTool(toolName); const browserAgentAutoExpand = isBrowserAgent && isPending && !isStreaming; const showBody = expanded || isStreaming || browserAgentAutoExpand; @@ -1217,6 +1264,38 @@ const ToolCallBubble: React.FC = React.memo( (parsedResult?.type === 'bash' && parsedResult.exitCode !== null && parsedResult.exitCode !== 0) || (parsedResult?.type === 'text' && parsedResult.isError); + const invokedSessionId = useMemo( + () => (isInvokeAgent && result ? parseInvokedSessionId(resultRawText) : null), + [isInvokeAgent, result, resultRawText], + ); + + const invokeAgentParsed = useMemo( + () => (isInvokeAgent && result ? parseInvokeAgentResult(resultRawText) : null), + [isInvokeAgent, result, resultRawText], + ); + + const handleRevealInvokedAgent = useCallback( + (e: React.MouseEvent) => { + e.stopPropagation(); + if (!invokedSessionId || !sessionId) return; + const parentCard = cards[sessionId]; + const targetX = parentCard + ? parentCard.x + parentCard.width + GRID_GAP * 3 + : 40; + const targetY = parentCard ? parentCard.y : 100; + dispatch(placeCard({ + sessionId: invokedSessionId, + x: targetX, + y: targetY, + width: DEFAULT_CARD_W, + height: DEFAULT_CARD_H, + })); + dispatch(expandSession(invokedSessionId)); + dispatch(setGlowingAgentCard({ sessionId: invokedSessionId, sourceId: sessionId })); + }, + [invokedSessionId, sessionId, cards, dispatch], + ); + const toggle = useCallback(() => { if (!isStreaming) setExpanded((v) => !v); }, [isStreaming]); @@ -1244,6 +1323,213 @@ const ToolCallBubble: React.FC = React.memo( 'data-select-meta': JSON.stringify({ tool: toolName, inputSummary }), }; + if (isInvokeAgent) { + const agentName = invokeAgentParsed?.agentName || input?.session_id || 'Agent'; + const responsePreview = invokeAgentParsed?.response || ''; + const costLabel = invokeAgentParsed?.cost ? `$${invokeAgentParsed.cost}` : null; + const hasResponse = !!invokeAgentParsed; + + return ( + + + + {/* Header */} + + + + InvokeAgent + + + + {agentName} + + + + {!hasResponse && !showTimer && } + + {hasResponse && responsePreview && !expanded && ( + + {responsePreview.slice(0, 100)}{responsePreview.length > 100 ? '…' : ''} + + )} + {expanded && } + + {isDenied && ( + + + denied + + )} + + {hasResponse && !isDenied && ( + + {isError ? ( + + ) : ( + + )} + {resultElapsedMs != null && ( + + {formatElapsed(resultElapsedMs)} + + )} + {costLabel && ( + + {costLabel} + + )} + + )} + + {showTimer && } + + {invokedSessionId && ( + + + + + + )} + + {hasResponse && ( + + {expanded ? : } + + )} + + + {/* Expanded body — markdown rendered, not terminal */} + + + ( + {children} + ), + }} + > + {responsePreview} + + + + + + ); + } + if (mcpCompact && mcpInfo.isMcp) { return ( diff --git a/frontend/src/app/pages/Dashboard/Dashboard.tsx b/frontend/src/app/pages/Dashboard/Dashboard.tsx index 5bae1868..0344662b 100644 --- a/frontend/src/app/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/app/pages/Dashboard/Dashboard.tsx @@ -308,7 +308,7 @@ const DashboardInner: React.FC = () => { useEffect(() => { if (!layoutInitialized) return; const dashboardSessionIds = Object.values(sessions) - .filter((s) => s.dashboard_id === dashboardId && s.mode !== 'browser-agent') + .filter((s) => s.dashboard_id === dashboardId && s.mode !== 'browser-agent' && s.mode !== 'invoked-agent') .map((s) => s.id); const liveIds = dashboardSessionIds.sort().join(','); if (liveIds === prevSessionIdsRef.current) return; diff --git a/frontend/src/app/pages/Settings/Settings.tsx b/frontend/src/app/pages/Settings/Settings.tsx index bae49d38..ee798ba6 100644 --- a/frontend/src/app/pages/Settings/Settings.tsx +++ b/frontend/src/app/pages/Settings/Settings.tsx @@ -736,6 +736,24 @@ const Settings: React.FC = () => { /> + {/* ── Advanced ── */} + Advanced + + + + Developer mode + Show transport details, environment variables, raw configs, and other technical metadata throughout the app. + + setForm({ ...form, dev_mode: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {/* ── About ── */} About diff --git a/frontend/src/app/pages/Tools/Tools.tsx b/frontend/src/app/pages/Tools/Tools.tsx index 2bdac126..e2c8ea32 100644 --- a/frontend/src/app/pages/Tools/Tools.tsx +++ b/frontend/src/app/pages/Tools/Tools.tsx @@ -56,6 +56,7 @@ import BlockIcon from '@mui/icons-material/Block'; import VisibilityIcon from '@mui/icons-material/Visibility'; import SecurityIcon from '@mui/icons-material/Security'; import PanToolIcon from '@mui/icons-material/PanTool'; +import CallSplitIcon from '@mui/icons-material/CallSplit'; import RefreshIcon from '@mui/icons-material/Refresh'; import { fetchTools, @@ -74,6 +75,8 @@ import { import { searchRegistry, fetchRegistryStats, + fetchServerDetail, + clearDetail, McpServer, } from '@/shared/state/mcpRegistrySlice'; import { @@ -156,7 +159,7 @@ const INTEGRATIONS: Integration[] = [ }, ]; -const CATEGORY_ORDER = ['filesystem', 'system', 'search', 'interaction', 'planning', 'scheduling']; +const CATEGORY_ORDER = ['filesystem', 'system', 'search', 'interaction', 'agents', 'planning', 'scheduling']; interface ToolForm { name: string; @@ -235,6 +238,7 @@ const ToolSection: React.FC = ({ interaction: { label: 'Interaction', color: '#a855f7', icon: }, planning: { label: 'Planning', color: '#ec4899', icon: }, scheduling: { label: 'Scheduling', color: '#14b8a6', icon: }, + agents: { label: 'Agents', color: '#f97316', icon: }, }; const PermToggle = ({ value, onChange, size = 16 }: { value: string; onChange: (v: string) => void; size?: number }) => ( @@ -343,7 +347,7 @@ const ToolSection: React.FC = ({ return ( - {bt.name} + {bt.display_name || bt.name} {bt.description && {firstSentence(bt.description)}} onPermissionChange(bt.name, v)} size={14} /> @@ -370,7 +374,8 @@ const Tools: React.FC = () => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); const { items, builtinTools, builtinPermissions, loading } = useAppSelector((s) => s.tools); - const { servers: regServers, total: regTotal, loading: regLoading, stats: regStats } = useAppSelector((s) => s.mcpRegistry); + const { servers: regServers, total: regTotal, loading: regLoading, stats: regStats, detail: regDetail, detailLoading: regDetailLoading } = useAppSelector((s) => s.mcpRegistry); + const devMode = useAppSelector((s) => s.settings.data.dev_mode); const outputItems = useAppSelector((s) => s.outputs.items); const outputs = useMemo(() => Object.values(outputItems), [outputItems]); const allTools = Object.values(items); @@ -531,6 +536,7 @@ const Tools: React.FC = () => { }; const [expandedServices, setExpandedServices] = useState>({}); + const [expandedSchema, setExpandedSchema] = useState(null); const [viewsSectionOpen, setViewsSectionOpen] = useState(false); const [browserSectionOpen, setBrowserSectionOpen] = useState(false); @@ -1083,7 +1089,7 @@ const Tools: React.FC = () => { return ( - {bt.name} + {bt.display_name || bt.name} {bt.description && {bt.description}} e.stopPropagation()}> @@ -1131,7 +1137,7 @@ const Tools: React.FC = () => { return ( - {bt.name} + {bt.display_name || bt.name} {bt.description && {bt.description}} e.stopPropagation()}> @@ -1220,6 +1226,7 @@ const Tools: React.FC = () => { const perms = tool.tool_permissions || {}; const services = perms._services as Record | undefined; const descriptions = (perms._tool_descriptions || {}) as Record; + const schemas = (perms._tool_schemas || {}) as Record; const serviceNames = services ? Object.keys(services) : []; const hasPerms = serviceNames.length > 0; const totalToolCount = serviceNames.reduce((acc, s) => acc + (services![s].read?.length || 0) + (services![s].write?.length || 0), 0); @@ -1292,15 +1299,36 @@ const Tools: React.FC = () => { handleGroupPermissionChange(tool.id, data.read!, v)} size={14} /> - {data.read!.map((name) => ( - - - {toDisplayName(name, serviceName)} - {descriptions[name] && {firstSentence(descriptions[name])}} + {data.read!.map((name) => { + const schemaKey = `${tool.id}:${name}`; + const schema = schemas[name]; + const schemaProps = schema?.properties as Record | undefined; + const schemaRequired = (schema?.required || []) as string[]; + return ( + + devMode && schema && setExpandedSchema((p) => p === schemaKey ? null : schemaKey)}> + + {toDisplayName(name, serviceName)} + {descriptions[name] && {firstSentence(descriptions[name])}} + + handlePermissionChange(tool.id, name, v)} size={14} /> + + {devMode && expandedSchema === schemaKey && schemaProps && ( + + Input Parameters + {Object.entries(schemaProps).map(([pName, pDef]: [string, any]) => ( + + {pName} + {pDef?.type || 'any'} + {schemaRequired.includes(pName) && } + {pDef?.description && {pDef.description}} + + ))} + + )} - handlePermissionChange(tool.id, name, v)} size={14} /> - - ))} + ); + })} )} {(data.write?.length || 0) > 0 && ( @@ -1313,15 +1341,36 @@ const Tools: React.FC = () => { handleGroupPermissionChange(tool.id, data.write!, v)} size={14} /> - {data.write!.map((name) => ( - - - {toDisplayName(name, serviceName)} - {descriptions[name] && {firstSentence(descriptions[name])}} + {data.write!.map((name) => { + const schemaKey = `${tool.id}:${name}`; + const schema = schemas[name]; + const schemaProps = schema?.properties as Record | undefined; + const schemaRequired = (schema?.required || []) as string[]; + return ( + + devMode && schema && setExpandedSchema((p) => p === schemaKey ? null : schemaKey)}> + + {toDisplayName(name, serviceName)} + {descriptions[name] && {firstSentence(descriptions[name])}} + + handlePermissionChange(tool.id, name, v)} size={14} /> + + {devMode && expandedSchema === schemaKey && schemaProps && ( + + Input Parameters + {Object.entries(schemaProps).map(([pName, pDef]: [string, any]) => ( + + {pName} + {pDef?.type || 'any'} + {schemaRequired.includes(pName) && } + {pDef?.description && {pDef.description}} + + ))} + + )} - handlePermissionChange(tool.id, name, v)} size={14} /> - - ))} + ); + })} )} @@ -1491,6 +1540,46 @@ const Tools: React.FC = () => { ))} )} + + {devMode && isMcp && ( + + + Developer Info + + + + MCP Config + + + {JSON.stringify(tool.mcp_config, null, 2)} + + + + + Auth type: + {tool.auth_type || 'none'} + + + Status: + {tool.auth_status || 'none'} + + {tool.connected_account_email && ( + + Account: + {tool.connected_account_email} + + )} + + {tool.credentials && Object.keys(tool.credentials).length > 0 && ( + + Credentials: + {Object.keys(tool.credentials).map((key) => ( + + ))} + + )} + + )} @@ -1527,17 +1616,24 @@ const Tools: React.FC = () => { MCP Registry {regStats && ( - + <> + + {devMode && regStats.lastUpdated > 0 && ( + + Synced {Math.round((Date.now() / 1000 - regStats.lastUpdated) / 60)}m ago + + )} + )} { {regServers.map((srv) => { const isExpanded = expandedServer === srv.name; + const isInstalled = allTools.some((t) => t.name === (srv.title || cleanServerName(srv.name))); return ( setExpandedServer(isExpanded ? null : srv.name)} + onClick={() => { + const next = isExpanded ? null : srv.name; + setExpandedServer(next); + if (next && devMode) { + dispatch(clearDetail()); + dispatch(fetchServerDetail(srv.name)); + } + }} sx={{ display: 'flex', alignItems: 'center', gap: 1.5, px: 1.5, py: 1, borderRadius: 1.5, cursor: 'pointer', @@ -1665,6 +1769,12 @@ const Tools: React.FC = () => { )} + {devMode && !srv.remoteType && ( + + )} + {isInstalled && ( + } label="Installed" size="small" sx={{ bgcolor: `${c.status.success}15`, color: c.status.success, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, color: c.status.success } }} /> + )} {srv.description} @@ -1719,6 +1829,46 @@ const Tools: React.FC = () => { + {devMode && ( + + {regDetailLoading && expandedServer === srv.name ? ( + + ) : regDetail && regDetail.name === srv.name ? ( + + {(regDetail.keywords?.length > 0 || regDetail.license) && ( + + {regDetail.license && ( + + )} + {regDetail.keywords?.map((kw) => ( + + ))} + + )} + {regDetail.environmentVariables?.length > 0 && ( + + + Required Environment Variables + + {regDetail.environmentVariables.map((ev) => ( + + + {ev.name} + + {ev.description && ( + + {ev.description} + + )} + + ))} + + )} + + ) : null} + + )} + - + + + {editingIsBuiltin && ( + + + + + + )} + + + + + diff --git a/frontend/src/app/pages/Views/ViewEditor.tsx b/frontend/src/app/pages/Views/ViewEditor.tsx index 8a9c2ac8..75ea08c7 100644 --- a/frontend/src/app/pages/Views/ViewEditor.tsx +++ b/frontend/src/app/pages/Views/ViewEditor.tsx @@ -381,7 +381,7 @@ interface FileTreeItemProps { c: ReturnType; } -const PROTECTED_FILES = new Set(['index.html', 'schema.json', 'meta.json']); +const PROTECTED_FILES = new Set(['index.html', 'schema.json', 'meta.json', 'SKILL.md']); const FileTreeItem: React.FC = ({ node, depth, activeFile, onSelect, onDelete, c }) => { const [open, setOpen] = useState(true); @@ -732,6 +732,7 @@ const ViewEditor: React.FC = ({ output, onClose }) => { const outputFiles = { ...files }; delete outputFiles['meta.json']; delete outputFiles['schema.json']; + delete outputFiles['SKILL.md']; return { name: name || 'Untitled App', @@ -948,7 +949,7 @@ const ViewEditor: React.FC = ({ output, onClose }) => { ? `${SERVE_BASE}/workspace/${workspaceId}/serve/index.html` : undefined; - const filePaths = useMemo(() => Object.keys(files).filter(p => p !== 'meta.json').sort(), [files]); + const filePaths = useMemo(() => Object.keys(files).filter(p => p !== 'meta.json' && p !== 'SKILL.md').sort(), [files]); const fileTree = useMemo(() => buildFileTree(filePaths), [filePaths]); const updateFile = useCallback((path: string, content: string) => { diff --git a/frontend/src/shared/state/modesSlice.ts b/frontend/src/shared/state/modesSlice.ts index e002ff10..9a294d64 100644 --- a/frontend/src/shared/state/modesSlice.ts +++ b/frontend/src/shared/state/modesSlice.ts @@ -18,18 +18,19 @@ export interface Mode { interface ModesState { items: Record; + builtinDefaults: Record; loading: boolean; loaded: boolean; } -const initialState: ModesState = { items: {}, loading: false, loaded: false }; +const initialState: ModesState = { items: {}, builtinDefaults: {}, loading: false, loaded: false }; export const fetchModes = createAsyncThunk( 'modes/fetch', async () => { const res = await fetch(`${MODES_API}/list`); const data = await res.json(); - return data.modes as Mode[]; + return { modes: data.modes as Mode[], builtinDefaults: (data.builtin_defaults ?? {}) as Record }; }, { condition: (_, { getState }) => !(getState() as { modes: ModesState }).modes.loading }, ); @@ -60,6 +61,15 @@ export const updateMode = createAsyncThunk( } ); +export const resetMode = createAsyncThunk( + 'modes/reset', + async (id: string) => { + const res = await fetch(`${MODES_API}/${id}/reset`, { method: 'POST' }); + const data = await res.json(); + return data.mode as Mode; + } +); + export const deleteMode = createAsyncThunk('modes/delete', async (id: string) => { await fetch(`${MODES_API}/${id}`, { method: 'DELETE' }); return id; @@ -76,11 +86,13 @@ const modesSlice = createSlice({ state.loading = false; state.loaded = true; state.items = {}; - for (const m of action.payload) state.items[m.id] = m; + for (const m of action.payload.modes) state.items[m.id] = m; + state.builtinDefaults = action.payload.builtinDefaults; }) .addCase(fetchModes.rejected, (state) => { state.loading = false; state.loaded = true; }) .addCase(createMode.fulfilled, (state, action) => { state.items[action.payload.id] = action.payload; }) .addCase(updateMode.fulfilled, (state, action) => { state.items[action.payload.id] = action.payload; }) + .addCase(resetMode.fulfilled, (state, action) => { state.items[action.payload.id] = action.payload; }) .addCase(deleteMode.fulfilled, (state, action) => { delete state.items[action.payload]; }); }, }); From 03c1cb1c910c100cefc847f49d48216eab2e45a7 Mon Sep 17 00:00:00 2001 From: haikdc Date: Fri, 20 Mar 2026 12:25:49 -0700 Subject: [PATCH 11/11] [Haik]: ckpt, auto spawn sub agent feature done and toggleable via settings --- backend/apps/settings/models.py | 1 + .../src/app/pages/Dashboard/Dashboard.tsx | 97 +++++++++++++++++++ frontend/src/app/pages/Settings/Settings.tsx | 17 +++- frontend/src/shared/state/settingsSlice.ts | 2 + 4 files changed, 116 insertions(+), 1 deletion(-) diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index bdd41e0c..e2e3061f 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -23,4 +23,5 @@ class AppSettings(BaseModel): browser_homepage: str = "https://www.google.com" auto_select_mode_on_new_agent: bool = False expand_new_chats_in_dashboard: bool = False + auto_reveal_sub_agents: bool = True dev_mode: bool = False diff --git a/frontend/src/app/pages/Dashboard/Dashboard.tsx b/frontend/src/app/pages/Dashboard/Dashboard.tsx index f6dab14e..064e12b4 100644 --- a/frontend/src/app/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/app/pages/Dashboard/Dashboard.tsx @@ -34,7 +34,9 @@ import { removeBrowserCard, pasteBrowserCard, placeCard, + removeCard, setGlowingAgentCard, + clearGlowingAgentCard, DEFAULT_CARD_W, DEFAULT_CARD_H, EXPANDED_CARD_MIN_H, @@ -97,6 +99,7 @@ const DashboardInner: React.FC = () => { const newAgentShortcut = useAppSelector((state) => state.settings.data.new_agent_shortcut); const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage); const expandNewChats = useAppSelector((state) => state.settings.data.expand_new_chats_in_dashboard); + const autoRevealSubAgents = useAppSelector((state) => state.settings.data.auto_reveal_sub_agents); const outputs = useAppSelector((state) => state.outputs.items); const glowingAgentCards = useAppSelector((state) => state.dashboardLayout.glowingAgentCards); const sessionList = Object.values(sessions); @@ -357,6 +360,100 @@ const DashboardInner: React.FC = () => { dispatch(reconcileSessions({ sessionIds: dashboardSessionIds, expandedSessionIds })); }, [sessions, layoutInitialized, dispatch, dashboardId, expandedSessionIds]); + // ---- Auto-reveal / collapse / unreveal sub-agent cards ---- + const autoRevealedRef = useRef(new Set()); + const prevSubStatusRef = useRef>({}); + const prevParentStatusRef = useRef>({}); + + useEffect(() => { + if (!layoutInitialized || !autoRevealSubAgents) return; + + const subSessions = Object.values(sessions).filter( + (s) => (s.mode === 'sub-agent' || s.mode === 'invoked-agent') && s.parent_session_id, + ); + + // 1) Auto-reveal newly spawned sub-agents (skip already-terminal ones on load) + for (const sub of subSessions) { + if (autoRevealedRef.current.has(sub.id)) continue; + if (cards[sub.id]) { + autoRevealedRef.current.add(sub.id); + continue; + } + const parentCard = cards[sub.parent_session_id!]; + if (!parentCard) continue; + + const isTerminal = sub.status === 'completed' || sub.status === 'error' || sub.status === 'stopped'; + const parentSession = sessions[sub.parent_session_id!]; + const parentTerminal = parentSession && + (parentSession.status === 'completed' || parentSession.status === 'error' || parentSession.status === 'stopped'); + if (isTerminal && parentTerminal) { + autoRevealedRef.current.add(sub.id); + continue; + } + + autoRevealedRef.current.add(sub.id); + + const targetX = parentCard.x + parentCard.width + GRID_GAP * 12; + let targetY = parentCard.y; + const columnCards = Object.values(cards).filter( + (c) => Math.abs(c.x - targetX) < 50 && c.session_id !== sub.id, + ); + if (columnCards.length > 0) { + const lowestBottom = Math.max( + ...columnCards.map((c) => c.y + Math.max(EXPANDED_CARD_MIN_H, c.height)), + ); + targetY = lowestBottom + GRID_GAP; + } + + dispatch(placeCard({ sessionId: sub.id, x: targetX, y: targetY, width: DEFAULT_CARD_W, height: DEFAULT_CARD_H })); + dispatch(expandSession(sub.id)); + const label = sub.mode === 'sub-agent' ? 'Create Agent' : 'Invoke Agent'; + dispatch(setGlowingAgentCard({ sessionId: sub.id, sourceId: sub.parent_session_id!, label })); + + if (sub.status === 'completed' || sub.status === 'error' || sub.status === 'stopped') { + const subId = sub.id; + setTimeout(() => dispatch(collapseSession(subId)), 2000); + } + } + + // 2) Auto-collapse sub-agents when they complete + const TERMINAL = new Set(['completed', 'error', 'stopped']); + for (const sub of subSessions) { + const prev = prevSubStatusRef.current[sub.id]; + if (prev !== sub.status && TERMINAL.has(sub.status) && cards[sub.id]) { + dispatch(collapseSession(sub.id)); + } + } + const newSubStatuses: Record = {}; + for (const sub of subSessions) { newSubStatuses[sub.id] = sub.status; } + prevSubStatusRef.current = newSubStatuses; + + // 3) Unreveal all sub-agent cards when parent finishes output + const parentIds = new Set(subSessions.map((s) => s.parent_session_id!)); + for (const pid of parentIds) { + const parent = sessions[pid]; + if (!parent) continue; + const prev = prevParentStatusRef.current[pid]; + if (prev !== parent.status && TERMINAL.has(parent.status)) { + const children = subSessions.filter((s) => s.parent_session_id === pid); + for (const child of children) { + if (!cards[child.id]) continue; + dispatch(collapseSession(child.id)); + dispatch(removeCard(child.id)); + setTimeout(() => { + dispatch(clearGlowingAgentCard(child.id)); + }, 500); + } + } + } + const newParentStatuses: Record = {}; + for (const pid of parentIds) { + const parent = sessions[pid]; + if (parent) newParentStatuses[pid] = parent.status; + } + prevParentStatusRef.current = newParentStatuses; + }, [sessions, cards, layoutInitialized, autoRevealSubAgents, dispatch]); + const skipInitialSave = useRef(true); const saveTimerRef = useRef | null>(null); const pendingSaveRef = useRef[0] | null>(null); diff --git a/frontend/src/app/pages/Settings/Settings.tsx b/frontend/src/app/pages/Settings/Settings.tsx index 06acc65c..79f743bc 100644 --- a/frontend/src/app/pages/Settings/Settings.tsx +++ b/frontend/src/app/pages/Settings/Settings.tsx @@ -569,7 +569,7 @@ const Settings: React.FC = () => { /> - + Default agent spawn state in dashboard When enabled, new agents spawn expanded instead of collapsed. @@ -584,6 +584,21 @@ const Settings: React.FC = () => { /> + + + Auto-reveal sub-agents on dashboard + Automatically show sub-agent cards (from CreateAgent / InvokeAgent) tethered to their parent on the dashboard. + + setForm({ ...form, auto_reveal_sub_agents: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {/* ── Browser ── */} Browser diff --git a/frontend/src/shared/state/settingsSlice.ts b/frontend/src/shared/state/settingsSlice.ts index a216149f..1f5921cf 100644 --- a/frontend/src/shared/state/settingsSlice.ts +++ b/frontend/src/shared/state/settingsSlice.ts @@ -24,6 +24,7 @@ export interface AppSettings { browser_homepage: string; auto_select_mode_on_new_agent: boolean; expand_new_chats_in_dashboard: boolean; + auto_reveal_sub_agents: boolean; dev_mode: boolean; } @@ -55,6 +56,7 @@ const initialState: SettingsState = { browser_homepage: 'https://www.google.com', auto_select_mode_on_new_agent: false, expand_new_chats_in_dashboard: false, + auto_reveal_sub_agents: true, dev_mode: false, }, loading: false,