[Haik]: (Browser actions shown in agent card output) (User can have item selection turned on by default in settings) (most browser related functionality works - altho spotify is tricky) (mcp discovery in prod shld worke now?)

This commit is contained in:
haikdc
2026-03-17 01:48:50 -07:00
parent 99f741c453
commit 6f02daff2d
29 changed files with 947 additions and 46 deletions
+1
View File
@@ -9,6 +9,7 @@ backend/data/**
electron/dist/
electron/python-env/
electron/node_modules/
electron/package-lock.json
# Frontend build output
frontend/dist/
+19
View File
@@ -621,6 +621,7 @@ class AgentManager:
"OPENSWARM_AGENT_MODEL": session.model,
"OPENSWARM_DASHBOARD_ID": session.dashboard_id or "",
"OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids),
"OPENSWARM_PARENT_SESSION_ID": session.id,
},
"type": "stdio",
}
@@ -1401,4 +1402,22 @@ class AgentManager:
def get_session(self, session_id: str) -> Optional[AgentSession]:
return self.sessions.get(session_id)
def get_browser_agent_children(self, parent_session_id: str) -> list[dict]:
"""Return browser-agent sessions for a parent, from memory or disk."""
results: list[dict] = []
seen: set[str] = set()
for s in self.sessions.values():
if s.mode == "browser-agent" and s.parent_session_id == parent_session_id:
results.append(s.model_dump(mode="json"))
seen.add(s.id)
for sid, data in _load_all_session_data():
if sid in seen:
continue
if data.get("mode") == "browser-agent" and data.get("parent_session_id") == parent_session_id:
results.append(data)
return results
agent_manager = AgentManager()
+5
View File
@@ -151,6 +151,11 @@ async def get_history(q: str = "", limit: int = 20, offset: int = 0, dashboard_i
dashboard_id=dashboard_id or None,
)
@agents.router.get("/sessions/{session_id}/browser-agents")
async def get_browser_agent_children(session_id: str):
children = agent_manager.get_browser_agent_children(session_id)
return {"sessions": children}
@agents.router.post("/sessions/{session_id}/resume")
async def resume_session(session_id: str):
try:
+4
View File
@@ -188,6 +188,7 @@ async def run_browser_agent(
tab_id: str = "",
pre_selected: bool = False,
initial_url: str | None = None,
parent_session_id: str | None = None,
) -> dict:
"""Run a browser sub-agent loop for a single browser card.
@@ -206,6 +207,7 @@ async def run_browser_agent(
dashboard_id=dashboard_id,
browser_id=browser_id,
system_prompt=SYSTEM_PROMPT,
parent_session_id=parent_session_id,
)
agent_manager.sessions[session_id] = session
@@ -425,6 +427,7 @@ async def run_browser_agents(
api_key: str,
dashboard_id: str | None = None,
pre_selected_browser_ids: list[str] | None = None,
parent_session_id: str | None = None,
) -> list[dict]:
"""Run multiple browser sub-agents in parallel.
@@ -451,6 +454,7 @@ async def run_browser_agents(
dashboard_id=dashboard_id,
pre_selected=is_pre_selected,
initial_url=url if url and browser_id not in pre_selected else None,
parent_session_id=parent_session_id,
)
results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True)
@@ -25,6 +25,7 @@ BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/browser-agent/run"
MODEL = os.environ.get("OPENSWARM_AGENT_MODEL", "sonnet")
DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "")
PRE_SELECTED_BROWSER_IDS = os.environ.get("OPENSWARM_PRE_SELECTED_BROWSER_IDS", "")
PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "")
TOOLS = [
{
@@ -119,6 +120,7 @@ def call_backend(tasks: list[dict]) -> dict:
"model": MODEL,
"dashboard_id": DASHBOARD_ID,
"pre_selected_browser_ids": pre_selected,
"parent_session_id": PARENT_SESSION_ID,
}).encode()
req = urllib.request.Request(
BACKEND_URL,
+1
View File
@@ -72,3 +72,4 @@ class AgentSession(BaseModel):
tool_group_meta: dict[str, "ToolGroupMeta"] = Field(default_factory=dict)
dashboard_id: Optional[str] = None
browser_id: Optional[str] = None
parent_session_id: Optional[str] = None
+1
View File
@@ -21,3 +21,4 @@ class AppSettings(BaseModel):
new_agent_shortcut: str = "Meta+l"
anthropic_api_key: Optional[str] = None
browser_homepage: str = "https://www.google.com"
auto_select_mode_on_new_agent: bool = False
+3 -1
View File
@@ -18,9 +18,11 @@ from backend.apps.tools_lib.models import ToolDefinition, ToolCreate, ToolUpdate
logger = logging.getLogger(__name__)
from backend.config.paths import BACKEND_DIR, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH
from backend.config.paths import BACKEND_DIR, DATA_ROOT, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH
load_dotenv(os.path.join(BACKEND_DIR, ".env"))
if os.environ.get("OPENSWARM_PACKAGED") == "1":
load_dotenv(os.path.join(os.path.dirname(DATA_ROOT), ".env"), override=True)
@asynccontextmanager
+2
View File
@@ -126,6 +126,7 @@ async def browser_agent_run(request: Request):
model = body.get("model", "sonnet")
dashboard_id = body.get("dashboard_id", "")
pre_selected_browser_ids = body.get("pre_selected_browser_ids", [])
parent_session_id = body.get("parent_session_id", "")
if not tasks:
return JSONResponse({"error": "tasks array is required"}, status_code=400)
@@ -140,6 +141,7 @@ async def browser_agent_run(request: Request):
api_key=settings.anthropic_api_key,
dashboard_id=dashboard_id or None,
pre_selected_browser_ids=pre_selected_browser_ids,
parent_session_id=parent_session_id or None,
)
return JSONResponse({"results": results})
+191 -12
View File
@@ -1,5 +1,6 @@
const { app, BrowserWindow, ipcMain, shell } = require('electron');
const { autoUpdater } = require('electron-updater');
const { app, components, BrowserWindow, ipcMain, shell, session } = require('electron');
let autoUpdater;
try { autoUpdater = require('electron-updater').autoUpdater; } catch (_) {}
const path = require('path');
const { spawn, execFileSync } = require('child_process');
const os = require('os');
@@ -7,6 +8,12 @@ const fs = require('fs');
const getPort = require('get-port');
const http = require('http');
app.commandLine.appendSwitch('disable-features', 'HardwareMediaKeyHandling');
app.commandLine.appendSwitch('ignore-gpu-blocklist');
app.commandLine.appendSwitch('enable-gpu-rasterization');
app.commandLine.appendSwitch('enable-zero-copy');
app.commandLine.appendSwitch('autoplay-policy', 'no-user-gesture-required');
let mainWindow = null;
let backendProcess = null;
let backendPort = null;
@@ -24,9 +31,10 @@ const iconPath = path.join(__dirname, 'build', 'icon.png');
function getShellPath() {
if (process.platform !== 'darwin' || isDev) return process.env.PATH || '';
// Strategy 1: ask the user's login shell for its PATH
try {
const shell = process.env.SHELL || '/bin/zsh';
const result = execFileSync(shell, ['-ilc', 'echo $PATH'], {
const userShell = process.env.SHELL || '/bin/zsh';
const result = execFileSync(userShell, ['-ilc', 'echo $PATH'], {
encoding: 'utf8',
timeout: 5000,
env: { ...process.env, HOME: os.homedir() },
@@ -35,19 +43,40 @@ function getShellPath() {
if (resolved) return resolved;
} catch (_) { /* fall through */ }
// Strategy 2: read macOS system PATH config (/etc/paths + /etc/paths.d/*)
const systemPaths = [];
try {
const base = fs.readFileSync('/etc/paths', 'utf8');
for (const line of base.split('\n')) {
const p = line.trim();
if (p) systemPaths.push(p);
}
} catch (_) { /* ignore */ }
try {
const pathsD = '/etc/paths.d';
if (fs.existsSync(pathsD)) {
for (const file of fs.readdirSync(pathsD).sort()) {
const content = fs.readFileSync(path.join(pathsD, file), 'utf8');
for (const line of content.split('\n')) {
const p = line.trim();
if (p) systemPaths.push(p);
}
}
}
} catch (_) { /* ignore */ }
// Strategy 3: well-known user-local bin directories
const home = os.homedir();
const fallbackDirs = [
path.join(home, '.nvm/versions/node'),
path.join(home, '.local/bin'),
path.join(home, '.volta/bin'),
path.join(home, '.fnm/aliases/default/bin'),
path.join(home, '.bun/bin'),
path.join(home, '.cargo/bin'),
path.join(home, '.local/bin'),
'/opt/homebrew/bin',
'/usr/local/bin',
];
// For nvm, resolve the current default version dynamically
const nvmDir = path.join(home, '.nvm/versions/node');
try {
if (fs.existsSync(nvmDir)) {
@@ -58,10 +87,14 @@ function getShellPath() {
}
} catch (_) { /* ignore */ }
const existing = fallbackDirs.filter((d) => {
try { return fs.statSync(d).isDirectory(); } catch { return false; }
});
return [...existing, process.env.PATH || ''].join(':');
const seen = new Set();
const dirs = [];
for (const d of [...fallbackDirs, ...systemPaths, ...(process.env.PATH || '').split(':')]) {
if (!d || seen.has(d)) continue;
seen.add(d);
try { if (fs.statSync(d).isDirectory()) dirs.push(d); } catch { /* skip */ }
}
return dirs.join(':');
}
function getResourcePath(...segments) {
@@ -190,6 +223,11 @@ function createWindow() {
mainWindow.loadFile(frontendPath);
}
mainWindow.webContents.on('will-attach-webview', (_event, webPreferences, _params) => {
webPreferences.plugins = true;
webPreferences.enableBlinkFeatures = 'EncryptedMedia';
});
mainWindow.on('closed', () => {
mainWindow = null;
});
@@ -202,6 +240,7 @@ function sendToRenderer(channel, ...args) {
}
function setupAutoUpdater() {
if (!autoUpdater) return;
autoUpdater.autoDownload = false;
autoUpdater.autoInstallOnAppQuit = false;
@@ -252,6 +291,68 @@ app.whenReady().then(async () => {
try { app.dock.setIcon(iconPath); } catch (_) {}
}
session.defaultSession.setPermissionRequestHandler((_wc, permission, callback) => {
const allowed = [
'media', 'mediaKeySystem', 'protected-media-identifier',
'geolocation', 'notifications', 'midi', 'midiSysex',
'clipboard-read', 'clipboard-sanitized-write',
'pointerLock', 'fullscreen', 'idle-detection',
];
console.log('Permission request:', permission, '->', allowed.includes(permission) ? 'granted' : 'denied');
callback(allowed.includes(permission));
});
session.defaultSession.setPermissionCheckHandler((_wc, permission) => {
const allowed = [
'media', 'mediaKeySystem', 'protected-media-identifier',
'clipboard-read', 'clipboard-sanitized-write',
'pointerLock', 'fullscreen', 'idle-detection',
];
return allowed.includes(permission);
});
// Read-only logging for DRM license requests — no modifying interceptors
// so the network stack can set Content-Type and other headers normally.
session.defaultSession.webRequest.onSendHeaders(
{ urls: ['*://*/*widevine*license*'] },
(details) => {
console.log(`[drm-req] ${details.method} ${details.url}`);
for (const [k, v] of Object.entries(details.requestHeaders || {})) {
if (/content-type|origin|referer|auth|accept/i.test(k)) {
console.log(`[drm-req] ${k}: ${v}`);
}
}
},
);
session.defaultSession.webRequest.onCompleted(
{ urls: ['*://*/*widevine*', '*://*/*license*'] },
(details) => {
console.log(`[drm-net] ${details.method} ${details.url}${details.statusCode}`);
},
);
session.defaultSession.webRequest.onErrorOccurred(
{ urls: ['*://*/*widevine*', '*://*/*license*'] },
(details) => {
console.log(`[drm-net] FAILED ${details.method} ${details.url}${details.error}`);
},
);
// Wait for the Widevine CDM to be downloaded/ready (CastLabs Component
// Updater Service). On first launch this downloads the CDM; subsequent
// launches use the cached version.
if (components && typeof components.whenReady === 'function') {
try {
await components.whenReady();
console.log('Widevine CDM ready');
if (typeof components.status === 'function') {
console.log('CDM component status:', JSON.stringify(components.status()));
}
} catch (err) {
console.warn('Widevine CDM not available:', err.message);
}
} else {
console.log('CastLabs components API not available — using standard Electron (no DRM)');
}
try {
if (isDev) {
backendPort = parseInt(process.env.OPENSWARM_PORT || '8324', 10);
@@ -269,6 +370,79 @@ app.whenReady().then(async () => {
}
});
app.on('web-contents-created', (_event, contents) => {
contents.setWindowOpenHandler(({ url, disposition }) => {
if (disposition === 'foreground-tab' || disposition === 'background-tab') {
if (mainWindow && !mainWindow.isDestroyed()) {
mainWindow.webContents.send('webview-new-window', url, contents.id);
}
return { action: 'deny' };
}
return {
action: 'allow',
overrideBrowserWindowOptions: {
parent: mainWindow || undefined,
},
};
});
contents.on('did-create-window', (childWindow) => {
if (mainWindow && !mainWindow.isDestroyed() && !childWindow.isDestroyed()) {
childWindow.setParentWindow(mainWindow);
}
});
if (contents.getType() === 'webview') {
contents.on('console-message', (_e, level, message, line, sourceId) => {
if (message.includes('widevine') || message.includes('drm') ||
message.includes('license') || message.includes('MediaKeySession') ||
message.includes('EME') || message.includes('[drm-diag]') || level >= 2) {
const tag = ['LOG', 'INFO', 'WARN', 'ERROR'][level] || 'LOG';
const src = sourceId ? sourceId.split('/').pop() : '';
console.log(`[webview:${tag}] ${message}${src ? ` (${src}:${line})` : ''}`);
}
});
contents.on('dom-ready', () => {
const url = contents.getURL();
if (url.includes('spotify')) {
contents.executeJavaScript(`
(function() {
const origFetch = window.fetch;
window.fetch = async function(...args) {
const resp = await origFetch.apply(this, args);
const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || '';
if (url.includes('widevine-license') && !resp.ok) {
const clone = resp.clone();
try {
const text = await clone.text();
console.log('[drm-diag] License response ' + resp.status + ': ' + text.substring(0, 500));
} catch(e) {}
}
return resp;
};
// Check EME availability
if (navigator.requestMediaKeySystemAccess) {
navigator.requestMediaKeySystemAccess('com.widevine.alpha', [{
initDataTypes: ['cenc'],
audioCapabilities: [{contentType: 'audio/mp4; codecs="mp4a.40.2"'}],
}]).then(function(access) {
console.log('[drm-diag] Widevine EME access: ' + access.keySystem);
}).catch(function(err) {
console.log('[drm-diag] Widevine EME FAILED: ' + err.message);
});
} else {
console.log('[drm-diag] EME API not available');
}
})();
`).catch(() => {});
}
});
}
});
app.on('window-all-closed', () => {
if (!isDev) killBackend();
app.quit();
@@ -286,9 +460,12 @@ app.on('activate', () => {
ipcMain.handle('get-backend-port', () => backendPort);
ipcMain.handle('get-app-version', () => app.getVersion());
ipcMain.handle('get-webview-preload-path', () => {
return `file://${path.join(__dirname, 'webview-preload.js')}`;
});
ipcMain.handle('check-for-updates', async () => {
if (!isPackaged) {
if (!autoUpdater || !isPackaged) {
sendToRenderer('update-error', 'Update check is only available in the packaged app.');
return { success: false, error: 'Not packaged' };
}
@@ -305,6 +482,7 @@ ipcMain.handle('check-for-updates', async () => {
});
ipcMain.handle('download-update', async () => {
if (!autoUpdater) return { success: false, error: 'Updater not available' };
try {
await autoUpdater.downloadUpdate();
return { success: true };
@@ -314,6 +492,7 @@ ipcMain.handle('download-update', async () => {
});
ipcMain.handle('install-update', () => {
if (!autoUpdater) return;
autoUpdater.quitAndInstall(false, true);
});
+12 -12
View File
@@ -1,19 +1,20 @@
{
"name": "openswarm",
"version": "1.0.3",
"version": "1.0.4",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "openswarm",
"version": "1.0.3",
"version": "1.0.4",
"hasInstallScript": true,
"dependencies": {
"electron-updater": "^6.3.0",
"get-port": "^5.1.1"
},
"devDependencies": {
"@electron/notarize": "^3.1.1",
"electron": "^33.0.0",
"electron": "castlabs/electron-releases#v33.4.11+wvcus",
"electron-builder": "^25.1.0"
}
},
@@ -2206,9 +2207,8 @@
}
},
"node_modules/electron": {
"version": "33.4.11",
"resolved": "https://registry.npmjs.org/electron/-/electron-33.4.11.tgz",
"integrity": "sha512-xmdAs5QWRkInC7TpXGNvzo/7exojubk+72jn1oJL7keNeIlw7xNglf8TGtJtkR4rWC5FJq0oXiIXPS9BcK2Irg==",
"version": "33.4.11+wvcus",
"resolved": "git+ssh://git@github.com/castlabs/electron-releases.git#d1cf58c11ec0a8a04f307ed362d7efde2816778d",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
@@ -3953,9 +3953,9 @@
}
},
"node_modules/node-abi": {
"version": "3.88.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.88.0.tgz",
"integrity": "sha512-At6b4UqIEVudaqPsXjmUO1r/N5BUr4yhDGs5PkBE8/oG5+TfLPhFechiskFsnT6Ql0VfUXbalUUCbfXxtj7K+w==",
"version": "3.89.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz",
"integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -4605,9 +4605,9 @@
}
},
"node_modules/sax": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.5.0.tgz",
"integrity": "sha512-21IYA3Q5cQf089Z6tgaUTr7lDAyzoTPx5HRtbhsME8Udispad8dC/+sziTNugOEx54ilvatQ9YCzl4KQLPcRHA==",
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz",
"integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==",
"license": "BlueOak-1.0.0",
"engines": {
"node": ">=11.0.0"
+7 -2
View File
@@ -1,11 +1,13 @@
{
"name": "openswarm",
"version": "1.0.3",
"version": "1.0.4",
"description": "OpenSwarm — AI Agent Orchestrator",
"main": "main.js",
"scripts": {
"start": "electron .",
"dev": "ELECTRON_DEV=1 electron .",
"postinstall": "bash scripts/sign-vmp.sh",
"sign-vmp": "bash scripts/sign-vmp.sh",
"dist": "electron-builder --mac --publish never",
"dist:publish": "electron-builder --mac --publish always",
"dist:all": "electron-builder --mac --win --linux"
@@ -16,12 +18,15 @@
},
"devDependencies": {
"@electron/notarize": "^3.1.1",
"electron": "^33.0.0",
"electron": "castlabs/electron-releases#v33.4.11+wvcus",
"electron-builder": "^25.1.0"
},
"build": {
"appId": "com.clusterlabs.openswarm",
"productName": "OpenSwarm",
"electronDownload": {
"mirror": "https://github.com/castlabs/electron-releases/releases/download/v"
},
"directories": {
"output": "dist"
},
+8
View File
@@ -2,11 +2,13 @@ const { contextBridge, ipcRenderer } = require('electron');
(async () => {
const port = await ipcRenderer.invoke('get-backend-port');
const webviewPreloadPath = await ipcRenderer.invoke('get-webview-preload-path');
contextBridge.exposeInMainWorld('__OPENSWARM_PORT__', port);
contextBridge.exposeInMainWorld('openswarm', {
getBackendPort: () => port,
getWebviewPreloadPath: () => webviewPreloadPath,
getAppVersion: () => ipcRenderer.invoke('get-app-version'),
openExternal: (url) => ipcRenderer.invoke('open-external', url),
@@ -40,5 +42,11 @@ const { contextBridge, ipcRenderer } = require('electron');
ipcRenderer.on('update-error', listener);
return () => ipcRenderer.removeListener('update-error', listener);
},
onWebviewNewWindow: (cb) => {
const listener = (_event, url, webContentsId) => cb(url, webContentsId);
ipcRenderer.on('webview-new-window', listener);
return () => ipcRenderer.removeListener('webview-new-window', listener);
},
});
})();
+61
View File
@@ -0,0 +1,61 @@
#!/bin/bash
# Signs the CastLabs Electron binary with a production VMP certificate via EVS,
# then repairs macOS framework symlinks that npm/signing may strip.
#
# First-time setup (one-time):
# pip3 install --user castlabs-evs
# python3 -m castlabs_evs.account signup
#
# After signup, this script runs automatically.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ELECTRON_DIR="$SCRIPT_DIR/../node_modules/electron/dist"
FW_BASE="$ELECTRON_DIR/Electron.app/Contents/Frameworks"
fix_framework_symlinks() {
[ -d "$FW_BASE" ] || return 0
for fw in "$FW_BASE"/*.framework; do
[ -d "$fw/Versions/A" ] || continue
local name
name=$(basename "$fw" .framework)
cd "$fw"
(cd Versions && ln -sf A Current 2>/dev/null)
ln -sf "Versions/Current/$name" "$name" 2>/dev/null
[ -d "Versions/A/Resources" ] && ln -sf Versions/Current/Resources Resources 2>/dev/null
[ -d "Versions/A/Libraries" ] && ln -sf Versions/Current/Libraries Libraries 2>/dev/null
[ -d "Versions/A/Helpers" ] && ln -sf Versions/Current/Helpers Helpers 2>/dev/null
done
}
if [ ! -d "$ELECTRON_DIR" ]; then
echo "[vmp] Electron dist not found at $ELECTRON_DIR — skipping VMP signing"
fix_framework_symlinks
exit 0
fi
# Always fix symlinks first (npm git installs strip them)
fix_framework_symlinks
if ! python3 -c "import castlabs_evs" 2>/dev/null; then
echo "[vmp] castlabs-evs not installed. Install with: pip3 install --user castlabs-evs"
echo "[vmp] Skipping VMP signing — DRM playback will be limited"
exit 0
fi
VERIFY_OUTPUT=$(python3 -m castlabs_evs.vmp verify-pkg "$ELECTRON_DIR" 2>&1)
if echo "$VERIFY_OUTPUT" | grep -q "Signature is valid" && ! echo "$VERIFY_OUTPUT" | grep -q "development only"; then
echo "[vmp] Electron already has a valid production VMP signature"
exit 0
fi
echo "[vmp] Signing Electron with production VMP certificate..."
if python3 -m castlabs_evs.vmp sign-pkg "$ELECTRON_DIR" 2>&1; then
echo "[vmp] VMP signing successful — full DRM playback enabled"
# Re-fix symlinks in case signing modified the bundle
fix_framework_symlinks
else
echo "[vmp] VMP signing failed — you may need to run: python3 -m castlabs_evs.account signup"
echo "[vmp] DRM playback will be limited to previews until signed"
fi
exit 0
+87
View File
@@ -0,0 +1,87 @@
/**
* Webview preload script — patches browser fingerprinting so sites like
* Spotify/Netflix don't detect an Electron shell and disable features.
* Loaded via the webview's `preload` attribute before any page script runs.
*/
'use strict';
// Hide webdriver flag
Object.defineProperty(navigator, 'webdriver', {
get: () => false,
configurable: true,
});
// Spoof navigator.plugins (Chrome has a few built-in ones)
const fakePlugins = {
0: { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
1: { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
2: { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
length: 3,
item: (i) => fakePlugins[i] || null,
namedItem: (name) => {
for (let i = 0; i < fakePlugins.length; i++) {
if (fakePlugins[i].name === name) return fakePlugins[i];
}
return null;
},
refresh: () => {},
[Symbol.iterator]: function* () {
for (let i = 0; i < this.length; i++) yield this[i];
},
};
try {
Object.defineProperty(navigator, 'plugins', {
get: () => fakePlugins,
configurable: true,
});
} catch (_) {}
// Ensure window.chrome exists (sites test for it)
if (!window.chrome) {
window.chrome = {};
}
if (!window.chrome.runtime) {
window.chrome.runtime = {
connect: () => {},
sendMessage: () => {},
onMessage: { addListener: () => {}, removeListener: () => {} },
};
}
// Ensure navigator.languages has sensible values
try {
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US', 'en'],
configurable: true,
});
} catch (_) {}
// Patch permissions.query to report 'granted' for common permissions
const originalQuery = navigator.permissions?.query?.bind(navigator.permissions);
if (originalQuery) {
navigator.permissions.query = (params) => {
if (params.name === 'notifications') {
return Promise.resolve({ state: 'granted', onchange: null });
}
return originalQuery(params).catch(() =>
Promise.resolve({ state: 'prompt', onchange: null })
);
};
}
// Prevent iframe detection heuristics
try {
Object.defineProperty(document, 'hidden', {
get: () => false,
configurable: true,
});
Object.defineProperty(document, 'visibilityState', {
get: () => 'visible',
configurable: true,
});
} catch (_) {}
// Fix console.debug detection (some sites use it as a breakpoint detector)
const noop = () => {};
if (!window.console.debug) window.console.debug = noop;
@@ -527,11 +527,11 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
{renderItems.map((item) => {
if (isToolGroup(item)) {
const groupMeta = session.tool_group_meta?.[item.id];
return <ToolGroupBubble key={item.id} group={item} isSessionRunning={sessionRunning} meta={groupMeta} />;
return <ToolGroupBubble key={item.id} group={item} isSessionRunning={sessionRunning} meta={groupMeta} sessionId={session.id} />;
}
if (isToolPair(item)) {
const isPending = item.result === null && sessionRunning;
return <ToolCallBubble key={item.id} call={item.call} result={item.result} isPending={isPending} />;
return <ToolCallBubble key={item.id} call={item.call} result={item.result} isPending={isPending} sessionId={session.id} />;
}
const msg = item;
const siblings = getSiblingBranches(msg.id);
@@ -566,6 +566,7 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
key={`streaming-${session.streamingMessage.id}`}
isStreaming
isPending
sessionId={session.id}
call={{
id: session.streamingMessage.id,
role: 'tool_call',
@@ -0,0 +1,392 @@
import React, { useEffect, useRef, useMemo } from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined';
import CheckCircleOutlineIcon from '@mui/icons-material/CheckCircleOutline';
import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline';
import LanguageIcon from '@mui/icons-material/Language';
import OpenInNewIcon from '@mui/icons-material/OpenInNew';
import TouchAppOutlinedIcon from '@mui/icons-material/TouchAppOutlined';
import KeyboardOutlinedIcon from '@mui/icons-material/KeyboardOutlined';
import CameraAltOutlinedIcon from '@mui/icons-material/CameraAltOutlined';
import ArticleOutlinedIcon from '@mui/icons-material/ArticleOutlined';
import AccountTreeOutlinedIcon from '@mui/icons-material/AccountTreeOutlined';
import CodeOutlinedIcon from '@mui/icons-material/CodeOutlined';
import BuildOutlinedIcon from '@mui/icons-material/BuildOutlined';
import { useAppSelector, useAppDispatch } from '@/shared/hooks';
import { AgentMessage, AgentSession, fetchBrowserAgentChildren } from '@/shared/state/agentsSlice';
import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext';
interface Props {
parentSessionId: string;
browserId?: string;
}
interface FeedEntry {
type: 'thought' | 'action' | 'result' | 'system';
text: string;
actionTool?: string;
sessionLabel?: string;
}
function formatMessage(msg: AgentMessage): FeedEntry | null {
if (msg.role === 'user') return null;
if (msg.role === 'assistant' && typeof msg.content === 'string') {
const trimmed = msg.content.trim();
if (!trimmed) return null;
return { type: 'thought', text: trimmed };
}
if (msg.role === 'tool_call') {
const content =
typeof msg.content === 'string'
? (() => { try { return JSON.parse(msg.content); } catch { return {}; } })()
: msg.content;
const tool = content?.tool || content?.name || '?';
const input = content?.input || {};
let brief = '';
switch (tool) {
case 'BrowserNavigate':
brief = `Navigate → ${input.url || '...'}`;
break;
case 'BrowserClick':
brief = `Click ${input.selector || '...'}`;
break;
case 'BrowserType': {
const txt = (input.text || '').slice(0, 40);
const ellipsis = (input.text || '').length > 40 ? '…' : '';
brief = `Type "${txt}${ellipsis}" into ${input.selector || '...'}`;
break;
}
case 'BrowserScreenshot':
brief = 'Screenshot';
break;
case 'BrowserGetText':
brief = 'Read page text';
break;
case 'BrowserGetElements':
brief = `Inspect elements${input.selector ? ` (${input.selector})` : ''}`;
break;
case 'BrowserEvaluate':
brief = `Evaluate JS`;
break;
default:
brief = `${tool}(${JSON.stringify(input).slice(0, 60)})`;
}
return { type: 'action', text: brief, actionTool: tool };
}
if (msg.role === 'tool_result') {
const content =
typeof msg.content === 'string'
? (() => { try { return JSON.parse(msg.content); } catch { return { text: msg.content }; } })()
: msg.content;
const toolName = content?.tool_name || '';
const elapsed = content?.elapsed_ms;
const text = content?.text || '';
if (toolName === 'BrowserScreenshot') {
return { type: 'result', text: `Screenshot captured${elapsed ? ` (${elapsed}ms)` : ''}` };
}
const preview = text.length > 120 ? text.slice(0, 120) + '…' : text;
return { type: 'result', text: `${preview}${elapsed ? ` (${elapsed}ms)` : ''}` };
}
if (msg.role === 'system') {
return { type: 'system', text: typeof msg.content === 'string' ? msg.content : '' };
}
return null;
}
type SvgIconComponent = typeof OpenInNewIcon;
function getActionIcon(tool?: string): SvgIconComponent {
switch (tool) {
case 'BrowserNavigate': return OpenInNewIcon;
case 'BrowserClick': return TouchAppOutlinedIcon;
case 'BrowserType': return KeyboardOutlinedIcon;
case 'BrowserScreenshot': return CameraAltOutlinedIcon;
case 'BrowserGetText': return ArticleOutlinedIcon;
case 'BrowserGetElements': return AccountTreeOutlinedIcon;
case 'BrowserEvaluate': return CodeOutlinedIcon;
default: return BuildOutlinedIcon;
}
}
interface FeedColors {
thought: string;
thoughtIcon: string;
result: string;
error: string;
errorIcon: string;
scrollThumb: string;
}
const darkFeedColors: FeedColors = {
thought: '#a0aab8',
thoughtIcon: '#555b6e',
result: '#555b6e',
error: '#ff8787',
errorIcon: '#ff8787',
scrollThumb: '#2a2d3e',
};
const lightFeedColors: FeedColors = {
thought: '#555550',
thoughtIcon: '#9e9c95',
result: '#9e9c95',
error: '#c03030',
errorIcon: '#c03030',
scrollThumb: '#ccc9c0',
};
const BrowserAgentInlineFeed: React.FC<Props> = ({ parentSessionId, browserId }) => {
const c = useClaudeTokens();
const dispatch = useAppDispatch();
const { mode } = useThemeMode();
const fc = mode === 'dark' ? darkFeedColors : lightFeedColors;
const scrollRef = useRef<HTMLDivElement>(null);
const fetchedRef = useRef(false);
const browserSessions = useAppSelector((state) => {
const all = state.agents.sessions;
return Object.values(all).filter(
(s): s is AgentSession =>
s.mode === 'browser-agent' &&
s.parent_session_id === parentSessionId &&
(!browserId || s.browser_id === browserId),
);
});
useEffect(() => {
if (browserSessions.length === 0 && !fetchedRef.current) {
fetchedRef.current = true;
dispatch(fetchBrowserAgentChildren(parentSessionId));
}
}, [browserSessions.length, parentSessionId, dispatch]);
const sessionsWithEntries = useMemo(() => {
return browserSessions.map((session) => {
const entries: FeedEntry[] = [];
for (const msg of session.messages) {
const entry = formatMessage(msg);
if (entry) entries.push(entry);
}
if (session.streamingMessage?.role === 'assistant' && session.streamingMessage.content) {
entries.push({ type: 'thought', text: session.streamingMessage.content });
}
return { session, entries };
});
}, [browserSessions]);
const totalMessages = browserSessions.reduce(
(n, s) => n + s.messages.length + (s.streamingMessage ? 1 : 0),
0,
);
useEffect(() => {
if (scrollRef.current) {
scrollRef.current.scrollTop = scrollRef.current.scrollHeight;
}
}, [totalMessages]);
if (browserSessions.length === 0) return null;
const showLabels = sessionsWithEntries.length > 1;
const accentColor = c.accent.primary;
return (
<Box
ref={scrollRef}
sx={{
maxHeight: 300,
overflowY: 'auto',
px: 1.5,
py: 1,
display: 'flex',
flexDirection: 'column',
gap: 0.25,
scrollbarWidth: 'thin',
scrollbarColor: `${fc.scrollThumb} transparent`,
'&::-webkit-scrollbar': { width: 4 },
'&::-webkit-scrollbar-thumb': {
background: fc.scrollThumb,
borderRadius: 2,
},
}}
>
{sessionsWithEntries.map(({ session, entries }, si) => (
<Box key={session.id} sx={{ display: 'flex', flexDirection: 'column', gap: 0.25 }}>
{showLabels && (
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5, mt: si > 0 ? 1 : 0, mb: 0.25 }}>
<LanguageIcon sx={{ fontSize: 12, color: accentColor, opacity: 0.7 }} />
<Typography
sx={{
fontSize: '0.65rem',
fontWeight: 600,
color: accentColor,
opacity: 0.8,
textTransform: 'uppercase',
letterSpacing: '0.04em',
}}
>
{session.browser_id || `Browser ${si + 1}`}
</Typography>
<SessionStatusChip status={session.status} />
</Box>
)}
{!showLabels && entries.length === 0 && session.status === 'running' && (
<Typography
sx={{
fontSize: '0.7rem',
color: c.text.tertiary,
fontStyle: 'italic',
fontFamily: c.font.mono,
}}
>
Starting browser agent...
</Typography>
)}
{entries.map((entry, i) => (
<EntryRow key={i} entry={entry} accentColor={accentColor} fc={fc} />
))}
{!showLabels && session.status === 'running' && entries.length > 0 && (
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5, mt: 0.25 }}>
<Box
sx={{
width: 6,
height: 6,
borderRadius: '50%',
bgcolor: accentColor,
animation: 'ba-feed-pulse 1.4s ease-in-out infinite',
'@keyframes ba-feed-pulse': {
'0%, 100%': { opacity: 0.3, transform: 'scale(0.8)' },
'50%': { opacity: 1, transform: 'scale(1.2)' },
},
}}
/>
</Box>
)}
</Box>
))}
</Box>
);
};
const EntryRow: React.FC<{ entry: FeedEntry; accentColor: string; fc: FeedColors }> = ({ entry, accentColor, fc }) => {
const c = useClaudeTokens();
if (entry.type === 'thought') {
return (
<Box sx={{ display: 'flex', gap: 0.5, alignItems: 'flex-start', minWidth: 0 }}>
<SmartToyOutlinedIcon
sx={{ fontSize: 10, color: fc.thoughtIcon, mt: '3px', flexShrink: 0 }}
/>
<Typography
sx={{
fontSize: '0.7rem',
color: fc.thought,
lineHeight: 1.45,
wordBreak: 'break-word',
fontFamily: c.font.mono,
}}
>
{entry.text}
</Typography>
</Box>
);
}
if (entry.type === 'action') {
const ActionIcon = getActionIcon(entry.actionTool);
return (
<Box sx={{ display: 'flex', gap: 0.5, alignItems: 'flex-start', minWidth: 0 }}>
<ActionIcon sx={{ fontSize: 11, color: accentColor, mt: '2px', flexShrink: 0 }} />
<Typography
sx={{
fontSize: '0.7rem',
fontFamily: c.font.mono,
color: accentColor,
lineHeight: 1.45,
wordBreak: 'break-word',
}}
>
{entry.text}
</Typography>
</Box>
);
}
if (entry.type === 'result') {
return (
<Box sx={{ display: 'flex', gap: 0.5, alignItems: 'flex-start', minWidth: 0, pl: 1.25 }}>
<Typography
sx={{
fontSize: '0.65rem',
fontFamily: c.font.mono,
color: fc.result,
lineHeight: 1.45,
wordBreak: 'break-word',
}}
>
{entry.text}
</Typography>
</Box>
);
}
if (entry.type === 'system') {
return (
<Box sx={{ display: 'flex', gap: 0.5, alignItems: 'center', minWidth: 0 }}>
<ErrorOutlineIcon sx={{ fontSize: 10, color: fc.errorIcon, flexShrink: 0 }} />
<Typography
sx={{
fontSize: '0.68rem',
fontFamily: c.font.mono,
color: fc.error,
lineHeight: 1.45,
}}
>
{entry.text}
</Typography>
</Box>
);
}
return null;
};
const SessionStatusChip: React.FC<{ status: string }> = ({ status }) => {
const c = useClaudeTokens();
if (status === 'running') {
return (
<Box
sx={{
width: 6,
height: 6,
borderRadius: '50%',
bgcolor: c.status.success,
animation: 'ba-feed-pulse 1.4s ease-in-out infinite',
'@keyframes ba-feed-pulse': {
'0%, 100%': { opacity: 0.3, transform: 'scale(0.8)' },
'50%': { opacity: 1, transform: 'scale(1.2)' },
},
}}
/>
);
}
if (status === 'completed') {
return <CheckCircleOutlineIcon sx={{ fontSize: 10, color: c.status.success }} />;
}
if (status === 'error') {
return <ErrorOutlineIcon sx={{ fontSize: 10, color: c.status.error }} />;
}
return null;
};
export default React.memo(BrowserAgentInlineFeed);
@@ -19,6 +19,7 @@ import ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
import { AgentMessage } from '@/shared/state/agentsSlice';
import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext';
import BrowserAgentInlineFeed from './BrowserAgentInlineFeed';
const GoogleServiceIcon: React.FC<{ service: string; size?: number }> = ({ service, size = 14 }) => {
if (service === 'gmail') {
@@ -466,6 +467,7 @@ interface ToolCallBubbleProps {
isPending?: boolean;
isStreaming?: boolean;
mcpCompact?: boolean;
sessionId?: string;
}
interface TermColors {
@@ -1169,8 +1171,14 @@ const McpResultCard: React.FC<{ parsed: ParsedMcpResult; compact?: boolean }> =
return <GenericMcpCard data={data} />;
};
function isBrowserAgentTool(name: string): boolean {
if (name === 'BrowserAgent' || name === 'BrowserAgents') return true;
const mcp = parseMcpToolName(name);
return mcp.isMcp && mcp.serverSlug === 'openswarm-browser-agent';
}
const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false }) => {
({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false, sessionId }) => {
const c = useClaudeTokens();
const tc = useTermColors();
const [expanded, setExpanded] = useState(false);
@@ -1180,7 +1188,10 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
const inputSummary = getInputSummary(toolName, input);
const formattedInput = useMemo(() => formatInputDisplay(toolName, input), [toolName, input]);
const showTimer = isPending && !isDenied && !isStreaming;
const showBody = expanded || isStreaming;
const isBrowserAgent = isBrowserAgentTool(toolName);
const browserAgentAutoExpand = isBrowserAgent && isPending && !isStreaming;
const showBody = expanded || isStreaming || browserAgentAutoExpand;
const resultContent = result?.content;
const hasStructuredResult =
@@ -1320,6 +1331,12 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
'&::-webkit-scrollbar-thumb': { background: tc.SCROLLBAR_THUMB, borderRadius: 3 },
}}
>
{isBrowserAgent && sessionId && (
<BrowserAgentInlineFeed
parentSessionId={sessionId}
browserId={input?.browser_id}
/>
)}
{parsedResult && parsedResult.type === 'mcp' ? (
<McpResultCard parsed={parsedResult} compact />
) : parsedResult ? (
@@ -1330,7 +1347,7 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
{parsedResult.type === 'text' ? parsedResult.content : ''}
</pre>
) : null}
{!parsedResult && isPending && !isStreaming && (
{!parsedResult && isPending && !isStreaming && !isBrowserAgent && (
<Box sx={{ px: 1.5, py: 1 }}>
<Box sx={{ width: 8, height: 2, bgcolor: tc.PROMPT_COLOR, animation: 'tool-pulse 1s ease-in-out infinite', borderRadius: 1 }} />
</Box>
@@ -1525,6 +1542,14 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
)}
</pre>
{/* Browser agent inline feed */}
{isBrowserAgent && sessionId && (
<BrowserAgentInlineFeed
parentSessionId={sessionId}
browserId={input?.browser_id}
/>
)}
{/* Output */}
{parsedResult && parsedResult.type === 'mcp' ? (
<McpResultCard parsed={parsedResult} />
@@ -1566,8 +1591,8 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
</pre>
) : null}
{/* Pending indicator when waiting for result */}
{!parsedResult && isPending && !isStreaming && (
{/* Pending indicator when waiting for result (skip for browser agent — feed replaces it) */}
{!parsedResult && isPending && !isStreaming && !isBrowserAgent && (
<Box sx={{ px: 1.5, pb: 1, pt: 0.5 }}>
<Box
sx={{
@@ -68,9 +68,10 @@ interface Props {
group: ToolGroup;
isSessionRunning?: boolean;
meta?: ToolGroupMeta;
sessionId?: string;
}
const ToolGroupBubble: React.FC<Props> = React.memo(({ group, isSessionRunning = false, meta }) => {
const ToolGroupBubble: React.FC<Props> = React.memo(({ group, isSessionRunning = false, meta, sessionId }) => {
const c = useClaudeTokens();
const isMcp = !!group.mcpServer;
const [expanded, setExpanded] = useState(isMcp);
@@ -186,6 +187,7 @@ const ToolGroupBubble: React.FC<Props> = React.memo(({ group, isSessionRunning =
result={pair.result}
isPending={pair.result === null && isSessionRunning}
mcpCompact
sessionId={sessionId}
/>
))}
</Box>
@@ -67,6 +67,14 @@ const HANDLE_DEFS: { dir: ResizeDir; sx: Record<string, any> }[] = [
const isElectron = navigator.userAgent.includes('Electron');
const chromeUserAgent = navigator.userAgent
.replace(/\s*Electron\/\S+/, '')
.replace(/\s*OpenSwarm\/\S+/, '');
const webviewPreloadPath: string | undefined = isElectron
? (window as any).openswarm?.getWebviewPreloadPath?.()
: undefined;
type WebviewElement = BrowserWebview;
interface TabLocalState {
@@ -192,10 +200,6 @@ const BrowserCard: React.FC<Props> = ({
onTitleUpdate();
};
const onNewWindow = (e: any) => {
if (e.url) dispatch(addBrowserTab({ browserId, url: e.url, makeActive: true }));
};
const onFaviconUpdate = (e: any) => {
const favicons = e.favicons || (e.detail && e.detail.favicons);
if (favicons?.[0]) {
@@ -208,7 +212,6 @@ const BrowserCard: React.FC<Props> = ({
wv.addEventListener('page-title-updated', onTitleUpdate);
wv.addEventListener('did-start-loading', onLoadStart);
wv.addEventListener('did-stop-loading', onLoadStop);
wv.addEventListener('new-window', onNewWindow);
wv.addEventListener('page-favicon-updated', onFaviconUpdate);
cleanups.push(() => {
@@ -218,7 +221,6 @@ const BrowserCard: React.FC<Props> = ({
wv.removeEventListener('page-title-updated', onTitleUpdate);
wv.removeEventListener('did-start-loading', onLoadStart);
wv.removeEventListener('did-stop-loading', onLoadStop);
wv.removeEventListener('new-window', onNewWindow);
wv.removeEventListener('page-favicon-updated', onFaviconUpdate);
});
}
@@ -988,6 +990,9 @@ const BrowserCard: React.FC<Props> = ({
data-tab-id={tab.id}
src="about:blank"
allowpopups="true"
useragent={chromeUserAgent}
{...(webviewPreloadPath ? { preload: webviewPreloadPath } : {})}
webpreferences="plugins=yes, autoplayPolicy=no-user-gesture-required"
style={{
position: 'absolute',
top: 0,
@@ -22,6 +22,7 @@ import {
tidyLayout,
addViewCard,
addBrowserCard,
addBrowserTab,
moveCards,
resetLayout,
setGlowingBrowserCards,
@@ -31,6 +32,7 @@ import { fetchOutputs } from '@/shared/state/outputsSlice';
import { generateDashboardName, updateDashboardThumbnail } from '@/shared/state/dashboardsSlice';
import { dashboardWs } from '@/shared/ws/WebSocketManager';
import { initBrowserCommandHandler } from '@/shared/browserCommandHandler';
import { findBrowserByWebContentsId } from '@/shared/browserRegistry';
import AgentCard from './AgentCard';
import DashboardViewCard from './DashboardViewCard';
import BrowserCard from './BrowserCard';
@@ -200,6 +202,26 @@ const DashboardInner: React.FC = () => {
return () => { cleanupBrowserHandler(); dashboardWs.disconnect(); };
}, [dispatch, dashboardId]);
useEffect(() => {
const w = window as any;
if (!w.openswarm?.onWebviewNewWindow) return;
let lastUrl = '';
let lastTime = 0;
return w.openswarm.onWebviewNewWindow((url: string, webContentsId: number) => {
const now = Date.now();
if (url === lastUrl && now - lastTime < 1000) return;
lastUrl = url;
lastTime = now;
const browserId = findBrowserByWebContentsId(webContentsId);
if (browserId) {
dispatch(addBrowserTab({ browserId, url, makeActive: true }));
} else {
dispatch(addBrowserCard({ url, expandedSessionIds }));
}
});
}, [dispatch, expandedSessionIds]);
// Capture a thumbnail screenshot of the dashboard.
// Uses Electron's native capturePage for pixel-perfect results.
// Captures current viewport as-is (no DOM mutation) to avoid visual flashes.
@@ -204,13 +204,17 @@ const DashboardToolbar = React.forwardRef<HTMLDivElement, Props>(
const isExpanded = inputOpen || viewPickerOpen || historyOpen;
const autoSelectOnNew = useAppSelector((s) => s.settings.data.auto_select_mode_on_new_agent);
const prevInputOpenRef = useRef(inputOpen);
useEffect(() => {
if (prevInputOpenRef.current && !inputOpen && elementSelection?.selectMode) {
elementSelection.setSelectMode(false);
}
if (!prevInputOpenRef.current && inputOpen && autoSelectOnNew) {
elementSelection?.setSelectMode(true);
}
prevInputOpenRef.current = inputOpen;
}, [inputOpen, elementSelection]);
}, [inputOpen, elementSelection, autoSelectOnNew]);
useEffect(() => {
if (!isExpanded) return;
+17 -1
View File
@@ -11,6 +11,7 @@ import InputAdornment from '@mui/material/InputAdornment';
import ToggleButton from '@mui/material/ToggleButton';
import ToggleButtonGroup from '@mui/material/ToggleButtonGroup';
import Slider from '@mui/material/Slider';
import Switch from '@mui/material/Switch';
import Snackbar from '@mui/material/Snackbar';
import Alert from '@mui/material/Alert';
import Tab from '@mui/material/Tab';
@@ -495,7 +496,7 @@ const Settings: React.FC = () => {
</Box>
</Box>
<Box sx={inlineRowLastSx}>
<Box sx={inlineRowSx}>
<Box sx={{ mr: 3 }}>
<Typography sx={labelSx}>New agent shortcut</Typography>
<Typography sx={descSx}>Keyboard shortcut to create an agent.</Typography>
@@ -553,6 +554,21 @@ const Settings: React.FC = () => {
</Box>
</Box>
<Box sx={inlineRowLastSx}>
<Box sx={{ mr: 3 }}>
<Typography sx={labelSx}>Auto-enable element selection</Typography>
<Typography sx={descSx}>Automatically enter element selection mode when creating a new agent.</Typography>
</Box>
<Switch
checked={form.auto_select_mode_on_new_agent}
onChange={(e) => setForm({ ...form, auto_select_mode_on_new_agent: e.target.checked })}
sx={{
'& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary },
'& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary },
}}
/>
</Box>
{/* ── Browser ── */}
<Typography sx={{ ...sectionSx, mt: 3 }}>Browser</Typography>
+2 -1
View File
@@ -488,7 +488,8 @@ const Tools: React.FC = () => {
if (discoverTools.fulfilled.match(result)) {
setSnackbar({ open: true, message: 'Actions discovered successfully' });
} else {
setSnackbar({ open: true, message: 'Discovery failed — is the MCP server running?', severity: 'error' });
const detail = (result as any).error?.message || 'Discovery failed — is the MCP server running?';
setSnackbar({ open: true, message: detail, severity: 'error' });
}
} finally {
setDiscovering(false);
+9
View File
@@ -51,6 +51,15 @@ export function getAllWebviews(): Map<string, BrowserWebview> {
return new Map(registry);
}
export function findBrowserByWebContentsId(wcId: number): string | undefined {
for (const [key, wv] of registry.entries()) {
if ((wv as any).getWebContentsId?.() === wcId) {
return key.split(':')[0];
}
}
return undefined;
}
export function unregisterAllForBrowser(browserId: string): void {
const prefix = `${browserId}:`;
for (const key of registry.keys()) {
+34 -1
View File
@@ -69,6 +69,7 @@ export interface AgentSession {
tool_group_meta: Record<string, ToolGroupMeta>;
dashboard_id?: string;
browser_id?: string | null;
parent_session_id?: string | null;
}
export interface AgentConfig {
@@ -380,6 +381,15 @@ export const resumeSession = createAsyncThunk(
}
);
export const fetchBrowserAgentChildren = createAsyncThunk(
'agents/fetchBrowserAgentChildren',
async (parentSessionId: string) => {
const res = await fetch(`${AGENTS_API}/sessions/${parentSessionId}/browser-agents`);
const data = await res.json();
return data.sessions as AgentSession[];
}
);
const agentsSlice = createSlice({
name: 'agents',
initialState,
@@ -629,7 +639,19 @@ const agentsSlice = createSlice({
closeSessionFromWs(state, action: PayloadAction<HistorySession>) {
const entry = action.payload;
state.history[entry.id] = entry;
delete state.sessions[entry.id];
const session = state.sessions[entry.id];
if (session?.mode === 'browser-agent' && session.parent_session_id) {
session.status = (entry.status as AgentSession['status']) || 'completed';
} else {
delete state.sessions[entry.id];
for (const [id, s] of Object.entries(state.sessions)) {
if (s.mode === 'browser-agent' && s.parent_session_id === entry.id) {
delete state.sessions[id];
}
}
}
if (state.activeSessionId === entry.id) {
state.activeSessionId = null;
}
@@ -816,6 +838,17 @@ const agentsSlice = createSlice({
};
}
})
.addCase(fetchBrowserAgentChildren.fulfilled, (state, action) => {
for (const session of action.payload) {
if (!state.sessions[session.id]) {
state.sessions[session.id] = {
...session,
streamingMessage: null,
tool_group_meta: session.tool_group_meta ?? {},
};
}
}
})
.addCase(searchHistory.pending, (state) => {
state.historySearch.loading = true;
})
@@ -22,6 +22,7 @@ export interface AppSettings {
new_agent_shortcut: string;
anthropic_api_key: string | null;
browser_homepage: string;
auto_select_mode_on_new_agent: boolean;
}
export interface BrowseResult {
@@ -50,6 +51,7 @@ const initialState: SettingsState = {
new_agent_shortcut: 'Meta+l',
anthropic_api_key: null,
browser_homepage: 'https://www.google.com',
auto_select_mode_on_new_agent: false,
},
loading: false,
loaded: false,
+4
View File
@@ -10,6 +10,8 @@ declare global {
partition?: string;
allowpopups?: string;
nodeintegration?: string;
webpreferences?: string;
useragent?: string;
},
HTMLElement
>;
@@ -31,6 +33,7 @@ declare global {
interface OpenSwarmAPI {
getBackendPort: () => number;
getWebviewPreloadPath: () => string;
getAppVersion: () => Promise<string>;
checkForUpdates: () => Promise<{ success: boolean; version?: string; error?: string }>;
downloadUpdate: () => Promise<{ success: boolean; error?: string }>;
@@ -40,6 +43,7 @@ declare global {
onDownloadProgress: (cb: (progress: OpenSwarmDownloadProgress) => void) => () => void;
onUpdateDownloaded: (cb: (info: OpenSwarmUpdateInfo) => void) => () => void;
onUpdateError: (cb: (message: string) => void) => () => void;
onWebviewNewWindow: (cb: (url: string, webContentsId: number) => void) => () => void;
}
interface Window {
+9 -1
View File
@@ -127,10 +127,18 @@ if (( frontend_elapsed >= FRONTEND_MAX_WAIT )); then
exit 1
fi
# --- Sign Electron VMP for DRM (if EVS account exists) ---
if [ -f "$PROJECT_ROOT/electron/scripts/sign-vmp.sh" ]; then
echo -e "${YELLOW}${BOLD}[vmp]${RESET} Checking VMP signature..."
bash "$PROJECT_ROOT/electron/scripts/sign-vmp.sh" 2>&1 | while IFS= read -r line; do
printf "${YELLOW}${BOLD}%s${RESET}\n" "$line"
done
fi
# --- Start Electron in dev mode ---
MAGENTA='\033[0;35m'
echo -e "${MAGENTA}${BOLD}[electron]${RESET} Launching Electron dev shell..."
(cd "$PROJECT_ROOT/electron" && ELECTRON_DEV=1 npx electron .) > >(
(cd "$PROJECT_ROOT/electron" && unset ELECTRON_RUN_AS_NODE && ELECTRON_DEV=1 npx electron .) > >(
while IFS= read -r line; do
printf "${MAGENTA}${BOLD}[electron]${RESET} %s\n" "$line"
done