[Haik]: Browser control

backend/apps/agents/agent_manager.py

@@ -2,6 +2,7 @@ import asyncio
 import json
 import logging
 import os
+import sys
 import time
 from datetime import datetime
 from uuid import uuid4
@@ -547,6 +548,17 @@ class AgentManager:
 
             mcp_servers = await self._build_mcp_servers(session.allowed_tools)
 
+            browser_server_path = os.path.join(
+                os.path.dirname(__file__), "browser_mcp_server.py"
+            )
+            backend_port = os.environ.get("OPENSWARM_PORT", "8324")
+            mcp_servers["openswarm-browser"] = {
+                "command": sys.executable,
+                "args": [browser_server_path],
+                "env": {"OPENSWARM_PORT": backend_port},
+                "type": "stdio",
+            }
+
             effective_allowed = [
                 t for t in session.allowed_tools
                 if _builtin_perms.get(t, "always_allow") == "always_allow"
@@ -569,6 +581,8 @@ class AgentManager:
                     else:
                         effective_allowed.append(f"mcp__{name}__*")
 
+            effective_allowed.append("mcp__openswarm-browser__*")
+
             options_kwargs = {
                 "model": session.model,
                 "can_use_tool": can_use_tool,

backend/apps/agents/browser_mcp_server.py

@@ -0,0 +1,303 @@
+#!/usr/bin/env python3
+"""
+Minimal stdio MCP server that exposes browser interaction tools.
+
+Launched as a subprocess by the Claude Agent SDK. Proxies tool calls
+to the OpenSwarm backend via HTTP, which bridges them to the Electron
+frontend via WebSocket where the actual webview lives.
+"""
+
+import base64
+import json
+import sys
+import os
+import urllib.request
+import urllib.error
+from io import BytesIO
+
+try:
+    from PIL import Image
+    HAS_PIL = True
+except ImportError:
+    HAS_PIL = False
+
+BACKEND_PORT = os.environ.get("OPENSWARM_PORT", "8324")
+BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/browser/command"
+
+TOOLS = [
+    {
+        "name": "BrowserScreenshot",
+        "description": (
+            "Capture a screenshot of the browser page. Returns the screenshot as a "
+            "base64-encoded PNG image. Use this to see what is currently displayed."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "browser_id": {
+                    "type": "string",
+                    "description": "The browser card ID to capture. Use the ID from the selected browser card context.",
+                },
+            },
+            "required": ["browser_id"],
+        },
+    },
+    {
+        "name": "BrowserGetText",
+        "description": (
+            "Get the visible text content of the browser page. Returns the page's "
+            "innerText (up to 15000 characters)."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "browser_id": {
+                    "type": "string",
+                    "description": "The browser card ID.",
+                },
+            },
+            "required": ["browser_id"],
+        },
+    },
+    {
+        "name": "BrowserNavigate",
+        "description": "Navigate the browser to a URL.",
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "browser_id": {
+                    "type": "string",
+                    "description": "The browser card ID.",
+                },
+                "url": {
+                    "type": "string",
+                    "description": "The URL to navigate to.",
+                },
+            },
+            "required": ["browser_id", "url"],
+        },
+    },
+    {
+        "name": "BrowserClick",
+        "description": (
+            "Click an element in the browser page identified by a CSS selector."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "browser_id": {
+                    "type": "string",
+                    "description": "The browser card ID.",
+                },
+                "selector": {
+                    "type": "string",
+                    "description": "CSS selector of the element to click.",
+                },
+            },
+            "required": ["browser_id", "selector"],
+        },
+    },
+    {
+        "name": "BrowserType",
+        "description": (
+            "Type text into an input element in the browser page. Clears the "
+            "existing value first, then types the new text."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "browser_id": {
+                    "type": "string",
+                    "description": "The browser card ID.",
+                },
+                "selector": {
+                    "type": "string",
+                    "description": "CSS selector of the input element.",
+                },
+                "text": {
+                    "type": "string",
+                    "description": "The text to type.",
+                },
+            },
+            "required": ["browser_id", "selector", "text"],
+        },
+    },
+    {
+        "name": "BrowserEvaluate",
+        "description": (
+            "Evaluate a JavaScript expression in the browser page and return the result. "
+            "The expression is run via executeJavaScript on the webview."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "browser_id": {
+                    "type": "string",
+                    "description": "The browser card ID.",
+                },
+                "expression": {
+                    "type": "string",
+                    "description": "JavaScript expression to evaluate.",
+                },
+            },
+            "required": ["browser_id", "expression"],
+        },
+    },
+]
+
+
+def send_response(id_, result=None, error=None):
+    msg = {"jsonrpc": "2.0", "id": id_}
+    if error is not None:
+        msg["error"] = error
+    else:
+        msg["result"] = result
+    sys.stdout.write(json.dumps(msg) + "\n")
+    sys.stdout.flush()
+
+
+def send_notification(method, params=None):
+    msg = {"jsonrpc": "2.0", "method": method}
+    if params is not None:
+        msg["params"] = params
+    sys.stdout.write(json.dumps(msg) + "\n")
+    sys.stdout.flush()
+
+
+def call_backend(action: str, browser_id: str, params: dict | None = None) -> dict:
+    payload = json.dumps({
+        "action": action,
+        "browser_id": browser_id,
+        "params": params or {},
+    }).encode()
+    req = urllib.request.Request(
+        BACKEND_URL,
+        data=payload,
+        headers={"Content-Type": "application/json"},
+        method="POST",
+    )
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return json.loads(resp.read().decode())
+    except urllib.error.HTTPError as e:
+        body = e.read().decode() if e.fp else str(e)
+        return {"error": f"HTTP {e.code}: {body}"}
+    except Exception as e:
+        return {"error": str(e)}
+
+
+MAX_IMAGE_B64_BYTES = 700_000
+
+
+def compress_screenshot(b64_png: str) -> tuple[str, str] | None:
+    """Resize and re-encode as JPEG to stay under the stdio buffer limit."""
+    if not HAS_PIL:
+        return None
+    try:
+        raw = base64.b64decode(b64_png)
+        img = Image.open(BytesIO(raw))
+        max_width = 1280
+        if img.width > max_width:
+            ratio = max_width / img.width
+            img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS)
+        buf = BytesIO()
+        img.convert("RGB").save(buf, format="JPEG", quality=55)
+        return base64.b64encode(buf.getvalue()).decode(), "image/jpeg"
+    except Exception:
+        return None
+
+
+def handle_tool_call(tool_name: str, arguments: dict) -> dict:
+    browser_id = arguments.get("browser_id", "")
+    if not browser_id:
+        return {"content": [{"type": "text", "text": "Error: browser_id is required"}], "isError": True}
+
+    action_map = {
+        "BrowserScreenshot": "screenshot",
+        "BrowserGetText": "get_text",
+        "BrowserNavigate": "navigate",
+        "BrowserClick": "click",
+        "BrowserType": "type",
+        "BrowserEvaluate": "evaluate",
+    }
+    action = action_map.get(tool_name)
+    if not action:
+        return {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True}
+
+    params = {k: v for k, v in arguments.items() if k != "browser_id"}
+    result = call_backend(action, browser_id, params)
+
+    if "error" in result:
+        return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True}
+
+    if action == "screenshot" and result.get("image"):
+        image_data = result["image"]
+        mime_type = "image/png"
+
+        if len(image_data) > MAX_IMAGE_B64_BYTES:
+            compressed = compress_screenshot(image_data)
+            if compressed:
+                image_data, mime_type = compressed
+
+        if len(image_data) > MAX_IMAGE_B64_BYTES:
+            return {
+                "content": [
+                    {"type": "text", "text": (
+                        f"Screenshot too large to return ({len(image_data)} bytes base64). "
+                        f"URL: {result.get('url', 'unknown')}. "
+                        "Use BrowserGetText to read the page content instead."
+                    )},
+                ],
+            }
+
+        return {
+            "content": [
+                {"type": "image", "data": image_data, "mimeType": mime_type},
+                {"type": "text", "text": f"Screenshot captured. URL: {result.get('url', 'unknown')}"},
+            ],
+        }
+
+    text = result.get("text", result.get("data", json.dumps(result)))
+    return {"content": [{"type": "text", "text": str(text)}]}
+
+
+def main():
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            msg = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+
+        method = msg.get("method")
+        id_ = msg.get("id")
+        params = msg.get("params", {})
+
+        if method == "initialize":
+            send_response(id_, {
+                "protocolVersion": "2024-11-05",
+                "capabilities": {"tools": {}},
+                "serverInfo": {
+                    "name": "openswarm-browser",
+                    "version": "1.0.0",
+                },
+            })
+        elif method == "notifications/initialized":
+            pass
+        elif method == "tools/list":
+            send_response(id_, {"tools": TOOLS})
+        elif method == "tools/call":
+            tool_name = params.get("name", "")
+            arguments = params.get("arguments", {})
+            result = handle_tool_call(tool_name, arguments)
+            send_response(id_, result)
+        elif method == "ping":
+            send_response(id_, {})
+        elif id_ is not None:
+            send_response(id_, error={"code": -32601, "message": f"Method not found: {method}"})
+
+
+if __name__ == "__main__":
+    main()

backend/apps/agents/ws_manager.py

@@ -12,6 +12,7 @@ class ConnectionManager:
         self.connections: dict[str, list[WebSocket]] = {}
         self.global_connections: list[WebSocket] = []
         self.pending_futures: dict[str, asyncio.Future] = {}
+        self.browser_futures: dict[str, asyncio.Future] = {}
 
     async def connect_session(self, session_id: str, websocket: WebSocket):
         await websocket.accept()
@@ -85,4 +86,32 @@ class ConnectionManager:
         if future and not future.done():
             future.set_result(decision)
 
+    async def send_browser_command(
+        self, request_id: str, action: str, browser_id: str, params: dict
+    ) -> dict:
+        """Send a browser command to the frontend and wait for the result."""
+        future = asyncio.get_event_loop().create_future()
+        self.browser_futures[request_id] = future
+
+        await self.broadcast_global("browser:command", {
+            "request_id": request_id,
+            "action": action,
+            "browser_id": browser_id,
+            "params": params,
+        })
+
+        try:
+            result = await asyncio.wait_for(future, timeout=30.0)
+            return result
+        except asyncio.TimeoutError:
+            return {"error": "Browser command timed out"}
+        finally:
+            self.browser_futures.pop(request_id, None)
+
+    def resolve_browser_command(self, request_id: str, result: dict):
+        """Resolve a pending browser command Future with the frontend's result."""
+        future = self.browser_futures.get(request_id)
+        if future and not future.done():
+            future.set_result(result)
+
 ws_manager = ConnectionManager()

backend/main.py

@@ -1,6 +1,8 @@
 import os
+from uuid import uuid4
 
 from fastapi.responses import JSONResponse
+from fastapi import Request
 from backend.config.Apps import MainApp
 from backend.apps.health.health import health
 from backend.apps.agents.agents import agents
@@ -85,9 +87,32 @@ async def websocket_dashboard(websocket: WebSocket):
                     "message": payload.get("message"),
                     "updated_input": payload.get("updated_input"),
                 })
+            elif event == "browser:result":
+                ws_manager.resolve_browser_command(
+                    payload.get("request_id", ""),
+                    payload,
+                )
     except WebSocketDisconnect:
         ws_manager.disconnect_global(websocket)
 
+
+@app.post("/api/browser/command")
+async def browser_command(request: Request):
+    """HTTP endpoint called by the browser MCP server subprocess.
+    Proxies commands to the frontend via WebSocket and waits for results."""
+    body = await request.json()
+    action = body.get("action", "")
+    browser_id = body.get("browser_id", "")
+    params = body.get("params", {})
+
+    if not action or not browser_id:
+        return JSONResponse({"error": "action and browser_id are required"}, status_code=400)
+
+    request_id = uuid4().hex
+    result = await ws_manager.send_browser_command(request_id, action, browser_id, params)
+    return JSONResponse(result)
+
+
 if __name__ == "__main__":
     import argparse
     import uvicorn

backend/requirements.txt

@@ -8,4 +8,5 @@ langchain-openai==0.3.12
 pytest==8.3.4
 pytest-asyncio==0.25.2
 typeguard==4.4.2
-python-dotenv==1.1.1
+python-dotenv==1.1.1
+Pillow

frontend/src/app/components/ElementSelectionContext.tsx

@@ -9,7 +9,7 @@ export interface SelectedElement {
   computedStyles: Record<string, string>;
   screenshot?: string;
   boundingRect: { x: number; y: number; width: number; height: number };
-  semanticType?: 'agent-card' | 'message' | 'tool-call' | 'tool-group' | 'view-card' | 'dom-element';
+  semanticType?: 'agent-card' | 'message' | 'tool-call' | 'tool-group' | 'view-card' | 'browser-card' | 'dom-element';
   semanticLabel?: string;
   semanticData?: Record<string, any>;
 }
@@ -18,6 +18,8 @@ interface ElementSelectionContextValue {
   selectMode: boolean;
   toggleSelectMode: () => void;
   setSelectMode: (active: boolean) => void;
+  excludeSelectId: string | null;
+  setExcludeSelectId: (id: string | null) => void;
   selectedElements: SelectedElement[];
   addSelectedElement: (el: SelectedElement) => void;
   updateSelectedElement: (id: string, patch: Partial<SelectedElement>) => void;
@@ -34,11 +36,15 @@ export function useElementSelection() {
 
 export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
   const [selectMode, setSelectMode] = useState(false);
+  const [excludeSelectId, setExcludeSelectId] = useState<string | null>(null);
   const [selectedElements, setSelectedElements] = useState<SelectedElement[]>([]);
   const iframeRef = useRef<HTMLIFrameElement | null>(null);
 
   const toggleSelectMode = useCallback(() => {
-    setSelectMode((prev) => !prev);
+    setSelectMode((prev) => {
+      if (prev) setExcludeSelectId(null);
+      return !prev;
+    });
   }, []);
 
   const addSelectedElement = useCallback((el: SelectedElement) => {
@@ -66,6 +72,8 @@ export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> =
         selectMode,
         toggleSelectMode,
         setSelectMode,
+        excludeSelectId,
+        setExcludeSelectId,
         selectedElements,
         addSelectedElement,
         updateSelectedElement,

frontend/src/app/components/useDomElementSelector.ts

@@ -31,12 +31,16 @@ const SEMANTIC_LABELS: Record<string, string> = {
   'tool-call': 'Tool Call',
   'tool-group': 'Tool Group',
   'view-card': 'View',
+  'browser-card': 'Browser',
 };
 
-function findSelectableAncestor(target: Element): Element | null {
+function findSelectableAncestor(target: Element, excludeId?: string | null): Element | null {
   let current: Element | null = target;
   while (current) {
-    if (current.hasAttribute(SELECT_ATTR)) return current;
+    if (current.hasAttribute(SELECT_ATTR)) {
+      if (excludeId && current.getAttribute(SELECT_ID_ATTR) === excludeId) return null;
+      return current;
+    }
     current = current.parentElement;
   }
   return null;
@@ -114,6 +118,11 @@ export function useDomElementSelector(): DomSelectorState {
   const isDraggingRef = useRef(false);
   const dragBoundsRef = useRef<{ left: number; top: number; right: number; bottom: number } | null>(null);
 
+  const excludeIdRef = useRef<string | null>(null);
+  useEffect(() => {
+    excludeIdRef.current = ctx?.excludeSelectId ?? null;
+  }, [ctx?.excludeSelectId]);
+
   const selectedIdsRef = useRef(new Map<string, string>());
   useEffect(() => {
     const map = new Map<string, string>();
@@ -161,10 +170,12 @@ export function useDomElementSelector(): DomSelectorState {
           const allSelectables = document.querySelectorAll(`[${SELECT_ATTR}]`);
           const preview: DragPreviewElement[] = [];
           const seen = new Set<string>();
+          const excId = excludeIdRef.current;
           allSelectables.forEach((el) => {
+            const selectId = el.getAttribute(SELECT_ID_ATTR) || '';
+            if (excId && selectId === excId) return;
             const rect = el.getBoundingClientRect();
             if (rectsIntersect(b, { left: rect.left, top: rect.top, right: rect.right, bottom: rect.bottom })) {
-              const selectId = el.getAttribute(SELECT_ID_ATTR) || '';
               if (seen.has(selectId)) return;
               seen.add(selectId);
               const type = el.getAttribute(SELECT_ATTR) || '';
@@ -200,7 +211,7 @@ export function useDomElementSelector(): DomSelectorState {
       return;
     }
 
-    const selectable = findSelectableAncestor(target);
+    const selectable = findSelectableAncestor(target, excludeIdRef.current);
     if (!selectable) {
       setOverlay(EMPTY_OVERLAY);
       hoveredRef.current = null;
@@ -231,7 +242,7 @@ export function useDomElementSelector(): DomSelectorState {
     if (e.button !== 0) return;
     const target = e.target as Element;
     // Only start drag on "empty" canvas areas (not on selectable elements)
-    if (target && findSelectableAncestor(target)) return;
+    if (target && findSelectableAncestor(target, excludeIdRef.current)) return;
     dragOriginRef.current = { x: e.clientX, y: e.clientY };
     isDraggingRef.current = false;
   }, []);
@@ -250,7 +261,10 @@ export function useDomElementSelector(): DomSelectorState {
       const allSelectables = document.querySelectorAll(`[${SELECT_ATTR}]`);
       const processed = new Set<string>();
 
+      const excId = excludeIdRef.current;
       allSelectables.forEach((el) => {
+        const selectId = el.getAttribute(SELECT_ID_ATTR) || '';
+        if (excId && selectId === excId) return;
         const rect = el.getBoundingClientRect();
         const elRect = {
           left: rect.left,
@@ -260,7 +274,6 @@ export function useDomElementSelector(): DomSelectorState {
         };
 
         if (rectsIntersect(dr, elRect)) {
-          const selectId = el.getAttribute(SELECT_ID_ATTR) || '';
           if (processed.has(selectId)) return;
           processed.add(selectId);
 

frontend/src/app/pages/AgentChat/AgentChat.tsx

@@ -629,6 +629,7 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
           isRunning={!isDraft && (session.status === 'running' || session.status === 'waiting_approval')}
           onStop={handleStop}
           contextEstimate={contextEstimate}
+          sessionId={id}
         />
       </Box>
     </Box>

frontend/src/app/pages/AgentChat/ChatInput.tsx

@@ -26,6 +26,7 @@ import AttachFileIcon from '@mui/icons-material/AttachFile';
 import AdsClickIcon from '@mui/icons-material/AdsClick';
 import CommandPicker, { CommandPickerItem, getToolGroupIcon } from '@/app/components/CommandPicker';
 import { useElementSelection, SelectedElement } from '@/app/components/ElementSelectionContext';
+import { getWebview } from '@/shared/browserRegistry';
 import { API_BASE } from '@/shared/config';
 import { ContextPath } from '@/app/components/DirectoryBrowser';
 import {
@@ -71,6 +72,7 @@ interface Props {
   contextEstimate?: { used: number; limit: number };
   embedded?: boolean;
   autoFocus?: boolean;
+  sessionId?: string;
 }
 
 export interface ChatInputHandle {
@@ -128,7 +130,7 @@ const ContextRing: React.FC<{ used: number; limit: number; accentColor: string;
   );
 };
 
-const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode, onModeChange, model, onModelChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus }, ref) => {
+const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode, onModeChange, model, onModelChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus, sessionId }, ref) => {
   const c = useClaudeTokens();
   const editorRef = useRef<HTMLDivElement>(null);
   const containerRef = useRef<HTMLDivElement>(null);
@@ -272,7 +274,7 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
     }
   }, []);
 
-  const handleSend = useCallback(() => {
+  const handleSend = useCallback(async () => {
     const editor = editorRef.current;
     if (!editor || disabled) return;
     const serialized = serializeEditorContent(editor, attachedSkillsRef.current);
@@ -286,14 +288,47 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
 
     if (selectedEls.length > 0) {
       const lines: string[] = ['\n\n---\nSelected UI Elements:\n'];
-      selectedEls.forEach((el, i) => {
-        if (el.semanticType && el.semanticData) {
+      for (let i = 0; i < selectedEls.length; i++) {
+        const el = selectedEls[i];
+
+        if (el.semanticType === 'browser-card' && el.semanticData?.selectId) {
+          const wv = getWebview(el.semanticData.selectId as string);
+          if (wv) {
+            const url = el.semanticData.url || wv.getURL();
+            const title = el.semanticData.name || wv.getTitle();
+            lines.push(`${i + 1}. [Browser Card] ${title}`);
+            lines.push(`   ID: ${el.semanticData.selectId}`);
+            lines.push(`   URL: ${url}`);
+
+            try {
+              const nativeImage = await wv.capturePage();
+              const dataUrl = nativeImage.toDataURL();
+              const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, '');
+              allImages.push({ data: base64, media_type: 'image/png' });
+              lines.push(`   [Screenshot captured and attached as image]`);
+            } catch { /* screenshot unavailable */ }
+
+            try {
+              const pageText: string = await wv.executeJavaScript(
+                'document.body.innerText.substring(0, 15000)'
+              );
+              if (pageText?.trim()) {
+                lines.push(`   Page text content:\n   ---\n${pageText.trim().split('\n').map(l => '   ' + l).join('\n')}\n   ---`);
+              }
+            } catch { /* text extraction unavailable */ }
+          } else {
+            lines.push(`${i + 1}. [Browser Card] ${el.semanticLabel || ''}`);
+            if (el.semanticData.selectId) lines.push(`   ID: ${el.semanticData.selectId}`);
+            if (el.semanticData.url) lines.push(`   URL: ${el.semanticData.url}`);
+          }
+        } else if (el.semanticType && el.semanticData) {
           const typeLabel = {
             'agent-card': 'Agent Card',
             'message': 'Message',
             'tool-call': 'Tool Call',
             'tool-group': 'Tool Group',
             'view-card': 'View Card',
+            'browser-card': 'Browser Card',
             'dom-element': 'Element',
           }[el.semanticType] || el.semanticType;
           lines.push(`${i + 1}. [${typeLabel}] ${el.semanticLabel || ''}`);
@@ -319,7 +354,7 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
           const base64 = el.screenshot.replace(/^data:image\/\w+;base64,/, '');
           allImages.push({ data: base64, media_type: 'image/png' });
         }
-      });
+      }
       trimmed += lines.join('\n');
     }
 
@@ -964,7 +999,12 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
           <Tooltip title={elementSelection.selectMode ? 'Exit select mode' : 'Select UI element'}>
             <IconButton
               size="small"
-              onClick={elementSelection.toggleSelectMode}
+              onClick={() => {
+                if (!elementSelection.selectMode && sessionId) {
+                  elementSelection.setExcludeSelectId(sessionId);
+                }
+                elementSelection.toggleSelectMode();
+              }}
               sx={{
                 p: 0.5,
                 ...(elementSelection.selectMode

frontend/src/app/pages/Dashboard/AgentCard.tsx

@@ -455,120 +455,131 @@ const AgentCard: React.FC<Props> = ({
         />
       ))}
 
-      {/* Header: always visible – entire bar is draggable */}
+      {/* Drag zone: header + metadata – entire region above separator is draggable */}
       <Box
         onPointerDown={handleDragPointerDown}
         onPointerMove={handleDragPointerMove}
         onPointerUp={handleDragPointerUp}
         sx={{
-          display: 'flex',
-          alignItems: 'center',
-          justifyContent: 'space-between',
-          mb: 1,
-          flexShrink: 0,
+          mx: -2,
+          mt: -2,
+          px: 2,
+          pt: 2,
+          pb: 1.5,
           cursor: isDragging ? 'grabbing' : 'grab',
           touchAction: 'none',
+          userSelect: 'none',
+          flexShrink: 0,
         }}
       >
         <Box
-          className="drag-handle"
           sx={{
             display: 'flex',
             alignItems: 'center',
-            mr: 0.5,
-            color: c.text.ghost,
+            justifyContent: 'space-between',
+            mb: 1,
+            flexShrink: 0,
           }}
         >
-          <DragIndicatorIcon sx={{ fontSize: 16 }} />
-        </Box>
-        <Box
-          sx={{
-            flex: 1,
-            minWidth: 0,
-            display: 'flex',
-            alignItems: 'center',
-            gap: 1,
-            borderRadius: 1,
-          }}
-        >
-          <Typography sx={{ color: c.text.primary, fontWeight: 600, fontSize: '0.95rem', overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
-            {session.name}
-          </Typography>
-          <Chip
-            label={session.status.replace('_', ' ')}
-            size="small"
+          <Box
+            className="drag-handle"
             sx={{
-              bgcolor: statusStyle.bg,
-              color: statusStyle.color,
-              fontWeight: 600,
-              fontSize: '0.7rem',
-              height: 22,
-              flexShrink: 0,
+              display: 'flex',
+              alignItems: 'center',
+              mr: 0.5,
+              color: c.text.ghost,
             }}
-          />
+          >
+            <DragIndicatorIcon sx={{ fontSize: 16 }} />
+          </Box>
+          <Box
+            sx={{
+              flex: 1,
+              minWidth: 0,
+              display: 'flex',
+              alignItems: 'center',
+              gap: 1,
+              borderRadius: 1,
+            }}
+          >
+            <Typography sx={{ color: c.text.primary, fontWeight: 600, fontSize: '0.95rem', overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
+              {session.name}
+            </Typography>
+            <Chip
+              label={session.status.replace('_', ' ')}
+              size="small"
+              sx={{
+                bgcolor: statusStyle.bg,
+                color: statusStyle.color,
+                fontWeight: 600,
+                fontSize: '0.7rem',
+                height: 22,
+                flexShrink: 0,
+              }}
+            />
+          </Box>
+          <Box
+            onPointerDown={(e) => e.stopPropagation()}
+            sx={{ display: 'flex', alignItems: 'center', gap: 0.5, flexShrink: 0, ml: 0.5 }}
+          >
+            {expanded ? (
+              <Tooltip title="Collapse">
+                <IconButton
+                  size="small"
+                  onClick={handleCollapse}
+                  onMouseDown={(e) => e.stopPropagation()}
+                  sx={{
+                    color: c.text.ghost,
+                    p: 0.5,
+                    '&:hover': { color: c.text.secondary, bgcolor: c.bg.secondary },
+                  }}
+                >
+                  <CloseIcon sx={{ fontSize: 16 }} />
+                </IconButton>
+              </Tooltip>
+            ) : (
+              <Tooltip title={isDraft ? 'Remove' : 'Close chat'}>
+                <IconButton
+                  size="small"
+                  onClick={handleRemove}
+                  onMouseDown={(e) => e.stopPropagation()}
+                  sx={{
+                    color: c.text.ghost,
+                    p: 0.5,
+                    '&:hover': { color: c.status.error, bgcolor: `${c.status.errorBg}` },
+                  }}
+                >
+                  <CloseIcon sx={{ fontSize: 16 }} />
+                </IconButton>
+              </Tooltip>
+            )}
+          </Box>
         </Box>
-        <Box
-          onPointerDown={(e) => e.stopPropagation()}
-          sx={{ display: 'flex', alignItems: 'center', gap: 0.5, flexShrink: 0, ml: 0.5 }}
-        >
-          {expanded ? (
-            <Tooltip title="Collapse">
-              <IconButton
-                size="small"
-                onClick={handleCollapse}
-                onMouseDown={(e) => e.stopPropagation()}
-                sx={{
-                  color: c.text.ghost,
-                  p: 0.5,
-                  '&:hover': { color: c.text.secondary, bgcolor: c.bg.secondary },
-                }}
-              >
-                <CloseIcon sx={{ fontSize: 16 }} />
-              </IconButton>
-            </Tooltip>
-          ) : (
-            <Tooltip title={isDraft ? 'Remove' : 'Close chat'}>
-              <IconButton
-                size="small"
-                onClick={handleRemove}
-                onMouseDown={(e) => e.stopPropagation()}
-                sx={{
-                  color: c.text.ghost,
-                  p: 0.5,
-                  '&:hover': { color: c.status.error, bgcolor: `${c.status.errorBg}` },
-                }}
-              >
-                <CloseIcon sx={{ fontSize: 16 }} />
-              </IconButton>
-            </Tooltip>
+
+        {/* Metadata row */}
+        <Box sx={{
+          display: isDraft && !expanded ? 'none' : 'flex',
+          gap: 1.5,
+          flexShrink: 0,
+          ...(isDraft && { visibility: 'hidden' }),
+        }}>
+          <Typography variant="caption" sx={{ color: c.text.tertiary }}>
+            {session.model}
+          </Typography>
+          <Typography variant="caption" sx={{ color: c.text.tertiary }}>
+            {session.mode}
+          </Typography>
+          <Typography variant="caption" sx={{ color: c.text.tertiary }}>
+            {formatDuration(session.created_at)}
+          </Typography>
+          {session.cost_usd > 0 && (
+            <Typography variant="caption" sx={{ color: c.accent.primary }}>
+              ${session.cost_usd.toFixed(4)}
+            </Typography>
           )}
         </Box>
       </Box>
 
-      {/* Metadata row */}
-      <Box sx={{
-        display: isDraft && !expanded ? 'none' : 'flex',
-        gap: 1.5,
-        mb: 1.5,
-        flexShrink: 0,
-        ...(isDraft && { visibility: 'hidden' }),
-      }}>
-        <Typography variant="caption" sx={{ color: c.text.tertiary }}>
-          {session.model}
-        </Typography>
-        <Typography variant="caption" sx={{ color: c.text.tertiary }}>
-          {session.mode}
-        </Typography>
-        <Typography variant="caption" sx={{ color: c.text.tertiary }}>
-          {formatDuration(session.created_at)}
-        </Typography>
-        {session.cost_usd > 0 && (
-          <Typography variant="caption" sx={{ color: c.accent.primary }}>
-            ${session.cost_usd.toFixed(4)}
-          </Typography>
-        )}
-      </Box>
-
       {/* Expanded: inline chat fills remaining space */}
       {expanded && (
         <Box

frontend/src/app/pages/Dashboard/BrowserCard.tsx

@@ -5,12 +5,14 @@ import IconButton from '@mui/material/IconButton';
 import Tooltip from '@mui/material/Tooltip';
 import InputBase from '@mui/material/InputBase';
 import LinearProgress from '@mui/material/LinearProgress';
+import CircularProgress from '@mui/material/CircularProgress';
 import LanguageIcon from '@mui/icons-material/Language';
 import ArrowBackIcon from '@mui/icons-material/ArrowBack';
 import ArrowForwardIcon from '@mui/icons-material/ArrowForward';
 import RefreshIcon from '@mui/icons-material/Refresh';
 import CloseIcon from '@mui/icons-material/Close';
 import LockIcon from '@mui/icons-material/Lock';
+import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined';
 import {
   setBrowserCardPosition,
   setBrowserCardSize,
@@ -19,6 +21,9 @@ import {
 } from '@/shared/state/dashboardLayoutSlice';
 import { useAppDispatch } from '@/shared/hooks';
 import { useClaudeTokens } from '@/shared/styles/ThemeContext';
+import { registerWebview, unregisterWebview, type BrowserWebview } from '@/shared/browserRegistry';
+import { useBrowserActivity } from '@/shared/useBrowserActivity';
+import { getActionLabel } from '@/shared/browserCommandHandler';
 
 type ResizeDir = 'n' | 's' | 'e' | 'w' | 'ne' | 'nw' | 'se' | 'sw';
 
@@ -45,19 +50,7 @@ const HANDLE_DEFS: { dir: ResizeDir; sx: Record<string, any> }[] = [
 
 const isElectron = navigator.userAgent.includes('Electron');
 
-interface WebviewElement extends HTMLElement {
-  src: string;
-  loadURL: (url: string) => Promise<void>;
-  goBack: () => void;
-  goForward: () => void;
-  reload: () => void;
-  canGoBack: () => boolean;
-  canGoForward: () => boolean;
-  getURL: () => string;
-  getTitle: () => string;
-  addEventListener: (event: string, listener: (...args: any[]) => void) => void;
-  removeEventListener: (event: string, listener: (...args: any[]) => void) => void;
-}
+type WebviewElement = BrowserWebview;
 
 interface Props {
   browserId: string;
@@ -89,6 +82,10 @@ const BrowserCard: React.FC<Props> = ({
   const c = useClaudeTokens();
   const dispatch = useAppDispatch();
   const webviewRef = useRef<WebviewElement | null>(null);
+  const activity = useBrowserActivity(browserId);
+  const agentActive = activity.active;
+  const agentAction = activity.action;
+  const lastAction = activity.lastAction;
 
   const [currentUrl, setCurrentUrl] = useState(url);
   const [urlBarValue, setUrlBarValue] = useState(url);
@@ -144,6 +141,14 @@ const BrowserCard: React.FC<Props> = ({
     };
   }, [browserId, dispatch]);
 
+  useEffect(() => {
+    if (!isElectron) return;
+    const wv = webviewRef.current;
+    if (!wv) return;
+    registerWebview(browserId, wv);
+    return () => { unregisterWebview(browserId); };
+  }, [browserId]);
+
   const navigate = useCallback((targetUrl: string) => {
     const finalUrl = ensureProtocol(targetUrl);
     setUrlBarValue(finalUrl);
@@ -306,10 +311,26 @@ const BrowserCard: React.FC<Props> = ({
 
   const isSecure = currentUrl.startsWith('https://');
 
+  const accentColor = c.accent.primary;
+  const accentHover = c.accent.hover;
+
+  const agentBorder = agentActive
+    ? `2px solid ${accentColor}`
+    : isSelected ? '2px solid #3b82f6' : `1px solid ${c.border.medium}`;
+
+  const agentShadow = agentActive
+    ? `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15`
+    : isDragging || isResizing
+      ? c.shadow.lg
+      : isSelected
+        ? `0 0 0 1px #3b82f6, ${c.shadow.md}`
+        : c.shadow.md;
+
   return (
     <Box
       data-select-type="browser-card"
       data-select-id={browserId}
+      data-select-meta={JSON.stringify({ name: pageTitle || 'Browser', url: currentUrl })}
       onClick={(e: React.MouseEvent) => {
         if (justDraggedRef.current) return;
         onCardSelect?.(browserId, 'browser', e.shiftKey);
@@ -321,21 +342,49 @@ const BrowserCard: React.FC<Props> = ({
         width: displayW,
         height: displayH,
         borderRadius: `${c.radius.lg}px`,
-        border: isSelected ? '2px solid #3b82f6' : `1px solid ${c.border.medium}`,
+        border: agentBorder,
         bgcolor: c.bg.surface,
-        boxShadow: isDragging || isResizing
-          ? c.shadow.lg
-          : isSelected
-            ? `0 0 0 1px #3b82f6, ${c.shadow.md}`
-            : c.shadow.md,
+        boxShadow: agentShadow,
         overflow: 'hidden',
         display: 'flex',
         flexDirection: 'column',
-        zIndex: (isDragging || isResizing) ? 100 : 1,
-        transition: noTransition ? 'none' : 'box-shadow 0.2s',
+        zIndex: (isDragging || isResizing) ? 100 : agentActive ? 50 : 1,
+        transition: noTransition ? 'none' : 'box-shadow 0.4s ease, border 0.3s ease',
         '&:hover .resize-handle': { opacity: 1 },
+        ...(agentActive && {
+          animation: 'agent-glow-pulse 2s ease-in-out infinite',
+          '@keyframes agent-glow-pulse': {
+            '0%, 100%': {
+              boxShadow: `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15`,
+            },
+            '50%': {
+              boxShadow: `0 0 0 3px ${accentColor}60, 0 0 28px ${accentColor}45, 0 0 56px ${accentColor}25`,
+            },
+          },
+        }),
       }}
     >
+      {/* Animated border glow (top edge overlay) */}
+      {agentActive && (
+        <Box
+          sx={{
+            position: 'absolute',
+            top: 0,
+            left: 0,
+            right: 0,
+            height: '2px',
+            zIndex: 20,
+            background: `linear-gradient(90deg, transparent, ${accentColor}, ${accentHover}, ${accentColor}, transparent)`,
+            backgroundSize: '200% 100%',
+            animation: 'border-shimmer 2s linear infinite',
+            '@keyframes border-shimmer': {
+              '0%': { backgroundPosition: '200% 0' },
+              '100%': { backgroundPosition: '-200% 0' },
+            },
+          }}
+        />
+      )}
+
       {/* Header / drag handle */}
       <Box
         onPointerDown={handleDragPointerDown}
@@ -347,12 +396,13 @@ const BrowserCard: React.FC<Props> = ({
           gap: 0.5,
           px: 1,
           py: 0.5,
-          bgcolor: c.bg.secondary,
-          borderBottom: `1px solid ${c.border.subtle}`,
+          bgcolor: agentActive ? `${accentColor}0a` : c.bg.secondary,
+          borderBottom: `1px solid ${agentActive ? `${accentColor}30` : c.border.subtle}`,
           cursor: isDragging ? 'grabbing' : 'grab',
           flexShrink: 0,
           minHeight: 36,
           userSelect: 'none',
+          transition: 'background 0.3s ease',
         }}
       >
         <LanguageIcon sx={{ fontSize: 16, color: c.accent.primary, flexShrink: 0 }} />
@@ -371,6 +421,44 @@ const BrowserCard: React.FC<Props> = ({
           {pageTitle || 'Browser'}
         </Typography>
 
+        {/* Agent activity badge */}
+        {agentActive && (
+          <Box
+            sx={{
+              display: 'inline-flex',
+              alignItems: 'center',
+              gap: 0.5,
+              px: 0.75,
+              py: 0.25,
+              borderRadius: '6px',
+              bgcolor: `${accentColor}18`,
+              border: `1px solid ${accentColor}30`,
+              animation: 'badge-fade-in 0.25s ease-out',
+              '@keyframes badge-fade-in': {
+                '0%': { opacity: 0, transform: 'scale(0.85)' },
+                '100%': { opacity: 1, transform: 'scale(1)' },
+              },
+            }}
+          >
+            <Box
+              sx={{
+                width: 6,
+                height: 6,
+                borderRadius: '50%',
+                bgcolor: accentColor,
+                animation: 'badge-dot-pulse 1.4s ease-in-out infinite',
+                '@keyframes badge-dot-pulse': {
+                  '0%, 100%': { opacity: 0.5, transform: 'scale(0.8)' },
+                  '50%': { opacity: 1, transform: 'scale(1.3)' },
+                },
+              }}
+            />
+            <Typography sx={{ fontSize: '0.65rem', fontWeight: 600, color: accentColor, lineHeight: 1 }}>
+              AI
+            </Typography>
+          </Box>
+        )}
+
         <Tooltip title="Back" placement="top">
           <span>
             <IconButton
@@ -457,14 +545,16 @@ const BrowserCard: React.FC<Props> = ({
         />
       </Box>
 
-      {/* Loading indicator */}
-      {loading && (
+      {/* Loading indicator — accent-colored when agent is navigating */}
+      {(loading || (agentActive && agentAction === 'navigate')) && (
         <LinearProgress
           sx={{
             height: 2,
             flexShrink: 0,
             bgcolor: 'transparent',
-            '& .MuiLinearProgress-bar': { bgcolor: c.accent.primary },
+            '& .MuiLinearProgress-bar': {
+              bgcolor: agentActive ? accentColor : c.accent.primary,
+            },
           }}
         />
       )}
@@ -506,6 +596,163 @@ const BrowserCard: React.FC<Props> = ({
             </Box>
           </Box>
         )}
+
+        {/* ===== Action micro-animations ===== */}
+
+        {/* Camera flash — screenshot */}
+        {(agentAction === 'screenshot' || lastAction === 'screenshot') && (
+          <Box
+            sx={{
+              position: 'absolute',
+              inset: 0,
+              bgcolor: '#fff',
+              pointerEvents: 'none',
+              zIndex: 15,
+              animation: 'camera-flash 0.4s ease-out forwards',
+              '@keyframes camera-flash': {
+                '0%': { opacity: 0.45 },
+                '100%': { opacity: 0 },
+              },
+            }}
+          />
+        )}
+
+        {/* Scanning line — get_text */}
+        {agentAction === 'get_text' && (
+          <Box
+            sx={{
+              position: 'absolute',
+              left: 0,
+              right: 0,
+              height: '3px',
+              zIndex: 15,
+              pointerEvents: 'none',
+              background: `linear-gradient(180deg, transparent, ${accentColor}90, transparent)`,
+              boxShadow: `0 0 12px ${accentColor}60`,
+              animation: 'scan-sweep 1.5s ease-in-out infinite',
+              '@keyframes scan-sweep': {
+                '0%': { top: '0%' },
+                '100%': { top: '100%' },
+              },
+            }}
+          />
+        )}
+
+        {/* Click ripple */}
+        {(agentAction === 'click' || lastAction === 'click') && (
+          <Box
+            sx={{
+              position: 'absolute',
+              top: '50%',
+              left: '50%',
+              width: 40,
+              height: 40,
+              borderRadius: '50%',
+              border: `2px solid ${accentColor}`,
+              transform: 'translate(-50%, -50%)',
+              pointerEvents: 'none',
+              zIndex: 15,
+              animation: 'click-ripple 0.5s ease-out forwards',
+              '@keyframes click-ripple': {
+                '0%': { opacity: 0.8, width: 10, height: 10, borderWidth: '2px' },
+                '100%': { opacity: 0, width: 60, height: 60, borderWidth: '1px' },
+              },
+            }}
+          />
+        )}
+
+        {/* Typing indicator */}
+        {agentAction === 'type' && (
+          <Box
+            sx={{
+              position: 'absolute',
+              bottom: 8,
+              left: '50%',
+              transform: 'translateX(-50%)',
+              display: 'flex',
+              gap: '4px',
+              alignItems: 'center',
+              px: 1,
+              py: 0.5,
+              borderRadius: '8px',
+              bgcolor: `${accentColor}20`,
+              border: `1px solid ${accentColor}40`,
+              zIndex: 15,
+              pointerEvents: 'none',
+            }}
+          >
+            {[0, 1, 2].map((i) => (
+              <Box
+                key={i}
+                sx={{
+                  width: 5,
+                  height: 5,
+                  borderRadius: '50%',
+                  bgcolor: accentColor,
+                  animation: `typing-dot 1s ease-in-out ${i * 0.15}s infinite`,
+                  '@keyframes typing-dot': {
+                    '0%, 60%, 100%': { opacity: 0.3, transform: 'scale(0.8)' },
+                    '30%': { opacity: 1, transform: 'scale(1.2)' },
+                  },
+                }}
+              />
+            ))}
+          </Box>
+        )}
+
+        {/* ===== Frosted glass overlay ===== */}
+        {agentActive && (
+          <Box
+            sx={{
+              position: 'absolute',
+              inset: 0,
+              zIndex: 16,
+              backdropFilter: 'blur(2px)',
+              bgcolor: 'rgba(0,0,0,0.15)',
+              display: 'flex',
+              flexDirection: 'column',
+              alignItems: 'center',
+              justifyContent: 'center',
+              gap: 1.5,
+              animation: 'overlay-fade-in 0.25s ease-out',
+              '@keyframes overlay-fade-in': {
+                '0%': { opacity: 0 },
+                '100%': { opacity: 1 },
+              },
+            }}
+          >
+            <CircularProgress
+              size={28}
+              thickness={3}
+              sx={{ color: accentColor }}
+            />
+            <Box
+              sx={{
+                display: 'flex',
+                alignItems: 'center',
+                gap: 0.75,
+                px: 1.5,
+                py: 0.75,
+                borderRadius: '10px',
+                bgcolor: 'rgba(0,0,0,0.55)',
+                backdropFilter: 'blur(8px)',
+                border: `1px solid ${accentColor}30`,
+              }}
+            >
+              <SmartToyOutlinedIcon sx={{ fontSize: 14, color: accentColor }} />
+              <Typography
+                sx={{
+                  fontSize: '0.75rem',
+                  fontWeight: 600,
+                  color: '#fff',
+                  letterSpacing: '0.02em',
+                }}
+              >
+                {getActionLabel(agentAction ?? '')}
+              </Typography>
+            </Box>
+          </Box>
+        )}
       </Box>
 
       {/* Resize handles */}

frontend/src/app/pages/Dashboard/Dashboard.tsx

@@ -29,6 +29,7 @@ import {
 import { fetchOutputs } from '@/shared/state/outputsSlice';
 import { generateDashboardName } from '@/shared/state/dashboardsSlice';
 import { dashboardWs } from '@/shared/ws/WebSocketManager';
+import { initBrowserCommandHandler } from '@/shared/browserCommandHandler';
 import AgentCard from './AgentCard';
 import DashboardViewCard from './DashboardViewCard';
 import BrowserCard from './BrowserCard';
@@ -168,7 +169,8 @@ const DashboardInner: React.FC = () => {
     dispatch(fetchLayout(dashboardId));
     dispatch(fetchOutputs());
     dashboardWs.connect();
-    return () => dashboardWs.disconnect();
+    const cleanupBrowserHandler = initBrowserCommandHandler();
+    return () => { cleanupBrowserHandler(); dashboardWs.disconnect(); };
   }, [dispatch, dashboardId]);
 
   useEffect(() => {

frontend/src/app/pages/Dashboard/DashboardToolbar.tsx

@@ -202,10 +202,12 @@ const DashboardToolbar = React.forwardRef<HTMLDivElement, Props>(
 
     const isExpanded = inputOpen || viewPickerOpen || historyOpen;
 
+    const prevInputOpenRef = useRef(inputOpen);
     useEffect(() => {
-      if (!inputOpen && elementSelection?.selectMode) {
+      if (prevInputOpenRef.current && !inputOpen && elementSelection?.selectMode) {
         elementSelection.setSelectMode(false);
       }
+      prevInputOpenRef.current = inputOpen;
     }, [inputOpen, elementSelection]);
 
     useEffect(() => {

frontend/src/shared/browserCommandHandler.ts

@@ -0,0 +1,159 @@
+import { getWebview, type BrowserWebview } from './browserRegistry';
+import { dashboardWs } from './ws/WebSocketManager';
+
+let initialized = false;
+
+export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate';
+
+export interface BrowserActivity {
+  action: BrowserAction;
+  detail?: string;
+}
+
+type ActivityListener = (browserId: string, activity: BrowserActivity | null) => void;
+
+const activityMap = new Map<string, BrowserActivity>();
+const listeners = new Set<ActivityListener>();
+
+function setActivity(browserId: string, activity: BrowserActivity | null) {
+  if (activity) {
+    activityMap.set(browserId, activity);
+  } else {
+    activityMap.delete(browserId);
+  }
+  listeners.forEach((fn) => fn(browserId, activity));
+}
+
+export function getActivity(browserId: string): BrowserActivity | null {
+  return activityMap.get(browserId) ?? null;
+}
+
+export function subscribeActivity(fn: ActivityListener): () => void {
+  listeners.add(fn);
+  return () => { listeners.delete(fn); };
+}
+
+const ACTION_LABELS: Record<string, string> = {
+  screenshot: 'Capturing...',
+  get_text: 'Reading...',
+  navigate: 'Navigating...',
+  click: 'Clicking...',
+  type: 'Typing...',
+  evaluate: 'Evaluating...',
+};
+
+export function getActionLabel(action: string): string {
+  return ACTION_LABELS[action] ?? 'Working...';
+}
+
+async function handleScreenshot(wv: BrowserWebview): Promise<Record<string, any>> {
+  const nativeImage = await wv.capturePage();
+  const dataUrl = nativeImage.toDataURL();
+  const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, '');
+  return { image: base64, url: wv.getURL(), title: wv.getTitle() };
+}
+
+async function handleGetText(wv: BrowserWebview): Promise<Record<string, any>> {
+  const text: string = await wv.executeJavaScript(
+    'document.body.innerText.substring(0, 15000)'
+  );
+  return { text, url: wv.getURL(), title: wv.getTitle() };
+}
+
+async function handleNavigate(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
+  const url = params.url as string;
+  if (!url) return { error: 'url parameter is required' };
+  await wv.loadURL(url);
+  return { text: `Navigated to ${url}`, url };
+}
+
+async function handleClick(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
+  const selector = params.selector as string;
+  if (!selector) return { error: 'selector parameter is required' };
+  const safeSelector = JSON.stringify(selector);
+  const code = `(()=>{const el=document.querySelector(${safeSelector});if(!el)return{error:'Element not found: '+${safeSelector}};el.click();return{text:'Clicked element: '+el.tagName.toLowerCase()+(el.id?'#'+el.id:'')}})()`;
+  const result = await wv.executeJavaScript(code);
+  return result;
+}
+
+async function handleType(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
+  const selector = params.selector as string;
+  const text = params.text as string;
+  if (!selector) return { error: 'selector parameter is required' };
+  if (text == null) return { error: 'text parameter is required' };
+  const safeSelector = JSON.stringify(selector);
+  const safeText = JSON.stringify(text);
+  const code = `(()=>{const el=document.querySelector(${safeSelector});if(!el)return{error:'Element not found: '+${safeSelector}};el.focus();el.value=${safeText};el.dispatchEvent(new Event('input',{bubbles:true}));el.dispatchEvent(new Event('change',{bubbles:true}));return{text:'Typed into: '+el.tagName.toLowerCase()+(el.id?'#'+el.id:'')}})()`;
+  const result = await wv.executeJavaScript(code);
+  return result;
+}
+
+async function handleEvaluate(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
+  const expression = params.expression as string;
+  if (!expression) return { error: 'expression parameter is required' };
+  try {
+    const result = await wv.executeJavaScript(expression);
+    const text = typeof result === 'string' ? result : JSON.stringify(result, null, 2);
+    return { text: text ?? 'undefined', url: wv.getURL() };
+  } catch (err: any) {
+    return { error: `JS evaluation error: ${err?.message || String(err)}` };
+  }
+}
+
+async function handleBrowserCommand(data: Record<string, any>) {
+  const { request_id, action, browser_id, params = {} } = data;
+  if (!request_id) return;
+
+  const wv = getWebview(browser_id);
+  if (!wv) {
+    dashboardWs.send('browser:result', {
+      request_id,
+      error: `Browser card '${browser_id}' not found or not an Electron webview`,
+    });
+    return;
+  }
+
+  const detail = params.url || params.selector || params.expression || undefined;
+  setActivity(browser_id, { action: action as BrowserAction, detail });
+
+  let result: Record<string, any>;
+  try {
+    switch (action) {
+      case 'screenshot':
+        result = await handleScreenshot(wv);
+        break;
+      case 'get_text':
+        result = await handleGetText(wv);
+        break;
+      case 'navigate':
+        result = await handleNavigate(wv, params);
+        break;
+      case 'click':
+        result = await handleClick(wv, params);
+        break;
+      case 'type':
+        result = await handleType(wv, params);
+        break;
+      case 'evaluate':
+        result = await handleEvaluate(wv, params);
+        break;
+      default:
+        result = { error: `Unknown browser action: ${action}` };
+    }
+  } catch (err: any) {
+    result = { error: `Browser command failed: ${err?.message || String(err)}` };
+  }
+
+  setActivity(browser_id, null);
+  dashboardWs.send('browser:result', { request_id, ...result });
+}
+
+export function initBrowserCommandHandler(): () => void {
+  if (initialized) return () => {};
+  initialized = true;
+  const unsub = dashboardWs.on('browser:command', handleBrowserCommand);
+  return () => {
+    unsub();
+    initialized = false;
+  };
+}

frontend/src/shared/browserRegistry.ts

@@ -0,0 +1,37 @@
+export interface BrowserWebview extends HTMLElement {
+  src: string;
+  loadURL: (url: string) => Promise<void>;
+  goBack: () => void;
+  goForward: () => void;
+  reload: () => void;
+  canGoBack: () => boolean;
+  canGoForward: () => boolean;
+  getURL: () => string;
+  getTitle: () => string;
+  capturePage: (rect?: { x: number; y: number; width: number; height: number }) => Promise<{
+    toDataURL: () => string;
+    toPNG: () => Buffer;
+  }>;
+  executeJavaScript: (code: string) => Promise<any>;
+  sendInputEvent: (event: any) => void;
+  addEventListener: (event: string, listener: (...args: any[]) => void) => void;
+  removeEventListener: (event: string, listener: (...args: any[]) => void) => void;
+}
+
+const registry = new Map<string, BrowserWebview>();
+
+export function registerWebview(browserId: string, wv: BrowserWebview): void {
+  registry.set(browserId, wv);
+}
+
+export function unregisterWebview(browserId: string): void {
+  registry.delete(browserId);
+}
+
+export function getWebview(browserId: string): BrowserWebview | undefined {
+  return registry.get(browserId);
+}
+
+export function getAllWebviews(): Map<string, BrowserWebview> {
+  return new Map(registry);
+}

frontend/src/shared/useBrowserActivity.ts

@@ -0,0 +1,66 @@
+import { useState, useEffect, useRef, useCallback } from 'react';
+import {
+  subscribeActivity,
+  getActivity,
+  type BrowserActivity,
+  type BrowserAction,
+} from './browserCommandHandler';
+
+export interface BrowserActivityState {
+  active: boolean;
+  action: BrowserAction | null;
+  detail: string | null;
+  /** The action that just completed — stays set briefly for exit animations */
+  lastAction: BrowserAction | null;
+}
+
+const EMPTY: BrowserActivityState = { active: false, action: null, detail: null, lastAction: null };
+
+export function useBrowserActivity(browserId: string): BrowserActivityState {
+  const [state, setState] = useState<BrowserActivityState>(() => {
+    const current = getActivity(browserId);
+    return current
+      ? { active: true, action: current.action, detail: current.detail ?? null, lastAction: null }
+      : EMPTY;
+  });
+
+  const lastActionTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
+
+  const handleChange = useCallback(
+    (changedId: string, activity: BrowserActivity | null) => {
+      if (changedId !== browserId) return;
+      if (activity) {
+        if (lastActionTimer.current) clearTimeout(lastActionTimer.current);
+        setState({
+          active: true,
+          action: activity.action,
+          detail: activity.detail ?? null,
+          lastAction: null,
+        });
+      } else {
+        setState((prev) => ({
+          active: false,
+          action: null,
+          detail: null,
+          lastAction: prev.action,
+        }));
+        lastActionTimer.current = setTimeout(() => {
+          setState((prev) => (prev.active ? prev : { ...prev, lastAction: null }));
+        }, 600);
+      }
+    },
+    [browserId],
+  );
+
+  useEffect(() => {
+    return subscribeActivity(handleChange);
+  }, [handleChange]);
+
+  useEffect(() => {
+    return () => {
+      if (lastActionTimer.current) clearTimeout(lastActionTimer.current);
+    };
+  }, []);
+
+  return state;
+}