From 87ef21919f5710aac021b7733eecd5b08d1f7962 Mon Sep 17 00:00:00 2001 From: haikdc Date: Sun, 15 Mar 2026 19:54:23 -0700 Subject: [PATCH] [Haik]: Browser control --- backend/apps/agents/agent_manager.py | 14 + backend/apps/agents/browser_mcp_server.py | 303 ++++++++++++++++++ backend/apps/agents/ws_manager.py | 29 ++ backend/main.py | 25 ++ backend/requirements.txt | 3 +- .../components/ElementSelectionContext.tsx | 12 +- .../app/components/useDomElementSelector.ts | 25 +- .../src/app/pages/AgentChat/AgentChat.tsx | 1 + .../src/app/pages/AgentChat/ChatInput.tsx | 52 ++- .../src/app/pages/Dashboard/AgentCard.tsx | 195 +++++------ .../src/app/pages/Dashboard/BrowserCard.tsx | 299 +++++++++++++++-- .../src/app/pages/Dashboard/Dashboard.tsx | 4 +- .../app/pages/Dashboard/DashboardToolbar.tsx | 4 +- frontend/src/shared/browserCommandHandler.ts | 159 +++++++++ frontend/src/shared/browserRegistry.ts | 37 +++ frontend/src/shared/useBrowserActivity.ts | 66 ++++ 16 files changed, 1093 insertions(+), 135 deletions(-) create mode 100644 backend/apps/agents/browser_mcp_server.py create mode 100644 frontend/src/shared/browserCommandHandler.ts create mode 100644 frontend/src/shared/browserRegistry.ts create mode 100644 frontend/src/shared/useBrowserActivity.ts diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index e7fee932..e6f06c51 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -2,6 +2,7 @@ import asyncio import json import logging import os +import sys import time from datetime import datetime from uuid import uuid4 @@ -547,6 +548,17 @@ class AgentManager: mcp_servers = await self._build_mcp_servers(session.allowed_tools) + browser_server_path = os.path.join( + os.path.dirname(__file__), "browser_mcp_server.py" + ) + backend_port = os.environ.get("OPENSWARM_PORT", "8324") + mcp_servers["openswarm-browser"] = { + "command": sys.executable, + "args": [browser_server_path], + "env": {"OPENSWARM_PORT": backend_port}, + "type": "stdio", + } + effective_allowed = [ t for t in session.allowed_tools if _builtin_perms.get(t, "always_allow") == "always_allow" @@ -569,6 +581,8 @@ class AgentManager: else: effective_allowed.append(f"mcp__{name}__*") + effective_allowed.append("mcp__openswarm-browser__*") + options_kwargs = { "model": session.model, "can_use_tool": can_use_tool, diff --git a/backend/apps/agents/browser_mcp_server.py b/backend/apps/agents/browser_mcp_server.py new file mode 100644 index 00000000..382178aa --- /dev/null +++ b/backend/apps/agents/browser_mcp_server.py @@ -0,0 +1,303 @@ +#!/usr/bin/env python3 +""" +Minimal stdio MCP server that exposes browser interaction tools. + +Launched as a subprocess by the Claude Agent SDK. Proxies tool calls +to the OpenSwarm backend via HTTP, which bridges them to the Electron +frontend via WebSocket where the actual webview lives. +""" + +import base64 +import json +import sys +import os +import urllib.request +import urllib.error +from io import BytesIO + +try: + from PIL import Image + HAS_PIL = True +except ImportError: + HAS_PIL = False + +BACKEND_PORT = os.environ.get("OPENSWARM_PORT", "8324") +BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/browser/command" + +TOOLS = [ + { + "name": "BrowserScreenshot", + "description": ( + "Capture a screenshot of the browser page. Returns the screenshot as a " + "base64-encoded PNG image. Use this to see what is currently displayed." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID to capture. Use the ID from the selected browser card context.", + }, + }, + "required": ["browser_id"], + }, + }, + { + "name": "BrowserGetText", + "description": ( + "Get the visible text content of the browser page. Returns the page's " + "innerText (up to 15000 characters)." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + }, + "required": ["browser_id"], + }, + }, + { + "name": "BrowserNavigate", + "description": "Navigate the browser to a URL.", + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "url": { + "type": "string", + "description": "The URL to navigate to.", + }, + }, + "required": ["browser_id", "url"], + }, + }, + { + "name": "BrowserClick", + "description": ( + "Click an element in the browser page identified by a CSS selector." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "selector": { + "type": "string", + "description": "CSS selector of the element to click.", + }, + }, + "required": ["browser_id", "selector"], + }, + }, + { + "name": "BrowserType", + "description": ( + "Type text into an input element in the browser page. Clears the " + "existing value first, then types the new text." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "selector": { + "type": "string", + "description": "CSS selector of the input element.", + }, + "text": { + "type": "string", + "description": "The text to type.", + }, + }, + "required": ["browser_id", "selector", "text"], + }, + }, + { + "name": "BrowserEvaluate", + "description": ( + "Evaluate a JavaScript expression in the browser page and return the result. " + "The expression is run via executeJavaScript on the webview." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "expression": { + "type": "string", + "description": "JavaScript expression to evaluate.", + }, + }, + "required": ["browser_id", "expression"], + }, + }, +] + + +def send_response(id_, result=None, error=None): + msg = {"jsonrpc": "2.0", "id": id_} + if error is not None: + msg["error"] = error + else: + msg["result"] = result + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def send_notification(method, params=None): + msg = {"jsonrpc": "2.0", "method": method} + if params is not None: + msg["params"] = params + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def call_backend(action: str, browser_id: str, params: dict | None = None) -> dict: + payload = json.dumps({ + "action": action, + "browser_id": browser_id, + "params": params or {}, + }).encode() + req = urllib.request.Request( + BACKEND_URL, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + body = e.read().decode() if e.fp else str(e) + return {"error": f"HTTP {e.code}: {body}"} + except Exception as e: + return {"error": str(e)} + + +MAX_IMAGE_B64_BYTES = 700_000 + + +def compress_screenshot(b64_png: str) -> tuple[str, str] | None: + """Resize and re-encode as JPEG to stay under the stdio buffer limit.""" + if not HAS_PIL: + return None + try: + raw = base64.b64decode(b64_png) + img = Image.open(BytesIO(raw)) + max_width = 1280 + if img.width > max_width: + ratio = max_width / img.width + img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) + buf = BytesIO() + img.convert("RGB").save(buf, format="JPEG", quality=55) + return base64.b64encode(buf.getvalue()).decode(), "image/jpeg" + except Exception: + return None + + +def handle_tool_call(tool_name: str, arguments: dict) -> dict: + browser_id = arguments.get("browser_id", "") + if not browser_id: + return {"content": [{"type": "text", "text": "Error: browser_id is required"}], "isError": True} + + action_map = { + "BrowserScreenshot": "screenshot", + "BrowserGetText": "get_text", + "BrowserNavigate": "navigate", + "BrowserClick": "click", + "BrowserType": "type", + "BrowserEvaluate": "evaluate", + } + action = action_map.get(tool_name) + if not action: + return {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True} + + params = {k: v for k, v in arguments.items() if k != "browser_id"} + result = call_backend(action, browser_id, params) + + if "error" in result: + return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} + + if action == "screenshot" and result.get("image"): + image_data = result["image"] + mime_type = "image/png" + + if len(image_data) > MAX_IMAGE_B64_BYTES: + compressed = compress_screenshot(image_data) + if compressed: + image_data, mime_type = compressed + + if len(image_data) > MAX_IMAGE_B64_BYTES: + return { + "content": [ + {"type": "text", "text": ( + f"Screenshot too large to return ({len(image_data)} bytes base64). " + f"URL: {result.get('url', 'unknown')}. " + "Use BrowserGetText to read the page content instead." + )}, + ], + } + + return { + "content": [ + {"type": "image", "data": image_data, "mimeType": mime_type}, + {"type": "text", "text": f"Screenshot captured. URL: {result.get('url', 'unknown')}"}, + ], + } + + text = result.get("text", result.get("data", json.dumps(result))) + return {"content": [{"type": "text", "text": str(text)}]} + + +def main(): + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + + method = msg.get("method") + id_ = msg.get("id") + params = msg.get("params", {}) + + if method == "initialize": + send_response(id_, { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": { + "name": "openswarm-browser", + "version": "1.0.0", + }, + }) + elif method == "notifications/initialized": + pass + elif method == "tools/list": + send_response(id_, {"tools": TOOLS}) + elif method == "tools/call": + tool_name = params.get("name", "") + arguments = params.get("arguments", {}) + result = handle_tool_call(tool_name, arguments) + send_response(id_, result) + elif method == "ping": + send_response(id_, {}) + elif id_ is not None: + send_response(id_, error={"code": -32601, "message": f"Method not found: {method}"}) + + +if __name__ == "__main__": + main() diff --git a/backend/apps/agents/ws_manager.py b/backend/apps/agents/ws_manager.py index 3d17b326..22651d8f 100644 --- a/backend/apps/agents/ws_manager.py +++ b/backend/apps/agents/ws_manager.py @@ -12,6 +12,7 @@ class ConnectionManager: self.connections: dict[str, list[WebSocket]] = {} self.global_connections: list[WebSocket] = [] self.pending_futures: dict[str, asyncio.Future] = {} + self.browser_futures: dict[str, asyncio.Future] = {} async def connect_session(self, session_id: str, websocket: WebSocket): await websocket.accept() @@ -85,4 +86,32 @@ class ConnectionManager: if future and not future.done(): future.set_result(decision) + async def send_browser_command( + self, request_id: str, action: str, browser_id: str, params: dict + ) -> dict: + """Send a browser command to the frontend and wait for the result.""" + future = asyncio.get_event_loop().create_future() + self.browser_futures[request_id] = future + + await self.broadcast_global("browser:command", { + "request_id": request_id, + "action": action, + "browser_id": browser_id, + "params": params, + }) + + try: + result = await asyncio.wait_for(future, timeout=30.0) + return result + except asyncio.TimeoutError: + return {"error": "Browser command timed out"} + finally: + self.browser_futures.pop(request_id, None) + + def resolve_browser_command(self, request_id: str, result: dict): + """Resolve a pending browser command Future with the frontend's result.""" + future = self.browser_futures.get(request_id) + if future and not future.done(): + future.set_result(result) + ws_manager = ConnectionManager() diff --git a/backend/main.py b/backend/main.py index 1f43b01e..451e4490 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,6 +1,8 @@ import os +from uuid import uuid4 from fastapi.responses import JSONResponse +from fastapi import Request from backend.config.Apps import MainApp from backend.apps.health.health import health from backend.apps.agents.agents import agents @@ -85,9 +87,32 @@ async def websocket_dashboard(websocket: WebSocket): "message": payload.get("message"), "updated_input": payload.get("updated_input"), }) + elif event == "browser:result": + ws_manager.resolve_browser_command( + payload.get("request_id", ""), + payload, + ) except WebSocketDisconnect: ws_manager.disconnect_global(websocket) + +@app.post("/api/browser/command") +async def browser_command(request: Request): + """HTTP endpoint called by the browser MCP server subprocess. + Proxies commands to the frontend via WebSocket and waits for results.""" + body = await request.json() + action = body.get("action", "") + browser_id = body.get("browser_id", "") + params = body.get("params", {}) + + if not action or not browser_id: + return JSONResponse({"error": "action and browser_id are required"}, status_code=400) + + request_id = uuid4().hex + result = await ws_manager.send_browser_command(request_id, action, browser_id, params) + return JSONResponse(result) + + if __name__ == "__main__": import argparse import uvicorn diff --git a/backend/requirements.txt b/backend/requirements.txt index 813faf0e..8ef474e6 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -8,4 +8,5 @@ langchain-openai==0.3.12 pytest==8.3.4 pytest-asyncio==0.25.2 typeguard==4.4.2 -python-dotenv==1.1.1 \ No newline at end of file +python-dotenv==1.1.1 +Pillow \ No newline at end of file diff --git a/frontend/src/app/components/ElementSelectionContext.tsx b/frontend/src/app/components/ElementSelectionContext.tsx index cfe88433..d42e5d1f 100644 --- a/frontend/src/app/components/ElementSelectionContext.tsx +++ b/frontend/src/app/components/ElementSelectionContext.tsx @@ -9,7 +9,7 @@ export interface SelectedElement { computedStyles: Record; screenshot?: string; boundingRect: { x: number; y: number; width: number; height: number }; - semanticType?: 'agent-card' | 'message' | 'tool-call' | 'tool-group' | 'view-card' | 'dom-element'; + semanticType?: 'agent-card' | 'message' | 'tool-call' | 'tool-group' | 'view-card' | 'browser-card' | 'dom-element'; semanticLabel?: string; semanticData?: Record; } @@ -18,6 +18,8 @@ interface ElementSelectionContextValue { selectMode: boolean; toggleSelectMode: () => void; setSelectMode: (active: boolean) => void; + excludeSelectId: string | null; + setExcludeSelectId: (id: string | null) => void; selectedElements: SelectedElement[]; addSelectedElement: (el: SelectedElement) => void; updateSelectedElement: (id: string, patch: Partial) => void; @@ -34,11 +36,15 @@ export function useElementSelection() { export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { const [selectMode, setSelectMode] = useState(false); + const [excludeSelectId, setExcludeSelectId] = useState(null); const [selectedElements, setSelectedElements] = useState([]); const iframeRef = useRef(null); const toggleSelectMode = useCallback(() => { - setSelectMode((prev) => !prev); + setSelectMode((prev) => { + if (prev) setExcludeSelectId(null); + return !prev; + }); }, []); const addSelectedElement = useCallback((el: SelectedElement) => { @@ -66,6 +72,8 @@ export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = selectMode, toggleSelectMode, setSelectMode, + excludeSelectId, + setExcludeSelectId, selectedElements, addSelectedElement, updateSelectedElement, diff --git a/frontend/src/app/components/useDomElementSelector.ts b/frontend/src/app/components/useDomElementSelector.ts index fb2f4068..79031c0d 100644 --- a/frontend/src/app/components/useDomElementSelector.ts +++ b/frontend/src/app/components/useDomElementSelector.ts @@ -31,12 +31,16 @@ const SEMANTIC_LABELS: Record = { 'tool-call': 'Tool Call', 'tool-group': 'Tool Group', 'view-card': 'View', + 'browser-card': 'Browser', }; -function findSelectableAncestor(target: Element): Element | null { +function findSelectableAncestor(target: Element, excludeId?: string | null): Element | null { let current: Element | null = target; while (current) { - if (current.hasAttribute(SELECT_ATTR)) return current; + if (current.hasAttribute(SELECT_ATTR)) { + if (excludeId && current.getAttribute(SELECT_ID_ATTR) === excludeId) return null; + return current; + } current = current.parentElement; } return null; @@ -114,6 +118,11 @@ export function useDomElementSelector(): DomSelectorState { const isDraggingRef = useRef(false); const dragBoundsRef = useRef<{ left: number; top: number; right: number; bottom: number } | null>(null); + const excludeIdRef = useRef(null); + useEffect(() => { + excludeIdRef.current = ctx?.excludeSelectId ?? null; + }, [ctx?.excludeSelectId]); + const selectedIdsRef = useRef(new Map()); useEffect(() => { const map = new Map(); @@ -161,10 +170,12 @@ export function useDomElementSelector(): DomSelectorState { const allSelectables = document.querySelectorAll(`[${SELECT_ATTR}]`); const preview: DragPreviewElement[] = []; const seen = new Set(); + const excId = excludeIdRef.current; allSelectables.forEach((el) => { + const selectId = el.getAttribute(SELECT_ID_ATTR) || ''; + if (excId && selectId === excId) return; const rect = el.getBoundingClientRect(); if (rectsIntersect(b, { left: rect.left, top: rect.top, right: rect.right, bottom: rect.bottom })) { - const selectId = el.getAttribute(SELECT_ID_ATTR) || ''; if (seen.has(selectId)) return; seen.add(selectId); const type = el.getAttribute(SELECT_ATTR) || ''; @@ -200,7 +211,7 @@ export function useDomElementSelector(): DomSelectorState { return; } - const selectable = findSelectableAncestor(target); + const selectable = findSelectableAncestor(target, excludeIdRef.current); if (!selectable) { setOverlay(EMPTY_OVERLAY); hoveredRef.current = null; @@ -231,7 +242,7 @@ export function useDomElementSelector(): DomSelectorState { if (e.button !== 0) return; const target = e.target as Element; // Only start drag on "empty" canvas areas (not on selectable elements) - if (target && findSelectableAncestor(target)) return; + if (target && findSelectableAncestor(target, excludeIdRef.current)) return; dragOriginRef.current = { x: e.clientX, y: e.clientY }; isDraggingRef.current = false; }, []); @@ -250,7 +261,10 @@ export function useDomElementSelector(): DomSelectorState { const allSelectables = document.querySelectorAll(`[${SELECT_ATTR}]`); const processed = new Set(); + const excId = excludeIdRef.current; allSelectables.forEach((el) => { + const selectId = el.getAttribute(SELECT_ID_ATTR) || ''; + if (excId && selectId === excId) return; const rect = el.getBoundingClientRect(); const elRect = { left: rect.left, @@ -260,7 +274,6 @@ export function useDomElementSelector(): DomSelectorState { }; if (rectsIntersect(dr, elRect)) { - const selectId = el.getAttribute(SELECT_ID_ATTR) || ''; if (processed.has(selectId)) return; processed.add(selectId); diff --git a/frontend/src/app/pages/AgentChat/AgentChat.tsx b/frontend/src/app/pages/AgentChat/AgentChat.tsx index a341c848..580fc187 100644 --- a/frontend/src/app/pages/AgentChat/AgentChat.tsx +++ b/frontend/src/app/pages/AgentChat/AgentChat.tsx @@ -629,6 +629,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose isRunning={!isDraft && (session.status === 'running' || session.status === 'waiting_approval')} onStop={handleStop} contextEstimate={contextEstimate} + sessionId={id} /> diff --git a/frontend/src/app/pages/AgentChat/ChatInput.tsx b/frontend/src/app/pages/AgentChat/ChatInput.tsx index 6822a267..6369464a 100644 --- a/frontend/src/app/pages/AgentChat/ChatInput.tsx +++ b/frontend/src/app/pages/AgentChat/ChatInput.tsx @@ -26,6 +26,7 @@ import AttachFileIcon from '@mui/icons-material/AttachFile'; import AdsClickIcon from '@mui/icons-material/AdsClick'; import CommandPicker, { CommandPickerItem, getToolGroupIcon } from '@/app/components/CommandPicker'; import { useElementSelection, SelectedElement } from '@/app/components/ElementSelectionContext'; +import { getWebview } from '@/shared/browserRegistry'; import { API_BASE } from '@/shared/config'; import { ContextPath } from '@/app/components/DirectoryBrowser'; import { @@ -71,6 +72,7 @@ interface Props { contextEstimate?: { used: number; limit: number }; embedded?: boolean; autoFocus?: boolean; + sessionId?: string; } export interface ChatInputHandle { @@ -128,7 +130,7 @@ const ContextRing: React.FC<{ used: number; limit: number; accentColor: string; ); }; -const ChatInput = forwardRef(({ onSend, disabled, mode, onModeChange, model, onModelChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus }, ref) => { +const ChatInput = forwardRef(({ onSend, disabled, mode, onModeChange, model, onModelChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus, sessionId }, ref) => { const c = useClaudeTokens(); const editorRef = useRef(null); const containerRef = useRef(null); @@ -272,7 +274,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, } }, []); - const handleSend = useCallback(() => { + const handleSend = useCallback(async () => { const editor = editorRef.current; if (!editor || disabled) return; const serialized = serializeEditorContent(editor, attachedSkillsRef.current); @@ -286,14 +288,47 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, if (selectedEls.length > 0) { const lines: string[] = ['\n\n---\nSelected UI Elements:\n']; - selectedEls.forEach((el, i) => { - if (el.semanticType && el.semanticData) { + for (let i = 0; i < selectedEls.length; i++) { + const el = selectedEls[i]; + + if (el.semanticType === 'browser-card' && el.semanticData?.selectId) { + const wv = getWebview(el.semanticData.selectId as string); + if (wv) { + const url = el.semanticData.url || wv.getURL(); + const title = el.semanticData.name || wv.getTitle(); + lines.push(`${i + 1}. [Browser Card] ${title}`); + lines.push(` ID: ${el.semanticData.selectId}`); + lines.push(` URL: ${url}`); + + try { + const nativeImage = await wv.capturePage(); + const dataUrl = nativeImage.toDataURL(); + const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, ''); + allImages.push({ data: base64, media_type: 'image/png' }); + lines.push(` [Screenshot captured and attached as image]`); + } catch { /* screenshot unavailable */ } + + try { + const pageText: string = await wv.executeJavaScript( + 'document.body.innerText.substring(0, 15000)' + ); + if (pageText?.trim()) { + lines.push(` Page text content:\n ---\n${pageText.trim().split('\n').map(l => ' ' + l).join('\n')}\n ---`); + } + } catch { /* text extraction unavailable */ } + } else { + lines.push(`${i + 1}. [Browser Card] ${el.semanticLabel || ''}`); + if (el.semanticData.selectId) lines.push(` ID: ${el.semanticData.selectId}`); + if (el.semanticData.url) lines.push(` URL: ${el.semanticData.url}`); + } + } else if (el.semanticType && el.semanticData) { const typeLabel = { 'agent-card': 'Agent Card', 'message': 'Message', 'tool-call': 'Tool Call', 'tool-group': 'Tool Group', 'view-card': 'View Card', + 'browser-card': 'Browser Card', 'dom-element': 'Element', }[el.semanticType] || el.semanticType; lines.push(`${i + 1}. [${typeLabel}] ${el.semanticLabel || ''}`); @@ -319,7 +354,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, const base64 = el.screenshot.replace(/^data:image\/\w+;base64,/, ''); allImages.push({ data: base64, media_type: 'image/png' }); } - }); + } trimmed += lines.join('\n'); } @@ -964,7 +999,12 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, { + if (!elementSelection.selectMode && sessionId) { + elementSelection.setExcludeSelectId(sessionId); + } + elementSelection.toggleSelectMode(); + }} sx={{ p: 0.5, ...(elementSelection.selectMode diff --git a/frontend/src/app/pages/Dashboard/AgentCard.tsx b/frontend/src/app/pages/Dashboard/AgentCard.tsx index b9e9639b..e15cd1c7 100644 --- a/frontend/src/app/pages/Dashboard/AgentCard.tsx +++ b/frontend/src/app/pages/Dashboard/AgentCard.tsx @@ -455,120 +455,131 @@ const AgentCard: React.FC = ({ /> ))} - {/* Header: always visible – entire bar is draggable */} + {/* Drag zone: header + metadata – entire region above separator is draggable */} - - - - - {session.name} - - + > + + + + + {session.name} + + + + e.stopPropagation()} + sx={{ display: 'flex', alignItems: 'center', gap: 0.5, flexShrink: 0, ml: 0.5 }} + > + {expanded ? ( + + e.stopPropagation()} + sx={{ + color: c.text.ghost, + p: 0.5, + '&:hover': { color: c.text.secondary, bgcolor: c.bg.secondary }, + }} + > + + + + ) : ( + + e.stopPropagation()} + sx={{ + color: c.text.ghost, + p: 0.5, + '&:hover': { color: c.status.error, bgcolor: `${c.status.errorBg}` }, + }} + > + + + + )} + - e.stopPropagation()} - sx={{ display: 'flex', alignItems: 'center', gap: 0.5, flexShrink: 0, ml: 0.5 }} - > - {expanded ? ( - - e.stopPropagation()} - sx={{ - color: c.text.ghost, - p: 0.5, - '&:hover': { color: c.text.secondary, bgcolor: c.bg.secondary }, - }} - > - - - - ) : ( - - e.stopPropagation()} - sx={{ - color: c.text.ghost, - p: 0.5, - '&:hover': { color: c.status.error, bgcolor: `${c.status.errorBg}` }, - }} - > - - - + + {/* Metadata row */} + + + {session.model} + + + {session.mode} + + + {formatDuration(session.created_at)} + + {session.cost_usd > 0 && ( + + ${session.cost_usd.toFixed(4)} + )} - {/* Metadata row */} - - - {session.model} - - - {session.mode} - - - {formatDuration(session.created_at)} - - {session.cost_usd > 0 && ( - - ${session.cost_usd.toFixed(4)} - - )} - - {/* Expanded: inline chat fills remaining space */} {expanded && ( }[] = [ const isElectron = navigator.userAgent.includes('Electron'); -interface WebviewElement extends HTMLElement { - src: string; - loadURL: (url: string) => Promise; - goBack: () => void; - goForward: () => void; - reload: () => void; - canGoBack: () => boolean; - canGoForward: () => boolean; - getURL: () => string; - getTitle: () => string; - addEventListener: (event: string, listener: (...args: any[]) => void) => void; - removeEventListener: (event: string, listener: (...args: any[]) => void) => void; -} +type WebviewElement = BrowserWebview; interface Props { browserId: string; @@ -89,6 +82,10 @@ const BrowserCard: React.FC = ({ const c = useClaudeTokens(); const dispatch = useAppDispatch(); const webviewRef = useRef(null); + const activity = useBrowserActivity(browserId); + const agentActive = activity.active; + const agentAction = activity.action; + const lastAction = activity.lastAction; const [currentUrl, setCurrentUrl] = useState(url); const [urlBarValue, setUrlBarValue] = useState(url); @@ -144,6 +141,14 @@ const BrowserCard: React.FC = ({ }; }, [browserId, dispatch]); + useEffect(() => { + if (!isElectron) return; + const wv = webviewRef.current; + if (!wv) return; + registerWebview(browserId, wv); + return () => { unregisterWebview(browserId); }; + }, [browserId]); + const navigate = useCallback((targetUrl: string) => { const finalUrl = ensureProtocol(targetUrl); setUrlBarValue(finalUrl); @@ -306,10 +311,26 @@ const BrowserCard: React.FC = ({ const isSecure = currentUrl.startsWith('https://'); + const accentColor = c.accent.primary; + const accentHover = c.accent.hover; + + const agentBorder = agentActive + ? `2px solid ${accentColor}` + : isSelected ? '2px solid #3b82f6' : `1px solid ${c.border.medium}`; + + const agentShadow = agentActive + ? `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15` + : isDragging || isResizing + ? c.shadow.lg + : isSelected + ? `0 0 0 1px #3b82f6, ${c.shadow.md}` + : c.shadow.md; + return ( { if (justDraggedRef.current) return; onCardSelect?.(browserId, 'browser', e.shiftKey); @@ -321,21 +342,49 @@ const BrowserCard: React.FC = ({ width: displayW, height: displayH, borderRadius: `${c.radius.lg}px`, - border: isSelected ? '2px solid #3b82f6' : `1px solid ${c.border.medium}`, + border: agentBorder, bgcolor: c.bg.surface, - boxShadow: isDragging || isResizing - ? c.shadow.lg - : isSelected - ? `0 0 0 1px #3b82f6, ${c.shadow.md}` - : c.shadow.md, + boxShadow: agentShadow, overflow: 'hidden', display: 'flex', flexDirection: 'column', - zIndex: (isDragging || isResizing) ? 100 : 1, - transition: noTransition ? 'none' : 'box-shadow 0.2s', + zIndex: (isDragging || isResizing) ? 100 : agentActive ? 50 : 1, + transition: noTransition ? 'none' : 'box-shadow 0.4s ease, border 0.3s ease', '&:hover .resize-handle': { opacity: 1 }, + ...(agentActive && { + animation: 'agent-glow-pulse 2s ease-in-out infinite', + '@keyframes agent-glow-pulse': { + '0%, 100%': { + boxShadow: `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15`, + }, + '50%': { + boxShadow: `0 0 0 3px ${accentColor}60, 0 0 28px ${accentColor}45, 0 0 56px ${accentColor}25`, + }, + }, + }), }} > + {/* Animated border glow (top edge overlay) */} + {agentActive && ( + + )} + {/* Header / drag handle */} = ({ gap: 0.5, px: 1, py: 0.5, - bgcolor: c.bg.secondary, - borderBottom: `1px solid ${c.border.subtle}`, + bgcolor: agentActive ? `${accentColor}0a` : c.bg.secondary, + borderBottom: `1px solid ${agentActive ? `${accentColor}30` : c.border.subtle}`, cursor: isDragging ? 'grabbing' : 'grab', flexShrink: 0, minHeight: 36, userSelect: 'none', + transition: 'background 0.3s ease', }} > @@ -371,6 +421,44 @@ const BrowserCard: React.FC = ({ {pageTitle || 'Browser'} + {/* Agent activity badge */} + {agentActive && ( + + + + AI + + + )} + = ({ /> - {/* Loading indicator */} - {loading && ( + {/* Loading indicator — accent-colored when agent is navigating */} + {(loading || (agentActive && agentAction === 'navigate')) && ( )} @@ -506,6 +596,163 @@ const BrowserCard: React.FC = ({ )} + + {/* ===== Action micro-animations ===== */} + + {/* Camera flash — screenshot */} + {(agentAction === 'screenshot' || lastAction === 'screenshot') && ( + + )} + + {/* Scanning line — get_text */} + {agentAction === 'get_text' && ( + + )} + + {/* Click ripple */} + {(agentAction === 'click' || lastAction === 'click') && ( + + )} + + {/* Typing indicator */} + {agentAction === 'type' && ( + + {[0, 1, 2].map((i) => ( + + ))} + + )} + + {/* ===== Frosted glass overlay ===== */} + {agentActive && ( + + + + + + {getActionLabel(agentAction ?? '')} + + + + )} {/* Resize handles */} diff --git a/frontend/src/app/pages/Dashboard/Dashboard.tsx b/frontend/src/app/pages/Dashboard/Dashboard.tsx index 4b44d41f..38dc4ae8 100644 --- a/frontend/src/app/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/app/pages/Dashboard/Dashboard.tsx @@ -29,6 +29,7 @@ import { import { fetchOutputs } from '@/shared/state/outputsSlice'; import { generateDashboardName } from '@/shared/state/dashboardsSlice'; import { dashboardWs } from '@/shared/ws/WebSocketManager'; +import { initBrowserCommandHandler } from '@/shared/browserCommandHandler'; import AgentCard from './AgentCard'; import DashboardViewCard from './DashboardViewCard'; import BrowserCard from './BrowserCard'; @@ -168,7 +169,8 @@ const DashboardInner: React.FC = () => { dispatch(fetchLayout(dashboardId)); dispatch(fetchOutputs()); dashboardWs.connect(); - return () => dashboardWs.disconnect(); + const cleanupBrowserHandler = initBrowserCommandHandler(); + return () => { cleanupBrowserHandler(); dashboardWs.disconnect(); }; }, [dispatch, dashboardId]); useEffect(() => { diff --git a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx index 7f05345a..dd179b36 100644 --- a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx @@ -202,10 +202,12 @@ const DashboardToolbar = React.forwardRef( const isExpanded = inputOpen || viewPickerOpen || historyOpen; + const prevInputOpenRef = useRef(inputOpen); useEffect(() => { - if (!inputOpen && elementSelection?.selectMode) { + if (prevInputOpenRef.current && !inputOpen && elementSelection?.selectMode) { elementSelection.setSelectMode(false); } + prevInputOpenRef.current = inputOpen; }, [inputOpen, elementSelection]); useEffect(() => { diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts new file mode 100644 index 00000000..eebdcfc8 --- /dev/null +++ b/frontend/src/shared/browserCommandHandler.ts @@ -0,0 +1,159 @@ +import { getWebview, type BrowserWebview } from './browserRegistry'; +import { dashboardWs } from './ws/WebSocketManager'; + +let initialized = false; + +export type BrowserAction = 'screenshot' | 'get_text' | 'navigate' | 'click' | 'type' | 'evaluate'; + +export interface BrowserActivity { + action: BrowserAction; + detail?: string; +} + +type ActivityListener = (browserId: string, activity: BrowserActivity | null) => void; + +const activityMap = new Map(); +const listeners = new Set(); + +function setActivity(browserId: string, activity: BrowserActivity | null) { + if (activity) { + activityMap.set(browserId, activity); + } else { + activityMap.delete(browserId); + } + listeners.forEach((fn) => fn(browserId, activity)); +} + +export function getActivity(browserId: string): BrowserActivity | null { + return activityMap.get(browserId) ?? null; +} + +export function subscribeActivity(fn: ActivityListener): () => void { + listeners.add(fn); + return () => { listeners.delete(fn); }; +} + +const ACTION_LABELS: Record = { + screenshot: 'Capturing...', + get_text: 'Reading...', + navigate: 'Navigating...', + click: 'Clicking...', + type: 'Typing...', + evaluate: 'Evaluating...', +}; + +export function getActionLabel(action: string): string { + return ACTION_LABELS[action] ?? 'Working...'; +} + +async function handleScreenshot(wv: BrowserWebview): Promise> { + const nativeImage = await wv.capturePage(); + const dataUrl = nativeImage.toDataURL(); + const base64 = dataUrl.replace(/^data:image\/\w+;base64,/, ''); + return { image: base64, url: wv.getURL(), title: wv.getTitle() }; +} + +async function handleGetText(wv: BrowserWebview): Promise> { + const text: string = await wv.executeJavaScript( + 'document.body.innerText.substring(0, 15000)' + ); + return { text, url: wv.getURL(), title: wv.getTitle() }; +} + +async function handleNavigate(wv: BrowserWebview, params: Record): Promise> { + const url = params.url as string; + if (!url) return { error: 'url parameter is required' }; + await wv.loadURL(url); + return { text: `Navigated to ${url}`, url }; +} + +async function handleClick(wv: BrowserWebview, params: Record): Promise> { + const selector = params.selector as string; + if (!selector) return { error: 'selector parameter is required' }; + const safeSelector = JSON.stringify(selector); + const code = `(()=>{const el=document.querySelector(${safeSelector});if(!el)return{error:'Element not found: '+${safeSelector}};el.click();return{text:'Clicked element: '+el.tagName.toLowerCase()+(el.id?'#'+el.id:'')}})()`; + const result = await wv.executeJavaScript(code); + return result; +} + +async function handleType(wv: BrowserWebview, params: Record): Promise> { + const selector = params.selector as string; + const text = params.text as string; + if (!selector) return { error: 'selector parameter is required' }; + if (text == null) return { error: 'text parameter is required' }; + const safeSelector = JSON.stringify(selector); + const safeText = JSON.stringify(text); + const code = `(()=>{const el=document.querySelector(${safeSelector});if(!el)return{error:'Element not found: '+${safeSelector}};el.focus();el.value=${safeText};el.dispatchEvent(new Event('input',{bubbles:true}));el.dispatchEvent(new Event('change',{bubbles:true}));return{text:'Typed into: '+el.tagName.toLowerCase()+(el.id?'#'+el.id:'')}})()`; + const result = await wv.executeJavaScript(code); + return result; +} + +async function handleEvaluate(wv: BrowserWebview, params: Record): Promise> { + const expression = params.expression as string; + if (!expression) return { error: 'expression parameter is required' }; + try { + const result = await wv.executeJavaScript(expression); + const text = typeof result === 'string' ? result : JSON.stringify(result, null, 2); + return { text: text ?? 'undefined', url: wv.getURL() }; + } catch (err: any) { + return { error: `JS evaluation error: ${err?.message || String(err)}` }; + } +} + +async function handleBrowserCommand(data: Record) { + const { request_id, action, browser_id, params = {} } = data; + if (!request_id) return; + + const wv = getWebview(browser_id); + if (!wv) { + dashboardWs.send('browser:result', { + request_id, + error: `Browser card '${browser_id}' not found or not an Electron webview`, + }); + return; + } + + const detail = params.url || params.selector || params.expression || undefined; + setActivity(browser_id, { action: action as BrowserAction, detail }); + + let result: Record; + try { + switch (action) { + case 'screenshot': + result = await handleScreenshot(wv); + break; + case 'get_text': + result = await handleGetText(wv); + break; + case 'navigate': + result = await handleNavigate(wv, params); + break; + case 'click': + result = await handleClick(wv, params); + break; + case 'type': + result = await handleType(wv, params); + break; + case 'evaluate': + result = await handleEvaluate(wv, params); + break; + default: + result = { error: `Unknown browser action: ${action}` }; + } + } catch (err: any) { + result = { error: `Browser command failed: ${err?.message || String(err)}` }; + } + + setActivity(browser_id, null); + dashboardWs.send('browser:result', { request_id, ...result }); +} + +export function initBrowserCommandHandler(): () => void { + if (initialized) return () => {}; + initialized = true; + const unsub = dashboardWs.on('browser:command', handleBrowserCommand); + return () => { + unsub(); + initialized = false; + }; +} diff --git a/frontend/src/shared/browserRegistry.ts b/frontend/src/shared/browserRegistry.ts new file mode 100644 index 00000000..97f65dc9 --- /dev/null +++ b/frontend/src/shared/browserRegistry.ts @@ -0,0 +1,37 @@ +export interface BrowserWebview extends HTMLElement { + src: string; + loadURL: (url: string) => Promise; + goBack: () => void; + goForward: () => void; + reload: () => void; + canGoBack: () => boolean; + canGoForward: () => boolean; + getURL: () => string; + getTitle: () => string; + capturePage: (rect?: { x: number; y: number; width: number; height: number }) => Promise<{ + toDataURL: () => string; + toPNG: () => Buffer; + }>; + executeJavaScript: (code: string) => Promise; + sendInputEvent: (event: any) => void; + addEventListener: (event: string, listener: (...args: any[]) => void) => void; + removeEventListener: (event: string, listener: (...args: any[]) => void) => void; +} + +const registry = new Map(); + +export function registerWebview(browserId: string, wv: BrowserWebview): void { + registry.set(browserId, wv); +} + +export function unregisterWebview(browserId: string): void { + registry.delete(browserId); +} + +export function getWebview(browserId: string): BrowserWebview | undefined { + return registry.get(browserId); +} + +export function getAllWebviews(): Map { + return new Map(registry); +} diff --git a/frontend/src/shared/useBrowserActivity.ts b/frontend/src/shared/useBrowserActivity.ts new file mode 100644 index 00000000..14a19f78 --- /dev/null +++ b/frontend/src/shared/useBrowserActivity.ts @@ -0,0 +1,66 @@ +import { useState, useEffect, useRef, useCallback } from 'react'; +import { + subscribeActivity, + getActivity, + type BrowserActivity, + type BrowserAction, +} from './browserCommandHandler'; + +export interface BrowserActivityState { + active: boolean; + action: BrowserAction | null; + detail: string | null; + /** The action that just completed — stays set briefly for exit animations */ + lastAction: BrowserAction | null; +} + +const EMPTY: BrowserActivityState = { active: false, action: null, detail: null, lastAction: null }; + +export function useBrowserActivity(browserId: string): BrowserActivityState { + const [state, setState] = useState(() => { + const current = getActivity(browserId); + return current + ? { active: true, action: current.action, detail: current.detail ?? null, lastAction: null } + : EMPTY; + }); + + const lastActionTimer = useRef | null>(null); + + const handleChange = useCallback( + (changedId: string, activity: BrowserActivity | null) => { + if (changedId !== browserId) return; + if (activity) { + if (lastActionTimer.current) clearTimeout(lastActionTimer.current); + setState({ + active: true, + action: activity.action, + detail: activity.detail ?? null, + lastAction: null, + }); + } else { + setState((prev) => ({ + active: false, + action: null, + detail: null, + lastAction: prev.action, + })); + lastActionTimer.current = setTimeout(() => { + setState((prev) => (prev.active ? prev : { ...prev, lastAction: null })); + }, 600); + } + }, + [browserId], + ); + + useEffect(() => { + return subscribeActivity(handleChange); + }, [handleChange]); + + useEffect(() => { + return () => { + if (lastActionTimer.current) clearTimeout(lastActionTimer.current); + }; + }, []); + + return state; +}