diff --git a/.gitignore b/.gitignore index 47256d70..76fb66c7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,9 @@ backend/data/** # Electron build artifacts electron/dist/ electron/python-env/ +electron/build-staging/ electron/node_modules/ +electron/package-lock.json # Frontend build output frontend/dist/ \ No newline at end of file diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index c92b6b10..a8307969 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -66,6 +66,8 @@ FULL_TOOLS = [ "TaskOutput", "TaskStop", "CronCreate", "CronList", "CronDelete", "RenderOutput", + "InvokeAgent", + "Agent", ] def _get_denied_tool_names(tool) -> set[str]: @@ -232,8 +234,12 @@ class AgentManager: + "\n" ) - def _build_browser_context(self, dashboard_id: str | None) -> str | None: - """Build a context block listing browser cards and delegation instructions.""" + def _build_browser_context(self, dashboard_id: str | None, selected_browser_ids: list[str] | None = None) -> str | None: + """Build a context block listing browser cards and delegation instructions. + + Only browser cards explicitly selected by the user are included. + If none are selected, no browser card details are exposed. + """ if not dashboard_id: return None try: @@ -246,34 +252,39 @@ class AgentManager: lines = [ "", - "You have access to browser automation through the BrowserAgent and BrowserAgents tools.", + "You have access to browser automation through the CreateBrowserAgent, BrowserAgent, and BrowserAgents tools.", "", - "- **BrowserAgent(task, browser_id?, url?)**: Delegate a single browser task to a dedicated browser agent. " + "- **CreateBrowserAgent(task, url?)**: Create a new browser card and run a task on it. " + "Use this when you need a fresh browser. Optionally provide a starting URL.", + "- **BrowserAgent(browser_id, task)**: Delegate a task to an existing browser card. " "The browser agent will autonomously navigate, click, type, and interact with the page, then return a summary and screenshot.", - "- **BrowserAgents(tasks)**: Run multiple browser tasks in parallel, each on a different browser.", + "- **BrowserAgents(tasks)**: Run multiple browser tasks in parallel on existing browser cards. " + "Each task requires a browser_id.", "", "You do NOT have direct access to low-level browser tools (click, type, screenshot, etc.). " "Instead, describe what you want accomplished and the browser agent will handle the details.", - "", - "If you omit browser_id, a new browser card will be auto-created. " - "If you provide a url without a browser_id, the new browser navigates there first.", ] - if browser_cards: - lines.append("") - lines.append("Available browser cards on the dashboard:") - for card in browser_cards.values(): - bid = card.get("browser_id", "") - tabs = card.get("tabs", []) - active_tab_id = card.get("activeTabId", "") - active_tab = next((t for t in tabs if t.get("id") == active_tab_id), None) - url = (active_tab or {}).get("url", card.get("url", "")) - title = (active_tab or {}).get("title", "") - lines.append(f"- browser_id: \"{bid}\"") - if title: - lines.append(f" Title: {title}") - if url: - lines.append(f" URL: {url}") + if browser_cards and selected_browser_ids: + visible_cards = [ + card for card in browser_cards.values() + if card.get("browser_id", "") in selected_browser_ids + ] + if visible_cards: + lines.append("") + lines.append("The user selected these browser cards for you to work with:") + for card in visible_cards: + bid = card.get("browser_id", "") + tabs = card.get("tabs", []) + active_tab_id = card.get("activeTabId", "") + active_tab = next((t for t in tabs if t.get("id") == active_tab_id), None) + url = (active_tab or {}).get("url", card.get("url", "")) + title = (active_tab or {}).get("title", "") + lines.append(f"- browser_id: \"{bid}\"") + if title: + lines.append(f" Title: {title}") + if url: + lines.append(f" URL: {url}") lines.append("") return "\n".join(lines) @@ -456,7 +467,7 @@ class AgentManager: }) return content - async def _run_agent_loop(self, session_id: str, prompt: str, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None): + async def _run_agent_loop(self, session_id: str, prompt: str, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None, fork_session: bool = False, selected_browser_ids: list[str] | None = None): """Run the Claude Agent SDK query loop for a session.""" session = self.sessions.get(session_id) if not session: @@ -481,58 +492,53 @@ class AgentManager: _builtin_perms = load_builtin_permissions() - def _check_tool_permission(tool_name: str) -> str | None: - """Check tool permissions for both builtin and MCP tools. - Returns 'always_allow', 'deny', or None (ask).""" + def _get_effective_policy(tool_name: str) -> str: + """Return 'always_allow', 'deny', or 'ask' for any tool.""" if tool_name in _builtin_perms: - policy = _builtin_perms[tool_name] - if policy in ("always_allow", "deny"): - return policy - return None + return _builtin_perms[tool_name] import re as _re + + bm = _re.match(r"mcp__openswarm-browser-agent__(.+)", tool_name) + if bm: + return _builtin_perms.get(bm.group(1), "always_allow") + + im = _re.match(r"mcp__openswarm-invoke-agent__(.+)", tool_name) + if im: + return _builtin_perms.get(im.group(1), "always_allow") + m = _re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name) - if not m: - return None - server_slug, mcp_tool_name = m.group(1), m.group(2) - for t in load_all_tools(): - if not t.mcp_config or not t.enabled: - continue - if _sanitize_server_name(t.name) == server_slug: - policy = t.tool_permissions.get(mcp_tool_name, "ask") - if policy in ("always_allow", "deny"): - return policy - return None - return None - - async def can_use_tool(tool_name, input_data, context): - if tool_name != "AskUserQuestion": - policy = _check_tool_permission(tool_name) - if policy == "always_allow": - return PermissionResultAllow(updated_input=input_data) - if policy == "deny": - return PermissionResultDeny(message="Tool denied by permission policy") + if m: + server_slug, mcp_tool_name = m.group(1), m.group(2) + for t in load_all_tools(): + if not t.mcp_config or not t.enabled: + continue + if _sanitize_server_name(t.name) == server_slug: + return t.tool_permissions.get(mcp_tool_name, "ask") + return "always_allow" + async def _request_user_approval(tool_name: str, tool_input) -> dict: + """Send an approval request via WebSocket and wait for the user's decision.""" + safe_input = tool_input if isinstance(tool_input, dict) else {} request_id = uuid4().hex approval_req = ApprovalRequest( id=request_id, session_id=session_id, tool_name=tool_name, - tool_input=input_data if isinstance(input_data, dict) else {}, + tool_input=safe_input, ) session.pending_approvals.append(approval_req) session.status = "waiting_approval" - + await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, "status": "waiting_approval", }) - + decision = await ws_manager.send_approval_request( - session_id, request_id, tool_name, - input_data if isinstance(input_data, dict) else {} + session_id, request_id, tool_name, safe_input ) - + session.pending_approvals = [ a for a in session.pending_approvals if a.id != request_id ] @@ -541,22 +547,67 @@ class AgentManager: "session_id": session_id, "status": "running", }) - + return decision + + async def can_use_tool(tool_name, input_data, context): + if tool_name != "AskUserQuestion": + policy = _get_effective_policy(tool_name) + if policy == "always_allow": + return PermissionResultAllow(updated_input=input_data) + if policy == "deny": + return PermissionResultDeny(message="Tool denied by permission policy") + + decision = await _request_user_approval(tool_name, input_data) if decision.get("behavior") == "allow": return PermissionResultAllow( updated_input=decision.get("updated_input", input_data) ) - else: - return PermissionResultDeny( - message=decision.get("message", "User denied this action") - ) + return PermissionResultDeny( + message=decision.get("message", "User denied this action") + ) tool_start_times: dict[str, float] = {} async def pre_tool_hook(input_data, tool_use_id, context): + tool_name = input_data.get("tool_name", "") + hook_event = input_data.get("hook_event_name", "PreToolUse") + + if tool_name and tool_name != "AskUserQuestion": + policy = _get_effective_policy(tool_name) + + if policy == "deny": + return { + "hookSpecificOutput": { + "hookEventName": hook_event, + "permissionDecision": "deny", + "permissionDecisionReason": "Tool denied by permission policy", + } + } + + if policy == "ask": + tool_input = input_data.get("tool_input", {}) + decision = await _request_user_approval(tool_name, tool_input) + + if decision.get("behavior") == "allow": + if tool_use_id: + tool_start_times[tool_use_id] = time.time() + return { + "hookSpecificOutput": { + "hookEventName": hook_event, + "permissionDecision": "allow", + } + } + return { + "hookSpecificOutput": { + "hookEventName": hook_event, + "permissionDecision": "deny", + "permissionDecisionReason": decision.get("message", "User denied this action"), + } + } + if tool_use_id: tool_start_times[tool_use_id] = time.time() - return {"continue_": True} + return {} async def post_tool_hook(input_data, tool_use_id, context): elapsed_ms = None @@ -590,7 +641,61 @@ class AgentManager: if elapsed_ms is not None: result_payload["elapsed_ms"] = elapsed_ms - result_msg = Message(role="tool_result", content=result_payload) + if hook_tool_name == "Agent": + tool_input = input_data.get("tool_input", {}) + agent_prompt = tool_input.get("prompt", tool_input.get("task", "")) + + sub_text = content + sub_cost = 0.0 + sub_tokens = {"input": 0, "output": 0} + sub_model = session.model + if isinstance(raw_response, dict): + blocks = raw_response.get("content") + if isinstance(blocks, list): + parts = [ + b.get("text", "") + for b in blocks + if isinstance(b, dict) and b.get("type") == "text" + ] + if parts: + sub_text = "\n".join(parts) if len(parts) > 1 else parts[0] + elif isinstance(raw_response.get("text"), str): + sub_text = raw_response["text"] + usage = raw_response.get("usage", {}) + if isinstance(usage, dict): + sub_tokens["input"] = usage.get("input_tokens", 0) + usage.get("cache_creation_input_tokens", 0) + usage.get("cache_read_input_tokens", 0) + sub_tokens["output"] = usage.get("output_tokens", 0) + if raw_response.get("model"): + sub_model = raw_response["model"] + + sub_session_id = uuid4().hex + sub_name = agent_prompt[:50] if agent_prompt else "Sub-agent" + sub_session = AgentSession( + id=sub_session_id, + name=sub_name, + status="completed", + model=sub_model, + mode="sub-agent", + cwd=session.cwd, + created_at=datetime.now(), + cost_usd=sub_cost, + tokens=sub_tokens, + messages=[ + Message(role="user", content=agent_prompt, branch_id="main"), + Message(role="assistant", content=sub_text, branch_id="main"), + ], + dashboard_id=session.dashboard_id, + parent_session_id=session_id, + ) + self.sessions[sub_session_id] = sub_session + await ws_manager.broadcast_global("agent:status", { + "session_id": sub_session_id, + "status": sub_session.status, + "session": sub_session.model_dump(mode="json"), + }) + result_payload["sub_session_id"] = sub_session_id + + result_msg = Message(role="tool_result", content=result_payload, branch_id=session.active_branch_id) session.messages.append(result_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -602,36 +707,95 @@ class AgentManager: _, mode_sys_prompt, _ = self._resolve_mode(session.mode) connected_tools_ctx = self._build_connected_tools_context(session.allowed_tools) outputs_ctx = self._build_outputs_context() - browser_ctx = self._build_browser_context(session.dashboard_id) + browser_ctx = self._build_browser_context(session.dashboard_id, selected_browser_ids=selected_browser_ids) global_settings = load_settings() composed_prompt = self._compose_system_prompt(global_settings.default_system_prompt, mode_sys_prompt, session.system_prompt, connected_tools_ctx, outputs_ctx, browser_ctx) + if session.mode == "view-builder": + from backend.apps.outputs.view_builder_templates import VIEW_BUILDER_SKILL + skill_block = f"\n{VIEW_BUILDER_SKILL}\n" + composed_prompt = f"{composed_prompt}\n\n{skill_block}" if composed_prompt else skill_block + mcp_servers = await self._build_mcp_servers(session.allowed_tools) - browser_agent_server_path = os.path.join( - os.path.dirname(__file__), "browser_agent_mcp_server.py" + _browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents"] + _browser_all_denied = all( + _builtin_perms.get(t, "always_allow") == "deny" + for t in _browser_delegation_tools ) - backend_port = os.environ.get("OPENSWARM_PORT", "8324") - pre_selected_bids = self._get_pre_selected_browser_ids(session.dashboard_id) - mcp_servers["openswarm-browser-agent"] = { - "command": sys.executable, - "args": [browser_agent_server_path], - "env": { - "OPENSWARM_PORT": backend_port, - "OPENSWARM_AGENT_MODEL": session.model, - "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", - "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), - }, - "type": "stdio", - } + + if not _browser_all_denied: + browser_agent_server_path = os.path.join( + os.path.dirname(__file__), "browser_agent_mcp_server.py" + ) + backend_port = os.environ.get("OPENSWARM_PORT", "8324") + pre_selected_bids = self._get_pre_selected_browser_ids(session.dashboard_id) + mcp_servers["openswarm-browser-agent"] = { + "command": sys.executable, + "args": [browser_agent_server_path], + "env": { + "OPENSWARM_PORT": backend_port, + "OPENSWARM_AGENT_MODEL": session.model, + "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", + "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), + "OPENSWARM_PARENT_SESSION_ID": session.id, + }, + "type": "stdio", + } + + _invoke_agent_tools = ["InvokeAgent"] + _invoke_all_denied = all( + _builtin_perms.get(t, "always_allow") == "deny" + for t in _invoke_agent_tools + ) + + if not _invoke_all_denied: + invoke_agent_server_path = os.path.join( + os.path.dirname(__file__), "invoke_agent_mcp_server.py" + ) + backend_port = os.environ.get("OPENSWARM_PORT", "8324") + mcp_servers["openswarm-invoke-agent"] = { + "command": sys.executable, + "args": [invoke_agent_server_path], + "env": { + "OPENSWARM_PORT": backend_port, + "OPENSWARM_PARENT_SESSION_ID": session.id, + "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", + }, + "type": "stdio", + } effective_allowed = [ t for t in session.allowed_tools - if _builtin_perms.get(t, "always_allow") == "always_allow" + if t in FULL_TOOLS and _builtin_perms.get(t, "always_allow") == "always_allow" ] + + effective_disallowed = [ + t for t in FULL_TOOLS + if _builtin_perms.get(t, "always_allow") == "deny" + ] + if mcp_servers: all_tools_list = load_all_tools() for name in mcp_servers: + if name == "openswarm-browser-agent": + for bt in _browser_delegation_tools: + policy = _builtin_perms.get(bt, "always_allow") + if policy == "always_allow": + effective_allowed.append(f"mcp__openswarm-browser-agent__{bt}") + elif policy == "deny": + effective_disallowed.append(f"mcp__openswarm-browser-agent__{bt}") + continue + + if name == "openswarm-invoke-agent": + for it in _invoke_agent_tools: + policy = _builtin_perms.get(it, "always_allow") + if policy == "always_allow": + effective_allowed.append(f"mcp__openswarm-invoke-agent__{it}") + elif policy == "deny": + effective_disallowed.append(f"mcp__openswarm-invoke-agent__{it}") + continue + tool_def = next( (t for t in all_tools_list if t.mcp_config and t.enabled and _sanitize_server_name(t.name) == name), @@ -644,20 +808,22 @@ class AgentManager: policy = tool_def.tool_permissions.get(tn, "ask") if policy == "always_allow": effective_allowed.append(f"mcp__{name}__{tn}") + for tn in denied: + effective_disallowed.append(f"mcp__{name}__{tn}") else: effective_allowed.append(f"mcp__{name}__*") - effective_allowed.append("mcp__openswarm-browser-agent__*") - options_kwargs = { "model": session.model, "max_buffer_size": 5 * 1024 * 1024, + "permission_mode": "default", "can_use_tool": can_use_tool, "hooks": { "PreToolUse": [HookMatcher(matcher=None, hooks=[pre_tool_hook])], "PostToolUse": [HookMatcher(matcher=None, hooks=[post_tool_hook])], }, "allowed_tools": effective_allowed, + "disallowed_tools": effective_disallowed, "include_partial_messages": True, } if not global_settings.anthropic_api_key: @@ -675,6 +841,8 @@ class AgentManager: if session.sdk_session_id: options_kwargs["resume"] = session.sdk_session_id + if fork_session: + options_kwargs["fork_session"] = True options = ClaudeAgentOptions(**options_kwargs) @@ -775,6 +943,7 @@ class AgentManager: id=stream_text_msg_id or uuid4().hex, role="assistant", content="\n".join(content_parts), + branch_id=session.active_branch_id, ) session.messages.append(asst_msg) await ws_manager.send_to_session(session_id, "agent:message", { @@ -784,7 +953,7 @@ class AgentManager: for i, tu in enumerate(tool_uses): msg_id = stream_tool_msg_ids_ordered[i] if i < len(stream_tool_msg_ids_ordered) else uuid4().hex - tool_msg = Message(id=msg_id, role="tool_call", content=tu) + tool_msg = Message(id=msg_id, role="tool_call", content=tu, branch_id=session.active_branch_id) session.messages.append(tool_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -811,7 +980,7 @@ class AgentManager: except Exception as e: logger.exception(f"Agent {session_id} error: {e}") session.status = "error" - error_msg = Message(role="system", content=f"Error: {str(e)}") + error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id) session.messages.append(error_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -912,7 +1081,7 @@ class AgentManager: session_id, tool_msg_id, "Bash", _json.dumps(tool_input_content["input"], indent=2), ) - tool_msg = Message(id=tool_msg_id, role="tool_call", content=tool_input_content) + tool_msg = Message(id=tool_msg_id, role="tool_call", content=tool_input_content, branch_id=session.active_branch_id) session.messages.append(tool_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -922,7 +1091,7 @@ class AgentManager: await asyncio.sleep(1) if decision.get("behavior") == "allow": - tool_result = Message(role="tool_result", content=f"Processing: {prompt}") + tool_result = Message(role="tool_result", content=f"Processing: {prompt}", branch_id=session.active_branch_id) session.messages.append(tool_result) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -940,7 +1109,7 @@ class AgentManager: asst_msg_id = uuid4().hex await self._stream_text(session_id, asst_msg_id, asst_text) - asst_msg = Message(id=asst_msg_id, role="assistant", content=asst_text) + asst_msg = Message(id=asst_msg_id, role="assistant", content=asst_text, branch_id=session.active_branch_id) session.messages.append(asst_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -969,6 +1138,8 @@ class AgentManager: context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None, + hidden: bool = False, + selected_browser_ids: list[str] | None = None, ): """Send a follow-up message to an existing session.""" session = self.sessions.get(session_id) @@ -1000,10 +1171,12 @@ class AgentManager: user_msg = Message( role="user", content=prompt, + branch_id=session.active_branch_id, context_paths=context_paths if context_paths else None, attached_skills=skill_meta, forced_tools=forced_tools if forced_tools else None, images=image_meta, + hidden=hidden, ) session.messages.append(user_msg) await ws_manager.send_to_session(session_id, "agent:message", { @@ -1011,7 +1184,14 @@ class AgentManager: "message": user_msg.model_dump(mode="json"), }) - task = asyncio.create_task(self._run_agent_loop(session_id, prompt, images=images, context_paths=context_paths, forced_tools=forced_tools, attached_skills=attached_skills)) + session.status = "running" + await ws_manager.send_to_session(session_id, "agent:status", { + "session_id": session_id, + "status": "running", + "session": session.model_dump(mode="json"), + }) + + task = asyncio.create_task(self._run_agent_loop(session_id, prompt, images=images, context_paths=context_paths, forced_tools=forced_tools, attached_skills=attached_skills, selected_browser_ids=selected_browser_ids)) self.tasks[session_id] = task async def stop_agent(self, session_id: str): @@ -1043,6 +1223,14 @@ class AgentManager: if not session: raise ValueError(f"Session {session_id} not found") + existing = self.tasks.get(session_id) + if existing and not existing.done(): + existing.cancel() + try: + await existing + except asyncio.CancelledError: + pass + target_msg = None for i, msg in enumerate(session.messages): if msg.id == message_id: @@ -1052,11 +1240,24 @@ class AgentManager: if not target_msg or target_msg.role != "user": raise ValueError("Can only edit user messages") - new_branch_id = uuid4().hex[:8] + fork_point_id = message_id + fork_parent_branch = target_msg.branch_id + + msg_branch = session.branches.get(target_msg.branch_id) + if msg_branch and msg_branch.fork_point_message_id: + branch_user_msgs = [ + m for m in session.messages + if m.branch_id == target_msg.branch_id and m.role == "user" + ] + if branch_user_msgs and branch_user_msgs[0].id == message_id: + fork_point_id = msg_branch.fork_point_message_id + fork_parent_branch = msg_branch.parent_branch_id or "main" + + new_branch_id = uuid4().hex new_branch = MessageBranch( id=new_branch_id, - parent_branch_id=target_msg.branch_id, - fork_point_message_id=message_id, + parent_branch_id=fork_parent_branch, + fork_point_message_id=fork_point_id, ) session.branches[new_branch_id] = new_branch session.active_branch_id = new_branch_id @@ -1066,6 +1267,10 @@ class AgentManager: content=new_content, branch_id=new_branch_id, parent_id=target_msg.parent_id, + images=target_msg.images, + context_paths=target_msg.context_paths, + forced_tools=target_msg.forced_tools, + attached_skills=target_msg.attached_skills, ) session.messages.append(edited_msg) @@ -1079,7 +1284,14 @@ class AgentManager: "active_branch_id": new_branch_id, }) - task = asyncio.create_task(self._run_agent_loop(session_id, new_content)) + session.sdk_session_id = None + task = asyncio.create_task(self._run_agent_loop( + session_id, new_content, + images=target_msg.images, + context_paths=target_msg.context_paths, + forced_tools=target_msg.forced_tools, + attached_skills=target_msg.attached_skills, + )) self.tasks[session_id] = task async def switch_branch(self, session_id: str, branch_id: str): @@ -1369,7 +1581,6 @@ class AgentManager: if session.status in ("running", "waiting_approval"): session.status = "stopped" session.pending_approvals = [] - session.closed_at = session.closed_at or datetime.now() doc_data = session.model_dump(mode="json") doc_data["search_text"] = self._build_search_text(session) _save_session(session_id, doc_data) @@ -1378,21 +1589,208 @@ class AgentManager: self.tasks.clear() async def restore_all_sessions(self) -> None: - """On startup, reload all persisted sessions from JSON files back into memory.""" + """On startup, reload all persisted sessions from JSON files back into memory. + + Only sessions without closed_at are restored (they were active at + shutdown). Sessions with closed_at were explicitly closed by the user + and stay on disk so the history endpoint can still serve them. + """ for sid, data in _load_all_session_data(): try: session = AgentSession(**data) except Exception as e: logger.warning(f"Skipping corrupt session file {sid}: {e}") continue + if session.closed_at is not None: + continue if session.status in ("running", "waiting_approval"): session.status = "stopped" - session.closed_at = None session.pending_approvals = [] self.sessions[session.id] = session _delete_session_file(sid) logger.info(f"Restored session {session.id}") + async def duplicate_session(self, session_id: str, dashboard_id: str | None = None, up_to_message_id: str | None = None) -> AgentSession: + """Create an independent copy of a session with the same chat history.""" + source = self.sessions.get(session_id) + if not source: + data = _load_session_data(session_id) + if data is None: + raise ValueError(f"Session {session_id} not found") + source = AgentSession(**data) + + source_messages = list(source.messages) + if up_to_message_id: + cut_idx = next( + (i for i, m in enumerate(source_messages) if m.id == up_to_message_id), + None, + ) + if cut_idx is not None: + source_messages = source_messages[: cut_idx + 1] + + old_to_new_msg: dict[str, str] = {} + new_messages: list[Message] = [] + for msg in source_messages: + new_id = uuid4().hex + old_to_new_msg[msg.id] = new_id + new_messages.append(Message( + id=new_id, + role=msg.role, + content=msg.content, + timestamp=msg.timestamp, + branch_id=msg.branch_id, + parent_id=old_to_new_msg.get(msg.parent_id) if msg.parent_id else None, + context_paths=msg.context_paths, + attached_skills=msg.attached_skills, + forced_tools=msg.forced_tools, + images=msg.images, + )) + + new_branches: dict[str, MessageBranch] = {} + for bid, branch in source.branches.items(): + new_branches[bid] = MessageBranch( + id=bid, + parent_branch_id=branch.parent_branch_id, + fork_point_message_id=old_to_new_msg.get(branch.fork_point_message_id) if branch.fork_point_message_id else None, + created_at=branch.created_at, + ) + + new_session = AgentSession( + id=uuid4().hex, + name=f"{source.name} (copy)", + status="stopped", + model=source.model, + mode=source.mode, + system_prompt=source.system_prompt, + allowed_tools=list(source.allowed_tools), + max_turns=source.max_turns, + cwd=source.cwd, + created_at=datetime.now(), + messages=new_messages, + branches=new_branches, + active_branch_id=source.active_branch_id, + tool_group_meta=dict(source.tool_group_meta), + dashboard_id=dashboard_id or source.dashboard_id, + ) + + self.sessions[new_session.id] = new_session + + await ws_manager.send_to_session(new_session.id, "agent:status", { + "session_id": new_session.id, + "status": new_session.status, + "session": new_session.model_dump(mode="json"), + }) + + return new_session + + async def invoke_agent( + self, + source_session_id: str, + message: str, + parent_session_id: str | None = None, + dashboard_id: str | None = None, + ) -> dict: + """Fork an existing session and send it a new message, returning the result.""" + source = self.sessions.get(source_session_id) + if not source: + data = _load_session_data(source_session_id) + if data is None: + raise ValueError(f"Session {source_session_id} not found") + source = AgentSession(**data) + + source_name = source.name + + old_to_new_msg: dict[str, str] = {} + new_messages: list[Message] = [] + for msg in source.messages: + new_id = uuid4().hex + old_to_new_msg[msg.id] = new_id + new_messages.append(Message( + id=new_id, + role=msg.role, + content=msg.content, + timestamp=msg.timestamp, + branch_id=msg.branch_id, + parent_id=old_to_new_msg.get(msg.parent_id) if msg.parent_id else None, + context_paths=msg.context_paths, + attached_skills=msg.attached_skills, + forced_tools=msg.forced_tools, + images=msg.images, + )) + + new_branches: dict[str, MessageBranch] = {} + for bid, branch in source.branches.items(): + new_branches[bid] = MessageBranch( + id=bid, + parent_branch_id=branch.parent_branch_id, + fork_point_message_id=( + old_to_new_msg.get(branch.fork_point_message_id) + if branch.fork_point_message_id else None + ), + created_at=branch.created_at, + ) + + fork = AgentSession( + id=uuid4().hex, + name=f"{source_name} (invoked)", + status="running", + model=source.model, + mode="invoked-agent", + sdk_session_id=source.sdk_session_id, + system_prompt=source.system_prompt, + allowed_tools=list(source.allowed_tools), + max_turns=source.max_turns or 25, + cwd=source.cwd, + created_at=datetime.now(), + messages=new_messages, + branches=new_branches, + active_branch_id=source.active_branch_id, + tool_group_meta=dict(source.tool_group_meta), + dashboard_id=dashboard_id or source.dashboard_id, + parent_session_id=parent_session_id, + ) + + self.sessions[fork.id] = fork + + await ws_manager.broadcast_global("agent:status", { + "session_id": fork.id, + "status": fork.status, + "session": fork.model_dump(mode="json"), + }) + + user_msg = Message( + role="user", + content=message, + branch_id=fork.active_branch_id, + ) + fork.messages.append(user_msg) + await ws_manager.send_to_session(fork.id, "agent:message", { + "session_id": fork.id, + "message": user_msg.model_dump(mode="json"), + }) + + await self._run_agent_loop(fork.id, message, fork_session=True) + + last_assistant = None + for msg in reversed(fork.messages): + if msg.role == "assistant": + content = msg.content + if isinstance(content, str): + last_assistant = content + elif isinstance(content, list): + texts = [b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text"] + last_assistant = "\n".join(texts) + else: + last_assistant = str(content) + break + + return { + "forked_session_id": fork.id, + "source_name": source_name, + "response": last_assistant or "No response from invoked agent.", + "cost_usd": fork.cost_usd, + } + def get_all_sessions(self, dashboard_id: str | None = None) -> list[AgentSession]: if dashboard_id: return [s for s in self.sessions.values() if s.dashboard_id == dashboard_id] @@ -1401,4 +1799,22 @@ class AgentManager: def get_session(self, session_id: str) -> Optional[AgentSession]: return self.sessions.get(session_id) + def get_browser_agent_children(self, parent_session_id: str) -> list[dict]: + """Return browser-agent sessions for a parent, from memory or disk.""" + results: list[dict] = [] + seen: set[str] = set() + + for s in self.sessions.values(): + if s.mode == "browser-agent" and s.parent_session_id == parent_session_id: + results.append(s.model_dump(mode="json")) + seen.add(s.id) + + for sid, data in _load_all_session_data(): + if sid in seen: + continue + if data.get("mode") == "browser-agent" and data.get("parent_session_id") == parent_session_id: + results.append(data) + + return results + agent_manager = AgentManager() diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 462b8fde..070e29c5 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -56,6 +56,8 @@ async def send_message(session_id: str, body: dict): context_paths=body.get("context_paths"), forced_tools=body.get("forced_tools"), attached_skills=body.get("attached_skills"), + hidden=body.get("hidden", False), + selected_browser_ids=body.get("selected_browser_ids"), ) return {"ok": True} @@ -131,6 +133,18 @@ async def get_branches(session_id: str): "active_branch_id": session.active_branch_id, } +@agents.router.post("/sessions/{session_id}/duplicate") +async def duplicate_session(session_id: str, body: dict = {}): + try: + session = await agent_manager.duplicate_session( + session_id, + dashboard_id=body.get("dashboard_id"), + up_to_message_id=body.get("up_to_message_id"), + ) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + return {"session": session.model_dump(mode="json")} + @agents.router.post("/sessions/{session_id}/close") async def close_session(session_id: str): try: @@ -151,6 +165,11 @@ async def get_history(q: str = "", limit: int = 20, offset: int = 0, dashboard_i dashboard_id=dashboard_id or None, ) +@agents.router.get("/sessions/{session_id}/browser-agents") +async def get_browser_agent_children(session_id: str): + children = agent_manager.get_browser_agent_children(session_id) + return {"sessions": children} + @agents.router.post("/sessions/{session_id}/resume") async def resume_session(session_id: str): try: diff --git a/backend/apps/agents/browser_agent.py b/backend/apps/agents/browser_agent.py index 11244acf..1508797a 100644 --- a/backend/apps/agents/browser_agent.py +++ b/backend/apps/agents/browser_agent.py @@ -15,8 +15,9 @@ from uuid import uuid4 import anthropic -from backend.apps.agents.models import AgentSession, Message +from backend.apps.agents.models import AgentSession, ApprovalRequest, Message from backend.apps.agents.ws_manager import ws_manager +from backend.apps.tools_lib.tools_lib import load_builtin_permissions logger = logging.getLogger(__name__) @@ -112,6 +113,48 @@ BROWSER_TOOLS_SCHEMA = [ "required": [], }, }, + { + "name": "BrowserScroll", + "description": ( + "Scroll the page up or down. Automatically finds the correct scrollable " + "container (works on SPAs like Notion, Gmail, etc. that use nested scroll " + "containers instead of window-level scrolling). Returns scroll position info " + "including whether top/bottom has been reached." + ), + "input_schema": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "enum": ["up", "down"], + "description": "Scroll direction. Defaults to 'down'.", + }, + "amount": { + "type": "number", + "description": "Pixels to scroll. Defaults to 500.", + }, + }, + "required": [], + }, + }, + { + "name": "BrowserWait", + "description": ( + "Wait for a specified duration. Useful after navigation or actions that " + "trigger page loads, animations, or async content rendering. " + "Min 100ms, max 10000ms." + ), + "input_schema": { + "type": "object", + "properties": { + "milliseconds": { + "type": "number", + "description": "Duration to wait in milliseconds. Defaults to 1000.", + }, + }, + "required": [], + }, + }, ] ACTION_MAP = { @@ -122,17 +165,29 @@ ACTION_MAP = { "BrowserType": "type", "BrowserEvaluate": "evaluate", "BrowserGetElements": "get_elements", + "BrowserScroll": "scroll", + "BrowserWait": "wait", } SYSTEM_PROMPT = ( "You are a browser automation agent. You control a single browser tab and " "execute the task you are given.\n\n" "Strategy:\n" - "1. Start by taking a screenshot or calling BrowserGetElements to understand the page.\n" - "2. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n" - "3. After performing actions, take a screenshot to verify the result.\n" - "4. If an action fails, try alternative selectors or approaches.\n" - "5. When the task is complete, provide a clear summary of what you accomplished.\n\n" + "1. Start by taking a screenshot to understand the page.\n" + "2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n" + "3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with " + "window.scrollBy() as many sites use nested scroll containers that BrowserScroll " + "handles automatically.\n" + "4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n" + "5. After performing actions, take a screenshot to verify the result.\n" + "6. If an action fails, try alternative selectors or approaches.\n" + "7. When the task is complete, provide a clear summary of what you accomplished.\n\n" + "Important notes:\n" + "- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n" + "- BrowserScroll returns position info including atTop/atBottom — use this to know when " + "you've reached the end of the page.\n" + "- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n" + "- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n" "You have access ONLY to browser tools. Do not ask the user questions — " "complete the task autonomously to the best of your ability." ) @@ -179,6 +234,40 @@ def _format_tool_result(result: dict, tool_name: str) -> list[dict]: return [{"type": "text", "text": str(text)}] +async def _request_browser_approval( + session: AgentSession, tool_name: str, tool_input: dict, +) -> dict: + """Send an approval request for a browser sub-agent tool and wait for the decision.""" + request_id = uuid4().hex + approval_req = ApprovalRequest( + id=request_id, + session_id=session.id, + tool_name=tool_name, + tool_input=tool_input, + ) + session.pending_approvals.append(approval_req) + session.status = "waiting_approval" + + await ws_manager.send_to_session(session.id, "agent:status", { + "session_id": session.id, + "status": "waiting_approval", + }) + + decision = await ws_manager.send_approval_request( + session.id, request_id, tool_name, tool_input, + ) + + session.pending_approvals = [ + a for a in session.pending_approvals if a.id != request_id + ] + session.status = "running" + await ws_manager.send_to_session(session.id, "agent:status", { + "session_id": session.id, + "status": "running", + }) + return decision + + async def run_browser_agent( task: str, browser_id: str, @@ -188,6 +277,7 @@ async def run_browser_agent( tab_id: str = "", pre_selected: bool = False, initial_url: str | None = None, + parent_session_id: str | None = None, ) -> dict: """Run a browser sub-agent loop for a single browser card. @@ -196,6 +286,8 @@ async def run_browser_agent( """ from backend.apps.agents.agent_manager import agent_manager + _browser_perms = load_builtin_permissions() + session_id = uuid4().hex session = AgentSession( id=session_id, @@ -206,6 +298,7 @@ async def run_browser_agent( dashboard_id=dashboard_id, browser_id=browser_id, system_prompt=SYSTEM_PROMPT, + parent_session_id=parent_session_id, ) agent_manager.sessions[session_id] = session @@ -291,6 +384,48 @@ async def run_browser_agent( tool_results = [] for tu in tool_uses: + policy = _browser_perms.get(tu.name, "always_allow") + + if policy == "deny": + denied_text = f"Tool {tu.name} is denied by permission policy." + tool_results.append({ + "type": "tool_result", + "tool_use_id": tu.id, + "content": [{"type": "text", "text": denied_text}], + }) + result_msg = Message( + role="tool_result", + content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0}, + ) + session.messages.append(result_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": result_msg.model_dump(mode="json"), + }) + continue + + if policy == "ask": + decision = await _request_browser_approval( + session, tu.name, tu.input, + ) + if decision.get("behavior") == "deny": + denied_text = decision.get("message") or f"Tool {tu.name} denied by user." + tool_results.append({ + "type": "tool_result", + "tool_use_id": tu.id, + "content": [{"type": "text", "text": denied_text}], + }) + result_msg = Message( + role="tool_result", + content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0}, + ) + session.messages.append(result_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": result_msg.model_dump(mode="json"), + }) + continue + start = time.time() result = await execute_browser_tool( tu.name, tu.input, browser_id, tab_id, @@ -407,6 +542,8 @@ async def _create_browser_card(dashboard_id: str, url: str) -> str: activeTabId=tab_id, x=40, y=100, + width=1280, + height=800, ) dashboard.layout.browser_cards[browser_id] = card dashboard.updated_at = datetime.now() @@ -425,6 +562,7 @@ async def run_browser_agents( api_key: str, dashboard_id: str | None = None, pre_selected_browser_ids: list[str] | None = None, + parent_session_id: str | None = None, ) -> list[dict]: """Run multiple browser sub-agents in parallel. @@ -451,6 +589,7 @@ async def run_browser_agents( dashboard_id=dashboard_id, pre_selected=is_pre_selected, initial_url=url if url and browser_id not in pre_selected else None, + parent_session_id=parent_session_id, ) results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True) diff --git a/backend/apps/agents/browser_agent_mcp_server.py b/backend/apps/agents/browser_agent_mcp_server.py index 2a46a139..0f7f4b6b 100644 --- a/backend/apps/agents/browser_agent_mcp_server.py +++ b/backend/apps/agents/browser_agent_mcp_server.py @@ -25,26 +25,20 @@ BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/browser-agent/run" MODEL = os.environ.get("OPENSWARM_AGENT_MODEL", "sonnet") DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "") PRE_SELECTED_BROWSER_IDS = os.environ.get("OPENSWARM_PRE_SELECTED_BROWSER_IDS", "") +PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") TOOLS = [ { - "name": "BrowserAgent", + "name": "CreateBrowserAgent", "description": ( - "Delegate a browser task to a dedicated browser agent. The browser agent " + "Create a new browser card and run a task on it. A dedicated browser agent " "will autonomously perform the task (navigating, clicking, typing, etc.) " "and return a summary of actions taken plus a final screenshot. " - "Use this for any task that requires interacting with a web page." + "Use this when you need a fresh browser for a new task." ), "inputSchema": { "type": "object", "properties": { - "browser_id": { - "type": "string", - "description": ( - "The ID of the browser card to use. If omitted, a new browser " - "card will be automatically created." - ), - }, "task": { "type": "string", "description": ( @@ -55,20 +49,47 @@ TOOLS = [ "url": { "type": "string", "description": ( - "Optional starting URL. If provided and no browser_id is given, " - "the new browser will navigate here first." + "Optional starting URL. The new browser will navigate here " + "before beginning the task." ), }, }, "required": ["task"], }, }, + { + "name": "BrowserAgent", + "description": ( + "Delegate a browser task to a dedicated browser agent on an existing " + "browser card. The browser agent will autonomously perform the task " + "(navigating, clicking, typing, etc.) and return a summary of actions " + "taken plus a final screenshot." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The ID of the existing browser card to use.", + }, + "task": { + "type": "string", + "description": ( + "The task for the browser agent to perform. Be specific and " + "detailed about what you want accomplished." + ), + }, + }, + "required": ["browser_id", "task"], + }, + }, { "name": "BrowserAgents", "description": ( - "Delegate multiple browser tasks to run in parallel, each on a different " - "browser. All tasks execute concurrently and results are returned together. " - "Use this when you need to perform tasks on multiple web pages simultaneously." + "Delegate multiple browser tasks to run in parallel, each on an existing " + "browser card. All tasks execute concurrently and results are returned " + "together. Use this when you need to perform tasks on multiple web pages " + "simultaneously." ), "inputSchema": { "type": "object", @@ -81,18 +102,14 @@ TOOLS = [ "properties": { "browser_id": { "type": "string", - "description": "Optional browser card ID. If omitted, a new browser will be created.", + "description": "The ID of the existing browser card to use.", }, "task": { "type": "string", "description": "The task for this browser agent.", }, - "url": { - "type": "string", - "description": "Optional starting URL.", - }, }, - "required": ["task"], + "required": ["browser_id", "task"], }, }, }, @@ -119,6 +136,7 @@ def call_backend(tasks: list[dict]) -> dict: "model": MODEL, "dashboard_id": DASHBOARD_ID, "pre_selected_browser_ids": pre_selected, + "parent_session_id": PARENT_SESSION_ID, }).encode() req = urllib.request.Request( BACKEND_URL, @@ -219,10 +237,10 @@ def format_batch_results(results: list[dict]) -> dict: def handle_tool_call(tool_name: str, arguments: dict) -> dict: - if tool_name == "BrowserAgent": + if tool_name == "CreateBrowserAgent": task_def = { "task": arguments.get("task", ""), - "browser_id": arguments.get("browser_id", ""), + "browser_id": "", "url": arguments.get("url", ""), } result = call_backend([task_def]) @@ -233,10 +251,30 @@ def handle_tool_call(tool_name: str, arguments: dict) -> dict: return format_result(results[0]) return {"content": [{"type": "text", "text": "No result returned."}], "isError": True} + elif tool_name == "BrowserAgent": + browser_id = arguments.get("browser_id", "") + if not browser_id: + return {"content": [{"type": "text", "text": "Error: browser_id is required"}], "isError": True} + task_def = { + "task": arguments.get("task", ""), + "browser_id": browser_id, + "url": "", + } + result = call_backend([task_def]) + if "error" in result: + return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} + results = result.get("results", [result]) + if results: + return format_result(results[0]) + return {"content": [{"type": "text", "text": "No result returned."}], "isError": True} + elif tool_name == "BrowserAgents": tasks = arguments.get("tasks", []) if not tasks: return {"content": [{"type": "text", "text": "Error: tasks array is empty"}], "isError": True} + for t in tasks: + if not t.get("browser_id"): + return {"content": [{"type": "text", "text": "Error: browser_id is required for each task"}], "isError": True} result = call_backend(tasks) if "error" in result: return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} diff --git a/backend/apps/agents/browser_mcp_server.py b/backend/apps/agents/browser_mcp_server.py index f32054c2..86f1ad5e 100644 --- a/backend/apps/agents/browser_mcp_server.py +++ b/backend/apps/agents/browser_mcp_server.py @@ -181,6 +181,58 @@ TOOLS = [ "required": ["browser_id"], }, }, + { + "name": "BrowserScroll", + "description": ( + "Scroll the page up or down. Automatically finds the correct scrollable " + "container (works on SPAs like Notion, Gmail, etc. that use nested scroll " + "containers instead of window-level scrolling). Returns scroll position info " + "including whether top/bottom has been reached." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "tab_id": TAB_ID_PROP, + "direction": { + "type": "string", + "enum": ["up", "down"], + "description": "Scroll direction. Defaults to 'down'.", + }, + "amount": { + "type": "number", + "description": "Pixels to scroll. Defaults to 500.", + }, + }, + "required": ["browser_id"], + }, + }, + { + "name": "BrowserWait", + "description": ( + "Wait for a specified duration. Useful after navigation or actions that " + "trigger page loads, animations, or async content rendering. " + "Min 100ms, max 10000ms." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "tab_id": TAB_ID_PROP, + "milliseconds": { + "type": "number", + "description": "Duration to wait in milliseconds. Defaults to 1000.", + }, + }, + "required": ["browser_id"], + }, + }, ] @@ -260,6 +312,8 @@ def handle_tool_call(tool_name: str, arguments: dict) -> dict: "BrowserType": "type", "BrowserEvaluate": "evaluate", "BrowserGetElements": "get_elements", + "BrowserScroll": "scroll", + "BrowserWait": "wait", } action = action_map.get(tool_name) if not action: diff --git a/backend/apps/agents/invoke_agent_mcp_server.py b/backend/apps/agents/invoke_agent_mcp_server.py new file mode 100644 index 00000000..cf614b03 --- /dev/null +++ b/backend/apps/agents/invoke_agent_mcp_server.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +""" +Stdio MCP server that exposes the InvokeAgent tool. + +Launched as a subprocess by the Claude Agent SDK. Proxies invocation +requests to the OpenSwarm backend via HTTP, which forks the target +agent session and runs it with the new message. +""" + +import json +import sys +import os +import urllib.request +import urllib.error + +BACKEND_PORT = os.environ.get("OPENSWARM_PORT", "8324") +BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/invoke-agent/run" +PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") +DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "") + +TOOLS = [ + { + "name": "InvokeAgent", + "description": ( + "Invoke a copy of an existing agent session with a new message. " + "The invoked agent will have full context of its prior conversation " + "and will process the new message independently. Use this when you " + "need to query another agent about its prior work or ask it to " + "perform a follow-up task." + ), + "inputSchema": { + "type": "object", + "properties": { + "session_id": { + "type": "string", + "description": ( + "The session ID of the agent to invoke. This is the ID " + "from a selected Agent Card in the context." + ), + }, + "message": { + "type": "string", + "description": ( + "The message to send to the invoked agent. Be specific " + "about what you need from it." + ), + }, + }, + "required": ["session_id", "message"], + }, + }, +] + + +def send_response(id_, result=None, error=None): + msg = {"jsonrpc": "2.0", "id": id_} + if error is not None: + msg["error"] = error + else: + msg["result"] = result + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def call_backend(session_id: str, message: str) -> dict: + payload = json.dumps({ + "session_id": session_id, + "message": message, + "parent_session_id": PARENT_SESSION_ID, + "dashboard_id": DASHBOARD_ID, + }).encode() + req = urllib.request.Request( + BACKEND_URL, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=600) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + body = e.read().decode() if e.fp else str(e) + return {"error": f"HTTP {e.code}: {body}"} + except Exception as e: + return {"error": str(e)} + + +def handle_tool_call(tool_name: str, arguments: dict) -> dict: + if tool_name != "InvokeAgent": + return {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True} + + session_id = arguments.get("session_id", "") + message = arguments.get("message", "") + + if not session_id: + return {"content": [{"type": "text", "text": "Error: session_id is required"}], "isError": True} + if not message: + return {"content": [{"type": "text", "text": "Error: message is required"}], "isError": True} + + result = call_backend(session_id, message) + + if "error" in result: + return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} + + forked_id = result.get("forked_session_id", "") + response = result.get("response", "No response from invoked agent.") + cost = result.get("cost_usd", 0) + source_name = result.get("source_name", "") + + lines = [f"**Invoked Agent Result** (forked session: {forked_id})"] + if source_name: + lines[0] = f"**Invoked Agent Result** — {source_name} (forked session: {forked_id})" + if cost > 0: + lines.append(f"*Cost: ${cost:.4f}*") + lines.append("") + lines.append(response) + + return {"content": [{"type": "text", "text": "\n".join(lines)}]} + + +def main(): + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + + method = msg.get("method") + id_ = msg.get("id") + params = msg.get("params", {}) + + if method == "initialize": + send_response(id_, { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": { + "name": "openswarm-invoke-agent", + "version": "1.0.0", + }, + }) + elif method == "notifications/initialized": + pass + elif method == "tools/list": + send_response(id_, {"tools": TOOLS}) + elif method == "tools/call": + tool_name = params.get("name", "") + arguments = params.get("arguments", {}) + result = handle_tool_call(tool_name, arguments) + send_response(id_, result) + elif method == "ping": + send_response(id_, {}) + elif id_ is not None: + send_response(id_, error={"code": -32601, "message": f"Method not found: {method}"}) + + +if __name__ == "__main__": + main() diff --git a/backend/apps/agents/models.py b/backend/apps/agents/models.py index a5f03df3..948a007c 100644 --- a/backend/apps/agents/models.py +++ b/backend/apps/agents/models.py @@ -37,6 +37,7 @@ class Message(BaseModel): attached_skills: Optional[list[dict]] = None forced_tools: Optional[list[str]] = None images: Optional[list[dict]] = None + hidden: bool = False class MessageBranch(BaseModel): id: str = Field(default_factory=lambda: uuid4().hex) @@ -72,3 +73,4 @@ class AgentSession(BaseModel): tool_group_meta: dict[str, "ToolGroupMeta"] = Field(default_factory=dict) dashboard_id: Optional[str] = None browser_id: Optional[str] = None + parent_session_id: Optional[str] = None diff --git a/backend/apps/dashboards/models.py b/backend/apps/dashboards/models.py index b6cd6bef..74fc4ee8 100644 --- a/backend/apps/dashboards/models.py +++ b/backend/apps/dashboards/models.py @@ -34,8 +34,8 @@ class BrowserCardPosition(BaseModel): activeTabId: str = "" x: float = 0 y: float = 0 - width: float = 640 - height: float = 480 + width: float = 1280 + height: float = 800 class DashboardLayout(BaseModel): diff --git a/backend/apps/modes/models.py b/backend/apps/modes/models.py index 03c201fb..0afc742c 100644 --- a/backend/apps/modes/models.py +++ b/backend/apps/modes/models.py @@ -76,43 +76,28 @@ BUILTIN_MODES: list[Mode] = [ ), Mode( id="view-builder", - name="View Builder", - description="Create and iterate on reusable View artifacts.", + name="App Builder", + description="Create and iterate on reusable App artifacts.", system_prompt=( - "You are helping the user build a reusable View — a self-contained " - "web app rendered in an iframe.\n\n" - "Your working directory is a dedicated workspace folder for this view. " - "You can create any file structure you need using the Write tool.\n\n" - "## Required files\n\n" - "1. **index.html** — The entry point. A complete HTML document. " - "React 18 is available via esm.sh CDN imports:\n" - ' \n' - " The structured input data is available at `window.OUTPUT_INPUT` (object) " - "and any server-side result at `window.OUTPUT_BACKEND_RESULT`.\n\n" - "2. **schema.json** — A JSON Schema object defining the structured input " - "the view accepts. Example:\n" - ' {"type":"object","properties":{"name":{"type":"string"}},"required":["name"]}\n\n' - "3. **meta.json** — Metadata for this view. Always write this file with " - "a short name and one-sentence description. Example:\n" - ' {"name":"Sales Dashboard","description":"Interactive dashboard showing sales metrics"}\n\n' - "## Optional files\n\n" - "- **backend.py** — Python code that receives `input_data` as " - "a global dict and must assign its result to a global `result` dict.\n" - "- **Any additional files** — You can create subdirectories and split code " - "across multiple files. For example:\n" - " - `components/Chart.js` — Reusable components\n" - " - `utils/helpers.js` — Utility functions\n" - " - `styles/main.css` — Stylesheets\n\n" - "Files are served from the workspace, so relative imports work naturally:\n" - ' ``\n' - ' ``\n' - " `import { helper } from './utils/helpers.js'` (in ES modules)\n\n" - "## Guidelines\n\n" - "Write files immediately when you have code ready. The user can see " - "a live preview that auto-refreshes from these files. Always write the " - "complete file content (do not use Edit for partial patches on first creation). " - "For complex views, split code into separate files to keep things organized." + "You are an App Builder — an AI assistant that creates self-contained " + "web apps rendered in an iframe preview.\n\n" + "Your working directory is a dedicated workspace folder pre-seeded with " + "template files. Read the existing files before making changes.\n\n" + "## Critical rules\n\n" + "- The entry point MUST be named `index.html`. Never rename it or create " + "a different HTML file as the main entry point.\n" + "- Write files immediately when you have code ready — the user sees a " + "live preview that auto-refreshes from these files.\n" + "- Always write the complete file content on first creation (do not use " + "Edit for partial patches on new files).\n" + "- For complex apps, split code into separate files (JS, CSS, etc.) " + "and reference them from index.html with relative paths.\n" + "- Always update meta.json with a short name and one-sentence description.\n" + "- Build beautiful, polished UIs with modern design — dark themes, smooth " + "transitions, proper spacing, and responsive layouts.\n\n" + "Read the SKILL.md reference in your workspace for the full technical " + "specification of the App platform (available globals, file conventions, " + "schema format, backend.py usage, and examples)." ), tools=None, default_next_mode=None, diff --git a/backend/apps/modes/modes.py b/backend/apps/modes/modes.py index 6d393143..37aabd90 100644 --- a/backend/apps/modes/modes.py +++ b/backend/apps/modes/modes.py @@ -59,7 +59,8 @@ def load_mode(mode_id: str) -> Mode | None: @modes.router.get("/list") async def list_modes(): - return {"modes": [m.model_dump() for m in _load_all()]} + builtin_defaults = {m.id: m.model_dump() for m in BUILTIN_MODES} + return {"modes": [m.model_dump() for m in _load_all()], "builtin_defaults": builtin_defaults} @modes.router.get("/{mode_id}") @@ -93,6 +94,16 @@ async def update_mode(mode_id: str, body: ModeUpdate): return {"ok": True, "mode": mode.model_dump()} +@modes.router.post("/{mode_id}/reset") +async def reset_mode(mode_id: str): + """Reset a built-in mode to its hardcoded defaults.""" + builtin = next((m for m in BUILTIN_MODES if m.id == mode_id), None) + if not builtin: + raise HTTPException(status_code=400, detail="Only built-in modes can be reset") + _save(builtin) + return {"ok": True, "mode": builtin.model_dump()} + + @modes.router.delete("/{mode_id}") async def delete_mode(mode_id: str): mode = _load(mode_id) diff --git a/backend/apps/outputs/outputs.py b/backend/apps/outputs/outputs.py index 206e29e8..22b3bdf7 100644 --- a/backend/apps/outputs/outputs.py +++ b/backend/apps/outputs/outputs.py @@ -15,6 +15,7 @@ from backend.apps.outputs.models import ( WorkspaceSeedRequest, ) from backend.apps.outputs.executor import execute_backend_code +from backend.apps.outputs.view_builder_templates import VIEW_BUILDER_SKILL, VIEW_TEMPLATE_FILES from backend.apps.settings.settings import load_settings logger = logging.getLogger(__name__) @@ -235,6 +236,14 @@ async def seed_workspace(body: WorkspaceSeedRequest): os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w") as f: f.write(content) + else: + for rel_path, content in VIEW_TEMPLATE_FILES.items(): + full_path = os.path.join(folder, rel_path) + with open(full_path, "w") as f: + f.write(content) + + with open(os.path.join(folder, "SKILL.md"), "w") as f: + f.write(VIEW_BUILDER_SKILL) if body.meta: with open(os.path.join(folder, "meta.json"), "w") as f: diff --git a/backend/apps/outputs/view_builder_skill.md b/backend/apps/outputs/view_builder_skill.md new file mode 100644 index 00000000..1e78377d --- /dev/null +++ b/backend/apps/outputs/view_builder_skill.md @@ -0,0 +1,223 @@ +# App Builder — Platform Reference + +You are building an **App**: a self-contained web app served in an iframe. +The workspace you're working in is the source of truth — every file you write +here is served directly to the live preview. + +--- + +## File conventions + +| File | Required | Purpose | +|------|----------|---------| +| `index.html` | **Yes** | Entry point. Must be a complete HTML document. This is the ONLY file the preview iframe loads — never rename it. | +| `meta.json` | **Yes** | `{"name":"…","description":"…"}` — displayed in the UI header. Always write this. | +| `schema.json` | Recommended | JSON Schema defining the input form (the "Test Input" tab). | +| `backend.py` | Optional | Server-side Python executed before rendering. | +| Everything else | Optional | JS, CSS, images, subdirectories — referenced from `index.html` via relative paths. | + +### ⚠️ Do NOT + +- Name the main HTML file anything other than `index.html` — the platform + will not find it and the preview will be blank. +- Use `document.write()` — it breaks the injected data globals. +- Assume any external server or API is available unless the user provides one. + +--- + +## Injected globals + +Before `index.html` loads, the platform injects two globals: + +```javascript +window.OUTPUT_INPUT // Object — structured input from the schema form +window.OUTPUT_BACKEND_RESULT // Object | null — result from backend.py execution +``` + +These are available immediately in any ` +``` + +ES module imports between JS files: + +```javascript +// components/Chart.js +import { formatNumber } from '../utils/helpers.js'; +``` + +--- + +## Using React + +React 18 is available via esm.sh CDN — no build step needed: + +```html + +
+ +``` + +Other CDN libraries work too — use `https://esm.sh/` or `https://cdn.jsdelivr.net/npm/` for any npm package. + +--- + +## Design guidelines + +- **Dark theme by default** — use dark backgrounds (#0f1117, #1a1d27) with + light text (#e2e8f0) unless the user requests otherwise. +- **Modern aesthetics** — rounded corners (8-12px), subtle borders, box shadows, + smooth transitions (0.15-0.3s ease). +- **Responsive** — use flexbox/grid, test at different sizes. +- **Typography** — system font stack for UI, monospace for code/data. +- **Color accents** — use a single accent color with variations for hover/active states. +- **Spacing** — consistent padding (12-20px), adequate whitespace between sections. +- **Interactivity** — hover effects, focus states, loading indicators where appropriate. + +--- + +## Complete minimal example + +```html + + + + + + My App + + + +
+

Loading…

+

+
+ + + +``` diff --git a/backend/apps/outputs/view_builder_templates.py b/backend/apps/outputs/view_builder_templates.py new file mode 100644 index 00000000..0aada357 --- /dev/null +++ b/backend/apps/outputs/view_builder_templates.py @@ -0,0 +1,74 @@ +"""Default template files seeded into new App Builder workspaces.""" + +import os + +_SKILL_PATH = os.path.join(os.path.dirname(__file__), "view_builder_skill.md") + +with open(_SKILL_PATH) as _f: + VIEW_BUILDER_SKILL = _f.read() + +VIEW_TEMPLATE_INDEX = """\ + + + + + + App + + + +
+

Ready

+

Describe what you want to build and the agent will update this app.

+
+ + + +""" + +VIEW_TEMPLATE_SCHEMA = """\ +{ + "type": "object", + "properties": {}, + "required": [] +} +""" + +VIEW_TEMPLATE_META = """\ +{ + "name": "", + "description": "" +} +""" + +VIEW_TEMPLATE_FILES = { + "index.html": VIEW_TEMPLATE_INDEX, + "schema.json": VIEW_TEMPLATE_SCHEMA, + "meta.json": VIEW_TEMPLATE_META, +} diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index 6030e1ab..34a589df 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -29,3 +29,8 @@ class AppSettings(BaseModel): deepgram_api_key: Optional[str] = None openai_api_key: Optional[str] = None webhook_base_url: Optional[str] = None + # Dashboard / UI preferences + auto_select_mode_on_new_agent: bool = False + expand_new_chats_in_dashboard: bool = False + auto_reveal_sub_agents: bool = True + dev_mode: bool = False diff --git a/backend/apps/tools_lib/models.py b/backend/apps/tools_lib/models.py index 3c0cc24e..857ccc3a 100644 --- a/backend/apps/tools_lib/models.py +++ b/backend/apps/tools_lib/models.py @@ -5,6 +5,7 @@ from uuid import uuid4 class BuiltinTool(BaseModel): name: str + display_name: Optional[str] = None description: str category: str = "filesystem" deferred: bool = False @@ -33,6 +34,23 @@ BUILTIN_TOOLS: list[BuiltinTool] = [ BuiltinTool(name="CronList", description="List all scheduled tasks", category="scheduling", deferred=True), BuiltinTool(name="CronDelete", description="Delete a scheduled task", category="scheduling", deferred=True), BuiltinTool(name="RenderOutput", description="Render a reusable View artifact with structured input data", category="views", deferred=True), + # Agent tools + BuiltinTool(name="Agent", display_name="CreateAgent", description="Spawn a sub-agent to handle a complex subtask", category="agents"), + BuiltinTool(name="InvokeAgent", description="Invoke a copy of an existing agent with a new message, preserving full conversation context", category="agents"), + # Browser delegation tools (Layer 1 — what the main agent calls) + BuiltinTool(name="CreateBrowserAgent", description="Create a new browser and run a task on it", category="browser_delegation"), + BuiltinTool(name="BrowserAgent", description="Delegate a browser task to an existing browser agent", category="browser_delegation"), + BuiltinTool(name="BrowserAgents", description="Run multiple browser tasks in parallel on existing browsers", category="browser_delegation"), + # Browser action tools (Layer 2 — what the sub-agent executes) + BuiltinTool(name="BrowserScreenshot", description="Capture a screenshot of the browser page", category="browser_action"), + BuiltinTool(name="BrowserNavigate", description="Navigate the browser to a URL", category="browser_action"), + BuiltinTool(name="BrowserClick", description="Click an element by CSS selector", category="browser_action"), + BuiltinTool(name="BrowserType", description="Type text into an input element", category="browser_action"), + BuiltinTool(name="BrowserEvaluate", description="Execute JavaScript in the browser", category="browser_action"), + BuiltinTool(name="BrowserGetText", description="Get visible text content of the page", category="browser_action"), + BuiltinTool(name="BrowserGetElements", description="List interactive elements with CSS selectors", category="browser_action"), + BuiltinTool(name="BrowserScroll", description="Scroll the page up or down", category="browser_action"), + BuiltinTool(name="BrowserWait", description="Wait for page loads or animations", category="browser_action"), ] diff --git a/backend/apps/tools_lib/tools_lib.py b/backend/apps/tools_lib/tools_lib.py index f19add53..0abb4bf7 100644 --- a/backend/apps/tools_lib/tools_lib.py +++ b/backend/apps/tools_lib/tools_lib.py @@ -18,9 +18,11 @@ from backend.apps.tools_lib.models import ToolDefinition, ToolCreate, ToolUpdate logger = logging.getLogger(__name__) -from backend.config.paths import BACKEND_DIR, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH +from backend.config.paths import BACKEND_DIR, DATA_ROOT, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH load_dotenv(os.path.join(BACKEND_DIR, ".env")) +if os.environ.get("OPENSWARM_PACKAGED") == "1": + load_dotenv(os.path.join(os.path.dirname(DATA_ROOT), ".env"), override=True) @asynccontextmanager @@ -380,6 +382,7 @@ def derive_mcp_config(tool: ToolDefinition) -> Optional[dict]: config["command"] = resolved env = config.setdefault("env", {}) env.setdefault("PATH", _augmented_path()) + env.setdefault("PYTHONPATH", "") return config @@ -510,7 +513,7 @@ async def _discover_mcp_tools_http(url: str, headers: dict | None = None) -> lis raise HTTPException(status_code=502, detail="Empty response from MCP server") tools_list = data.get("result", {}).get("tools", []) - return [{"name": t.get("name", ""), "description": t.get("description", "")} for t in tools_list] + return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list] async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list[dict]: @@ -533,7 +536,7 @@ async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list ) as session: await session.initialize() result = await session.list_tools() - return [{"name": t.name, "description": t.description or ""} for t in result.tools] + return [{"name": t.name, "description": t.description or "", "inputSchema": t.inputSchema if t.inputSchema else None} for t in result.tools] except BaseExceptionGroup as eg: first = eg.exceptions[0] if eg.exceptions else eg raise HTTPException(status_code=502, detail=f"SSE discovery failed: {first}") from first @@ -546,6 +549,7 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, raise HTTPException(status_code=400, detail=f"Command '{command}' not found on PATH or common install locations") proc_env = {**os.environ, **(env or {}), "PATH": _augmented_path()} + proc_env.pop("PYTHONPATH", None) proc = await asyncio.create_subprocess_exec( cmd_path, *(args or []), @@ -601,7 +605,7 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, data = await _recv() tools_list = data.get("result", {}).get("tools", []) - return [{"name": t.get("name", ""), "description": t.get("description", "")} for t in tools_list] + return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list] except HTTPException: raise @@ -616,12 +620,34 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, proc.terminate() await asyncio.wait_for(proc.wait(), timeout=5.0) except Exception: - proc.kill() + try: + proc.kill() + except Exception: + pass @tools_lib.router.post("/{tool_id}/discover") async def discover_tools(tool_id: str): tool = _load(tool_id) + + if tool.auth_type == "oauth2" and tool.auth_status == "connected": + refreshed = await refresh_google_token(tool) + if not refreshed and tool.oauth_tokens.get("access_token"): + expiry = tool.oauth_tokens.get("token_expiry", 0) + if time.time() >= expiry - 60: + client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "") + if not client_id: + raise HTTPException( + status_code=400, + detail="OAuth token expired and GOOGLE_OAUTH_CLIENT_ID is not set. " + "In the packaged app, create ~/.openswarm.env or " + "~/Library/Application Support/OpenSwarm/.env with your Google OAuth credentials.", + ) + raise HTTPException( + status_code=502, + detail="OAuth token expired and refresh failed. Try reconnecting Google.", + ) + config = derive_mcp_config(tool) if not config: raise HTTPException(status_code=400, detail="Cannot derive MCP config for tool") @@ -655,8 +681,11 @@ async def discover_tools(tool_id: str): except HTTPException: raise except Exception as e: - logger.warning(f"MCP tool discovery failed for {tool.name}: {e}") - raise HTTPException(status_code=502, detail=f"Discovery failed: {e}") + msg = str(e).strip() + if not msg: + msg = type(e).__name__ + logger.warning(f"MCP tool discovery failed for {tool.name}: {msg}", exc_info=True) + raise HTTPException(status_code=502, detail=f"Discovery failed: {msg}") services: dict[str, dict[str, list[str]]] = {} service_groups: dict[str, list[str]] = {} @@ -681,6 +710,7 @@ async def discover_tools(tool_id: str): permissions["_services"] = services permissions["_service_groups"] = service_groups permissions["_tool_descriptions"] = {t["name"]: t["description"] for t in raw_tools} + permissions["_tool_schemas"] = {t["name"]: t.get("inputSchema") for t in raw_tools if t.get("inputSchema")} tool.tool_permissions = permissions _save(tool) diff --git a/backend/main.py b/backend/main.py index 4e5b45f1..1320084c 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,6 +1,9 @@ +import logging import os from uuid import uuid4 +logger = logging.getLogger(__name__) + from fastapi.responses import JSONResponse from fastapi import Request from backend.config.Apps import MainApp @@ -133,6 +136,7 @@ async def browser_agent_run(request: Request): model = body.get("model", "sonnet") dashboard_id = body.get("dashboard_id", "") pre_selected_browser_ids = body.get("pre_selected_browser_ids", []) + parent_session_id = body.get("parent_session_id", "") if not tasks: return JSONResponse({"error": "tasks array is required"}, status_code=400) @@ -147,10 +151,42 @@ async def browser_agent_run(request: Request): api_key=settings.anthropic_api_key, dashboard_id=dashboard_id or None, pre_selected_browser_ids=pre_selected_browser_ids, + parent_session_id=parent_session_id or None, ) return JSONResponse({"results": results}) +@app.post("/api/invoke-agent/run") +async def invoke_agent_run(request: Request): + """Fork an existing agent session and send it a new message. + Called by the invoke_agent_mcp_server stdio subprocess.""" + body = await request.json() + session_id = body.get("session_id", "") + message = body.get("message", "") + parent_session_id = body.get("parent_session_id", "") + dashboard_id = body.get("dashboard_id", "") + + if not session_id: + return JSONResponse({"error": "session_id is required"}, status_code=400) + if not message: + return JSONResponse({"error": "message is required"}, status_code=400) + + try: + from backend.apps.agents.agent_manager import agent_manager + result = await agent_manager.invoke_agent( + source_session_id=session_id, + message=message, + parent_session_id=parent_session_id or None, + dashboard_id=dashboard_id or None, + ) + return JSONResponse(result) + except ValueError as e: + return JSONResponse({"error": str(e)}, status_code=404) + except Exception as e: + logger.exception("invoke_agent_run failed") + return JSONResponse({"error": str(e)}, status_code=500) + + if __name__ == "__main__": import argparse import uvicorn diff --git a/electron/main.js b/electron/main.js index d5f79a99..c209718e 100644 --- a/electron/main.js +++ b/electron/main.js @@ -1,5 +1,6 @@ -const { app, BrowserWindow, ipcMain, shell } = require('electron'); -const { autoUpdater } = require('electron-updater'); +const { app, components, BrowserWindow, ipcMain, shell, session } = require('electron'); +let autoUpdater; +try { autoUpdater = require('electron-updater').autoUpdater; } catch (_) {} const path = require('path'); const { spawn, execFileSync } = require('child_process'); const os = require('os'); @@ -7,9 +8,16 @@ const fs = require('fs'); const getPort = require('get-port'); const http = require('http'); +app.commandLine.appendSwitch('disable-features', 'HardwareMediaKeyHandling'); +app.commandLine.appendSwitch('ignore-gpu-blocklist'); +app.commandLine.appendSwitch('enable-gpu-rasterization'); +app.commandLine.appendSwitch('enable-zero-copy'); +app.commandLine.appendSwitch('autoplay-policy', 'no-user-gesture-required'); + let mainWindow = null; let backendProcess = null; let backendPort = null; +let cachedUpdateStatus = { status: 'idle', info: null, error: null }; const isPackaged = app.isPackaged; const isDev = process.env.ELECTRON_DEV === '1'; @@ -24,9 +32,10 @@ const iconPath = path.join(__dirname, 'build', 'icon.png'); function getShellPath() { if (process.platform !== 'darwin' || isDev) return process.env.PATH || ''; + // Strategy 1: ask the user's login shell for its PATH try { - const shell = process.env.SHELL || '/bin/zsh'; - const result = execFileSync(shell, ['-ilc', 'echo $PATH'], { + const userShell = process.env.SHELL || '/bin/zsh'; + const result = execFileSync(userShell, ['-ilc', 'echo $PATH'], { encoding: 'utf8', timeout: 5000, env: { ...process.env, HOME: os.homedir() }, @@ -35,19 +44,40 @@ function getShellPath() { if (resolved) return resolved; } catch (_) { /* fall through */ } + // Strategy 2: read macOS system PATH config (/etc/paths + /etc/paths.d/*) + const systemPaths = []; + try { + const base = fs.readFileSync('/etc/paths', 'utf8'); + for (const line of base.split('\n')) { + const p = line.trim(); + if (p) systemPaths.push(p); + } + } catch (_) { /* ignore */ } + try { + const pathsD = '/etc/paths.d'; + if (fs.existsSync(pathsD)) { + for (const file of fs.readdirSync(pathsD).sort()) { + const content = fs.readFileSync(path.join(pathsD, file), 'utf8'); + for (const line of content.split('\n')) { + const p = line.trim(); + if (p) systemPaths.push(p); + } + } + } + } catch (_) { /* ignore */ } + + // Strategy 3: well-known user-local bin directories const home = os.homedir(); const fallbackDirs = [ - path.join(home, '.nvm/versions/node'), + path.join(home, '.local/bin'), path.join(home, '.volta/bin'), path.join(home, '.fnm/aliases/default/bin'), path.join(home, '.bun/bin'), path.join(home, '.cargo/bin'), - path.join(home, '.local/bin'), '/opt/homebrew/bin', '/usr/local/bin', ]; - // For nvm, resolve the current default version dynamically const nvmDir = path.join(home, '.nvm/versions/node'); try { if (fs.existsSync(nvmDir)) { @@ -58,10 +88,14 @@ function getShellPath() { } } catch (_) { /* ignore */ } - const existing = fallbackDirs.filter((d) => { - try { return fs.statSync(d).isDirectory(); } catch { return false; } - }); - return [...existing, process.env.PATH || ''].join(':'); + const seen = new Set(); + const dirs = []; + for (const d of [...fallbackDirs, ...systemPaths, ...(process.env.PATH || '').split(':')]) { + if (!d || seen.has(d)) continue; + seen.add(d); + try { if (fs.statSync(d).isDirectory()) dirs.push(d); } catch { /* skip */ } + } + return dirs.join(':'); } function getResourcePath(...segments) { @@ -190,6 +224,18 @@ function createWindow() { mainWindow.loadFile(frontendPath); } + mainWindow.webContents.on('will-attach-webview', (_event, webPreferences, _params) => { + webPreferences.plugins = true; + webPreferences.enableBlinkFeatures = 'EncryptedMedia'; + }); + + mainWindow.webContents.on('will-navigate', (event, url) => { + if (isDev && url.startsWith('http://localhost:3000')) return; + if (url.startsWith('file://')) return; + event.preventDefault(); + mainWindow.webContents.send('webview-new-window', url, mainWindow.webContents.id); + }); + mainWindow.on('closed', () => { mainWindow = null; }); @@ -202,30 +248,36 @@ function sendToRenderer(channel, ...args) { } function setupAutoUpdater() { + if (!autoUpdater) return; autoUpdater.autoDownload = false; autoUpdater.autoInstallOnAppQuit = false; autoUpdater.on('update-available', (info) => { console.log(`Update available: ${info.version}`); + cachedUpdateStatus = { status: 'available', info, error: null }; sendToRenderer('update-available', info); }); autoUpdater.on('update-not-available', (info) => { console.log('App is up to date'); + cachedUpdateStatus = { status: 'not-available', info, error: null }; sendToRenderer('update-not-available', info); }); autoUpdater.on('download-progress', (progress) => { + cachedUpdateStatus = { status: 'downloading', info: progress, error: null }; sendToRenderer('download-progress', progress); }); autoUpdater.on('update-downloaded', (info) => { console.log(`Update downloaded: ${info.version}`); + cachedUpdateStatus = { status: 'downloaded', info, error: null }; sendToRenderer('update-downloaded', info); }); autoUpdater.on('error', (err) => { console.error('Auto-update error:', err); + cachedUpdateStatus = { status: 'error', info: null, error: err?.message || String(err) }; sendToRenderer('update-error', err?.message || String(err)); }); @@ -252,6 +304,68 @@ app.whenReady().then(async () => { try { app.dock.setIcon(iconPath); } catch (_) {} } + session.defaultSession.setPermissionRequestHandler((_wc, permission, callback) => { + const allowed = [ + 'media', 'mediaKeySystem', 'protected-media-identifier', + 'geolocation', 'notifications', 'midi', 'midiSysex', + 'clipboard-read', 'clipboard-sanitized-write', + 'pointerLock', 'fullscreen', 'idle-detection', + ]; + console.log('Permission request:', permission, '->', allowed.includes(permission) ? 'granted' : 'denied'); + callback(allowed.includes(permission)); + }); + session.defaultSession.setPermissionCheckHandler((_wc, permission) => { + const allowed = [ + 'media', 'mediaKeySystem', 'protected-media-identifier', + 'clipboard-read', 'clipboard-sanitized-write', + 'pointerLock', 'fullscreen', 'idle-detection', + ]; + return allowed.includes(permission); + }); + + // Read-only logging for DRM license requests — no modifying interceptors + // so the network stack can set Content-Type and other headers normally. + session.defaultSession.webRequest.onSendHeaders( + { urls: ['*://*/*widevine*license*'] }, + (details) => { + console.log(`[drm-req] ${details.method} ${details.url}`); + for (const [k, v] of Object.entries(details.requestHeaders || {})) { + if (/content-type|origin|referer|auth|accept/i.test(k)) { + console.log(`[drm-req] ${k}: ${v}`); + } + } + }, + ); + session.defaultSession.webRequest.onCompleted( + { urls: ['*://*/*widevine*', '*://*/*license*'] }, + (details) => { + console.log(`[drm-net] ${details.method} ${details.url} → ${details.statusCode}`); + }, + ); + session.defaultSession.webRequest.onErrorOccurred( + { urls: ['*://*/*widevine*', '*://*/*license*'] }, + (details) => { + console.log(`[drm-net] FAILED ${details.method} ${details.url} → ${details.error}`); + }, + ); + + // Wait for the Widevine CDM to be downloaded/ready (CastLabs Component + // Updater Service). On first launch this downloads the CDM; subsequent + // launches use the cached version. + if (components && typeof components.whenReady === 'function') { + try { + await components.whenReady(); + console.log('Widevine CDM ready'); + if (typeof components.status === 'function') { + console.log('CDM component status:', JSON.stringify(components.status())); + } + } catch (err) { + console.warn('Widevine CDM not available:', err.message); + } + } else { + console.log('CastLabs components API not available — using standard Electron (no DRM)'); + } + try { if (isDev) { backendPort = parseInt(process.env.OPENSWARM_PORT || '8324', 10); @@ -262,6 +376,13 @@ app.whenReady().then(async () => { createWindow(); if (!isDev) { setupAutoUpdater(); + mainWindow.webContents.on('did-finish-load', () => { + if (cachedUpdateStatus.status === 'available') { + sendToRenderer('update-available', cachedUpdateStatus.info); + } else if (cachedUpdateStatus.status === 'downloaded') { + sendToRenderer('update-downloaded', cachedUpdateStatus.info); + } + }); } } catch (err) { console.error('Failed to start:', err); @@ -269,6 +390,79 @@ app.whenReady().then(async () => { } }); +app.on('web-contents-created', (_event, contents) => { + contents.setWindowOpenHandler(({ url, disposition }) => { + if (disposition === 'foreground-tab' || disposition === 'background-tab') { + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send('webview-new-window', url, contents.id); + } + return { action: 'deny' }; + } + + return { + action: 'allow', + overrideBrowserWindowOptions: { + parent: mainWindow || undefined, + }, + }; + }); + + contents.on('did-create-window', (childWindow) => { + if (mainWindow && !mainWindow.isDestroyed() && !childWindow.isDestroyed()) { + childWindow.setParentWindow(mainWindow); + } + }); + + if (contents.getType() === 'webview') { + contents.on('console-message', (_e, level, message, line, sourceId) => { + if (message.includes('widevine') || message.includes('drm') || + message.includes('license') || message.includes('MediaKeySession') || + message.includes('EME') || message.includes('[drm-diag]') || level >= 2) { + const tag = ['LOG', 'INFO', 'WARN', 'ERROR'][level] || 'LOG'; + const src = sourceId ? sourceId.split('/').pop() : ''; + console.log(`[webview:${tag}] ${message}${src ? ` (${src}:${line})` : ''}`); + } + }); + + contents.on('dom-ready', () => { + const url = contents.getURL(); + if (url.includes('spotify')) { + contents.executeJavaScript(` + (function() { + const origFetch = window.fetch; + window.fetch = async function(...args) { + const resp = await origFetch.apply(this, args); + const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || ''; + if (url.includes('widevine-license') && !resp.ok) { + const clone = resp.clone(); + try { + const text = await clone.text(); + console.log('[drm-diag] License response ' + resp.status + ': ' + text.substring(0, 500)); + } catch(e) {} + } + return resp; + }; + + // Check EME availability + if (navigator.requestMediaKeySystemAccess) { + navigator.requestMediaKeySystemAccess('com.widevine.alpha', [{ + initDataTypes: ['cenc'], + audioCapabilities: [{contentType: 'audio/mp4; codecs="mp4a.40.2"'}], + }]).then(function(access) { + console.log('[drm-diag] Widevine EME access: ' + access.keySystem); + }).catch(function(err) { + console.log('[drm-diag] Widevine EME FAILED: ' + err.message); + }); + } else { + console.log('[drm-diag] EME API not available'); + } + })(); + `).catch(() => {}); + } + }); + } +}); + app.on('window-all-closed', () => { if (!isDev) killBackend(); app.quit(); @@ -286,9 +480,14 @@ app.on('activate', () => { ipcMain.handle('get-backend-port', () => backendPort); ipcMain.handle('get-app-version', () => app.getVersion()); +ipcMain.handle('get-webview-preload-path', () => { + return `file://${path.join(__dirname, 'webview-preload.js')}`; +}); + +ipcMain.handle('get-update-status', () => cachedUpdateStatus); ipcMain.handle('check-for-updates', async () => { - if (!isPackaged) { + if (!autoUpdater || !isPackaged) { sendToRenderer('update-error', 'Update check is only available in the packaged app.'); return { success: false, error: 'Not packaged' }; } @@ -305,6 +504,7 @@ ipcMain.handle('check-for-updates', async () => { }); ipcMain.handle('download-update', async () => { + if (!autoUpdater) return { success: false, error: 'Updater not available' }; try { await autoUpdater.downloadUpdate(); return { success: true }; @@ -314,6 +514,7 @@ ipcMain.handle('download-update', async () => { }); ipcMain.handle('install-update', () => { + if (!autoUpdater) return; autoUpdater.quitAndInstall(false, true); }); diff --git a/electron/package-lock.json b/electron/package-lock.json index 3358b1a3..f3ebd483 100644 --- a/electron/package-lock.json +++ b/electron/package-lock.json @@ -1,19 +1,20 @@ { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.9", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.9", + "hasInstallScript": true, "dependencies": { "electron-updater": "^6.3.0", "get-port": "^5.1.1" }, "devDependencies": { "@electron/notarize": "^3.1.1", - "electron": "^33.0.0", + "electron": "castlabs/electron-releases#v33.4.11+wvcus", "electron-builder": "^25.1.0" } }, @@ -2206,9 +2207,8 @@ } }, "node_modules/electron": { - "version": "33.4.11", - "resolved": "https://registry.npmjs.org/electron/-/electron-33.4.11.tgz", - "integrity": "sha512-xmdAs5QWRkInC7TpXGNvzo/7exojubk+72jn1oJL7keNeIlw7xNglf8TGtJtkR4rWC5FJq0oXiIXPS9BcK2Irg==", + "version": "33.4.11+wvcus", + "resolved": "git+ssh://git@github.com/castlabs/electron-releases.git#d1cf58c11ec0a8a04f307ed362d7efde2816778d", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3953,9 +3953,9 @@ } }, "node_modules/node-abi": { - "version": "3.88.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.88.0.tgz", - "integrity": "sha512-At6b4UqIEVudaqPsXjmUO1r/N5BUr4yhDGs5PkBE8/oG5+TfLPhFechiskFsnT6Ql0VfUXbalUUCbfXxtj7K+w==", + "version": "3.89.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", + "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", "dev": true, "license": "MIT", "dependencies": { @@ -4605,9 +4605,9 @@ } }, "node_modules/sax": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.5.0.tgz", - "integrity": "sha512-21IYA3Q5cQf089Z6tgaUTr7lDAyzoTPx5HRtbhsME8Udispad8dC/+sziTNugOEx54ilvatQ9YCzl4KQLPcRHA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", + "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", "license": "BlueOak-1.0.0", "engines": { "node": ">=11.0.0" diff --git a/electron/package.json b/electron/package.json index 48ee49a7..53e18f84 100644 --- a/electron/package.json +++ b/electron/package.json @@ -1,11 +1,13 @@ { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.10", "description": "OpenSwarm — AI Agent Orchestrator", "main": "main.js", "scripts": { "start": "electron .", "dev": "ELECTRON_DEV=1 electron .", + "postinstall": "bash scripts/sign-vmp.sh", + "sign-vmp": "bash scripts/sign-vmp.sh", "dist": "electron-builder --mac --publish never", "dist:publish": "electron-builder --mac --publish always", "dist:all": "electron-builder --mac --win --linux" @@ -16,12 +18,15 @@ }, "devDependencies": { "@electron/notarize": "^3.1.1", - "electron": "^33.0.0", + "electron": "castlabs/electron-releases#v33.4.11+wvcus", "electron-builder": "^25.1.0" }, "build": { "appId": "com.clusterlabs.openswarm", "productName": "OpenSwarm", + "electronDownload": { + "mirror": "https://github.com/castlabs/electron-releases/releases/download/v" + }, "directories": { "output": "dist" }, @@ -38,6 +43,7 @@ "entitlementsInherit": "build/entitlements.mac.plist" }, "dmg": { + "artifactName": "OpenSwarm-${arch}.${ext}", "title": "OpenSwarm", "contents": [ { @@ -54,34 +60,24 @@ }, "extraResources": [ { - "from": "../frontend/dist", + "from": "build-staging/frontend", "to": "frontend", "filter": [ "**/*" ] }, { - "from": "../backend", + "from": "build-staging/backend", "to": "backend", "filter": [ - "**/*", - "!__pycache__/**", - "!**/__pycache__/**", - "!.venv/**", - "!*.pyc" + "**/*" ] }, { - "from": "../debugger", + "from": "build-staging/debugger", "to": "debugger", "filter": [ - "**/*", - "!__pycache__/**", - "!**/__pycache__/**", - "!*.pyc", - "!.venv/**", - "!**/.venv/**", - "!**/node_modules/**" + "**/*" ] }, { diff --git a/electron/preload.js b/electron/preload.js index 2f9037be..534c540f 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -2,15 +2,18 @@ const { contextBridge, ipcRenderer } = require('electron'); (async () => { const port = await ipcRenderer.invoke('get-backend-port'); + const webviewPreloadPath = await ipcRenderer.invoke('get-webview-preload-path'); contextBridge.exposeInMainWorld('__OPENSWARM_PORT__', port); contextBridge.exposeInMainWorld('openswarm', { getBackendPort: () => port, + getWebviewPreloadPath: () => webviewPreloadPath, getAppVersion: () => ipcRenderer.invoke('get-app-version'), openExternal: (url) => ipcRenderer.invoke('open-external', url), capturePage: (rect) => ipcRenderer.invoke('capture-page', rect), + getUpdateStatus: () => ipcRenderer.invoke('get-update-status'), checkForUpdates: () => ipcRenderer.invoke('check-for-updates'), downloadUpdate: () => ipcRenderer.invoke('download-update'), installUpdate: () => ipcRenderer.invoke('install-update'), @@ -40,5 +43,11 @@ const { contextBridge, ipcRenderer } = require('electron'); ipcRenderer.on('update-error', listener); return () => ipcRenderer.removeListener('update-error', listener); }, + + onWebviewNewWindow: (cb) => { + const listener = (_event, url, webContentsId) => cb(url, webContentsId); + ipcRenderer.on('webview-new-window', listener); + return () => ipcRenderer.removeListener('webview-new-window', listener); + }, }); })(); diff --git a/electron/scripts/sign-vmp.sh b/electron/scripts/sign-vmp.sh new file mode 100755 index 00000000..ba5eb833 --- /dev/null +++ b/electron/scripts/sign-vmp.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Signs the CastLabs Electron binary with a production VMP certificate via EVS, +# then repairs macOS framework symlinks that npm/signing may strip. +# +# First-time setup (one-time): +# pip3 install --user castlabs-evs +# python3 -m castlabs_evs.account signup +# +# After signup, this script runs automatically. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ELECTRON_DIR="$SCRIPT_DIR/../node_modules/electron/dist" +FW_BASE="$ELECTRON_DIR/Electron.app/Contents/Frameworks" + +fix_framework_symlinks() { + [ -d "$FW_BASE" ] || return 0 + for fw in "$FW_BASE"/*.framework; do + [ -d "$fw/Versions/A" ] || continue + local name + name=$(basename "$fw" .framework) + cd "$fw" + (cd Versions && ln -sf A Current 2>/dev/null) + ln -sf "Versions/Current/$name" "$name" 2>/dev/null + [ -d "Versions/A/Resources" ] && ln -sf Versions/Current/Resources Resources 2>/dev/null + [ -d "Versions/A/Libraries" ] && ln -sf Versions/Current/Libraries Libraries 2>/dev/null + [ -d "Versions/A/Helpers" ] && ln -sf Versions/Current/Helpers Helpers 2>/dev/null + done +} + +if [ ! -d "$ELECTRON_DIR" ]; then + echo "[vmp] Electron dist not found at $ELECTRON_DIR — skipping VMP signing" + fix_framework_symlinks + exit 0 +fi + +# Always fix symlinks first (npm git installs strip them) +fix_framework_symlinks + +if ! python3 -c "import castlabs_evs" 2>/dev/null; then + echo "[vmp] castlabs-evs not installed. Install with: pip3 install --user castlabs-evs" + echo "[vmp] Skipping VMP signing — DRM playback will be limited" + exit 0 +fi + +VERIFY_OUTPUT=$(python3 -m castlabs_evs.vmp verify-pkg "$ELECTRON_DIR" 2>&1) +if echo "$VERIFY_OUTPUT" | grep -q "Signature is valid" && ! echo "$VERIFY_OUTPUT" | grep -q "development only"; then + echo "[vmp] Electron already has a valid production VMP signature" + exit 0 +fi + +echo "[vmp] Signing Electron with production VMP certificate..." +if python3 -m castlabs_evs.vmp sign-pkg "$ELECTRON_DIR" 2>&1; then + echo "[vmp] VMP signing successful — full DRM playback enabled" + # Re-fix symlinks in case signing modified the bundle + fix_framework_symlinks +else + echo "[vmp] VMP signing failed — you may need to run: python3 -m castlabs_evs.account signup" + echo "[vmp] DRM playback will be limited to previews until signed" +fi + +exit 0 diff --git a/electron/webview-preload.js b/electron/webview-preload.js new file mode 100644 index 00000000..95a93d66 --- /dev/null +++ b/electron/webview-preload.js @@ -0,0 +1,87 @@ +/** + * Webview preload script — patches browser fingerprinting so sites like + * Spotify/Netflix don't detect an Electron shell and disable features. + * Loaded via the webview's `preload` attribute before any page script runs. + */ + +'use strict'; + +// Hide webdriver flag +Object.defineProperty(navigator, 'webdriver', { + get: () => false, + configurable: true, +}); + +// Spoof navigator.plugins (Chrome has a few built-in ones) +const fakePlugins = { + 0: { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, + 1: { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' }, + 2: { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }, + length: 3, + item: (i) => fakePlugins[i] || null, + namedItem: (name) => { + for (let i = 0; i < fakePlugins.length; i++) { + if (fakePlugins[i].name === name) return fakePlugins[i]; + } + return null; + }, + refresh: () => {}, + [Symbol.iterator]: function* () { + for (let i = 0; i < this.length; i++) yield this[i]; + }, +}; +try { + Object.defineProperty(navigator, 'plugins', { + get: () => fakePlugins, + configurable: true, + }); +} catch (_) {} + +// Ensure window.chrome exists (sites test for it) +if (!window.chrome) { + window.chrome = {}; +} +if (!window.chrome.runtime) { + window.chrome.runtime = { + connect: () => {}, + sendMessage: () => {}, + onMessage: { addListener: () => {}, removeListener: () => {} }, + }; +} + +// Ensure navigator.languages has sensible values +try { + Object.defineProperty(navigator, 'languages', { + get: () => ['en-US', 'en'], + configurable: true, + }); +} catch (_) {} + +// Patch permissions.query to report 'granted' for common permissions +const originalQuery = navigator.permissions?.query?.bind(navigator.permissions); +if (originalQuery) { + navigator.permissions.query = (params) => { + if (params.name === 'notifications') { + return Promise.resolve({ state: 'granted', onchange: null }); + } + return originalQuery(params).catch(() => + Promise.resolve({ state: 'prompt', onchange: null }) + ); + }; +} + +// Prevent iframe detection heuristics +try { + Object.defineProperty(document, 'hidden', { + get: () => false, + configurable: true, + }); + Object.defineProperty(document, 'visibilityState', { + get: () => 'visible', + configurable: true, + }); +} catch (_) {} + +// Fix console.debug detection (some sites use it as a breakpoint detector) +const noop = () => {}; +if (!window.console.debug) window.console.debug = noop; diff --git a/frontend/src/app/Main.tsx b/frontend/src/app/Main.tsx index c7c3e68b..50e051d8 100644 --- a/frontend/src/app/Main.tsx +++ b/frontend/src/app/Main.tsx @@ -3,7 +3,7 @@ import { Provider } from 'react-redux'; import { HashRouter, Routes, Route } from 'react-router-dom'; import { ThemeProvider as MuiThemeProvider, createTheme, CssBaseline } from '@mui/material'; import { store } from '../shared/state/store'; -import { useAppDispatch } from '@/shared/hooks'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { fetchSettings } from '@/shared/state/settingsSlice'; import { setAppVersion, @@ -156,9 +156,15 @@ const ShortcutsProvider: React.FC<{ children: React.ReactNode }> = ({ children } const SettingsLoader: React.FC<{ children: React.ReactNode }> = ({ children }) => { const dispatch = useAppDispatch(); + const { setMode: setThemeMode } = useThemeMode(); + const theme = useAppSelector((s) => s.settings.data.theme); + const loaded = useAppSelector((s) => s.settings.loaded); useEffect(() => { dispatch(fetchSettings()); }, [dispatch]); + useEffect(() => { + if (loaded) setThemeMode(theme as 'light' | 'dark'); + }, [loaded, theme, setThemeMode]); return <>{children}; }; @@ -171,6 +177,21 @@ const UpdateListener: React.FC<{ children: React.ReactNode }> = ({ children }) = api.getAppVersion().then((v: string) => dispatch(setAppVersion(v))); + api.getUpdateStatus?.().then((cached) => { + if (!cached) return; + if (cached.status === 'available' && cached.info?.version) { + dispatch(setUpdateAvailable(cached.info.version)); + } else if (cached.status === 'not-available') { + dispatch(setUpdateNotAvailable()); + } else if (cached.status === 'downloading' && cached.info?.percent != null) { + dispatch(setDownloading(cached.info.percent)); + } else if (cached.status === 'downloaded') { + dispatch(setUpdateDownloaded()); + } else if (cached.status === 'error' && cached.error) { + dispatch(setUpdateError(cached.error)); + } + }); + const cleanups = [ api.onUpdateAvailable?.((info: OpenSwarmUpdateInfo) => dispatch(setUpdateAvailable(info.version))), api.onUpdateNotAvailable?.(() => dispatch(setUpdateNotAvailable())), diff --git a/frontend/src/app/components/ElementSelectionContext.tsx b/frontend/src/app/components/ElementSelectionContext.tsx index d42e5d1f..5db25ba1 100644 --- a/frontend/src/app/components/ElementSelectionContext.tsx +++ b/frontend/src/app/components/ElementSelectionContext.tsx @@ -1,4 +1,4 @@ -import React, { createContext, useContext, useState, useRef, useCallback, RefObject } from 'react'; +import React, { createContext, useContext, useState, useRef, useCallback, useMemo, RefObject } from 'react'; export interface SelectedElement { id: string; @@ -20,11 +20,17 @@ interface ElementSelectionContextValue { setSelectMode: (active: boolean) => void; excludeSelectId: string | null; setExcludeSelectId: (id: string | null) => void; + activeOwnerId: string | null; + setActiveOwnerId: (id: string | null) => void; selectedElements: SelectedElement[]; addSelectedElement: (el: SelectedElement) => void; updateSelectedElement: (id: string, patch: Partial) => void; removeSelectedElement: (id: string) => void; clearSelectedElements: () => void; + elementsByOwner: Record; + addElementForOwner: (ownerId: string, el: SelectedElement) => void; + removeOwnerElement: (ownerId: string, elementId: string) => void; + clearOwnerElements: (ownerId: string) => void; iframeRef: RefObject; } @@ -37,9 +43,18 @@ export function useElementSelection() { export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { const [selectMode, setSelectMode] = useState(false); const [excludeSelectId, setExcludeSelectId] = useState(null); - const [selectedElements, setSelectedElements] = useState([]); + const [activeOwnerId, setActiveOwnerId] = useState(null); + const [elementsByOwner, setElementsByOwner] = useState>({}); const iframeRef = useRef(null); + const activeOwnerIdRef = useRef(activeOwnerId); + activeOwnerIdRef.current = activeOwnerId; + + const selectedElements = useMemo( + () => (activeOwnerId ? elementsByOwner[activeOwnerId] ?? [] : []), + [activeOwnerId, elementsByOwner], + ); + const toggleSelectMode = useCallback(() => { setSelectMode((prev) => { if (prev) setExcludeSelectId(null); @@ -48,22 +63,65 @@ export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = }, []); const addSelectedElement = useCallback((el: SelectedElement) => { - setSelectedElements((prev) => { - if (prev.some((e) => e.id === el.id)) return prev; - return [...prev, el]; + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + const existing = prev[ownerId] ?? []; + if (existing.some((e) => e.id === el.id)) return prev; + return { ...prev, [ownerId]: [...existing, el] }; }); }, []); const updateSelectedElement = useCallback((id: string, patch: Partial) => { - setSelectedElements((prev) => prev.map((e) => e.id === id ? { ...e, ...patch } : e)); + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + const existing = prev[ownerId]; + if (!existing) return prev; + return { ...prev, [ownerId]: existing.map((e) => (e.id === id ? { ...e, ...patch } : e)) }; + }); }, []); const removeSelectedElement = useCallback((id: string) => { - setSelectedElements((prev) => prev.filter((e) => e.id !== id)); + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + const existing = prev[ownerId]; + if (!existing) return prev; + return { ...prev, [ownerId]: existing.filter((e) => e.id !== id) }; + }); }, []); const clearSelectedElements = useCallback(() => { - setSelectedElements([]); + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + if (!prev[ownerId]?.length) return prev; + return { ...prev, [ownerId]: [] }; + }); + }, []); + + const addElementForOwner = useCallback((ownerId: string, el: SelectedElement) => { + setElementsByOwner((prev) => { + const existing = prev[ownerId] ?? []; + if (existing.some((e) => e.semanticData?.selectId === el.semanticData?.selectId)) return prev; + return { ...prev, [ownerId]: [...existing, el] }; + }); + }, []); + + const removeOwnerElement = useCallback((ownerId: string, elementId: string) => { + setElementsByOwner((prev) => { + const existing = prev[ownerId]; + if (!existing) return prev; + return { ...prev, [ownerId]: existing.filter((e) => e.id !== elementId) }; + }); + }, []); + + const clearOwnerElements = useCallback((ownerId: string) => { + setElementsByOwner((prev) => { + if (!prev[ownerId]?.length) return prev; + return { ...prev, [ownerId]: [] }; + }); }, []); return ( @@ -74,11 +132,17 @@ export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = setSelectMode, excludeSelectId, setExcludeSelectId, + activeOwnerId, + setActiveOwnerId, selectedElements, addSelectedElement, updateSelectedElement, removeSelectedElement, clearSelectedElements, + elementsByOwner, + addElementForOwner, + removeOwnerElement, + clearOwnerElements, iframeRef, }} > diff --git a/frontend/src/app/components/GlobalApprovalOverlay.tsx b/frontend/src/app/components/GlobalApprovalOverlay.tsx index 4405e7bf..a369ba1b 100644 --- a/frontend/src/app/components/GlobalApprovalOverlay.tsx +++ b/frontend/src/app/components/GlobalApprovalOverlay.tsx @@ -3,11 +3,23 @@ import Box from '@mui/material/Box'; import Typography from '@mui/material/Typography'; import Chip from '@mui/material/Chip'; import IconButton from '@mui/material/IconButton'; +import Tooltip from '@mui/material/Tooltip'; import ExpandLessIcon from '@mui/icons-material/ExpandLess'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import NotificationsActiveIcon from '@mui/icons-material/NotificationsActive'; +import StopCircleOutlinedIcon from '@mui/icons-material/StopCircleOutlined'; +import CloseIcon from '@mui/icons-material/Close'; +import { useNavigate } from 'react-router-dom'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; -import { handleApproval, ApprovalRequest } from '@/shared/state/agentsSlice'; +import { + handleApproval, + stopAgent, + dismissAgentNotification, + ApprovalRequest, + AgentSession, + HistorySession, +} from '@/shared/state/agentsSlice'; +import { setPendingFocusAgentId } from '@/shared/state/tempStateSlice'; import ApprovalBar, { BatchApprovalBar } from '@/app/pages/AgentChat/ApprovalBar'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; @@ -17,16 +29,133 @@ interface SessionApprovalGroup { approvals: ApprovalRequest[]; } +type TrackedAgent = { + id: string; + name: string; + status: AgentSession['status'] | string; + dashboardId?: string; +}; + +const STATUS_CONFIG: Record = { + running: { color: '', label: 'Running', tokenKey: 'success' }, + waiting_approval: { color: '', label: 'Waiting', tokenKey: 'warning' }, + completed: { color: '', label: 'Done', tokenKey: 'success' }, + error: { color: '', label: 'Error', tokenKey: 'error' }, + stopped: { color: '', label: 'Stopped', tokenKey: 'info' }, +}; + +const StatusDot: React.FC<{ status: string; c: ReturnType }> = ({ status, c }) => { + const cfg = STATUS_CONFIG[status]; + const color = cfg?.tokenKey ? (c.status as any)[cfg.tokenKey] : c.text.ghost; + const isActive = status === 'running'; + return ( + + ); +}; + +const AgentStatusRow: React.FC<{ + agent: TrackedAgent; + c: ReturnType; + onStop: (id: string) => void; + onDismiss: (id: string) => void; + onNavigate: (dashboardId: string, agentId: string) => void; +}> = ({ agent, c, onStop, onDismiss, onNavigate }) => { + const isActive = agent.status === 'running' || agent.status === 'waiting_approval'; + const cfg = STATUS_CONFIG[agent.status] ?? { label: agent.status }; + + return ( + agent.dashboardId && onNavigate(agent.dashboardId, agent.id)} + sx={{ + display: 'flex', + alignItems: 'center', + gap: 1, + px: 2, + py: 0.75, + cursor: agent.dashboardId ? 'pointer' : 'default', + '&:hover': { bgcolor: `${c.text.ghost}10` }, + transition: 'background-color 0.15s', + minHeight: 36, + }} + > + + + {agent.name} + + + {cfg.label} + + {isActive ? ( + + { e.stopPropagation(); onStop(agent.id); }} + sx={{ p: 0.25, color: c.status.error, '&:hover': { bgcolor: `${c.status.error}15` } }} + > + + + + ) : ( + + { e.stopPropagation(); onDismiss(agent.id); }} + sx={{ p: 0.25, color: c.text.ghost, '&:hover': { bgcolor: `${c.text.ghost}15` } }} + > + + + + )} + + ); +}; + const GlobalApprovalOverlay: React.FC = () => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const navigate = useNavigate(); const sessions = useAppSelector((state) => state.agents.sessions); + const history = useAppSelector((state) => state.agents.history); + const trackedIds = useAppSelector((state) => state.agents.trackedNotificationIds); const [collapsed, setCollapsed] = useState(false); const groups: SessionApprovalGroup[] = useMemo(() => { const result: SessionApprovalGroup[] = []; for (const [sessionId, session] of Object.entries(sessions)) { - if (session.pending_approvals.length > 0) { + if (session.pending_approvals?.length > 0) { result.push({ sessionId, sessionName: session.name || 'Agent', @@ -42,11 +171,38 @@ const GlobalApprovalOverlay: React.FC = () => { [groups], ); + const trackedAgents: TrackedAgent[] = useMemo(() => { + return trackedIds + .map((id): TrackedAgent | null => { + const session = sessions[id]; + if (session && session.status !== 'draft') { + return { id, name: session.name, status: session.status, dashboardId: session.dashboard_id }; + } + const hist: HistorySession | undefined = history[id]; + if (hist) { + return { id, name: hist.name, status: hist.status, dashboardId: hist.dashboard_id }; + } + return null; + }) + .filter((a): a is TrackedAgent => a !== null); + }, [trackedIds, sessions, history]); + + const activeAgents = useMemo( + () => trackedAgents.filter((a) => a.status === 'running' || a.status === 'waiting_approval'), + [trackedAgents], + ); + const finishedAgents = useMemo( + () => trackedAgents.filter((a) => a.status !== 'running' && a.status !== 'waiting_approval'), + [trackedAgents], + ); + + const totalBadge = totalApprovals + activeAgents.length; + useEffect(() => { - if (totalApprovals > 0) { + if (totalApprovals > 0 || activeAgents.length > 0) { setCollapsed(false); } - }, [totalApprovals]); + }, [totalApprovals, activeAgents.length]); const onApprove = useCallback( (requestId: string, updatedInput?: Record) => { @@ -62,7 +218,39 @@ const GlobalApprovalOverlay: React.FC = () => { [dispatch], ); - if (totalApprovals === 0) return null; + const onStopAgent = useCallback( + (sessionId: string) => { + dispatch(stopAgent({ sessionId })); + }, + [dispatch], + ); + + const onDismissAgent = useCallback( + (sessionId: string) => { + dispatch(dismissAgentNotification(sessionId)); + }, + [dispatch], + ); + + const onNavigateToDashboard = useCallback( + (dashboardId: string, agentId: string) => { + dispatch(setPendingFocusAgentId(agentId)); + navigate(`/dashboard/${dashboardId}`); + }, + [navigate, dispatch], + ); + + if (totalApprovals === 0 && trackedAgents.length === 0) return null; + + const hasApprovals = totalApprovals > 0; + const hasAgents = trackedAgents.length > 0; + const headerTitle = hasApprovals && !hasAgents + ? 'Approval Required' + : hasAgents && !hasApprovals + ? 'Agents' + : 'Notifications'; + const headerColor = hasApprovals ? c.status.warning : c.status.info; + const headerBg = hasApprovals ? c.status.warningBg : c.status.infoBg; return ( { flexDirection: 'column', borderRadius: `${c.radius.xl}px`, bgcolor: c.bg.surface, - border: `1px solid ${c.status.warning}40`, - boxShadow: `0 8px 32px rgba(0,0,0,0.25), 0 0 0 1px ${c.status.warning}20`, + border: `1px solid ${headerColor}40`, + boxShadow: `0 8px 32px rgba(0,0,0,0.25), 0 0 0 1px ${headerColor}20`, overflow: 'hidden', animation: 'approvalSlideIn 0.25s ease-out', '@keyframes approvalSlideIn': { @@ -97,19 +285,19 @@ const GlobalApprovalOverlay: React.FC = () => { gap: 1, px: 2, py: 1.25, - bgcolor: c.status.warningBg, - borderBottom: collapsed ? 'none' : `1px solid ${c.status.warning}20`, + bgcolor: headerBg, + borderBottom: collapsed ? 'none' : `1px solid ${headerColor}20`, cursor: 'pointer', userSelect: 'none', - '&:hover': { bgcolor: `${c.status.warning}18` }, + '&:hover': { bgcolor: `${headerColor}18` }, transition: 'background-color 0.15s', }} > { }, }} /> - - Approval Required + + {headerTitle} - + {totalBadge > 0 && ( + + )} {collapsed ? : } @@ -146,7 +336,6 @@ const GlobalApprovalOverlay: React.FC = () => { { scrollbarColor: `${c.border.medium} transparent`, }} > - {groups.map((group) => ( - - {groups.length > 1 && ( + {/* Approvals section */} + {hasApprovals && ( + + {hasAgents && ( - {group.sessionName} + Approvals )} - {group.approvals.length > 1 ? ( - - ) : ( - group.approvals.map((req) => ( - - )) - )} + {groups.map((group) => ( + + {groups.length > 1 && ( + + {group.sessionName} + + )} + {group.approvals.length > 1 ? ( + + ) : ( + group.approvals.map((req) => ( + + )) + )} + + ))} - ))} + )} + + {/* Divider between sections */} + {hasApprovals && hasAgents && ( + + )} + + {/* Agent status section */} + {hasAgents && ( + + {hasApprovals && ( + + Agents + + )} + {activeAgents.map((agent) => ( + + ))} + {finishedAgents.map((agent) => ( + + ))} + + )} )} diff --git a/frontend/src/app/components/KeyboardShortcutsHelp.tsx b/frontend/src/app/components/KeyboardShortcutsHelp.tsx index 1447a635..23e9aa29 100644 --- a/frontend/src/app/components/KeyboardShortcutsHelp.tsx +++ b/frontend/src/app/components/KeyboardShortcutsHelp.tsx @@ -11,6 +11,7 @@ const shortcuts = [ { key: 't', description: 'Go to Templates' }, { key: '1-9', description: 'Open agent by position' }, { key: '⌘M', description: 'Add App' }, + { key: '⌘N', description: 'New Browser' }, { key: '⌘O', description: 'History' }, { key: 'Shift+A', description: 'Approve all pending' }, { key: 'Shift+D', description: 'Deny all pending' }, diff --git a/frontend/src/app/components/Layout/AppShell.tsx b/frontend/src/app/components/Layout/AppShell.tsx index ac9f8740..cd3e51a0 100644 --- a/frontend/src/app/components/Layout/AppShell.tsx +++ b/frontend/src/app/components/Layout/AppShell.tsx @@ -28,17 +28,24 @@ import ViewSidebarOutlinedIcon from '@mui/icons-material/ViewSidebarOutlined'; import ArrowBackOutlinedIcon from '@mui/icons-material/ArrowBackOutlined'; import ArrowForwardOutlinedIcon from '@mui/icons-material/ArrowForwardOutlined'; import RestartAltIcon from '@mui/icons-material/RestartAlt'; +import SystemUpdateAltIcon from '@mui/icons-material/SystemUpdateAlt'; +import CloseIcon from '@mui/icons-material/Close'; +import LinearProgress from '@mui/material/LinearProgress'; import Settings from '@/app/pages/Settings/Settings'; import GlobalApprovalOverlay from '@/app/components/GlobalApprovalOverlay'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { fetchDashboards, createDashboard, renameDashboard } from '@/shared/state/dashboardsSlice'; +import { addBrowserCard, addBrowserTab } from '@/shared/state/dashboardLayoutSlice'; +import { setPendingBrowserUrl } from '@/shared/state/tempStateSlice'; import { fetchOutputs } from '@/shared/state/outputsSlice'; +import { findBrowserByWebContentsId } from '@/shared/browserRegistry'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; const SIDEBAR_MIN = 160; const SIDEBAR_MAX = 400; const SIDEBAR_DEFAULT = 220; const SIDEBAR_WIDTH_KEY = 'openswarm-sidebar-width'; +const UPDATE_DISMISS_KEY = 'openswarm-update-dismissed'; const CUSTOMIZATION_ITEMS = [ { label: 'Prompts', path: '/templates', icon: }, @@ -76,10 +83,34 @@ const AppShell: React.FC = () => { const updateStatus = useAppSelector((state) => state.update.status); const availableVersion = useAppSelector((state) => state.update.availableVersion); - const [updateBannerDismissed, setUpdateBannerDismissed] = useState(false); + const downloadPercent = useAppSelector((state) => state.update.downloadPercent); - const showUpdateDot = updateStatus === 'available' || updateStatus === 'downloaded'; - const showUpdateBanner = updateStatus === 'downloaded' && !updateBannerDismissed; + const [dismissedVersion, setDismissedVersion] = useState(() => { + try { return localStorage.getItem(UPDATE_DISMISS_KEY); } catch { return null; } + }); + const [snackbarDismissed, setSnackbarDismissed] = useState(false); + + const bannerDismissedForVersion = availableVersion != null && dismissedVersion === availableVersion; + const isUpdateActionable = updateStatus === 'available' || updateStatus === 'downloaded' || updateStatus === 'downloading'; + + const showUpdateDot = (updateStatus === 'available' || updateStatus === 'downloaded') && !bannerDismissedForVersion; + const showUpdateBanner = isUpdateActionable && !bannerDismissedForVersion; + const showUpdateSnackbar = (updateStatus === 'available' || updateStatus === 'downloaded') && !bannerDismissedForVersion && !snackbarDismissed; + + const handleDismissBanner = useCallback(() => { + if (availableVersion) { + try { localStorage.setItem(UPDATE_DISMISS_KEY, availableVersion); } catch {} + setDismissedVersion(availableVersion); + } + }, [availableVersion]); + + const handleDownloadUpdate = useCallback(async () => { + try { await (window as any).openswarm?.downloadUpdate(); } catch {} + }, []); + + const handleInstallUpdate = useCallback(() => { + (window as any).openswarm?.installUpdate(); + }, []); const dashboardItems = useAppSelector((state) => state.dashboards.items); const dashboardList = Object.values(dashboardItems).sort( @@ -96,6 +127,75 @@ const AppShell: React.FC = () => { dispatch(fetchOutputs()); }, [dispatch]); + const openUrlInBrowser = useCallback((url: string, webContentsId?: number) => { + const dashMatch = location.pathname.match(/^\/dashboard\/(.+)/); + if (dashMatch) { + if (webContentsId != null) { + const browserId = findBrowserByWebContentsId(webContentsId); + if (browserId) { + dispatch(addBrowserTab({ browserId, url, makeActive: true })); + return; + } + } + dispatch(addBrowserCard({ url })); + } else { + dispatch(setPendingBrowserUrl(url)); + const lastId = (window as any).__openswarm_last_dashboard_id as string | undefined; + const firstDashboard = dashboardList[0]; + const targetId = lastId || firstDashboard?.id; + if (targetId) { + navigate(`/dashboard/${targetId}`); + } else { + dispatch(createDashboard('Untitled Dashboard')).then((result: any) => { + if (createDashboard.fulfilled.match(result)) { + navigate(`/dashboard/${result.payload.id}`); + } + }); + } + } + }, [location.pathname, dashboardList, dispatch, navigate]); + + useEffect(() => { + let lastUrl = ''; + let lastTime = 0; + + const handleClick = (e: MouseEvent) => { + const anchor = (e.target as HTMLElement)?.closest?.('a'); + if (!anchor) return; + const href = anchor.getAttribute('href'); + if (!href) return; + if (!/^https?:\/\//i.test(href)) return; + if (href.startsWith('http://localhost:')) return; + + e.preventDefault(); + e.stopPropagation(); + + const now = Date.now(); + if (href === lastUrl && now - lastTime < 1000) return; + lastUrl = href; + lastTime = now; + + openUrlInBrowser(href); + }; + + document.addEventListener('click', handleClick, true); + return () => document.removeEventListener('click', handleClick, true); + }, [openUrlInBrowser]); + + useEffect(() => { + const w = window as any; + if (!w.openswarm?.onWebviewNewWindow) return; + let lastUrl = ''; + let lastTime = 0; + return w.openswarm.onWebviewNewWindow((url: string, webContentsId: number) => { + const now = Date.now(); + if (url === lastUrl && now - lastTime < 1000) return; + lastUrl = url; + lastTime = now; + openUrlInBrowser(url, webContentsId); + }); + }, [openUrlInBrowser]); + useEffect(() => { try { localStorage.setItem(SIDEBAR_WIDTH_KEY, String(sidebarWidth)); } catch {} }, [sidebarWidth]); @@ -281,6 +381,98 @@ const AppShell: React.FC = () => { + {showUpdateBanner && ( + + + + {updateStatus === 'available' && `OpenSwarm ${availableVersion} is available`} + {updateStatus === 'downloading' && `Downloading OpenSwarm ${availableVersion}…`} + {updateStatus === 'downloaded' && `OpenSwarm ${availableVersion} is ready to install`} + + {updateStatus === 'downloading' && ( + + )} + {updateStatus === 'downloading' && ( + + {Math.round(downloadPercent)}% + + )} + {updateStatus === 'available' && ( + + )} + {updateStatus === 'downloaded' && ( + + )} + + + + + )} + {!sidebarCollapsed && ( <> @@ -769,36 +961,60 @@ const AppShell: React.FC = () => { setSnackbarDismissed(true)} anchorOrigin={{ vertical: 'bottom', horizontal: 'center' }} > } + icon={updateStatus === 'downloaded' + ? + : + } action={ - + {updateStatus === 'available' && ( + + )} + {updateStatus === 'downloaded' && ( + + )} } sx={{ @@ -809,7 +1025,8 @@ const AppShell: React.FC = () => { '& .MuiAlert-icon': { color: c.accent.primary }, }} > - OpenSwarm {availableVersion} downloaded — restart to update + {updateStatus === 'available' && `OpenSwarm ${availableVersion} is available`} + {updateStatus === 'downloaded' && `OpenSwarm ${availableVersion} downloaded — restart to update`} diff --git a/frontend/src/app/components/useDomElementSelector.ts b/frontend/src/app/components/useDomElementSelector.ts index b4535e19..8115769b 100644 --- a/frontend/src/app/components/useDomElementSelector.ts +++ b/frontend/src/app/components/useDomElementSelector.ts @@ -120,6 +120,7 @@ export function useDomElementSelector(): DomSelectorState { const dragOriginRef = useRef<{ x: number; y: number } | null>(null); const isDraggingRef = useRef(false); const dragBoundsRef = useRef<{ left: number; top: number; right: number; bottom: number } | null>(null); + const preDragFocusRef = useRef(null); const excludeIdRef = useRef(null); useEffect(() => { @@ -245,8 +246,11 @@ export function useDomElementSelector(): DomSelectorState { if (e.button !== 0) return; if (e.metaKey || e.ctrlKey) return; const target = e.target as Element; - // Only start drag on "empty" canvas areas (not on selectable elements) - if (target && findSelectableAncestor(target, excludeIdRef.current)) return; + if (target && findSelectableAncestor(target, excludeIdRef.current)) { + e.preventDefault(); + return; + } + preDragFocusRef.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; dragOriginRef.current = { x: e.clientX, y: e.clientY }; isDraggingRef.current = false; }, []); @@ -291,12 +295,17 @@ export function useDomElementSelector(): DomSelectorState { }); } + const wasDragging = isDraggingRef.current; dragOriginRef.current = null; isDraggingRef.current = false; dragBoundsRef.current = null; setDragRect(EMPTY_DRAG); setDragPreview([]); if (dragPreviewRafRef.current) cancelAnimationFrame(dragPreviewRafRef.current); + if (wasDragging && preDragFocusRef.current) { + preDragFocusRef.current.focus(); + } + preDragFocusRef.current = null; }, [ctx]); const handleClick = useCallback((e: MouseEvent) => { @@ -327,6 +336,7 @@ export function useDomElementSelector(): DomSelectorState { dragOriginRef.current = null; dragBoundsRef.current = null; isDraggingRef.current = false; + preDragFocusRef.current = null; return; } @@ -353,6 +363,7 @@ export function useDomElementSelector(): DomSelectorState { dragOriginRef.current = null; dragBoundsRef.current = null; isDraggingRef.current = false; + preDragFocusRef.current = null; }; }, [ctx?.selectMode, handleMouseMove, handleMouseDown, handleMouseUp, handleClick]); diff --git a/frontend/src/app/pages/AgentChat/AgentChat.tsx b/frontend/src/app/pages/AgentChat/AgentChat.tsx index 0c5c43c5..6d1764cd 100644 --- a/frontend/src/app/pages/AgentChat/AgentChat.tsx +++ b/frontend/src/app/pages/AgentChat/AgentChat.tsx @@ -5,8 +5,16 @@ import Typography from '@mui/material/Typography'; import Chip from '@mui/material/Chip'; import IconButton from '@mui/material/IconButton'; import Tooltip from '@mui/material/Tooltip'; +import TextField from '@mui/material/TextField'; +import ClickAwayListener from '@mui/material/ClickAwayListener'; import CloseIcon from '@mui/icons-material/Close'; import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'; +import KeyboardArrowUpIcon from '@mui/icons-material/KeyboardArrowUp'; +import PlayArrowIcon from '@mui/icons-material/PlayArrow'; +import EditOutlinedIcon from '@mui/icons-material/EditOutlined'; +import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; +import CheckIcon from '@mui/icons-material/Check'; +import DragIndicatorIcon from '@mui/icons-material/DragIndicator'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { sendMessage as sendMessageThunk, @@ -17,6 +25,8 @@ import { handleApproval, editMessage, switchBranch, + duplicateSession, + setActiveSession, updateSessionModel, updateSessionMode, fetchSession, @@ -25,12 +35,12 @@ import { import { fetchModes } from '@/shared/state/modesSlice'; import { createSessionWs } from '@/shared/ws/WebSocketManager'; import MessageBubble from './MessageBubble'; +import MessageActionBar from './MessageActionBar'; import ToolCallBubble, { ToolPair } from './ToolCallBubble'; import ToolGroupBubble, { RenderItem, ToolGroup, isToolGroup, isToolPair } from './ToolGroupBubble'; import ApprovalBar, { BatchApprovalBar } from './ApprovalBar'; import ChatInput, { ChatInputHandle } from './ChatInput'; import { ContextPath } from '@/app/components/DirectoryBrowser'; -import BranchNavigator from './BranchNavigator'; import DiffViewer from './DiffViewer'; import { setGlowingBrowserCards, clearGlowingBrowserCards } from '@/shared/state/dashboardLayoutSlice'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; @@ -91,14 +101,27 @@ const ThinkingBubble: React.FC = () => { ); }; +interface QueuedMessage { + prompt: string; + images?: Array<{ data: string; media_type: string }>; + contextPaths?: Array<{ path: string; type: 'file' | 'directory' }>; + forcedTools?: string[]; + attachedSkills?: Array<{ id: string; name: string; content: string }>; + selectedBrowserIds?: string[]; +} + interface AgentChatProps { sessionId?: string; onClose?: () => void; embedded?: boolean; + autoFocus?: boolean; + isGlowing?: boolean; + onDismissGlow?: () => void; initialContextPaths?: ContextPath[]; + onBranch?: (newSessionId: string) => void; } -const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose, embedded, initialContextPaths }) => { +const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose, embedded, autoFocus, isGlowing, onDismissGlow, initialContextPaths, onBranch }) => { const c = useClaudeTokens(); const STATUS_STYLES: Record = { running: { color: c.status.success, bg: c.status.successBg }, @@ -116,11 +139,20 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const chatInputRef = useRef(null); const isAtBottomRef = useRef(true); const [showScrollButton, setShowScrollButton] = useState(false); + const [showResumeBubble, setShowResumeBubble] = useState(false); + const [awaitingResponse, setAwaitingResponse] = useState(false); const [mode, setMode] = useState('agent'); const [model, setModel] = useState('sonnet'); const wsRef = useRef | null>(null); const initialContextApplied = useRef(false); + const messageQueueRef = useRef([]); + const [queueLength, setQueueLength] = useState(0); + const [queueExpanded, setQueueExpanded] = useState(false); + const [editingQueueIdx, setEditingQueueIdx] = useState(null); + const [editingQueueText, setEditingQueueText] = useState(''); + const [dragIdx, setDragIdx] = useState(null); + const [dropTargetIdx, setDropTargetIdx] = useState(null); const isDraft = session?.status === 'draft'; @@ -157,13 +189,64 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose if (Object.keys(modesMap).length === 0) dispatch(fetchModes()); }, [dispatch, modesMap]); + const dispatchMessage = useCallback((msg: QueuedMessage) => { + if (!id) return; + setShowResumeBubble(false); + setAwaitingResponse(true); + if (isDraft) { + const config: Record = { model, mode }; + if (session?.system_prompt) config.system_prompt = session.system_prompt; + if (session?.target_directory) config.target_directory = session.target_directory; + dispatch( + launchAndSendFirstMessage({ draftId: id, config, prompt: msg.prompt, mode, model, images: msg.images, contextPaths: msg.contextPaths, forcedTools: msg.forcedTools, attachedSkills: msg.attachedSkills, selectedBrowserIds: msg.selectedBrowserIds }) + ).then((action) => { + if (launchAndSendFirstMessage.fulfilled.match(action)) { + const realId = action.payload.session.id; + dispatch(generateTitle({ sessionId: realId, prompt: msg.prompt })); + if (msg.selectedBrowserIds?.length) { + dispatch(setGlowingBrowserCards({ browserIds: msg.selectedBrowserIds, sessionId: realId })); + } + } + }); + } else { + if (msg.selectedBrowserIds?.length) { + dispatch(setGlowingBrowserCards({ browserIds: msg.selectedBrowserIds, sessionId: id })); + } + dispatch(sendMessageThunk({ sessionId: id, prompt: msg.prompt, mode, model, images: msg.images, contextPaths: msg.contextPaths, forcedTools: msg.forcedTools, attachedSkills: msg.attachedSkills, selectedBrowserIds: msg.selectedBrowserIds })) + .then((action) => { + if (sendMessageThunk.rejected.match(action)) { + setAwaitingResponse(false); + } + }); + } + }, [id, isDraft, mode, model, session?.system_prompt, session?.target_directory, dispatch]); + + const agentBusy = awaitingResponse || (!isDraft && (session?.status === 'running' || session?.status === 'waiting_approval')); + const prevStatusRef = useRef(session?.status); useEffect(() => { const prev = prevStatusRef.current; const curr = session?.status; prevStatusRef.current = curr; - if (prev === 'running' && (curr === 'completed' || curr === 'stopped' || curr === 'error')) { + let didDispatchQueued = false; + + const wasActive = prev === 'running' || prev === 'waiting_approval'; + const isTerminal = curr === 'completed' || curr === 'stopped' || curr === 'error'; + + if (wasActive && isTerminal) { if (id) dispatch(clearGlowingBrowserCards(id)); + + const nextQueued = messageQueueRef.current.shift(); + if (nextQueued) { + setQueueLength(messageQueueRef.current.length); + dispatchMessage(nextQueued); + didDispatchQueued = true; + } else { + if (curr === 'stopped') { + setShowResumeBubble(true); + } + } + const currentMode = modesMap[mode]; if (currentMode?.default_next_mode && modesMap[currentMode.default_next_mode]) { setMode(currentMode.default_next_mode); @@ -172,7 +255,13 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose } } } - }, [session?.status, mode, modesMap, id, isDraft, dispatch]); + if (curr === 'running') { + setShowResumeBubble(false); + } + if (curr !== 'draft' && !didDispatchQueued) { + setAwaitingResponse(false); + } + }, [session?.status, mode, modesMap, id, isDraft, dispatch, dispatchMessage]); const SCROLL_THRESHOLD = 50; @@ -201,27 +290,13 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const handleSend = (prompt: string, images?: Array<{ data: string; media_type: string }>, contextPaths?: Array<{ path: string; type: 'file' | 'directory' }>, forcedTools?: string[], attachedSkills?: Array<{ id: string; name: string; content: string }>, selectedBrowserIds?: string[]) => { if (!id) return; - if (isDraft) { - const config: Record = { model, mode }; - if (session?.system_prompt) config.system_prompt = session.system_prompt; - if (session?.target_directory) config.target_directory = session.target_directory; - dispatch( - launchAndSendFirstMessage({ draftId: id, config, prompt, mode, model, images, contextPaths, forcedTools, attachedSkills }) - ).then((action) => { - if (launchAndSendFirstMessage.fulfilled.match(action)) { - const realId = action.payload.session.id; - dispatch(generateTitle({ sessionId: realId, prompt })); - if (selectedBrowserIds?.length) { - dispatch(setGlowingBrowserCards({ browserIds: selectedBrowserIds, sessionId: realId })); - } - } - }); - } else { - if (selectedBrowserIds?.length) { - dispatch(setGlowingBrowserCards({ browserIds: selectedBrowserIds, sessionId: id })); - } - dispatch(sendMessageThunk({ sessionId: id, prompt, mode, model, images, contextPaths, forcedTools, attachedSkills })); + const msg: QueuedMessage = { prompt, images, contextPaths, forcedTools, attachedSkills, selectedBrowserIds }; + if (agentBusy) { + messageQueueRef.current.push(msg); + setQueueLength(messageQueueRef.current.length); + return; } + dispatchMessage(msg); }; const handleModeChange = useCallback((newMode: string) => { @@ -247,14 +322,33 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose dispatch(stopAgent({ sessionId: id })); }; - const handleEdit = useCallback( + const handleResume = useCallback(() => { + if (!id) return; + setShowResumeBubble(false); + dispatch(sendMessageThunk({ + sessionId: id, + prompt: "Continue where you left off. Start you're response EXACTLY with 'Sorry, let me pick up where I left off", + mode, + model, + hidden: true, + })); + }, [id, mode, model, dispatch]); + + const [editingMessageId, setEditingMessageId] = useState(null); + + const handleSaveEdit = useCallback( (messageId: string, newContent: string) => { if (!id) return; dispatch(editMessage({ sessionId: id, messageId, content: newContent })); + setEditingMessageId(null); }, [id, dispatch] ); + const handleCancelEdit = useCallback(() => { + setEditingMessageId(null); + }, []); + const activeBranchMessages = useMemo(() => { if (!session) return []; const branchId = session.active_branch_id || 'main'; @@ -264,14 +358,74 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose return session.messages.filter((m) => m.branch_id === 'main' || m.branch_id === branchId); } - const forkIdx = session.messages.findIndex((m) => m.id === branch.fork_point_message_id); - const preMessages = session.messages - .slice(0, forkIdx) - .filter((m) => m.branch_id === (branch.parent_branch_id || 'main')); - const branchMessages = session.messages.filter((m) => m.branch_id === branchId); - return [...preMessages, ...branchMessages]; + const segments: Array<{ branchId: string; upToMessageId?: string }> = []; + let cur = branch; + let curId = branchId; + while (cur && cur.fork_point_message_id) { + segments.unshift({ branchId: curId, upToMessageId: cur.fork_point_message_id }); + curId = cur.parent_branch_id || 'main'; + cur = session.branches?.[curId]; + } + segments.unshift({ branchId: curId }); + + const result: typeof session.messages = []; + for (let i = 0; i < segments.length; i++) { + const seg = segments[i]; + const nextForkMsgId = seg.upToMessageId; + if (nextForkMsgId) { + const forkIdx = session.messages.findIndex((m) => m.id === nextForkMsgId); + const pre = session.messages + .slice(0, forkIdx) + .filter((m) => m.branch_id === seg.branchId); + result.push(...pre); + } else if (i < segments.length - 1) { + const nextFork = segments[i + 1].upToMessageId; + const forkIdx = nextFork + ? session.messages.findIndex((m) => m.id === nextFork) + : session.messages.length; + result.push( + ...session.messages.slice(0, forkIdx).filter((m) => m.branch_id === seg.branchId) + ); + } else { + result.push(...session.messages.filter((m) => m.branch_id === seg.branchId)); + } + } + const leafMsgs = session.messages.filter((m) => m.branch_id === branchId); + if (!result.some((m) => m.branch_id === branchId)) { + result.push(...leafMsgs); + } + return result; }, [session?.messages, session?.active_branch_id, session?.branches]); + const handleRegenerate = useCallback( + (assistantMsg: AgentMessage) => { + if (!id) return; + const idx = activeBranchMessages.findIndex((m) => m.id === assistantMsg.id); + for (let i = idx - 1; i >= 0; i--) { + if (activeBranchMessages[i].role === 'user') { + const userMsg = activeBranchMessages[i]; + const content = typeof userMsg.content === 'string' ? userMsg.content : JSON.stringify(userMsg.content); + dispatch(editMessage({ sessionId: id, messageId: userMsg.id, content })); + break; + } + } + }, + [id, activeBranchMessages, dispatch] + ); + + const handleBranchChat = useCallback(async (upToMessageId: string) => { + if (!id) return; + const dashId = session?.dashboard_id; + const action = await dispatch(duplicateSession({ sessionId: id, dashboardId: dashId, upToMessageId })); + if (duplicateSession.fulfilled.match(action)) { + if (onBranch) { + onBranch(action.payload.id); + } else { + dispatch(setActiveSession(action.payload.id)); + } + } + }, [id, dispatch, onBranch, session?.dashboard_id]); + const contextEstimate = useMemo(() => { const limit = CONTEXT_WINDOWS[model] || 200_000; let totalChars = 0; @@ -373,13 +527,33 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose items.push(...outputItems); } else { - items.push(msg); + if (!msg.hidden) { + items.push(msg); + } i++; } } return items; }, [activeBranchMessages]); + const lastAssistantIdsInTurn = useMemo(() => { + const ids = new Set(); + let lastAssistantId: string | null = null; + for (const item of renderItems) { + if (!isToolGroup(item) && !isToolPair(item)) { + const msg = item as AgentMessage; + if (msg.role === 'assistant') { + lastAssistantId = msg.id; + } else if (msg.role === 'user') { + if (lastAssistantId) ids.add(lastAssistantId); + lastAssistantId = null; + } + } + } + if (lastAssistantId) ids.add(lastAssistantId); + return ids; + }, [renderItems]); + const groupMetaRequestedRef = useRef>(new Set()); const groupMetaRefinedRef = useRef>(new Set()); @@ -427,11 +601,33 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const getSiblingBranches = useCallback( (messageId: string): string[] => { if (!session?.branches) return []; - return Object.values(session.branches) + + const directForks = Object.values(session.branches) .filter((b) => b.fork_point_message_id === messageId) .map((b) => b.id); + if (directForks.length > 0) { + const originalMsg = session.messages.find((m) => m.id === messageId); + const parentBranchId = originalMsg?.branch_id || 'main'; + return [parentBranchId, ...directForks]; + } + + const msg = session.messages.find((m) => m.id === messageId); + if (!msg || msg.role !== 'user') return []; + const msgBranch = session.branches[msg.branch_id]; + if (!msgBranch?.fork_point_message_id) return []; + const branchUserMsgs = session.messages.filter( + (m) => m.branch_id === msg.branch_id && m.role === 'user' + ); + if (branchUserMsgs.length === 0 || branchUserMsgs[0].id !== messageId) return []; + + const forkPointId = msgBranch.fork_point_message_id; + const siblingBranches = Object.values(session.branches) + .filter((b) => b.fork_point_message_id === forkPointId) + .map((b) => b.id); + const parentBranchId = msgBranch.parent_branch_id || 'main'; + return [parentBranchId, ...siblingBranches]; }, - [session?.branches] + [session?.branches, session?.messages] ); if (!session) { @@ -527,37 +723,55 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose {renderItems.map((item) => { if (isToolGroup(item)) { const groupMeta = session.tool_group_meta?.[item.id]; - return ; + return ; } if (isToolPair(item)) { const isPending = item.result === null && sessionRunning; - return ; + return ; } const msg = item; + const isEditing = editingMessageId === msg.id; const siblings = getSiblingBranches(msg.id); const hasBranches = siblings.length > 0; const currentBranchIdx = hasBranches ? siblings.indexOf(session.active_branch_id || 'main') : 0; + const rawText = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content); return ( - - - {hasBranches && ( - { - const prevBranch = siblings[Math.max(0, currentBranchIdx - 1)]; - if (prevBranch && id) dispatch(switchBranch({ sessionId: id, branchId: prevBranch })); - }} - onNext={() => { - const nextBranch = siblings[Math.min(siblings.length - 1, currentBranchIdx + 1)]; - if (nextBranch && id) dispatch(switchBranch({ sessionId: id, branchId: nextBranch })); - }} + + + {!isEditing && (msg.role === 'user' || (msg.role === 'assistant' && lastAssistantIdsInTurn.has(msg.id))) && ( + navigator.clipboard.writeText(rawText)} + onEdit={msg.role === 'user' ? () => setEditingMessageId(msg.id) : undefined} + onRegenerate={msg.role === 'assistant' ? () => handleRegenerate(msg) : undefined} + onBranch={msg.role === 'assistant' ? () => handleBranchChat(msg.id) : undefined} + branchNav={ + hasBranches + ? { + currentIndex: Math.max(0, currentBranchIdx), + totalBranches: siblings.length, + onPrevious: () => { + const prevBranch = siblings[Math.max(0, currentBranchIdx - 1)]; + if (prevBranch && id) dispatch(switchBranch({ sessionId: id, branchId: prevBranch })); + }, + onNext: () => { + const nextBranch = siblings[Math.min(siblings.length - 1, currentBranchIdx + 1)]; + if (nextBranch && id) dispatch(switchBranch({ sessionId: id, branchId: nextBranch })); + }, + } + : undefined + } /> )} - + ); })} {session.streamingMessage && ( @@ -566,6 +780,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose key={`streaming-${session.streamingMessage.id}`} isStreaming isPending + sessionId={session.id} call={{ id: session.streamingMessage.id, role: 'tool_call', @@ -590,9 +805,37 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose /> ) )} - {session.status === 'running' && !session.streamingMessage && ( + {(awaitingResponse || (session.status === 'running' && !session.streamingMessage)) && ( )} + {showResumeBubble && session.status === 'stopped' && ( + + + + + Resume Agent Response + + + + )} {showScrollButton && ( @@ -627,19 +870,262 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose )) )} - + {isGlowing ? ( + { e.stopPropagation(); onDismissGlow?.(); }} + sx={{ + mx: 1.5, + mb: 1.5, + py: 1.25, + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + borderRadius: 2.5, + cursor: 'pointer', + fontWeight: 600, + fontSize: '0.85rem', + color: c.accent.primary, + border: `1.5px solid ${c.accent.primary}`, + background: `${c.accent.primary}08`, + boxShadow: `0 0 12px ${c.accent.primary}25, inset 0 0 12px ${c.accent.primary}08`, + animation: 'continue-chat-glow 2s ease-in-out infinite', + transition: 'background 0.15s, box-shadow 0.15s', + '@keyframes continue-chat-glow': { + '0%, 100%': { + boxShadow: `0 0 12px ${c.accent.primary}25, inset 0 0 12px ${c.accent.primary}08`, + }, + '50%': { + boxShadow: `0 0 20px ${c.accent.primary}40, inset 0 0 20px ${c.accent.primary}15`, + }, + }, + '&:hover': { + background: `${c.accent.primary}14`, + boxShadow: `0 0 24px ${c.accent.primary}50, inset 0 0 20px ${c.accent.primary}18`, + }, + }} + > + Continue chat + + ) : ( + { if (queueExpanded) { setQueueExpanded(false); setEditingQueueIdx(null); } }}> + + {queueLength > 0 && ( + + { setQueueExpanded((v) => !v); setEditingQueueIdx(null); }} + sx={{ + display: 'inline-flex', + alignItems: 'center', + gap: 0.5, + px: 1.25, + py: 0.25, + borderRadius: '8px 8px 0 0', + bgcolor: c.bg.surface, + border: `1px solid ${c.border.subtle}`, + borderBottom: 'none', + cursor: 'pointer', + userSelect: 'none', + '&:hover': { bgcolor: c.bg.secondary }, + transition: 'background 0.12s', + }} + > + {queueExpanded + ? + : + } + + {queueLength} queued + + + { e.stopPropagation(); messageQueueRef.current = []; setQueueLength(0); setQueueExpanded(false); setEditingQueueIdx(null); }} + sx={{ p: 0.15, color: c.text.tertiary, '&:hover': { color: c.status.error } }} + > + + + + + + {queueExpanded && ( + + {messageQueueRef.current.map((msg, idx) => ( + { + setDragIdx(idx); + e.dataTransfer.effectAllowed = 'move'; + }} + onDragOver={(e) => { + e.preventDefault(); + e.dataTransfer.dropEffect = 'move'; + if (dragIdx !== null && dragIdx !== idx) setDropTargetIdx(idx); + }} + onDragLeave={() => { if (dropTargetIdx === idx) setDropTargetIdx(null); }} + onDrop={(e) => { + e.preventDefault(); + if (dragIdx !== null && dragIdx !== idx) { + const q = messageQueueRef.current; + const [item] = q.splice(dragIdx, 1); + q.splice(idx, 0, item); + setQueueLength(q.length); + } + setDragIdx(null); + setDropTargetIdx(null); + }} + onDragEnd={() => { setDragIdx(null); setDropTargetIdx(null); }} + sx={{ + display: 'flex', + alignItems: 'flex-start', + gap: 0.75, + px: 1.5, + py: 1, + borderBottom: idx < queueLength - 1 ? `1px solid ${c.border.subtle}` : 'none', + '&:hover': { bgcolor: c.bg.secondary }, + transition: 'background 0.1s, opacity 0.15s', + ...(dragIdx === idx ? { opacity: 0.35 } : {}), + ...(dropTargetIdx === idx && dragIdx !== null && dragIdx !== idx + ? { borderTop: `2px solid ${c.accent.primary}` } + : {}), + }} + > + + + + {editingQueueIdx === idx ? ( + + setEditingQueueText(e.target.value)} + autoFocus + onKeyDown={(e) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + const trimmed = editingQueueText.trim(); + if (trimmed) { + messageQueueRef.current[idx] = { ...messageQueueRef.current[idx], prompt: trimmed }; + setQueueLength(messageQueueRef.current.length); + } + setEditingQueueIdx(null); + } + if (e.key === 'Escape') setEditingQueueIdx(null); + }} + sx={{ + '& .MuiOutlinedInput-root': { + fontSize: '0.78rem', + color: c.text.primary, + '& fieldset': { borderColor: c.border.medium }, + '&.Mui-focused fieldset': { borderColor: c.accent.primary }, + }, + }} + /> + { + const trimmed = editingQueueText.trim(); + if (trimmed) { + messageQueueRef.current[idx] = { ...messageQueueRef.current[idx], prompt: trimmed }; + setQueueLength(messageQueueRef.current.length); + } + setEditingQueueIdx(null); + }} + sx={{ p: 0.25, color: c.accent.primary, mt: 0.25 }} + > + + + + ) : ( + + {msg.prompt} + + )} + {editingQueueIdx !== idx && ( + + + { setEditingQueueIdx(idx); setEditingQueueText(msg.prompt); }} + sx={{ p: 0.25, color: c.text.tertiary, '&:hover': { color: c.text.primary } }} + > + + + + + { + messageQueueRef.current.splice(idx, 1); + setQueueLength(messageQueueRef.current.length); + if (messageQueueRef.current.length === 0) setQueueExpanded(false); + }} + sx={{ p: 0.25, color: c.text.tertiary, '&:hover': { color: c.status.error } }} + > + + + + + )} + + ))} + + )} + + )} + + + + )} ); diff --git a/frontend/src/app/pages/AgentChat/BranchNavigator.tsx b/frontend/src/app/pages/AgentChat/BranchNavigator.tsx index 4dbabb95..e506f8c5 100644 --- a/frontend/src/app/pages/AgentChat/BranchNavigator.tsx +++ b/frontend/src/app/pages/AgentChat/BranchNavigator.tsx @@ -21,31 +21,38 @@ const BranchNavigator: React.FC = ({ currentIndex, totalBranches, onPrevi - - - - - {currentIndex + 1}/{totalBranches} - - - - + + + + + {currentIndex + 1} / {totalBranches} + + + + + ); }; diff --git a/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx b/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx new file mode 100644 index 00000000..010c1782 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx @@ -0,0 +1,403 @@ +import React, { useEffect, useRef, useMemo } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined'; +import CheckCircleOutlineIcon from '@mui/icons-material/CheckCircleOutline'; +import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; +import LanguageIcon from '@mui/icons-material/Language'; +import OpenInNewIcon from '@mui/icons-material/OpenInNew'; +import TouchAppOutlinedIcon from '@mui/icons-material/TouchAppOutlined'; +import KeyboardOutlinedIcon from '@mui/icons-material/KeyboardOutlined'; +import CameraAltOutlinedIcon from '@mui/icons-material/CameraAltOutlined'; +import ArticleOutlinedIcon from '@mui/icons-material/ArticleOutlined'; +import AccountTreeOutlinedIcon from '@mui/icons-material/AccountTreeOutlined'; +import CodeOutlinedIcon from '@mui/icons-material/CodeOutlined'; +import BuildOutlinedIcon from '@mui/icons-material/BuildOutlined'; +import { createSelector } from '@reduxjs/toolkit'; +import { useAppSelector, useAppDispatch } from '@/shared/hooks'; +import { AgentMessage, AgentSession, fetchBrowserAgentChildren } from '@/shared/state/agentsSlice'; +import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; +import type { RootState } from '@/shared/state/store'; + +interface Props { + parentSessionId: string; + browserId?: string; +} + +interface FeedEntry { + type: 'thought' | 'action' | 'result' | 'system'; + text: string; + actionTool?: string; + sessionLabel?: string; +} + +function formatMessage(msg: AgentMessage): FeedEntry | null { + if (msg.role === 'user') return null; + + if (msg.role === 'assistant' && typeof msg.content === 'string') { + const trimmed = msg.content.trim(); + if (!trimmed) return null; + return { type: 'thought', text: trimmed }; + } + + if (msg.role === 'tool_call') { + const content = + typeof msg.content === 'string' + ? (() => { try { return JSON.parse(msg.content); } catch { return {}; } })() + : msg.content; + const tool = content?.tool || content?.name || '?'; + const input = content?.input || {}; + let brief = ''; + switch (tool) { + case 'BrowserNavigate': + brief = `Navigate → ${input.url || '...'}`; + break; + case 'BrowserClick': + brief = `Click ${input.selector || '...'}`; + break; + case 'BrowserType': { + const txt = (input.text || '').slice(0, 40); + const ellipsis = (input.text || '').length > 40 ? '…' : ''; + brief = `Type "${txt}${ellipsis}" into ${input.selector || '...'}`; + break; + } + case 'BrowserScreenshot': + brief = 'Screenshot'; + break; + case 'BrowserGetText': + brief = 'Read page text'; + break; + case 'BrowserGetElements': + brief = `Inspect elements${input.selector ? ` (${input.selector})` : ''}`; + break; + case 'BrowserEvaluate': + brief = `Evaluate JS`; + break; + default: + brief = `${tool}(${JSON.stringify(input).slice(0, 60)})`; + } + return { type: 'action', text: brief, actionTool: tool }; + } + + if (msg.role === 'tool_result') { + const content = + typeof msg.content === 'string' + ? (() => { try { return JSON.parse(msg.content); } catch { return { text: msg.content }; } })() + : msg.content; + const toolName = content?.tool_name || ''; + const elapsed = content?.elapsed_ms; + const text = content?.text || ''; + + if (toolName === 'BrowserScreenshot') { + return { type: 'result', text: `Screenshot captured${elapsed ? ` (${elapsed}ms)` : ''}` }; + } + const preview = text.length > 120 ? text.slice(0, 120) + '…' : text; + return { type: 'result', text: `${preview}${elapsed ? ` (${elapsed}ms)` : ''}` }; + } + + if (msg.role === 'system') { + return { type: 'system', text: typeof msg.content === 'string' ? msg.content : '' }; + } + + return null; +} + +type SvgIconComponent = typeof OpenInNewIcon; + +function getActionIcon(tool?: string): SvgIconComponent { + switch (tool) { + case 'BrowserNavigate': return OpenInNewIcon; + case 'BrowserClick': return TouchAppOutlinedIcon; + case 'BrowserType': return KeyboardOutlinedIcon; + case 'BrowserScreenshot': return CameraAltOutlinedIcon; + case 'BrowserGetText': return ArticleOutlinedIcon; + case 'BrowserGetElements': return AccountTreeOutlinedIcon; + case 'BrowserEvaluate': return CodeOutlinedIcon; + default: return BuildOutlinedIcon; + } +} + +interface FeedColors { + thought: string; + thoughtIcon: string; + result: string; + error: string; + errorIcon: string; + scrollThumb: string; +} + +const darkFeedColors: FeedColors = { + thought: '#a0aab8', + thoughtIcon: '#555b6e', + result: '#555b6e', + error: '#ff8787', + errorIcon: '#ff8787', + scrollThumb: '#2a2d3e', +}; + +const lightFeedColors: FeedColors = { + thought: '#555550', + thoughtIcon: '#9e9c95', + result: '#9e9c95', + error: '#c03030', + errorIcon: '#c03030', + scrollThumb: '#ccc9c0', +}; + +const selectBrowserSessions = createSelector( + [(state: RootState) => state.agents.sessions, + (_: RootState, parentSessionId: string) => parentSessionId, + (_: RootState, __: string, browserId?: string) => browserId], + (sessions, parentSessionId, browserId) => + Object.values(sessions).filter( + (s): s is AgentSession => + s.mode === 'browser-agent' && + s.parent_session_id === parentSessionId && + (!browserId || s.browser_id === browserId), + ), +); + +const BrowserAgentInlineFeed: React.FC = ({ parentSessionId, browserId }) => { + const c = useClaudeTokens(); + const dispatch = useAppDispatch(); + const { mode } = useThemeMode(); + const fc = mode === 'dark' ? darkFeedColors : lightFeedColors; + const scrollRef = useRef(null); + const fetchedForSession = useRef(null); + + const browserSessions = useAppSelector((state) => + selectBrowserSessions(state, parentSessionId, browserId), + ); + + useEffect(() => { + if (browserSessions.length === 0 && fetchedForSession.current !== parentSessionId) { + fetchedForSession.current = parentSessionId; + dispatch(fetchBrowserAgentChildren(parentSessionId)) + .unwrap() + .catch(() => { fetchedForSession.current = null; }); + } + }, [browserSessions.length, parentSessionId, dispatch]); + + const sessionsWithEntries = useMemo(() => { + return browserSessions.map((session) => { + const entries: FeedEntry[] = []; + for (const msg of session.messages) { + const entry = formatMessage(msg); + if (entry) entries.push(entry); + } + if (session.streamingMessage?.role === 'assistant' && session.streamingMessage.content) { + entries.push({ type: 'thought', text: session.streamingMessage.content }); + } + return { session, entries }; + }); + }, [browserSessions]); + + const totalMessages = browserSessions.reduce( + (n, s) => n + s.messages.length + (s.streamingMessage ? 1 : 0), + 0, + ); + + useEffect(() => { + if (scrollRef.current) { + scrollRef.current.scrollTop = scrollRef.current.scrollHeight; + } + }, [totalMessages]); + + if (browserSessions.length === 0) return null; + + const showLabels = sessionsWithEntries.length > 1; + const accentColor = c.accent.primary; + + return ( + + {sessionsWithEntries.map(({ session, entries }, si) => ( + + {showLabels && ( + 0 ? 1 : 0, mb: 0.25 }}> + + + {session.browser_id || `Browser ${si + 1}`} + + + + )} + + {!showLabels && entries.length === 0 && session.status === 'running' && ( + + Starting browser agent... + + )} + + {entries.map((entry, i) => ( + + ))} + + {!showLabels && session.status === 'running' && entries.length > 0 && ( + + + + )} + + ))} + + ); +}; + +const EntryRow: React.FC<{ entry: FeedEntry; accentColor: string; fc: FeedColors }> = ({ entry, accentColor, fc }) => { + const c = useClaudeTokens(); + + if (entry.type === 'thought') { + return ( + + + + {entry.text} + + + ); + } + + if (entry.type === 'action') { + const ActionIcon = getActionIcon(entry.actionTool); + return ( + + + + {entry.text} + + + ); + } + + if (entry.type === 'result') { + return ( + + + ↳ {entry.text} + + + ); + } + + if (entry.type === 'system') { + return ( + + + + {entry.text} + + + ); + } + + return null; +}; + +const SessionStatusChip: React.FC<{ status: string }> = ({ status }) => { + const c = useClaudeTokens(); + if (status === 'running') { + return ( + + ); + } + if (status === 'completed') { + return ; + } + if (status === 'error') { + return ; + } + return null; +}; + +export default React.memo(BrowserAgentInlineFeed); diff --git a/frontend/src/app/pages/AgentChat/ChatInput.tsx b/frontend/src/app/pages/AgentChat/ChatInput.tsx index ff028b5d..d07e4cb1 100644 --- a/frontend/src/app/pages/AgentChat/ChatInput.tsx +++ b/frontend/src/app/pages/AgentChat/ChatInput.tsx @@ -26,6 +26,7 @@ import AttachFileIcon from '@mui/icons-material/AttachFile'; import AdsClickIcon from '@mui/icons-material/AdsClick'; import CommandPicker, { CommandPickerItem, getToolGroupIcon } from '@/app/components/CommandPicker'; import { useElementSelection, SelectedElement } from '@/app/components/ElementSelectionContext'; +import { getClipboardCards, clearClipboard } from '@/shared/dashboardClipboard'; import { getWebview } from '@/shared/browserRegistry'; import { API_BASE } from '@/shared/config'; import { ContextPath } from '@/app/components/DirectoryBrowser'; @@ -73,6 +74,7 @@ interface Props { embedded?: boolean; autoFocus?: boolean; sessionId?: string; + queueLength?: number; } export interface ChatInputHandle { @@ -130,7 +132,7 @@ const ContextRing: React.FC<{ used: number; limit: number; accentColor: string; ); }; -const ChatInput = forwardRef(({ onSend, disabled, mode, onModeChange, model, onModelChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus, sessionId }, ref) => { +const ChatInput = forwardRef(({ onSend, disabled, mode, onModeChange, model, onModelChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus, sessionId, queueLength = 0 }, ref) => { const c = useClaudeTokens(); const editorRef = useRef(null); const containerRef = useRef(null); @@ -138,6 +140,9 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, const dispatch = useAppDispatch(); const elementSelection = useElementSelection(); + const fallbackOwnerIdRef = useRef(`input-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`); + const ownerId = sessionId || fallbackOwnerIdRef.current; + useEffect(() => { if (autoFocus) editorRef.current?.focus(); }, [autoFocus]); @@ -281,7 +286,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, let trimmed = serialized.trim(); if (!trimmed) return; - const selectedEls = elementSelection?.selectedElements ?? []; + const selectedEls = elementSelection?.elementsByOwner?.[ownerId] ?? []; let allImages = images.length > 0 ? images.map(({ data, media_type }) => ({ data, media_type })) : []; @@ -298,7 +303,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, lines.push(`${i + 1}. [Browser Card] ${title}`); lines.push(` browser_id: ${el.semanticData.selectId}`); if (url) lines.push(` URL: ${url}`); - lines.push(` (Use BrowserAgent with this browser_id to interact with it)`); + lines.push(` (Use BrowserAgent with this browser_id to interact with it, or CreateBrowserAgent for a new browser)`); } else if (el.semanticType && el.semanticData) { const typeLabel = { 'agent-card': 'Agent Card', @@ -317,6 +322,9 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, .map(([k, v]) => `${k}: ${typeof v === 'string' ? v : JSON.stringify(v)}`) .join(', '); if (metaStr) lines.push(` ${metaStr}`); + if (el.semanticType === 'agent-card' && selectId) { + lines.push(` (Use InvokeAgent with session_id "${selectId}" to query this agent with full conversation context)`); + } } else { const styleStr = Object.entries(el.computedStyles) .map(([k, v]) => `${k}: ${v}`) @@ -359,8 +367,8 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, setForcedTools([]); setAttachedSkills({}); setHasContent(false); - elementSelection?.clearSelectedElements(); - }, [disabled, images, contextPaths, forcedTools, onSend, elementSelection]); + elementSelection?.clearOwnerElements(ownerId); + }, [disabled, images, contextPaths, forcedTools, onSend, elementSelection, ownerId]); const detectTrigger = useCallback(() => { const result = detectEditorTrigger(); @@ -469,6 +477,41 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, }; const handlePaste = useCallback((e: React.ClipboardEvent) => { + const copied = getClipboardCards(); + if (copied.length > 0 && elementSelection) { + e.preventDefault(); + for (const card of copied) { + const semanticTypeMap: Record = { + agent: 'agent-card', + view: 'view-card', + browser: 'browser-card', + }; + const semanticType = semanticTypeMap[card.type]; + if (!semanticType) continue; + const labelMap: Record = { + 'agent-card': 'Agent', + 'view-card': 'View', + 'browser-card': 'Browser', + }; + const semanticLabel = (labelMap[semanticType] || semanticType) + ': ' + card.name; + const el: SelectedElement = { + id: `sel-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + selectorPath: `[data-select-type="${semanticType}"][data-select-id="${card.id}"]`, + tagName: 'DIV', + className: '', + outerHTML: '', + computedStyles: {}, + boundingRect: { x: 0, y: 0, width: 0, height: 0 }, + semanticType, + semanticLabel, + semanticData: { ...card.meta, selectId: card.id }, + }; + elementSelection.addElementForOwner(ownerId, el); + } + clearClipboard(); + return; + } + const items = e.clipboardData?.items; if (!items) return; const imageFiles: File[] = []; @@ -486,7 +529,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, e.preventDefault(); const plain = e.clipboardData.getData('text/plain'); if (plain) document.execCommand('insertText', false, plain); - }, [addImageFiles]); + }, [addImageFiles, elementSelection, ownerId]); const handleDragOver = useCallback((e: React.DragEvent) => { e.preventDefault(); @@ -535,7 +578,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, }, }; - const selectedElements = elementSelection?.selectedElements ?? []; + const selectedElements = elementSelection?.elementsByOwner?.[ownerId] ?? []; const hasAttachments = images.length > 0 || contextPaths.length > 0 || forcedTools.length > 0 || selectedElements.length > 0; return ( @@ -783,7 +826,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, icon={} label={chipLabel} size="small" - onDelete={() => elementSelection?.removeSelectedElement(el.id)} + onDelete={() => elementSelection?.removeOwnerElement(ownerId, el.id)} sx={{ bgcolor: 'rgba(59, 130, 246, 0.1)', color: '#3b82f6', @@ -849,7 +892,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, userSelect: 'none', }} > - {disabled ? 'Agent is working...' : autoRunMode ? 'Describe what data to generate…' : `${modeConf.label}, @ for context, / for commands`} + {disabled ? 'Agent is working...' : autoRunMode ? 'Describe what data to generate…' : isRunning ? (queueLength > 0 ? `${queueLength} queued — type another or wait…` : 'Agent is working — messages will queue…') : `${modeConf.label}, @ for context, / for commands`} )} @@ -977,40 +1020,54 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, /> )} - {elementSelection && !autoRunMode && ( - - { - if (!elementSelection.selectMode && sessionId) { - elementSelection.setExcludeSelectId(sessionId); - } - elementSelection.toggleSelectMode(); - }} - sx={{ - p: 0.5, - ...(elementSelection.selectMode - ? { - bgcolor: '#3b82f6', - color: '#fff', - '&:hover': { bgcolor: '#2563eb' }, - animation: 'selectBtnPulse 2s ease-in-out infinite', - '@keyframes selectBtnPulse': { - '0%, 100%': { boxShadow: '0 0 0 0 rgba(59,130,246,0.4)' }, - '50%': { boxShadow: '0 0 0 4px rgba(59,130,246,0.1)' }, - }, + {elementSelection && !autoRunMode && (() => { + const isMySelectMode = elementSelection.selectMode && elementSelection.activeOwnerId === ownerId; + return ( + + e.preventDefault()} + onClick={() => { + if (isMySelectMode) { + elementSelection.setSelectMode(false); + } else { + if (elementSelection.activeOwnerId !== ownerId) { + elementSelection.clearOwnerElements(ownerId); } - : { - color: c.text.tertiary, - '&:hover': { color: c.text.secondary, bgcolor: 'rgba(0,0,0,0.04)' }, - }), - transition: 'background-color 0.15s, color 0.15s', - }} - > - - - - )} + elementSelection.setActiveOwnerId(ownerId); + if (sessionId) { + elementSelection.setExcludeSelectId(sessionId); + } else { + elementSelection.setExcludeSelectId(null); + } + elementSelection.setSelectMode(true); + } + }} + sx={{ + p: 0.5, + ...(isMySelectMode + ? { + bgcolor: '#3b82f6', + color: '#fff', + '&:hover': { bgcolor: '#2563eb' }, + animation: 'selectBtnPulse 2s ease-in-out infinite', + '@keyframes selectBtnPulse': { + '0%, 100%': { boxShadow: '0 0 0 0 rgba(59,130,246,0.4)' }, + '50%': { boxShadow: '0 0 0 4px rgba(59,130,246,0.1)' }, + }, + } + : { + color: c.text.tertiary, + '&:hover': { color: c.text.secondary, bgcolor: 'rgba(0,0,0,0.04)' }, + }), + transition: 'background-color 0.15s, color 0.15s', + }} + > + + + + ); + })()} (({ onSend, disabled, mode, - {!autoRunMode && (isRunning ? ( - - - - - - ) : hasContent ? ( - - - - - - ) : ( - - - - - - - - ))} + {!autoRunMode && ( + + {hasContent && ( + + + + + + )} + {isRunning ? ( + + + + + + ) : !hasContent ? ( + + + + + + + + ) : null} + + )} {selectedTemplate && ( diff --git a/frontend/src/app/pages/AgentChat/MessageActionBar.tsx b/frontend/src/app/pages/AgentChat/MessageActionBar.tsx new file mode 100644 index 00000000..5ff19921 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/MessageActionBar.tsx @@ -0,0 +1,153 @@ +import React, { useState } from 'react'; +import Box from '@mui/material/Box'; +import IconButton from '@mui/material/IconButton'; +import Typography from '@mui/material/Typography'; +import Tooltip from '@mui/material/Tooltip'; +import ContentCopyIcon from '@mui/icons-material/ContentCopy'; +import CheckIcon from '@mui/icons-material/Check'; +import EditIcon from '@mui/icons-material/Edit'; +import BookmarkBorderIcon from '@mui/icons-material/BookmarkBorder'; +import ReplayIcon from '@mui/icons-material/Replay'; +import CallSplitIcon from '@mui/icons-material/CallSplit'; +import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; +import ChevronRightIcon from '@mui/icons-material/ChevronRight'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; + +interface BranchNavProps { + currentIndex: number; + totalBranches: number; + onPrevious: () => void; + onNext: () => void; +} + +interface Props { + role: 'user' | 'assistant'; + onCopy: () => void; + onEdit?: () => void; + onRegenerate?: () => void; + onBranch?: () => void; + branchNav?: BranchNavProps; +} + +const btnSx = (c: ReturnType) => ({ + color: c.text.tertiary, + p: 0.4, + '&:hover': { color: c.text.secondary, bgcolor: 'transparent' }, + '&.Mui-disabled': { color: c.border.medium }, +}); + +const MessageActionBar: React.FC = ({ + role, + onCopy, + onEdit, + onRegenerate, + onBranch, + branchNav, +}) => { + const c = useClaudeTokens(); + const [copied, setCopied] = useState(false); + + const handleCopy = () => { + onCopy(); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + }; + + const isUser = role === 'user'; + + return ( + + {isUser ? ( + <> + + + + + + + + + + {copied ? : } + + + {onEdit && ( + + + + + + )} + {branchNav && branchNav.totalBranches > 1 && ( + + + + + + {branchNav.currentIndex + 1} / {branchNav.totalBranches} + + + + + + )} + + ) : ( + <> + + + {copied ? : } + + + {onRegenerate && ( + + + + + + )} + {onBranch && ( + + + + + + )} + + )} + + ); +}; + +export default MessageActionBar; diff --git a/frontend/src/app/pages/AgentChat/MessageBubble.tsx b/frontend/src/app/pages/AgentChat/MessageBubble.tsx index 5fbd998e..11cfb126 100644 --- a/frontend/src/app/pages/AgentChat/MessageBubble.tsx +++ b/frontend/src/app/pages/AgentChat/MessageBubble.tsx @@ -8,7 +8,6 @@ import Chip from '@mui/material/Chip'; import Tooltip from '@mui/material/Tooltip'; import Collapse from '@mui/material/Collapse'; import Modal from '@mui/material/Modal'; -import EditIcon from '@mui/icons-material/Edit'; import CloseIcon from '@mui/icons-material/Close'; import AdsClickIcon from '@mui/icons-material/AdsClick'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; @@ -391,13 +390,14 @@ const MessageImageThumbnails: React.FC<{ interface Props { message: AgentMessage; - onEdit?: (messageId: string, newContent: string) => void; + editing?: boolean; + onSaveEdit?: (messageId: string, newContent: string) => void; + onCancelEdit?: () => void; isStreaming?: boolean; } -const MessageBubble: React.FC = React.memo(({ message, onEdit, isStreaming }) => { +const MessageBubble: React.FC = React.memo(({ message, editing = false, onSaveEdit, onCancelEdit, isStreaming }) => { const c = useClaudeTokens(); - const [editing, setEditing] = useState(false); const [editText, setEditText] = useState(''); const { role, content } = message; @@ -440,23 +440,22 @@ const MessageBubble: React.FC = React.memo(({ message, onEdit, isStreamin ? parseElementContext(rawText) : { userMessage: rawText, elements: [] }; - const handleStartEdit = () => { - setEditText(rawText); - setEditing(true); - }; + React.useEffect(() => { + if (editing) setEditText(rawText); + }, [editing, rawText]); const handleCancelEdit = () => { - setEditing(false); setEditText(''); + onCancelEdit?.(); }; const handleSaveEdit = () => { const trimmed = editText.trim(); - if (trimmed && trimmed !== rawText && onEdit) { - onEdit(message.id, trimmed); + if (trimmed && trimmed !== rawText && onSaveEdit) { + onSaveEdit(message.id, trimmed); } - setEditing(false); setEditText(''); + onCancelEdit?.(); }; const truncatedContent = typeof content === 'string' @@ -472,26 +471,8 @@ const MessageBubble: React.FC = React.memo(({ message, onEdit, isStreamin display: 'flex', justifyContent: isUser ? 'flex-end' : 'flex-start', my: 0.75, - '&:hover .edit-btn': { opacity: 1 }, }} > - {isUser && onEdit && !editing && ( - - - - )} = React.memo(({ message, onEdit, isStreamin '& a': { color: c.accent.primary }, }} > - {rawText} + ( + {children} + ), + }} + >{rawText} {isStreaming && } )} diff --git a/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx b/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx index f058b69e..1b2bf7ef 100644 --- a/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx +++ b/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx @@ -1,8 +1,9 @@ -import React, { useState, useEffect, useCallback, useMemo } from 'react'; +import React, { useState, useEffect, useCallback, useMemo, useRef } from 'react'; import Box from '@mui/material/Box'; import Typography from '@mui/material/Typography'; import Collapse from '@mui/material/Collapse'; import IconButton from '@mui/material/IconButton'; +import Tooltip from '@mui/material/Tooltip'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import ExpandLessIcon from '@mui/icons-material/ExpandLess'; import TerminalIcon from '@mui/icons-material/Terminal'; @@ -15,10 +16,14 @@ import FolderIcon from '@mui/icons-material/Folder'; import AttachFileIcon from '@mui/icons-material/AttachFile'; import SearchIcon from '@mui/icons-material/Search'; import SendIcon from '@mui/icons-material/Send'; +import CallSplitIcon from '@mui/icons-material/CallSplit'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; -import { AgentMessage } from '@/shared/state/agentsSlice'; +import { AgentMessage, expandSession, collapseSession, fetchSession } from '@/shared/state/agentsSlice'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; +import { placeCard, removeCard, setGlowingAgentCard, clearGlowingAgentCard, DEFAULT_CARD_W, DEFAULT_CARD_H, EXPANDED_CARD_MIN_H, GRID_GAP } from '@/shared/state/dashboardLayoutSlice'; import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; +import BrowserAgentInlineFeed from './BrowserAgentInlineFeed'; const GoogleServiceIcon: React.FC<{ service: string; size?: number }> = ({ service, size = 14 }) => { if (service === 'gmail') { @@ -68,7 +73,13 @@ export interface ToolPair { result: AgentMessage | null; } -const pulsingKeyframes = ` +let toolCallKeyframesInjected = false; +function ensureToolCallKeyframes() { + if (toolCallKeyframesInjected) return; + toolCallKeyframesInjected = true; + const style = document.createElement('style'); + style.setAttribute('data-tool-call-keyframes', ''); + style.textContent = ` @keyframes tool-pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.4; } @@ -77,14 +88,13 @@ const pulsingKeyframes = ` 0%, 100% { box-shadow: 0 0 0 0 rgba(var(--glow-rgb), 0); } 50% { box-shadow: 0 0 10px 2px rgba(var(--glow-rgb), 0.12); } } -`; - -const streamingCursorKeyframes = ` @keyframes blink-cursor { 0%, 100% { opacity: 1; } 50% { opacity: 0; } } `; + document.head.appendChild(style); +} const ElapsedTimer: React.FC<{ startTime: string }> = ({ startTime }) => { const c = useClaudeTokens(); @@ -466,6 +476,7 @@ interface ToolCallBubbleProps { isPending?: boolean; isStreaming?: boolean; mcpCompact?: boolean; + sessionId?: string; } interface TermColors { @@ -929,7 +940,7 @@ const GmailCard: React.FC<{ data: Record; action: string; hideSubje }}> {children} }} + components={{ a: ({ children, ...props }) => {children} }} > {email.bodyPreview || email.snippet} @@ -1169,18 +1180,92 @@ const McpResultCard: React.FC<{ parsed: ParsedMcpResult; compact?: boolean }> = return ; }; +function isBrowserAgentTool(name: string): boolean { + if (name === 'CreateBrowserAgent' || name === 'BrowserAgent' || name === 'BrowserAgents') return true; + const mcp = parseMcpToolName(name); + return mcp.isMcp && mcp.serverSlug === 'openswarm-browser-agent'; +} + +function isInvokeAgentTool(name: string): boolean { + if (name === 'InvokeAgent') return true; + const mcp = parseMcpToolName(name); + return mcp.isMcp && mcp.serverSlug === 'openswarm-invoke-agent'; +} + +function isCreateAgentTool(name: string): boolean { + return name === 'Agent'; +} + +function parseInvokedSessionId(rawText: string): string | null { + const match = rawText.match(/\(forked session:\s*([a-f0-9]+)\)/); + return match ? match[1] : null; +} + +interface InvokeAgentParsed { + agentName: string; + sessionId: string | null; + cost: string | null; + response: string; +} + +function parseCreateAgentResult(rawText: string): string { + if (!rawText) return ''; + try { + const parsed = JSON.parse(rawText); + if (typeof parsed === 'string') return parsed; + if (typeof parsed === 'object' && parsed !== null) { + if (parsed.text) return parsed.text; + if (parsed.content) return typeof parsed.content === 'string' ? parsed.content : JSON.stringify(parsed.content); + if (parsed.result) return typeof parsed.result === 'string' ? parsed.result : JSON.stringify(parsed.result); + } + } catch {} + return rawText; +} + +function parseInvokeAgentResult(rawText: string): InvokeAgentParsed | null { + const headerMatch = rawText.match( + /\*\*Invoked Agent Result\*\*(?:\s*—\s*(.+?))?\s*\(forked session:\s*([a-f0-9]+)\)/, + ); + if (!headerMatch) return null; + + const agentName = headerMatch[1]?.trim() || 'Agent'; + const sessionId = headerMatch[2]; + + const costMatch = rawText.match(/\*Cost:\s*\$([0-9.]+)\*/); + const cost = costMatch ? costMatch[1] : null; + + const bodyStart = rawText.indexOf('\n\n'); + let response = bodyStart >= 0 ? rawText.slice(bodyStart + 2).trim() : ''; + if (response.startsWith('*Cost:')) { + const afterCost = response.indexOf('\n'); + response = afterCost >= 0 ? response.slice(afterCost + 1).trim() : ''; + } + + return { agentName, sessionId, cost, response }; +} + const ToolCallBubble: React.FC = React.memo( - ({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false }) => { + ({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false, sessionId }) => { + ensureToolCallKeyframes(); + const c = useClaudeTokens(); const tc = useTermColors(); + const dispatch = useAppDispatch(); + const cards = useAppSelector((s) => s.dashboardLayout.cards); const [expanded, setExpanded] = useState(false); + const bubbleRef = useRef(null); const { toolName, input, isDenied } = getToolData(call); const mcpInfo = useMemo(() => parseMcpToolName(toolName), [toolName]); const inputSummary = getInputSummary(toolName, input); const formattedInput = useMemo(() => formatInputDisplay(toolName, input), [toolName, input]); const showTimer = isPending && !isDenied && !isStreaming; - const showBody = expanded || isStreaming; + + const isBrowserAgent = isBrowserAgentTool(toolName); + const isInvokeAgent = isInvokeAgentTool(toolName); + const isCreateAgent = isCreateAgentTool(toolName); + const browserAgentAutoExpand = isBrowserAgent && isPending && !isStreaming; + const showBody = expanded || isStreaming || browserAgentAutoExpand; const resultContent = result?.content; const hasStructuredResult = @@ -1206,6 +1291,95 @@ const ToolCallBubble: React.FC = React.memo( (parsedResult?.type === 'bash' && parsedResult.exitCode !== null && parsedResult.exitCode !== 0) || (parsedResult?.type === 'text' && parsedResult.isError); + const invokedSessionId = useMemo( + () => (isInvokeAgent && result ? parseInvokedSessionId(resultRawText) : null), + [isInvokeAgent, result, resultRawText], + ); + + const invokeAgentParsed = useMemo( + () => (isInvokeAgent && result ? parseInvokeAgentResult(resultRawText) : null), + [isInvokeAgent, result, resultRawText], + ); + + const createAgentResponse = useMemo( + () => (isCreateAgent && result ? parseCreateAgentResult(resultRawText) : ''), + [isCreateAgent, result, resultRawText], + ); + + const createAgentSessionId: string | null = useMemo( + () => (isCreateAgent && hasStructuredResult && resultContent?.sub_session_id) ? resultContent.sub_session_id : null, + [isCreateAgent, hasStructuredResult, resultContent], + ); + + const revealTargetSessionId = invokedSessionId || createAgentSessionId; + + const sessions = useAppSelector((s) => s.agents.sessions); + + const handleRevealAgent = useCallback( + (e: React.MouseEvent) => { + e.stopPropagation(); + if (!revealTargetSessionId || !sessionId) return; + + if (cards[revealTargetSessionId]) { + dispatch(collapseSession(revealTargetSessionId)); + dispatch(removeCard(revealTargetSessionId)); + setTimeout(() => { + dispatch(clearGlowingAgentCard(revealTargetSessionId)); + }, 500); + return; + } + + let sourceYRatio: number | undefined; + if (bubbleRef.current) { + const bubbleEl = bubbleRef.current; + const cardEl = bubbleEl.closest('[data-select-type="agent-card"]') as HTMLElement | null; + if (cardEl) { + const cardRect = cardEl.getBoundingClientRect(); + const bubbleRect = bubbleEl.getBoundingClientRect(); + const bubbleCenterY = bubbleRect.top + bubbleRect.height / 2; + const ratio = (bubbleCenterY - cardRect.top) / cardRect.height; + sourceYRatio = Math.max(0, Math.min(1, ratio)); + } + } + + const doPlace = () => { + const parentCard = cards[sessionId]; + const targetX = parentCard + ? parentCard.x + parentCard.width + GRID_GAP * 12 + : 40; + let targetY = parentCard ? parentCard.y : 100; + if (parentCard) { + const columnCards = Object.values(cards).filter( + (c) => Math.abs(c.x - targetX) < 50 && c.session_id !== revealTargetSessionId, + ); + if (columnCards.length > 0) { + const lowestBottom = Math.max( + ...columnCards.map((c) => c.y + Math.max(EXPANDED_CARD_MIN_H, c.height)), + ); + targetY = lowestBottom + GRID_GAP; + } + } + dispatch(placeCard({ + sessionId: revealTargetSessionId, + x: targetX, + y: targetY, + width: DEFAULT_CARD_W, + height: DEFAULT_CARD_H, + })); + dispatch(expandSession(revealTargetSessionId)); + const label = isCreateAgent ? 'Create Agent' : isInvokeAgent ? 'Invoke Agent' : 'Agent'; + dispatch(setGlowingAgentCard({ sessionId: revealTargetSessionId, sourceId: sessionId, sourceYRatio, label })); + }; + + if (!sessions[revealTargetSessionId]) { + dispatch(fetchSession(revealTargetSessionId)).then(doPlace); + } else { + doPlace(); + } + }, + [revealTargetSessionId, sessionId, cards, sessions, dispatch], + ); + const toggle = useCallback(() => { if (!isStreaming) setExpanded((v) => !v); }, [isStreaming]); @@ -1233,10 +1407,415 @@ const ToolCallBubble: React.FC = React.memo( 'data-select-meta': JSON.stringify({ tool: toolName, inputSummary }), }; + if (isInvokeAgent) { + const agentName = invokeAgentParsed?.agentName || input?.session_id || 'Agent'; + const responsePreview = invokeAgentParsed?.response || ''; + const costLabel = invokeAgentParsed?.cost ? `$${invokeAgentParsed.cost}` : null; + const hasResponse = !!invokeAgentParsed; + + return ( + + + {/* Header */} + + + + InvokeAgent + + + + {agentName} + + + + {!hasResponse && !showTimer && } + + {hasResponse && responsePreview && !expanded && ( + + {responsePreview.slice(0, 100)}{responsePreview.length > 100 ? '…' : ''} + + )} + {expanded && } + + {isDenied && ( + + + denied + + )} + + {hasResponse && !isDenied && ( + + {isError ? ( + + ) : ( + + )} + {resultElapsedMs != null && ( + + {formatElapsed(resultElapsedMs)} + + )} + {costLabel && ( + + {costLabel} + + )} + + )} + + {showTimer && } + + {invokedSessionId && ( + + + + + + )} + + {hasResponse && ( + + {expanded ? : } + + )} + + + {/* Expanded body — markdown rendered, not terminal */} + + + ( + {children} + ), + }} + > + {responsePreview} + + + + + + ); + } + + if (isCreateAgent) { + const taskPrompt = input?.prompt || input?.task || input?.message || ''; + const taskLabel = taskPrompt + ? taskPrompt.length > 40 ? taskPrompt.slice(0, 40) + '…' : taskPrompt + : 'Sub-agent'; + const hasResponse = !!createAgentResponse; + + return ( + + + + + + CreateAgent + + + + {taskLabel} + + + + {!hasResponse && !showTimer && } + + {hasResponse && createAgentResponse && !expanded && ( + + {createAgentResponse.slice(0, 100)}{createAgentResponse.length > 100 ? '…' : ''} + + )} + {expanded && } + + {isDenied && ( + + + denied + + )} + + {hasResponse && !isDenied && ( + + {isError ? ( + + ) : ( + + )} + {resultElapsedMs != null && ( + + {formatElapsed(resultElapsedMs)} + + )} + + )} + + {showTimer && } + + {createAgentSessionId && ( + + + + + + )} + + {hasResponse && ( + + {expanded ? : } + + )} + + + + + ( + {children} + ), + }} + > + {createAgentResponse} + + + + + + ); + } + if (mcpCompact && mcpInfo.isMcp) { return ( - = React.memo( '&::-webkit-scrollbar-thumb': { background: tc.SCROLLBAR_THUMB, borderRadius: 3 }, }} > + {isBrowserAgent && sessionId && ( + + )} {parsedResult && parsedResult.type === 'mcp' ? ( ) : parsedResult ? ( @@ -1330,7 +1915,7 @@ const ToolCallBubble: React.FC = React.memo( {parsedResult.type === 'text' ? parsedResult.content : ''} ) : null} - {!parsedResult && isPending && !isStreaming && ( + {!parsedResult && isPending && !isStreaming && !isBrowserAgent && ( @@ -1343,8 +1928,6 @@ const ToolCallBubble: React.FC = React.memo( return ( - - {isStreaming && } = React.memo( )} + {/* Browser agent inline feed */} + {isBrowserAgent && sessionId && ( + + )} + {/* Output */} {parsedResult && parsedResult.type === 'mcp' ? ( @@ -1566,8 +2157,8 @@ const ToolCallBubble: React.FC = React.memo( ) : null} - {/* Pending indicator when waiting for result */} - {!parsedResult && isPending && !isStreaming && ( + {/* Pending indicator when waiting for result (skip for browser agent — feed replaces it) */} + {!parsedResult && isPending && !isStreaming && !isBrowserAgent && ( = React.memo(({ group, isSessionRunning = false, meta }) => { +const ToolGroupBubble: React.FC = React.memo(({ group, isSessionRunning = false, meta, sessionId }) => { const c = useClaudeTokens(); const isMcp = !!group.mcpServer; const [expanded, setExpanded] = useState(isMcp); @@ -186,6 +187,7 @@ const ToolGroupBubble: React.FC = React.memo(({ group, isSessionRunning = result={pair.result} isPending={pair.result === null && isSessionRunning} mcpCompact + sessionId={sessionId} /> ))} diff --git a/frontend/src/app/pages/Dashboard/AgentCard.tsx b/frontend/src/app/pages/Dashboard/AgentCard.tsx index 31c2e07f..751d09ea 100644 --- a/frontend/src/app/pages/Dashboard/AgentCard.tsx +++ b/frontend/src/app/pages/Dashboard/AgentCard.tsx @@ -22,12 +22,16 @@ import { import { setCardPosition, setCardSize, + fadeGlowingAgentCard, + clearGlowingAgentCard, + removeCard, } from '@/shared/state/dashboardLayoutSlice'; -import { useAppDispatch } from '@/shared/hooks'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { QuestionForm } from '@/app/pages/AgentChat/ApprovalBar'; import AgentChat from '@/app/pages/AgentChat/AgentChat'; import { parseMcpToolName, getMcpShortAction } from '@/app/pages/AgentChat/ToolCallBubble'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { useOverlayScrollPassthrough } from './useOverlayScrollPassthrough'; // --------------------------------------------------------------------------- // Helper components & functions (unchanged) @@ -168,7 +172,8 @@ interface Props { cardWidth: number; cardHeight: number; zoom?: number; - spawnFrom?: { x: number; y: number }; + spawnFrom?: { x: number; y: number; type?: 'branch' }; + exitTarget?: { x: number; y: number }; isSelected?: boolean; isHighlighted?: boolean; multiDragDelta?: { dx: number; dy: number } | null; @@ -176,6 +181,10 @@ interface Props { onDragStart?: (id: string, type: 'agent' | 'view') => void; onDragMove?: (dx: number, dy: number) => void; onDragEnd?: (dx: number, dy: number, didDrag: boolean) => void; + onBranch?: (sourceSessionId: string, newSessionId: string) => void; + onMeasuredHeight?: (sessionId: string, height: number) => void; + snapColumn?: { x: number; width: number }; + autoFocusInput?: boolean; } const MIN_W = 480; @@ -183,13 +192,54 @@ const MIN_H = 120; const EXPANDED_OVERLAY_H = 620; const SPAWN_SPRING = { type: 'spring' as const, stiffness: 400, damping: 28, mass: 0.6 }; +const BRANCH_SPRING = { type: 'spring' as const, stiffness: 300, damping: 26, mass: 0.8 }; +const EXIT_SPRING = { type: 'spring' as const, stiffness: 350, damping: 30, mass: 0.7 }; +const GLOW_FADE_MS = 2500; + +const SNAP_THRESHOLD = 60; const AgentCard: React.FC = ({ - session, expanded, cardX, cardY, cardWidth, cardHeight, zoom = 1, spawnFrom, + session, expanded, cardX, cardY, cardWidth, cardHeight, zoom = 1, spawnFrom, exitTarget, isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd, + onBranch, onMeasuredHeight, snapColumn, autoFocusInput, }) => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); + + const cardBoxRef = useRef(null); + useEffect(() => { + const el = cardBoxRef.current; + if (!el || !onMeasuredHeight) return; + const ro = new ResizeObserver((entries) => { + for (const entry of entries) { + onMeasuredHeight(session.id, entry.contentRect.height); + } + }); + ro.observe(el); + return () => ro.disconnect(); + }, [session.id, onMeasuredHeight]); + + // ---- Glow state (for branched cards) ---- + const glowEntry = useAppSelector((s) => s.dashboardLayout.glowingAgentCards[session.id]); + const isGlowingRedux = !!glowEntry; + const glowFading = glowEntry?.fading ?? false; + const glowFadeTimer = useRef | null>(null); + + const dismissGlow = useCallback(() => { + if (!isGlowingRedux || glowFading) return; + dispatch(fadeGlowingAgentCard(session.id)); + glowFadeTimer.current = setTimeout(() => { + dispatch(clearGlowingAgentCard(session.id)); + }, GLOW_FADE_MS + 300); + }, [isGlowingRedux, glowFading, dispatch, session.id]); + + useEffect(() => () => { + if (glowFadeTimer.current) clearTimeout(glowFadeTimer.current); + }, []); + + const accentColor = c.accent.primary; + const accentHover = c.accent.hover; const STATUS_COLORS: Record = { running: { color: c.status.success, bg: c.status.successBg }, @@ -212,6 +262,7 @@ const AgentCard: React.FC = ({ const justDraggedRef = useRef(false); const handleDragPointerDown = useCallback((e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY }; @@ -241,11 +292,15 @@ const AgentCard: React.FC = ({ const dx = (e.clientX - dragState.current.startX) / zoom; const dy = (e.clientY - dragState.current.startY) / zoom; if (didDrag.current) { - dispatch(setCardPosition({ - sessionId: session.id, - x: dragState.current.origX + dx, - y: dragState.current.origY + dy, - })); + let finalX = dragState.current.origX + dx; + const finalY = dragState.current.origY + dy; + + if (snapColumn && Math.abs(finalX - snapColumn.x) < SNAP_THRESHOLD) { + finalX = snapColumn.x; + dispatch(setCardSize({ sessionId: session.id, width: snapColumn.width, height: cardHeight })); + } + + dispatch(setCardPosition({ sessionId: session.id, x: finalX, y: finalY })); justDraggedRef.current = true; requestAnimationFrame(() => { justDraggedRef.current = false; }); } @@ -255,7 +310,7 @@ const AgentCard: React.FC = ({ setLocalDragPos(null); setIsDragging(false); (e.currentTarget as HTMLElement).releasePointerCapture(e.pointerId); - }, [zoom, dispatch, session.id, onDragEnd]); + }, [zoom, dispatch, session.id, onDragEnd, snapColumn, cardHeight]); // ---- Unified edge / corner resize ---- const resizeRef = useRef<{ @@ -272,6 +327,7 @@ const AgentCard: React.FC = ({ const handleResizeDown = useCallback( (dir: ResizeDir) => (e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); const effectiveW = Math.max(cardWidth, MIN_W); @@ -337,14 +393,17 @@ const AgentCard: React.FC = ({ const handleRemove = (e: React.MouseEvent) => { e.stopPropagation(); e.preventDefault(); - dispatch(closeSession({ sessionId: session.id })); + dispatch(collapseSession(session.id)); + dispatch(removeCard(session.id)); + if (glowEntry) { + setTimeout(() => { + dispatch(clearGlowingAgentCard(session.id)); + }, 500); + } else { + dispatch(closeSession({ sessionId: session.id })); + } }; - const handleCollapse = (e: React.MouseEvent) => { - e.stopPropagation(); - e.preventDefault(); - dispatch(collapseSession(session.id)); - }; useEffect(() => { if (session.status === 'running' || session.status === 'waiting_approval') { @@ -376,33 +435,53 @@ const AgentCard: React.FC = ({ const activeW = localResize?.w ?? cardWidth; const activeH = localResize?.h ?? cardHeight; + const isBranchSpawn = spawnFrom?.type === 'branch'; + const spawnInitial = spawnFrom + ? isBranchSpawn + ? { opacity: 0.5, scale: 0.92, left: spawnFrom.x, top: spawnFrom.y } + : { opacity: 0, scale: 0.3, left: spawnFrom.x, top: spawnFrom.y } + : false; + const spawnTransition = noTransition || !spawnFrom + ? { duration: 0 } + : isBranchSpawn + ? { left: BRANCH_SPRING, top: BRANCH_SPRING, scale: BRANCH_SPRING, opacity: { duration: 0.25 } } + : { left: SPAWN_SPRING, top: SPAWN_SPRING, scale: SPAWN_SPRING, opacity: { duration: 0.12 } }; + + const exitAnimation = exitTarget + ? { + opacity: 0, + scale: 0.3, + left: exitTarget.x, + top: exitTarget.y, + transition: { left: EXIT_SPRING, top: EXIT_SPRING, scale: EXIT_SPRING, opacity: { duration: 0.2 } }, + } + : { opacity: 0, scale: 0.85, transition: { duration: 0.2 } }; + return ( { if (justDraggedRef.current) return; + if (!isSelected && !e.shiftKey) { + dispatch(toggleExpandSession(session.id)); + } onCardSelect?.(session.id, 'agent', e.shiftKey); }} - onDoubleClick={() => dispatch(toggleExpandSession(session.id))} sx={{ position: 'relative', width: localResize ? activeW : Math.max(cardWidth, MIN_W), @@ -410,26 +489,34 @@ const AgentCard: React.FC = ({ bgcolor: c.bg.surface, border: isHighlighted ? `2px solid ${c.accent.primary}` - : isSelected - ? '2px solid #3b82f6' - : hasPending && !expanded - ? `1px solid ${c.status.warning}` - : expanded - ? `1px solid ${c.border.strong}` - : `1px solid ${c.border.subtle}`, + : (isGlowingRedux && !glowFading) + ? `2px solid ${accentColor}` + : isSelected + ? '2px solid #3b82f6' + : hasPending && !expanded + ? `1px solid ${c.status.warning}` + : expanded + ? `1px solid ${c.border.strong}` + : `1px solid ${c.border.subtle}`, borderRadius: 3, p: 2, cursor: expanded ? 'default' : 'pointer', - transition: noTransition ? 'none' : c.transition, + transition: noTransition + ? 'none' + : glowFading + ? `border ${GLOW_FADE_MS}ms ease-out, box-shadow ${GLOW_FADE_MS}ms ease-out` + : c.transition, boxShadow: isHighlighted ? `0 0 0 3px ${c.accent.primary}50, 0 0 20px ${c.accent.primary}35, 0 0 40px ${c.accent.primary}15` - : isDragging - ? c.shadow.lg - : isSelected - ? `0 0 0 1px #3b82f6, ${c.shadow.md}` - : expanded - ? c.shadow.md - : c.shadow.sm, + : (isGlowingRedux && !glowFading) + ? `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15` + : isDragging + ? c.shadow.lg + : isSelected + ? `0 0 0 1px #3b82f6, ${c.shadow.md}` + : expanded + ? c.shadow.md + : c.shadow.sm, display: 'flex', flexDirection: 'column', overflow: 'hidden', @@ -454,7 +541,18 @@ const AgentCard: React.FC = ({ }, zIndex: 50, }), - ...(!isHighlighted && !expanded && !isDragging && !isSelected && { + ...(!isHighlighted && isGlowingRedux && !glowFading && { + animation: 'agent-card-glow-pulse 2s ease-in-out infinite', + '@keyframes agent-card-glow-pulse': { + '0%, 100%': { + boxShadow: `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15`, + }, + '50%': { + boxShadow: `0 0 0 3px ${accentColor}60, 0 0 28px ${accentColor}45, 0 0 56px ${accentColor}25`, + }, + }, + }), + ...(!isHighlighted && !(isGlowingRedux && !glowFading) && !expanded && !isDragging && !isSelected && { '&:hover': { boxShadow: c.shadow.md, borderColor: hasPending ? c.status.warning : c.border.strong, @@ -462,6 +560,82 @@ const AgentCard: React.FC = ({ }), }} > + {/* Glow overlays for branched cards */} + {isGlowingRedux && ( + + {/* Rotating conic gradient border */} + + {/* Top edge shimmer */} + + {/* Inner shadow overlay */} + + + )} + {/* Resize handles: 4 edges + 4 corners */} {HANDLE_DEFS.map(({ dir, sx }) => ( = ({ /> ))} - {/* Selection overlay – blocks content interaction while selected, enabling drag from anywhere */} + {/* Selection overlay – blocks click interaction while selected, enabling drag from anywhere */} {isSelected && ( = ({ if (justDraggedRef.current) return; onCardSelect?.(session.id, 'agent', e.shiftKey); }} - onDoubleClick={() => dispatch(toggleExpandSession(session.id))} sx={{ position: 'absolute', inset: 0, @@ -571,37 +745,20 @@ const AgentCard: React.FC = ({ onPointerDown={(e) => e.stopPropagation()} sx={{ display: 'flex', alignItems: 'center', gap: 0.5, flexShrink: 0, ml: 0.5 }} > - {expanded ? ( - - e.stopPropagation()} - sx={{ - color: c.text.ghost, - p: 0.5, - '&:hover': { color: c.text.secondary, bgcolor: c.bg.secondary }, - }} - > - - - - ) : ( - - e.stopPropagation()} - sx={{ - color: c.text.ghost, - p: 0.5, - '&:hover': { color: c.status.error, bgcolor: `${c.status.errorBg}` }, - }} - > - - - - )} + + e.stopPropagation()} + sx={{ + color: c.text.ghost, + p: 0.5, + '&:hover': { color: c.status.error, bgcolor: `${c.status.errorBg}` }, + }} + > + + + @@ -649,6 +806,10 @@ const AgentCard: React.FC = ({ sessionId={session.id} onClose={() => dispatch(collapseSession(session.id))} embedded + autoFocus={autoFocusInput} + isGlowing={isGlowingRedux && !glowFading} + onDismissGlow={dismissGlow} + onBranch={onBranch ? (newId: string) => onBranch(session.id, newId) : undefined} /> )} diff --git a/frontend/src/app/pages/Dashboard/BrowserAgentOverlay.tsx b/frontend/src/app/pages/Dashboard/BrowserAgentOverlay.tsx index e7a02858..7e25af27 100644 --- a/frontend/src/app/pages/Dashboard/BrowserAgentOverlay.tsx +++ b/frontend/src/app/pages/Dashboard/BrowserAgentOverlay.tsx @@ -58,8 +58,10 @@ const BrowserAgentOverlay: React.FC = ({ session, browserWidth, browserHe const [expanded, setExpanded] = useState(false); const [confirmStop, setConfirmStop] = useState(false); const [fadeOut, setFadeOut] = useState(false); + const [hidden, setHidden] = useState(false); const confirmTimer = useRef | null>(null); const fadeTimer = useRef | null>(null); + const hideTimer = useRef | null>(null); const isRunning = session.status === 'running'; const isDone = session.status === 'completed' || session.status === 'error' || session.status === 'stopped'; @@ -71,6 +73,13 @@ const BrowserAgentOverlay: React.FC = ({ session, browserWidth, browserHe return () => { if (fadeTimer.current) clearTimeout(fadeTimer.current); }; }, [isDone]); + useEffect(() => { + if (fadeOut) { + hideTimer.current = setTimeout(() => setHidden(true), 400); + } + return () => { if (hideTimer.current) clearTimeout(hideTimer.current); }; + }, [fadeOut]); + useEffect(() => { if (scrollRef.current) { scrollRef.current.scrollTop = scrollRef.current.scrollHeight; @@ -111,7 +120,7 @@ const BrowserAgentOverlay: React.FC = ({ session, browserWidth, browserHe const panelW = expanded ? expandedW : collapsedW; const panelH = expanded ? expandedH : collapsedH; - if (fadeOut) return null; + if (hidden) return null; return ( = ({ session, browserWidth, browserHe flexDirection: 'column', overflow: 'hidden', transition: 'width 0.25s ease, height 0.25s ease, opacity 0.4s ease', - opacity: isDone && !fadeOut ? 0.7 : 1, + opacity: fadeOut ? 0 : isDone ? 0.7 : 1, animation: 'overlay-enter 0.3s ease-out', '@keyframes overlay-enter': { '0%': { opacity: 0, transform: 'translateY(8px) scale(0.95)' }, diff --git a/frontend/src/app/pages/Dashboard/BrowserCard.tsx b/frontend/src/app/pages/Dashboard/BrowserCard.tsx index d7772294..9ed10122 100644 --- a/frontend/src/app/pages/Dashboard/BrowserCard.tsx +++ b/frontend/src/app/pages/Dashboard/BrowserCard.tsx @@ -40,6 +40,7 @@ import { useBrowserActivity } from '@/shared/useBrowserActivity'; import { getActionLabel } from '@/shared/browserCommandHandler'; import { resolveInput, isGoogleSearch } from '@/shared/resolveUrl'; import BrowserAgentOverlay from './BrowserAgentOverlay'; +import { useOverlayScrollPassthrough } from './useOverlayScrollPassthrough'; import { useElementSelection } from '@/app/components/ElementSelectionContext'; type ResizeDir = 'n' | 's' | 'e' | 'w' | 'ne' | 'nw' | 'se' | 'sw'; @@ -67,6 +68,14 @@ const HANDLE_DEFS: { dir: ResizeDir; sx: Record }[] = [ const isElectron = navigator.userAgent.includes('Electron'); +const chromeUserAgent = navigator.userAgent + .replace(/\s*Electron\/\S+/, '') + .replace(/\s*OpenSwarm\/\S+/, ''); + +const webviewPreloadPath: string | undefined = isElectron + ? (window as any).openswarm?.getWebviewPreloadPath?.() + : undefined; + type WebviewElement = BrowserWebview; interface TabLocalState { @@ -84,6 +93,7 @@ interface Props { cardWidth: number; cardHeight: number; zoom?: number; + cmdHeld?: boolean; isSelected?: boolean; isHighlighted?: boolean; multiDragDelta?: { dx: number; dy: number } | null; @@ -95,11 +105,12 @@ interface Props { const BrowserCard: React.FC = ({ - browserId, tabs, activeTabId, cardX, cardY, cardWidth, cardHeight, zoom = 1, + browserId, tabs, activeTabId, cardX, cardY, cardWidth, cardHeight, zoom = 1, cmdHeld = false, isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd, }) => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage); const elementSelectionCtx = useElementSelection(); const isElementSelectMode = elementSelectionCtx?.selectMode ?? false; @@ -192,10 +203,6 @@ const BrowserCard: React.FC = ({ onTitleUpdate(); }; - const onNewWindow = (e: any) => { - if (e.url) dispatch(addBrowserTab({ browserId, url: e.url, makeActive: true })); - }; - const onFaviconUpdate = (e: any) => { const favicons = e.favicons || (e.detail && e.detail.favicons); if (favicons?.[0]) { @@ -208,7 +215,6 @@ const BrowserCard: React.FC = ({ wv.addEventListener('page-title-updated', onTitleUpdate); wv.addEventListener('did-start-loading', onLoadStart); wv.addEventListener('did-stop-loading', onLoadStop); - wv.addEventListener('new-window', onNewWindow); wv.addEventListener('page-favicon-updated', onFaviconUpdate); cleanups.push(() => { @@ -218,7 +224,6 @@ const BrowserCard: React.FC = ({ wv.removeEventListener('page-title-updated', onTitleUpdate); wv.removeEventListener('did-start-loading', onLoadStart); wv.removeEventListener('did-stop-loading', onLoadStop); - wv.removeEventListener('new-window', onNewWindow); wv.removeEventListener('page-favicon-updated', onFaviconUpdate); }); } @@ -371,6 +376,7 @@ const BrowserCard: React.FC = ({ const justDraggedRef = useRef(false); const handleDragPointerDown = useCallback((e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY }; @@ -426,6 +432,7 @@ const BrowserCard: React.FC = ({ const handleResizeDown = useCallback( (dir: ResizeDir) => (e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); resizeRef.current = { @@ -491,20 +498,13 @@ const BrowserCard: React.FC = ({ const accentColor = c.accent.primary; const accentHover = c.accent.hover; + const accentRgb = accentColor.replace('#', '').match(/.{2}/g)?.map(h => parseInt(h, 16)).join(',') || '189,100,57'; // ---- Glow state ---- const glowingBrowserCards = useAppSelector((s) => s.dashboardLayout.glowingBrowserCards); const isGlowingFromRedux = !!glowingBrowserCards[browserId]; - const [hasBeenTouched, setHasBeenTouched] = useState(false); - useEffect(() => { - if (isGlowingFromRedux && agentActive) setHasBeenTouched(true); - }, [isGlowingFromRedux, agentActive]); - useEffect(() => { - if (!isGlowingFromRedux) setHasBeenTouched(false); - }, [isGlowingFromRedux]); - - const showGlow = isGlowingFromRedux && hasBeenTouched; + const showGlow = isGlowingFromRedux; const agentBorder = isHighlighted ? `2px solid ${c.accent.primary}` @@ -576,8 +576,8 @@ const BrowserCard: React.FC = ({ }, }), ...(!isHighlighted && (agentActive || showGlow) && { - animation: 'agent-glow-pulse 2s ease-in-out infinite', - '@keyframes agent-glow-pulse': { + animation: `agent-glow-${browserId} 2s ease-in-out infinite`, + [`@keyframes agent-glow-${browserId}`]: { '0%, 100%': { boxShadow: `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15${innerGlow}`, }, @@ -588,9 +588,10 @@ const BrowserCard: React.FC = ({ }), }} > - {/* Selection overlay – blocks content interaction while selected, enabling drag from anywhere */} + {/* Selection overlay – blocks click interaction while selected, enabling drag from anywhere */} {isSelected && ( = ({ {isElementSelectMode && ( )} + {cmdHeld && !isSelected && ( + + )} {isElectron ? ( tabs.map((tab) => ( = ({ data-tab-id={tab.id} src="about:blank" allowpopups="true" + useragent={chromeUserAgent} + {...(webviewPreloadPath ? { preload: webviewPreloadPath } : {})} + webpreferences="plugins=yes, autoplayPolicy=no-user-gesture-required" style={{ position: 'absolute', top: 0, @@ -1035,6 +1042,7 @@ const BrowserCard: React.FC = ({ {/* Camera flash — screenshot */} {(agentAction === 'screenshot' || lastAction === 'screenshot') && ( = ({ pointerEvents: 'none', background: `linear-gradient(180deg, transparent, ${accentColor}90, transparent)`, boxShadow: `0 0 12px ${accentColor}60`, - animation: 'scan-sweep 1.5s ease-in-out infinite', + animation: 'scan-sweep 1.5s ease-in-out infinite alternate', '@keyframes scan-sweep': { '0%': { top: '0%' }, - '100%': { top: '100%' }, + '100%': { top: 'calc(100% - 3px)' }, }, }} /> @@ -1074,10 +1082,11 @@ const BrowserCard: React.FC = ({ {/* Click ripple */} {(agentAction === 'click' || lastAction === 'click') && ( = ({ )} - {/* Orange inner shadow overlay for selection / streaming glow */} + {/* Accent inner shadow overlay for selection / streaming glow */} {showGlow && !agentActive && ( = ({ zIndex: 14, pointerEvents: 'none', borderRadius: 'inherit', - boxShadow: 'inset 0 0 40px rgba(255,140,0,0.35), inset 0 0 80px rgba(255,100,0,0.15)', - animation: 'orange-glow-pulse 2s ease-in-out infinite', - '@keyframes orange-glow-pulse': { + boxShadow: `inset 0 0 40px rgba(${accentRgb},0.35), inset 0 0 80px rgba(${accentRgb},0.15)`, + animation: `accent-glow-${browserId} 2s ease-in-out infinite`, + [`@keyframes accent-glow-${browserId}`]: { '0%, 100%': { - boxShadow: 'inset 0 0 40px rgba(255,140,0,0.35), inset 0 0 80px rgba(255,100,0,0.15)', + boxShadow: `inset 0 0 40px rgba(${accentRgb},0.35), inset 0 0 80px rgba(${accentRgb},0.15)`, }, '50%': { - boxShadow: 'inset 0 0 50px rgba(255,140,0,0.45), inset 0 0 100px rgba(255,100,0,0.22)', + boxShadow: `inset 0 0 50px rgba(${accentRgb},0.45), inset 0 0 100px rgba(${accentRgb},0.22)`, }, }, }} diff --git a/frontend/src/app/pages/Dashboard/Dashboard.tsx b/frontend/src/app/pages/Dashboard/Dashboard.tsx index 2aa9393b..064e12b4 100644 --- a/frontend/src/app/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/app/pages/Dashboard/Dashboard.tsx @@ -1,4 +1,5 @@ -import React, { useEffect, useCallback, useRef, useState } from 'react'; +import React, { useEffect, useCallback, useRef, useState, useMemo } from 'react'; +import { AnimatePresence, motion } from 'framer-motion'; import { useParams } from 'react-router-dom'; import Box from '@mui/material/Box'; import Typography from '@mui/material/Typography'; @@ -9,10 +10,14 @@ import { fetchSessions, fetchHistory, collapseSession, + closeSession, + duplicateSession, + expandSession, launchAndSendFirstMessage, generateTitle, resumeSession, setExpandedSessionIds, + toggleExpandSession, } from '@/shared/state/agentsSlice'; import type { AgentConfig } from '@/shared/state/agentsSlice'; import { @@ -25,12 +30,23 @@ import { moveCards, resetLayout, setGlowingBrowserCards, + removeViewCard, + removeBrowserCard, + pasteBrowserCard, + placeCard, + removeCard, + setGlowingAgentCard, + clearGlowingAgentCard, + DEFAULT_CARD_W, + DEFAULT_CARD_H, EXPANDED_CARD_MIN_H, + GRID_GAP, } from '@/shared/state/dashboardLayoutSlice'; import { fetchOutputs } from '@/shared/state/outputsSlice'; import { generateDashboardName, updateDashboardThumbnail } from '@/shared/state/dashboardsSlice'; import { dashboardWs } from '@/shared/ws/WebSocketManager'; import { initBrowserCommandHandler } from '@/shared/browserCommandHandler'; +import { clearPendingBrowserUrl, clearPendingFocusAgentId } from '@/shared/state/tempStateSlice'; import AgentCard from './AgentCard'; import DashboardViewCard from './DashboardViewCard'; import BrowserCard from './BrowserCard'; @@ -45,6 +61,7 @@ import type { ContextPath } from '@/app/components/DirectoryBrowser'; import { ElementSelectionProvider, useElementSelection } from '@/app/components/ElementSelectionContext'; import { useDomElementSelector } from '@/app/components/useDomElementSelector'; import SelectionOverlay from '@/app/components/SelectionOverlay'; +import { setClipboardCards, getClipboardCards, type ClipboardCard } from '@/shared/dashboardClipboard'; const SELECT_ATTR = 'data-select-type'; @@ -81,7 +98,10 @@ const DashboardInner: React.FC = () => { const zoomSensitivity = useAppSelector((state) => state.settings.data.zoom_sensitivity); const newAgentShortcut = useAppSelector((state) => state.settings.data.new_agent_shortcut); const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage); + const expandNewChats = useAppSelector((state) => state.settings.data.expand_new_chats_in_dashboard); + const autoRevealSubAgents = useAppSelector((state) => state.settings.data.auto_reveal_sub_agents); const outputs = useAppSelector((state) => state.outputs.items); + const glowingAgentCards = useAppSelector((state) => state.dashboardLayout.glowingAgentCards); const sessionList = Object.values(sessions); const canvas = useCanvasControls(zoomSensitivity); @@ -96,6 +116,8 @@ const DashboardInner: React.FC = () => { const [toolbarOpen, setToolbarOpen] = useState(false); const [highlightedCardId, setHighlightedCardId] = useState(null); const highlightTimerRef = useRef | null>(null); + const [autoFocusSessionId, setAutoFocusSessionId] = useState(null); + const [pendingSelectSessionId, setPendingSelectSessionId] = useState(null); const handleHighlightCard = useCallback((cardId: string) => { if (highlightTimerRef.current) clearTimeout(highlightTimerRef.current); @@ -106,7 +128,35 @@ const DashboardInner: React.FC = () => { }, 2000); }, []); - const spawnOriginsRef = useRef>({}); + useEffect(() => { + if (autoFocusSessionId) { + const timer = setTimeout(() => setAutoFocusSessionId(null), 1500); + return () => clearTimeout(timer); + } + }, [autoFocusSessionId]); + + useEffect(() => { + if (!pendingSelectSessionId) return; + if (!cards[pendingSelectSessionId]) return; + setPendingSelectSessionId(null); + selection.selectCard(pendingSelectSessionId, 'agent', false); + }, [pendingSelectSessionId, cards, selection]); + + const spawnOriginsRef = useRef>({}); + const measuredHeightsRef = useRef>({}); + const [measuredHeightsTick, setMeasuredHeightsTick] = useState(0); + const handleMeasuredHeight = useCallback((sessionId: string, height: number) => { + if (measuredHeightsRef.current[sessionId] !== height) { + measuredHeightsRef.current[sessionId] = height; + setMeasuredHeightsTick((t) => t + 1); + } + }, []); + const revealSpawnedRef = useRef(new Set()); + useEffect(() => { + revealSpawnedRef.current.forEach((id) => { + if (!cards[id]) revealSpawnedRef.current.delete(id); + }); + }, [cards]); const hasFittedRef = useRef(false); const restoredExpandedRef = useRef(false); const canvasStateRef = useRef({ panX: canvas.panX, panY: canvas.panY, zoom: canvas.zoom }); @@ -114,16 +164,16 @@ const DashboardInner: React.FC = () => { // ---- Multi-drag coordination ---- const [multiDragDelta, setMultiDragDelta] = useState<{ dx: number; dy: number } | null>(null); + const [liveDragInfo, setLiveDragInfo] = useState<{ cardId: string; dx: number; dy: number } | null>(null); const activeDragCardRef = useRef(null); const isMultiDragRef = useRef(false); const handleCardDragStart = useCallback((id: string, _type: CardType) => { + activeDragCardRef.current = id; if (selection.isSelected(id)) { - activeDragCardRef.current = id; isMultiDragRef.current = true; } else { selection.deselectAll(); - activeDragCardRef.current = null; isMultiDragRef.current = false; } }, [selection]); @@ -132,6 +182,9 @@ const DashboardInner: React.FC = () => { if (isMultiDragRef.current) { setMultiDragDelta({ dx, dy }); } + if (activeDragCardRef.current) { + setLiveDragInfo({ cardId: activeDragCardRef.current, dx, dy }); + } }, []); const handleCardDragEnd = useCallback((dx: number, dy: number, didDrag: boolean) => { @@ -145,6 +198,7 @@ const DashboardInner: React.FC = () => { activeDragCardRef.current = null; isMultiDragRef.current = false; setMultiDragDelta(null); + setLiveDragInfo(null); }, [selection, dispatch]); const handleCardSelect = useCallback((id: string, type: CardType, shiftKey: boolean) => { @@ -157,11 +211,17 @@ const DashboardInner: React.FC = () => { canvas.handlers.onMouseDown(e); return; } + + if (e.button === 2) { + e.preventDefault(); + canvas.handlers.onMouseDown(e); + return; + } + if (e.button !== 0) return; if (isCardTarget(e.target, e.currentTarget)) return; if (isElementSelectMode) { - // Cmd/Ctrl held → allow panning even in element select mode if (e.metaKey || e.ctrlKey) { canvas.handlers.onMouseDown(e); } @@ -169,10 +229,10 @@ const DashboardInner: React.FC = () => { } if (e.metaKey || e.ctrlKey || canvas.spaceHeld) { - selection.handleCanvasMouseDown(e.nativeEvent); - } else { selection.deselectAll(); canvas.handlers.onMouseDown(e); + } else { + selection.handleCanvasMouseDown(e.nativeEvent); } }, [canvas.handlers, canvas.spaceHeld, selection, isElementSelectMode]); @@ -200,6 +260,20 @@ const DashboardInner: React.FC = () => { return () => { cleanupBrowserHandler(); dashboardWs.disconnect(); }; }, [dispatch, dashboardId]); + const pendingBrowserUrl = useAppSelector((state) => state.tempState.pendingBrowserUrl); + const pendingFocusAgentId = useAppSelector((state) => state.tempState.pendingFocusAgentId); + + useEffect(() => { + if (!dashboardId) return; + (window as any).__openswarm_last_dashboard_id = dashboardId; + }, [dashboardId]); + + useEffect(() => { + if (!pendingBrowserUrl || !layoutInitialized) return; + dispatch(addBrowserCard({ url: pendingBrowserUrl, expandedSessionIds })); + dispatch(clearPendingBrowserUrl()); + }, [pendingBrowserUrl, layoutInitialized, dispatch, expandedSessionIds]); + // Capture a thumbnail screenshot of the dashboard. // Uses Electron's native capturePage for pixel-perfect results. // Captures current viewport as-is (no DOM mutation) to avoid visual flashes. @@ -247,10 +321,25 @@ const DashboardInner: React.FC = () => { useEffect(() => { if (!layoutInitialized || hasFittedRef.current) return; + if (pendingFocusAgentId) return; hasFittedRef.current = true; const timer = setTimeout(() => canvas.actions.fitToView(), 150); return () => clearTimeout(timer); - }, [layoutInitialized, canvas.actions]); + }, [layoutInitialized, canvas.actions, pendingFocusAgentId]); + + useEffect(() => { + if (!pendingFocusAgentId || !layoutInitialized) return; + const agentId = pendingFocusAgentId; + dispatch(clearPendingFocusAgentId()); + hasFittedRef.current = true; + setTimeout(() => { + const card = store.getState().dashboardLayout.cards[agentId]; + if (card) { + canvas.actions.fitToCards([{ x: card.x, y: card.y, width: card.width, height: card.height }], 1.0, true); + handleHighlightCard(agentId); + } + }, 350); + }, [pendingFocusAgentId, layoutInitialized, dispatch, canvas.actions, handleHighlightCard]); useEffect(() => { if (!layoutInitialized || restoredExpandedRef.current) return; @@ -263,7 +352,7 @@ const DashboardInner: React.FC = () => { useEffect(() => { if (!layoutInitialized) return; const dashboardSessionIds = Object.values(sessions) - .filter((s) => s.dashboard_id === dashboardId && s.mode !== 'browser-agent') + .filter((s) => s.dashboard_id === dashboardId && s.mode !== 'browser-agent' && s.mode !== 'invoked-agent' && s.mode !== 'sub-agent') .map((s) => s.id); const liveIds = dashboardSessionIds.sort().join(','); if (liveIds === prevSessionIdsRef.current) return; @@ -271,6 +360,100 @@ const DashboardInner: React.FC = () => { dispatch(reconcileSessions({ sessionIds: dashboardSessionIds, expandedSessionIds })); }, [sessions, layoutInitialized, dispatch, dashboardId, expandedSessionIds]); + // ---- Auto-reveal / collapse / unreveal sub-agent cards ---- + const autoRevealedRef = useRef(new Set()); + const prevSubStatusRef = useRef>({}); + const prevParentStatusRef = useRef>({}); + + useEffect(() => { + if (!layoutInitialized || !autoRevealSubAgents) return; + + const subSessions = Object.values(sessions).filter( + (s) => (s.mode === 'sub-agent' || s.mode === 'invoked-agent') && s.parent_session_id, + ); + + // 1) Auto-reveal newly spawned sub-agents (skip already-terminal ones on load) + for (const sub of subSessions) { + if (autoRevealedRef.current.has(sub.id)) continue; + if (cards[sub.id]) { + autoRevealedRef.current.add(sub.id); + continue; + } + const parentCard = cards[sub.parent_session_id!]; + if (!parentCard) continue; + + const isTerminal = sub.status === 'completed' || sub.status === 'error' || sub.status === 'stopped'; + const parentSession = sessions[sub.parent_session_id!]; + const parentTerminal = parentSession && + (parentSession.status === 'completed' || parentSession.status === 'error' || parentSession.status === 'stopped'); + if (isTerminal && parentTerminal) { + autoRevealedRef.current.add(sub.id); + continue; + } + + autoRevealedRef.current.add(sub.id); + + const targetX = parentCard.x + parentCard.width + GRID_GAP * 12; + let targetY = parentCard.y; + const columnCards = Object.values(cards).filter( + (c) => Math.abs(c.x - targetX) < 50 && c.session_id !== sub.id, + ); + if (columnCards.length > 0) { + const lowestBottom = Math.max( + ...columnCards.map((c) => c.y + Math.max(EXPANDED_CARD_MIN_H, c.height)), + ); + targetY = lowestBottom + GRID_GAP; + } + + dispatch(placeCard({ sessionId: sub.id, x: targetX, y: targetY, width: DEFAULT_CARD_W, height: DEFAULT_CARD_H })); + dispatch(expandSession(sub.id)); + const label = sub.mode === 'sub-agent' ? 'Create Agent' : 'Invoke Agent'; + dispatch(setGlowingAgentCard({ sessionId: sub.id, sourceId: sub.parent_session_id!, label })); + + if (sub.status === 'completed' || sub.status === 'error' || sub.status === 'stopped') { + const subId = sub.id; + setTimeout(() => dispatch(collapseSession(subId)), 2000); + } + } + + // 2) Auto-collapse sub-agents when they complete + const TERMINAL = new Set(['completed', 'error', 'stopped']); + for (const sub of subSessions) { + const prev = prevSubStatusRef.current[sub.id]; + if (prev !== sub.status && TERMINAL.has(sub.status) && cards[sub.id]) { + dispatch(collapseSession(sub.id)); + } + } + const newSubStatuses: Record = {}; + for (const sub of subSessions) { newSubStatuses[sub.id] = sub.status; } + prevSubStatusRef.current = newSubStatuses; + + // 3) Unreveal all sub-agent cards when parent finishes output + const parentIds = new Set(subSessions.map((s) => s.parent_session_id!)); + for (const pid of parentIds) { + const parent = sessions[pid]; + if (!parent) continue; + const prev = prevParentStatusRef.current[pid]; + if (prev !== parent.status && TERMINAL.has(parent.status)) { + const children = subSessions.filter((s) => s.parent_session_id === pid); + for (const child of children) { + if (!cards[child.id]) continue; + dispatch(collapseSession(child.id)); + dispatch(removeCard(child.id)); + setTimeout(() => { + dispatch(clearGlowingAgentCard(child.id)); + }, 500); + } + } + } + const newParentStatuses: Record = {}; + for (const pid of parentIds) { + const parent = sessions[pid]; + if (parent) newParentStatuses[pid] = parent.status; + } + prevParentStatusRef.current = newParentStatuses; + }, [sessions, cards, layoutInitialized, autoRevealSubAgents, dispatch]); + const skipInitialSave = useRef(true); const saveTimerRef = useRef | null>(null); const pendingSaveRef = useRef[0] | null>(null); @@ -326,6 +509,187 @@ const DashboardInner: React.FC = () => { return () => window.removeEventListener('keydown', handleShortcut); }, [newAgentShortcut]); + useEffect(() => { + const handleEnter = (e: KeyboardEvent) => { + if (e.key !== 'Enter') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + if (selection.selectedIds.size !== 1) return; + const [id, type] = selection.selectedIds.entries().next().value!; + if (type !== 'agent') return; + e.preventDefault(); + dispatch(toggleExpandSession(id)); + }; + window.addEventListener('keydown', handleEnter); + return () => window.removeEventListener('keydown', handleEnter); + }, [selection.selectedIds, dispatch]); + + useEffect(() => { + const handleDelete = (e: KeyboardEvent) => { + if (e.key !== 'Backspace' && e.key !== 'Delete') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + if (selection.selectedIds.size === 0) return; + e.preventDefault(); + for (const [id, type] of selection.selectedIds) { + if (type === 'agent') { + dispatch(closeSession({ sessionId: id })); + } else if (type === 'view') { + dispatch(removeViewCard(id)); + } else if (type === 'browser') { + dispatch(removeBrowserCard(id)); + } + } + selection.deselectAll(); + }; + window.addEventListener('keydown', handleDelete); + return () => window.removeEventListener('keydown', handleDelete); + }, [selection, dispatch]); + + useEffect(() => { + const handleCopy = (e: KeyboardEvent) => { + if (!(e.metaKey || e.ctrlKey) || e.key.toLowerCase() !== 'c') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + if (selection.selectedIds.size === 0) return; + + e.preventDefault(); + const copied: ClipboardCard[] = []; + const names: string[] = []; + for (const [id, type] of selection.selectedIds) { + if (type === 'agent') { + const session = sessions[id]; + const card = cards[id]; + if (!session || !card) continue; + copied.push({ + type, id, name: session.name || id, + meta: { name: session.name, status: session.status, model: session.model, mode: session.mode }, + x: card.x, y: card.y, width: card.width, height: card.height, + expanded: expandedSessionIds.includes(id), + }); + names.push(session.name || id); + } else if (type === 'view') { + const output = outputs[id]; + const vc = viewCards[id]; + if (!output || !vc) continue; + copied.push({ + type, id, name: output.name, + meta: { name: output.name, description: output.description }, + x: vc.x, y: vc.y, width: vc.width, height: vc.height, + }); + names.push(output.name); + } else if (type === 'browser') { + const bc = browserCards[id]; + if (!bc) continue; + const activeTab = bc.tabs.find((t) => t.id === bc.activeTabId); + const title = activeTab?.title || 'Browser'; + copied.push({ + type, id, name: title, + meta: { name: title, url: activeTab?.url || bc.url, tabs: bc.tabs }, + x: bc.x, y: bc.y, width: bc.width, height: bc.height, + }); + names.push(title); + } + } + setClipboardCards(copied); + navigator.clipboard.writeText(names.join(', ')).catch(() => {}); + }; + window.addEventListener('keydown', handleCopy); + return () => window.removeEventListener('keydown', handleCopy); + }, [selection.selectedIds, sessions, cards, viewCards, browserCards, outputs, expandedSessionIds]); + + useEffect(() => { + const PASTE_OFFSET = 40; + const handlePaste = async (e: KeyboardEvent) => { + if (!(e.metaKey || e.ctrlKey) || e.key.toLowerCase() !== 'v') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + + const copied = getClipboardCards(); + if (copied.length === 0) return; + e.preventDefault(); + + selection.deselectAll(); + const newSelection = new Map(); + + for (const card of copied) { + const px = card.x + PASTE_OFFSET; + const py = card.y - PASTE_OFFSET; + + if (card.type === 'agent') { + const action = await dispatch(duplicateSession({ sessionId: card.id, dashboardId })); + if (duplicateSession.fulfilled.match(action)) { + const newId = action.payload.id; + dispatch(placeCard({ sessionId: newId, x: px, y: py, width: card.width, height: card.height })); + if (card.expanded) { + dispatch(expandSession(newId)); + } + newSelection.set(newId, 'agent'); + } + } else if (card.type === 'view') { + dispatch(addViewCard({ outputId: card.id, expandedSessionIds, x: px, y: py, width: card.width, height: card.height })); + newSelection.set(card.id, 'view'); + } else if (card.type === 'browser') { + const browserId = `browser-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`; + dispatch(pasteBrowserCard({ + id: browserId, tabs: card.meta.tabs || [], url: card.meta.url || '', + x: px, y: py, width: card.width, height: card.height, + })); + newSelection.set(browserId, 'browser'); + } + } + + if (newSelection.size > 0) { + for (const [id, type] of newSelection) { + selection.selectCard(id, type, true); + } + } + }; + window.addEventListener('keydown', handlePaste); + return () => window.removeEventListener('keydown', handlePaste); + }, [dispatch, dashboardId, expandedSessionIds, selection]); + + const handleBranchFromCard = useCallback( + (sourceSessionId: string, newSessionId: string) => { + const sourceCard = cards[sourceSessionId]; + if (!sourceCard) return; + + const targetX = sourceCard.x + sourceCard.width + GRID_GAP * 12; + let targetY = sourceCard.y; + + const columnCards = Object.values(cards).filter( + (c) => Math.abs(c.x - targetX) < 50 && c.session_id !== newSessionId, + ); + if (columnCards.length > 0) { + const lowestBottom = Math.max( + ...columnCards.map((c) => c.y + Math.max(EXPANDED_CARD_MIN_H, c.height)), + ); + targetY = lowestBottom + GRID_GAP; + } + + spawnOriginsRef.current[newSessionId] = { + x: sourceCard.x, + y: sourceCard.y, + type: 'branch' as const, + }; + + dispatch(placeCard({ + sessionId: newSessionId, + x: targetX, + y: targetY, + width: DEFAULT_CARD_W, + height: DEFAULT_CARD_H, + })); + + if (expandedSessionIds.includes(sourceSessionId)) { + dispatch(expandSession(newSessionId)); + } + + dispatch(setGlowingAgentCard({ sessionId: newSessionId, sourceId: sourceSessionId, label: 'Branch' })); + }, + [cards, dispatch, expandedSessionIds], + ); + const handleNewAgent = useCallback(() => { setToolbarOpen(true); }, []); @@ -376,7 +740,7 @@ const DashboardInner: React.FC = () => { contextPaths: contextPaths?.map((cp) => ({ path: cp.path, type: cp.type })), forcedTools, attachedSkills, - expand: false, + expand: expandNewChats, }), ).then((action) => { if (launchAndSendFirstMessage.fulfilled.match(action)) { @@ -388,6 +752,22 @@ const DashboardInner: React.FC = () => { spawnOriginsRef.current[realId] = spawnOriginsRef.current[draftId]; delete spawnOriginsRef.current[draftId]; + if (expandNewChats) { + setAutoFocusSessionId(realId); + } else { + setPendingSelectSessionId(realId); + } + + setTimeout(() => { + const card = store.getState().dashboardLayout.cards[realId]; + if (card) { + const isExp = store.getState().agents.expandedSessionIds.includes(realId); + const height = isExp ? Math.max(EXPANDED_CARD_MIN_H, card.height) : card.height; + canvas.actions.fitToCards([{ x: card.x, y: card.y, width: card.width, height }], 1.0, true); + handleHighlightCard(realId); + } + }, 200); + if (dashboardId) { const currentSessions = store.getState().agents.sessions; const agentCount = Object.values(currentSessions).filter( @@ -408,24 +788,49 @@ const DashboardInner: React.FC = () => { } }); }, - [canvas.viewportRef, dispatch, dashboardId], + [canvas.viewportRef, canvas.actions, dispatch, dashboardId, expandNewChats, handleHighlightCard], ); const handleAddView = useCallback((outputId: string) => { dispatch(addViewCard({ outputId, expandedSessionIds })); - }, [dispatch, expandedSessionIds]); + setTimeout(() => { + const card = store.getState().dashboardLayout.viewCards[outputId]; + if (card) { + canvas.actions.fitToCards([{ x: card.x, y: card.y, width: card.width, height: card.height }], 1.0, true); + handleHighlightCard(outputId); + } + }, 200); + }, [dispatch, expandedSessionIds, canvas.actions, handleHighlightCard]); const handleAddBrowser = useCallback(() => { + const prevIds = new Set(Object.keys(store.getState().dashboardLayout.browserCards)); dispatch(addBrowserCard({ url: browserHomepage, expandedSessionIds })); - }, [dispatch, browserHomepage, expandedSessionIds]); + setTimeout(() => { + const allBrowserCards = store.getState().dashboardLayout.browserCards; + const newId = Object.keys(allBrowserCards).find((id) => !prevIds.has(id)); + if (newId) { + const card = allBrowserCards[newId]; + canvas.actions.fitToCards([{ x: card.x, y: card.y, width: card.width, height: card.height }], 1.0, true); + handleHighlightCard(newId); + } + }, 200); + }, [dispatch, browserHomepage, expandedSessionIds, canvas.actions, handleHighlightCard]); const handleHistoryResume = useCallback((sessionId: string) => { dispatch(resumeSession({ sessionId })).then((action) => { if (resumeSession.fulfilled.match(action)) { - dispatch(collapseSession(sessionId)); + dispatch(expandSession(sessionId)); + setAutoFocusSessionId(sessionId); + setTimeout(() => { + const card = store.getState().dashboardLayout.cards[sessionId]; + if (card) { + canvas.actions.fitToCards([{ x: card.x, y: card.y, width: card.width, height: card.height }], 1.0, true); + handleHighlightCard(sessionId); + } + }, 200); } }); - }, [dispatch]); + }, [dispatch, canvas.actions, handleHighlightCard, setAutoFocusSessionId]); const handleTidy = useCallback(() => { const currentExpanded = store.getState().agents.expandedSessionIds; @@ -444,6 +849,112 @@ const DashboardInner: React.FC = () => { canvas.actions.fitToCards(allRects); }, [dispatch, canvas.actions]); + useEffect(() => { + const DRIFT_THRESHOLD = 60; + + // Group tethered sub-agent cards by source, only including those still in the spawn column + const sourceToSiblings = new Map(); + for (const [id, glow] of Object.entries(glowingAgentCards)) { + const card = cards[id]; + if (!card) continue; + const sourceCard = cards[glow.sourceId]; + if (!sourceCard) continue; + const expectedX = sourceCard.x + sourceCard.width + GRID_GAP * 12; + if (Math.abs(card.x - expectedX) > DRIFT_THRESHOLD) continue; + const list = sourceToSiblings.get(glow.sourceId) ?? []; + list.push(id); + sourceToSiblings.set(glow.sourceId, list); + } + + for (const siblings of sourceToSiblings.values()) { + if (siblings.length < 2) continue; + siblings.sort((a, b) => cards[a].y - cards[b].y); + + let cursor = cards[siblings[0]].y; + for (const id of siblings) { + const card = cards[id]; + const dy = cursor - card.y; + if (Math.abs(dy) > 1) { + dispatch(moveCards({ items: [{ id, type: 'agent' as const }], dx: 0, dy })); + } + const isExpanded = expandedSessionIds.includes(id); + const h = isExpanded + ? Math.max(EXPANDED_CARD_MIN_H, card.height) + : (measuredHeightsRef.current[id] ?? card.height); + cursor += h + GRID_GAP * 2; + } + } + // measuredHeightsTick in deps ensures we re-run once ResizeObserver reports + // the new height after a collapse (avoids stale-height no-ops) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [expandedSessionIds, glowingAgentCards, cards, dispatch, measuredHeightsTick]); + + const TETHER_FADE_MS = 2500; + + const tethers = useMemo(() => { + const ELBOW_RADIUS = 16; + + function elbowPath(x1: number, y1: number, x2: number, y2: number): string { + const dx = x2 - x1; + const dy = y2 - y1; + const midX = x1 + dx / 2; + const r = (Math.abs(dy) < 1 || Math.abs(dx) < ELBOW_RADIUS * 2) + ? 0 + : Math.min(ELBOW_RADIUS, Math.abs(dy) / 2, Math.abs(dx) / 4); + const sy = dy >= 0 ? 1 : -1; + const sx = dx >= 0 ? 1 : -1; + + return [ + `M ${x1},${y1}`, + `H ${midX - sx * r}`, + `Q ${midX},${y1} ${midX},${y1 + sy * r}`, + `V ${y2 - sy * r}`, + `Q ${midX},${y2} ${midX + sx * r},${y2}`, + `H ${x2}`, + ].join(' '); + } + + return Object.entries(glowingAgentCards).map(([copyId, { sourceId, fading, sourceYRatio, label }]) => { + const src = cards[sourceId]; + const dst = cards[copyId]; + if (!src || !dst) return null; + + let srcX = src.x, srcY = src.y; + let dstX = dst.x, dstY = dst.y; + if (liveDragInfo) { + if (liveDragInfo.cardId === sourceId) { srcX += liveDragInfo.dx; srcY += liveDragInfo.dy; } + if (liveDragInfo.cardId === copyId) { dstX += liveDragInfo.dx; dstY += liveDragInfo.dy; } + } + + const srcMeasured = measuredHeightsRef.current[sourceId]; + const srcH = srcMeasured ?? (expandedSessionIds.includes(sourceId) + ? Math.max(EXPANDED_CARD_MIN_H, src.height) + : src.height); + const dstMeasured = measuredHeightsRef.current[copyId]; + const dstH = dstMeasured ?? (expandedSessionIds.includes(copyId) + ? Math.max(EXPANDED_CARD_MIN_H, dst.height) + : dst.height); + + const x1 = srcX + src.width; + const y1 = srcY + srcH * 0.54; + const x2 = dstX; + const y2 = dstY + dstH * (expandedSessionIds.includes(copyId) ? 0.54 : 0.79); + const midX = x1 + (x2 - x1) / 2; + const labelX = midX + (x2 - midX) * 0.15; + const labelY = y2; + + return { + key: copyId, + path: elbowPath(x1, y1, x2, y2), + labelX, + labelY, + label: label || '', + fading, + }; + }).filter(Boolean) as Array<{ key: string; path: string; labelX: number; labelY: number; label: string; fading: boolean }>; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [glowingAgentCards, cards, expandedSessionIds, liveDragInfo, measuredHeightsTick]); + const dotSize = Math.max(1, 1.5 * canvas.zoom); const dotSpacing = 24 * canvas.zoom; @@ -486,6 +997,7 @@ const DashboardInner: React.FC = () => { onMouseDown={handleViewportMouseDown} onMouseMove={handleViewportMouseMove} onMouseUp={handleViewportMouseUp} + onContextMenu={(e) => e.preventDefault()} sx={{ position: 'absolute', inset: 0, @@ -493,7 +1005,7 @@ const DashboardInner: React.FC = () => { cursor: canvas.isPanning ? 'grabbing' : (canvas.spaceHeld || canvas.cmdHeld) - ? 'crosshair' + ? 'grab' : selection.marquee ? 'crosshair' : 'default', @@ -540,11 +1052,178 @@ const DashboardInner: React.FC = () => { position: 'relative', }} > + {/* Tether lines between branched cards */} + {tethers.length > 0 && ( + + + + + + + + + + + + + + + {tethers.map((t) => ( + + + + + {t.label && ( + + + + {t.label} + + + )} + + ))} + + )} + {Object.values(cards).map((card) => { const session = sessions[card.session_id]; if (!session) return null; - const origin = spawnOriginsRef.current[session.id]; - if (origin) delete spawnOriginsRef.current[session.id]; + + let origin = spawnOriginsRef.current[session.id]; + if (origin) { + delete spawnOriginsRef.current[session.id]; + } else { + const glow = glowingAgentCards[session.id]; + if (glow && !revealSpawnedRef.current.has(session.id)) { + revealSpawnedRef.current.add(session.id); + const srcCard = cards[glow.sourceId]; + if (srcCard) { + const srcH = measuredHeightsRef.current[glow.sourceId] + ?? (expandedSessionIds.includes(glow.sourceId) + ? Math.max(EXPANDED_CARD_MIN_H, srcCard.height) + : srcCard.height); + origin = { + x: srcCard.x + srcCard.width, + y: srcCard.y + srcH / 2, + type: 'branch' as const, + }; + } + } + } + + let exitTarget: { x: number; y: number } | undefined; + const glow = glowingAgentCards[session.id]; + if (glow) { + const srcCard = cards[glow.sourceId]; + if (srcCard) { + const srcH = measuredHeightsRef.current[glow.sourceId] + ?? (expandedSessionIds.includes(glow.sourceId) + ? Math.max(EXPANDED_CARD_MIN_H, srcCard.height) + : srcCard.height); + exitTarget = { + x: srcCard.x + srcCard.width, + y: srcCard.y + srcH / 2, + }; + } + } + + let snapColumn: { x: number; width: number } | undefined; + if (glow) { + const srcCard = cards[glow.sourceId]; + if (srcCard) { + snapColumn = { + x: srcCard.x + srcCard.width + GRID_GAP * 12, + width: DEFAULT_CARD_W, + }; + } + } + return ( { cardHeight={card.height} zoom={canvas.zoom} spawnFrom={origin} + exitTarget={exitTarget} isSelected={selection.isSelected(session.id)} isHighlighted={highlightedCardId === session.id} multiDragDelta={multiDragDelta} @@ -563,9 +1243,14 @@ const DashboardInner: React.FC = () => { onDragStart={handleCardDragStart} onDragMove={handleCardDragMove} onDragEnd={handleCardDragEnd} + onBranch={handleBranchFromCard} + onMeasuredHeight={handleMeasuredHeight} + snapColumn={snapColumn} + autoFocusInput={autoFocusSessionId === session.id} /> ); })} + {Object.values(viewCards).map((vc) => { const output = outputs[vc.output_id]; if (!output) return null; @@ -578,6 +1263,7 @@ const DashboardInner: React.FC = () => { cardWidth={vc.width} cardHeight={vc.height} zoom={canvas.zoom} + cmdHeld={canvas.cmdHeld} isSelected={selection.isSelected(vc.output_id)} isHighlighted={highlightedCardId === vc.output_id} multiDragDelta={multiDragDelta} @@ -599,6 +1285,7 @@ const DashboardInner: React.FC = () => { cardWidth={bc.width} cardHeight={bc.height} zoom={canvas.zoom} + cmdHeld={canvas.cmdHeld} isSelected={selection.isSelected(bc.browser_id)} isHighlighted={highlightedCardId === bc.browser_id} multiDragDelta={multiDragDelta} diff --git a/frontend/src/app/pages/Dashboard/DashboardHeader.tsx b/frontend/src/app/pages/Dashboard/DashboardHeader.tsx index 417b6315..b6429b9f 100644 --- a/frontend/src/app/pages/Dashboard/DashboardHeader.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardHeader.tsx @@ -92,7 +92,7 @@ const DashboardHeader: React.FC = ({ const handleFocus = useCallback( (cardId: string, card: { x: number; y: number; width: number; height: number }) => { - canvasActions.fitToCards([card], 1.0); + canvasActions.fitToCards([card], 1.0, true); onHighlightCard?.(cardId); setExpanded(false); }, diff --git a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx index 38b000b0..9dad9329 100644 --- a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx @@ -42,6 +42,7 @@ interface Props { dashboardId?: string; } +const TOOLBAR_OWNER_ID = '__toolbar__'; const BTN = 40; const WarmTooltip = styled( @@ -90,8 +91,18 @@ const DashboardToolbar = React.forwardRef( const searchInputRef = useRef(null); const historyInputRef = useRef(null); const historyListRef = useRef(null); - const [mode, setMode] = useState('agent'); - const [model, setModel] = useState('sonnet'); + const defaultMode = useAppSelector((s) => s.settings.data.default_mode); + const defaultModel = useAppSelector((s) => s.settings.data.default_model); + const [mode, setMode] = useState(defaultMode || 'agent'); + const [model, setModel] = useState(defaultModel || 'sonnet'); + const settingsApplied = useRef(false); + useEffect(() => { + if (!settingsApplied.current) { + setMode(defaultMode || 'agent'); + setModel(defaultModel || 'sonnet'); + settingsApplied.current = true; + } + }, [defaultMode, defaultModel]); const [viewPickerOpen, setViewPickerOpen] = useState(false); const [viewSearch, setViewSearch] = useState(''); const [historyOpen, setHistoryOpen] = useState(false); @@ -204,13 +215,23 @@ const DashboardToolbar = React.forwardRef( const isExpanded = inputOpen || viewPickerOpen || historyOpen; + const autoSelectOnNew = useAppSelector((s) => s.settings.data.auto_select_mode_on_new_agent); const prevInputOpenRef = useRef(inputOpen); useEffect(() => { - if (prevInputOpenRef.current && !inputOpen && elementSelection?.selectMode) { - elementSelection.setSelectMode(false); + if (prevInputOpenRef.current && !inputOpen && elementSelection) { + elementSelection.clearOwnerElements(TOOLBAR_OWNER_ID); + if (elementSelection.selectMode && elementSelection.activeOwnerId === TOOLBAR_OWNER_ID) { + elementSelection.setSelectMode(false); + } + } + if (!prevInputOpenRef.current && inputOpen && autoSelectOnNew && elementSelection) { + elementSelection.clearOwnerElements(TOOLBAR_OWNER_ID); + elementSelection.setActiveOwnerId(TOOLBAR_OWNER_ID); + elementSelection.setExcludeSelectId(null); + elementSelection.setSelectMode(true); } prevInputOpenRef.current = inputOpen; - }, [inputOpen, elementSelection]); + }, [inputOpen, elementSelection, autoSelectOnNew]); useEffect(() => { if (!isExpanded) return; @@ -226,21 +247,44 @@ const DashboardToolbar = React.forwardRef( useEffect(() => { if (!isExpanded) return; - const handleClick = (e: MouseEvent) => { - if (elementSelection?.selectMode) return; + let downPos: { x: number; y: number; target: Node } | null = null; + const DRAG_THRESHOLD = 5; + + const handleDown = (e: MouseEvent) => { const target = e.target as Node; if (containerRef.current && !containerRef.current.contains(target)) { - const el = target instanceof Element ? target : target.parentElement; - if (el?.closest('[role="dialog"], [role="presentation"], .MuiModal-root, .MuiPopover-root')) { - return; - } - handleDismiss(); + downPos = { x: e.clientX, y: e.clientY, target }; + } else { + downPos = null; } }; - const t = setTimeout(() => document.addEventListener('mousedown', handleClick), 50); + + const handleUp = (e: MouseEvent) => { + if (!downPos) return; + const dx = e.clientX - downPos.x; + const dy = e.clientY - downPos.y; + const target = downPos.target; + downPos = null; + if (Math.abs(dx) > DRAG_THRESHOLD || Math.abs(dy) > DRAG_THRESHOLD) return; + + const el = target instanceof Element ? target : (target as Node).parentElement; + if (el?.closest('[role="dialog"], [role="presentation"], .MuiModal-root, .MuiPopover-root')) { + return; + } + if (elementSelection?.selectMode && el?.closest('[data-select-type]')) { + return; + } + handleDismiss(); + }; + + const t = setTimeout(() => { + document.addEventListener('mousedown', handleDown, true); + document.addEventListener('mouseup', handleUp, true); + }, 50); return () => { clearTimeout(t); - document.removeEventListener('mousedown', handleClick); + document.removeEventListener('mousedown', handleDown, true); + document.removeEventListener('mouseup', handleUp, true); }; }, [isExpanded, handleDismiss, elementSelection?.selectMode]); @@ -266,10 +310,14 @@ const DashboardToolbar = React.forwardRef( e.preventDefault(); handleOpenHistory(); } + if (e.metaKey && e.key.toLowerCase() === 'n' && !e.ctrlKey && !e.shiftKey && !e.altKey) { + e.preventDefault(); + onAddBrowser(); + } }; window.addEventListener('keydown', handleKey); return () => window.removeEventListener('keydown', handleKey); - }, [handleOpenViewPicker, handleOpenHistory]); + }, [handleOpenViewPicker, handleOpenHistory, onAddBrowser]); useEffect(() => { if (!historyOpen) return; @@ -319,6 +367,7 @@ const DashboardToolbar = React.forwardRef( onModelChange={setModel} embedded autoFocus + sessionId={TOOLBAR_OWNER_ID} /> ) : historyOpen ? ( @@ -596,6 +645,39 @@ const DashboardToolbar = React.forwardRef( + + Browser ⌘N + + } + > + + + + + ( - - Browser - - } - > - - - - - {placeholderItems.map(({ icon: PlaceholderIcon, label, sub }) => ( = ({ - output, cardX, cardY, cardWidth, cardHeight, zoom = 1, + output, cardX, cardY, cardWidth, cardHeight, zoom = 1, cmdHeld = false, isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd, }) => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); const previewRef = useRef(null); const [inputData, setInputData] = useState>(() => getDefault(output.input_schema)); @@ -77,6 +80,7 @@ const DashboardViewCard: React.FC = ({ const justDraggedRef = useRef(false); const handleDragPointerDown = useCallback((e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY }; @@ -132,6 +136,7 @@ const DashboardViewCard: React.FC = ({ const handleResizeDown = useCallback( (dir: ResizeDir) => (e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); resizeRef.current = { @@ -304,9 +309,10 @@ const DashboardViewCard: React.FC = ({ }), }} > - {/* Selection overlay – blocks content interaction while selected, enabling drag from anywhere */} + {/* Selection overlay – blocks click interaction while selected, enabling drag from anywhere */} {isSelected && ( = ({ {/* Preview body */} + {cmdHeld && !isSelected && ( + + )} (null); // Wheel zoom centered on cursor useEffect(() => { @@ -207,6 +208,10 @@ export function useCanvasControls(zoomSensitivity: number = 50) { return () => window.removeEventListener('mouseup', onUp); }, []); + useEffect(() => { + return () => { if (animFrameRef.current) cancelAnimationFrame(animFrameRef.current); }; + }, []); + const zoomIn = useCallback(() => { setState((prev) => { const newZoom = clamp(prev.zoom * ZOOM_IN_FACTOR, MIN_ZOOM, MAX_ZOOM); @@ -276,7 +281,9 @@ export function useCanvasControls(zoomSensitivity: number = 50) { }); }, []); - const fitToCards = useCallback((cardRects: Array<{ x: number; y: number; width: number; height: number }>, maxZoom?: number) => { + const fitToCards = useCallback((cardRects: Array<{ x: number; y: number; width: number; height: number }>, maxZoom?: number, animate?: boolean) => { + if (animFrameRef.current) { cancelAnimationFrame(animFrameRef.current); animFrameRef.current = null; } + const viewport = viewportRef.current; if (!viewport || cardRects.length === 0) { setState({ panX: 0, panY: 0, zoom: 1 }); @@ -303,11 +310,34 @@ export function useCanvasControls(zoomSensitivity: number = 50) { const availW = vRect.width - FIT_PADDING * 2; const availH = vRect.height - FIT_PADDING * 2; const ceiling = maxZoom ?? MAX_ZOOM; - const newZoom = clamp(Math.min(availW / contentWidth, availH / contentHeight), MIN_ZOOM, ceiling); - const newPanX = (vRect.width - contentWidth * newZoom) / 2 - minX * newZoom; - const newPanY = (vRect.height - contentHeight * newZoom) / 2 - minY * newZoom; + const targetZoom = clamp(Math.min(availW / contentWidth, availH / contentHeight), MIN_ZOOM, ceiling); + const targetPanX = (vRect.width - contentWidth * targetZoom) / 2 - minX * targetZoom; + const targetPanY = (vRect.height - contentHeight * targetZoom) / 2 - minY * targetZoom; - setState({ panX: newPanX, panY: newPanY, zoom: newZoom }); + if (!animate) { + setState({ panX: targetPanX, panY: targetPanY, zoom: targetZoom }); + return; + } + + const start = { ...stateRef.current }; + const startTime = performance.now(); + const duration = 320; + + const step = (now: number) => { + const t = Math.min((now - startTime) / duration, 1); + const ease = 1 - Math.pow(1 - t, 3); + setState({ + panX: start.panX + (targetPanX - start.panX) * ease, + panY: start.panY + (targetPanY - start.panY) * ease, + zoom: start.zoom + (targetZoom - start.zoom) * ease, + }); + if (t < 1) { + animFrameRef.current = requestAnimationFrame(step); + } else { + animFrameRef.current = null; + } + }; + animFrameRef.current = requestAnimationFrame(step); }, []); const handlers = useMemo(() => ({ diff --git a/frontend/src/app/pages/Dashboard/useDashboardSelection.ts b/frontend/src/app/pages/Dashboard/useDashboardSelection.ts index 6344d975..aa84f6e2 100644 --- a/frontend/src/app/pages/Dashboard/useDashboardSelection.ts +++ b/frontend/src/app/pages/Dashboard/useDashboardSelection.ts @@ -79,7 +79,10 @@ export function useDashboardSelection( } return next; } - return new Map([[id, type]]); + if (prev.has(id)) { + return new Map(); + } + return prev; }); }, [], @@ -152,7 +155,7 @@ export function useDashboardSelection( const handleCanvasMouseDown = useCallback( (e: MouseEvent) => { - if (e.button !== 0) return; + if (e.button !== 0 && e.button !== 2) return; marqueeOriginRef.current = { screenX: e.clientX, screenY: e.clientY }; isDraggingMarqueeRef.current = false; diff --git a/frontend/src/app/pages/Dashboard/useOverlayScrollPassthrough.ts b/frontend/src/app/pages/Dashboard/useOverlayScrollPassthrough.ts new file mode 100644 index 00000000..9107b8f5 --- /dev/null +++ b/frontend/src/app/pages/Dashboard/useOverlayScrollPassthrough.ts @@ -0,0 +1,80 @@ +import { useRef, useEffect } from 'react'; + +/** + * Attaches a native wheel listener to an overlay element that forwards scroll + * events to whatever scrollable content sits beneath it, while still letting + * the overlay capture pointer events (click / drag). Pinch-zoom (ctrl/meta + + * wheel) is left alone so the canvas zoom still works. + * + * Handles two cases: + * 1. Regular DOM scrollable containers — uses `scrollBy` directly. + * 2. Electron `` elements — executes JS inside the webview to scroll + * the element at the cursor position. + */ +export function useOverlayScrollPassthrough(active: boolean) { + const ref = useRef(null); + + useEffect(() => { + const el = ref.current; + if (!active || !el) return; + + const handleWheel = (e: WheelEvent) => { + if (e.ctrlKey || e.metaKey) return; + + el.style.pointerEvents = 'none'; + const underneath = document.elementFromPoint(e.clientX, e.clientY); + el.style.pointerEvents = ''; + + let dx = e.deltaX; + let dy = e.deltaY; + if (e.deltaMode === 1) { + dx *= 20; + dy *= 20; + } + + let node = underneath as HTMLElement | null; + while (node) { + if (node.tagName === 'WEBVIEW') { + e.stopPropagation(); + e.preventDefault(); + const rect = node.getBoundingClientRect(); + const relX = Math.round(e.clientX - rect.left); + const relY = Math.round(e.clientY - rect.top); + (node as any).executeJavaScript?.( + `(function(){` + + `var el=document.elementFromPoint(${relX},${relY});` + + `while(el){` + + `var s=getComputedStyle(el);` + + `if((s.overflowY==='auto'||s.overflowY==='scroll')&&el.scrollHeight>el.clientHeight){el.scrollBy(${dx},${dy});return}` + + `if((s.overflowX==='auto'||s.overflowX==='scroll')&&el.scrollWidth>el.clientWidth){el.scrollBy(${dx},${dy});return}` + + `el=el.parentElement}` + + `window.scrollBy(${dx},${dy})` + + `})()` + ).catch(() => {}); + return; + } + + const cs = getComputedStyle(node); + const canScrollY = + node.scrollHeight > node.clientHeight && + (cs.overflowY === 'auto' || cs.overflowY === 'scroll'); + const canScrollX = + node.scrollWidth > node.clientWidth && + (cs.overflowX === 'auto' || cs.overflowX === 'scroll'); + + if (canScrollY || canScrollX) { + e.stopPropagation(); + e.preventDefault(); + node.scrollBy(dx, dy); + return; + } + node = node.parentElement; + } + }; + + el.addEventListener('wheel', handleWheel, { passive: false }); + return () => el.removeEventListener('wheel', handleWheel); + }, [active]); + + return ref; +} diff --git a/frontend/src/app/pages/Modes/Modes.tsx b/frontend/src/app/pages/Modes/Modes.tsx index e7e75627..2b024b81 100644 --- a/frontend/src/app/pages/Modes/Modes.tsx +++ b/frontend/src/app/pages/Modes/Modes.tsx @@ -27,6 +27,7 @@ import DeleteIcon from '@mui/icons-material/Delete'; import TuneIcon from '@mui/icons-material/Tune'; import LockIcon from '@mui/icons-material/Lock'; import ArrowForwardIcon from '@mui/icons-material/ArrowForward'; +import RestoreIcon from '@mui/icons-material/Restore'; import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined'; import QuestionAnswerOutlinedIcon from '@mui/icons-material/QuestionAnswerOutlined'; import MapOutlinedIcon from '@mui/icons-material/MapOutlined'; @@ -37,6 +38,7 @@ import { createMode, updateMode, deleteMode, + resetMode, Mode, } from '@/shared/state/modesSlice'; import { fetchBuiltinTools, fetchTools } from '@/shared/state/toolsSlice'; @@ -105,7 +107,7 @@ const ALL_BUILTIN_TOOL_NAMES = ['Read', 'Edit', 'Write', 'Bash', 'Glob', 'Grep', const Modes: React.FC = () => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); - const { items, loading } = useAppSelector((s) => s.modes); + const { items, builtinDefaults, loading } = useAppSelector((s) => s.modes); const toolItems = useAppSelector((s) => s.tools.items); const modes = useMemo(() => Object.values(items), [items]); @@ -174,6 +176,45 @@ const Modes: React.FC = () => { await dispatch(deleteMode(id)); }; + const editingIsBuiltin = editingId ? items[editingId]?.is_builtin ?? false : false; + + const hasDiverged = useMemo(() => { + if (!editingId || !editingIsBuiltin) return false; + const defaults = builtinDefaults[editingId]; + if (!defaults) return false; + const current = items[editingId]; + if (!current) return false; + return ( + current.name !== defaults.name || + current.description !== defaults.description || + (current.system_prompt ?? '') !== (defaults.system_prompt ?? '') || + JSON.stringify(current.tools) !== JSON.stringify(defaults.tools) || + (current.default_next_mode ?? '') !== (defaults.default_next_mode ?? '') || + current.icon !== defaults.icon || + current.color !== defaults.color || + (current.default_folder ?? '') !== (defaults.default_folder ?? '') + ); + }, [editingId, editingIsBuiltin, items, builtinDefaults]); + + const handleReset = async () => { + if (!editingId) return; + const action = await dispatch(resetMode(editingId)); + if (resetMode.fulfilled.match(action)) { + const m = action.payload; + setForm({ + name: m.name, + description: m.description, + system_prompt: m.system_prompt ?? '', + tools: m.tools ?? [], + toolsEnabled: m.tools !== null, + default_next_mode: m.default_next_mode ?? '', + icon: m.icon, + color: m.color, + default_folder: m.default_folder ?? '', + }); + } + }; + const otherModes = modes.filter((m) => m.id !== editingId); return ( @@ -505,23 +546,46 @@ const Modes: React.FC = () => { - - - + + + {editingIsBuiltin && ( + + + + + + )} + + + + + diff --git a/frontend/src/app/pages/Settings/Settings.tsx b/frontend/src/app/pages/Settings/Settings.tsx index 70a14a8d..79f743bc 100644 --- a/frontend/src/app/pages/Settings/Settings.tsx +++ b/frontend/src/app/pages/Settings/Settings.tsx @@ -11,6 +11,7 @@ import InputAdornment from '@mui/material/InputAdornment'; import ToggleButton from '@mui/material/ToggleButton'; import ToggleButtonGroup from '@mui/material/ToggleButtonGroup'; import Slider from '@mui/material/Slider'; +import Switch from '@mui/material/Switch'; import Snackbar from '@mui/material/Snackbar'; import Alert from '@mui/material/Alert'; import Tab from '@mui/material/Tab'; @@ -495,7 +496,7 @@ const Settings: React.FC = () => { - + New agent shortcut Keyboard shortcut to create an agent. @@ -553,6 +554,51 @@ const Settings: React.FC = () => { + + + Auto-enable element selection + Automatically enter element selection mode when creating a new agent. + + setForm({ ...form, auto_select_mode_on_new_agent: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + + + + Default agent spawn state in dashboard + When enabled, new agents spawn expanded instead of collapsed. + + setForm({ ...form, expand_new_chats_in_dashboard: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + + + + Auto-reveal sub-agents on dashboard + Automatically show sub-agent cards (from CreateAgent / InvokeAgent) tethered to their parent on the dashboard. + + setForm({ ...form, auto_reveal_sub_agents: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {/* ── Browser ── */} Browser @@ -639,15 +685,8 @@ const Settings: React.FC = () => { {step.title} {step.link && ( { - const w = window as any; - if (w.openswarm?.openExternal) { - w.openswarm.openExternal(step.link); - } else { - window.open(step.link, '_blank', 'noopener'); - } - }} + component="a" + href={step.link} sx={{ color: c.accent.primary, fontSize: '0.75rem', @@ -657,6 +696,7 @@ const Settings: React.FC = () => { alignItems: 'center', gap: 0.3, verticalAlign: 'middle', + textDecoration: 'none', '&:hover': { textDecoration: 'underline' }, }} > @@ -705,6 +745,24 @@ const Settings: React.FC = () => { /> + {/* ── Advanced ── */} + Advanced + + + + Developer mode + Show transport details, environment variables, raw configs, and other technical metadata throughout the app. + + setForm({ ...form, dev_mode: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {/* ── About ── */} About diff --git a/frontend/src/app/pages/Skills/Skills.tsx b/frontend/src/app/pages/Skills/Skills.tsx index 7424cac5..89b1feaa 100644 --- a/frontend/src/app/pages/Skills/Skills.tsx +++ b/frontend/src/app/pages/Skills/Skills.tsx @@ -579,8 +579,6 @@ const Skills: React.FC = () => { size="small" component="a" href={selectedReg.repositoryUrl} - target="_blank" - rel="noopener" sx={{ color: c.text.tertiary, '&:hover': { color: c.text.primary } }} > diff --git a/frontend/src/app/pages/Tools/Tools.tsx b/frontend/src/app/pages/Tools/Tools.tsx index a6e05224..272b7688 100644 --- a/frontend/src/app/pages/Tools/Tools.tsx +++ b/frontend/src/app/pages/Tools/Tools.tsx @@ -56,6 +56,7 @@ import BlockIcon from '@mui/icons-material/Block'; import VisibilityIcon from '@mui/icons-material/Visibility'; import SecurityIcon from '@mui/icons-material/Security'; import PanToolIcon from '@mui/icons-material/PanTool'; +import CallSplitIcon from '@mui/icons-material/CallSplit'; import RefreshIcon from '@mui/icons-material/Refresh'; import { fetchTools, @@ -74,6 +75,8 @@ import { import { searchRegistry, fetchRegistryStats, + fetchServerDetail, + clearDetail, McpServer, } from '@/shared/state/mcpRegistrySlice'; import { @@ -156,7 +159,7 @@ const INTEGRATIONS: Integration[] = [ }, ]; -const CATEGORY_ORDER = ['filesystem', 'system', 'search', 'interaction', 'planning', 'scheduling']; +const CATEGORY_ORDER = ['filesystem', 'system', 'search', 'interaction', 'agents', 'planning', 'scheduling']; interface ToolForm { name: string; @@ -235,6 +238,7 @@ const ToolSection: React.FC = ({ interaction: { label: 'Interaction', color: '#a855f7', icon: }, planning: { label: 'Planning', color: '#ec4899', icon: }, scheduling: { label: 'Scheduling', color: '#14b8a6', icon: }, + agents: { label: 'Agents', color: '#f97316', icon: }, }; const PermToggle = ({ value, onChange, size = 16 }: { value: string; onChange: (v: string) => void; size?: number }) => ( @@ -343,7 +347,7 @@ const ToolSection: React.FC = ({ return ( - {bt.name} + {bt.display_name || bt.name} {bt.description && {firstSentence(bt.description)}} onPermissionChange(bt.name, v)} size={14} /> @@ -370,7 +374,8 @@ const Tools: React.FC = () => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); const { items, builtinTools, builtinPermissions, loading } = useAppSelector((s) => s.tools); - const { servers: regServers, total: regTotal, loading: regLoading, stats: regStats } = useAppSelector((s) => s.mcpRegistry); + const { servers: regServers, total: regTotal, loading: regLoading, stats: regStats, detail: regDetail, detailLoading: regDetailLoading } = useAppSelector((s) => s.mcpRegistry); + const devMode = useAppSelector((s) => s.settings.data.dev_mode); const outputItems = useAppSelector((s) => s.outputs.items); const outputs = useMemo(() => Object.values(outputItems), [outputItems]); const allTools = Object.values(items); @@ -488,7 +493,8 @@ const Tools: React.FC = () => { if (discoverTools.fulfilled.match(result)) { setSnackbar({ open: true, message: 'Actions discovered successfully' }); } else { - setSnackbar({ open: true, message: 'Discovery failed — is the MCP server running?', severity: 'error' }); + const detail = (result as any).error?.message || 'Discovery failed — is the MCP server running?'; + setSnackbar({ open: true, message: detail, severity: 'error' }); } } finally { setDiscovering(false); @@ -530,8 +536,11 @@ const Tools: React.FC = () => { }; const [expandedServices, setExpandedServices] = useState>({}); + const [expandedSchema, setExpandedSchema] = useState(null); const [viewsSectionOpen, setViewsSectionOpen] = useState(false); + const [browserSectionOpen, setBrowserSectionOpen] = useState(false); + const [browserCollapsed, setBrowserCollapsed] = useState>({ browser_delegation: true, browser_action: true }); const [builtinSectionOpen, setBuiltinSectionOpen] = useState(true); useEffect(() => { @@ -556,8 +565,12 @@ const Tools: React.FC = () => { }; // Built-in tool grouping - const coreTools = useMemo(() => builtinTools.filter((bt) => !bt.deferred), [builtinTools]); - const deferredTools = useMemo(() => builtinTools.filter((bt) => bt.deferred), [builtinTools]); + const BROWSER_CATEGORIES = new Set(['browser_delegation', 'browser_action']); + const coreTools = useMemo(() => builtinTools.filter((bt) => !bt.deferred && !BROWSER_CATEGORIES.has(bt.category)), [builtinTools]); + const deferredTools = useMemo(() => builtinTools.filter((bt) => bt.deferred && !BROWSER_CATEGORIES.has(bt.category)), [builtinTools]); + const browserTools = useMemo(() => builtinTools.filter((bt) => BROWSER_CATEGORIES.has(bt.category)), [builtinTools]); + const browserDelegationTools = useMemo(() => browserTools.filter((bt) => bt.category === 'browser_delegation'), [browserTools]); + const browserActionTools = useMemo(() => browserTools.filter((bt) => bt.category === 'browser_action'), [browserTools]); const groupTools = (list: BuiltinTool[]) => { const g: Record = {}; for (const bt of list) { if (!g[bt.category]) g[bt.category] = []; g[bt.category].push(bt); } @@ -578,10 +591,14 @@ const Tools: React.FC = () => { () => !outputs.every((o) => o.permission === 'deny'), [outputs], ); + const browserSectionEnabled = useMemo( + () => browserTools.length > 0 && !browserTools.every((t) => builtinPermissions[t.name] === 'deny'), + [browserTools, builtinPermissions], + ); const handleSectionEnabledChange = async (tools: BuiltinTool[], enabled: boolean) => { const perms: Record = {}; - for (const t of tools) perms[t.name] = enabled ? 'ask' : 'deny'; + for (const t of tools) perms[t.name] = enabled ? 'always_allow' : 'deny'; await dispatch(updateBuiltinPermissions(perms)); }; @@ -894,7 +911,7 @@ const Tools: React.FC = () => { {builtinSectionOpen ? : } Built-in Action Sets - + @@ -992,6 +1009,156 @@ const Tools: React.FC = () => { )} + {/* Browser */} + {browserTools.length > 0 && ( + + + browserSectionEnabled && setBrowserSectionOpen((v) => !v)} + sx={{ display: 'flex', alignItems: 'center', gap: 2, cursor: browserSectionEnabled ? 'pointer' : 'default' }} + > + + + + + + Browser + + + Browser automation delegation and individual browser actions + + e.stopPropagation()}> + handleSectionEnabledChange(browserTools, checked)} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {browserSectionEnabled && ( + + + + )} + + + + + + + + Action Permissions + + + + + {/* Delegation group */} + {browserDelegationTools.length > 0 && (() => { + const delegationPolicies = browserDelegationTools.map((t) => builtinPermissions[t.name] || 'always_allow'); + const groupPolicy = delegationPolicies.every((p) => p === 'always_allow') ? 'always_allow' + : delegationPolicies.every((p) => p === 'deny') ? 'deny' + : delegationPolicies.every((p) => p === 'ask') ? 'ask' : 'ask'; + const isOpen = !browserCollapsed.browser_delegation; + return ( + + setBrowserCollapsed((p) => ({ ...p, browser_delegation: !p.browser_delegation }))} + > + + + Delegation + + + e.stopPropagation()}> + handleBuiltinCategoryPermissionChange(browserDelegationTools.map((t) => t.name), 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: groupPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinCategoryPermissionChange(browserDelegationTools.map((t) => t.name), 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: groupPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinCategoryPermissionChange(browserDelegationTools.map((t) => t.name), 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: groupPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + + + {browserDelegationTools.map((bt) => { + const toolPolicy = builtinPermissions[bt.name] || 'always_allow'; + return ( + + + {bt.display_name || bt.name} + {bt.description && {bt.description}} + + e.stopPropagation()}> + handleBuiltinPermissionChange(bt.name, 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: toolPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinPermissionChange(bt.name, 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: toolPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinPermissionChange(bt.name, 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: toolPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + ); + })} + + + + ); + })()} + + {/* Browser Actions group */} + {browserActionTools.length > 0 && (() => { + const actionPolicies = browserActionTools.map((t) => builtinPermissions[t.name] || 'always_allow'); + const groupPolicy = actionPolicies.every((p) => p === 'always_allow') ? 'always_allow' + : actionPolicies.every((p) => p === 'deny') ? 'deny' + : actionPolicies.every((p) => p === 'ask') ? 'ask' : 'ask'; + const isOpen = !browserCollapsed.browser_action; + return ( + + setBrowserCollapsed((p) => ({ ...p, browser_action: !p.browser_action }))} + > + + + Browser Actions + + + e.stopPropagation()}> + handleBuiltinCategoryPermissionChange(browserActionTools.map((t) => t.name), 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: groupPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinCategoryPermissionChange(browserActionTools.map((t) => t.name), 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: groupPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinCategoryPermissionChange(browserActionTools.map((t) => t.name), 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: groupPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + + + {browserActionTools.map((bt) => { + const toolPolicy = builtinPermissions[bt.name] || 'always_allow'; + return ( + + + {bt.display_name || bt.name} + {bt.description && {bt.description}} + + e.stopPropagation()}> + handleBuiltinPermissionChange(bt.name, 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: toolPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinPermissionChange(bt.name, 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: toolPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinPermissionChange(bt.name, 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: toolPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + ); + })} + + + + ); + })()} + + + + + )} + @@ -1029,7 +1196,7 @@ const Tools: React.FC = () => { {ig.name} - } label="docs" size="small" sx={{ bgcolor: c.bg.secondary, color: c.text.ghost, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, fontSize: 10 } }} /> + } label="docs" size="small" sx={{ bgcolor: c.bg.secondary, color: c.text.ghost, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, fontSize: 10 } }} /> {ig.description} @@ -1059,6 +1226,7 @@ const Tools: React.FC = () => { const perms = tool.tool_permissions || {}; const services = perms._services as Record | undefined; const descriptions = (perms._tool_descriptions || {}) as Record; + const schemas = (perms._tool_schemas || {}) as Record; const serviceNames = services ? Object.keys(services) : []; const hasPerms = serviceNames.length > 0; const totalToolCount = serviceNames.reduce((acc, s) => acc + (services![s].read?.length || 0) + (services![s].write?.length || 0), 0); @@ -1131,15 +1299,36 @@ const Tools: React.FC = () => { handleGroupPermissionChange(tool.id, data.read!, v)} size={14} /> - {data.read!.map((name) => ( - - - {toDisplayName(name, serviceName)} - {descriptions[name] && {firstSentence(descriptions[name])}} + {data.read!.map((name) => { + const schemaKey = `${tool.id}:${name}`; + const schema = schemas[name]; + const schemaProps = schema?.properties as Record | undefined; + const schemaRequired = (schema?.required || []) as string[]; + return ( + + devMode && schema && setExpandedSchema((p) => p === schemaKey ? null : schemaKey)}> + + {toDisplayName(name, serviceName)} + {descriptions[name] && {firstSentence(descriptions[name])}} + + handlePermissionChange(tool.id, name, v)} size={14} /> + + {devMode && expandedSchema === schemaKey && schemaProps && ( + + Input Parameters + {Object.entries(schemaProps).map(([pName, pDef]: [string, any]) => ( + + {pName} + {pDef?.type || 'any'} + {schemaRequired.includes(pName) && } + {pDef?.description && {pDef.description}} + + ))} + + )} - handlePermissionChange(tool.id, name, v)} size={14} /> - - ))} + ); + })} )} {(data.write?.length || 0) > 0 && ( @@ -1152,15 +1341,36 @@ const Tools: React.FC = () => { handleGroupPermissionChange(tool.id, data.write!, v)} size={14} /> - {data.write!.map((name) => ( - - - {toDisplayName(name, serviceName)} - {descriptions[name] && {firstSentence(descriptions[name])}} + {data.write!.map((name) => { + const schemaKey = `${tool.id}:${name}`; + const schema = schemas[name]; + const schemaProps = schema?.properties as Record | undefined; + const schemaRequired = (schema?.required || []) as string[]; + return ( + + devMode && schema && setExpandedSchema((p) => p === schemaKey ? null : schemaKey)}> + + {toDisplayName(name, serviceName)} + {descriptions[name] && {firstSentence(descriptions[name])}} + + handlePermissionChange(tool.id, name, v)} size={14} /> + + {devMode && expandedSchema === schemaKey && schemaProps && ( + + Input Parameters + {Object.entries(schemaProps).map(([pName, pDef]: [string, any]) => ( + + {pName} + {pDef?.type || 'any'} + {schemaRequired.includes(pName) && } + {pDef?.description && {pDef.description}} + + ))} + + )} - handlePermissionChange(tool.id, name, v)} size={14} /> - - ))} + ); + })} )} @@ -1202,7 +1412,7 @@ const Tools: React.FC = () => { )} {ig && ( - } label="docs" size="small" sx={{ bgcolor: c.bg.secondary, color: c.text.ghost, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, fontSize: 10 } }} /> + } label="docs" size="small" sx={{ bgcolor: c.bg.secondary, color: c.text.ghost, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, fontSize: 10 } }} /> )} {tool.description && {tool.description}} @@ -1330,6 +1540,46 @@ const Tools: React.FC = () => { ))} )} + + {devMode && isMcp && ( + + + Developer Info + + + + MCP Config + + + {JSON.stringify(tool.mcp_config, null, 2)} + + + + + Auth type: + {tool.auth_type || 'none'} + + + Status: + {tool.auth_status || 'none'} + + {tool.connected_account_email && ( + + Account: + {tool.connected_account_email} + + )} + + {tool.credentials && Object.keys(tool.credentials).length > 0 && ( + + Credentials: + {Object.keys(tool.credentials).map((key) => ( + + ))} + + )} + + )} @@ -1366,17 +1616,24 @@ const Tools: React.FC = () => { MCP Registry {regStats && ( - + <> + + {devMode && regStats.lastUpdated > 0 && ( + + Synced {Math.round((Date.now() / 1000 - regStats.lastUpdated) / 60)}m ago + + )} + )} { {regServers.map((srv) => { const isExpanded = expandedServer === srv.name; + const isInstalled = allTools.some((t) => t.name === (srv.title || cleanServerName(srv.name))); return ( setExpandedServer(isExpanded ? null : srv.name)} + onClick={() => { + const next = isExpanded ? null : srv.name; + setExpandedServer(next); + if (next && devMode) { + dispatch(clearDetail()); + dispatch(fetchServerDetail(srv.name)); + } + }} sx={{ display: 'flex', alignItems: 'center', gap: 1.5, px: 1.5, py: 1, borderRadius: 1.5, cursor: 'pointer', @@ -1504,6 +1769,12 @@ const Tools: React.FC = () => { )} + {devMode && !srv.remoteType && ( + + )} + {isInstalled && ( + } label="Installed" size="small" sx={{ bgcolor: `${c.status.success}15`, color: c.status.success, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, color: c.status.success } }} /> + )} {srv.description} @@ -1533,8 +1804,6 @@ const Tools: React.FC = () => { } label="Website" @@ -1546,8 +1815,6 @@ const Tools: React.FC = () => { } label="Repository" @@ -1558,6 +1825,46 @@ const Tools: React.FC = () => { + {devMode && ( + + {regDetailLoading && expandedServer === srv.name ? ( + + ) : regDetail && regDetail.name === srv.name ? ( + + {(regDetail.keywords?.length > 0 || regDetail.license) && ( + + {regDetail.license && ( + + )} + {regDetail.keywords?.map((kw) => ( + + ))} + + )} + {regDetail.environmentVariables?.length > 0 && ( + + + Required Environment Variables + + {regDetail.environmentVariables.map((ev) => ( + + + {ev.name} + + {ev.description && ( + + {ev.description} + + )} + + ))} + + )} + + ) : null} + + )} +