diff --git a/.gitignore b/.gitignore index 47256d70..76fb66c7 100644 --- a/.gitignore +++ b/.gitignore @@ -8,7 +8,9 @@ backend/data/** # Electron build artifacts electron/dist/ electron/python-env/ +electron/build-staging/ electron/node_modules/ +electron/package-lock.json # Frontend build output frontend/dist/ \ No newline at end of file diff --git a/README.md b/README.md index c9a6122b..73ad2cfb 100644 --- a/README.md +++ b/README.md @@ -229,14 +229,6 @@ Please open an issue first for larger changes so we can discuss the approach.
-## Community - -- [Twitter / X](https://twitter.com/openswarm_ai) -- [Discord](https://discord.gg/openswarm) -- [Website](https://openswarm.ai) - -
- ## License MIT — see [LICENSE](LICENSE) for details. diff --git a/backend/.env.example b/backend/.env.example index ded015af..c6ae0550 100644 --- a/backend/.env.example +++ b/backend/.env.example @@ -17,3 +17,25 @@ APPLE_TEAM_ID=ABCDE12345 # GitHub Releases (required for --publish) # ============================================================================= GH_TOKEN=ghp_your-github-personal-access-token + +# ============================================================================= +# Channels: SMS, WhatsApp, Voice Calling +# ============================================================================= +TWILIO_ACCOUNT_SID=your-twilio-account-sid +TWILIO_AUTH_TOKEN=your-twilio-auth-token +TWILIO_PHONE_NUMBER=+1234567890 + +TELNYX_API_KEY=your-telnyx-api-key +TELNYX_PUBLIC_KEY=your-telnyx-public-key + +# ============================================================================= +# TTS / STT Providers +# ============================================================================= +ELEVENLABS_API_KEY=your-elevenlabs-api-key +DEEPGRAM_API_KEY=your-deepgram-api-key +OPENAI_API_KEY=your-openai-api-key + +# ============================================================================= +# Webhook URL (required for inbound SMS/calls — use ngrok or Tailscale) +# ============================================================================= +WEBHOOK_BASE_URL=https://your-public-url.ngrok.io diff --git a/backend/apps/agents/agent_loop.py b/backend/apps/agents/agent_loop.py new file mode 100644 index 00000000..92410408 --- /dev/null +++ b/backend/apps/agents/agent_loop.py @@ -0,0 +1,331 @@ +"""Owned agent loop — replaces claude_agent_sdk's query() function. + +Generalizes the pattern from browser_agent.py (lines 243-334) into a +provider-agnostic, streaming, HITL-aware tool-use loop. +""" + +from __future__ import annotations + +import json +import logging +import time +from typing import Any, Callable, Awaitable +from uuid import uuid4 + +from backend.apps.agents.providers.base import ( + BaseProvider, ContentBlock, ModelResponse, ProviderMessage, + StreamEvent, ToolCall, ToolSchema, +) + +logger = logging.getLogger(__name__) + +# Type aliases for callbacks +ToolExecutor = Callable[[str, dict], Awaitable[list[dict]]] +# hitl_handler(tool_name, tool_input) -> (approved, updated_input_or_None) +HITLHandler = Callable[[str, dict], Awaitable[tuple[bool, dict | None]]] +# ws_emitter(event_type, data) -> None +WSEmitter = Callable[[str, dict], Awaitable[None]] + + +class AgentLoop: + """Provider-agnostic agent loop with streaming and HITL support. + + The loop: + 1. Sends user message to the model + 2. Streams the response (emitting WebSocket events) + 3. If the model requests tool use: + a. For each tool call: check HITL permission → execute → collect result + b. Append tool results → go to step 2 + 4. If the model stops (end_turn/max_tokens): done + """ + + def __init__( + self, + session_id: str, + provider: BaseProvider, + model: str, + system_prompt: str | None, + tools: list[ToolSchema], + tool_executor: ToolExecutor, + hitl_handler: HITLHandler, + ws_emitter: WSEmitter, + max_turns: int | None = None, + cwd: str | None = None, + ): + self.session_id = session_id + self.provider = provider + self.model = model + self.system_prompt = system_prompt + self.tools = tools + self.tool_executor = tool_executor + self.hitl_handler = hitl_handler + self.ws_emitter = ws_emitter + self.max_turns = max_turns + self.cwd = cwd + + # Conversation history in provider-agnostic format + self.messages: list[ProviderMessage] = [] + + # Token tracking + self.total_input_tokens = 0 + self.total_output_tokens = 0 + + async def run(self, user_content: Any) -> None: + """Run the agent loop for a single user turn.""" + # Append user message + user_msg = self.provider.format_user_message(user_content) + self.messages.append(user_msg) + + turn = 0 + while True: + if self.max_turns and turn >= self.max_turns: + logger.info(f"Agent {self.session_id}: max turns ({self.max_turns}) reached") + break + turn += 1 + + # Stream the model response and collect it + response = await self._stream_and_collect() + + # Track usage + self.total_input_tokens += response.usage.get("input_tokens", 0) + self.total_output_tokens += response.usage.get("output_tokens", 0) + + # Append assistant message to conversation history + assistant_msg = self.provider.format_assistant_message(response) + self.messages.append(assistant_msg) + + # If no tool use, we're done + if response.stop_reason != "tool_use": + break + + # Execute tools + tool_results = await self._execute_tools(response) + if not tool_results: + break + + # Append tool results + self.messages.append(ProviderMessage(role="tool_result", content=tool_results)) + + async def _stream_and_collect(self) -> ModelResponse: + """Stream model output, emit WebSocket events, collect full response.""" + collected_content: list[ContentBlock] = [] + collected_usage: dict[str, int] = {} + stop_reason = "end_turn" + + # Track streaming state for WS emissions + stream_text_msg_id: str | None = None + stream_tool_msg_ids: dict[int, str] = {} # block index -> msg_id + block_index_map: dict[int, str] = {} # block index -> msg_id + + # Buffers for collecting content + text_buffers: dict[int, str] = {} + json_buffers: dict[int, str] = {} + tool_names: dict[int, str] = {} + tool_ids: dict[int, str] = {} + block_types: dict[int, str] = {} + + async for event in self.provider.stream_message( + model=self.model, + system=self.system_prompt, + messages=self.messages, + tools=self.tools, + ): + if event.type == "content_block_start": + if event.block_type == "text": + if stream_text_msg_id is None: + stream_text_msg_id = uuid4().hex + await self.ws_emitter("agent:stream_start", { + "message_id": stream_text_msg_id, + "role": "assistant", + }) + block_index_map[event.index] = stream_text_msg_id + block_types[event.index] = "text" + text_buffers[event.index] = "" + + elif event.block_type == "tool_use": + tool_msg_id = uuid4().hex + stream_tool_msg_ids[event.index] = tool_msg_id + block_index_map[event.index] = tool_msg_id + block_types[event.index] = "tool_use" + tool_names[event.index] = event.tool_name + tool_ids[event.index] = event.tool_id + json_buffers[event.index] = "" + + await self.ws_emitter("agent:stream_start", { + "message_id": tool_msg_id, + "role": "tool_call", + "tool_name": event.tool_name, + }) + + elif event.type == "content_block_delta": + msg_id = block_index_map.get(event.index) + if not msg_id: + continue + + if event.delta_type == "text_delta": + text_buffers.setdefault(event.index, "") + text_buffers[event.index] += event.text + await self.ws_emitter("agent:stream_delta", { + "message_id": msg_id, + "delta": event.text, + }) + + elif event.delta_type == "input_json_delta": + json_buffers.setdefault(event.index, "") + json_buffers[event.index] += event.text + await self.ws_emitter("agent:stream_delta", { + "message_id": msg_id, + "delta": event.text, + }) + + elif event.type == "content_block_stop": + msg_id = block_index_map.get(event.index) + bt = block_types.get(event.index, "") + + if bt == "text": + collected_content.append( + ContentBlock(type="text", text=text_buffers.get(event.index, "")) + ) + elif bt == "tool_use": + try: + tool_input = json.loads(json_buffers.get(event.index, "{}")) + except json.JSONDecodeError: + tool_input = {} + collected_content.append(ContentBlock( + type="tool_use", + tool_call=ToolCall( + id=tool_ids.get(event.index, uuid4().hex), + name=tool_names.get(event.index, ""), + input=tool_input, + ), + )) + + # Send stream_end for tool blocks (text block ends at message_stop) + if msg_id and bt == "tool_use": + await self.ws_emitter("agent:stream_end", { + "message_id": msg_id, + }) + + elif event.type == "usage": + # Accumulate token usage from provider stream + for k, v in event.usage.items(): + collected_usage[k] = collected_usage.get(k, 0) + v + + elif event.type == "message_stop": + # Check if any tool calls means stop_reason is tool_use + has_tool_use = any(b.type == "tool_use" for b in collected_content) + if has_tool_use: + stop_reason = "tool_use" + + # End text stream + if stream_text_msg_id: + await self.ws_emitter("agent:stream_end", { + "message_id": stream_text_msg_id, + }) + + # Build and emit the collected messages + await self._emit_collected_messages( + collected_content, stream_text_msg_id, stream_tool_msg_ids, + ) + + return ModelResponse( + content=collected_content, + stop_reason=stop_reason, + usage=collected_usage, + ) + + async def _emit_collected_messages( + self, + content: list[ContentBlock], + text_msg_id: str | None, + tool_msg_ids: dict[int, str], + ) -> None: + """Emit finalized agent:message events for the collected response.""" + from backend.apps.agents.models import Message + + # Emit text message + text_parts = [b.text for b in content if b.type == "text" and b.text] + if text_parts: + msg = Message( + id=text_msg_id or uuid4().hex, + role="assistant", + content="\n".join(text_parts), + ) + await self.ws_emitter("agent:message", { + "message": msg.model_dump(mode="json"), + }) + + # Emit tool call messages + tool_blocks = [b for b in content if b.type == "tool_use" and b.tool_call] + tool_id_list = sorted(tool_msg_ids.items(), key=lambda x: x[0]) + for i, block in enumerate(tool_blocks): + tc = block.tool_call + msg_id = tool_id_list[i][1] if i < len(tool_id_list) else uuid4().hex + msg = Message( + id=msg_id, + role="tool_call", + content={ + "id": tc.id, + "tool": tc.name, + "input": tc.input, + }, + ) + await self.ws_emitter("agent:message", { + "message": msg.model_dump(mode="json"), + }) + + async def _execute_tools(self, response: ModelResponse) -> list[dict]: + """Execute all tool calls from a response, respecting HITL permissions. + + Returns a list of tool result dicts formatted for the provider. + """ + from backend.apps.agents.models import Message + + results = [] + for block in response.content: + if block.type != "tool_use" or not block.tool_call: + continue + + tc = block.tool_call + start_time = time.time() + + # HITL permission check + approved, updated_input = await self.hitl_handler(tc.name, tc.input) + + if not approved: + result_content = [{"type": "text", "text": "Tool use was denied by the user."}] + else: + tool_input = updated_input if updated_input else tc.input + try: + result_content = await self.tool_executor(tc.name, tool_input) + except Exception as e: + logger.warning(f"Tool execution error: {tc.name}: {e}") + result_content = [{"type": "text", "text": f"Error executing {tc.name}: {e}"}] + + elapsed_ms = int((time.time() - start_time) * 1000) + + # Emit tool result to frontend + result_text = "" + for block_item in result_content: + if isinstance(block_item, dict) and block_item.get("type") == "text": + result_text = block_item.get("text", "") + break + + result_msg = Message( + role="tool_result", + content={ + "text": result_text[:15000] if result_text else "Done.", + "tool_name": tc.name, + "elapsed_ms": elapsed_ms, + }, + ) + await self.ws_emitter("agent:message", { + "message": result_msg.model_dump(mode="json"), + }) + + # Format for provider + results.append( + self.provider.format_tool_result(tc.id, result_content) + ) + + return results diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index c92b6b10..04e5c472 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -23,10 +23,34 @@ from backend.apps.tools_lib.tools_lib import ( refresh_google_token, ) from backend.config.paths import SESSIONS_DIR +from backend.apps.analytics.collector import record as _analytics logger = logging.getLogger(__name__) -os.environ.setdefault("CLAUDE_CODE_STREAM_CLOSE_TIMEOUT", "3600000") +_TOOL_CATEGORIES: dict[str, str] = { + "gmail": "email", "calendar": "calendar", "drive": "files", + "sheet": "files", "doc": "files", "slide": "files", + "tweet": "social", "twitter": "social", "reddit": "social", + "Bash": "coding", "Edit": "coding", "Write": "coding", "Read": "coding", + "Glob": "coding", "Grep": "coding", + "BrowserAgent": "browsing", "BrowserAgents": "browsing", + "WebSearch": "research", "WebFetch": "research", +} + + +def _infer_task_category(tool_names: list[str]) -> str: + """Infer what the user was doing from the tools used.""" + if not tool_names: + return "chat" + counts: dict[str, int] = {} + for name in tool_names: + for keyword, category in _TOOL_CATEGORIES.items(): + if keyword.lower() in name.lower(): + counts[category] = counts.get(category, 0) + 1 + break + if not counts: + return "other" + return max(counts, key=counts.get) def _save_session(session_id: str, doc_data: dict): @@ -66,6 +90,8 @@ FULL_TOOLS = [ "TaskOutput", "TaskStop", "CronCreate", "CronList", "CronDelete", "RenderOutput", + "InvokeAgent", + "Agent", ] def _get_denied_tool_names(tool) -> set[str]: @@ -232,8 +258,12 @@ class AgentManager: + "\n" ) - def _build_browser_context(self, dashboard_id: str | None) -> str | None: - """Build a context block listing browser cards and delegation instructions.""" + def _build_browser_context(self, dashboard_id: str | None, selected_browser_ids: list[str] | None = None) -> str | None: + """Build a context block listing browser cards and delegation instructions. + + Only browser cards explicitly selected by the user are included. + If none are selected, no browser card details are exposed. + """ if not dashboard_id: return None try: @@ -246,34 +276,39 @@ class AgentManager: lines = [ "", - "You have access to browser automation through the BrowserAgent and BrowserAgents tools.", + "You have access to browser automation through the CreateBrowserAgent, BrowserAgent, and BrowserAgents tools.", "", - "- **BrowserAgent(task, browser_id?, url?)**: Delegate a single browser task to a dedicated browser agent. " + "- **CreateBrowserAgent(task, url?)**: Create a new browser card and run a task on it. " + "Use this when you need a fresh browser. Optionally provide a starting URL.", + "- **BrowserAgent(browser_id, task)**: Delegate a task to an existing browser card. " "The browser agent will autonomously navigate, click, type, and interact with the page, then return a summary and screenshot.", - "- **BrowserAgents(tasks)**: Run multiple browser tasks in parallel, each on a different browser.", + "- **BrowserAgents(tasks)**: Run multiple browser tasks in parallel on existing browser cards. " + "Each task requires a browser_id.", "", "You do NOT have direct access to low-level browser tools (click, type, screenshot, etc.). " "Instead, describe what you want accomplished and the browser agent will handle the details.", - "", - "If you omit browser_id, a new browser card will be auto-created. " - "If you provide a url without a browser_id, the new browser navigates there first.", ] - if browser_cards: - lines.append("") - lines.append("Available browser cards on the dashboard:") - for card in browser_cards.values(): - bid = card.get("browser_id", "") - tabs = card.get("tabs", []) - active_tab_id = card.get("activeTabId", "") - active_tab = next((t for t in tabs if t.get("id") == active_tab_id), None) - url = (active_tab or {}).get("url", card.get("url", "")) - title = (active_tab or {}).get("title", "") - lines.append(f"- browser_id: \"{bid}\"") - if title: - lines.append(f" Title: {title}") - if url: - lines.append(f" URL: {url}") + if browser_cards and selected_browser_ids: + visible_cards = [ + card for card in browser_cards.values() + if card.get("browser_id", "") in selected_browser_ids + ] + if visible_cards: + lines.append("") + lines.append("The user selected these browser cards for you to work with:") + for card in visible_cards: + bid = card.get("browser_id", "") + tabs = card.get("tabs", []) + active_tab_id = card.get("activeTabId", "") + active_tab = next((t for t in tabs if t.get("id") == active_tab_id), None) + url = (active_tab or {}).get("url", card.get("url", "")) + title = (active_tab or {}).get("title", "") + lines.append(f"- browser_id: \"{bid}\"") + if title: + lines.append(f" Title: {title}") + if url: + lines.append(f" URL: {url}") lines.append("") return "\n".join(lines) @@ -317,6 +352,7 @@ class AgentManager: session = AgentSession( id=session_id, name=config.name, + provider=config.provider, model=config.model, mode=config.mode, system_prompt=config.system_prompt, @@ -326,13 +362,20 @@ class AgentManager: dashboard_id=config.dashboard_id, ) self.sessions[session_id] = session - + + _analytics("session.started", { + "model": session.model, + "provider": session.provider, + "mode": session.mode, + "tool_count": len(tools), + }, session_id=session_id, dashboard_id=config.dashboard_id) + await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, "status": "running", "session": session.model_dump(mode="json"), }) - + return session def _resolve_context_paths(self, context_paths: list | None) -> str: @@ -456,83 +499,75 @@ class AgentManager: }) return content - async def _run_agent_loop(self, session_id: str, prompt: str, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None): - """Run the Claude Agent SDK query loop for a session.""" + async def _run_agent_loop(self, session_id: str, prompt: str, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None, fork_session: bool = False, selected_browser_ids: list[str] | None = None): + """Run the owned agent loop for a session (multi-provider).""" + from backend.apps.agents.agent_loop import AgentLoop + from backend.apps.agents.mcp_client import MCPClientManager + from backend.apps.agents.providers.registry import create_provider, calculate_cost + from backend.apps.settings.credentials import validate_credentials + session = self.sessions.get(session_id) if not session: return - + prompt_content = self._build_prompt_content(prompt, images, context_paths, forced_tools, attached_skills) - try: - from claude_agent_sdk import ( - query, ClaudeAgentOptions, AssistantMessage, ResultMessage, - ) - from claude_agent_sdk.types import ( - HookMatcher, PermissionResultAllow, PermissionResultDeny, - TextBlock, ToolUseBlock, StreamEvent, - ) - except ImportError: - logger.warning("claude_agent_sdk not installed, running in mock mode") - await self._run_mock_agent(session_id, prompt) - return - session.status = "running" - + _builtin_perms = load_builtin_permissions() - def _check_tool_permission(tool_name: str) -> str | None: - """Check tool permissions for both builtin and MCP tools. - Returns 'always_allow', 'deny', or None (ask).""" + def _get_effective_policy(tool_name: str) -> str: + """Return 'always_allow', 'deny', or 'ask' for any tool.""" if tool_name in _builtin_perms: - policy = _builtin_perms[tool_name] - if policy in ("always_allow", "deny"): - return policy - return None + return _builtin_perms[tool_name] import re as _re + + bm = _re.match(r"mcp__openswarm-browser-agent__(.+)", tool_name) + if bm: + return _builtin_perms.get(bm.group(1), "always_allow") + + im = _re.match(r"mcp__openswarm-invoke-agent__(.+)", tool_name) + if im: + return _builtin_perms.get(im.group(1), "always_allow") + m = _re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name) - if not m: - return None - server_slug, mcp_tool_name = m.group(1), m.group(2) - for t in load_all_tools(): - if not t.mcp_config or not t.enabled: - continue - if _sanitize_server_name(t.name) == server_slug: - policy = t.tool_permissions.get(mcp_tool_name, "ask") - if policy in ("always_allow", "deny"): - return policy - return None - return None - - async def can_use_tool(tool_name, input_data, context): - if tool_name != "AskUserQuestion": - policy = _check_tool_permission(tool_name) - if policy == "always_allow": - return PermissionResultAllow(updated_input=input_data) - if policy == "deny": - return PermissionResultDeny(message="Tool denied by permission policy") + if m: + server_slug, mcp_tool_name = m.group(1), m.group(2) + for t in load_all_tools(): + if not t.mcp_config or not t.enabled: + continue + if _sanitize_server_name(t.name) == server_slug: + return t.tool_permissions.get(mcp_tool_name, "ask") + return "always_allow" + async def _request_user_approval(tool_name: str, tool_input) -> dict: + """Send an approval request via WebSocket and wait for the user's decision.""" + safe_input = tool_input if isinstance(tool_input, dict) else {} request_id = uuid4().hex approval_req = ApprovalRequest( id=request_id, session_id=session_id, tool_name=tool_name, - tool_input=input_data if isinstance(input_data, dict) else {}, + tool_input=safe_input, ) session.pending_approvals.append(approval_req) session.status = "waiting_approval" - + await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, "status": "waiting_approval", }) - + decision = await ws_manager.send_approval_request( - session_id, request_id, tool_name, - input_data if isinstance(input_data, dict) else {} + session_id, request_id, tool_name, safe_input ) - + + _analytics("tool.approval_resolved", { + "tool_name": tool_name, + "decision": decision.get("behavior", "deny"), + }, session_id=session_id) + session.pending_approvals = [ a for a in session.pending_approvals if a.id != request_id ] @@ -541,269 +576,216 @@ class AgentManager: "session_id": session_id, "status": "running", }) - - if decision.get("behavior") == "allow": - return PermissionResultAllow( - updated_input=decision.get("updated_input", input_data) - ) - else: - return PermissionResultDeny( - message=decision.get("message", "User denied this action") - ) + return decision - tool_start_times: dict[str, float] = {} - - async def pre_tool_hook(input_data, tool_use_id, context): - if tool_use_id: - tool_start_times[tool_use_id] = time.time() - return {"continue_": True} - - async def post_tool_hook(input_data, tool_use_id, context): - elapsed_ms = None - if tool_use_id and tool_use_id in tool_start_times: - elapsed_ms = int((time.time() - tool_start_times.pop(tool_use_id)) * 1000) - - raw_response = input_data.get("tool_response", "") - - if isinstance(raw_response, list) and raw_response: - text_parts = [ - block.get("text", "") - for block in raw_response - if isinstance(block, dict) and block.get("type") == "text" - ] - if text_parts: - raw_response = "\n".join(text_parts) if len(text_parts) > 1 else text_parts[0] - - if isinstance(raw_response, str): - content = raw_response - else: - try: - import json as _json - content = _json.dumps(raw_response, indent=2, default=str) - except Exception: - content = str(raw_response) - - result_payload = {"text": content} - hook_tool_name = input_data.get("tool_name", "") - if hook_tool_name: - result_payload["tool_name"] = hook_tool_name - if elapsed_ms is not None: - result_payload["elapsed_ms"] = elapsed_ms - - result_msg = Message(role="tool_result", content=result_payload) - session.messages.append(result_msg) - await ws_manager.send_to_session(session_id, "agent:message", { - "session_id": session_id, - "message": result_msg.model_dump(mode="json"), - }) - return {"continue_": True} + mcp_manager: MCPClientManager | None = None try: _, mode_sys_prompt, _ = self._resolve_mode(session.mode) connected_tools_ctx = self._build_connected_tools_context(session.allowed_tools) outputs_ctx = self._build_outputs_context() - browser_ctx = self._build_browser_context(session.dashboard_id) + browser_ctx = self._build_browser_context(session.dashboard_id, selected_browser_ids=selected_browser_ids) global_settings = load_settings() composed_prompt = self._compose_system_prompt(global_settings.default_system_prompt, mode_sys_prompt, session.system_prompt, connected_tools_ctx, outputs_ctx, browser_ctx) + if session.mode == "view-builder": + from backend.apps.outputs.view_builder_templates import VIEW_BUILDER_SKILL + skill_block = f"\n{VIEW_BUILDER_SKILL}\n" + composed_prompt = f"{composed_prompt}\n\n{skill_block}" if composed_prompt else skill_block + + # Validate credentials for the provider + validate_credentials(global_settings, session.provider) + + # Create the provider adapter + provider = create_provider(session.provider, global_settings) + + # Build MCP servers config mcp_servers = await self._build_mcp_servers(session.allowed_tools) - browser_agent_server_path = os.path.join( - os.path.dirname(__file__), "browser_agent_mcp_server.py" + _browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents"] + _browser_all_denied = all( + _builtin_perms.get(t, "always_allow") == "deny" + for t in _browser_delegation_tools ) - backend_port = os.environ.get("OPENSWARM_PORT", "8324") - pre_selected_bids = self._get_pre_selected_browser_ids(session.dashboard_id) - mcp_servers["openswarm-browser-agent"] = { - "command": sys.executable, - "args": [browser_agent_server_path], - "env": { - "OPENSWARM_PORT": backend_port, - "OPENSWARM_AGENT_MODEL": session.model, - "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", - "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), - }, - "type": "stdio", - } - effective_allowed = [ - t for t in session.allowed_tools - if _builtin_perms.get(t, "always_allow") == "always_allow" - ] - if mcp_servers: - all_tools_list = load_all_tools() - for name in mcp_servers: - tool_def = next( - (t for t in all_tools_list - if t.mcp_config and t.enabled and _sanitize_server_name(t.name) == name), - None, - ) - if tool_def: - denied = _get_denied_tool_names(tool_def) - known = _get_all_known_tool_names(tool_def) - for tn in known - denied: - policy = tool_def.tool_permissions.get(tn, "ask") - if policy == "always_allow": - effective_allowed.append(f"mcp__{name}__{tn}") - else: - effective_allowed.append(f"mcp__{name}__*") - - effective_allowed.append("mcp__openswarm-browser-agent__*") - - options_kwargs = { - "model": session.model, - "max_buffer_size": 5 * 1024 * 1024, - "can_use_tool": can_use_tool, - "hooks": { - "PreToolUse": [HookMatcher(matcher=None, hooks=[pre_tool_hook])], - "PostToolUse": [HookMatcher(matcher=None, hooks=[post_tool_hook])], - }, - "allowed_tools": effective_allowed, - "include_partial_messages": True, - } - if not global_settings.anthropic_api_key: - raise ValueError("Anthropic API key not configured. Set it in Settings.") - options_kwargs["env"] = {"ANTHROPIC_API_KEY": global_settings.anthropic_api_key} - if mcp_servers: - options_kwargs["mcp_servers"] = mcp_servers - if composed_prompt: - options_kwargs["system_prompt"] = composed_prompt - if session.max_turns: - options_kwargs["max_turns"] = session.max_turns - - if session.cwd: - options_kwargs["cwd"] = session.cwd - - if session.sdk_session_id: - options_kwargs["resume"] = session.sdk_session_id - - options = ClaudeAgentOptions(**options_kwargs) - - async def prompt_stream(): - yield { - "type": "user", - "message": {"role": "user", "content": prompt_content}, + if not _browser_all_denied: + browser_agent_server_path = os.path.join( + os.path.dirname(__file__), "browser_agent_mcp_server.py" + ) + backend_port = os.environ.get("OPENSWARM_PORT", "8324") + pre_selected_bids = self._get_pre_selected_browser_ids(session.dashboard_id) + mcp_servers["openswarm-browser-agent"] = { + "command": sys.executable, + "args": [browser_agent_server_path], + "env": { + "OPENSWARM_PORT": backend_port, + "OPENSWARM_AGENT_MODEL": session.model, + "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", + "OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids), + "OPENSWARM_PARENT_SESSION_ID": session.id, + }, + "type": "stdio", } - stream_text_msg_id = None - stream_tool_msg_ids_ordered = [] - stream_block_index_map = {} + _invoke_agent_tools = ["InvokeAgent"] + _invoke_all_denied = all( + _builtin_perms.get(t, "always_allow") == "deny" + for t in _invoke_agent_tools + ) - async for message in query( - prompt=prompt_stream(), - options=options, - ): - if isinstance(message, StreamEvent): - event = message.event - event_type = event.get("type") + if not _invoke_all_denied: + invoke_agent_server_path = os.path.join( + os.path.dirname(__file__), "invoke_agent_mcp_server.py" + ) + backend_port = os.environ.get("OPENSWARM_PORT", "8324") + mcp_servers["openswarm-invoke-agent"] = { + "command": sys.executable, + "args": [invoke_agent_server_path], + "env": { + "OPENSWARM_PORT": backend_port, + "OPENSWARM_PARENT_SESSION_ID": session.id, + "OPENSWARM_DASHBOARD_ID": session.dashboard_id or "", + }, + "type": "stdio", + } - if event_type == "content_block_start": - block = event.get("content_block", {}) - index = event.get("index") - block_type = block.get("type") + # Connect MCP servers in parallel with timeouts — get real schemas + mcp_manager = MCPClientManager() + await mcp_manager.__aenter__() - if block_type == "text": - if stream_text_msg_id is None: - stream_text_msg_id = uuid4().hex - await ws_manager.send_to_session(session_id, "agent:stream_start", { - "session_id": session_id, - "message_id": stream_text_msg_id, - "role": "assistant", - }) - stream_block_index_map[index] = stream_text_msg_id + from backend.apps.agents.providers.base import ToolSchema - elif block_type == "tool_use": - tool_msg_id = uuid4().hex - stream_tool_msg_ids_ordered.append(tool_msg_id) - stream_block_index_map[index] = tool_msg_id - await ws_manager.send_to_session(session_id, "agent:stream_start", { - "session_id": session_id, - "message_id": tool_msg_id, - "role": "tool_call", - "tool_name": block.get("name", ""), - }) + _pending_mcp_servers: dict[str, dict] = dict(mcp_servers) + _connected_servers: set[str] = set() - elif event_type == "content_block_delta": - index = event.get("index") - delta = event.get("delta", {}) - delta_type = delta.get("type") - msg_id = stream_block_index_map.get(index) + async def _connect_one(name: str, config: dict) -> list[ToolSchema]: + try: + tools = await asyncio.wait_for(mcp_manager.connect(name, config), timeout=10.0) + _connected_servers.add(name) + return tools + except asyncio.TimeoutError: + logger.warning(f"MCP server {name} timed out (10s)") + return [] + except Exception as e: + logger.warning(f"MCP server {name} failed: {e}") + return [] - if msg_id and delta_type == "text_delta": - await ws_manager.send_to_session(session_id, "agent:stream_delta", { - "session_id": session_id, - "message_id": msg_id, - "delta": delta.get("text", ""), - }) - elif msg_id and delta_type == "input_json_delta": - await ws_manager.send_to_session(session_id, "agent:stream_delta", { - "session_id": session_id, - "message_id": msg_id, - "delta": delta.get("partial_json", ""), - }) + results = await asyncio.gather( + *[_connect_one(n, c) for n, c in mcp_servers.items()] + ) + mcp_tool_schemas: list[ToolSchema] = [] + for tools in results: + mcp_tool_schemas.extend(tools) - elif event_type == "content_block_stop": - index = event.get("index") - msg_id = stream_block_index_map.get(index) - if msg_id and msg_id != stream_text_msg_id: - await ws_manager.send_to_session(session_id, "agent:stream_end", { - "session_id": session_id, - "message_id": msg_id, - }) + # Collect builtin + MCP tool schemas + from backend.apps.agents.tools.registry import get_all_tool_schemas + all_tool_schemas = list(get_all_tool_schemas()) + list(mcp_tool_schemas) - elif event_type == "message_stop": - if stream_text_msg_id: - await ws_manager.send_to_session(session_id, "agent:stream_end", { - "session_id": session_id, - "message_id": stream_text_msg_id, - }) + # HITL handler for the agent loop + async def hitl_handler(tool_name: str, tool_input: dict) -> tuple[bool, dict | None]: + policy = _get_effective_policy(tool_name) + if policy == "always_allow": + return True, None + if policy == "deny": + return False, None + # policy == "ask" + decision = await _request_user_approval(tool_name, tool_input) + if decision.get("behavior") == "allow": + return True, decision.get("updated_input") + return False, None - elif isinstance(message, AssistantMessage): - content_parts = [] - tool_uses = [] - for block in message.content: - if isinstance(block, TextBlock): - content_parts.append(block.text) - elif isinstance(block, ToolUseBlock): - tool_uses.append({ - "id": block.id, - "tool": block.name, - "input": block.input, - }) + # Tool executor — routes to builtins or MCP servers + async def tool_executor(tool_name: str, tool_input: dict) -> list[dict]: + t0 = time.time() - if content_parts: - asst_msg = Message( - id=stream_text_msg_id or uuid4().hex, - role="assistant", - content="\n".join(content_parts), - ) - session.messages.append(asst_msg) - await ws_manager.send_to_session(session_id, "agent:message", { - "session_id": session_id, - "message": asst_msg.model_dump(mode="json"), - }) + # Check builtin tools first + from backend.apps.agents.tools.registry import get_tool + from backend.apps.agents.tools.base import ToolContext + builtin = get_tool(tool_name) + if builtin: + ctx = ToolContext(cwd=session.cwd or os.path.expanduser("~"), session_id=session_id) + result = await builtin.execute(tool_input, ctx) + elapsed_ms = int((time.time() - t0) * 1000) + _analytics("tool.called", { + "tool_name": tool_name, + "tool_type": "builtin", + "elapsed_ms": elapsed_ms, + "input_summary": str(tool_input)[:200], + }, session_id=session_id) + return result - for i, tu in enumerate(tool_uses): - msg_id = stream_tool_msg_ids_ordered[i] if i < len(stream_tool_msg_ids_ordered) else uuid4().hex - tool_msg = Message(id=msg_id, role="tool_call", content=tu) - session.messages.append(tool_msg) - await ws_manager.send_to_session(session_id, "agent:message", { - "session_id": session_id, - "message": tool_msg.model_dump(mode="json"), - }) + # Check MCP tools + parsed = mcp_manager.parse_mcp_tool_name(tool_name) + if parsed: + server_name, bare_name = parsed - stream_text_msg_id = None - stream_tool_msg_ids_ordered = [] - stream_block_index_map = {} + # Lazy connect: if this server hasn't been connected yet, connect now + if server_name not in _connected_servers and server_name in _pending_mcp_servers: + try: + await asyncio.wait_for( + mcp_manager.connect(server_name, _pending_mcp_servers[server_name]), + timeout=15.0, + ) + _connected_servers.add(server_name) + except Exception as e: + logger.warning(f"Lazy MCP connect failed for {server_name}: {e}") + return [{"type": "text", "text": f"Failed to connect to {server_name}: {e}"}] - elif isinstance(message, ResultMessage): - session.sdk_session_id = getattr(message, "session_id", None) - cost = getattr(message, "total_cost_usd", None) - if cost is not None: - session.cost_usd = cost - await ws_manager.send_to_session(session_id, "agent:cost_update", { - "session_id": session_id, - "cost_usd": session.cost_usd, - }) + result = await mcp_manager.call_tool(server_name, bare_name, tool_input) + elapsed_ms = int((time.time() - t0) * 1000) + _analytics("tool.called", { + "tool_name": tool_name, + "tool_short_name": bare_name, + "tool_type": "mcp", + "mcp_server": server_name, + "elapsed_ms": elapsed_ms, + "input_summary": str(tool_input)[:200], + }, session_id=session_id) + return result + + return [{"type": "text", "text": f"Unknown tool: {tool_name}"}] + + # WebSocket emitter — captures messages into session + async def ws_emitter(event_type: str, data: dict) -> None: + data["session_id"] = session_id + await ws_manager.send_to_session(session_id, event_type, data) + # Capture finalized messages into session.messages + if event_type == "agent:message" and "message" in data: + msg_data = data["message"] + try: + msg = Message(**msg_data) + msg.branch_id = session.active_branch_id + session.messages.append(msg) + except Exception: + pass + + # Create and run the agent loop + loop = AgentLoop( + session_id=session_id, + provider=provider, + model=session.model, + system_prompt=composed_prompt, + tools=all_tool_schemas, + tool_executor=tool_executor, + hitl_handler=hitl_handler, + ws_emitter=ws_emitter, + max_turns=session.max_turns, + cwd=session.cwd, + ) + + await loop.run(prompt_content) + + # Update cost from token usage + session.tokens["input"] += loop.total_input_tokens + session.tokens["output"] += loop.total_output_tokens + session.cost_usd += calculate_cost( + session.provider, session.model, + loop.total_input_tokens, loop.total_output_tokens, + ) + await ws_manager.send_to_session(session_id, "agent:cost_update", { + "session_id": session_id, + "cost_usd": session.cost_usd, + }) session.status = "completed" except asyncio.CancelledError: @@ -811,14 +793,72 @@ class AgentManager: except Exception as e: logger.exception(f"Agent {session_id} error: {e}") session.status = "error" - error_msg = Message(role="system", content=f"Error: {str(e)}") + _analytics("error.occurred", { + "error_category": type(e).__name__, + "error_message": str(e)[:200], + }, session_id=session_id) + error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id) session.messages.append(error_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": error_msg.model_dump(mode="json"), }) finally: + if mcp_manager: + try: + await mcp_manager.__aexit__(None, None, None) + except Exception as e: + logger.warning(f"MCP cleanup error: {e}") if session_id in self.sessions: + # Track session completion + duration = 0.0 + if session.created_at: + duration = (datetime.now() - session.created_at).total_seconds() + msg_count = len([m for m in session.messages if m.role in ("user", "assistant")]) + tool_names = [m.content.get("tool", "") for m in session.messages if m.role == "tool_call" and isinstance(m.content, dict)] + task_category = _infer_task_category(tool_names) + + # Collect rich session data for analytics — full messages, no truncation + user_messages = [ + m.content if isinstance(m.content, str) else str(m.content) + for m in session.messages if m.role == "user" + ] + assistant_messages = [ + m.content if isinstance(m.content, str) else str(m.content) + for m in session.messages if m.role == "assistant" + ] + # Unique tools used with call counts + tool_call_counts: dict[str, int] = {} + for tn in tool_names: + short = tn.split("__")[-1] if "__" in tn else tn + tool_call_counts[short] = tool_call_counts.get(short, 0) + 1 + # MCP servers used + mcp_servers_used = list(set( + tn.split("__")[1] for tn in tool_names + if tn.startswith("mcp__") and len(tn.split("__")) >= 3 + )) + + _analytics("session.completed", { + "model": session.model, + "provider": session.provider, + "mode": session.mode, + "cost_usd": session.cost_usd, + "message_count": msg_count, + "duration_seconds": round(duration, 1), + "status": session.status, + "task_category": task_category, + "tool_count": len(tool_names), + # Rich data + "session_title": session.name, + "user_messages": user_messages, + "assistant_messages": assistant_messages, + "first_user_message": user_messages[0] if user_messages else "", + "tools_used": tool_call_counts, + "tools_list": list(tool_call_counts.keys()), + "mcp_servers_used": mcp_servers_used, + "system_prompt_length": len(session.system_prompt or ""), + }, session_id=session_id, dashboard_id=session.dashboard_id) + await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, "status": session.status, @@ -912,7 +952,7 @@ class AgentManager: session_id, tool_msg_id, "Bash", _json.dumps(tool_input_content["input"], indent=2), ) - tool_msg = Message(id=tool_msg_id, role="tool_call", content=tool_input_content) + tool_msg = Message(id=tool_msg_id, role="tool_call", content=tool_input_content, branch_id=session.active_branch_id) session.messages.append(tool_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -922,7 +962,7 @@ class AgentManager: await asyncio.sleep(1) if decision.get("behavior") == "allow": - tool_result = Message(role="tool_result", content=f"Processing: {prompt}") + tool_result = Message(role="tool_result", content=f"Processing: {prompt}", branch_id=session.active_branch_id) session.messages.append(tool_result) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -940,7 +980,7 @@ class AgentManager: asst_msg_id = uuid4().hex await self._stream_text(session_id, asst_msg_id, asst_text) - asst_msg = Message(id=asst_msg_id, role="assistant", content=asst_text) + asst_msg = Message(id=asst_msg_id, role="assistant", content=asst_text, branch_id=session.active_branch_id) session.messages.append(asst_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, @@ -965,21 +1005,27 @@ class AgentManager: prompt: str, mode: str | None = None, model: str | None = None, + provider: str | None = None, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None, + hidden: bool = False, + selected_browser_ids: list[str] | None = None, ): """Send a follow-up message to an existing session.""" session = self.sessions.get(session_id) if not session: raise ValueError(f"Session {session_id} not found") - + existing = self.tasks.get(session_id) if existing and not existing.done(): return session_changed = False + if provider and provider != session.provider: + session.provider = provider + session_changed = True if model and model != session.model: session.model = model session_changed = True @@ -1000,10 +1046,12 @@ class AgentManager: user_msg = Message( role="user", content=prompt, + branch_id=session.active_branch_id, context_paths=context_paths if context_paths else None, attached_skills=skill_meta, forced_tools=forced_tools if forced_tools else None, images=image_meta, + hidden=hidden, ) session.messages.append(user_msg) await ws_manager.send_to_session(session_id, "agent:message", { @@ -1011,11 +1059,18 @@ class AgentManager: "message": user_msg.model_dump(mode="json"), }) - task = asyncio.create_task(self._run_agent_loop(session_id, prompt, images=images, context_paths=context_paths, forced_tools=forced_tools, attached_skills=attached_skills)) + session.status = "running" + await ws_manager.send_to_session(session_id, "agent:status", { + "session_id": session_id, + "status": "running", + "session": session.model_dump(mode="json"), + }) + + task = asyncio.create_task(self._run_agent_loop(session_id, prompt, images=images, context_paths=context_paths, forced_tools=forced_tools, attached_skills=attached_skills, selected_browser_ids=selected_browser_ids)) self.tasks[session_id] = task async def stop_agent(self, session_id: str): - """Stop a running agent.""" + """Stop a running agent and all its browser-agent children.""" task = self.tasks.get(session_id) if task and not task.done(): task.cancel() @@ -1026,6 +1081,13 @@ class AgentManager: session = self.sessions.get(session_id) if session: + for req in list(session.pending_approvals): + ws_manager.resolve_approval(req.id, {"behavior": "deny", "message": "Agent stopped"}) + session.pending_approvals = [] + + if hasattr(session, '_cancel_event'): + session._cancel_event.set() + session.status = "stopped" await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, @@ -1033,6 +1095,13 @@ class AgentManager: "session": session.model_dump(mode="json"), }) + children = [ + s for s in self.sessions.values() + if s.parent_session_id == session_id and s.mode == "browser-agent" + ] + for child in children: + await self.stop_agent(child.id) + def handle_approval(self, request_id: str, decision: dict): """Resolve a pending HITL approval.""" ws_manager.resolve_approval(request_id, decision) @@ -1043,6 +1112,14 @@ class AgentManager: if not session: raise ValueError(f"Session {session_id} not found") + existing = self.tasks.get(session_id) + if existing and not existing.done(): + existing.cancel() + try: + await existing + except asyncio.CancelledError: + pass + target_msg = None for i, msg in enumerate(session.messages): if msg.id == message_id: @@ -1052,11 +1129,24 @@ class AgentManager: if not target_msg or target_msg.role != "user": raise ValueError("Can only edit user messages") - new_branch_id = uuid4().hex[:8] + fork_point_id = message_id + fork_parent_branch = target_msg.branch_id + + msg_branch = session.branches.get(target_msg.branch_id) + if msg_branch and msg_branch.fork_point_message_id: + branch_user_msgs = [ + m for m in session.messages + if m.branch_id == target_msg.branch_id and m.role == "user" + ] + if branch_user_msgs and branch_user_msgs[0].id == message_id: + fork_point_id = msg_branch.fork_point_message_id + fork_parent_branch = msg_branch.parent_branch_id or "main" + + new_branch_id = uuid4().hex new_branch = MessageBranch( id=new_branch_id, - parent_branch_id=target_msg.branch_id, - fork_point_message_id=message_id, + parent_branch_id=fork_parent_branch, + fork_point_message_id=fork_point_id, ) session.branches[new_branch_id] = new_branch session.active_branch_id = new_branch_id @@ -1066,6 +1156,10 @@ class AgentManager: content=new_content, branch_id=new_branch_id, parent_id=target_msg.parent_id, + images=target_msg.images, + context_paths=target_msg.context_paths, + forced_tools=target_msg.forced_tools, + attached_skills=target_msg.attached_skills, ) session.messages.append(edited_msg) @@ -1079,7 +1173,21 @@ class AgentManager: "active_branch_id": new_branch_id, }) - task = asyncio.create_task(self._run_agent_loop(session_id, new_content)) + session.sdk_session_id = None + session.status = "running" + await ws_manager.send_to_session(session_id, "agent:status", { + "session_id": session_id, + "status": "running", + "session": session.model_dump(mode="json"), + }) + + task = asyncio.create_task(self._run_agent_loop( + session_id, new_content, + images=target_msg.images, + context_paths=target_msg.context_paths, + forced_tools=target_msg.forced_tools, + attached_skills=target_msg.attached_skills, + )) self.tasks[session_id] = task async def switch_branch(self, session_id: str, branch_id: str): @@ -1234,7 +1342,14 @@ class AgentManager: async def close_session(self, session_id: str) -> None: """Close a session: pause the agent if running, persist to JSON file, - and remove from in-memory state.""" + and remove from in-memory state. Also stops browser-agent children.""" + children = [ + s for s in self.sessions.values() + if s.parent_session_id == session_id and s.mode == "browser-agent" + ] + for child in children: + await self.stop_agent(child.id) + task = self.tasks.get(session_id) if task and not task.done(): task.cancel() @@ -1250,8 +1365,14 @@ class AgentManager: if session.status in ("running", "waiting_approval"): session.status = "stopped" session.closed_at = datetime.now() + + for req in list(session.pending_approvals): + ws_manager.resolve_approval(req.id, {"behavior": "deny", "message": "Session closed"}) session.pending_approvals = [] + if hasattr(session, '_cancel_event'): + session._cancel_event.set() + doc_data = session.model_dump(mode="json") doc_data["search_text"] = self._build_search_text(session) @@ -1274,7 +1395,15 @@ class AgentManager: logger.info(f"Session {session_id} closed and persisted") async def delete_session(self, session_id: str) -> None: - """Permanently delete a session: remove from memory and JSON file.""" + """Permanently delete a session: remove from memory and JSON file. + Also stops browser-agent children first.""" + children = [ + s for s in self.sessions.values() + if s.parent_session_id == session_id and s.mode == "browser-agent" + ] + for child in children: + await self.stop_agent(child.id) + task = self.tasks.get(session_id) if task and not task.done(): task.cancel() @@ -1368,8 +1497,9 @@ class AgentManager: for session_id, session in list(self.sessions.items()): if session.status in ("running", "waiting_approval"): session.status = "stopped" + for req in list(session.pending_approvals): + ws_manager.resolve_approval(req.id, {"behavior": "deny", "message": "Server shutting down"}) session.pending_approvals = [] - session.closed_at = session.closed_at or datetime.now() doc_data = session.model_dump(mode="json") doc_data["search_text"] = self._build_search_text(session) _save_session(session_id, doc_data) @@ -1378,21 +1508,208 @@ class AgentManager: self.tasks.clear() async def restore_all_sessions(self) -> None: - """On startup, reload all persisted sessions from JSON files back into memory.""" + """On startup, reload all persisted sessions from JSON files back into memory. + + Only sessions without closed_at are restored (they were active at + shutdown). Sessions with closed_at were explicitly closed by the user + and stay on disk so the history endpoint can still serve them. + """ for sid, data in _load_all_session_data(): try: session = AgentSession(**data) except Exception as e: logger.warning(f"Skipping corrupt session file {sid}: {e}") continue + if session.closed_at is not None: + continue if session.status in ("running", "waiting_approval"): session.status = "stopped" - session.closed_at = None session.pending_approvals = [] self.sessions[session.id] = session _delete_session_file(sid) logger.info(f"Restored session {session.id}") + async def duplicate_session(self, session_id: str, dashboard_id: str | None = None, up_to_message_id: str | None = None) -> AgentSession: + """Create an independent copy of a session with the same chat history.""" + source = self.sessions.get(session_id) + if not source: + data = _load_session_data(session_id) + if data is None: + raise ValueError(f"Session {session_id} not found") + source = AgentSession(**data) + + source_messages = list(source.messages) + if up_to_message_id: + cut_idx = next( + (i for i, m in enumerate(source_messages) if m.id == up_to_message_id), + None, + ) + if cut_idx is not None: + source_messages = source_messages[: cut_idx + 1] + + old_to_new_msg: dict[str, str] = {} + new_messages: list[Message] = [] + for msg in source_messages: + new_id = uuid4().hex + old_to_new_msg[msg.id] = new_id + new_messages.append(Message( + id=new_id, + role=msg.role, + content=msg.content, + timestamp=msg.timestamp, + branch_id=msg.branch_id, + parent_id=old_to_new_msg.get(msg.parent_id) if msg.parent_id else None, + context_paths=msg.context_paths, + attached_skills=msg.attached_skills, + forced_tools=msg.forced_tools, + images=msg.images, + )) + + new_branches: dict[str, MessageBranch] = {} + for bid, branch in source.branches.items(): + new_branches[bid] = MessageBranch( + id=bid, + parent_branch_id=branch.parent_branch_id, + fork_point_message_id=old_to_new_msg.get(branch.fork_point_message_id) if branch.fork_point_message_id else None, + created_at=branch.created_at, + ) + + new_session = AgentSession( + id=uuid4().hex, + name=f"{source.name} (copy)", + status="stopped", + model=source.model, + mode=source.mode, + system_prompt=source.system_prompt, + allowed_tools=list(source.allowed_tools), + max_turns=source.max_turns, + cwd=source.cwd, + created_at=datetime.now(), + messages=new_messages, + branches=new_branches, + active_branch_id=source.active_branch_id, + tool_group_meta=dict(source.tool_group_meta), + dashboard_id=dashboard_id or source.dashboard_id, + ) + + self.sessions[new_session.id] = new_session + + await ws_manager.send_to_session(new_session.id, "agent:status", { + "session_id": new_session.id, + "status": new_session.status, + "session": new_session.model_dump(mode="json"), + }) + + return new_session + + async def invoke_agent( + self, + source_session_id: str, + message: str, + parent_session_id: str | None = None, + dashboard_id: str | None = None, + ) -> dict: + """Fork an existing session and send it a new message, returning the result.""" + source = self.sessions.get(source_session_id) + if not source: + data = _load_session_data(source_session_id) + if data is None: + raise ValueError(f"Session {source_session_id} not found") + source = AgentSession(**data) + + source_name = source.name + + old_to_new_msg: dict[str, str] = {} + new_messages: list[Message] = [] + for msg in source.messages: + new_id = uuid4().hex + old_to_new_msg[msg.id] = new_id + new_messages.append(Message( + id=new_id, + role=msg.role, + content=msg.content, + timestamp=msg.timestamp, + branch_id=msg.branch_id, + parent_id=old_to_new_msg.get(msg.parent_id) if msg.parent_id else None, + context_paths=msg.context_paths, + attached_skills=msg.attached_skills, + forced_tools=msg.forced_tools, + images=msg.images, + )) + + new_branches: dict[str, MessageBranch] = {} + for bid, branch in source.branches.items(): + new_branches[bid] = MessageBranch( + id=bid, + parent_branch_id=branch.parent_branch_id, + fork_point_message_id=( + old_to_new_msg.get(branch.fork_point_message_id) + if branch.fork_point_message_id else None + ), + created_at=branch.created_at, + ) + + fork = AgentSession( + id=uuid4().hex, + name=f"{source_name} (invoked)", + status="running", + model=source.model, + mode="invoked-agent", + sdk_session_id=source.sdk_session_id, + system_prompt=source.system_prompt, + allowed_tools=list(source.allowed_tools), + max_turns=source.max_turns or 25, + cwd=source.cwd, + created_at=datetime.now(), + messages=new_messages, + branches=new_branches, + active_branch_id=source.active_branch_id, + tool_group_meta=dict(source.tool_group_meta), + dashboard_id=dashboard_id or source.dashboard_id, + parent_session_id=parent_session_id, + ) + + self.sessions[fork.id] = fork + + await ws_manager.broadcast_global("agent:status", { + "session_id": fork.id, + "status": fork.status, + "session": fork.model_dump(mode="json"), + }) + + user_msg = Message( + role="user", + content=message, + branch_id=fork.active_branch_id, + ) + fork.messages.append(user_msg) + await ws_manager.send_to_session(fork.id, "agent:message", { + "session_id": fork.id, + "message": user_msg.model_dump(mode="json"), + }) + + await self._run_agent_loop(fork.id, message, fork_session=True) + + last_assistant = None + for msg in reversed(fork.messages): + if msg.role == "assistant": + content = msg.content + if isinstance(content, str): + last_assistant = content + elif isinstance(content, list): + texts = [b.get("text", "") for b in content if isinstance(b, dict) and b.get("type") == "text"] + last_assistant = "\n".join(texts) + else: + last_assistant = str(content) + break + + return { + "forked_session_id": fork.id, + "source_name": source_name, + "response": last_assistant or "No response from invoked agent.", + "cost_usd": fork.cost_usd, + } + def get_all_sessions(self, dashboard_id: str | None = None) -> list[AgentSession]: if dashboard_id: return [s for s in self.sessions.values() if s.dashboard_id == dashboard_id] @@ -1401,4 +1718,22 @@ class AgentManager: def get_session(self, session_id: str) -> Optional[AgentSession]: return self.sessions.get(session_id) + def get_browser_agent_children(self, parent_session_id: str) -> list[dict]: + """Return browser-agent sessions for a parent, from memory or disk.""" + results: list[dict] = [] + seen: set[str] = set() + + for s in self.sessions.values(): + if s.mode == "browser-agent" and s.parent_session_id == parent_session_id: + results.append(s.model_dump(mode="json")) + seen.add(s.id) + + for sid, data in _load_all_session_data(): + if sid in seen: + continue + if data.get("mode") == "browser-agent" and data.get("parent_session_id") == parent_session_id: + results.append(data) + + return results + agent_manager = AgentManager() diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 462b8fde..c6bee62f 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -52,10 +52,13 @@ async def send_message(session_id: str, body: dict): prompt, mode=body.get("mode"), model=body.get("model"), + provider=body.get("provider"), images=body.get("images"), context_paths=body.get("context_paths"), forced_tools=body.get("forced_tools"), attached_skills=body.get("attached_skills"), + hidden=body.get("hidden", False), + selected_browser_ids=body.get("selected_browser_ids"), ) return {"ok": True} @@ -131,6 +134,18 @@ async def get_branches(session_id: str): "active_branch_id": session.active_branch_id, } +@agents.router.post("/sessions/{session_id}/duplicate") +async def duplicate_session(session_id: str, body: dict = {}): + try: + session = await agent_manager.duplicate_session( + session_id, + dashboard_id=body.get("dashboard_id"), + up_to_message_id=body.get("up_to_message_id"), + ) + except ValueError as e: + raise HTTPException(status_code=404, detail=str(e)) + return {"session": session.model_dump(mode="json")} + @agents.router.post("/sessions/{session_id}/close") async def close_session(session_id: str): try: @@ -151,6 +166,11 @@ async def get_history(q: str = "", limit: int = 20, offset: int = 0, dashboard_i dashboard_id=dashboard_id or None, ) +@agents.router.get("/sessions/{session_id}/browser-agents") +async def get_browser_agent_children(session_id: str): + children = agent_manager.get_browser_agent_children(session_id) + return {"sessions": children} + @agents.router.post("/sessions/{session_id}/resume") async def resume_session(session_id: str): try: @@ -159,3 +179,199 @@ async def resume_session(session_id: str): raise HTTPException(status_code=404, detail=str(e)) return {"session": session.model_dump(mode="json")} + +@agents.router.get("/models") +async def list_models(): + """Return available models grouped by provider, filtered by configured credentials.""" + from backend.apps.agents.providers.registry import get_available_models + from backend.apps.settings.settings import load_settings + settings = load_settings() + return {"models": get_available_models(settings)} + + +# ── GitHub Copilot Auth ── + +@agents.router.post("/copilot/start-auth") +async def copilot_start_auth(): + """Start GitHub device flow for Copilot auth.""" + from backend.apps.agents.copilot_auth import start_device_flow + try: + result = await start_device_flow() + return result + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@agents.router.post("/copilot/poll-auth") +async def copilot_poll_auth(body: dict): + """Poll for GitHub auth completion. Returns token on success.""" + from backend.apps.agents.copilot_auth import poll_for_token, exchange_for_copilot_token, get_github_username, list_copilot_models + from backend.apps.settings.settings import load_settings, _save_settings + + device_code = body.get("device_code", "") + if not device_code: + raise HTTPException(status_code=400, detail="device_code required") + + try: + github_token = await poll_for_token(device_code) + if github_token is None: + return {"status": "pending"} + + # Got GitHub token — exchange for Copilot token + copilot_result = await exchange_for_copilot_token(github_token) + username = await get_github_username(github_token) + + # Fetch available models + models = await list_copilot_models(copilot_result["token"]) + + # Save to settings + settings = load_settings() + settings.copilot_github_token = github_token + settings.copilot_token = copilot_result["token"] + settings.copilot_token_expires = copilot_result["expires_at"] + _save_settings(settings) + + return { + "status": "connected", + "username": username, + "models": models, + } + except ValueError as e: + raise HTTPException(status_code=400, detail=str(e)) + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@agents.router.get("/copilot/models") +async def copilot_models(): + """List models available through Copilot.""" + from backend.apps.agents.copilot_auth import list_copilot_models, get_copilot_token + from backend.apps.settings.settings import load_settings, _save_settings + + settings = load_settings() + github_token = getattr(settings, "copilot_github_token", None) + if not github_token: + return {"models": []} + + try: + result = await get_copilot_token( + github_token, + getattr(settings, "copilot_token", None), + getattr(settings, "copilot_token_expires", None), + ) + settings.copilot_token = result["token"] + settings.copilot_token_expires = result["expires_at"] + _save_settings(settings) + + models = await list_copilot_models(result["token"]) + return {"models": models} + except Exception as e: + return {"models": [], "error": str(e)} + + +@agents.router.post("/copilot/disconnect") +async def copilot_disconnect(): + """Clear Copilot tokens.""" + from backend.apps.settings.settings import load_settings, _save_settings + + settings = load_settings() + settings.copilot_github_token = None + settings.copilot_token = None + settings.copilot_token_expires = None + _save_settings(settings) + return {"ok": True} + + +# ── Subscription Management (via 9Router) ── + +@agents.router.get("/subscriptions/status") +async def subscriptions_status(): + """Check if 9Router is running and list connected providers.""" + from backend.apps.nine_router import is_running, get_providers, get_models + if not is_running(): + return {"running": False, "providers": [], "models": []} + providers = await get_providers() + models = await get_models() + return {"running": True, "providers": providers, "models": models} + + +@agents.router.post("/subscriptions/connect") +async def subscriptions_connect(body: dict): + """Start OAuth flow for a subscription provider.""" + from backend.apps.nine_router import is_running, ensure_running, start_oauth + provider = body.get("provider", "") + if not provider: + raise HTTPException(status_code=400, detail="provider required") + + if not is_running(): + import asyncio + await ensure_running() + if not is_running(): + raise HTTPException(status_code=503, detail="9Router not available. Please install Node.js.") + + try: + result = await start_oauth(provider) + + # For auth_code flows, store pending state so the callback can exchange + if result.get("flow") == "authorization_code" and result.get("state"): + from backend.main import _pending_oauth + _pending_oauth[result["state"]] = { + "provider": provider, + "code_verifier": result.get("code_verifier", ""), + "redirect_uri": result.get("redirect_uri", ""), + } + + return result + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@agents.router.post("/subscriptions/poll") +async def subscriptions_poll(body: dict): + """Poll for OAuth completion.""" + from backend.apps.nine_router import poll_oauth + provider = body.get("provider", "") + device_code = body.get("device_code", "") + if not provider or not device_code: + raise HTTPException(status_code=400, detail="provider and device_code required") + + try: + result = await poll_oauth( + provider, device_code, + code_verifier=body.get("code_verifier"), + extra_data=body.get("extra_data"), + ) + return result + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@agents.router.post("/subscriptions/exchange") +async def subscriptions_exchange(body: dict): + """Exchange OAuth code for tokens via 9Router.""" + from backend.apps.nine_router import exchange_oauth + provider = body.get("provider", "") + code = body.get("code", "") + redirect_uri = body.get("redirect_uri", "") + code_verifier = body.get("code_verifier", "") + state = body.get("state", "") + + if not provider or not code: + raise HTTPException(status_code=400, detail="provider and code required") + + try: + result = await exchange_oauth(provider, code, redirect_uri, code_verifier, state) + return result + except Exception as e: + raise HTTPException(status_code=500, detail=str(e)) + + +@agents.router.get("/subscriptions/models") +async def subscriptions_models(): + """List all models available through connected subscriptions.""" + from backend.apps.nine_router import is_running, get_models + if not is_running(): + return {"models": []} + models = await get_models() + return {"models": models} + diff --git a/backend/apps/agents/browser_agent.py b/backend/apps/agents/browser_agent.py index 11244acf..1056c4f1 100644 --- a/backend/apps/agents/browser_agent.py +++ b/backend/apps/agents/browser_agent.py @@ -15,15 +15,16 @@ from uuid import uuid4 import anthropic -from backend.apps.agents.models import AgentSession, Message +from backend.apps.agents.models import AgentSession, ApprovalRequest, Message from backend.apps.agents.ws_manager import ws_manager +from backend.apps.tools_lib.tools_lib import load_builtin_permissions logger = logging.getLogger(__name__) MODEL_MAP = { "sonnet": "claude-sonnet-4-20250514", "opus": "claude-opus-4-20250514", - "haiku": "claude-haiku-4-20250414", + "haiku": "claude-haiku-4-5-20251001", } BROWSER_TOOLS_SCHEMA = [ @@ -112,6 +113,48 @@ BROWSER_TOOLS_SCHEMA = [ "required": [], }, }, + { + "name": "BrowserScroll", + "description": ( + "Scroll the page up or down. Automatically finds the correct scrollable " + "container (works on SPAs like Notion, Gmail, etc. that use nested scroll " + "containers instead of window-level scrolling). Returns scroll position info " + "including whether top/bottom has been reached." + ), + "input_schema": { + "type": "object", + "properties": { + "direction": { + "type": "string", + "enum": ["up", "down"], + "description": "Scroll direction. Defaults to 'down'.", + }, + "amount": { + "type": "number", + "description": "Pixels to scroll. Defaults to 500.", + }, + }, + "required": [], + }, + }, + { + "name": "BrowserWait", + "description": ( + "Wait for a specified duration. Useful after navigation or actions that " + "trigger page loads, animations, or async content rendering. " + "Min 100ms, max 10000ms." + ), + "input_schema": { + "type": "object", + "properties": { + "milliseconds": { + "type": "number", + "description": "Duration to wait in milliseconds. Defaults to 1000.", + }, + }, + "required": [], + }, + }, ] ACTION_MAP = { @@ -122,17 +165,29 @@ ACTION_MAP = { "BrowserType": "type", "BrowserEvaluate": "evaluate", "BrowserGetElements": "get_elements", + "BrowserScroll": "scroll", + "BrowserWait": "wait", } SYSTEM_PROMPT = ( "You are a browser automation agent. You control a single browser tab and " "execute the task you are given.\n\n" "Strategy:\n" - "1. Start by taking a screenshot or calling BrowserGetElements to understand the page.\n" - "2. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n" - "3. After performing actions, take a screenshot to verify the result.\n" - "4. If an action fails, try alternative selectors or approaches.\n" - "5. When the task is complete, provide a clear summary of what you accomplished.\n\n" + "1. Start by taking a screenshot to understand the page.\n" + "2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n" + "3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with " + "window.scrollBy() as many sites use nested scroll containers that BrowserScroll " + "handles automatically.\n" + "4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n" + "5. After performing actions, take a screenshot to verify the result.\n" + "6. If an action fails, try alternative selectors or approaches.\n" + "7. When the task is complete, provide a clear summary of what you accomplished.\n\n" + "Important notes:\n" + "- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n" + "- BrowserScroll returns position info including atTop/atBottom — use this to know when " + "you've reached the end of the page.\n" + "- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n" + "- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n" "You have access ONLY to browser tools. Do not ask the user questions — " "complete the task autonomously to the best of your ability." ) @@ -143,16 +198,29 @@ MAX_TURNS = 25 async def execute_browser_tool( tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "", ) -> dict: - """Execute a browser tool via ws_manager directly (no MCP/HTTP round-trip).""" + """Execute a browser tool. Tries Electron webview first, falls back to headless Playwright.""" action = ACTION_MAP.get(tool_name) if not action: return {"error": f"Unknown browser tool: {tool_name}"} params = {k: v for k, v in tool_input.items()} request_id = uuid4().hex + + # Try Electron webview first result = await ws_manager.send_browser_command( request_id, action, browser_id, params, tab_id=tab_id, ) + + # If Electron webview not available, fall back to headless Playwright + if isinstance(result, dict) and result.get("error") and "not found" in result.get("error", "").lower(): + try: + from backend.apps.agents.headless_browser import execute as headless_execute + result = await headless_execute(browser_id, action, params) + except ImportError: + return {"error": "Browser not available. Install playwright: pip install playwright && playwright install chromium"} + except Exception as e: + return {"error": f"Headless browser error: {e}"} + return result @@ -179,6 +247,40 @@ def _format_tool_result(result: dict, tool_name: str) -> list[dict]: return [{"type": "text", "text": str(text)}] +async def _request_browser_approval( + session: AgentSession, tool_name: str, tool_input: dict, +) -> dict: + """Send an approval request for a browser sub-agent tool and wait for the decision.""" + request_id = uuid4().hex + approval_req = ApprovalRequest( + id=request_id, + session_id=session.id, + tool_name=tool_name, + tool_input=tool_input, + ) + session.pending_approvals.append(approval_req) + session.status = "waiting_approval" + + await ws_manager.send_to_session(session.id, "agent:status", { + "session_id": session.id, + "status": "waiting_approval", + }) + + decision = await ws_manager.send_approval_request( + session.id, request_id, tool_name, tool_input, + ) + + session.pending_approvals = [ + a for a in session.pending_approvals if a.id != request_id + ] + session.status = "running" + await ws_manager.send_to_session(session.id, "agent:status", { + "session_id": session.id, + "status": "running", + }) + return decision + + async def run_browser_agent( task: str, browser_id: str, @@ -188,6 +290,9 @@ async def run_browser_agent( tab_id: str = "", pre_selected: bool = False, initial_url: str | None = None, + parent_session_id: str | None = None, + auth_token: str | None = None, + base_url: str | None = None, ) -> dict: """Run a browser sub-agent loop for a single browser card. @@ -196,7 +301,10 @@ async def run_browser_agent( """ from backend.apps.agents.agent_manager import agent_manager + _browser_perms = load_builtin_permissions() + session_id = uuid4().hex + cancel_event = asyncio.Event() session = AgentSession( id=session_id, name=f"Browser Agent", @@ -206,7 +314,9 @@ async def run_browser_agent( dashboard_id=dashboard_id, browser_id=browser_id, system_prompt=SYSTEM_PROMPT, + parent_session_id=parent_session_id, ) + session._cancel_event = cancel_event agent_manager.sessions[session_id] = session await ws_manager.send_to_session(session_id, "agent:status", { @@ -222,7 +332,17 @@ async def run_browser_agent( logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}") api_model = MODEL_MAP.get(model, model) - client = anthropic.AsyncAnthropic(api_key=api_key) + + # Use OpenAI client for 9Router, Anthropic client for direct API + _use_openai_client = base_url is not None + if _use_openai_client: + from openai import AsyncOpenAI + # Map to 9Router model IDs + _9r_map = {"sonnet": "cc/claude-sonnet-4-6", "opus": "cc/claude-opus-4-6", "haiku": "cc/claude-haiku-4-5-20251001"} + api_model = _9r_map.get(model, f"cc/{api_model}" if not api_model.startswith("cc/") else api_model) + client = AsyncOpenAI(api_key=api_key, base_url=base_url) + else: + client = anthropic.AsyncAnthropic(api_key=api_key) messages: list[dict] = [{"role": "user", "content": task}] action_log: list[dict] = [] @@ -237,30 +357,69 @@ async def run_browser_agent( try: for turn in range(MAX_TURNS): - response = await client.messages.create( - model=api_model, - max_tokens=4096, - system=SYSTEM_PROMPT, - tools=BROWSER_TOOLS_SCHEMA, - messages=messages, - ) + if cancel_event.is_set(): + break - assistant_content = [] - text_parts = [] - tool_uses = [] + if _use_openai_client: + # OpenAI-compatible format (9Router) + import json as _json + oai_tools = [{"type": "function", "function": {"name": t["name"], "description": t["description"], "parameters": t["input_schema"]}} for t in BROWSER_TOOLS_SCHEMA] + oai_messages = [{"role": "system", "content": SYSTEM_PROMPT}] + messages + resp = await client.chat.completions.create(model=api_model, max_tokens=4096, tools=oai_tools, messages=oai_messages) + choice = resp.choices[0] - for block in response.content: - if block.type == "text": - text_parts.append(block.text) - assistant_content.append({"type": "text", "text": block.text}) - elif block.type == "tool_use": - tool_uses.append(block) - assistant_content.append({ - "type": "tool_use", - "id": block.id, - "name": block.name, - "input": block.input, - }) + assistant_content = [] + text_parts = [] + tool_uses = [] + + if choice.message.content: + text_parts.append(choice.message.content) + assistant_content.append({"type": "text", "text": choice.message.content}) + + if choice.message.tool_calls: + for tc in choice.message.tool_calls: + try: + inp = _json.loads(tc.function.arguments) + except Exception: + inp = {} + # Create a simple object with .id, .name, .input + class _TC: + pass + tool_obj = _TC() + tool_obj.id = tc.id + tool_obj.name = tc.function.name + tool_obj.input = inp + tool_uses.append(tool_obj) + assistant_content.append({"type": "tool_use", "id": tc.id, "name": tc.function.name, "input": inp}) + + stop_reason = "tool_use" if choice.message.tool_calls else "end_turn" + else: + # Anthropic format (direct API) + response = await client.messages.create( + model=api_model, + max_tokens=4096, + system=SYSTEM_PROMPT, + tools=BROWSER_TOOLS_SCHEMA, + messages=messages, + ) + + assistant_content = [] + text_parts = [] + tool_uses = [] + stop_reason = response.stop_reason + + for block in response.content: + if block.type == "text": + text_parts.append(block.text) + assistant_content.append({"type": "text", "text": block.text}) + elif block.type == "tool_use": + tool_uses.append(block) + assistant_content.append({ + "type": "tool_use", + "id": block.id, + "name": block.name, + "input": block.input, + }) if text_parts: asst_msg = Message( @@ -284,13 +443,67 @@ async def run_browser_agent( "message": tool_msg.model_dump(mode="json"), }) - messages.append({"role": "assistant", "content": assistant_content}) + if _use_openai_client: + # OpenAI format: assistant message with tool_calls + asst_api_msg: dict = {"role": "assistant", "content": choice.message.content} + if choice.message.tool_calls: + asst_api_msg["tool_calls"] = [{"id": tc.id, "type": "function", "function": {"name": tc.function.name, "arguments": tc.function.arguments}} for tc in (choice.message.tool_calls or [])] + messages.append(asst_api_msg) + else: + messages.append({"role": "assistant", "content": assistant_content}) - if response.stop_reason != "tool_use": + if stop_reason != "tool_use": break tool_results = [] + cancelled = False for tu in tool_uses: + if cancel_event.is_set(): + cancelled = True + break + + policy = _browser_perms.get(tu.name, "always_allow") + + if policy == "deny": + denied_text = f"Tool {tu.name} is denied by permission policy." + tool_results.append({ + "type": "tool_result", + "tool_use_id": tu.id, + "content": [{"type": "text", "text": denied_text}], + }) + result_msg = Message( + role="tool_result", + content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0}, + ) + session.messages.append(result_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": result_msg.model_dump(mode="json"), + }) + continue + + if policy == "ask": + decision = await _request_browser_approval( + session, tu.name, tu.input, + ) + if decision.get("behavior") == "deny": + denied_text = decision.get("message") or f"Tool {tu.name} denied by user." + tool_results.append({ + "type": "tool_result", + "tool_use_id": tu.id, + "content": [{"type": "text", "text": denied_text}], + }) + result_msg = Message( + role="tool_result", + content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0}, + ) + session.messages.append(result_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": result_msg.model_dump(mode="json"), + }) + continue + start = time.time() result = await execute_browser_tool( tu.name, tu.input, browser_id, tab_id, @@ -325,7 +538,26 @@ async def run_browser_agent( "message": result_msg.model_dump(mode="json"), }) - messages.append({"role": "user", "content": tool_results}) + if _use_openai_client: + # OpenAI format: each tool result is a separate message + for tr in tool_results: + text_content = "" + for block in (tr.get("content") or []): + if isinstance(block, dict) and block.get("type") == "text": + text_content += block.get("text", "") + messages.append({"role": "tool", "tool_call_id": tr["tool_use_id"], "content": text_content or "Done."}) + else: + messages.append({"role": "user", "content": tool_results}) + + if cancelled: + break + + if cancel_event.is_set(): + session.status = "stopped" + await ws_manager.send_to_session(session_id, "agent:status", { + "session_id": session_id, + "status": "stopped", + }) summary_parts = text_parts if text_parts else ["Task completed."] summary = "\n".join(summary_parts) @@ -407,6 +639,8 @@ async def _create_browser_card(dashboard_id: str, url: str) -> str: activeTabId=tab_id, x=40, y=100, + width=1280, + height=800, ) dashboard.layout.browser_cards[browser_id] = card dashboard.updated_at = datetime.now() @@ -425,6 +659,9 @@ async def run_browser_agents( api_key: str, dashboard_id: str | None = None, pre_selected_browser_ids: list[str] | None = None, + parent_session_id: str | None = None, + auth_token: str | None = None, + base_url: str | None = None, ) -> list[dict]: """Run multiple browser sub-agents in parallel. @@ -451,6 +688,9 @@ async def run_browser_agents( dashboard_id=dashboard_id, pre_selected=is_pre_selected, initial_url=url if url and browser_id not in pre_selected else None, + parent_session_id=parent_session_id, + auth_token=auth_token, + base_url=base_url, ) results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True) diff --git a/backend/apps/agents/browser_agent_mcp_server.py b/backend/apps/agents/browser_agent_mcp_server.py index 2a46a139..0f7f4b6b 100644 --- a/backend/apps/agents/browser_agent_mcp_server.py +++ b/backend/apps/agents/browser_agent_mcp_server.py @@ -25,26 +25,20 @@ BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/browser-agent/run" MODEL = os.environ.get("OPENSWARM_AGENT_MODEL", "sonnet") DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "") PRE_SELECTED_BROWSER_IDS = os.environ.get("OPENSWARM_PRE_SELECTED_BROWSER_IDS", "") +PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") TOOLS = [ { - "name": "BrowserAgent", + "name": "CreateBrowserAgent", "description": ( - "Delegate a browser task to a dedicated browser agent. The browser agent " + "Create a new browser card and run a task on it. A dedicated browser agent " "will autonomously perform the task (navigating, clicking, typing, etc.) " "and return a summary of actions taken plus a final screenshot. " - "Use this for any task that requires interacting with a web page." + "Use this when you need a fresh browser for a new task." ), "inputSchema": { "type": "object", "properties": { - "browser_id": { - "type": "string", - "description": ( - "The ID of the browser card to use. If omitted, a new browser " - "card will be automatically created." - ), - }, "task": { "type": "string", "description": ( @@ -55,20 +49,47 @@ TOOLS = [ "url": { "type": "string", "description": ( - "Optional starting URL. If provided and no browser_id is given, " - "the new browser will navigate here first." + "Optional starting URL. The new browser will navigate here " + "before beginning the task." ), }, }, "required": ["task"], }, }, + { + "name": "BrowserAgent", + "description": ( + "Delegate a browser task to a dedicated browser agent on an existing " + "browser card. The browser agent will autonomously perform the task " + "(navigating, clicking, typing, etc.) and return a summary of actions " + "taken plus a final screenshot." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The ID of the existing browser card to use.", + }, + "task": { + "type": "string", + "description": ( + "The task for the browser agent to perform. Be specific and " + "detailed about what you want accomplished." + ), + }, + }, + "required": ["browser_id", "task"], + }, + }, { "name": "BrowserAgents", "description": ( - "Delegate multiple browser tasks to run in parallel, each on a different " - "browser. All tasks execute concurrently and results are returned together. " - "Use this when you need to perform tasks on multiple web pages simultaneously." + "Delegate multiple browser tasks to run in parallel, each on an existing " + "browser card. All tasks execute concurrently and results are returned " + "together. Use this when you need to perform tasks on multiple web pages " + "simultaneously." ), "inputSchema": { "type": "object", @@ -81,18 +102,14 @@ TOOLS = [ "properties": { "browser_id": { "type": "string", - "description": "Optional browser card ID. If omitted, a new browser will be created.", + "description": "The ID of the existing browser card to use.", }, "task": { "type": "string", "description": "The task for this browser agent.", }, - "url": { - "type": "string", - "description": "Optional starting URL.", - }, }, - "required": ["task"], + "required": ["browser_id", "task"], }, }, }, @@ -119,6 +136,7 @@ def call_backend(tasks: list[dict]) -> dict: "model": MODEL, "dashboard_id": DASHBOARD_ID, "pre_selected_browser_ids": pre_selected, + "parent_session_id": PARENT_SESSION_ID, }).encode() req = urllib.request.Request( BACKEND_URL, @@ -219,10 +237,10 @@ def format_batch_results(results: list[dict]) -> dict: def handle_tool_call(tool_name: str, arguments: dict) -> dict: - if tool_name == "BrowserAgent": + if tool_name == "CreateBrowserAgent": task_def = { "task": arguments.get("task", ""), - "browser_id": arguments.get("browser_id", ""), + "browser_id": "", "url": arguments.get("url", ""), } result = call_backend([task_def]) @@ -233,10 +251,30 @@ def handle_tool_call(tool_name: str, arguments: dict) -> dict: return format_result(results[0]) return {"content": [{"type": "text", "text": "No result returned."}], "isError": True} + elif tool_name == "BrowserAgent": + browser_id = arguments.get("browser_id", "") + if not browser_id: + return {"content": [{"type": "text", "text": "Error: browser_id is required"}], "isError": True} + task_def = { + "task": arguments.get("task", ""), + "browser_id": browser_id, + "url": "", + } + result = call_backend([task_def]) + if "error" in result: + return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} + results = result.get("results", [result]) + if results: + return format_result(results[0]) + return {"content": [{"type": "text", "text": "No result returned."}], "isError": True} + elif tool_name == "BrowserAgents": tasks = arguments.get("tasks", []) if not tasks: return {"content": [{"type": "text", "text": "Error: tasks array is empty"}], "isError": True} + for t in tasks: + if not t.get("browser_id"): + return {"content": [{"type": "text", "text": "Error: browser_id is required for each task"}], "isError": True} result = call_backend(tasks) if "error" in result: return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} diff --git a/backend/apps/agents/browser_mcp_server.py b/backend/apps/agents/browser_mcp_server.py index f32054c2..86f1ad5e 100644 --- a/backend/apps/agents/browser_mcp_server.py +++ b/backend/apps/agents/browser_mcp_server.py @@ -181,6 +181,58 @@ TOOLS = [ "required": ["browser_id"], }, }, + { + "name": "BrowserScroll", + "description": ( + "Scroll the page up or down. Automatically finds the correct scrollable " + "container (works on SPAs like Notion, Gmail, etc. that use nested scroll " + "containers instead of window-level scrolling). Returns scroll position info " + "including whether top/bottom has been reached." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "tab_id": TAB_ID_PROP, + "direction": { + "type": "string", + "enum": ["up", "down"], + "description": "Scroll direction. Defaults to 'down'.", + }, + "amount": { + "type": "number", + "description": "Pixels to scroll. Defaults to 500.", + }, + }, + "required": ["browser_id"], + }, + }, + { + "name": "BrowserWait", + "description": ( + "Wait for a specified duration. Useful after navigation or actions that " + "trigger page loads, animations, or async content rendering. " + "Min 100ms, max 10000ms." + ), + "inputSchema": { + "type": "object", + "properties": { + "browser_id": { + "type": "string", + "description": "The browser card ID.", + }, + "tab_id": TAB_ID_PROP, + "milliseconds": { + "type": "number", + "description": "Duration to wait in milliseconds. Defaults to 1000.", + }, + }, + "required": ["browser_id"], + }, + }, ] @@ -260,6 +312,8 @@ def handle_tool_call(tool_name: str, arguments: dict) -> dict: "BrowserType": "type", "BrowserEvaluate": "evaluate", "BrowserGetElements": "get_elements", + "BrowserScroll": "scroll", + "BrowserWait": "wait", } action = action_map.get(tool_name) if not action: diff --git a/backend/apps/agents/copilot_auth.py b/backend/apps/agents/copilot_auth.py new file mode 100644 index 00000000..d7e0d472 --- /dev/null +++ b/backend/apps/agents/copilot_auth.py @@ -0,0 +1,175 @@ +"""GitHub Copilot device flow OAuth + token management. + +Handles the full flow: +1. Start device flow → get user_code for user to enter at github.com/login/device +2. Poll until user authorizes → get GitHub access token +3. Exchange GitHub token → Copilot JWT (expires every ~30min) +4. Auto-refresh Copilot token before expiry +""" + +import logging +import time + +import httpx + +logger = logging.getLogger(__name__) + +CLIENT_ID = "Iv1.b507a08c87ecfe98" # Copilot's public OAuth app ID +DEVICE_CODE_URL = "https://github.com/login/device/code" +TOKEN_URL = "https://github.com/login/oauth/access_token" +COPILOT_TOKEN_URL = "https://api.github.com/copilot_internal/v2/token" +COPILOT_API_BASE = "https://api.githubcopilot.com" + +HEADERS = { + "accept": "application/json", + "content-type": "application/json", + "editor-version": "vscode/1.100.0", + "editor-plugin-version": "copilot-chat/0.30.0", + "user-agent": "GithubCopilot/1.200.0", +} + + +async def start_device_flow() -> dict: + """Start GitHub device flow. + + Returns: {user_code, verification_uri, device_code, expires_in, interval} + """ + async with httpx.AsyncClient(timeout=15.0) as client: + resp = await client.post( + DEVICE_CODE_URL, + headers=HEADERS, + json={"client_id": CLIENT_ID, "scope": "read:user"}, + ) + resp.raise_for_status() + data = resp.json() + return { + "user_code": data["user_code"], + "verification_uri": data["verification_uri"], + "device_code": data["device_code"], + "expires_in": data.get("expires_in", 900), + "interval": data.get("interval", 5), + } + + +async def poll_for_token(device_code: str) -> str | None: + """Poll GitHub for token after user authorizes. + + Returns the GitHub access token (gho_xxx), or None if still pending. + Raises on error (expired, denied, etc.) + """ + async with httpx.AsyncClient(timeout=15.0) as client: + resp = await client.post( + TOKEN_URL, + headers=HEADERS, + json={ + "client_id": CLIENT_ID, + "device_code": device_code, + "grant_type": "urn:ietf:params:oauth:grant-type:device_code", + }, + ) + data = resp.json() + + if "access_token" in data: + return data["access_token"] + + error = data.get("error", "") + if error == "authorization_pending": + return None # Still waiting + if error == "slow_down": + return None # Need to slow down polling + if error in ("expired_token", "access_denied"): + raise ValueError(f"GitHub auth failed: {error}") + + return None + + +async def exchange_for_copilot_token(github_token: str) -> dict: + """Exchange GitHub OAuth token for Copilot JWT. + + Returns: {token, expires_at} + """ + async with httpx.AsyncClient(timeout=15.0) as client: + resp = await client.get( + COPILOT_TOKEN_URL, + headers={ + **HEADERS, + "authorization": f"token {github_token}", + }, + ) + if resp.status_code == 401: + raise ValueError("GitHub token invalid or expired. Please re-authenticate.") + if resp.status_code == 403: + raise ValueError("No Copilot subscription found for this GitHub account.") + resp.raise_for_status() + + data = resp.json() + token = data.get("token", "") + + # Extract expiry from token (format: tid=xxx;exp=1234567890;...) + expires_at = time.time() + 25 * 60 # Default 25 min + if "exp=" in token: + try: + for pair in token.split(";"): + if pair.strip().startswith("exp="): + expires_at = int(pair.strip().split("=")[1]) + break + except (ValueError, IndexError): + pass + + return {"token": token, "expires_at": expires_at} + + +async def get_copilot_token(github_token: str, current_token: str | None = None, expires_at: float | None = None) -> dict: + """Get a valid Copilot token, refreshing if needed. + + Returns: {token, expires_at} + """ + # If current token is still valid (with 2 min buffer), return it + if current_token and expires_at and time.time() < expires_at - 120: + return {"token": current_token, "expires_at": expires_at} + + # Otherwise refresh + return await exchange_for_copilot_token(github_token) + + +async def list_copilot_models(copilot_token: str) -> list[dict]: + """Fetch available models from Copilot API.""" + async with httpx.AsyncClient(timeout=15.0) as client: + resp = await client.get( + f"{COPILOT_API_BASE}/models", + headers={ + "authorization": f"Bearer {copilot_token}", + "copilot-integration-id": "vscode-chat", + **HEADERS, + }, + ) + if resp.status_code != 200: + logger.warning(f"Failed to list Copilot models: {resp.status_code}") + return [] + + data = resp.json() + models = data.get("data", data.get("models", [])) + return [ + { + "value": m.get("id", m.get("name", "")), + "label": m.get("name", m.get("id", "")), + "context_window": m.get("context_window", 128_000), + } + for m in models + if isinstance(m, dict) + ] + + +async def get_github_username(github_token: str) -> str | None: + """Get the GitHub username for display.""" + try: + async with httpx.AsyncClient(timeout=10.0) as client: + resp = await client.get( + "https://api.github.com/user", + headers={"authorization": f"token {github_token}", "accept": "application/json"}, + ) + if resp.status_code == 200: + return resp.json().get("login") + except Exception: + pass + return None diff --git a/backend/apps/agents/headless_browser.py b/backend/apps/agents/headless_browser.py new file mode 100644 index 00000000..0b601272 --- /dev/null +++ b/backend/apps/agents/headless_browser.py @@ -0,0 +1,149 @@ +"""Headless browser fallback using Playwright. + +Used when Electron's is not available (running in regular browser). +Provides the same browser actions as the Electron webview bridge: +navigate, click, type, screenshot, get_text, get_elements, evaluate. +""" + +import asyncio +import base64 +import logging +from typing import Any + +logger = logging.getLogger(__name__) + +_playwright = None +_browser = None +_pages: dict[str, Any] = {} # browser_id -> Page + + +async def _ensure_browser(): + """Start Playwright browser if not running.""" + global _playwright, _browser + if _browser and _browser.is_connected(): + return + + from playwright.async_api import async_playwright + _playwright = await async_playwright().start() + _browser = await _playwright.chromium.launch( + headless=True, + args=["--no-sandbox", "--disable-gpu"], + ) + logger.info("Headless browser started (Playwright/Chromium)") + + +async def _get_page(browser_id: str) -> Any: + """Get or create a page for a browser_id.""" + await _ensure_browser() + if browser_id not in _pages or _pages[browser_id].is_closed(): + page = await _browser.new_page() + await page.set_viewport_size({"width": 1280, "height": 800}) + _pages[browser_id] = page + return _pages[browser_id] + + +async def execute(browser_id: str, action: str, params: dict) -> dict: + """Execute a browser action. Returns same format as Electron webview bridge.""" + try: + page = await _get_page(browser_id) + + if action == "navigate": + url = params.get("url", "") + if url: + await page.goto(url, wait_until="domcontentloaded", timeout=30000) + await page.wait_for_timeout(1000) # Let JS settle + return {"text": f"Navigated to {page.url}", "url": page.url} + + elif action == "screenshot": + screenshot_bytes = await page.screenshot(type="png") + b64 = base64.b64encode(screenshot_bytes).decode() + return {"image": b64, "url": page.url} + + elif action == "get_text": + text = await page.evaluate("document.body?.innerText || ''") + return {"text": text[:15000]} + + elif action == "click": + selector = params.get("selector", "") + if selector: + await page.click(selector, timeout=5000) + await page.wait_for_timeout(500) + return {"text": f"Clicked {selector}"} + + elif action == "type": + selector = params.get("selector", "") + text = params.get("text", "") + if selector and text: + await page.fill(selector, text) + return {"text": f"Typed into {selector}"} + + elif action == "evaluate": + expression = params.get("expression", "") + result = await page.evaluate(expression) + return {"text": str(result) if result is not None else "undefined"} + + elif action == "get_elements": + selector = params.get("selector", "body") + elements = await page.evaluate(f""" + (() => {{ + const els = document.querySelectorAll('{selector} a, {selector} button, {selector} input, {selector} select, {selector} textarea, {selector} [role="button"], {selector} [onclick]'); + return Array.from(els).slice(0, 50).map(el => ({{ + tag: el.tagName.toLowerCase(), + text: (el.textContent || '').trim().slice(0, 100), + selector: el.id ? '#' + el.id : (el.className ? '.' + el.className.split(' ')[0] : el.tagName.toLowerCase()), + href: el.href || null, + type: el.type || null, + placeholder: el.placeholder || null, + }})); + }})() + """) + lines = [] + for el in (elements or []): + desc = f"{el['tag']}" + if el.get("text"): + desc += f" \"{el['text'][:50]}\"" + if el.get("href"): + desc += f" → {el['href'][:80]}" + desc += f" selector: {el.get('selector', '?')}" + lines.append(desc) + return {"text": "\n".join(lines) if lines else "No interactive elements found"} + + else: + return {"error": f"Unknown action: {action}"} + + except Exception as e: + logger.warning(f"Headless browser error ({action}): {e}") + return {"error": str(e)} + + +async def close_page(browser_id: str): + """Close a specific page.""" + page = _pages.pop(browser_id, None) + if page and not page.is_closed(): + await page.close() + + +async def shutdown(): + """Close all pages and the browser.""" + global _browser, _playwright + for page in _pages.values(): + try: + if not page.is_closed(): + await page.close() + except Exception: + pass + _pages.clear() + + if _browser: + try: + await _browser.close() + except Exception: + pass + _browser = None + + if _playwright: + try: + await _playwright.stop() + except Exception: + pass + _playwright = None diff --git a/backend/apps/agents/invoke_agent_mcp_server.py b/backend/apps/agents/invoke_agent_mcp_server.py new file mode 100644 index 00000000..cf614b03 --- /dev/null +++ b/backend/apps/agents/invoke_agent_mcp_server.py @@ -0,0 +1,160 @@ +#!/usr/bin/env python3 +""" +Stdio MCP server that exposes the InvokeAgent tool. + +Launched as a subprocess by the Claude Agent SDK. Proxies invocation +requests to the OpenSwarm backend via HTTP, which forks the target +agent session and runs it with the new message. +""" + +import json +import sys +import os +import urllib.request +import urllib.error + +BACKEND_PORT = os.environ.get("OPENSWARM_PORT", "8324") +BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/invoke-agent/run" +PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") +DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "") + +TOOLS = [ + { + "name": "InvokeAgent", + "description": ( + "Invoke a copy of an existing agent session with a new message. " + "The invoked agent will have full context of its prior conversation " + "and will process the new message independently. Use this when you " + "need to query another agent about its prior work or ask it to " + "perform a follow-up task." + ), + "inputSchema": { + "type": "object", + "properties": { + "session_id": { + "type": "string", + "description": ( + "The session ID of the agent to invoke. This is the ID " + "from a selected Agent Card in the context." + ), + }, + "message": { + "type": "string", + "description": ( + "The message to send to the invoked agent. Be specific " + "about what you need from it." + ), + }, + }, + "required": ["session_id", "message"], + }, + }, +] + + +def send_response(id_, result=None, error=None): + msg = {"jsonrpc": "2.0", "id": id_} + if error is not None: + msg["error"] = error + else: + msg["result"] = result + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def call_backend(session_id: str, message: str) -> dict: + payload = json.dumps({ + "session_id": session_id, + "message": message, + "parent_session_id": PARENT_SESSION_ID, + "dashboard_id": DASHBOARD_ID, + }).encode() + req = urllib.request.Request( + BACKEND_URL, + data=payload, + headers={"Content-Type": "application/json"}, + method="POST", + ) + try: + with urllib.request.urlopen(req, timeout=600) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + body = e.read().decode() if e.fp else str(e) + return {"error": f"HTTP {e.code}: {body}"} + except Exception as e: + return {"error": str(e)} + + +def handle_tool_call(tool_name: str, arguments: dict) -> dict: + if tool_name != "InvokeAgent": + return {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True} + + session_id = arguments.get("session_id", "") + message = arguments.get("message", "") + + if not session_id: + return {"content": [{"type": "text", "text": "Error: session_id is required"}], "isError": True} + if not message: + return {"content": [{"type": "text", "text": "Error: message is required"}], "isError": True} + + result = call_backend(session_id, message) + + if "error" in result: + return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} + + forked_id = result.get("forked_session_id", "") + response = result.get("response", "No response from invoked agent.") + cost = result.get("cost_usd", 0) + source_name = result.get("source_name", "") + + lines = [f"**Invoked Agent Result** (forked session: {forked_id})"] + if source_name: + lines[0] = f"**Invoked Agent Result** — {source_name} (forked session: {forked_id})" + if cost > 0: + lines.append(f"*Cost: ${cost:.4f}*") + lines.append("") + lines.append(response) + + return {"content": [{"type": "text", "text": "\n".join(lines)}]} + + +def main(): + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + + method = msg.get("method") + id_ = msg.get("id") + params = msg.get("params", {}) + + if method == "initialize": + send_response(id_, { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": { + "name": "openswarm-invoke-agent", + "version": "1.0.0", + }, + }) + elif method == "notifications/initialized": + pass + elif method == "tools/list": + send_response(id_, {"tools": TOOLS}) + elif method == "tools/call": + tool_name = params.get("name", "") + arguments = params.get("arguments", {}) + result = handle_tool_call(tool_name, arguments) + send_response(id_, result) + elif method == "ping": + send_response(id_, {}) + elif id_ is not None: + send_response(id_, error={"code": -32601, "message": f"Method not found: {method}"}) + + +if __name__ == "__main__": + main() diff --git a/backend/apps/agents/mcp_client.py b/backend/apps/agents/mcp_client.py new file mode 100644 index 00000000..303078ab --- /dev/null +++ b/backend/apps/agents/mcp_client.py @@ -0,0 +1,360 @@ +"""Standalone MCP client manager for agent sessions. + +Replaces claude_agent_sdk's internal MCP server management. +One MCPClientManager instance per agent session — manages connections +to stdio/http/sse MCP servers, discovers tools, and routes tool calls. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from contextlib import AsyncExitStack +from dataclasses import dataclass, field +from typing import Any + +from backend.apps.agents.providers.base import ToolSchema + +logger = logging.getLogger(__name__) + + +@dataclass +class MCPConnection: + """A live connection to an MCP server.""" + server_name: str + session: Any # mcp.ClientSession + tools: list[ToolSchema] = field(default_factory=list) + + +class MCPClientManager: + """Manages connections to MCP servers for a single agent session.""" + + def __init__(self): + self._connections: dict[str, MCPConnection] = {} + self._exit_stack = AsyncExitStack() + self._started = False + + async def __aenter__(self): + await self._exit_stack.__aenter__() + self._started = True + return self + + async def __aexit__(self, *exc): + await self.disconnect_all() + try: + await self._exit_stack.__aexit__(*exc) + except (BaseExceptionGroup, ExceptionGroup, Exception) as e: + # MCP subprocess cleanup errors are non-fatal + logger.warning(f"MCP cleanup error (non-fatal): {e}") + self._started = False + + async def connect(self, server_name: str, config: dict, timeout: float = 30.0) -> list[ToolSchema]: + """Connect to an MCP server and return its available tools. + + The tools are returned with names prefixed as mcp____. + """ + transport = config.get("type", "stdio") + try: + if transport == "stdio": + coro = self._connect_stdio(server_name, config) + elif transport == "sse": + coro = self._connect_sse(server_name, config) + elif transport == "http": + coro = self._connect_http(server_name, config) + else: + logger.warning(f"Unsupported MCP transport: {transport} for {server_name}") + return [] + + conn = await asyncio.wait_for(coro, timeout=timeout) + self._connections[server_name] = conn + logger.info(f"MCP connected: {server_name} ({len(conn.tools)} tools)") + return conn.tools + + except asyncio.TimeoutError: + logger.warning(f"MCP server {server_name} connection timed out after {timeout}s") + return [] + except Exception as e: + logger.warning(f"Failed to connect MCP server {server_name}: {e}") + return [] + + async def _connect_stdio(self, server_name: str, config: dict) -> MCPConnection: + """Connect to a stdio MCP server (spawns a subprocess).""" + from mcp import ClientSession + from mcp.client.stdio import stdio_client, StdioServerParameters + + command = config.get("command", "") + args = config.get("args", []) + env = config.get("env") + + params = StdioServerParameters( + command=command, + args=args, + env=env, + ) + + transport = await self._exit_stack.enter_async_context( + stdio_client(params) + ) + read_stream, write_stream = transport + session = await self._exit_stack.enter_async_context( + ClientSession(read_stream, write_stream) + ) + await session.initialize() + + result = await session.list_tools() + tools = [ + ToolSchema( + name=f"mcp__{server_name}__{t.name}", + description=t.description or "", + input_schema=t.inputSchema if hasattr(t, "inputSchema") else (t.input_schema if hasattr(t, "input_schema") else {}), + ) + for t in result.tools + ] + + return MCPConnection(server_name=server_name, session=session, tools=tools) + + async def _connect_sse(self, server_name: str, config: dict) -> MCPConnection: + """Connect to an SSE MCP server.""" + from mcp import ClientSession + from mcp.client.sse import sse_client + + url = config.get("url", "") + headers = config.get("headers") + + transport = await self._exit_stack.enter_async_context( + sse_client(url=url, headers=headers, timeout=30, sse_read_timeout=300) + ) + read_stream, write_stream = transport + session = await self._exit_stack.enter_async_context( + ClientSession(read_stream, write_stream) + ) + await session.initialize() + + result = await session.list_tools() + tools = [ + ToolSchema( + name=f"mcp__{server_name}__{t.name}", + description=t.description or "", + input_schema=t.inputSchema if hasattr(t, "inputSchema") else (t.input_schema if hasattr(t, "input_schema") else {}), + ) + for t in result.tools + ] + + return MCPConnection(server_name=server_name, session=session, tools=tools) + + async def _connect_http(self, server_name: str, config: dict) -> MCPConnection: + """Connect to a Streamable HTTP MCP server. + + Falls back to SSE if streamable HTTP fails. + """ + url = config.get("url", "") + headers = config.get("headers") + + # Try streamable HTTP first, fall back to SSE + try: + return await self._connect_http_streamable(server_name, url, headers) + except Exception as e: + logger.info(f"Streamable HTTP failed for {server_name}, trying SSE: {e}") + return await self._connect_sse(server_name, config) + + async def _connect_http_streamable( + self, server_name: str, url: str, headers: dict | None, + ) -> MCPConnection: + """Connect via Streamable HTTP (JSON-RPC POST).""" + import httpx + from mcp import ClientSession + + # Use httpx for streamable HTTP — keep client alive in the exit stack + client = await self._exit_stack.enter_async_context( + httpx.AsyncClient(timeout=30.0) + ) + + h = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + **(headers or {}), + } + + # Initialize + init_resp = await client.post(url, headers=h, json={ + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "self-swarm", "version": "0.1.0"}, + }, + }) + if init_resp.status_code not in (200, 201): + raise ConnectionError(f"MCP initialize failed: {init_resp.status_code}") + + session_id = init_resp.headers.get("mcp-session-id", "") + if session_id: + h["mcp-session-id"] = session_id + + # Notify initialized + await client.post(url, headers=h, json={ + "jsonrpc": "2.0", "method": "notifications/initialized", + }) + + # List tools + list_resp = await client.post(url, headers=h, json={ + "jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}, + }) + if list_resp.status_code not in (200, 201): + raise ConnectionError(f"MCP tools/list failed: {list_resp.status_code}") + + ct = list_resp.headers.get("content-type", "") + if "text/event-stream" in ct: + data = self._parse_sse_json(list_resp.text) + else: + data = list_resp.json() + + if not data: + raise ConnectionError("Empty response from MCP server") + + tools_list = data.get("result", {}).get("tools", []) + tools = [ + ToolSchema( + name=f"mcp__{server_name}__{t.get('name', '')}", + description=t.get("description", ""), + input_schema=t.get("inputSchema", t.get("input_schema", {})), + ) + for t in tools_list + ] + + # Store the HTTP client info for call_tool + conn = MCPConnection(server_name=server_name, session=None, tools=tools) + conn._http_client = client # type: ignore[attr-defined] + conn._http_url = url # type: ignore[attr-defined] + conn._http_headers = h # type: ignore[attr-defined] + conn._next_id = 3 # type: ignore[attr-defined] + return conn + + @staticmethod + def _parse_sse_json(text: str) -> dict | None: + """Extract JSON from an SSE response body.""" + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("data:"): + payload = stripped[len("data:"):].strip() + if payload: + try: + return json.loads(payload) + except json.JSONDecodeError: + continue + try: + return json.loads(text) + except json.JSONDecodeError: + return None + + async def call_tool( + self, server_name: str, tool_name: str, arguments: dict, + ) -> list[dict]: + """Call a tool on a specific MCP server. + + Args: + server_name: The MCP server name (e.g. "google-workspace") + tool_name: The bare tool name (without mcp__prefix) + arguments: Tool input arguments + + Returns: + List of content blocks: [{"type": "text", "text": "..."}] + """ + conn = self._connections.get(server_name) + if not conn: + return [{"type": "text", "text": f"MCP server {server_name} not connected"}] + + try: + if conn.session is not None: + # stdio or SSE — use MCP ClientSession + result = await conn.session.call_tool(tool_name, arguments) + return self._format_mcp_result(result) + elif hasattr(conn, "_http_client"): + # Streamable HTTP — use JSON-RPC + return await self._call_tool_http(conn, tool_name, arguments) + else: + return [{"type": "text", "text": f"No session for MCP server {server_name}"}] + + except Exception as e: + logger.warning(f"MCP tool call failed: {server_name}/{tool_name}: {e}") + return [{"type": "text", "text": f"Error calling {tool_name}: {e}"}] + + async def _call_tool_http( + self, conn: MCPConnection, tool_name: str, arguments: dict, + ) -> list[dict]: + """Call a tool via Streamable HTTP.""" + client = conn._http_client # type: ignore[attr-defined] + url = conn._http_url # type: ignore[attr-defined] + headers = conn._http_headers # type: ignore[attr-defined] + req_id = conn._next_id # type: ignore[attr-defined] + conn._next_id = req_id + 1 # type: ignore[attr-defined] + + resp = await client.post(url, headers=headers, json={ + "jsonrpc": "2.0", + "id": req_id, + "method": "tools/call", + "params": {"name": tool_name, "arguments": arguments}, + }, timeout=300.0) + + ct = resp.headers.get("content-type", "") + if "text/event-stream" in ct: + data = self._parse_sse_json(resp.text) + else: + data = resp.json() + + if not data: + return [{"type": "text", "text": "Empty response from MCP server"}] + + if "error" in data: + return [{"type": "text", "text": f"MCP error: {data['error']}"}] + + result = data.get("result", {}) + content = result.get("content", []) + return content if content else [{"type": "text", "text": json.dumps(result)}] + + @staticmethod + def _format_mcp_result(result: Any) -> list[dict]: + """Convert an MCP CallToolResult to content blocks.""" + if hasattr(result, "content"): + blocks = [] + for item in result.content: + if hasattr(item, "text"): + blocks.append({"type": "text", "text": item.text}) + elif hasattr(item, "data"): + blocks.append({ + "type": "image", + "source": { + "type": "base64", + "media_type": getattr(item, "mimeType", "image/png"), + "data": item.data, + }, + }) + else: + blocks.append({"type": "text", "text": str(item)}) + return blocks if blocks else [{"type": "text", "text": "Done."}] + + return [{"type": "text", "text": str(result)}] + + def get_all_tool_schemas(self) -> list[ToolSchema]: + """Return tool schemas from all connected MCP servers.""" + schemas = [] + for conn in self._connections.values(): + schemas.extend(conn.tools) + return schemas + + def parse_mcp_tool_name(self, full_name: str) -> tuple[str, str] | None: + """Parse mcp____ into (server_name, tool_name). + + Returns None if the name doesn't match the MCP naming convention. + """ + import re + m = re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", full_name) + if m: + return m.group(1), m.group(2) + return None + + async def disconnect_all(self): + """Disconnect all MCP servers. Called on session end.""" + self._connections.clear() + # The AsyncExitStack handles actual cleanup of transports/sessions diff --git a/backend/apps/agents/models.py b/backend/apps/agents/models.py index a5f03df3..d5e6ccfe 100644 --- a/backend/apps/agents/models.py +++ b/backend/apps/agents/models.py @@ -5,6 +5,7 @@ from uuid import uuid4 class AgentConfig(BaseModel): name: str = Field(default_factory=lambda: f"Agent-{uuid4().hex[:6]}") + provider: str = "anthropic" model: str = "sonnet" mode: str = "agent" system_prompt: Optional[str] = None @@ -37,6 +38,7 @@ class Message(BaseModel): attached_skills: Optional[list[dict]] = None forced_tools: Optional[list[str]] = None images: Optional[list[dict]] = None + hidden: bool = False class MessageBranch(BaseModel): id: str = Field(default_factory=lambda: uuid4().hex) @@ -54,6 +56,7 @@ class AgentSession(BaseModel): id: str = Field(default_factory=lambda: uuid4().hex) name: str status: Literal["running", "waiting_approval", "completed", "error", "stopped"] = "running" + provider: str = "anthropic" model: str = "sonnet" mode: str = "agent" sdk_session_id: Optional[str] = None @@ -72,3 +75,4 @@ class AgentSession(BaseModel): tool_group_meta: dict[str, "ToolGroupMeta"] = Field(default_factory=dict) dashboard_id: Optional[str] = None browser_id: Optional[str] = None + parent_session_id: Optional[str] = None diff --git a/backend/apps/agents/providers/__init__.py b/backend/apps/agents/providers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/agents/providers/anthropic.py b/backend/apps/agents/providers/anthropic.py new file mode 100644 index 00000000..bd347ead --- /dev/null +++ b/backend/apps/agents/providers/anthropic.py @@ -0,0 +1,260 @@ +"""Anthropic provider adapter using the native Anthropic SDK.""" + +from __future__ import annotations + +import json +import logging +from typing import Any, AsyncIterator + +import anthropic + +from backend.apps.agents.providers.base import ( + BaseProvider, ContentBlock, ModelResponse, ProviderMessage, + StreamEvent, ToolCall, ToolSchema, +) + +logger = logging.getLogger(__name__) + +MODEL_MAP = { + "sonnet": "claude-sonnet-4-6", + "opus": "claude-opus-4-6", + "haiku": "claude-haiku-4-5", +} + + +class AnthropicProvider(BaseProvider): + """Provider adapter for Anthropic's Messages API.""" + + def __init__( + self, + api_key: str | None = None, + auth_token: str | None = None, + base_url: str | None = None, + ): + kwargs: dict[str, Any] = {} + if auth_token: + kwargs["auth_token"] = auth_token + elif api_key: + kwargs["api_key"] = api_key + if base_url: + kwargs["base_url"] = base_url + self.client = anthropic.AsyncAnthropic(**kwargs) + + def get_model_id(self, short_name: str) -> str: + return MODEL_MAP.get(short_name, short_name) + + def clean_tool_schema(self, schema: ToolSchema) -> dict: + return { + "name": schema.name, + "description": schema.description, + "input_schema": schema.input_schema, + } + + def format_tool_result(self, tool_use_id: str, content: list[dict]) -> dict: + return { + "type": "tool_result", + "tool_use_id": tool_use_id, + "content": content, + } + + def format_user_message(self, content: Any) -> ProviderMessage: + return ProviderMessage(role="user", content=content) + + def format_assistant_message(self, response: ModelResponse) -> ProviderMessage: + blocks = [] + for block in response.content: + if block.type == "text": + blocks.append({"type": "text", "text": block.text}) + elif block.type == "tool_use" and block.tool_call: + blocks.append({ + "type": "tool_use", + "id": block.tool_call.id, + "name": block.tool_call.name, + "input": block.tool_call.input, + }) + return ProviderMessage(role="assistant", content=blocks) + + def _build_messages(self, messages: list[ProviderMessage]) -> list[dict]: + """Convert ProviderMessages to Anthropic API format.""" + result = [] + for msg in messages: + if msg.role == "tool_result": + # Tool results: content is a list of tool_result dicts + if isinstance(msg.content, list): + result.append({"role": "user", "content": msg.content}) + else: + result.append({"role": "user", "content": [msg.content]}) + elif msg.role == "assistant": + result.append({"role": "assistant", "content": msg.content}) + elif msg.role == "user": + result.append({"role": "user", "content": msg.content}) + return result + + async def create_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> ModelResponse: + kwargs: dict[str, Any] = { + "model": self.get_model_id(model), + "max_tokens": max_tokens, + "messages": self._build_messages(messages), + } + if system: + kwargs["system"] = system + if tools: + kwargs["tools"] = [self.clean_tool_schema(t) for t in tools] + + resp = await self.client.messages.create(**kwargs) + + content = [] + for block in resp.content: + if block.type == "text": + content.append(ContentBlock(type="text", text=block.text)) + elif block.type == "tool_use": + content.append(ContentBlock( + type="tool_use", + tool_call=ToolCall( + id=block.id, + name=block.name, + input=block.input, + ), + )) + + return ModelResponse( + content=content, + stop_reason="tool_use" if resp.stop_reason == "tool_use" else "end_turn", + usage={ + "input_tokens": resp.usage.input_tokens, + "output_tokens": resp.usage.output_tokens, + }, + ) + + async def stream_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> AsyncIterator[StreamEvent]: + kwargs: dict[str, Any] = { + "model": self.get_model_id(model), + "max_tokens": max_tokens, + "messages": self._build_messages(messages), + } + if system: + kwargs["system"] = system + if tools: + kwargs["tools"] = [self.clean_tool_schema(t) for t in tools] + + # Use create() with stream=True for raw SSE events + kwargs["stream"] = True + raw_stream = await self.client.messages.create(**kwargs) + + current_block_type: dict[int, str] = {} + current_tool_name: dict[int, str] = {} + current_tool_id: dict[int, str] = {} + current_text: dict[int, str] = {} + current_json: dict[int, str] = {} + + async for event in raw_stream: + event_type = getattr(event, "type", "") + + if event_type == "content_block_start": + index = event.index + block = event.content_block + block_type = block.type + current_block_type[index] = block_type + + if block_type == "text": + current_text[index] = "" + yield StreamEvent( + type="content_block_start", + index=index, + block_type="text", + ) + elif block_type == "tool_use": + current_tool_name[index] = block.name + current_tool_id[index] = block.id + current_json[index] = "" + yield StreamEvent( + type="content_block_start", + index=index, + block_type="tool_use", + tool_name=block.name, + tool_id=block.id, + ) + + elif event_type == "content_block_delta": + index = event.index + delta = event.delta + delta_type = delta.type + + if delta_type == "text_delta": + current_text.setdefault(index, "") + current_text[index] += delta.text + yield StreamEvent( + type="content_block_delta", + index=index, + delta_type="text_delta", + text=delta.text, + ) + elif delta_type == "input_json_delta": + current_json.setdefault(index, "") + current_json[index] += delta.partial_json + yield StreamEvent( + type="content_block_delta", + index=index, + delta_type="input_json_delta", + text=delta.partial_json, + ) + + elif event_type == "content_block_stop": + yield StreamEvent(type="content_block_stop", index=event.index) + + elif event_type == "message_delta": + # Extract output token usage from the final delta + usage_data = {} + delta_usage = getattr(event, "usage", None) + if delta_usage: + output_tokens = getattr(delta_usage, "output_tokens", 0) + if output_tokens: + usage_data["output_tokens"] = output_tokens + if usage_data: + yield StreamEvent(type="usage", usage=usage_data) + + elif event_type == "message_start": + # Extract input token usage from the message start + msg = getattr(event, "message", None) + if msg: + msg_usage = getattr(msg, "usage", None) + if msg_usage: + usage_data = {} + input_tokens = getattr(msg_usage, "input_tokens", 0) + output_tokens = getattr(msg_usage, "output_tokens", 0) + if input_tokens: + usage_data["input_tokens"] = input_tokens + if output_tokens: + usage_data["output_tokens"] = output_tokens + if usage_data: + yield StreamEvent(type="usage", usage=usage_data) + + yield StreamEvent(type="message_stop") + + async def stream_and_collect( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> tuple[AsyncIterator[StreamEvent], ModelResponse]: + """Helper: stream events and also return the full collected response. + + Not used directly — the AgentLoop handles collection. + """ + raise NotImplementedError("Use stream_message() directly; AgentLoop collects.") diff --git a/backend/apps/agents/providers/base.py b/backend/apps/agents/providers/base.py new file mode 100644 index 00000000..ef2f746e --- /dev/null +++ b/backend/apps/agents/providers/base.py @@ -0,0 +1,135 @@ +"""Provider-agnostic base classes for multi-model support. + +All provider adapters (Anthropic, OpenAI, Gemini, OpenAI-compatible) +implement BaseProvider, translating their native APIs into these +common data structures. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any, AsyncIterator + + +@dataclass +class ToolSchema: + """Provider-agnostic tool definition.""" + name: str + description: str + input_schema: dict[str, Any] + + +@dataclass +class ToolCall: + """A tool invocation requested by the model.""" + id: str + name: str + input: dict[str, Any] + + +@dataclass +class ContentBlock: + """A block of content from the model response.""" + type: str # "text" | "tool_use" + text: str = "" + tool_call: ToolCall | None = None + + +@dataclass +class ModelResponse: + """Complete (non-streaming) response from a provider.""" + content: list[ContentBlock] + stop_reason: str # "end_turn" | "tool_use" | "max_tokens" + usage: dict[str, int] = field(default_factory=dict) + + +@dataclass +class StreamEvent: + """A single streaming event, normalized across providers. + + The event types match what the frontend already expects via WebSocket: + content_block_start, content_block_delta, content_block_stop, message_stop. + """ + type: str + index: int = 0 + block_type: str = "" # "text" | "tool_use" + delta_type: str = "" # "text_delta" | "input_json_delta" + text: str = "" + tool_name: str = "" + tool_id: str = "" + usage: dict[str, int] = field(default_factory=dict) + + +@dataclass +class ProviderMessage: + """Provider-agnostic message for conversation history. + + Each provider adapter converts these to/from its native format. + """ + role: str # "user" | "assistant" | "tool_result" + content: Any # str, list[dict], or provider-specific content + + +class BaseProvider(ABC): + """Abstract base for LLM provider adapters.""" + + @abstractmethod + async def stream_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> AsyncIterator[StreamEvent]: + """Stream a model response, yielding normalized StreamEvents.""" + ... + + @abstractmethod + async def create_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> ModelResponse: + """Non-streaming message creation.""" + ... + + @abstractmethod + def format_tool_result( + self, + tool_use_id: str, + content: list[dict], + ) -> dict: + """Format a tool result in this provider's expected message format.""" + ... + + @abstractmethod + def format_user_message(self, content: Any) -> ProviderMessage: + """Wrap user content (str or multimodal blocks) into a ProviderMessage.""" + ... + + @abstractmethod + def format_assistant_message(self, response: ModelResponse) -> ProviderMessage: + """Convert a ModelResponse into a ProviderMessage for conversation history.""" + ... + + @abstractmethod + def get_model_id(self, short_name: str) -> str: + """Resolve a short model name to the full API model ID.""" + ... + + def clean_tool_schema(self, schema: ToolSchema) -> dict: + """Convert a ToolSchema to the provider's native tool format. + + Default: Anthropic-style format. Override for providers that need + different formats or schema cleaning (e.g. Gemini). + """ + return { + "name": schema.name, + "description": schema.description, + "input_schema": schema.input_schema, + } diff --git a/backend/apps/agents/providers/copilot.py b/backend/apps/agents/providers/copilot.py new file mode 100644 index 00000000..84e0a0ff --- /dev/null +++ b/backend/apps/agents/providers/copilot.py @@ -0,0 +1,121 @@ +"""GitHub Copilot provider — routes through Copilot's OpenAI-compatible API. + +Uses the user's GitHub Copilot subscription to access Claude, GPT, and other models. +Extends OpenAICompatProvider since Copilot's API speaks the OpenAI format. +""" + +from __future__ import annotations + +import logging +import time +from typing import Any, AsyncIterator + +from openai import AsyncOpenAI + +from backend.apps.agents.providers.base import ( + BaseProvider, ProviderMessage, StreamEvent, ToolSchema, ModelResponse, +) +from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + +logger = logging.getLogger(__name__) + +COPILOT_API_BASE = "https://api.githubcopilot.com" + + +class CopilotProvider(OpenAICompatProvider): + """Provider that routes through GitHub Copilot's API.""" + + def __init__(self, copilot_token: str): + # Initialize OpenAI client pointing at Copilot's API + self.client = AsyncOpenAI( + api_key=copilot_token, + base_url=COPILOT_API_BASE, + ) + # Store token for header injection + self._copilot_token = copilot_token + + def get_model_id(self, short_name: str) -> str: + # Copilot uses same model IDs — pass through + return short_name + + async def stream_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> AsyncIterator[StreamEvent]: + """Stream with Copilot-specific headers.""" + kwargs: dict[str, Any] = { + "model": self.get_model_id(model), + "max_tokens": max_tokens, + "messages": self._build_messages(system, messages), + "stream": True, + "extra_headers": { + "copilot-integration-id": "vscode-chat", + }, + } + if tools: + kwargs["tools"] = [self.clean_tool_schema(t) for t in tools] + + stream = await self.client.chat.completions.create(**kwargs) + + # Reuse parent's stream parsing logic + text_started = False + text_index = 0 + tool_indices: dict[int, dict] = {} + next_block_index = 0 + + from uuid import uuid4 + + async for chunk in stream: + if not chunk.choices: + continue + + delta = chunk.choices[0].delta + finish_reason = chunk.choices[0].finish_reason + + if delta.content is not None: + if not text_started: + text_started = True + text_index = next_block_index + next_block_index += 1 + yield StreamEvent(type="content_block_start", index=text_index, block_type="text") + yield StreamEvent(type="content_block_delta", index=text_index, delta_type="text_delta", text=delta.content) + + if delta.tool_calls: + for tc_delta in delta.tool_calls: + tc_idx = tc_delta.index + if tc_idx not in tool_indices: + if text_started: + yield StreamEvent(type="content_block_stop", index=text_index) + text_started = False + block_idx = next_block_index + next_block_index += 1 + tool_indices[tc_idx] = { + "block_index": block_idx, + "id": tc_delta.id or uuid4().hex, + "name": tc_delta.function.name if tc_delta.function else "", + "json_buf": "", + } + yield StreamEvent( + type="content_block_start", index=block_idx, block_type="tool_use", + tool_name=tool_indices[tc_idx]["name"], tool_id=tool_indices[tc_idx]["id"], + ) + info = tool_indices[tc_idx] + if tc_delta.function and tc_delta.function.name: + info["name"] = tc_delta.function.name + if tc_delta.function and tc_delta.function.arguments: + info["json_buf"] += tc_delta.function.arguments + yield StreamEvent( + type="content_block_delta", index=info["block_index"], + delta_type="input_json_delta", text=tc_delta.function.arguments, + ) + + if finish_reason is not None: + if text_started: + yield StreamEvent(type="content_block_stop", index=text_index) + for info in tool_indices.values(): + yield StreamEvent(type="content_block_stop", index=info["block_index"]) + yield StreamEvent(type="message_stop") diff --git a/backend/apps/agents/providers/gemini.py b/backend/apps/agents/providers/gemini.py new file mode 100644 index 00000000..7abb6a20 --- /dev/null +++ b/backend/apps/agents/providers/gemini.py @@ -0,0 +1,600 @@ +"""Gemini provider adapter using the new google-genai SDK.""" + +from __future__ import annotations + +import base64 +import json +import logging +from copy import deepcopy +from typing import Any, AsyncIterator +from uuid import uuid4 + +from google import genai +from google.genai import types + +from backend.apps.agents.providers.base import ( + BaseProvider, + ContentBlock, + ModelResponse, + ProviderMessage, + StreamEvent, + ToolCall, + ToolSchema, +) + +logger = logging.getLogger(__name__) + +MODEL_MAP = { + "flash": "gemini-2.5-flash", + "pro": "gemini-2.5-pro", +} + +# JSON Schema keywords that Gemini does not support +_UNSUPPORTED_VALIDATION_KEYS = frozenset({ + "minLength", + "maxLength", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "pattern", + "minItems", + "maxItems", + "uniqueItems", + "minProperties", + "maxProperties", + "multipleOf", + "format", + "const", +}) + +_UNSUPPORTED_STRUCTURAL_KEYS = frozenset({ + "$ref", + "$defs", + "patternProperties", +}) + + +# --------------------------------------------------------------------------- +# Schema cleaning — port of OpenClaw's clean-for-gemini.ts logic +# --------------------------------------------------------------------------- + +def _resolve_ref(ref: str, root_defs: dict[str, Any]) -> dict[str, Any]: + """Attempt to resolve a $ref pointer like '#/$defs/Foo'.""" + if ref.startswith("#/$defs/"): + name = ref[len("#/$defs/"):] + if name in root_defs: + return deepcopy(root_defs[name]) + # Cannot resolve — return empty object + return {"type": "object"} + + +def _clean_schema_node(node: dict[str, Any], root_defs: dict[str, Any]) -> dict[str, Any]: + """Recursively clean a single JSON Schema node for Gemini compatibility.""" + if not isinstance(node, dict): + return node + + # If this node is just a $ref, resolve it first then clean the result + if "$ref" in node and len(node) <= 2: # $ref possibly with description + resolved = _resolve_ref(node["$ref"], root_defs) + # Carry over description if the ref node had one + if "description" in node: + resolved["description"] = node["description"] + return _clean_schema_node(resolved, root_defs) + + result: dict[str, Any] = {} + + for key, value in node.items(): + # Drop unsupported keys + if key in _UNSUPPORTED_VALIDATION_KEYS: + continue + if key in _UNSUPPORTED_STRUCTURAL_KEYS: + continue + + # Handle additionalProperties: drop if boolean, recurse if schema + if key == "additionalProperties": + if isinstance(value, bool): + continue + # It's a schema dict — clean and keep it + result[key] = _clean_schema_node(value, root_defs) + continue + + # Handle anyOf / oneOf: flatten into something Gemini can use + if key in ("anyOf", "oneOf"): + if isinstance(value, list): + flattened = _flatten_union(value, root_defs) + if flattened is not None: + result.update(flattened) + continue + + # Handle allOf: merge all members + if key == "allOf": + if isinstance(value, list): + merged = _merge_all_of(value, root_defs) + result.update(merged) + continue + + # Recurse into properties + if key == "properties" and isinstance(value, dict): + result[key] = { + prop_name: _clean_schema_node(prop_schema, root_defs) + for prop_name, prop_schema in value.items() + } + continue + + # Recurse into items + if key == "items": + if isinstance(value, dict): + result[key] = _clean_schema_node(value, root_defs) + elif isinstance(value, list): + result[key] = [_clean_schema_node(item, root_defs) for item in value] + else: + result[key] = value + continue + + # Keep everything else (type, description, title, default, enum, required, etc.) + result[key] = value + + return result + + +def _flatten_union( + variants: list[dict[str, Any]], + root_defs: dict[str, Any], +) -> dict[str, Any] | None: + """Collapse anyOf/oneOf into a Gemini-compatible schema. + + Strategies (applied in order): + 1. If all variants are literal types (with const or single-value enums), + collapse into a single enum. + 2. If there's a null variant mixed with non-null variants, strip the null + variant and return the remaining schema (nullable). + 3. If only one non-null variant remains after stripping, unwrap it. + 4. Otherwise return the first variant (best-effort). + """ + if not variants: + return None + + # Clean each variant first (resolve refs, etc.) + cleaned = [_clean_schema_node(v, root_defs) for v in variants] + + # Strategy 1: all literal types → collapse to enum + enum_values: list[Any] = [] + all_literals = True + for v in cleaned: + if "const" in v: + enum_values.append(v["const"]) + elif "enum" in v and isinstance(v["enum"], list) and len(v["enum"]) == 1: + enum_values.append(v["enum"][0]) + else: + all_literals = False + break + if all_literals and enum_values: + return {"type": "string", "enum": enum_values} + + # Strategy 2 & 3: strip null variants + non_null = [v for v in cleaned if v.get("type") != "null"] + + if len(non_null) == 0: + # All variants are null + return {"type": "string"} + + if len(non_null) == 1: + # Single non-null variant — unwrap it, mark as nullable + result = non_null[0].copy() + result["nullable"] = True + return result + + # Strategy 4: multiple non-null variants, just use first (best-effort) + return non_null[0] + + +def _merge_all_of( + members: list[dict[str, Any]], + root_defs: dict[str, Any], +) -> dict[str, Any]: + """Merge allOf members into a single cleaned schema.""" + merged: dict[str, Any] = {} + for member in members: + cleaned = _clean_schema_node(member, root_defs) + for key, value in cleaned.items(): + if key == "properties" and key in merged: + merged[key].update(value) + elif key == "required" and key in merged: + existing = set(merged[key]) + existing.update(value) + merged[key] = sorted(existing) + else: + merged[key] = value + return merged + + +def clean_schema_for_gemini(schema: dict[str, Any]) -> dict[str, Any]: + """Top-level entry point: clean a full JSON Schema for Gemini compatibility.""" + root_defs = schema.get("$defs", schema.get("definitions", {})) + return _clean_schema_node(schema, root_defs) + + +# --------------------------------------------------------------------------- +# Provider implementation +# --------------------------------------------------------------------------- + +class GeminiProvider(BaseProvider): + """Provider adapter for Google Gemini via the google-genai SDK.""" + + def __init__(self, api_key: str): + self.client = genai.Client(api_key=api_key) + + def get_model_id(self, short_name: str) -> str: + return MODEL_MAP.get(short_name, short_name) + + def clean_tool_schema(self, schema: ToolSchema) -> dict: + """Convert a ToolSchema to Gemini function declaration format. + + Cleans the JSON Schema of unsupported features before sending. + """ + cleaned_params = clean_schema_for_gemini(schema.input_schema) + return { + "name": schema.name, + "description": schema.description, + "parameters": cleaned_params, + } + + def format_tool_result(self, tool_use_id: str, content: list[dict]) -> dict: + """Format a tool result for Gemini conversation history. + + Gemini uses FunctionResponse parts; we store enough info to reconstruct them. + """ + # Extract text content for the function response + text_parts = [] + for block in content: + if block.get("type") == "text": + text_parts.append(block.get("text", "")) + elif block.get("type") == "image": + text_parts.append("[image]") + else: + text_parts.append(json.dumps(block)) + + return { + "tool_use_id": tool_use_id, + "output": "\n".join(text_parts) if text_parts else "Done.", + } + + def format_user_message(self, content: Any) -> ProviderMessage: + if isinstance(content, str): + return ProviderMessage(role="user", content=content) + return ProviderMessage(role="user", content=content) + + def format_assistant_message(self, response: ModelResponse) -> ProviderMessage: + """Convert a ModelResponse into a ProviderMessage for conversation history.""" + blocks = [] + for block in response.content: + if block.type == "text": + blocks.append({"type": "text", "text": block.text}) + elif block.type == "tool_use" and block.tool_call: + blocks.append({ + "type": "tool_use", + "id": block.tool_call.id, + "name": block.tool_call.name, + "input": block.tool_call.input, + }) + return ProviderMessage(role="assistant", content=blocks) + + # ------------------------------------------------------------------ + # Internal helpers for building Gemini API messages + # ------------------------------------------------------------------ + + def _build_tools(self, tools: list[ToolSchema]) -> list[types.Tool] | None: + """Convert ToolSchemas to Gemini Tool objects.""" + if not tools: + return None + declarations = [] + for t in tools: + cleaned = self.clean_tool_schema(t) + declarations.append(types.FunctionDeclaration( + name=cleaned["name"], + description=cleaned["description"], + parameters=cleaned["parameters"], + )) + return [types.Tool(function_declarations=declarations)] + + def _build_contents( + self, messages: list[ProviderMessage], + ) -> list[types.Content]: + """Convert ProviderMessages into a list of Gemini Content objects. + + Gemini requires: + - Conversation starts with a user message + - Strict alternating user/model turns + We merge consecutive same-role messages to satisfy this. + """ + raw_contents: list[types.Content] = [] + + for msg in messages: + if msg.role == "user": + parts = self._user_content_to_parts(msg.content) + raw_contents.append(types.Content(role="user", parts=parts)) + + elif msg.role == "assistant": + parts = self._assistant_content_to_parts(msg.content) + raw_contents.append(types.Content(role="model", parts=parts)) + + elif msg.role == "tool_result": + # Tool results become user turns with FunctionResponse parts + parts = self._tool_result_to_parts(msg.content) + raw_contents.append(types.Content(role="user", parts=parts)) + + # Ensure conversation starts with user + if raw_contents and raw_contents[0].role != "user": + raw_contents.insert( + 0, + types.Content( + role="user", + parts=[types.Part.from_text("Hello.")], + ), + ) + + # Merge consecutive same-role turns + merged: list[types.Content] = [] + for content in raw_contents: + if merged and merged[-1].role == content.role: + merged[-1].parts.extend(content.parts) + else: + merged.append(content) + + return merged + + def _user_content_to_parts(self, content: Any) -> list[types.Part]: + """Convert user message content to Gemini Part objects.""" + if isinstance(content, str): + return [types.Part.from_text(content)] + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, str): + parts.append(types.Part.from_text(block)) + elif isinstance(block, dict): + if block.get("type") == "text": + parts.append(types.Part.from_text(block.get("text", ""))) + elif block.get("type") == "image": + source = block.get("source", {}) + media_type = source.get("media_type", "image/png") + data = source.get("data", "") + parts.append(types.Part.from_bytes( + data=base64.b64decode(data), + mime_type=media_type, + )) + else: + parts.append(types.Part.from_text(json.dumps(block))) + return parts if parts else [types.Part.from_text("")] + return [types.Part.from_text(str(content))] + + def _assistant_content_to_parts(self, content: Any) -> list[types.Part]: + """Convert assistant message content to Gemini Part objects.""" + if isinstance(content, str): + return [types.Part.from_text(content)] + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + text = block.get("text", "") + if text: + parts.append(types.Part.from_text(text)) + elif block.get("type") == "tool_use": + parts.append(types.Part( + function_call=types.FunctionCall( + name=block.get("name", ""), + args=block.get("input", {}), + ) + )) + return parts if parts else [types.Part.from_text("")] + return [types.Part.from_text(str(content))] + + def _tool_result_to_parts(self, content: Any) -> list[types.Part]: + """Convert tool result content to Gemini FunctionResponse parts.""" + parts = [] + if isinstance(content, list): + for item in content: + if isinstance(item, dict) and "tool_use_id" in item: + # The tool_use_id is the function name in our format_tool_result + func_name = item.get("tool_use_id", "unknown") + output = item.get("output", "Done.") + parts.append(types.Part( + function_response=types.FunctionResponse( + name=func_name, + response={"result": output}, + ) + )) + elif isinstance(content, dict) and "tool_use_id" in content: + func_name = content.get("tool_use_id", "unknown") + output = content.get("output", "Done.") + parts.append(types.Part( + function_response=types.FunctionResponse( + name=func_name, + response={"result": output}, + ) + )) + + return parts if parts else [types.Part.from_text("Tool result unavailable.")] + + # ------------------------------------------------------------------ + # Non-streaming message creation + # ------------------------------------------------------------------ + + async def create_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> ModelResponse: + contents = self._build_contents(messages) + gemini_tools = self._build_tools(tools) + + config = types.GenerateContentConfig( + max_output_tokens=max_tokens, + ) + if system: + config.system_instruction = system + if gemini_tools: + config.tools = gemini_tools + + resp = await self.client.aio.models.generate_content( + model=self.get_model_id(model), + contents=contents, + config=config, + ) + + return self._parse_response(resp) + + def _parse_response(self, resp: Any) -> ModelResponse: + """Parse a Gemini GenerateContentResponse into a ModelResponse.""" + content: list[ContentBlock] = [] + has_tool_calls = False + + if resp.candidates: + candidate = resp.candidates[0] + if candidate.content and candidate.content.parts: + for part in candidate.content.parts: + if part.text is not None: + content.append(ContentBlock(type="text", text=part.text)) + elif part.function_call is not None: + has_tool_calls = True + fc = part.function_call + content.append(ContentBlock( + type="tool_use", + tool_call=ToolCall( + id=uuid4().hex, + name=fc.name, + input=dict(fc.args) if fc.args else {}, + ), + )) + + stop_reason = "tool_use" if has_tool_calls else "end_turn" + + usage = {} + if resp.usage_metadata: + usage = { + "input_tokens": getattr(resp.usage_metadata, "prompt_token_count", 0) or 0, + "output_tokens": getattr(resp.usage_metadata, "candidates_token_count", 0) or 0, + } + + return ModelResponse(content=content, stop_reason=stop_reason, usage=usage) + + # ------------------------------------------------------------------ + # Streaming message creation + # ------------------------------------------------------------------ + + async def stream_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> AsyncIterator[StreamEvent]: + contents = self._build_contents(messages) + gemini_tools = self._build_tools(tools) + + config = types.GenerateContentConfig( + max_output_tokens=max_tokens, + ) + if system: + config.system_instruction = system + if gemini_tools: + config.tools = gemini_tools + + stream = self.client.aio.models.generate_content_stream( + model=self.get_model_id(model), + contents=contents, + config=config, + ) + + # Track streaming state + block_index = 0 + text_block_open = False + tool_blocks: dict[str, int] = {} # tool_name -> block_index (for dedup) + + async for chunk in stream: + if not chunk.candidates: + continue + + candidate = chunk.candidates[0] + if not candidate.content or not candidate.content.parts: + continue + + for part in candidate.content.parts: + if part.text is not None: + text = part.text + if not text_block_open: + text_block_open = True + yield StreamEvent( + type="content_block_start", + index=block_index, + block_type="text", + ) + + yield StreamEvent( + type="content_block_delta", + index=block_index, + delta_type="text_delta", + text=text, + ) + + elif part.function_call is not None: + fc = part.function_call + + # Close text block if open + if text_block_open: + yield StreamEvent( + type="content_block_stop", + index=block_index, + ) + block_index += 1 + text_block_open = False + + tool_id = uuid4().hex + tool_block_idx = block_index + block_index += 1 + + args = dict(fc.args) if fc.args else {} + args_json = json.dumps(args) + + yield StreamEvent( + type="content_block_start", + index=tool_block_idx, + block_type="tool_use", + tool_name=fc.name, + tool_id=tool_id, + ) + yield StreamEvent( + type="content_block_delta", + index=tool_block_idx, + delta_type="input_json_delta", + text=args_json, + ) + yield StreamEvent( + type="content_block_stop", + index=tool_block_idx, + ) + + # Close any remaining open text block + if text_block_open: + yield StreamEvent( + type="content_block_stop", + index=block_index, + ) + + # Emit usage from the last chunk if available + if chunk and hasattr(chunk, 'usage_metadata') and chunk.usage_metadata: + um = chunk.usage_metadata + usage_data = { + "input_tokens": getattr(um, "prompt_token_count", 0) or 0, + "output_tokens": getattr(um, "candidates_token_count", 0) or 0, + } + if any(v > 0 for v in usage_data.values()): + yield StreamEvent(type="usage", usage=usage_data) + + yield StreamEvent(type="message_stop") diff --git a/backend/apps/agents/providers/openai_compat.py b/backend/apps/agents/providers/openai_compat.py new file mode 100644 index 00000000..bdc4a3b6 --- /dev/null +++ b/backend/apps/agents/providers/openai_compat.py @@ -0,0 +1,330 @@ +"""OpenAI-compatible provider adapter. + +Works with ANY endpoint that speaks the OpenAI Chat Completions API: +OpenAI, OpenRouter, Together, Groq, Fireworks, Mistral, Ollama, vLLM, etc. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any, AsyncIterator +from uuid import uuid4 + +from openai import AsyncOpenAI + +from backend.apps.agents.providers.base import ( + BaseProvider, ContentBlock, ModelResponse, ProviderMessage, + StreamEvent, ToolCall, ToolSchema, +) + +logger = logging.getLogger(__name__) + + +class OpenAICompatProvider(BaseProvider): + """Provider adapter for any OpenAI-compatible API endpoint.""" + + def __init__( + self, + api_key: str = "", + base_url: str | None = None, + ): + kwargs: dict[str, Any] = {} + # Always set api_key — use "none" as placeholder if empty (some endpoints don't need real keys) + kwargs["api_key"] = api_key if api_key else "none" + if base_url: + kwargs["base_url"] = base_url + self.client = AsyncOpenAI(**kwargs) + + def get_model_id(self, short_name: str) -> str: + # Pass through — user selects exact model ID + return short_name + + def clean_tool_schema(self, schema: ToolSchema) -> dict: + """Convert to OpenAI function calling format.""" + return { + "type": "function", + "function": { + "name": schema.name, + "description": schema.description, + "parameters": schema.input_schema, + }, + } + + def format_tool_result(self, tool_use_id: str, content: list[dict]) -> dict: + """Format tool result as OpenAI expects.""" + # OpenAI wants a single string for tool results + text_parts = [] + for block in content: + if block.get("type") == "text": + text_parts.append(block.get("text", "")) + elif block.get("type") == "image": + text_parts.append("[image]") + else: + text_parts.append(json.dumps(block)) + return { + "role": "tool", + "tool_call_id": tool_use_id, + "content": "\n".join(text_parts) if text_parts else "Done.", + } + + def format_user_message(self, content: Any) -> ProviderMessage: + """Convert user content to OpenAI format.""" + if isinstance(content, str): + return ProviderMessage(role="user", content=content) + # Multimodal content (text + images) + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + parts.append({"type": "text", "text": block["text"]}) + elif block.get("type") == "image": + source = block.get("source", {}) + media_type = source.get("media_type", "image/png") + data = source.get("data", "") + parts.append({ + "type": "image_url", + "image_url": {"url": f"data:{media_type};base64,{data}"}, + }) + elif isinstance(block, str): + parts.append({"type": "text", "text": block}) + return ProviderMessage(role="user", content=parts) + return ProviderMessage(role="user", content=str(content)) + + def format_assistant_message(self, response: ModelResponse) -> ProviderMessage: + """Convert ModelResponse to OpenAI assistant message format.""" + text_parts = [] + tool_calls = [] + for block in response.content: + if block.type == "text": + text_parts.append(block.text) + elif block.type == "tool_use" and block.tool_call: + tool_calls.append({ + "id": block.tool_call.id, + "type": "function", + "function": { + "name": block.tool_call.name, + "arguments": json.dumps(block.tool_call.input), + }, + }) + msg: dict[str, Any] = {"role": "assistant"} + if text_parts: + msg["content"] = "\n".join(text_parts) + else: + msg["content"] = None + if tool_calls: + msg["tool_calls"] = tool_calls + return ProviderMessage(role="assistant", content=msg) + + def _build_messages( + self, + system: str | None, + messages: list[ProviderMessage], + ) -> list[dict]: + """Convert ProviderMessages to OpenAI API format.""" + result = [] + if system: + result.append({"role": "system", "content": system}) + + for msg in messages: + if msg.role == "assistant": + # Assistant messages are already in OpenAI format from format_assistant_message + if isinstance(msg.content, dict) and "role" in msg.content: + result.append(msg.content) + else: + # Raw content blocks from provider-agnostic format + text_parts = [] + tool_calls = [] + if isinstance(msg.content, list): + for block in msg.content: + if isinstance(block, dict): + if block.get("type") == "text": + text_parts.append(block["text"]) + elif block.get("type") == "tool_use": + tool_calls.append({ + "id": block.get("id", uuid4().hex), + "type": "function", + "function": { + "name": block.get("name", ""), + "arguments": json.dumps(block.get("input", {})), + }, + }) + api_msg: dict[str, Any] = { + "role": "assistant", + "content": "\n".join(text_parts) if text_parts else None, + } + if tool_calls: + api_msg["tool_calls"] = tool_calls + result.append(api_msg) + + elif msg.role == "tool_result": + # Tool results: content is a list of tool result dicts + if isinstance(msg.content, list): + for tr in msg.content: + if isinstance(tr, dict) and "tool_call_id" in tr: + result.append(tr) + elif isinstance(msg.content, dict) and "tool_call_id" in msg.content: + result.append(msg.content) + + elif msg.role == "user": + result.append({"role": "user", "content": msg.content}) + + return result + + async def create_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> ModelResponse: + kwargs: dict[str, Any] = { + "model": self.get_model_id(model), + "max_tokens": max_tokens, + "messages": self._build_messages(system, messages), + } + if tools: + kwargs["tools"] = [self.clean_tool_schema(t) for t in tools] + + resp = await self.client.chat.completions.create(**kwargs) + choice = resp.choices[0] + message = choice.message + + content: list[ContentBlock] = [] + if message.content: + content.append(ContentBlock(type="text", text=message.content)) + + if message.tool_calls: + for tc in message.tool_calls: + try: + args = json.loads(tc.function.arguments) + except json.JSONDecodeError: + args = {} + content.append(ContentBlock( + type="tool_use", + tool_call=ToolCall( + id=tc.id, + name=tc.function.name, + input=args, + ), + )) + + stop = "end_turn" + if choice.finish_reason == "tool_calls": + stop = "tool_use" + elif message.tool_calls: + stop = "tool_use" + + usage_dict = {} + if resp.usage: + usage_dict = { + "input_tokens": resp.usage.prompt_tokens, + "output_tokens": resp.usage.completion_tokens, + } + + return ModelResponse(content=content, stop_reason=stop, usage=usage_dict) + + async def stream_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> AsyncIterator[StreamEvent]: + kwargs: dict[str, Any] = { + "model": self.get_model_id(model), + "max_tokens": max_tokens, + "messages": self._build_messages(system, messages), + "stream": True, + "stream_options": {"include_usage": True}, + } + if tools: + kwargs["tools"] = [self.clean_tool_schema(t) for t in tools] + + stream = await self.client.chat.completions.create(**kwargs) + + # Track streaming state to emit normalized events + text_started = False + text_index = 0 + tool_indices: dict[int, dict] = {} # openai tool_call index -> {name, id, json_buf} + next_block_index = 0 + + async for chunk in stream: + if not chunk.choices: + # Usage-only chunk at the end + if chunk.usage: + yield StreamEvent(type="usage", usage={ + "input_tokens": chunk.usage.prompt_tokens or 0, + "output_tokens": chunk.usage.completion_tokens or 0, + }) + continue + + delta = chunk.choices[0].delta + finish_reason = chunk.choices[0].finish_reason + + # Text content + if delta.content is not None: + if not text_started: + text_started = True + text_index = next_block_index + next_block_index += 1 + yield StreamEvent( + type="content_block_start", + index=text_index, + block_type="text", + ) + yield StreamEvent( + type="content_block_delta", + index=text_index, + delta_type="text_delta", + text=delta.content, + ) + + # Tool calls + if delta.tool_calls: + for tc_delta in delta.tool_calls: + tc_idx = tc_delta.index + if tc_idx not in tool_indices: + # New tool call starting + if text_started: + yield StreamEvent(type="content_block_stop", index=text_index) + text_started = False + + block_idx = next_block_index + next_block_index += 1 + tool_indices[tc_idx] = { + "block_index": block_idx, + "id": tc_delta.id or uuid4().hex, + "name": tc_delta.function.name if tc_delta.function else "", + "json_buf": "", + } + yield StreamEvent( + type="content_block_start", + index=block_idx, + block_type="tool_use", + tool_name=tool_indices[tc_idx]["name"], + tool_id=tool_indices[tc_idx]["id"], + ) + + info = tool_indices[tc_idx] + if tc_delta.function and tc_delta.function.name: + info["name"] = tc_delta.function.name + if tc_delta.function and tc_delta.function.arguments: + info["json_buf"] += tc_delta.function.arguments + yield StreamEvent( + type="content_block_delta", + index=info["block_index"], + delta_type="input_json_delta", + text=tc_delta.function.arguments, + ) + + # Finish + if finish_reason is not None: + if text_started: + yield StreamEvent(type="content_block_stop", index=text_index) + for info in tool_indices.values(): + yield StreamEvent(type="content_block_stop", index=info["block_index"]) + yield StreamEvent(type="message_stop") diff --git a/backend/apps/agents/providers/registry.py b/backend/apps/agents/providers/registry.py new file mode 100644 index 00000000..4e16c80c --- /dev/null +++ b/backend/apps/agents/providers/registry.py @@ -0,0 +1,362 @@ +"""Provider factory and model registry. + +Two-tier system: +1. Built-in providers (Anthropic, OpenAI, Gemini) with curated model lists +2. User-configured custom providers (any OpenAI-compatible endpoint) + - Includes built-in OpenRouter integration for 300+ models +""" + +from __future__ import annotations + +import logging +from typing import Any, TYPE_CHECKING + +from backend.apps.agents.providers.base import BaseProvider + +if TYPE_CHECKING: + from backend.apps.settings.models import AppSettings + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Tier 1: Built-in models (curated, we know their quirks) +# --------------------------------------------------------------------------- + +BUILTIN_MODELS: dict[str, list[dict[str, Any]]] = { + # ── Native API providers (use direct SDK) ── + "Anthropic": [ + {"value": "sonnet", "label": "Claude Sonnet 4.6", "context_window": 1_000_000, "model_id": "claude-sonnet-4-6", "api": "anthropic"}, + {"value": "opus", "label": "Claude Opus 4.6", "context_window": 1_000_000, "model_id": "claude-opus-4-6", "api": "anthropic"}, + {"value": "haiku", "label": "Claude Haiku 4.5", "context_window": 200_000, "model_id": "claude-haiku-4-5", "api": "anthropic"}, + ], + "OpenAI": [ + {"value": "gpt-5.4", "label": "GPT-5.4", "context_window": 1_000_000, "api": "openai"}, + {"value": "gpt-5.4-mini", "label": "GPT-5.4 Mini", "context_window": 400_000, "api": "openai"}, + {"value": "o3", "label": "o3", "context_window": 200_000, "api": "openai"}, + {"value": "o4-mini", "label": "o4-mini", "context_window": 200_000, "api": "openai"}, + ], + "Google": [ + {"value": "gemini-2.5-pro", "label": "Gemini 2.5 Pro", "context_window": 1_048_576, "api": "gemini"}, + {"value": "gemini-2.5-flash", "label": "Gemini 2.5 Flash", "context_window": 1_048_576, "api": "gemini"}, + ], + # ── Via OpenRouter (need OpenRouter API key) ── + "xAI": [ + {"value": "x-ai/grok-4-0214", "label": "Grok 4", "context_window": 2_000_000, "api": "openrouter"}, + ], + "Meta": [ + {"value": "meta-llama/llama-4-maverick", "label": "Llama 4 Maverick", "context_window": 1_000_000, "api": "openrouter"}, + {"value": "meta-llama/llama-4-scout", "label": "Llama 4 Scout", "context_window": 10_000_000, "api": "openrouter"}, + ], + "DeepSeek": [ + {"value": "deepseek/deepseek-chat-v3-0324", "label": "DeepSeek V3", "context_window": 163_840, "api": "openrouter"}, + {"value": "deepseek/deepseek-r1", "label": "DeepSeek R1", "context_window": 163_840, "api": "openrouter"}, + ], + "Mistral": [ + {"value": "mistralai/mistral-large-2501", "label": "Mistral Large", "context_window": 256_000, "api": "openrouter"}, + {"value": "mistralai/mistral-small-3.1-24b-instruct", "label": "Mistral Small 3.1", "context_window": 128_000, "api": "openrouter"}, + ], + "Qwen": [ + {"value": "qwen/qwen3-coder", "label": "Qwen3 Coder 480B", "context_window": 262_144, "api": "openrouter"}, + {"value": "qwen/qwen3-235b-a22b", "label": "Qwen3 235B", "context_window": 131_072, "api": "openrouter"}, + ], + "Cohere": [ + {"value": "cohere/command-a-03-2025", "label": "Command A", "context_window": 256_000, "api": "openrouter"}, + ], +} + +# --------------------------------------------------------------------------- +# OpenRouter: built-in integration for 300+ models +# --------------------------------------------------------------------------- + +OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1" + +_9router_cache: dict = {"available": None, "checked_at": 0} + + +def _is_9router_available() -> bool: + """Check if 9Router is running on localhost:20128. Caches for 30 seconds.""" + import time as _time + now = _time.time() + if _9router_cache["available"] is not None and now - _9router_cache["checked_at"] < 30: + return _9router_cache["available"] + try: + import httpx + r = httpx.get("http://localhost:20128/v1/models", timeout=2.0) + available = r.status_code == 200 + except Exception: + available = False + _9router_cache["available"] = available + _9router_cache["checked_at"] = now + return available + + +# --------------------------------------------------------------------------- +# Provider factory +# --------------------------------------------------------------------------- + +def create_provider( + provider_name: str, + settings: AppSettings, + provider_config: dict | None = None, +) -> BaseProvider: + """Create a provider adapter. + + Routes based on the 'api' field in BUILTIN_MODELS: + - "anthropic" → native Anthropic SDK + - "openai" → native OpenAI SDK (direct API) + - "gemini" → native Google GenAI SDK + - "openrouter" → OpenAI-compat via openrouter.ai (Meta, Mistral, DeepSeek, Qwen, xAI, etc.) + Custom providers use OpenAI-compat with user's base_url. + """ + api_type = _get_api_type(provider_name) + + # Check for 9Router first + if provider_name in ("9Router", "9router"): + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + return OpenAICompatProvider(api_key="9router", base_url="http://localhost:20128/v1") + + # Check for GitHub Copilot + if provider_name in ("GitHub Copilot", "copilot"): + from backend.apps.agents.providers.copilot import CopilotProvider + copilot_token = getattr(settings, "copilot_token", None) + if not copilot_token: + raise ValueError("GitHub Copilot not connected. Sign in via Settings → Models.") + # Auto-refresh if expired + import time as _time + expires = getattr(settings, "copilot_token_expires", None) + if expires and _time.time() > expires - 120: + github_token = getattr(settings, "copilot_github_token", None) + if github_token: + import asyncio + from backend.apps.agents.copilot_auth import exchange_for_copilot_token + try: + loop = asyncio.get_event_loop() + result = loop.run_until_complete(exchange_for_copilot_token(github_token)) + copilot_token = result["token"] + settings.copilot_token = copilot_token + settings.copilot_token_expires = result["expires_at"] + from backend.apps.settings.settings import _save_settings + _save_settings(settings) + except Exception as e: + logger.warning(f"Copilot token refresh failed: {e}") + return CopilotProvider(copilot_token=copilot_token) + + if api_type == "anthropic": + from backend.apps.agents.providers.anthropic import AnthropicProvider + if getattr(settings, "connection_mode", "own_key") == "managed": + return AnthropicProvider( + auth_token=getattr(settings, "openswarm_auth_token", None), + base_url=getattr(settings, "openswarm_proxy_url", None) or "https://api.openswarm.ai", + ) + if settings.anthropic_api_key: + return AnthropicProvider(api_key=settings.anthropic_api_key) + # No API key — try 9Router as fallback + if _is_9router_available(): + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + provider = OpenAICompatProvider(api_key="9router", base_url="http://localhost:20128/v1") + # Override get_model_id to map our short names to 9Router's cc/ prefixed IDs + _original_get_model = provider.get_model_id + _9r_model_map = { + "sonnet": "cc/claude-sonnet-4-6", + "opus": "cc/claude-opus-4-6", + "haiku": "cc/claude-haiku-4-5-20251001", + } + provider.get_model_id = lambda name: _9r_model_map.get(name, f"cc/{name}" if not name.startswith("cc/") else name) + return provider + raise ValueError("Anthropic API key not configured. Set it in Settings, or connect 9Router.") + + if api_type == "openai": + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + if settings.openai_api_key: + return OpenAICompatProvider(api_key=settings.openai_api_key, base_url="https://api.openai.com/v1") + # No API key — try 9Router as fallback + if _is_9router_available(): + return OpenAICompatProvider(api_key="9router", base_url="http://localhost:20128/v1") + raise ValueError("OpenAI API key not configured. Set it in Settings, or connect 9Router.") + + if api_type == "gemini": + from backend.apps.agents.providers.gemini import GeminiProvider + if settings.google_api_key: + return GeminiProvider(api_key=settings.google_api_key) + # No API key — try 9Router as fallback + if _is_9router_available(): + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + return OpenAICompatProvider(api_key="9router", base_url="http://localhost:20128/v1") + raise ValueError("Google API key not configured. Set it in Settings, or connect 9Router.") + + if api_type == "openrouter": + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + openrouter_key = getattr(settings, "openrouter_api_key", None) + if openrouter_key: + return OpenAICompatProvider(api_key=openrouter_key, base_url=OPENROUTER_BASE_URL) + # No OpenRouter key — try 9Router as fallback + if _is_9router_available(): + return OpenAICompatProvider(api_key="9router", base_url="http://localhost:20128/v1") + raise ValueError(f"OpenRouter API key not configured for {provider_name}. Set it in Settings, or connect a subscription.") + + # Custom provider — look up in settings.custom_providers + if provider_config: + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + return OpenAICompatProvider( + api_key=provider_config.get("api_key", ""), + base_url=provider_config.get("base_url", ""), + ) + + for cp in getattr(settings, "custom_providers", []): + if cp.name == provider_name: + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + return OpenAICompatProvider( + api_key=cp.api_key, + base_url=cp.base_url, + ) + + raise ValueError(f"Unknown provider: {provider_name}") + + +def _get_api_type(provider_name: str) -> str: + """Get the API type for a provider from BUILTIN_MODELS. + + Accepts both display names ('Anthropic') and lowercase API names ('anthropic'). + """ + # Direct lookup first (display name like 'Anthropic', 'OpenAI', etc.) + models = BUILTIN_MODELS.get(provider_name, []) + if models: + return models[0].get("api", "openrouter") + + # Lowercase API name mapping + _API_NAME_MAP = { + "anthropic": "anthropic", + "openai": "openai", + "gemini": "gemini", + "google": "gemini", + "openrouter": "openrouter", + } + if provider_name.lower() in _API_NAME_MAP: + return _API_NAME_MAP[provider_name.lower()] + + # Case-insensitive lookup into BUILTIN_MODELS + lower = provider_name.lower() + for key, models in BUILTIN_MODELS.items(): + if key.lower() == lower: + return models[0].get("api", "openrouter") + + return "openrouter" + + +def _has_credentials(provider_name: str, settings: AppSettings) -> bool: + """Check if a provider has credentials configured.""" + api_type = _get_api_type(provider_name) + + if api_type == "anthropic": + if getattr(settings, "connection_mode", "own_key") == "managed": + return bool(getattr(settings, "openswarm_auth_token", None)) + return bool(settings.anthropic_api_key) + if api_type == "openai": + return bool(settings.openai_api_key) + if api_type == "gemini": + return bool(getattr(settings, "google_api_key", None)) + if api_type == "openrouter": + return bool(getattr(settings, "openrouter_api_key", None)) + return False + + +def get_available_models(settings: AppSettings) -> dict[str, list[dict]]: + """Return all models — always show everything, mark which have keys configured. + + Like Cursor: show all models upfront, prompt for key when user tries to use one. + Returns: {"provider_name": [{"value": ..., "label": ..., "context_window": ..., "configured": bool}, ...]} + """ + result: dict[str, list[dict]] = {} + + # Built-in providers — always show all + for provider_name, models in BUILTIN_MODELS.items(): + configured = _has_credentials(provider_name, settings) + result[provider_name] = [ + {**m, "configured": configured} + for m in models + ] + + # Custom providers + for cp in getattr(settings, "custom_providers", []): + if cp.models: + result[cp.name] = [ + { + "value": m.get("value", m.get("id", "")), + "label": m.get("label", m.get("value", m.get("id", ""))), + "context_window": m.get("context_window", 128_000), + "configured": True, + } + for m in cp.models + ] + + return result + + +def get_context_window(provider: str, model: str, settings: AppSettings | None = None) -> int: + """Look up context window for any model.""" + # Check built-in models first + for models in BUILTIN_MODELS.values(): + for m in models: + if m["value"] == model: + return m.get("context_window", 128_000) + + # Check custom providers + if settings: + for cp in getattr(settings, "custom_providers", []): + for m in cp.models: + if m.get("value") == model or m.get("id") == model: + return m.get("context_window", 128_000) + + return 128_000 # safe default + + +# --------------------------------------------------------------------------- +# Cost tracking +# --------------------------------------------------------------------------- + +COST_PER_1M_TOKENS: dict[tuple[str, str], tuple[float, float]] = { + # (provider, model): (input_cost_per_1M, output_cost_per_1M) + # Anthropic + ("Anthropic", "sonnet"): (3.0, 15.0), + ("Anthropic", "opus"): (5.0, 25.0), + ("Anthropic", "haiku"): (1.0, 5.0), + # OpenAI + ("OpenAI", "gpt-5.4"): (2.50, 15.0), + ("OpenAI", "gpt-5.4-mini"): (0.75, 3.0), + ("OpenAI", "o3"): (2.0, 8.0), + ("OpenAI", "o4-mini"): (1.10, 4.40), + # Google + ("Google", "gemini-2.5-flash"): (0.15, 0.60), + ("Google", "gemini-2.5-pro"): (1.25, 10.0), + # OpenRouter-backed (approximate) + ("xAI", "x-ai/grok-4-0214"): (3.0, 15.0), + ("Meta", "meta-llama/llama-4-maverick"): (0.50, 0.70), + ("Meta", "meta-llama/llama-4-scout"): (0.15, 0.40), + ("DeepSeek", "deepseek/deepseek-chat-v3-0324"): (0.30, 0.90), + ("DeepSeek", "deepseek/deepseek-r1"): (0.80, 2.40), + ("Mistral", "mistralai/mistral-large-2501"): (2.0, 6.0), + ("Mistral", "mistralai/mistral-small-3.1-24b-instruct"): (0.10, 0.30), + ("Qwen", "qwen/qwen3-coder"): (0.0, 0.0), + ("Qwen", "qwen/qwen3-235b-a22b"): (0.20, 0.70), + ("Cohere", "cohere/command-a-03-2025"): (2.50, 10.0), +} + + +def calculate_cost( + provider: str, model: str, + input_tokens: int, output_tokens: int, +) -> float: + """Calculate cost in USD from token counts.""" + # Direct lookup first + rates = COST_PER_1M_TOKENS.get((provider, model)) + if not rates: + # Case-insensitive provider lookup + lower = provider.lower() + for (p, m), r in COST_PER_1M_TOKENS.items(): + if p.lower() == lower and m == model: + rates = r + break + if not rates: + return 0.0 + input_rate, output_rate = rates + return (input_tokens * input_rate + output_tokens * output_rate) / 1_000_000 diff --git a/backend/apps/agents/tools/__init__.py b/backend/apps/agents/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/agents/tools/base.py b/backend/apps/agents/tools/base.py new file mode 100644 index 00000000..6190a170 --- /dev/null +++ b/backend/apps/agents/tools/base.py @@ -0,0 +1,47 @@ +"""Base classes for builtin tool implementations.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any + + +@dataclass +class ToolContext: + """Runtime context passed to every tool execution.""" + cwd: str + session_id: str + + +class BaseTool(ABC): + """Abstract base for all builtin tools. + + Subclasses must set ``name`` and ``description`` as class attributes and + implement ``get_schema`` (JSON Schema for tool input) and ``execute``. + """ + + name: str + description: str + + @abstractmethod + def get_schema(self) -> dict: + """Return JSON Schema for this tool's input parameters.""" + ... + + @abstractmethod + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + """Execute the tool. + + Returns a list of content blocks, e.g. + ``[{"type": "text", "text": "..."}]``. + """ + ... + + def to_tool_schema(self): + """Convert to the provider-agnostic ``ToolSchema`` used everywhere.""" + from backend.apps.agents.providers.base import ToolSchema + + return ToolSchema( + name=self.name, + description=self.description, + input_schema=self.get_schema(), + ) diff --git a/backend/apps/agents/tools/filesystem.py b/backend/apps/agents/tools/filesystem.py new file mode 100644 index 00000000..5ea3c253 --- /dev/null +++ b/backend/apps/agents/tools/filesystem.py @@ -0,0 +1,476 @@ +"""Filesystem tools: Read, Write, Edit, Glob, Grep.""" + +from __future__ import annotations + +import asyncio +import base64 +import mimetypes +import os +import re +from pathlib import Path +from typing import Any + +from backend.apps.agents.tools.base import BaseTool, ToolContext + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg"} +_MAX_OUTPUT_BYTES = 50 * 1024 # ~50 KB cap for grep output + + +def _resolve(file_path: str, cwd: str) -> Path: + """Resolve *file_path* against *cwd* when it is relative.""" + p = Path(file_path) + if not p.is_absolute(): + p = Path(cwd) / p + return p.resolve() + + +def _text_block(text: str) -> list[dict]: + return [{"type": "text", "text": text}] + + +# ─────────────────────────────────────────────────────────────────────────── +# ReadTool +# ─────────────────────────────────────────────────────────────────────────── + + +class ReadTool(BaseTool): + name = "Read" + description = ( + "Read a file from the filesystem. Returns lines with line numbers " + "(cat -n style). For image files returns base64 content. Supports " + "offset and limit parameters for reading portions of large files." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Absolute or relative path to the file to read.", + }, + "offset": { + "type": "integer", + "description": "1-based line number to start reading from.", + }, + "limit": { + "type": "integer", + "description": "Maximum number of lines to return (default 2000).", + }, + }, + "required": ["file_path"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + file_path = _resolve(input_data["file_path"], context.cwd) + + if not file_path.exists(): + return _text_block(f"Error: file not found: {file_path}") + + if not file_path.is_file(): + return _text_block(f"Error: not a regular file: {file_path}") + + # Binary / image files → base64 + ext = file_path.suffix.lower() + if ext in _IMAGE_EXTENSIONS: + try: + raw = file_path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + media = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream" + return [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": media, + "data": b64, + }, + } + ] + except Exception as exc: + return _text_block(f"Error reading image {file_path}: {exc}") + + # Text files + offset = max(input_data.get("offset", 1), 1) + limit = input_data.get("limit", 2000) + if limit <= 0: + limit = 2000 + + try: + with open(file_path, "r", errors="replace") as fh: + lines: list[str] = [] + for lineno, line in enumerate(fh, start=1): + if lineno < offset: + continue + if len(lines) >= limit: + break + # cat -n style: right-justified line number + tab + content + lines.append(f"{lineno:>6}\t{line.rstrip()}") + if not lines: + return _text_block(f"(file is empty or offset beyond end of file: {file_path})") + return _text_block("\n".join(lines)) + except Exception as exc: + return _text_block(f"Error reading {file_path}: {exc}") + + +# ─────────────────────────────────────────────────────────────────────────── +# WriteTool +# ─────────────────────────────────────────────────────────────────────────── + + +class WriteTool(BaseTool): + name = "Write" + description = ( + "Write content to a file. Creates parent directories if they do not " + "exist. Overwrites the file if it already exists." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Absolute or relative path to the file to write.", + }, + "content": { + "type": "string", + "description": "The full content to write to the file.", + }, + }, + "required": ["file_path", "content"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + file_path = _resolve(input_data["file_path"], context.cwd) + content: str = input_data["content"] + + try: + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(content, encoding="utf-8") + return _text_block(f"Successfully wrote {len(content)} bytes to {file_path}") + except Exception as exc: + return _text_block(f"Error writing {file_path}: {exc}") + + +# ─────────────────────────────────────────────────────────────────────────── +# EditTool +# ─────────────────────────────────────────────────────────────────────────── + + +class EditTool(BaseTool): + name = "Edit" + description = ( + "Perform exact string replacements in a file. By default the " + "old_string must appear exactly once (not unique → error). Pass " + "replace_all=true to replace every occurrence." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Absolute or relative path to the file to edit.", + }, + "old_string": { + "type": "string", + "description": "The exact text to find in the file.", + }, + "new_string": { + "type": "string", + "description": "The text to replace old_string with.", + }, + "replace_all": { + "type": "boolean", + "description": "If true, replace all occurrences. Default false.", + "default": False, + }, + }, + "required": ["file_path", "old_string", "new_string"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + file_path = _resolve(input_data["file_path"], context.cwd) + old_string: str = input_data["old_string"] + new_string: str = input_data["new_string"] + replace_all: bool = input_data.get("replace_all", False) + + if not file_path.exists(): + return _text_block(f"Error: file not found: {file_path}") + if not file_path.is_file(): + return _text_block(f"Error: not a regular file: {file_path}") + + try: + content = file_path.read_text(encoding="utf-8") + except Exception as exc: + return _text_block(f"Error reading {file_path}: {exc}") + + count = content.count(old_string) + if count == 0: + return _text_block( + f"Error: old_string not found in {file_path}. " + "Make sure the string matches exactly, including whitespace and indentation." + ) + + if not replace_all and count > 1: + return _text_block( + f"Error: old_string appears {count} times in {file_path}. " + "Provide more surrounding context to make the match unique, " + "or set replace_all=true to replace every occurrence." + ) + + if replace_all: + new_content = content.replace(old_string, new_string) + else: + # Replace only the first (and only) occurrence + new_content = content.replace(old_string, new_string, 1) + + try: + file_path.write_text(new_content, encoding="utf-8") + except Exception as exc: + return _text_block(f"Error writing {file_path}: {exc}") + + replacements = count if replace_all else 1 + return _text_block( + f"Successfully edited {file_path} ({replacements} replacement{'s' if replacements != 1 else ''})." + ) + + +# ─────────────────────────────────────────────────────────────────────────── +# GlobTool +# ─────────────────────────────────────────────────────────────────────────── + + +class GlobTool(BaseTool): + name = "Glob" + description = ( + "Fast file pattern matching. Supports glob patterns like '**/*.py'. " + "Returns matching file paths sorted by modification time (newest first)." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Glob pattern to match files (e.g. '**/*.py', 'src/**/*.ts').", + }, + "path": { + "type": "string", + "description": "Directory to search in. Defaults to the working directory.", + }, + }, + "required": ["pattern"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + pattern: str = input_data["pattern"] + base = Path(input_data.get("path") or context.cwd) + + if not base.is_dir(): + return _text_block(f"Error: directory not found: {base}") + + try: + matches: list[Path] = [] + for p in base.glob(pattern): + if p.is_file(): + matches.append(p) + if len(matches) >= 500: + break + + # Sort by modification time, newest first + matches.sort(key=lambda p: p.stat().st_mtime, reverse=True) + + if not matches: + return _text_block(f"No files matched pattern '{pattern}' in {base}") + + result = "\n".join(str(p) for p in matches) + return _text_block(result) + except Exception as exc: + return _text_block(f"Error during glob '{pattern}' in {base}: {exc}") + + +# ─────────────────────────────────────────────────────────────────────────── +# GrepTool +# ─────────────────────────────────────────────────────────────────────────── + + +class GrepTool(BaseTool): + name = "Grep" + description = ( + "Search file contents using regular expressions. Uses ripgrep (rg) " + "when available, otherwise falls back to Python's re module. " + "Supports output modes: files_with_matches, content, count." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Regular expression pattern to search for.", + }, + "path": { + "type": "string", + "description": "File or directory to search in. Defaults to the working directory.", + }, + "glob": { + "type": "string", + "description": "Glob pattern to filter files (e.g. '*.py', '*.{ts,tsx}').", + }, + "output_mode": { + "type": "string", + "enum": ["files_with_matches", "content", "count"], + "description": "Output mode. Default: files_with_matches.", + "default": "files_with_matches", + }, + }, + "required": ["pattern"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + pattern: str = input_data["pattern"] + search_path: str = input_data.get("path") or context.cwd + file_glob: str | None = input_data.get("glob") + output_mode: str = input_data.get("output_mode", "files_with_matches") + + # Try ripgrep first + try: + result = await self._run_rg(pattern, search_path, file_glob, output_mode) + if result is not None: + return result + except FileNotFoundError: + pass # rg not installed, fall through to Python fallback + + # Python fallback + return await self._python_grep(pattern, search_path, file_glob, output_mode) + + async def _run_rg( + self, + pattern: str, + search_path: str, + file_glob: str | None, + output_mode: str, + ) -> list[dict] | None: + """Run ripgrep and return results, or None if rg is not available.""" + cmd = ["rg", "--no-heading", "--color=never"] + + if output_mode == "files_with_matches": + cmd.append("--files-with-matches") + elif output_mode == "count": + cmd.append("--count") + else: + cmd.extend(["--line-number"]) + + if file_glob: + cmd.extend(["--glob", file_glob]) + + cmd.append(pattern) + cmd.append(search_path) + + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30) + except FileNotFoundError: + raise # re-raise so caller knows rg is missing + except asyncio.TimeoutError: + return _text_block("Error: grep timed out after 30 seconds.") + except Exception as exc: + return _text_block(f"Error running ripgrep: {exc}") + + output = stdout.decode("utf-8", errors="replace") + + if proc.returncode not in (0, 1): + err = stderr.decode("utf-8", errors="replace").strip() + if err: + return _text_block(f"Grep error: {err}") + + if not output.strip(): + return _text_block(f"No matches found for pattern '{pattern}'.") + + # Truncate if too large + if len(output) > _MAX_OUTPUT_BYTES: + output = output[:_MAX_OUTPUT_BYTES] + "\n... (output truncated)" + + return _text_block(output.rstrip()) + + async def _python_grep( + self, + pattern: str, + search_path: str, + file_glob: str | None, + output_mode: str, + ) -> list[dict]: + """Pure-Python grep fallback using the re module.""" + try: + regex = re.compile(pattern) + except re.error as exc: + return _text_block(f"Invalid regex pattern: {exc}") + + base = Path(search_path) + if base.is_file(): + files = [base] + elif base.is_dir(): + glob_pat = file_glob or "**/*" + files = [p for p in base.glob(glob_pat) if p.is_file()] + else: + return _text_block(f"Error: path not found: {search_path}") + + lines_out: list[str] = [] + total_bytes = 0 + truncated = False + + for fp in sorted(files): + try: + text = fp.read_text(encoding="utf-8", errors="replace") + except Exception: + continue + + file_matches: list[tuple[int, str]] = [] + for lineno, line in enumerate(text.splitlines(), start=1): + if regex.search(line): + file_matches.append((lineno, line)) + + if not file_matches: + continue + + if output_mode == "files_with_matches": + entry = str(fp) + elif output_mode == "count": + entry = f"{fp}:{len(file_matches)}" + else: + parts = [f"{fp}:{ln}:{txt}" for ln, txt in file_matches] + entry = "\n".join(parts) + + total_bytes += len(entry) + if total_bytes > _MAX_OUTPUT_BYTES: + truncated = True + break + + lines_out.append(entry) + + if not lines_out: + return _text_block(f"No matches found for pattern '{pattern}'.") + + result = "\n".join(lines_out) + if truncated: + result += "\n... (output truncated)" + + return _text_block(result) diff --git a/backend/apps/agents/tools/registry.py b/backend/apps/agents/tools/registry.py new file mode 100644 index 00000000..91cd6b5b --- /dev/null +++ b/backend/apps/agents/tools/registry.py @@ -0,0 +1,61 @@ +"""Central tool registry. + +Importing this module automatically registers all builtin tools. +""" + +from __future__ import annotations + +from backend.apps.agents.tools.base import BaseTool +from backend.apps.agents.providers.base import ToolSchema + +_TOOLS: dict[str, BaseTool] = {} + + +def register_tool(tool: BaseTool) -> None: + """Register a tool instance by its name.""" + _TOOLS[tool.name] = tool + + +def get_tool(name: str) -> BaseTool | None: + """Look up a registered tool by name. Returns None if not found.""" + return _TOOLS.get(name) + + +def get_all_tools() -> list[BaseTool]: + """Return all registered tool instances.""" + return list(_TOOLS.values()) + + +def get_all_tool_schemas() -> list[ToolSchema]: + """Return provider-agnostic ToolSchema for every registered tool.""" + return [t.to_tool_schema() for t in _TOOLS.values()] + + +def init_tools() -> None: + """Import and register all builtin tools.""" + from backend.apps.agents.tools.filesystem import ( + ReadTool, + WriteTool, + EditTool, + GlobTool, + GrepTool, + ) + from backend.apps.agents.tools.system import BashTool, AskUserQuestionTool + from backend.apps.agents.tools.web import WebSearchTool, WebFetchTool + + for tool_cls in [ + ReadTool, + WriteTool, + EditTool, + GlobTool, + GrepTool, + BashTool, + AskUserQuestionTool, + WebSearchTool, + WebFetchTool, + ]: + register_tool(tool_cls()) + + +# Auto-register on import +init_tools() diff --git a/backend/apps/agents/tools/system.py b/backend/apps/agents/tools/system.py new file mode 100644 index 00000000..3a394328 --- /dev/null +++ b/backend/apps/agents/tools/system.py @@ -0,0 +1,125 @@ +"""System tools: Bash and AskUserQuestion.""" + +from __future__ import annotations + +import asyncio + +from backend.apps.agents.tools.base import BaseTool, ToolContext + +_MAX_OUTPUT_BYTES = 100 * 1024 # ~100 KB cap + + +class BashTool(BaseTool): + name = "Bash" + description = ( + "Execute a shell command and return its output. The command runs in " + "the session's working directory. Supports an optional timeout " + "(default 120 000 ms). Stdout and stderr are captured and returned." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "The shell command to execute.", + }, + "timeout": { + "type": "integer", + "description": "Timeout in milliseconds (default 120000, max 600000).", + "default": 120000, + }, + "description": { + "type": "string", + "description": "Optional human-readable description of what this command does.", + }, + }, + "required": ["command"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + command: str = input_data["command"] + timeout_ms: int = min(input_data.get("timeout", 120000), 600000) + timeout_s: float = timeout_ms / 1000.0 + + try: + proc = await asyncio.create_subprocess_shell( + command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=context.cwd, + ) + except Exception as exc: + return [{"type": "text", "text": f"Error starting command: {exc}"}] + + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_s) + except asyncio.TimeoutError: + # Attempt to kill the process + try: + proc.kill() + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=5) + except Exception: + stdout, stderr = b"", b"" + + partial = self._decode(stdout, stderr) + msg = ( + f"Command timed out after {timeout_ms}ms.\n" + f"Partial output:\n{partial}" + ) + return [{"type": "text", "text": self._truncate(msg)}] + except Exception as exc: + return [{"type": "text", "text": f"Error executing command: {exc}"}] + + output = self._decode(stdout, stderr) + + if proc.returncode != 0: + output = f"Exit code: {proc.returncode}\n{output}" + + if not output.strip(): + output = f"(command completed with exit code {proc.returncode})" + + return [{"type": "text", "text": self._truncate(output)}] + + @staticmethod + def _decode(stdout: bytes, stderr: bytes) -> str: + parts: list[str] = [] + if stdout: + parts.append(stdout.decode("utf-8", errors="replace")) + if stderr: + parts.append(stderr.decode("utf-8", errors="replace")) + return "\n".join(parts) + + @staticmethod + def _truncate(text: str) -> str: + if len(text) > _MAX_OUTPUT_BYTES: + return text[:_MAX_OUTPUT_BYTES] + "\n... (output truncated)" + return text + + +class AskUserQuestionTool(BaseTool): + name = "AskUserQuestion" + description = ( + "Ask the user a clarifying question. The actual blocking/HITL " + "interaction is handled by the agent loop's hitl_handler; this tool " + "simply surfaces the question text." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "question": { + "type": "string", + "description": "The question to ask the user.", + }, + }, + "required": ["question"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + question: str = input_data.get("question", "") + return [{"type": "text", "text": question}] diff --git a/backend/apps/agents/tools/web.py b/backend/apps/agents/tools/web.py new file mode 100644 index 00000000..d1eec3b0 --- /dev/null +++ b/backend/apps/agents/tools/web.py @@ -0,0 +1,214 @@ +"""Web tools: WebSearch and WebFetch.""" + +from __future__ import annotations + +import html +import re +from typing import Any + +import httpx + +from backend.apps.agents.tools.base import BaseTool, ToolContext + +_MAX_OUTPUT_BYTES = 100 * 1024 # ~100 KB +_HTTP_TIMEOUT = 30 # seconds +_USER_AGENT = ( + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) " + "AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36" +) + + +def _truncate(text: str, limit: int = _MAX_OUTPUT_BYTES) -> str: + if len(text) > limit: + return text[:limit] + "\n... (output truncated)" + return text + + +def _strip_html(raw_html: str) -> str: + """Naive but effective HTML → plain-text conversion.""" + # Remove script/style blocks + text = re.sub(r"<(script|style)[^>]*>.*?", "", raw_html, flags=re.DOTALL | re.IGNORECASE) + # Remove HTML tags + text = re.sub(r"<[^>]+>", " ", text) + # Decode HTML entities + text = html.unescape(text) + # Collapse whitespace + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() + + +# ─────────────────────────────────────────────────────────────────────────── +# WebSearchTool +# ─────────────────────────────────────────────────────────────────────────── + + +class WebSearchTool(BaseTool): + name = "WebSearch" + description = ( + "Search the web using DuckDuckGo and return titles, URLs, and " + "snippets for the top results." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The search query.", + }, + "num_results": { + "type": "integer", + "description": "Maximum number of results to return (default 5).", + "default": 5, + }, + }, + "required": ["query"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + query: str = input_data["query"] + num_results: int = input_data.get("num_results", 5) + + try: + results = await self._search_ddg(query, num_results) + if not results: + return [{"type": "text", "text": f"No search results found for: {query}"}] + return [{"type": "text", "text": results}] + except Exception as exc: + return [{"type": "text", "text": f"Web search error: {exc}"}] + + @staticmethod + async def _search_ddg(query: str, num_results: int) -> str: + """Query DuckDuckGo HTML endpoint and parse results.""" + async with httpx.AsyncClient( + timeout=_HTTP_TIMEOUT, + follow_redirects=True, + headers={"User-Agent": _USER_AGENT}, + ) as client: + resp = await client.post( + "https://html.duckduckgo.com/html/", + data={"q": query}, + ) + resp.raise_for_status() + + body = resp.text + + # Parse result blocks – DuckDuckGo wraps each result in + #
...
+ result_blocks = re.findall( + r']*class="[^"]*result[^"]*"[^>]*>(.*?)\s*(?=]*class="[^"]*result|$)', + body, + flags=re.DOTALL, + ) + + entries: list[str] = [] + for block in result_blocks: + if len(entries) >= num_results: + break + + # Title + URL — handle both class-before-href and href-before-class + link_match = re.search( + r']*class="[^"]*result__a[^"]*"[^>]*href="([^"]*)"[^>]*>(.*?)', + block, + flags=re.DOTALL, + ) + if not link_match: + # Try reversed attribute order + link_match = re.search( + r']*href="([^"]*)"[^>]*class="[^"]*result__a[^"]*"[^>]*>(.*?)', + block, + flags=re.DOTALL, + ) + if not link_match: + continue + + raw_url = html.unescape(link_match.group(1)) + title = _strip_html(link_match.group(2)).strip() + + # Snippet + snippet_match = re.search( + r']*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)', + block, + flags=re.DOTALL, + ) + snippet = _strip_html(snippet_match.group(1)).strip() if snippet_match else "" + + # DuckDuckGo wraps URLs through a redirect; try to extract the real URL + real_url_match = re.search(r"uddg=([^&]+)", raw_url) + if real_url_match: + from urllib.parse import unquote + url = unquote(real_url_match.group(1)) + else: + url = raw_url + + entry = f"[{len(entries) + 1}] {title}\n {url}" + if snippet: + entry += f"\n {snippet}" + entries.append(entry) + + return "\n\n".join(entries) + + +# ─────────────────────────────────────────────────────────────────────────── +# WebFetchTool +# ─────────────────────────────────────────────────────────────────────────── + + +class WebFetchTool(BaseTool): + name = "WebFetch" + description = ( + "Fetch the contents of a URL and return the extracted text. " + "HTML is stripped to plain text. Output is truncated to ~100 KB." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to fetch.", + }, + "prompt": { + "type": "string", + "description": "Optional prompt/context describing what information to look for.", + }, + }, + "required": ["url"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + url: str = input_data["url"] + prompt: str | None = input_data.get("prompt") + + try: + async with httpx.AsyncClient( + timeout=_HTTP_TIMEOUT, + follow_redirects=True, + headers={"User-Agent": _USER_AGENT}, + ) as client: + resp = await client.get(url) + resp.raise_for_status() + except httpx.HTTPStatusError as exc: + return [{"type": "text", "text": f"HTTP error {exc.response.status_code} fetching {url}"}] + except Exception as exc: + return [{"type": "text", "text": f"Error fetching {url}: {exc}"}] + + content_type = resp.headers.get("content-type", "") + + if "html" in content_type or resp.text.strip().startswith(" dict: """Send an approval request and wait for the user's response. - Returns the approval decision dict.""" + Returns the approval decision dict. Times out after *timeout* seconds + (default 10 minutes) to prevent permanently stuck agents.""" future = asyncio.get_event_loop().create_future() self.pending_futures[request_id] = future @@ -75,8 +77,11 @@ class ConnectionManager: }) try: - result = await future + result = await asyncio.wait_for(future, timeout=timeout) return result + except asyncio.TimeoutError: + logger.warning("Approval %s for session %s timed out after %ss", request_id, session_id, timeout) + return {"behavior": "deny", "message": "Approval timed out"} finally: self.pending_futures.pop(request_id, None) diff --git a/backend/apps/analytics/__init__.py b/backend/apps/analytics/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/analytics/analytics.py b/backend/apps/analytics/analytics.py new file mode 100644 index 00000000..600ccbcd --- /dev/null +++ b/backend/apps/analytics/analytics.py @@ -0,0 +1,168 @@ +"""Analytics SubApp: PostHog for product analytics + local usage summary from session data.""" + +import json +import logging +import os +import platform +from collections import Counter +from contextlib import asynccontextmanager + +from backend.config.Apps import SubApp +from backend.config.paths import SESSIONS_DIR +from backend.apps.analytics.collector import init as init_collector, shutdown as shutdown_collector, record, identify + +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def analytics_lifespan(): + init_collector() + logger.info("PostHog analytics initialised") + + try: + from backend.apps.settings.settings import load_settings + settings = load_settings() + + providers = [] + if getattr(settings, "anthropic_api_key", None): + providers.append("anthropic") + if getattr(settings, "openai_api_key", None): + providers.append("openai") + if getattr(settings, "google_api_key", None): + providers.append("gemini") + if getattr(settings, "openrouter_api_key", None): + providers.append("openrouter") + for cp in getattr(settings, "custom_providers", []): + providers.append(cp.name) + + record("app.opened", { + "os": platform.system(), + "platform": platform.platform(), + "provider_count": len(providers), + "providers": providers, + "connection_mode": getattr(settings, "connection_mode", "own_key"), + }) + + identify({ + "providers_configured": providers, + "provider_count": len(providers), + "connection_mode": getattr(settings, "connection_mode", "own_key"), + }) + except Exception as e: + logger.debug(f"Analytics startup event failed (non-critical): {e}") + + # Auto-start 9Router for subscription access + try: + from backend.apps.nine_router import ensure_running as ensure_9router, stop as stop_9router + await ensure_9router() + except Exception as e: + logger.debug(f"9Router auto-start skipped: {e}") + + yield + + # Stop 9Router + try: + from backend.apps.nine_router import stop as stop_9router + stop_9router() + except Exception: + pass + + shutdown_collector() + logger.info("PostHog analytics shut down") + + +analytics = SubApp("analytics", analytics_lifespan) + + +def _load_all_sessions() -> list[dict]: + """Load all persisted session JSON files.""" + results = [] + if not os.path.exists(SESSIONS_DIR): + return results + for fname in os.listdir(SESSIONS_DIR): + if fname.endswith(".json"): + try: + with open(os.path.join(SESSIONS_DIR, fname)) as f: + results.append(json.load(f)) + except Exception: + pass + return results + + +@analytics.router.get("/usage-summary") +async def usage_summary(): + """Compute usage stats from persisted sessions for the Settings page.""" + from backend.apps.agents.agent_manager import agent_manager + + # Combine persisted + active sessions + sessions = _load_all_sessions() + for s in agent_manager.get_all_sessions(): + sessions.append(s.model_dump(mode="json")) + + total_sessions = len(sessions) + total_cost = sum(s.get("cost_usd", 0) for s in sessions) + total_messages = 0 + total_tool_calls = 0 + total_duration = 0.0 + model_counts: Counter = Counter() + provider_counts: Counter = Counter() + tool_counts: Counter = Counter() + status_counts: Counter = Counter() + + for s in sessions: + messages = s.get("messages", []) + user_msgs = [m for m in messages if m.get("role") in ("user", "assistant")] + tool_msgs = [m for m in messages if m.get("role") == "tool_call"] + total_messages += len(user_msgs) + total_tool_calls += len(tool_msgs) + + model_counts[s.get("model", "unknown")] += 1 + provider_counts[s.get("provider", "anthropic")] += 1 + status_counts[s.get("status", "unknown")] += 1 + + # Duration + created = s.get("created_at") + closed = s.get("closed_at") + if created and closed: + try: + from datetime import datetime + fmt = "%Y-%m-%dT%H:%M:%S" + c_str = created[:19] + cl_str = closed[:19] + dur = (datetime.fromisoformat(cl_str) - datetime.fromisoformat(c_str)).total_seconds() + if dur > 0: + total_duration += dur + except Exception: + pass + + # Count individual tools + for m in tool_msgs: + content = m.get("content", {}) + if isinstance(content, dict): + tool_name = content.get("tool", "") + if tool_name: + tool_counts[tool_name] += 1 + + avg_duration = total_duration / total_sessions if total_sessions > 0 else 0 + avg_cost = total_cost / total_sessions if total_sessions > 0 else 0 + completed = status_counts.get("completed", 0) + completion_rate = completed / total_sessions if total_sessions > 0 else 0 + + return { + "total_sessions": total_sessions, + "total_cost_usd": round(total_cost, 4), + "total_messages": total_messages, + "total_tool_calls": total_tool_calls, + "avg_duration_seconds": round(avg_duration, 1), + "avg_cost_per_session": round(avg_cost, 4), + "completion_rate": round(completion_rate, 3), + "models_used": dict(model_counts.most_common(10)), + "providers_used": dict(provider_counts.most_common(10)), + "top_tools": dict(tool_counts.most_common(15)), + "status_breakdown": dict(status_counts), + } + + +@analytics.router.get("/status") +async def analytics_status(): + return {"status": "posthog", "enabled": True} diff --git a/backend/apps/analytics/collector.py b/backend/apps/analytics/collector.py new file mode 100644 index 00000000..dddcedc5 --- /dev/null +++ b/backend/apps/analytics/collector.py @@ -0,0 +1,123 @@ +"""PostHog-only analytics collector. + +All events go directly to PostHog. No local SQLite storage. + +Usage from any module: + from backend.apps.analytics.collector import record + record("session.started", {"model": "opus"}, session_id="abc123") +""" + +import logging +import platform +from uuid import uuid4 + +from posthog import Posthog + +logger = logging.getLogger(__name__) + +POSTHOG_API_KEY = "phc_KdVLvAdjCuHeacFoDm1CM1Gb23XikewRqlX67Mj6TNB" +POSTHOG_HOST = "https://us.i.posthog.com" + +_posthog: Posthog | None = None +_installation_id: str | None = None + + +def init(): + """Initialise PostHog. Called once at app startup.""" + global _posthog + if _posthog is None: + _posthog = Posthog( + project_api_key=POSTHOG_API_KEY, + host=POSTHOG_HOST, + ) + return _posthog + + +def shutdown(): + """Flush and close. Called at app shutdown.""" + global _posthog + if _posthog: + try: + _posthog.shutdown() + except Exception: + pass + _posthog = None + + +def _get_installation_id() -> str: + """Get or create a stable anonymous installation ID.""" + global _installation_id + if _installation_id: + return _installation_id + try: + from backend.apps.settings.settings import load_settings, _save_settings + settings = load_settings() + iid = getattr(settings, "installation_id", None) + if not iid: + iid = uuid4().hex + settings.installation_id = iid + _save_settings(settings) + _installation_id = iid + except Exception: + _installation_id = uuid4().hex + return _installation_id + + +def _is_opted_in() -> bool: + """Check if user has opted in to analytics.""" + try: + from backend.apps.settings.settings import load_settings + return getattr(load_settings(), "analytics_opt_in", True) + except Exception: + return True + + +def record( + event_type: str, + properties: dict | None = None, + session_id: str | None = None, + dashboard_id: str | None = None, +): + """Record an analytics event to PostHog.""" + if not _posthog or not _is_opted_in(): + return + + props = {**(properties or {})} + if session_id: + props["session_id"] = session_id + if dashboard_id: + props["dashboard_id"] = dashboard_id + props["os"] = platform.system() + props["platform"] = platform.platform() + + try: + _posthog.capture( + event_type, + distinct_id=_get_installation_id(), + properties=props, + ) + except Exception as e: + logger.debug(f"PostHog capture failed (non-critical): {e}") + + +def identify(extra_properties: dict | None = None): + """Identify the current installation with properties.""" + if not _posthog or not _is_opted_in(): + return + + try: + _posthog.identify( + _get_installation_id(), + properties={ + "os": platform.system(), + "platform": platform.platform(), + **(extra_properties or {}), + }, + ) + except Exception as e: + logger.debug(f"PostHog identify failed (non-critical): {e}") + + +def get_collector(): + """Backward compat — returns None since we no longer have a local collector.""" + return None diff --git a/backend/apps/analytics/models.py b/backend/apps/analytics/models.py new file mode 100644 index 00000000..a8696dde --- /dev/null +++ b/backend/apps/analytics/models.py @@ -0,0 +1,37 @@ +from pydantic import BaseModel +from typing import Optional + + +class AnalyticsEvent(BaseModel): + id: Optional[int] = None + timestamp: str + event_type: str + properties: dict + session_id: Optional[str] = None + dashboard_id: Optional[str] = None + + +class UsageSummary(BaseModel): + total_sessions: int = 0 + total_cost_usd: float = 0.0 + total_messages: int = 0 + total_tool_calls: int = 0 + avg_session_duration_seconds: float = 0.0 + session_completion_rate: float = 0.0 + approval_rate: float = 0.0 + models_used: dict[str, int] = {} + modes_used: dict[str, int] = {} + top_tools: list[list] = [] + + +class TimeSeriesPoint(BaseModel): + date: str + value: float + + +class ExportPayload(BaseModel): + export_version: str = "1.0" + exported_at: str = "" + app_version: str = "unknown" + period: dict = {} + summary: dict = {} diff --git a/backend/apps/auth/__init__.py b/backend/apps/auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/auth/auth.py b/backend/apps/auth/auth.py new file mode 100644 index 00000000..1110fa02 --- /dev/null +++ b/backend/apps/auth/auth.py @@ -0,0 +1,168 @@ +"""Auth SubApp — handles managed-mode authentication with Open Swarm service. + +All endpoints are currently stubbed with mock responses so the full UI flow +works end-to-end without a real proxy server. +""" + +from contextlib import asynccontextmanager +from uuid import uuid4 + +from pydantic import BaseModel +from typing import Optional + +from backend.config.Apps import SubApp +from backend.apps.settings.settings import load_settings, update_settings + + +# ── Models ────────────────────────────────────────────────────────────── + +class LoginRequest(BaseModel): + email: str + password: str + + +class GoogleCallbackRequest(BaseModel): + code: str + redirect_uri: str = "" + + +class LoginResponse(BaseModel): + ok: bool + token: Optional[str] = None + email: Optional[str] = None + proxy_url: Optional[str] = None + error: Optional[str] = None + + +class ValidateResponse(BaseModel): + valid: bool + email: Optional[str] = None + + +class UsageResponse(BaseModel): + used_usd: float + quota_usd: float + reset_date: str + + +# ── SubApp setup ──────────────────────────────────────────────────────── + +@asynccontextmanager +async def _lifespan(): + yield + +auth = SubApp("auth", _lifespan) +router = auth.router + + +# ── Endpoints ─────────────────────────────────────────────────────────── + +@router.post("/login") +async def login(req: LoginRequest) -> LoginResponse: + """Authenticate with email + password. + + TODO: replace with real API call to Open Swarm auth server. + """ + if not req.email or not req.password: + return LoginResponse(ok=False, error="Email and password are required") + + # Stub: generate a mock token for any valid-looking input + mock_token = f"osw_{uuid4().hex}" + proxy_url = "https://api.openswarm.ai" + + # Persist credentials to settings + settings = load_settings() + settings.connection_mode = "managed" + settings.openswarm_auth_token = mock_token + settings.openswarm_proxy_url = proxy_url + settings.openswarm_user_email = req.email + await _save_settings(settings) + + return LoginResponse(ok=True, token=mock_token, email=req.email, proxy_url=proxy_url) + + +@router.post("/google-url") +async def google_auth_url() -> dict: + """Return the Google OAuth authorize URL. + + TODO: replace with real Google OAuth URL construction. + """ + # Stub: return a placeholder URL + return { + "url": "https://accounts.google.com/o/oauth2/v2/auth?client_id=PLACEHOLDER&response_type=code&scope=email+profile&redirect_uri=http://localhost:8324/api/auth/google-callback" + } + + +@router.post("/google-callback") +async def google_callback(req: GoogleCallbackRequest) -> LoginResponse: + """Exchange Google OAuth code for a session token. + + TODO: replace with real OAuth code exchange + Open Swarm auth server call. + """ + if not req.code: + return LoginResponse(ok=False, error="Authorization code is required") + + # Stub: generate a mock token + mock_token = f"osw_{uuid4().hex}" + proxy_url = "https://api.openswarm.ai" + mock_email = "user@gmail.com" + + settings = load_settings() + settings.connection_mode = "managed" + settings.openswarm_auth_token = mock_token + settings.openswarm_proxy_url = proxy_url + settings.openswarm_user_email = mock_email + await _save_settings(settings) + + return LoginResponse(ok=True, token=mock_token, email=mock_email, proxy_url=proxy_url) + + +@router.post("/validate") +async def validate_token() -> ValidateResponse: + """Check if the stored auth token is still valid. + + TODO: replace with real validation call to Open Swarm auth server. + """ + settings = load_settings() + if not settings.openswarm_auth_token: + return ValidateResponse(valid=False) + + # Stub: always return valid + return ValidateResponse(valid=True, email=settings.openswarm_user_email) + + +@router.post("/logout") +async def logout() -> dict: + """Clear managed-mode credentials from settings.""" + settings = load_settings() + settings.connection_mode = "own_key" + settings.openswarm_auth_token = None + settings.openswarm_proxy_url = None + settings.openswarm_user_email = None + await _save_settings(settings) + return {"ok": True} + + +@router.get("/usage") +async def get_usage() -> UsageResponse: + """Fetch usage and quota information for the current managed-mode user. + + TODO: replace with real API call to Open Swarm proxy server. + """ + settings = load_settings() + if not settings.openswarm_auth_token: + return UsageResponse(used_usd=0, quota_usd=0, reset_date="") + + # Stub: return mock usage data + return UsageResponse(used_usd=0, quota_usd=50, reset_date="2026-04-01") + + +# ── Helpers ───────────────────────────────────────────────────────────── + +async def _save_settings(settings): + """Persist settings to disk (reuses the settings module's update logic).""" + import json + from backend.apps.settings.settings import SETTINGS_FILE + + with open(SETTINGS_FILE, "w") as f: + json.dump(settings.model_dump(), f, indent=2) diff --git a/backend/apps/channels/__init__.py b/backend/apps/channels/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/channels/adapters/__init__.py b/backend/apps/channels/adapters/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/channels/adapters/telnyx_adapter.py b/backend/apps/channels/adapters/telnyx_adapter.py new file mode 100644 index 00000000..e5cf037f --- /dev/null +++ b/backend/apps/channels/adapters/telnyx_adapter.py @@ -0,0 +1,146 @@ +"""Telnyx implementation of BaseChannelAdapter. + +Uses Telnyx Call Control v2 for voice and Messaging API for SMS. +""" +import asyncio +import hashlib +import hmac +import logging +from typing import Optional +from functools import partial + +from backend.apps.channels.base_adapter import BaseChannelAdapter + +logger = logging.getLogger(__name__) + + +class TelnyxAdapter(BaseChannelAdapter): + + def __init__(self, api_key: str, public_key: str = ""): + self._api_key = api_key + self._public_key = public_key + self._telnyx = None + + def _get_telnyx(self): + if self._telnyx is None: + import telnyx + telnyx.api_key = self._api_key + self._telnyx = telnyx + return self._telnyx + + async def send_sms(self, to: str, from_: str, body: str) -> dict: + telnyx = self._get_telnyx() + loop = asyncio.get_event_loop() + msg = await loop.run_in_executor( + None, + partial( + telnyx.Message.create, + to=to, + from_=from_, + text=body, + ), + ) + return {"id": msg.id, "status": getattr(msg, "status", "queued")} + + async def send_whatsapp(self, to: str, from_: str, body: str) -> dict: + # Telnyx WhatsApp uses the same messaging API with messaging_profile_id + return await self.send_sms(to, from_, body) + + async def initiate_call( + self, to: str, from_: str, webhook_url: str, greeting: str = "" + ) -> dict: + telnyx = self._get_telnyx() + loop = asyncio.get_event_loop() + call = await loop.run_in_executor( + None, + partial( + telnyx.Call.create, + to=to, + from_=from_, + connection_id=self._api_key, # connection_id should be set separately + webhook_url=webhook_url, + ), + ) + return {"call_control_id": call.call_control_id, "status": "initiated"} + + def verify_webhook_signature( + self, request_url: str, params: dict, signature: str, auth_token: str + ) -> bool: + if not self._public_key: + # Fail closed: no public key means reject + logger.error("Telnyx public key not configured — rejecting webhook") + return False + try: + # Telnyx uses Ed25519 signature verification + # The signature and timestamp are in webhook headers + import base64 + from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey + from cryptography.hazmat.primitives.serialization import load_pem_public_key + + public_key = load_pem_public_key(self._public_key.encode()) + sig_bytes = base64.b64decode(signature) + payload = params.get("_raw_body", "") + timestamp = params.get("_timestamp", "") + signed_payload = f"{timestamp}|{payload}" + public_key.verify(sig_bytes, signed_payload.encode()) + return True + except Exception: + logger.exception("Telnyx signature verification failed") + return False + + def generate_twiml_gather( + self, + prompt: str, + action_url: str, + voice: str = "Polly.Joanna", + language: str = "en-US", + timeout: int = 10, + ) -> str: + # Telnyx uses TeXML (Twilio-compatible XML) + return ( + '' + "" + f'' + f'{_escape_xml(prompt)}' + "" + f'I didn\'t hear anything. Goodbye.' + "" + ) + + def generate_twiml_say( + self, text: str, voice: str = "Polly.Joanna", language: str = "en-US" + ) -> str: + return ( + '' + "" + f'{_escape_xml(text)}' + "" + ) + + def generate_twiml_hangup(self) -> str: + return ( + '' + "" + ) + + async def get_recording_audio(self, recording_url: str, auth_token: str) -> bytes: + import httpx + async with httpx.AsyncClient() as client: + resp = await client.get( + recording_url, + headers={"Authorization": f"Bearer {self._api_key}"}, + follow_redirects=True, + ) + resp.raise_for_status() + return resp.content + + +def _escape_xml(text: str) -> str: + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) diff --git a/backend/apps/channels/adapters/twilio_adapter.py b/backend/apps/channels/adapters/twilio_adapter.py new file mode 100644 index 00000000..7246bea1 --- /dev/null +++ b/backend/apps/channels/adapters/twilio_adapter.py @@ -0,0 +1,139 @@ +"""Twilio implementation of BaseChannelAdapter. + +Handles SMS, WhatsApp, and Voice via the Twilio Python SDK. +""" +import asyncio +import logging +from typing import Optional +from functools import partial + +from backend.apps.channels.base_adapter import BaseChannelAdapter + +logger = logging.getLogger(__name__) + + +class TwilioAdapter(BaseChannelAdapter): + + def __init__(self, account_sid: str, auth_token: str): + self._account_sid = account_sid + self._auth_token = auth_token + self._client = None + + def _get_client(self): + if self._client is None: + from twilio.rest import Client + self._client = Client(self._account_sid, self._auth_token) + return self._client + + async def send_sms(self, to: str, from_: str, body: str) -> dict: + client = self._get_client() + loop = asyncio.get_event_loop() + msg = await loop.run_in_executor( + None, + partial( + client.messages.create, + to=to, + from_=from_, + body=body, + ), + ) + return {"sid": msg.sid, "status": msg.status} + + async def send_whatsapp(self, to: str, from_: str, body: str) -> dict: + wa_to = to if to.startswith("whatsapp:") else f"whatsapp:{to}" + wa_from = from_ if from_.startswith("whatsapp:") else f"whatsapp:{from_}" + client = self._get_client() + loop = asyncio.get_event_loop() + msg = await loop.run_in_executor( + None, + partial( + client.messages.create, + to=wa_to, + from_=wa_from, + body=body, + ), + ) + return {"sid": msg.sid, "status": msg.status} + + async def initiate_call( + self, to: str, from_: str, webhook_url: str, greeting: str = "" + ) -> dict: + client = self._get_client() + loop = asyncio.get_event_loop() + call = await loop.run_in_executor( + None, + partial( + client.calls.create, + to=to, + from_=from_, + url=webhook_url, + ), + ) + return {"sid": call.sid, "status": call.status} + + def verify_webhook_signature( + self, request_url: str, params: dict, signature: str, auth_token: str + ) -> bool: + try: + from twilio.request_validator import RequestValidator + validator = RequestValidator(auth_token) + return validator.validate(request_url, params, signature) + except Exception: + logger.exception("Twilio signature verification failed") + return False + + def generate_twiml_gather( + self, + prompt: str, + action_url: str, + voice: str = "Polly.Joanna", + language: str = "en-US", + timeout: int = 10, + ) -> str: + return ( + '' + "" + f'' + f'{_escape_xml(prompt)}' + "" + f'I didn\'t hear anything. Goodbye.' + "" + ) + + def generate_twiml_say( + self, text: str, voice: str = "Polly.Joanna", language: str = "en-US" + ) -> str: + return ( + '' + "" + f'{_escape_xml(text)}' + "" + ) + + def generate_twiml_hangup(self) -> str: + return ( + '' + "" + ) + + async def get_recording_audio(self, recording_url: str, auth_token: str) -> bytes: + import httpx + async with httpx.AsyncClient() as client: + resp = await client.get( + recording_url, + auth=(self._account_sid, auth_token), + follow_redirects=True, + ) + resp.raise_for_status() + return resp.content + + +def _escape_xml(text: str) -> str: + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) diff --git a/backend/apps/channels/base_adapter.py b/backend/apps/channels/base_adapter.py new file mode 100644 index 00000000..bafeeac7 --- /dev/null +++ b/backend/apps/channels/base_adapter.py @@ -0,0 +1,88 @@ +from abc import ABC, abstractmethod +from typing import Optional + + +class BaseChannelAdapter(ABC): + """Provider-agnostic interface for telephony operations.""" + + @abstractmethod + async def send_sms(self, to: str, from_: str, body: str) -> dict: + """Send an SMS message. Returns provider response dict.""" + ... + + @abstractmethod + async def send_whatsapp(self, to: str, from_: str, body: str) -> dict: + """Send a WhatsApp message. Returns provider response dict.""" + ... + + @abstractmethod + async def initiate_call( + self, to: str, from_: str, webhook_url: str, greeting: str = "" + ) -> dict: + """Initiate an outbound voice call. Returns provider response dict.""" + ... + + @abstractmethod + def verify_webhook_signature( + self, request_url: str, params: dict, signature: str, auth_token: str + ) -> bool: + """Verify that an inbound webhook is authentic.""" + ... + + @abstractmethod + def generate_twiml_gather( + self, + prompt: str, + action_url: str, + voice: str = "Polly.Joanna", + language: str = "en-US", + timeout: int = 10, + ) -> str: + """Generate TwiML (or equivalent) to play a prompt and gather speech.""" + ... + + @abstractmethod + def generate_twiml_say( + self, text: str, voice: str = "Polly.Joanna", language: str = "en-US" + ) -> str: + """Generate TwiML (or equivalent) to speak text.""" + ... + + @abstractmethod + def generate_twiml_hangup(self) -> str: + """Generate TwiML (or equivalent) to end a call.""" + ... + + @abstractmethod + async def get_recording_audio(self, recording_url: str, auth_token: str) -> bytes: + """Download audio from a recording URL.""" + ... + + def chunk_message(self, text: str, max_length: int = 1600) -> list[str]: + """Split a long message into chunks respecting sentence boundaries.""" + if len(text) <= max_length: + return [text] + + chunks: list[str] = [] + remaining = text + + while remaining: + if len(remaining) <= max_length: + chunks.append(remaining) + break + + # Try to split at sentence boundary + split_at = -1 + for sep in [". ", "! ", "? ", "\n\n", "\n", " "]: + idx = remaining.rfind(sep, 0, max_length) + if idx > 0: + split_at = idx + len(sep) + break + + if split_at <= 0: + split_at = max_length + + chunks.append(remaining[:split_at].rstrip()) + remaining = remaining[split_at:].lstrip() + + return chunks diff --git a/backend/apps/channels/call_state.py b/backend/apps/channels/call_state.py new file mode 100644 index 00000000..6a3abc95 --- /dev/null +++ b/backend/apps/channels/call_state.py @@ -0,0 +1,129 @@ +import logging +from datetime import datetime +from typing import Optional, Literal + +logger = logging.getLogger(__name__) + +CallStatus = Literal[ + "ringing", "connected", "gathering", "processing", "responding", "completed", "failed" +] + +VALID_TRANSITIONS: dict[CallStatus, set[CallStatus]] = { + "ringing": {"connected", "completed", "failed"}, + "connected": {"gathering", "completed", "failed"}, + "gathering": {"processing", "completed", "failed"}, + "processing": {"responding", "completed", "failed"}, + "responding": {"gathering", "completed", "failed"}, + "completed": set(), + "failed": set(), +} + + +class CallState: + """Tracks the lifecycle of a single voice call.""" + + def __init__( + self, + call_sid: str, + channel_id: str, + from_number: str, + to_number: str, + ): + self.call_sid = call_sid + self.channel_id = channel_id + self.from_number = from_number + self.to_number = to_number + self.agent_session_id: Optional[str] = None + self.status: CallStatus = "ringing" + self.turns: list[dict] = [] + self.created_at = datetime.now() + self.last_activity = datetime.now() + self.error: Optional[str] = None + + def transition(self, new_status: CallStatus) -> bool: + """Attempt a state transition. Returns True if valid.""" + if new_status in VALID_TRANSITIONS.get(self.status, set()): + logger.info( + "Call %s: %s -> %s", self.call_sid, self.status, new_status + ) + self.status = new_status + self.last_activity = datetime.now() + return True + logger.warning( + "Call %s: invalid transition %s -> %s", + self.call_sid, self.status, new_status, + ) + return False + + def add_turn(self, role: str, content: str): + self.turns.append({ + "role": role, + "content": content, + "timestamp": datetime.now().isoformat(), + }) + self.last_activity = datetime.now() + + @property + def is_active(self) -> bool: + return self.status not in ("completed", "failed") + + @property + def duration_seconds(self) -> float: + return (datetime.now() - self.created_at).total_seconds() + + def to_dict(self) -> dict: + return { + "call_sid": self.call_sid, + "channel_id": self.channel_id, + "from_number": self.from_number, + "to_number": self.to_number, + "agent_session_id": self.agent_session_id, + "status": self.status, + "turns": self.turns, + "created_at": self.created_at.isoformat(), + "last_activity": self.last_activity.isoformat(), + "duration_seconds": self.duration_seconds, + "error": self.error, + } + + +class CallManager: + """Manages all active voice calls.""" + + def __init__(self): + self.calls: dict[str, CallState] = {} + + def create_call( + self, + call_sid: str, + channel_id: str, + from_number: str, + to_number: str, + ) -> CallState: + call = CallState(call_sid, channel_id, from_number, to_number) + self.calls[call_sid] = call + return call + + def get_call(self, call_sid: str) -> Optional[CallState]: + return self.calls.get(call_sid) + + def end_call(self, call_sid: str, status: CallStatus = "completed"): + call = self.calls.get(call_sid) + if call: + call.transition(status) + + def cleanup_stale(self, max_duration_seconds: int = 3600): + """Remove calls that have exceeded max duration.""" + stale = [ + sid + for sid, call in self.calls.items() + if not call.is_active or call.duration_seconds > max_duration_seconds + ] + for sid in stale: + if self.calls[sid].is_active: + self.calls[sid].transition("failed") + self.calls[sid].error = "Exceeded max call duration" + del self.calls[sid] + + def get_active_calls(self) -> list[dict]: + return [c.to_dict() for c in self.calls.values() if c.is_active] diff --git a/backend/apps/channels/channels.py b/backend/apps/channels/channels.py new file mode 100644 index 00000000..60da8283 --- /dev/null +++ b/backend/apps/channels/channels.py @@ -0,0 +1,387 @@ +"""Channels SubApp — REST endpoints and Twilio/Telnyx webhooks.""" +import logging +import os +from contextlib import asynccontextmanager +from datetime import datetime + +from fastapi import HTTPException, Request +from fastapi.responses import JSONResponse, Response + +from backend.config.Apps import SubApp +from backend.apps.channels.models import ( + ChannelConfig, ChannelCreate, ChannelUpdate, VoiceConfig, TTSConfig, STTConfig, +) +from backend.apps.channels.orchestrator import channel_orchestrator +from backend.apps.channels import ws_events + +logger = logging.getLogger(__name__) + + +@asynccontextmanager +async def channels_lifespan(): + logger.info("Channels sub-app starting") + await channel_orchestrator.restore_all() + yield + logger.info("Channels sub-app shutting down") + await channel_orchestrator.persist_all() + + +channels = SubApp("channels", channels_lifespan) + + +# ─── CRUD Endpoints ────────────────────────────────────────────── + + +@channels.router.get("/list") +async def list_channels(): + configs = list(channel_orchestrator.configs.values()) + return { + "channels": [c.model_dump(mode="json") for c in configs], + } + + +@channels.router.get("/{channel_id}") +async def get_channel(channel_id: str): + config = channel_orchestrator.configs.get(channel_id) + if not config: + raise HTTPException(404, "Channel not found") + return config.model_dump(mode="json") + + +@channels.router.post("/create") +async def create_channel(body: ChannelCreate): + config = ChannelConfig( + name=body.name, + channel_type=body.channel_type, + provider=body.provider, + phone_number=body.phone_number, + credentials=body.credentials, + ) + if body.agent_config: + config.agent_config = body.agent_config + if body.security: + config.security = body.security + if body.voice_config: + config.voice_config = body.voice_config + if body.tts_config: + config.tts_config = body.tts_config + if body.stt_config: + config.stt_config = body.stt_config + + channel_orchestrator.save_config(config) + return {"channel": config.model_dump(mode="json")} + + +@channels.router.put("/{channel_id}") +async def update_channel(channel_id: str, body: ChannelUpdate): + config = channel_orchestrator.configs.get(channel_id) + if not config: + raise HTTPException(404, "Channel not found") + + updates = body.model_dump(exclude_none=True) + for key, val in updates.items(): + setattr(config, key, val) + + # Re-create adapter if credentials changed + if "credentials" in updates or "provider" in updates: + channel_orchestrator.adapters.pop(channel_id, None) + + channel_orchestrator.save_config(config) + return {"channel": config.model_dump(mode="json")} + + +@channels.router.delete("/{channel_id}") +async def delete_channel(channel_id: str): + if channel_id not in channel_orchestrator.configs: + raise HTTPException(404, "Channel not found") + channel_orchestrator.delete_config(channel_id) + return {"ok": True} + + +# ─── Enable / Disable / Test ───────────────────────────────────── + + +@channels.router.post("/{channel_id}/enable") +async def enable_channel(channel_id: str): + config = channel_orchestrator.configs.get(channel_id) + if not config: + raise HTTPException(404, "Channel not found") + + try: + channel_orchestrator.get_adapter(config) + config.enabled = True + config.status = "active" + config.status_message = None + channel_orchestrator.save_config(config) + await ws_events.emit_channel_status(channel_id, "active") + return {"ok": True, "status": "active"} + except Exception as e: + config.status = "error" + config.status_message = str(e) + channel_orchestrator.save_config(config) + raise HTTPException(400, f"Failed to enable channel: {e}") + + +@channels.router.post("/{channel_id}/disable") +async def disable_channel(channel_id: str): + config = channel_orchestrator.configs.get(channel_id) + if not config: + raise HTTPException(404, "Channel not found") + + config.enabled = False + config.status = "inactive" + channel_orchestrator.adapters.pop(channel_id, None) + channel_orchestrator.save_config(config) + await ws_events.emit_channel_status(channel_id, "inactive") + return {"ok": True} + + +@channels.router.post("/{channel_id}/test") +async def test_channel(channel_id: str, body: dict | None = None): + config = channel_orchestrator.configs.get(channel_id) + if not config: + raise HTTPException(404, "Channel not found") + + to_number = (body or {}).get("to_number", "") + if not to_number: + raise HTTPException(400, "to_number is required for test") + + try: + adapter = channel_orchestrator.get_adapter(config) + if config.channel_type == "whatsapp": + result = await adapter.send_whatsapp(to_number, config.phone_number, "Test message from Open Swarm") + elif config.channel_type == "voice": + result = {"message": "Voice test: configure webhook and call the number"} + else: + result = await adapter.send_sms(to_number, config.phone_number, "Test message from Open Swarm") + return {"ok": True, "result": result} + except Exception as e: + raise HTTPException(400, f"Test failed: {e}") + + +# ─── Conversations ──────────────────────────────────────────────── + + +@channels.router.get("/{channel_id}/conversations") +async def list_conversations(channel_id: str): + convs = [ + c.model_dump(mode="json") + for c in channel_orchestrator.conversations.values() + if c.channel_id == channel_id + ] + return {"conversations": convs} + + +@channels.router.get("/{channel_id}/conversations/{conversation_id}") +async def get_conversation(channel_id: str, conversation_id: str): + for conv in channel_orchestrator.conversations.values(): + if conv.id == conversation_id and conv.channel_id == channel_id: + return conv.model_dump(mode="json") + raise HTTPException(404, "Conversation not found") + + +# ─── Outbound ───────────────────────────────────────────────────── + + +@channels.router.post("/{channel_id}/send") +async def send_outbound(channel_id: str, body: dict): + to_number = body.get("to_number", "") + message = body.get("message", "") + if not to_number or not message: + raise HTTPException(400, "to_number and message are required") + try: + result = await channel_orchestrator.send_outbound(channel_id, to_number, message) + return result + except ValueError as e: + raise HTTPException(404, str(e)) + + +@channels.router.post("/{channel_id}/call") +async def initiate_call(channel_id: str, body: dict): + to_number = body.get("to_number", "") + if not to_number: + raise HTTPException(400, "to_number is required") + try: + result = await channel_orchestrator.initiate_outbound_call(channel_id, to_number) + return result + except ValueError as e: + raise HTTPException(404, str(e)) + + +# ─── Twilio Webhooks ───────────────────────────────────────────── + + +@channels.router.post("/webhooks/twilio/sms") +async def twilio_sms_webhook(request: Request): + """Inbound SMS webhook from Twilio.""" + form = await request.form() + channel_id = request.query_params.get("channel_id", "") + + # Find channel by phone number if channel_id not provided + if not channel_id: + to_number = form.get("To", "") + for cfg in channel_orchestrator.configs.values(): + if cfg.phone_number == to_number and cfg.channel_type == "sms": + channel_id = cfg.id + break + + config = channel_orchestrator.configs.get(channel_id) + if not config: + return Response(status_code=404) + + # Verify signature + if config.security.verify_signatures: + adapter = channel_orchestrator.get_adapter(config) + sig = request.headers.get("X-Twilio-Signature", "") + url = str(request.url) + if not adapter.verify_webhook_signature(url, dict(form), sig, config.credentials.get("auth_token", "")): + logger.warning("Invalid Twilio signature for channel %s", channel_id) + return Response(status_code=403) + + from_number = form.get("From", "") + body = form.get("Body", "") + num_media = int(form.get("NumMedia", "0")) + media_urls = [form.get(f"MediaUrl{i}", "") for i in range(num_media)] + media_urls = [u for u in media_urls if u] + + await channel_orchestrator.handle_inbound_sms(channel_id, from_number, body, media_urls) + + # Return empty TwiML (Twilio expects XML response) + return Response( + content='', + media_type="application/xml", + ) + + +@channels.router.post("/webhooks/twilio/whatsapp") +async def twilio_whatsapp_webhook(request: Request): + """Inbound WhatsApp webhook from Twilio.""" + form = await request.form() + channel_id = request.query_params.get("channel_id", "") + + if not channel_id: + to_number = form.get("To", "").replace("whatsapp:", "") + for cfg in channel_orchestrator.configs.values(): + if cfg.phone_number == to_number and cfg.channel_type == "whatsapp": + channel_id = cfg.id + break + + config = channel_orchestrator.configs.get(channel_id) + if not config: + return Response(status_code=404) + + if config.security.verify_signatures: + adapter = channel_orchestrator.get_adapter(config) + sig = request.headers.get("X-Twilio-Signature", "") + if not adapter.verify_webhook_signature(str(request.url), dict(form), sig, config.credentials.get("auth_token", "")): + return Response(status_code=403) + + from_number = form.get("From", "").replace("whatsapp:", "") + body = form.get("Body", "") + num_media = int(form.get("NumMedia", "0")) + media_urls = [form.get(f"MediaUrl{i}", "") for i in range(num_media)] + + await channel_orchestrator.handle_inbound_sms(channel_id, from_number, body, media_urls or None) + + return Response( + content='', + media_type="application/xml", + ) + + +@channels.router.post("/webhooks/twilio/voice") +async def twilio_voice_webhook(request: Request): + """Inbound voice call webhook from Twilio.""" + form = await request.form() + channel_id = request.query_params.get("channel_id", "") + + if not channel_id: + to_number = form.get("To", "") + for cfg in channel_orchestrator.configs.values(): + if cfg.phone_number == to_number and cfg.channel_type == "voice": + channel_id = cfg.id + break + + call_sid = form.get("CallSid", "") + from_number = form.get("From", "") + to_number = form.get("To", "") + + twiml = await channel_orchestrator.handle_inbound_call( + channel_id, call_sid, from_number, to_number + ) + + return Response(content=twiml, media_type="application/xml") + + +@channels.router.post("/webhooks/twilio/voice/gather") +async def twilio_voice_gather_webhook(request: Request): + """Speech gathered from a voice call.""" + form = await request.form() + channel_id = request.query_params.get("channel_id", "") + call_sid = request.query_params.get("call_sid", "") or form.get("CallSid", "") + + speech_result = form.get("SpeechResult", "") + + if not speech_result: + # No speech detected, ask again or hang up + config = channel_orchestrator.configs.get(channel_id) + if config: + adapter = channel_orchestrator.get_adapter(config) + voice_cfg = config.voice_config or VoiceConfig() + twiml = adapter.generate_twiml_say( + "I didn't catch that. Goodbye.", voice=voice_cfg.voice + ) + else: + twiml = 'Goodbye.' + return Response(content=twiml, media_type="application/xml") + + twiml = await channel_orchestrator.handle_voice_gather( + channel_id, call_sid, speech_result + ) + + return Response(content=twiml, media_type="application/xml") + + +@channels.router.post("/webhooks/twilio/voice/status") +async def twilio_voice_status_webhook(request: Request): + """Call status update from Twilio.""" + form = await request.form() + call_sid = form.get("CallSid", "") + status = form.get("CallStatus", "") + + channel_orchestrator.handle_call_status(call_sid, status) + return Response(status_code=204) + + +# ─── Telnyx Webhook ─────────────────────────────────────────────── + + +@channels.router.post("/webhooks/telnyx") +async def telnyx_webhook(request: Request): + """Unified Telnyx webhook for SMS and Voice events.""" + body = await request.json() + event_type = body.get("data", {}).get("event_type", "") + payload = body.get("data", {}).get("payload", {}) + + channel_id = request.query_params.get("channel_id", "") + + if event_type == "message.received": + from_number = payload.get("from", {}).get("phone_number", "") + text = payload.get("text", "") + await channel_orchestrator.handle_inbound_sms(channel_id, from_number, text) + elif event_type in ("call.initiated", "call.answered"): + call_sid = payload.get("call_control_id", "") + from_number = payload.get("from", "") + to_number = payload.get("to", "") + # Telnyx voice uses Call Control commands rather than TwiML + logger.info("Telnyx call event: %s for %s", event_type, call_sid) + + return JSONResponse({"ok": True}) + + +# ─── Active Calls ───────────────────────────────────────────────── + + +@channels.router.get("/calls/active") +async def list_active_calls(): + return {"calls": channel_orchestrator.call_manager.get_active_calls()} diff --git a/backend/apps/channels/media_handler.py b/backend/apps/channels/media_handler.py new file mode 100644 index 00000000..71055e4c --- /dev/null +++ b/backend/apps/channels/media_handler.py @@ -0,0 +1,60 @@ +"""Audio attachment processing for WhatsApp voice notes and media messages.""" +import logging +from typing import Optional + +import httpx + +from backend.apps.channels.models import STTConfig +from backend.apps.channels import stt_service + +logger = logging.getLogger(__name__) + +SUPPORTED_FORMATS = { + "audio/ogg", "audio/mpeg", "audio/wav", "audio/mp4", + "audio/flac", "audio/webm", "audio/x-wav", +} +MAX_MEDIA_BYTES = 20 * 1024 * 1024 + + +async def process_audio_attachment( + url: str, + content_type: str, + stt_config: STTConfig, + auth: tuple[str, str] | None = None, +) -> Optional[str]: + """Download an audio attachment and return its transcript. + + Args: + url: URL to download the audio from. + content_type: MIME type of the audio. + stt_config: STT configuration for transcription. + auth: Optional (username, password) tuple for basic auth (e.g. Twilio). + + Returns: + Transcript string, or None on failure. + """ + if content_type not in SUPPORTED_FORMATS: + logger.warning("Unsupported audio format: %s", content_type) + return None + + try: + async with httpx.AsyncClient(timeout=60) as client: + kwargs = {"follow_redirects": True} + if auth: + kwargs["auth"] = auth + resp = await client.get(url, **kwargs) + resp.raise_for_status() + audio_bytes = resp.content + except Exception: + logger.exception("Failed to download audio from %s", url) + return None + + if len(audio_bytes) > MAX_MEDIA_BYTES: + logger.warning("Audio attachment exceeds %d bytes", MAX_MEDIA_BYTES) + return None + + if len(audio_bytes) < 1024: + logger.debug("Audio attachment too small, skipping") + return None + + return await stt_service.transcribe(audio_bytes, stt_config, content_type) diff --git a/backend/apps/channels/models.py b/backend/apps/channels/models.py new file mode 100644 index 00000000..3c0b9502 --- /dev/null +++ b/backend/apps/channels/models.py @@ -0,0 +1,116 @@ +from pydantic import BaseModel, Field +from typing import Optional, Literal, Any +from datetime import datetime +from uuid import uuid4 + + +class ChannelAgentConfig(BaseModel): + mode: str = "agent" + model: str = "sonnet" + system_prompt: Optional[str] = None + max_turns: int = 10 + allowed_tools: Optional[list[str]] = None + + +class ChannelSecurityConfig(BaseModel): + verify_signatures: bool = True + allowlist: list[str] = Field(default_factory=list) + blocklist: list[str] = Field(default_factory=list) + rate_limit_per_minute: int = 10 + rate_limit_per_hour: int = 60 + + +class VoiceConfig(BaseModel): + mode: Literal["conversation", "notify"] = "conversation" + greeting_message: str = "Hello, how can I help you?" + silence_timeout_ms: int = 700 + max_call_duration_seconds: int = 600 + gather_timeout_seconds: int = 10 + voice: str = "Polly.Joanna" + language: str = "en-US" + + +class TTSConfig(BaseModel): + provider: Literal["twilio_say", "elevenlabs", "openai_tts", "edge_tts"] = "twilio_say" + auto_tts_mode: Literal["off", "always", "inbound", "tagged"] = "off" + elevenlabs_voice_id: Optional[str] = None + elevenlabs_model_id: str = "eleven_v3" + openai_voice: str = "alloy" + skip_short_text: bool = True + summarize_long_replies: bool = True + max_tts_chars: int = 4000 + + +class STTConfig(BaseModel): + provider: Literal["twilio_builtin", "deepgram", "openai_whisper"] = "twilio_builtin" + deepgram_model: str = "nova-3" + language: str = "en-US" + fallback_chain: list[str] = Field(default_factory=lambda: ["twilio_builtin"]) + + +class ChannelConfig(BaseModel): + id: str = Field(default_factory=lambda: uuid4().hex) + name: str = "" + channel_type: Literal["sms", "whatsapp", "voice"] = "sms" + provider: Literal["twilio", "telnyx"] = "twilio" + enabled: bool = False + phone_number: str = "" + credentials: dict[str, str] = Field(default_factory=dict) + agent_config: ChannelAgentConfig = Field(default_factory=ChannelAgentConfig) + security: ChannelSecurityConfig = Field(default_factory=ChannelSecurityConfig) + voice_config: Optional[VoiceConfig] = None + tts_config: Optional[TTSConfig] = None + stt_config: Optional[STTConfig] = None + status: Literal["inactive", "active", "error"] = "inactive" + status_message: Optional[str] = None + created_at: str = Field(default_factory=lambda: datetime.now().isoformat()) + updated_at: str = Field(default_factory=lambda: datetime.now().isoformat()) + last_message_at: Optional[str] = None + message_count: int = 0 + + +class ChannelMessage(BaseModel): + id: str = Field(default_factory=lambda: uuid4().hex) + direction: Literal["inbound", "outbound"] = "inbound" + content: str = "" + media_urls: list[str] = Field(default_factory=list) + timestamp: str = Field(default_factory=lambda: datetime.now().isoformat()) + channel_type: str = "" + provider_message_id: Optional[str] = None + + +class ChannelConversation(BaseModel): + id: str = Field(default_factory=lambda: uuid4().hex) + channel_id: str = "" + phone_number: str = "" + agent_session_id: Optional[str] = None + messages: list[ChannelMessage] = Field(default_factory=list) + created_at: str = Field(default_factory=lambda: datetime.now().isoformat()) + updated_at: str = Field(default_factory=lambda: datetime.now().isoformat()) + status: Literal["active", "closed"] = "active" + + +class ChannelCreate(BaseModel): + name: str + channel_type: Literal["sms", "whatsapp", "voice"] = "sms" + provider: Literal["twilio", "telnyx"] = "twilio" + phone_number: str = "" + credentials: dict[str, str] = Field(default_factory=dict) + agent_config: Optional[ChannelAgentConfig] = None + security: Optional[ChannelSecurityConfig] = None + voice_config: Optional[VoiceConfig] = None + tts_config: Optional[TTSConfig] = None + stt_config: Optional[STTConfig] = None + + +class ChannelUpdate(BaseModel): + name: Optional[str] = None + channel_type: Optional[Literal["sms", "whatsapp", "voice"]] = None + provider: Optional[Literal["twilio", "telnyx"]] = None + phone_number: Optional[str] = None + credentials: Optional[dict[str, str]] = None + agent_config: Optional[ChannelAgentConfig] = None + security: Optional[ChannelSecurityConfig] = None + voice_config: Optional[VoiceConfig] = None + tts_config: Optional[TTSConfig] = None + stt_config: Optional[STTConfig] = None diff --git a/backend/apps/channels/orchestrator.py b/backend/apps/channels/orchestrator.py new file mode 100644 index 00000000..4b06979e --- /dev/null +++ b/backend/apps/channels/orchestrator.py @@ -0,0 +1,559 @@ +"""Channel orchestrator — routes inbound messages/calls to agent sessions. + +This is the central routing layer that bridges telephony events to the +existing AgentManager. Each phone number gets its own ChannelConversation +which maps to an AgentSession. +""" +import asyncio +import json +import logging +import os +import time +from datetime import datetime +from typing import Optional + +from backend.apps.channels.models import ( + ChannelConfig, ChannelConversation, ChannelMessage, +) +from backend.apps.channels.call_state import CallManager, CallState +from backend.apps.channels.base_adapter import BaseChannelAdapter +from backend.apps.channels import ws_events +from backend.apps.agents.models import AgentConfig +from backend.config.paths import DATA_ROOT + +logger = logging.getLogger(__name__) + +CHANNELS_DIR = os.path.join(DATA_ROOT, "channels") +CHANNELS_SESSIONS_DIR = os.path.join(DATA_ROOT, "channels", "sessions") + +PLATFORM_MAX_LENGTH = { + "sms": 1600, + "whatsapp": 4096, + "voice": 100000, +} + + +class RateLimiter: + """Simple token-bucket rate limiter per phone number.""" + + def __init__(self): + self._buckets: dict[str, list[float]] = {} + + def check(self, key: str, per_minute: int, per_hour: int) -> bool: + now = time.time() + if key not in self._buckets: + self._buckets[key] = [] + + # Prune old entries + self._buckets[key] = [t for t in self._buckets[key] if now - t < 3600] + + recent_minute = sum(1 for t in self._buckets[key] if now - t < 60) + recent_hour = len(self._buckets[key]) + + if recent_minute >= per_minute or recent_hour >= per_hour: + return False + + self._buckets[key].append(now) + return True + + +class ChannelOrchestrator: + """Manages channel configs, conversations, and message routing.""" + + def __init__(self): + self.configs: dict[str, ChannelConfig] = {} + self.conversations: dict[str, ChannelConversation] = {} # key: "{channel_id}:{phone}" + self.adapters: dict[str, BaseChannelAdapter] = {} + self.call_manager = CallManager() + self.rate_limiter = RateLimiter() + self._agent_listeners: dict[str, asyncio.Task] = {} + + # ─── Config persistence ─────────────────────────────────────── + + def _ensure_dirs(self): + os.makedirs(CHANNELS_DIR, exist_ok=True) + os.makedirs(CHANNELS_SESSIONS_DIR, exist_ok=True) + + def _config_path(self, channel_id: str) -> str: + return os.path.join(CHANNELS_DIR, f"{channel_id}.json") + + def _conv_path(self, channel_id: str) -> str: + return os.path.join(CHANNELS_SESSIONS_DIR, f"{channel_id}.json") + + def save_config(self, config: ChannelConfig): + self._ensure_dirs() + config.updated_at = datetime.now().isoformat() + self.configs[config.id] = config + with open(self._config_path(config.id), "w") as f: + json.dump(config.model_dump(mode="json"), f, indent=2) + + def delete_config(self, channel_id: str): + self.configs.pop(channel_id, None) + self.adapters.pop(channel_id, None) + path = self._config_path(channel_id) + if os.path.exists(path): + os.remove(path) + + def load_all_configs(self): + self._ensure_dirs() + self.configs.clear() + for fname in os.listdir(CHANNELS_DIR): + if fname.endswith(".json"): + try: + with open(os.path.join(CHANNELS_DIR, fname)) as f: + data = json.load(f) + config = ChannelConfig(**data) + self.configs[config.id] = config + except Exception: + logger.exception("Failed to load channel config: %s", fname) + + # ─── Conversation persistence ───────────────────────────────── + + def save_conversations(self, channel_id: str): + self._ensure_dirs() + convs = [ + c.model_dump(mode="json") + for c in self.conversations.values() + if c.channel_id == channel_id + ] + with open(self._conv_path(channel_id), "w") as f: + json.dump(convs, f, indent=2) + + def load_all_conversations(self): + self._ensure_dirs() + self.conversations.clear() + for fname in os.listdir(CHANNELS_SESSIONS_DIR): + if fname.endswith(".json"): + try: + with open(os.path.join(CHANNELS_SESSIONS_DIR, fname)) as f: + convs = json.load(f) + for data in convs: + conv = ChannelConversation(**data) + key = f"{conv.channel_id}:{conv.phone_number}" + self.conversations[key] = conv + except Exception: + logger.exception("Failed to load conversations: %s", fname) + + # ─── Adapter management ─────────────────────────────────────── + + def get_adapter(self, config: ChannelConfig) -> BaseChannelAdapter: + if config.id not in self.adapters: + self.adapters[config.id] = self._create_adapter(config) + return self.adapters[config.id] + + def _create_adapter(self, config: ChannelConfig) -> BaseChannelAdapter: + if config.provider == "twilio": + from backend.apps.channels.adapters.twilio_adapter import TwilioAdapter + return TwilioAdapter( + account_sid=config.credentials.get("account_sid", ""), + auth_token=config.credentials.get("auth_token", ""), + ) + elif config.provider == "telnyx": + from backend.apps.channels.adapters.telnyx_adapter import TelnyxAdapter + return TelnyxAdapter( + api_key=config.credentials.get("api_key", ""), + public_key=config.credentials.get("public_key", ""), + ) + raise ValueError(f"Unknown provider: {config.provider}") + + # ─── Security checks ───────────────────────────────────────── + + def _check_allowlist(self, config: ChannelConfig, phone: str) -> bool: + sec = config.security + if phone in sec.blocklist: + return False + if sec.allowlist and phone not in sec.allowlist: + return False + return True + + def _check_rate_limit(self, config: ChannelConfig, phone: str) -> bool: + sec = config.security + return self.rate_limiter.check( + phone, sec.rate_limit_per_minute, sec.rate_limit_per_hour + ) + + # ─── Inbound SMS / WhatsApp ─────────────────────────────────── + + async def handle_inbound_sms( + self, + channel_id: str, + from_number: str, + body: str, + media_urls: list[str] | None = None, + ) -> Optional[str]: + """Handle an inbound SMS or WhatsApp message. Returns agent response or None.""" + config = self.configs.get(channel_id) + if not config or not config.enabled: + logger.warning("Channel %s not found or disabled", channel_id) + return None + + if not self._check_allowlist(config, from_number): + logger.info("Blocked message from %s (not in allowlist)", from_number) + return None + + if not self._check_rate_limit(config, from_number): + logger.info("Rate limited: %s", from_number) + return None + + # Process media attachments (voice notes) + if media_urls and config.stt_config: + from backend.apps.channels.media_handler import process_audio_attachment + for url in media_urls: + transcript = await process_audio_attachment( + url, "audio/ogg", config.stt_config, + auth=( + config.credentials.get("account_sid", ""), + config.credentials.get("auth_token", ""), + ) if config.provider == "twilio" else None, + ) + if transcript: + body = f"{body}\n\n[Voice Note Transcript]: {transcript}" if body else transcript + + # Get or create conversation + conv_key = f"{channel_id}:{from_number}" + conv = self.conversations.get(conv_key) + if not conv: + conv = ChannelConversation( + channel_id=channel_id, + phone_number=from_number, + ) + self.conversations[conv_key] = conv + + # Record inbound message + inbound_msg = ChannelMessage( + direction="inbound", + content=body, + media_urls=media_urls or [], + channel_type=config.channel_type, + ) + conv.messages.append(inbound_msg) + conv.updated_at = datetime.now().isoformat() + + await ws_events.emit_channel_message( + channel_id, conv.id, inbound_msg.model_dump(mode="json") + ) + + # Launch or reuse agent session + agent_response = await self._route_to_agent(config, conv, body) + + if agent_response: + # Send response back via SMS/WhatsApp + adapter = self.get_adapter(config) + max_len = PLATFORM_MAX_LENGTH.get(config.channel_type, 1600) + chunks = adapter.chunk_message(agent_response, max_len) + + for chunk in chunks: + if config.channel_type == "whatsapp": + await adapter.send_whatsapp(from_number, config.phone_number, chunk) + else: + await adapter.send_sms(from_number, config.phone_number, chunk) + + outbound_msg = ChannelMessage( + direction="outbound", + content=agent_response, + channel_type=config.channel_type, + ) + conv.messages.append(outbound_msg) + conv.updated_at = datetime.now().isoformat() + config.message_count += 1 + config.last_message_at = datetime.now().isoformat() + + await ws_events.emit_channel_message( + channel_id, conv.id, outbound_msg.model_dump(mode="json") + ) + + self.save_conversations(channel_id) + self.save_config(config) + + return agent_response + + # ─── Inbound Voice ──────────────────────────────────────────── + + async def handle_inbound_call( + self, channel_id: str, call_sid: str, from_number: str, to_number: str + ) -> str: + """Handle an inbound voice call. Returns initial TwiML.""" + config = self.configs.get(channel_id) + if not config or not config.enabled: + adapter = self._fallback_adapter(config) + return adapter.generate_twiml_hangup() + + if not self._check_allowlist(config, from_number): + adapter = self.get_adapter(config) + return adapter.generate_twiml_say("Sorry, you are not authorized to call this number.") + + voice_cfg = config.voice_config or VoiceConfig() + adapter = self.get_adapter(config) + + # Create call state + call = self.call_manager.create_call(call_sid, channel_id, from_number, to_number) + call.transition("connected") + call.transition("gathering") + + await ws_events.emit_call_event(channel_id, call_sid, "call_started", { + "from": from_number, "to": to_number, + }) + + # Return TwiML to greet and gather speech + from backend.apps.settings.settings import load_settings + settings = load_settings() + webhook_base = getattr(settings, "webhook_base_url", "") or "" + gather_url = f"{webhook_base}/api/channels/webhooks/twilio/voice/gather?channel_id={channel_id}&call_sid={call_sid}" + + return adapter.generate_twiml_gather( + prompt=voice_cfg.greeting_message, + action_url=gather_url, + voice=voice_cfg.voice, + language=voice_cfg.language, + timeout=voice_cfg.gather_timeout_seconds, + ) + + async def handle_voice_gather( + self, channel_id: str, call_sid: str, speech_result: str + ) -> str: + """Handle gathered speech from a voice call. Returns response TwiML.""" + config = self.configs.get(channel_id) + if not config: + return '' + + call = self.call_manager.get_call(call_sid) + if not call or not call.is_active: + adapter = self.get_adapter(config) + return adapter.generate_twiml_hangup() + + call.transition("processing") + call.add_turn("user", speech_result) + + voice_cfg = config.voice_config or VoiceConfig() + adapter = self.get_adapter(config) + + # Route speech to agent + conv_key = f"{channel_id}:{call.from_number}" + conv = self.conversations.get(conv_key) + if not conv: + conv = ChannelConversation( + channel_id=channel_id, + phone_number=call.from_number, + ) + self.conversations[conv_key] = conv + + agent_response = await self._route_to_agent(config, conv, speech_result) + + if not agent_response: + agent_response = "I'm sorry, I couldn't process that. Could you try again?" + + call.transition("responding") + call.add_turn("assistant", agent_response) + + await ws_events.emit_call_event(channel_id, call_sid, "turn_complete", { + "user": speech_result, "assistant": agent_response, + }) + + # Check if we should continue or end + if voice_cfg.mode == "notify": + call.transition("completed") + return adapter.generate_twiml_say(agent_response, voice=voice_cfg.voice) + + # Conversation mode: say response then gather again + from backend.apps.settings.settings import load_settings + settings = load_settings() + webhook_base = getattr(settings, "webhook_base_url", "") or "" + gather_url = f"{webhook_base}/api/channels/webhooks/twilio/voice/gather?channel_id={channel_id}&call_sid={call_sid}" + + call.transition("gathering") + + return ( + '' + "" + f'{_escape_xml(agent_response)}' + f'' + "" + f'Are you still there? Goodbye.' + "" + ) + + def handle_call_status(self, call_sid: str, status: str): + """Handle Twilio call status callback.""" + call = self.call_manager.get_call(call_sid) + if not call: + return + if status in ("completed", "busy", "no-answer", "canceled", "failed"): + final = "failed" if status == "failed" else "completed" + call.transition(final) + asyncio.create_task( + ws_events.emit_call_event(call.channel_id, call_sid, "call_ended", { + "status": status, + }) + ) + + # ─── Agent routing ──────────────────────────────────────────── + + async def _route_to_agent( + self, config: ChannelConfig, conv: ChannelConversation, text: str + ) -> Optional[str]: + """Send a message to an agent session and wait for the response.""" + from backend.apps.agents.agent_manager import agent_manager + from backend.apps.agents.ws_manager import ws_manager + + # Launch agent if no session exists + if not conv.agent_session_id or not agent_manager.get_session(conv.agent_session_id): + ac = config.agent_config + agent_cfg = AgentConfig( + name=f"{config.channel_type}: {conv.phone_number}", + model=ac.model, + mode=ac.mode, + system_prompt=ac.system_prompt, + max_turns=ac.max_turns, + ) + if ac.allowed_tools: + agent_cfg.allowed_tools = ac.allowed_tools + + session = await agent_manager.launch_agent(agent_cfg) + conv.agent_session_id = session.id + + session_id = conv.agent_session_id + + # Set up a future to capture the agent's response + response_future: asyncio.Future[str] = asyncio.get_event_loop().create_future() + + async def _on_agent_event(event: str, data: dict): + if response_future.done(): + return + if event == "agent:message": + msg = data.get("message", {}) + if msg.get("role") == "assistant": + content = msg.get("content", "") + if isinstance(content, list): + # Extract text from content blocks + parts = [ + b.get("text", "") + for b in content + if isinstance(b, dict) and b.get("type") == "text" + ] + content = "\n".join(parts) + if content and not response_future.done(): + response_future.set_result(content) + elif event == "agent:status": + status = data.get("status", "") + if status in ("completed", "error", "stopped") and not response_future.done(): + response_future.set_result("") + + # Register listener for this session's events + # We tap into ws_manager's send_to_session by monkey-patching temporarily + original_send = ws_manager.send_to_session + + async def _hooked_send(sid: str, event: str, data: dict): + await original_send(sid, event, data) + if sid == session_id: + await _on_agent_event(event, data) + + ws_manager.send_to_session = _hooked_send + + try: + await agent_manager.send_message(session_id, text) + response = await asyncio.wait_for(response_future, timeout=120) + return response if response else None + except asyncio.TimeoutError: + logger.warning("Agent response timed out for session %s", session_id) + return None + except Exception: + logger.exception("Error routing to agent") + return None + finally: + ws_manager.send_to_session = original_send + + def _fallback_adapter(self, config: Optional[ChannelConfig] = None) -> BaseChannelAdapter: + """Return a minimal adapter for generating hangup TwiML.""" + from backend.apps.channels.adapters.twilio_adapter import TwilioAdapter + return TwilioAdapter("", "") + + # ─── Outbound ───────────────────────────────────────────────── + + async def send_outbound( + self, channel_id: str, to_number: str, message: str + ) -> dict: + config = self.configs.get(channel_id) + if not config: + raise ValueError(f"Channel {channel_id} not found") + + adapter = self.get_adapter(config) + max_len = PLATFORM_MAX_LENGTH.get(config.channel_type, 1600) + chunks = adapter.chunk_message(message, max_len) + results = [] + + for chunk in chunks: + if config.channel_type == "whatsapp": + r = await adapter.send_whatsapp(to_number, config.phone_number, chunk) + else: + r = await adapter.send_sms(to_number, config.phone_number, chunk) + results.append(r) + + # Record outbound + conv_key = f"{channel_id}:{to_number}" + conv = self.conversations.get(conv_key) + if not conv: + conv = ChannelConversation(channel_id=channel_id, phone_number=to_number) + self.conversations[conv_key] = conv + + conv.messages.append(ChannelMessage( + direction="outbound", content=message, channel_type=config.channel_type, + )) + conv.updated_at = datetime.now().isoformat() + self.save_conversations(channel_id) + + return {"sent": len(chunks), "results": results} + + async def initiate_outbound_call( + self, channel_id: str, to_number: str + ) -> dict: + config = self.configs.get(channel_id) + if not config: + raise ValueError(f"Channel {channel_id} not found") + + from backend.apps.settings.settings import load_settings + settings = load_settings() + webhook_base = getattr(settings, "webhook_base_url", "") or "" + voice_webhook = f"{webhook_base}/api/channels/webhooks/twilio/voice?channel_id={channel_id}" + + adapter = self.get_adapter(config) + result = await adapter.initiate_call( + to=to_number, + from_=config.phone_number, + webhook_url=voice_webhook, + ) + return result + + # ─── Lifecycle ──────────────────────────────────────────────── + + async def restore_all(self): + self.load_all_configs() + self.load_all_conversations() + for config in self.configs.values(): + if config.enabled: + try: + self.get_adapter(config) + config.status = "active" + except Exception: + config.status = "error" + config.status_message = "Failed to initialize adapter" + + async def persist_all(self): + for config in self.configs.values(): + self.save_config(config) + self.save_conversations(config.id) + + +def _escape_xml(text: str) -> str: + return ( + text.replace("&", "&") + .replace("<", "<") + .replace(">", ">") + .replace('"', """) + .replace("'", "'") + ) + + +# Singleton +from backend.apps.channels.models import VoiceConfig # noqa: E402 +channel_orchestrator = ChannelOrchestrator() diff --git a/backend/apps/channels/stt_service.py b/backend/apps/channels/stt_service.py new file mode 100644 index 00000000..5b9a1aed --- /dev/null +++ b/backend/apps/channels/stt_service.py @@ -0,0 +1,142 @@ +"""Provider-abstracted Speech-to-Text service. + +Supports: Twilio built-in (via Gather), Deepgram Nova-3, OpenAI Whisper. +""" +import logging +from typing import Optional + +import httpx + +from backend.apps.channels.models import STTConfig +from backend.apps.settings.settings import load_settings + +logger = logging.getLogger(__name__) + +SUPPORTED_AUDIO_FORMATS = { + "audio/ogg", "audio/mpeg", "audio/wav", "audio/mp4", + "audio/flac", "audio/webm", "audio/x-wav", +} +MAX_MEDIA_BYTES = 20 * 1024 * 1024 # 20 MB + + +async def transcribe( + audio_bytes: bytes, + config: STTConfig, + content_type: str = "audio/wav", +) -> Optional[str]: + """Transcribe audio bytes to text using the configured provider chain.""" + if len(audio_bytes) > MAX_MEDIA_BYTES: + logger.warning("Audio exceeds %d bytes limit", MAX_MEDIA_BYTES) + return None + if len(audio_bytes) < 1024: + logger.debug("Audio too short, skipping") + return None + + providers = config.fallback_chain or [config.provider] + + for provider in providers: + try: + if provider == "twilio_builtin": + # Twilio STT is handled inline by — no bytes to process + continue + elif provider == "deepgram": + result = await _deepgram_transcribe(audio_bytes, config, content_type) + elif provider == "openai_whisper": + result = await _openai_transcribe(audio_bytes, config, content_type) + else: + logger.warning("Unknown STT provider: %s", provider) + continue + + if result: + return result + except Exception: + logger.exception("STT provider %s failed, trying next", provider) + + return None + + +async def transcribe_from_url( + url: str, config: STTConfig, content_type: str = "audio/ogg" +) -> Optional[str]: + """Download audio from URL and transcribe.""" + try: + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.get(url, follow_redirects=True) + resp.raise_for_status() + return await transcribe(resp.content, config, content_type) + except Exception: + logger.exception("Failed to download audio from %s", url) + return None + + +async def _deepgram_transcribe( + audio_bytes: bytes, config: STTConfig, content_type: str +) -> Optional[str]: + settings = load_settings() + api_key = settings.deepgram_api_key if hasattr(settings, "deepgram_api_key") else None + if not api_key: + logger.warning("Deepgram API key not configured") + return None + + try: + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.post( + "https://api.deepgram.com/v1/listen", + headers={ + "Authorization": f"Token {api_key}", + "Content-Type": content_type, + }, + params={ + "model": config.deepgram_model, + "language": config.language, + "smart_format": "true", + "punctuate": "true", + }, + content=audio_bytes, + ) + resp.raise_for_status() + data = resp.json() + return ( + data.get("results", {}) + .get("channels", [{}])[0] + .get("alternatives", [{}])[0] + .get("transcript", "") + ) + except Exception: + logger.exception("Deepgram transcription failed") + return None + + +async def _openai_transcribe( + audio_bytes: bytes, config: STTConfig, content_type: str +) -> Optional[str]: + settings = load_settings() + api_key = settings.openai_api_key if hasattr(settings, "openai_api_key") else None + if not api_key: + logger.warning("OpenAI API key not configured") + return None + + ext_map = { + "audio/ogg": "ogg", + "audio/mpeg": "mp3", + "audio/wav": "wav", + "audio/x-wav": "wav", + "audio/mp4": "m4a", + "audio/flac": "flac", + "audio/webm": "webm", + } + ext = ext_map.get(content_type, "wav") + + try: + async with httpx.AsyncClient(timeout=60) as client: + resp = await client.post( + "https://api.openai.com/v1/audio/transcriptions", + headers={"Authorization": f"Bearer {api_key}"}, + files={"file": (f"audio.{ext}", audio_bytes, content_type)}, + data={"model": "whisper-1", "language": config.language[:2]}, + ) + resp.raise_for_status() + return resp.json().get("text", "") + except Exception: + logger.exception("OpenAI Whisper transcription failed") + return None diff --git a/backend/apps/channels/talk_mode.py b/backend/apps/channels/talk_mode.py new file mode 100644 index 00000000..c4a0e88f --- /dev/null +++ b/backend/apps/channels/talk_mode.py @@ -0,0 +1,163 @@ +"""Browser-based Talk Mode — continuous voice conversation via WebSocket. + +Pipeline: Mic → WebSocket → STT → Agent → TTS → WebSocket → Speaker + +This runs as a separate WebSocket endpoint /ws/talk/{session_id} that +streams audio bidirectionally between the browser and the STT/TTS services. +""" +import asyncio +import json +import logging +from typing import Optional + +from fastapi import WebSocket, WebSocketDisconnect + +from backend.apps.channels import stt_service, tts_service +from backend.apps.channels.models import STTConfig, TTSConfig +from backend.apps.agents.agent_manager import agent_manager +from backend.apps.agents.ws_manager import ws_manager +from backend.apps.settings.settings import load_settings + +logger = logging.getLogger(__name__) + +# Default configs for talk mode +DEFAULT_STT = STTConfig( + provider="openai_whisper", + fallback_chain=["openai_whisper", "deepgram"], +) +DEFAULT_TTS = TTSConfig( + provider="elevenlabs", + skip_short_text=False, +) + + +async def handle_talk_session(websocket: WebSocket, session_id: str): + """Handle a talk-mode WebSocket connection. + + Protocol: + - Client sends: {"type": "audio", "data": "", "format": "webm"} + - Client sends: {"type": "config", "stt": {...}, "tts": {...}} + - Client sends: {"type": "end_utterance"} when silence detected + - Server sends: {"type": "transcript", "text": "..."} + - Server sends: {"type": "audio", "data": "", "format": "mp3"} + - Server sends: {"type": "agent_response", "text": "..."} + - Server sends: {"type": "status", "status": "listening|processing|speaking"} + """ + await websocket.accept() + + stt_config = DEFAULT_STT + tts_config = DEFAULT_TTS + audio_buffer = bytearray() + + try: + while True: + data = await websocket.receive_text() + msg = json.loads(data) + msg_type = msg.get("type", "") + + if msg_type == "config": + if msg.get("stt"): + stt_config = STTConfig(**msg["stt"]) + if msg.get("tts"): + tts_config = TTSConfig(**msg["tts"]) + await websocket.send_text(json.dumps({"type": "status", "status": "listening"})) + + elif msg_type == "audio": + import base64 + chunk = base64.b64decode(msg.get("data", "")) + audio_buffer.extend(chunk) + + elif msg_type == "end_utterance": + if not audio_buffer: + continue + + await websocket.send_text(json.dumps({"type": "status", "status": "processing"})) + + audio_bytes = bytes(audio_buffer) + audio_buffer.clear() + + audio_format = msg.get("format", "webm") + content_type = f"audio/{audio_format}" + + # STT + transcript = await stt_service.transcribe( + audio_bytes, stt_config, content_type + ) + + if not transcript: + await websocket.send_text(json.dumps({"type": "status", "status": "listening"})) + continue + + await websocket.send_text(json.dumps({ + "type": "transcript", "text": transcript, + })) + + # Route to agent + agent_response = await _get_agent_response(session_id, transcript) + + if agent_response: + await websocket.send_text(json.dumps({ + "type": "agent_response", "text": agent_response, + })) + + # TTS + await websocket.send_text(json.dumps({"type": "status", "status": "speaking"})) + + audio = await tts_service.synthesize(agent_response, tts_config) + if audio: + import base64 as b64 + await websocket.send_text(json.dumps({ + "type": "audio", + "data": b64.b64encode(audio).decode(), + "format": "mp3", + })) + + await websocket.send_text(json.dumps({"type": "status", "status": "listening"})) + + elif msg_type == "stop": + break + + except WebSocketDisconnect: + logger.info("Talk mode disconnected for session %s", session_id) + except Exception: + logger.exception("Talk mode error for session %s", session_id) + + +async def _get_agent_response(session_id: str, text: str) -> Optional[str]: + """Send text to agent and wait for response.""" + session = agent_manager.get_session(session_id) + if not session: + return None + + response_future: asyncio.Future[str] = asyncio.get_event_loop().create_future() + + original_send = ws_manager.send_to_session + + async def _hooked_send(sid: str, event: str, data: dict): + await original_send(sid, event, data) + if sid == session_id and not response_future.done(): + if event == "agent:message": + msg = data.get("message", {}) + if msg.get("role") == "assistant": + content = msg.get("content", "") + if isinstance(content, list): + parts = [ + b.get("text", "") + for b in content + if isinstance(b, dict) and b.get("type") == "text" + ] + content = "\n".join(parts) + if content: + response_future.set_result(content) + elif event == "agent:status": + if data.get("status") in ("completed", "error", "stopped"): + response_future.set_result("") + + ws_manager.send_to_session = _hooked_send + try: + await agent_manager.send_message(session_id, text) + return await asyncio.wait_for(response_future, timeout=120) or None + except asyncio.TimeoutError: + return None + finally: + ws_manager.send_to_session = original_send diff --git a/backend/apps/channels/tts_service.py b/backend/apps/channels/tts_service.py new file mode 100644 index 00000000..c3731724 --- /dev/null +++ b/backend/apps/channels/tts_service.py @@ -0,0 +1,127 @@ +"""Provider-abstracted Text-to-Speech service. + +Supports: Twilio built-in Say, ElevenLabs, OpenAI TTS, Microsoft Edge TTS. +""" +import logging +from typing import Optional + +import httpx + +from backend.apps.channels.models import TTSConfig +from backend.apps.settings.settings import load_settings + +logger = logging.getLogger(__name__) + + +async def synthesize(text: str, config: TTSConfig) -> Optional[bytes]: + """Convert text to audio bytes. Returns None for twilio_say (handled in TwiML).""" + if should_skip(text, config): + return None + + if len(text) > config.max_tts_chars and config.summarize_long_replies: + text = text[: config.max_tts_chars] + + provider = config.provider + if provider == "twilio_say": + # Twilio renders speech inline via — no audio bytes needed + return None + elif provider == "elevenlabs": + return await _elevenlabs_synthesize(text, config) + elif provider == "openai_tts": + return await _openai_synthesize(text, config) + elif provider == "edge_tts": + return await _edge_synthesize(text, config) + + logger.warning("Unknown TTS provider: %s", provider) + return None + + +def should_skip(text: str, config: TTSConfig) -> bool: + if config.skip_short_text and len(text.strip()) < 20: + return True + return False + + +async def _elevenlabs_synthesize(text: str, config: TTSConfig) -> Optional[bytes]: + settings = load_settings() + api_key = settings.elevenlabs_api_key if hasattr(settings, "elevenlabs_api_key") else None + if not api_key: + logger.warning("ElevenLabs API key not configured, falling back to edge_tts") + return await _edge_synthesize(text, config) + + voice_id = config.elevenlabs_voice_id or "21m00Tcm4TlvDq8ikWAM" # Rachel default + model_id = config.elevenlabs_model_id + + try: + async with httpx.AsyncClient(timeout=30) as client: + resp = await client.post( + f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}", + headers={ + "xi-api-key": api_key, + "Content-Type": "application/json", + "Accept": "audio/mpeg", + }, + json={ + "text": text, + "model_id": model_id, + "voice_settings": { + "stability": 0.5, + "similarity_boost": 0.75, + }, + }, + ) + resp.raise_for_status() + return resp.content + except Exception: + logger.exception("ElevenLabs TTS failed, falling back to edge_tts") + return await _edge_synthesize(text, config) + + +async def _openai_synthesize(text: str, config: TTSConfig) -> Optional[bytes]: + settings = load_settings() + api_key = settings.openai_api_key if hasattr(settings, "openai_api_key") else None + if not api_key: + logger.warning("OpenAI API key not configured, falling back to edge_tts") + return await _edge_synthesize(text, config) + + try: + async with httpx.AsyncClient(timeout=30) as client: + resp = await client.post( + "https://api.openai.com/v1/audio/speech", + headers={ + "Authorization": f"Bearer {api_key}", + "Content-Type": "application/json", + }, + json={ + "model": "tts-1", + "input": text, + "voice": config.openai_voice, + "response_format": "mp3", + }, + ) + resp.raise_for_status() + return resp.content + except Exception: + logger.exception("OpenAI TTS failed, falling back to edge_tts") + return await _edge_synthesize(text, config) + + +async def _edge_synthesize(text: str, config: TTSConfig) -> Optional[bytes]: + """Free fallback TTS via Microsoft Edge neural voices. No API key needed.""" + try: + import edge_tts + import tempfile + import os + + communicate = edge_tts.Communicate(text, "en-US-JennyNeural") + with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f: + tmp_path = f.name + + await communicate.save(tmp_path) + with open(tmp_path, "rb") as f: + audio = f.read() + os.unlink(tmp_path) + return audio + except Exception: + logger.exception("Edge TTS failed") + return None diff --git a/backend/apps/channels/wake_word.py b/backend/apps/channels/wake_word.py new file mode 100644 index 00000000..a79f70cc --- /dev/null +++ b/backend/apps/channels/wake_word.py @@ -0,0 +1,68 @@ +"""Voice Wake Word Detection — scaffolded interface. + +Matches OpenClaw's current state: the interface is defined but full +implementation is deferred. Supports future integration with Vosk +(offline) or Porcupine wake word engines. + +Usage: + This module defines the configuration and interface. Actual wake word + detection runs on the client device (macOS/iOS/Android) and sends + a "wake" event to the gateway when triggered. +""" +import logging +from typing import Optional +from pydantic import BaseModel, Field + +logger = logging.getLogger(__name__) + + +class WakeWordConfig(BaseModel): + """Configuration for wake word detection.""" + enabled: bool = False + wake_words: list[str] = Field(default_factory=lambda: ["hey swarm", "open swarm"]) + sensitivity: float = 0.5 # 0.0 - 1.0 + engine: str = "vosk" # "vosk" | "porcupine" + + +class WakeWordManager: + """Manages wake word detection state. + + In the current scaffolded implementation, this stores configuration + and handles wake events from client devices. The actual audio + processing runs on the client side. + """ + + def __init__(self): + self.config = WakeWordConfig() + self._active_devices: dict[str, bool] = {} + + def update_config(self, **kwargs): + for k, v in kwargs.items(): + if hasattr(self.config, k): + setattr(self.config, k, v) + + def register_device(self, device_id: str): + self._active_devices[device_id] = True + logger.info("Wake word device registered: %s", device_id) + + def unregister_device(self, device_id: str): + self._active_devices.pop(device_id, None) + + def handle_wake_event(self, device_id: str, wake_word: str) -> bool: + """Called when a client device detects a wake word. + + Returns True if the wake event should trigger a talk session. + """ + if not self.config.enabled: + return False + if device_id not in self._active_devices: + return False + if wake_word.lower() not in [w.lower() for w in self.config.wake_words]: + return False + + logger.info("Wake word detected: '%s' from device %s", wake_word, device_id) + return True + + +# Singleton +wake_word_manager = WakeWordManager() diff --git a/backend/apps/channels/ws_events.py b/backend/apps/channels/ws_events.py new file mode 100644 index 00000000..75e0853b --- /dev/null +++ b/backend/apps/channels/ws_events.py @@ -0,0 +1,40 @@ +"""WebSocket event emitters for channel events. + +Uses the existing ws_manager.broadcast_global() — no new WebSocket +infrastructure needed. +""" +import logging +from backend.apps.agents.ws_manager import ws_manager + +logger = logging.getLogger(__name__) + + +async def emit_channel_message( + channel_id: str, conversation_id: str, message: dict +): + await ws_manager.broadcast_global("channel:message", { + "channel_id": channel_id, + "conversation_id": conversation_id, + "message": message, + }) + + +async def emit_channel_status( + channel_id: str, status: str, detail: str = "" +): + await ws_manager.broadcast_global("channel:status", { + "channel_id": channel_id, + "status": status, + "detail": detail, + }) + + +async def emit_call_event( + channel_id: str, call_sid: str, event: str, data: dict | None = None +): + await ws_manager.broadcast_global("channel:call_event", { + "channel_id": channel_id, + "call_sid": call_sid, + "event": event, + **(data or {}), + }) diff --git a/backend/apps/dashboards/dashboards.py b/backend/apps/dashboards/dashboards.py index 29d32af3..5a21d4a6 100644 --- a/backend/apps/dashboards/dashboards.py +++ b/backend/apps/dashboards/dashboards.py @@ -123,8 +123,10 @@ async def list_dashboards(): @dashboards.router.post("/create") async def create_dashboard(body: DashboardCreate): + from backend.apps.analytics.collector import record as _analytics dashboard = Dashboard(name=body.name) _save(dashboard) + _analytics("dashboard.created", {}, dashboard_id=dashboard.id) return dashboard.model_dump(mode="json") @@ -151,13 +153,10 @@ async def generate_name(dashboard_id: str): fallback = prompts[0][:40] try: - import anthropic from backend.apps.settings.settings import load_settings + from backend.apps.settings.credentials import get_anthropic_client global_settings = load_settings() - if not global_settings.anthropic_api_key: - raise ValueError("API key not configured") - - client = anthropic.AsyncAnthropic(api_key=global_settings.anthropic_api_key) + client = get_anthropic_client(global_settings) if len(prompts) == 1: system = ( @@ -173,7 +172,7 @@ async def generate_name(dashboard_id: str): user_content = "\n".join(f"- {p}" for p in prompts) resp = await client.messages.create( - model="claude-haiku-4-20250414", + model="claude-haiku-4-5-20251001", max_tokens=30, system=system, messages=[{"role": "user", "content": user_content}], diff --git a/backend/apps/dashboards/models.py b/backend/apps/dashboards/models.py index b6cd6bef..74fc4ee8 100644 --- a/backend/apps/dashboards/models.py +++ b/backend/apps/dashboards/models.py @@ -34,8 +34,8 @@ class BrowserCardPosition(BaseModel): activeTabId: str = "" x: float = 0 y: float = 0 - width: float = 640 - height: float = 480 + width: float = 1280 + height: float = 800 class DashboardLayout(BaseModel): diff --git a/backend/apps/modes/models.py b/backend/apps/modes/models.py index 03c201fb..0afc742c 100644 --- a/backend/apps/modes/models.py +++ b/backend/apps/modes/models.py @@ -76,43 +76,28 @@ BUILTIN_MODES: list[Mode] = [ ), Mode( id="view-builder", - name="View Builder", - description="Create and iterate on reusable View artifacts.", + name="App Builder", + description="Create and iterate on reusable App artifacts.", system_prompt=( - "You are helping the user build a reusable View — a self-contained " - "web app rendered in an iframe.\n\n" - "Your working directory is a dedicated workspace folder for this view. " - "You can create any file structure you need using the Write tool.\n\n" - "## Required files\n\n" - "1. **index.html** — The entry point. A complete HTML document. " - "React 18 is available via esm.sh CDN imports:\n" - ' \n' - " The structured input data is available at `window.OUTPUT_INPUT` (object) " - "and any server-side result at `window.OUTPUT_BACKEND_RESULT`.\n\n" - "2. **schema.json** — A JSON Schema object defining the structured input " - "the view accepts. Example:\n" - ' {"type":"object","properties":{"name":{"type":"string"}},"required":["name"]}\n\n' - "3. **meta.json** — Metadata for this view. Always write this file with " - "a short name and one-sentence description. Example:\n" - ' {"name":"Sales Dashboard","description":"Interactive dashboard showing sales metrics"}\n\n' - "## Optional files\n\n" - "- **backend.py** — Python code that receives `input_data` as " - "a global dict and must assign its result to a global `result` dict.\n" - "- **Any additional files** — You can create subdirectories and split code " - "across multiple files. For example:\n" - " - `components/Chart.js` — Reusable components\n" - " - `utils/helpers.js` — Utility functions\n" - " - `styles/main.css` — Stylesheets\n\n" - "Files are served from the workspace, so relative imports work naturally:\n" - ' ``\n' - ' ``\n' - " `import { helper } from './utils/helpers.js'` (in ES modules)\n\n" - "## Guidelines\n\n" - "Write files immediately when you have code ready. The user can see " - "a live preview that auto-refreshes from these files. Always write the " - "complete file content (do not use Edit for partial patches on first creation). " - "For complex views, split code into separate files to keep things organized." + "You are an App Builder — an AI assistant that creates self-contained " + "web apps rendered in an iframe preview.\n\n" + "Your working directory is a dedicated workspace folder pre-seeded with " + "template files. Read the existing files before making changes.\n\n" + "## Critical rules\n\n" + "- The entry point MUST be named `index.html`. Never rename it or create " + "a different HTML file as the main entry point.\n" + "- Write files immediately when you have code ready — the user sees a " + "live preview that auto-refreshes from these files.\n" + "- Always write the complete file content on first creation (do not use " + "Edit for partial patches on new files).\n" + "- For complex apps, split code into separate files (JS, CSS, etc.) " + "and reference them from index.html with relative paths.\n" + "- Always update meta.json with a short name and one-sentence description.\n" + "- Build beautiful, polished UIs with modern design — dark themes, smooth " + "transitions, proper spacing, and responsive layouts.\n\n" + "Read the SKILL.md reference in your workspace for the full technical " + "specification of the App platform (available globals, file conventions, " + "schema format, backend.py usage, and examples)." ), tools=None, default_next_mode=None, diff --git a/backend/apps/modes/modes.py b/backend/apps/modes/modes.py index 6d393143..37aabd90 100644 --- a/backend/apps/modes/modes.py +++ b/backend/apps/modes/modes.py @@ -59,7 +59,8 @@ def load_mode(mode_id: str) -> Mode | None: @modes.router.get("/list") async def list_modes(): - return {"modes": [m.model_dump() for m in _load_all()]} + builtin_defaults = {m.id: m.model_dump() for m in BUILTIN_MODES} + return {"modes": [m.model_dump() for m in _load_all()], "builtin_defaults": builtin_defaults} @modes.router.get("/{mode_id}") @@ -93,6 +94,16 @@ async def update_mode(mode_id: str, body: ModeUpdate): return {"ok": True, "mode": mode.model_dump()} +@modes.router.post("/{mode_id}/reset") +async def reset_mode(mode_id: str): + """Reset a built-in mode to its hardcoded defaults.""" + builtin = next((m for m in BUILTIN_MODES if m.id == mode_id), None) + if not builtin: + raise HTTPException(status_code=400, detail="Only built-in modes can be reset") + _save(builtin) + return {"ok": True, "mode": builtin.model_dump()} + + @modes.router.delete("/{mode_id}") async def delete_mode(mode_id: str): mode = _load(mode_id) diff --git a/backend/apps/nine_router.py b/backend/apps/nine_router.py new file mode 100644 index 00000000..c53bf52a --- /dev/null +++ b/backend/apps/nine_router.py @@ -0,0 +1,199 @@ +"""Auto-start and manage 9Router subprocess. + +9Router is a free AI subscription proxy that lets users connect their +Claude/ChatGPT/Gemini subscriptions to OpenSwarm without API keys. + +It runs silently in the background on port 20128 and exposes an +OpenAI-compatible API at localhost:20128/v1. +""" + +import asyncio +import logging +import os +import shutil +import subprocess + +import httpx + +logger = logging.getLogger(__name__) + +NINE_ROUTER_PORT = 20128 +NINE_ROUTER_URL = f"http://localhost:{NINE_ROUTER_PORT}" +NINE_ROUTER_API = f"{NINE_ROUTER_URL}/api" +NINE_ROUTER_V1 = f"{NINE_ROUTER_URL}/v1" + +_process: subprocess.Popen | None = None + + +def is_running() -> bool: + """Check if 9Router is running.""" + try: + r = httpx.get(f"{NINE_ROUTER_V1}/models", timeout=2.0) + return r.status_code == 200 + except Exception: + return False + + +async def ensure_running(): + """Start 9Router if not already running.""" + global _process + if is_running(): + logger.info("9Router already running on port %d", NINE_ROUTER_PORT) + return + + npx = shutil.which("npx") + if not npx: + logger.warning("npx not found — cannot auto-start 9Router. Install Node.js or run 9Router manually.") + return + + logger.info("Starting 9Router on port %d...", NINE_ROUTER_PORT) + try: + env = {**os.environ, "PORT": str(NINE_ROUTER_PORT)} + _process = subprocess.Popen( + [npx, "9router"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + env=env, + ) + + # Wait up to 15 seconds for it to start + for _ in range(30): + await asyncio.sleep(0.5) + if is_running(): + logger.info("9Router started successfully") + return + + logger.warning("9Router did not start within 15 seconds") + except Exception as e: + logger.warning(f"Failed to start 9Router: {e}") + + +def stop(): + """Stop the 9Router subprocess.""" + global _process + if _process: + try: + _process.terminate() + _process.wait(timeout=5) + except Exception: + try: + _process.kill() + except Exception: + pass + _process = None + logger.info("9Router stopped") + + +# --------------------------------------------------------------------------- +# API proxy helpers — call 9Router's API from OpenSwarm +# --------------------------------------------------------------------------- + +async def get_providers() -> list[dict]: + """Get all providers and their connection status from 9Router.""" + try: + async with httpx.AsyncClient(timeout=5.0) as client: + r = await client.get(f"{NINE_ROUTER_API}/providers") + if r.status_code == 200: + return r.json() + except Exception as e: + logger.debug(f"9Router providers fetch failed: {e}") + return [] + + +async def start_oauth(provider: str) -> dict: + """Start OAuth flow for a provider. + + For device_code providers (github, qwen, kiro): returns {user_code, verification_uri, device_code} + For authorization_code providers (claude, codex, gemini-cli): returns {authUrl, codeVerifier, state} + """ + async with httpx.AsyncClient(timeout=15.0) as client: + # Try device-code flow first + try: + r = await client.get(f"{NINE_ROUTER_API}/oauth/{provider}/device-code") + if r.status_code == 200: + data = r.json() + return { + "flow": "device_code", + "user_code": data.get("user_code", ""), + "verification_uri": data.get("verification_uri", data.get("verification_uri_complete", "")), + "device_code": data.get("device_code", ""), + "code_verifier": data.get("codeVerifier", ""), + "extra_data": {k: v for k, v in data.items() if k.startswith("_")}, + } + except Exception: + pass + + # Authorization code flow — redirect to 9Router's own callback page + # (Anthropic only accepts redirect URIs registered with 9Router's client ID) + callback_url = f"http://localhost:{NINE_ROUTER_PORT}/callback" + r = await client.get( + f"{NINE_ROUTER_API}/oauth/{provider}/authorize", + params={"redirect_uri": callback_url}, + ) + r.raise_for_status() + data = r.json() + return { + "flow": "authorization_code", + "auth_url": data.get("authUrl", ""), + "code_verifier": data.get("codeVerifier", ""), + "state": data.get("state", ""), + "redirect_uri": callback_url, + } + + +async def poll_oauth(provider: str, device_code: str, code_verifier: str | None = None, extra_data: dict | None = None) -> dict: + """Poll for OAuth completion. + + Returns: {success: true, connection: {...}} or {success: false, pending: true} + """ + body: dict = {"deviceCode": device_code} + if code_verifier: + body["codeVerifier"] = code_verifier + if extra_data: + body["extraData"] = extra_data + + async with httpx.AsyncClient(timeout=15.0) as client: + r = await client.post( + f"{NINE_ROUTER_API}/oauth/{provider}/poll", + json=body, + ) + r.raise_for_status() + return r.json() + + +async def exchange_oauth(provider: str, code: str, redirect_uri: str, code_verifier: str, state: str = "") -> dict: + """Exchange OAuth code for tokens via 9Router.""" + async with httpx.AsyncClient(timeout=15.0) as client: + r = await client.post( + f"{NINE_ROUTER_API}/oauth/{provider}/exchange", + json={ + "code": code, + "redirectUri": redirect_uri, + "codeVerifier": code_verifier, + "state": state, + }, + ) + r.raise_for_status() + return r.json() + + +async def get_models() -> list[dict]: + """Get all available models from 9Router.""" + try: + async with httpx.AsyncClient(timeout=5.0) as client: + r = await client.get(f"{NINE_ROUTER_V1}/models") + if r.status_code == 200: + data = r.json() + models = data.get("data", []) + return [ + { + "value": m.get("id", ""), + "label": m.get("id", "").split("/")[-1] if "/" in m.get("id", "") else m.get("id", ""), + "context_window": 200_000, + "provider": m.get("owned_by", "subscription"), + } + for m in models + ] + except Exception as e: + logger.debug(f"9Router models fetch failed: {e}") + return [] diff --git a/backend/apps/outputs/outputs.py b/backend/apps/outputs/outputs.py index 206e29e8..668e8b55 100644 --- a/backend/apps/outputs/outputs.py +++ b/backend/apps/outputs/outputs.py @@ -15,6 +15,7 @@ from backend.apps.outputs.models import ( WorkspaceSeedRequest, ) from backend.apps.outputs.executor import execute_backend_code +from backend.apps.outputs.view_builder_templates import VIEW_BUILDER_SKILL, VIEW_TEMPLATE_FILES from backend.apps.settings.settings import load_settings logger = logging.getLogger(__name__) @@ -22,7 +23,7 @@ logger = logging.getLogger(__name__) MODEL_MAP = { "sonnet": "claude-sonnet-4-20250514", "opus": "claude-opus-4-20250514", - "haiku": "claude-haiku-4-20250414", + "haiku": "claude-haiku-4-5-20251001", } @@ -31,13 +32,11 @@ def _resolve_model(short_name: str) -> str: def _get_anthropic_client(): - """Create an AsyncAnthropic client using the API key from app settings.""" - import anthropic + """Create an AsyncAnthropic client using credentials from app settings.""" + from backend.apps.settings.credentials import get_anthropic_client settings = load_settings() - if not settings.anthropic_api_key: - raise ValueError("Anthropic API key not configured. Set it in Settings.") - return anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key) + return get_anthropic_client(settings) def _validate_against_schema(data: dict, schema: dict) -> str | None: @@ -235,6 +234,14 @@ async def seed_workspace(body: WorkspaceSeedRequest): os.makedirs(os.path.dirname(full_path), exist_ok=True) with open(full_path, "w") as f: f.write(content) + else: + for rel_path, content in VIEW_TEMPLATE_FILES.items(): + full_path = os.path.join(folder, rel_path) + with open(full_path, "w") as f: + f.write(content) + + with open(os.path.join(folder, "SKILL.md"), "w") as f: + f.write(VIEW_BUILDER_SKILL) if body.meta: with open(os.path.join(folder, "meta.json"), "w") as f: diff --git a/backend/apps/outputs/view_builder_skill.md b/backend/apps/outputs/view_builder_skill.md new file mode 100644 index 00000000..1e78377d --- /dev/null +++ b/backend/apps/outputs/view_builder_skill.md @@ -0,0 +1,223 @@ +# App Builder — Platform Reference + +You are building an **App**: a self-contained web app served in an iframe. +The workspace you're working in is the source of truth — every file you write +here is served directly to the live preview. + +--- + +## File conventions + +| File | Required | Purpose | +|------|----------|---------| +| `index.html` | **Yes** | Entry point. Must be a complete HTML document. This is the ONLY file the preview iframe loads — never rename it. | +| `meta.json` | **Yes** | `{"name":"…","description":"…"}` — displayed in the UI header. Always write this. | +| `schema.json` | Recommended | JSON Schema defining the input form (the "Test Input" tab). | +| `backend.py` | Optional | Server-side Python executed before rendering. | +| Everything else | Optional | JS, CSS, images, subdirectories — referenced from `index.html` via relative paths. | + +### ⚠️ Do NOT + +- Name the main HTML file anything other than `index.html` — the platform + will not find it and the preview will be blank. +- Use `document.write()` — it breaks the injected data globals. +- Assume any external server or API is available unless the user provides one. + +--- + +## Injected globals + +Before `index.html` loads, the platform injects two globals: + +```javascript +window.OUTPUT_INPUT // Object — structured input from the schema form +window.OUTPUT_BACKEND_RESULT // Object | null — result from backend.py execution +``` + +These are available immediately in any ` +``` + +ES module imports between JS files: + +```javascript +// components/Chart.js +import { formatNumber } from '../utils/helpers.js'; +``` + +--- + +## Using React + +React 18 is available via esm.sh CDN — no build step needed: + +```html + +
+ +``` + +Other CDN libraries work too — use `https://esm.sh/` or `https://cdn.jsdelivr.net/npm/` for any npm package. + +--- + +## Design guidelines + +- **Dark theme by default** — use dark backgrounds (#0f1117, #1a1d27) with + light text (#e2e8f0) unless the user requests otherwise. +- **Modern aesthetics** — rounded corners (8-12px), subtle borders, box shadows, + smooth transitions (0.15-0.3s ease). +- **Responsive** — use flexbox/grid, test at different sizes. +- **Typography** — system font stack for UI, monospace for code/data. +- **Color accents** — use a single accent color with variations for hover/active states. +- **Spacing** — consistent padding (12-20px), adequate whitespace between sections. +- **Interactivity** — hover effects, focus states, loading indicators where appropriate. + +--- + +## Complete minimal example + +```html + + + + + + My App + + + +
+

Loading…

+

+
+ + + +``` diff --git a/backend/apps/outputs/view_builder_templates.py b/backend/apps/outputs/view_builder_templates.py new file mode 100644 index 00000000..0aada357 --- /dev/null +++ b/backend/apps/outputs/view_builder_templates.py @@ -0,0 +1,74 @@ +"""Default template files seeded into new App Builder workspaces.""" + +import os + +_SKILL_PATH = os.path.join(os.path.dirname(__file__), "view_builder_skill.md") + +with open(_SKILL_PATH) as _f: + VIEW_BUILDER_SKILL = _f.read() + +VIEW_TEMPLATE_INDEX = """\ + + + + + + App + + + +
+

Ready

+

Describe what you want to build and the agent will update this app.

+
+ + + +""" + +VIEW_TEMPLATE_SCHEMA = """\ +{ + "type": "object", + "properties": {}, + "required": [] +} +""" + +VIEW_TEMPLATE_META = """\ +{ + "name": "", + "description": "" +} +""" + +VIEW_TEMPLATE_FILES = { + "index.html": VIEW_TEMPLATE_INDEX, + "schema.json": VIEW_TEMPLATE_SCHEMA, + "meta.json": VIEW_TEMPLATE_META, +} diff --git a/backend/apps/settings/credentials.py b/backend/apps/settings/credentials.py new file mode 100644 index 00000000..9d3c5e14 --- /dev/null +++ b/backend/apps/settings/credentials.py @@ -0,0 +1,141 @@ +"""Centralized credential resolution for LLM API calls. + +Supports multiple providers: Anthropic (native), OpenAI, Gemini, +OpenRouter, and user-configured custom providers. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import anthropic + from backend.apps.settings.models import AppSettings + +OPENSWARM_DEFAULT_PROXY_URL = "https://api.openswarm.ai" + + +def _check_9router() -> bool: + """Check if 9Router is running locally.""" + try: + import httpx + r = httpx.get("http://localhost:20128/v1/models", timeout=2.0) + return r.status_code == 200 + except Exception: + return False + + +def validate_credentials(settings: AppSettings, provider: str = "anthropic") -> None: + """Raise ValueError if credentials are missing for the given provider. + + Allows through if 9Router is running as a fallback. + Handles both display names ('Anthropic') and lowercase ('anthropic'). + """ + p = provider.lower().strip() + + # 9Router or GitHub Copilot providers don't need traditional credentials + if p in ("9router", "github copilot", "copilot"): + return + + # If 9Router is running, all providers are accessible + if _check_9router(): + return + + if p == "anthropic": + if getattr(settings, "connection_mode", "own_key") == "managed": + if not getattr(settings, "openswarm_auth_token", None): + raise ValueError("Open Swarm account not connected. Sign in via Settings → API.") + return + if settings.anthropic_api_key: + return + raise ValueError("Anthropic API key not configured. Set it in Settings, or connect a subscription.") + elif p == "openai": + if settings.openai_api_key: + return + raise ValueError("OpenAI API key not configured. Set it in Settings, or connect a subscription.") + elif p in ("gemini", "google"): + if getattr(settings, "google_api_key", None): + return + raise ValueError("Google API key not configured. Set it in Settings, or connect a subscription.") + elif p == "openrouter": + if getattr(settings, "openrouter_api_key", None): + return + raise ValueError("OpenRouter API key not configured. Set it in Settings.") + elif p in ("xai", "meta", "deepseek", "mistral", "qwen", "cohere"): + # These route through OpenRouter — need either OpenRouter key or 9Router + if getattr(settings, "openrouter_api_key", None): + return + raise ValueError(f"{provider} requires an OpenRouter API key, or connect a subscription via 9Router.") + else: + # Custom provider — check if it exists in custom_providers + for cp in getattr(settings, "custom_providers", []): + if cp.name.lower() == p: + return + # Unknown provider — allow through (create_provider will handle the error) + return + + +def get_provider_credentials(settings: AppSettings, provider: str) -> dict[str, str]: + """Return credential dict for a specific provider.""" + validate_credentials(settings, provider) + + if provider == "anthropic": + if getattr(settings, "connection_mode", "own_key") == "managed": + return { + "auth_token": getattr(settings, "openswarm_auth_token", "") or "", + "base_url": getattr(settings, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL, + } + return {"api_key": settings.anthropic_api_key or ""} + + if provider == "openai": + return {"api_key": settings.openai_api_key or ""} + + if provider == "gemini": + return {"api_key": getattr(settings, "google_api_key", "") or ""} + + if provider == "openrouter": + return {"api_key": getattr(settings, "openrouter_api_key", "") or ""} + + # Custom provider + for cp in getattr(settings, "custom_providers", []): + if cp.name == provider: + return {"api_key": cp.api_key, "base_url": cp.base_url} + + raise ValueError(f"No credentials for provider: {provider}") + + +# --------------------------------------------------------------------------- +# Legacy helpers (kept for backward compat during migration) +# --------------------------------------------------------------------------- + +def get_agent_sdk_env(settings: AppSettings) -> dict[str, str]: + """Return the env dict for ClaudeAgentOptions based on connection mode. + + DEPRECATED: Use create_provider() from providers.registry instead. + """ + validate_credentials(settings, "anthropic") + + if getattr(settings, "connection_mode", "own_key") == "managed": + proxy_url = getattr(settings, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL + return { + "ANTHROPIC_AUTH_TOKEN": getattr(settings, "openswarm_auth_token", ""), + "ANTHROPIC_BASE_URL": proxy_url, + } + + return {"ANTHROPIC_API_KEY": settings.anthropic_api_key} + + +def get_anthropic_client(settings: AppSettings) -> anthropic.AsyncAnthropic: + """Return a configured AsyncAnthropic client based on connection mode.""" + import anthropic + + validate_credentials(settings, "anthropic") + + if getattr(settings, "connection_mode", "own_key") == "managed": + proxy_url = getattr(settings, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL + return anthropic.AsyncAnthropic( + auth_token=getattr(settings, "openswarm_auth_token", None), + base_url=proxy_url, + ) + + return anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key) diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index 343b245e..d69c6e52 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -1,12 +1,22 @@ -from pydantic import BaseModel -from typing import Optional +from pydantic import BaseModel, Field +from typing import Optional, Any DEFAULT_SYSTEM_PROMPT = ( - '"Ask the user as many follow ups as needed in order to eliminate any possible ambiguity. ' - "When asking the user questions, use the AskUserQuestion tool.\n\n" - "You are an unstopable Agent that does whatever is needed to achieve the task. " - "You are particularly gifted at coding, so when needed, transpose ordinary tasks into coding tasks.\n\n" - 'If a Browser is selected, prioritize this over other tools when it makes sense (so the user also has observability)."' + "You are a personal AI assistant running inside OpenSwarm.\n\n" + "## Tool Priority\n" + "When a dedicated MCP tool exists for a task, use it directly — do not use the browser for things MCP tools can handle.\n" + "Priority order:\n" + "1. MCP tools first (Reddit, Google Workspace, Twitter, etc.) — fastest and most reliable\n" + "2. WebSearch / WebFetch — for general web lookups without a dedicated MCP\n" + "3. BrowserAgent — only when you need to visually interact with a website, fill forms, or do something no other tool can handle\n\n" + "## Tool Call Style\n" + "Default: do not narrate routine tool calls — just call the tool.\n" + "Narrate only when it helps: multi-step work, complex problems, or when the user explicitly asks.\n" + "Keep narration brief. Use plain language.\n\n" + "## Interaction Style\n" + "Be direct and action-oriented. Do not ask clarifying questions unless genuinely ambiguous — " + "make reasonable assumptions and act. If you need to ask, use the AskUserQuestion tool.\n" + "Do not over-explain what you are about to do. Just do it and show the results.\n" ) @@ -21,3 +31,37 @@ class AppSettings(BaseModel): new_agent_shortcut: str = "Meta+l" anthropic_api_key: Optional[str] = None browser_homepage: str = "https://www.google.com" + # Telephony / Channel credentials + twilio_account_sid: Optional[str] = None + twilio_auth_token: Optional[str] = None + telnyx_api_key: Optional[str] = None + elevenlabs_api_key: Optional[str] = None + deepgram_api_key: Optional[str] = None + openai_api_key: Optional[str] = None + google_api_key: Optional[str] = None + openrouter_api_key: Optional[str] = None + custom_providers: list["CustomProvider"] = Field(default_factory=list) + webhook_base_url: Optional[str] = None + # Dashboard / UI preferences + auto_select_mode_on_new_agent: bool = False + expand_new_chats_in_dashboard: bool = False + auto_reveal_sub_agents: bool = True + dev_mode: bool = False + # Subscription tokens (from CLI tools — alternative to API keys) + claude_subscription_token: Optional[str] = None + openai_subscription_token: Optional[str] = None + gemini_subscription_token: Optional[str] = None + # GitHub Copilot + copilot_github_token: Optional[str] = None + copilot_token: Optional[str] = None + copilot_token_expires: Optional[float] = None + # Analytics: opted in by default, user can toggle off + analytics_opt_in: bool = True + installation_id: Optional[str] = None + + +class CustomProvider(BaseModel): + name: str + base_url: str + api_key: str = "" + models: list[dict[str, Any]] = Field(default_factory=list) diff --git a/backend/apps/settings/settings.py b/backend/apps/settings/settings.py index e590d9a4..d0c8ca41 100644 --- a/backend/apps/settings/settings.py +++ b/backend/apps/settings/settings.py @@ -38,6 +38,13 @@ def load_settings() -> AppSettings: return AppSettings() +def _save_settings(settings: AppSettings): + """Persist settings to JSON file.""" + os.makedirs(DATA_DIR, exist_ok=True) + with open(SETTINGS_FILE, "w") as f: + json.dump(settings.model_dump(), f, indent=2) + + @settings.router.get("") async def get_settings(): return load_settings().model_dump() diff --git a/backend/apps/tools_lib/models.py b/backend/apps/tools_lib/models.py index 3c0cc24e..857ccc3a 100644 --- a/backend/apps/tools_lib/models.py +++ b/backend/apps/tools_lib/models.py @@ -5,6 +5,7 @@ from uuid import uuid4 class BuiltinTool(BaseModel): name: str + display_name: Optional[str] = None description: str category: str = "filesystem" deferred: bool = False @@ -33,6 +34,23 @@ BUILTIN_TOOLS: list[BuiltinTool] = [ BuiltinTool(name="CronList", description="List all scheduled tasks", category="scheduling", deferred=True), BuiltinTool(name="CronDelete", description="Delete a scheduled task", category="scheduling", deferred=True), BuiltinTool(name="RenderOutput", description="Render a reusable View artifact with structured input data", category="views", deferred=True), + # Agent tools + BuiltinTool(name="Agent", display_name="CreateAgent", description="Spawn a sub-agent to handle a complex subtask", category="agents"), + BuiltinTool(name="InvokeAgent", description="Invoke a copy of an existing agent with a new message, preserving full conversation context", category="agents"), + # Browser delegation tools (Layer 1 — what the main agent calls) + BuiltinTool(name="CreateBrowserAgent", description="Create a new browser and run a task on it", category="browser_delegation"), + BuiltinTool(name="BrowserAgent", description="Delegate a browser task to an existing browser agent", category="browser_delegation"), + BuiltinTool(name="BrowserAgents", description="Run multiple browser tasks in parallel on existing browsers", category="browser_delegation"), + # Browser action tools (Layer 2 — what the sub-agent executes) + BuiltinTool(name="BrowserScreenshot", description="Capture a screenshot of the browser page", category="browser_action"), + BuiltinTool(name="BrowserNavigate", description="Navigate the browser to a URL", category="browser_action"), + BuiltinTool(name="BrowserClick", description="Click an element by CSS selector", category="browser_action"), + BuiltinTool(name="BrowserType", description="Type text into an input element", category="browser_action"), + BuiltinTool(name="BrowserEvaluate", description="Execute JavaScript in the browser", category="browser_action"), + BuiltinTool(name="BrowserGetText", description="Get visible text content of the page", category="browser_action"), + BuiltinTool(name="BrowserGetElements", description="List interactive elements with CSS selectors", category="browser_action"), + BuiltinTool(name="BrowserScroll", description="Scroll the page up or down", category="browser_action"), + BuiltinTool(name="BrowserWait", description="Wait for page loads or animations", category="browser_action"), ] diff --git a/backend/apps/tools_lib/tools_lib.py b/backend/apps/tools_lib/tools_lib.py index f19add53..0abb4bf7 100644 --- a/backend/apps/tools_lib/tools_lib.py +++ b/backend/apps/tools_lib/tools_lib.py @@ -18,9 +18,11 @@ from backend.apps.tools_lib.models import ToolDefinition, ToolCreate, ToolUpdate logger = logging.getLogger(__name__) -from backend.config.paths import BACKEND_DIR, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH +from backend.config.paths import BACKEND_DIR, DATA_ROOT, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH load_dotenv(os.path.join(BACKEND_DIR, ".env")) +if os.environ.get("OPENSWARM_PACKAGED") == "1": + load_dotenv(os.path.join(os.path.dirname(DATA_ROOT), ".env"), override=True) @asynccontextmanager @@ -380,6 +382,7 @@ def derive_mcp_config(tool: ToolDefinition) -> Optional[dict]: config["command"] = resolved env = config.setdefault("env", {}) env.setdefault("PATH", _augmented_path()) + env.setdefault("PYTHONPATH", "") return config @@ -510,7 +513,7 @@ async def _discover_mcp_tools_http(url: str, headers: dict | None = None) -> lis raise HTTPException(status_code=502, detail="Empty response from MCP server") tools_list = data.get("result", {}).get("tools", []) - return [{"name": t.get("name", ""), "description": t.get("description", "")} for t in tools_list] + return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list] async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list[dict]: @@ -533,7 +536,7 @@ async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list ) as session: await session.initialize() result = await session.list_tools() - return [{"name": t.name, "description": t.description or ""} for t in result.tools] + return [{"name": t.name, "description": t.description or "", "inputSchema": t.inputSchema if t.inputSchema else None} for t in result.tools] except BaseExceptionGroup as eg: first = eg.exceptions[0] if eg.exceptions else eg raise HTTPException(status_code=502, detail=f"SSE discovery failed: {first}") from first @@ -546,6 +549,7 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, raise HTTPException(status_code=400, detail=f"Command '{command}' not found on PATH or common install locations") proc_env = {**os.environ, **(env or {}), "PATH": _augmented_path()} + proc_env.pop("PYTHONPATH", None) proc = await asyncio.create_subprocess_exec( cmd_path, *(args or []), @@ -601,7 +605,7 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, data = await _recv() tools_list = data.get("result", {}).get("tools", []) - return [{"name": t.get("name", ""), "description": t.get("description", "")} for t in tools_list] + return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list] except HTTPException: raise @@ -616,12 +620,34 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, proc.terminate() await asyncio.wait_for(proc.wait(), timeout=5.0) except Exception: - proc.kill() + try: + proc.kill() + except Exception: + pass @tools_lib.router.post("/{tool_id}/discover") async def discover_tools(tool_id: str): tool = _load(tool_id) + + if tool.auth_type == "oauth2" and tool.auth_status == "connected": + refreshed = await refresh_google_token(tool) + if not refreshed and tool.oauth_tokens.get("access_token"): + expiry = tool.oauth_tokens.get("token_expiry", 0) + if time.time() >= expiry - 60: + client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "") + if not client_id: + raise HTTPException( + status_code=400, + detail="OAuth token expired and GOOGLE_OAUTH_CLIENT_ID is not set. " + "In the packaged app, create ~/.openswarm.env or " + "~/Library/Application Support/OpenSwarm/.env with your Google OAuth credentials.", + ) + raise HTTPException( + status_code=502, + detail="OAuth token expired and refresh failed. Try reconnecting Google.", + ) + config = derive_mcp_config(tool) if not config: raise HTTPException(status_code=400, detail="Cannot derive MCP config for tool") @@ -655,8 +681,11 @@ async def discover_tools(tool_id: str): except HTTPException: raise except Exception as e: - logger.warning(f"MCP tool discovery failed for {tool.name}: {e}") - raise HTTPException(status_code=502, detail=f"Discovery failed: {e}") + msg = str(e).strip() + if not msg: + msg = type(e).__name__ + logger.warning(f"MCP tool discovery failed for {tool.name}: {msg}", exc_info=True) + raise HTTPException(status_code=502, detail=f"Discovery failed: {msg}") services: dict[str, dict[str, list[str]]] = {} service_groups: dict[str, list[str]] = {} @@ -681,6 +710,7 @@ async def discover_tools(tool_id: str): permissions["_services"] = services permissions["_service_groups"] = service_groups permissions["_tool_descriptions"] = {t["name"]: t["description"] for t in raw_tools} + permissions["_tool_schemas"] = {t["name"]: t.get("inputSchema") for t in raw_tools if t.get("inputSchema")} tool.tool_permissions = permissions _save(tool) diff --git a/backend/config/paths.py b/backend/config/paths.py index 083ebdfb..62259a68 100644 --- a/backend/config/paths.py +++ b/backend/config/paths.py @@ -35,5 +35,8 @@ OUTPUTS_WORKSPACE_DIR = os.path.join(DATA_ROOT, "outputs_workspace") SKILLS_WORKSPACE_DIR = os.path.join(DATA_ROOT, "skills_workspace") DASHBOARD_LAYOUT_DIR = os.path.join(DATA_ROOT, "dashboard_layout") BUILTIN_PERMISSIONS_PATH = os.path.join(DATA_ROOT, "builtin_permissions.json") +CHANNELS_DIR = os.path.join(DATA_ROOT, "channels") +CHANNELS_SESSIONS_DIR = os.path.join(DATA_ROOT, "channels", "sessions") +ANALYTICS_DIR = os.path.join(DATA_ROOT, "analytics") BACKEND_DIR = _BACKEND_DIR diff --git a/backend/main.py b/backend/main.py index d7f30cee..c07fd7c2 100644 --- a/backend/main.py +++ b/backend/main.py @@ -1,8 +1,14 @@ +import logging import os from uuid import uuid4 -from fastapi.responses import JSONResponse +logger = logging.getLogger(__name__) + +from fastapi.responses import JSONResponse, HTMLResponse from fastapi import Request + +# In-memory store for pending OAuth flows (state → {provider, code_verifier, redirect_uri}) +_pending_oauth: dict[str, dict] = {} from backend.config.Apps import MainApp from backend.apps.health.health import health from backend.apps.agents.agents import agents @@ -16,11 +22,14 @@ from backend.apps.mcp_registry.mcp_registry import mcp_registry from backend.apps.skill_registry.skill_registry import skill_registry from backend.apps.outputs.outputs import outputs from backend.apps.dashboards.dashboards import dashboards +from backend.apps.channels.channels import channels +from backend.apps.analytics.analytics import analytics +from backend.apps.auth.auth import auth from fastapi.middleware.cors import CORSMiddleware from fastapi import WebSocket, WebSocketDisconnect import json -main_app = MainApp([health, agents, templates, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, dashboards]) +main_app = MainApp([health, agents, templates, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, dashboards, channels, analytics, auth]) app = main_app.app app.add_middleware( @@ -48,6 +57,7 @@ async def websocket_session(websocket: WebSocket, session_id: str): payload.get("prompt", ""), mode=payload.get("mode"), model=payload.get("model"), + provider=payload.get("provider"), images=payload.get("images"), ) elif event == "agent:approval_response": @@ -96,6 +106,12 @@ async def websocket_dashboard(websocket: WebSocket): ws_manager.disconnect_global(websocket) +@app.websocket("/ws/talk/{session_id}") +async def websocket_talk_mode(websocket: WebSocket, session_id: str): + from backend.apps.channels.talk_mode import handle_talk_session + await handle_talk_session(websocket, session_id) + + @app.post("/api/browser/command") async def browser_command(request: Request): """HTTP endpoint called by the browser MCP server subprocess. @@ -114,6 +130,53 @@ async def browser_command(request: Request): return JSONResponse(result) +@app.get("/api/subscriptions/pending/{state}") +async def subscriptions_pending(state: str): + """Return pending OAuth data for a state param. Called by 9Router's callback page.""" + pending = _pending_oauth.get(state) + if not pending: + return JSONResponse({"error": "not found"}, status_code=404, + headers={"Access-Control-Allow-Origin": "*"}) + return JSONResponse({ + "provider": pending["provider"], + "code_verifier": pending["code_verifier"], + "redirect_uri": pending["redirect_uri"], + }, headers={"Access-Control-Allow-Origin": "*"}) + + +@app.get("/api/subscriptions/callback") +async def subscriptions_callback(request: Request): + """Catch OAuth redirect from provider, exchange code via 9Router, close window.""" + code = request.query_params.get("code", "") + state = request.query_params.get("state", "") + error = request.query_params.get("error", "") + + if error: + desc = request.query_params.get("error_description", error) + return HTMLResponse(f'

Authorization failed

{desc}

') + + pending = _pending_oauth.pop(state, None) + if not pending: + return HTMLResponse('

Session expired

Please try connecting again.

') + + from backend.apps.nine_router import exchange_oauth + try: + await exchange_oauth(pending["provider"], code, pending["redirect_uri"], pending["code_verifier"], state) + except Exception as e: + return HTMLResponse(f'

Connection failed

{e}

') + + return HTMLResponse( + '' + '
' + '
✓
' + '

Connected!

' + '

You can close this window

' + '
' + '' + '' + ) + + @app.post("/api/browser-agent/run") async def browser_agent_run(request: Request): """Run one or more browser sub-agents in parallel. @@ -126,24 +189,72 @@ async def browser_agent_run(request: Request): model = body.get("model", "sonnet") dashboard_id = body.get("dashboard_id", "") pre_selected_browser_ids = body.get("pre_selected_browser_ids", []) + parent_session_id = body.get("parent_session_id", "") if not tasks: return JSONResponse({"error": "tasks array is required"}, status_code=400) settings = load_settings() - if not settings.anthropic_api_key: - return JSONResponse({"error": "Anthropic API key not configured"}, status_code=400) + + # Determine API credentials — check API key, then 9Router + api_key = settings.anthropic_api_key + auth_token = None + base_url = None + + if not api_key: + # Try 9Router + from backend.apps.nine_router import is_running as _9r_running + if _9r_running(): + api_key = "9router" + base_url = "http://localhost:20128/v1" + auth_token = None + else: + return JSONResponse({"error": "No AI provider configured. Set an API key or connect a subscription."}, status_code=400) results = await run_browser_agents( tasks=tasks, model=model, - api_key=settings.anthropic_api_key, + api_key=api_key, dashboard_id=dashboard_id or None, pre_selected_browser_ids=pre_selected_browser_ids, + parent_session_id=parent_session_id or None, + auth_token=auth_token, + base_url=base_url, ) return JSONResponse({"results": results}) +@app.post("/api/invoke-agent/run") +async def invoke_agent_run(request: Request): + """Fork an existing agent session and send it a new message. + Called by the invoke_agent_mcp_server stdio subprocess.""" + body = await request.json() + session_id = body.get("session_id", "") + message = body.get("message", "") + parent_session_id = body.get("parent_session_id", "") + dashboard_id = body.get("dashboard_id", "") + + if not session_id: + return JSONResponse({"error": "session_id is required"}, status_code=400) + if not message: + return JSONResponse({"error": "message is required"}, status_code=400) + + try: + from backend.apps.agents.agent_manager import agent_manager + result = await agent_manager.invoke_agent( + source_session_id=session_id, + message=message, + parent_session_id=parent_session_id or None, + dashboard_id=dashboard_id or None, + ) + return JSONResponse(result) + except ValueError as e: + return JSONResponse({"error": str(e)}, status_code=404) + except Exception as e: + logger.exception("invoke_agent_run failed") + return JSONResponse({"error": str(e)}, status_code=500) + + if __name__ == "__main__": import argparse import uvicorn diff --git a/backend/requirements.txt b/backend/requirements.txt index 8ef474e6..c9792e56 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,5 +1,8 @@ anthropic -claude-agent-sdk +openai +google-genai +mcp +posthog jsonschema fastapi[standard] pydantic==2.10.5 @@ -9,4 +12,11 @@ pytest==8.3.4 pytest-asyncio==0.25.2 typeguard==4.4.2 python-dotenv==1.1.1 -Pillow \ No newline at end of file +Pillow +# Channels: SMS, WhatsApp, Voice +twilio>=9.0.0 +telnyx>=2.0.0 +edge-tts>=6.1.0 +httpx>=0.27.0 +playwright +cryptography>=42.0.0 \ No newline at end of file diff --git a/electron/main.js b/electron/main.js index d5f79a99..c209718e 100644 --- a/electron/main.js +++ b/electron/main.js @@ -1,5 +1,6 @@ -const { app, BrowserWindow, ipcMain, shell } = require('electron'); -const { autoUpdater } = require('electron-updater'); +const { app, components, BrowserWindow, ipcMain, shell, session } = require('electron'); +let autoUpdater; +try { autoUpdater = require('electron-updater').autoUpdater; } catch (_) {} const path = require('path'); const { spawn, execFileSync } = require('child_process'); const os = require('os'); @@ -7,9 +8,16 @@ const fs = require('fs'); const getPort = require('get-port'); const http = require('http'); +app.commandLine.appendSwitch('disable-features', 'HardwareMediaKeyHandling'); +app.commandLine.appendSwitch('ignore-gpu-blocklist'); +app.commandLine.appendSwitch('enable-gpu-rasterization'); +app.commandLine.appendSwitch('enable-zero-copy'); +app.commandLine.appendSwitch('autoplay-policy', 'no-user-gesture-required'); + let mainWindow = null; let backendProcess = null; let backendPort = null; +let cachedUpdateStatus = { status: 'idle', info: null, error: null }; const isPackaged = app.isPackaged; const isDev = process.env.ELECTRON_DEV === '1'; @@ -24,9 +32,10 @@ const iconPath = path.join(__dirname, 'build', 'icon.png'); function getShellPath() { if (process.platform !== 'darwin' || isDev) return process.env.PATH || ''; + // Strategy 1: ask the user's login shell for its PATH try { - const shell = process.env.SHELL || '/bin/zsh'; - const result = execFileSync(shell, ['-ilc', 'echo $PATH'], { + const userShell = process.env.SHELL || '/bin/zsh'; + const result = execFileSync(userShell, ['-ilc', 'echo $PATH'], { encoding: 'utf8', timeout: 5000, env: { ...process.env, HOME: os.homedir() }, @@ -35,19 +44,40 @@ function getShellPath() { if (resolved) return resolved; } catch (_) { /* fall through */ } + // Strategy 2: read macOS system PATH config (/etc/paths + /etc/paths.d/*) + const systemPaths = []; + try { + const base = fs.readFileSync('/etc/paths', 'utf8'); + for (const line of base.split('\n')) { + const p = line.trim(); + if (p) systemPaths.push(p); + } + } catch (_) { /* ignore */ } + try { + const pathsD = '/etc/paths.d'; + if (fs.existsSync(pathsD)) { + for (const file of fs.readdirSync(pathsD).sort()) { + const content = fs.readFileSync(path.join(pathsD, file), 'utf8'); + for (const line of content.split('\n')) { + const p = line.trim(); + if (p) systemPaths.push(p); + } + } + } + } catch (_) { /* ignore */ } + + // Strategy 3: well-known user-local bin directories const home = os.homedir(); const fallbackDirs = [ - path.join(home, '.nvm/versions/node'), + path.join(home, '.local/bin'), path.join(home, '.volta/bin'), path.join(home, '.fnm/aliases/default/bin'), path.join(home, '.bun/bin'), path.join(home, '.cargo/bin'), - path.join(home, '.local/bin'), '/opt/homebrew/bin', '/usr/local/bin', ]; - // For nvm, resolve the current default version dynamically const nvmDir = path.join(home, '.nvm/versions/node'); try { if (fs.existsSync(nvmDir)) { @@ -58,10 +88,14 @@ function getShellPath() { } } catch (_) { /* ignore */ } - const existing = fallbackDirs.filter((d) => { - try { return fs.statSync(d).isDirectory(); } catch { return false; } - }); - return [...existing, process.env.PATH || ''].join(':'); + const seen = new Set(); + const dirs = []; + for (const d of [...fallbackDirs, ...systemPaths, ...(process.env.PATH || '').split(':')]) { + if (!d || seen.has(d)) continue; + seen.add(d); + try { if (fs.statSync(d).isDirectory()) dirs.push(d); } catch { /* skip */ } + } + return dirs.join(':'); } function getResourcePath(...segments) { @@ -190,6 +224,18 @@ function createWindow() { mainWindow.loadFile(frontendPath); } + mainWindow.webContents.on('will-attach-webview', (_event, webPreferences, _params) => { + webPreferences.plugins = true; + webPreferences.enableBlinkFeatures = 'EncryptedMedia'; + }); + + mainWindow.webContents.on('will-navigate', (event, url) => { + if (isDev && url.startsWith('http://localhost:3000')) return; + if (url.startsWith('file://')) return; + event.preventDefault(); + mainWindow.webContents.send('webview-new-window', url, mainWindow.webContents.id); + }); + mainWindow.on('closed', () => { mainWindow = null; }); @@ -202,30 +248,36 @@ function sendToRenderer(channel, ...args) { } function setupAutoUpdater() { + if (!autoUpdater) return; autoUpdater.autoDownload = false; autoUpdater.autoInstallOnAppQuit = false; autoUpdater.on('update-available', (info) => { console.log(`Update available: ${info.version}`); + cachedUpdateStatus = { status: 'available', info, error: null }; sendToRenderer('update-available', info); }); autoUpdater.on('update-not-available', (info) => { console.log('App is up to date'); + cachedUpdateStatus = { status: 'not-available', info, error: null }; sendToRenderer('update-not-available', info); }); autoUpdater.on('download-progress', (progress) => { + cachedUpdateStatus = { status: 'downloading', info: progress, error: null }; sendToRenderer('download-progress', progress); }); autoUpdater.on('update-downloaded', (info) => { console.log(`Update downloaded: ${info.version}`); + cachedUpdateStatus = { status: 'downloaded', info, error: null }; sendToRenderer('update-downloaded', info); }); autoUpdater.on('error', (err) => { console.error('Auto-update error:', err); + cachedUpdateStatus = { status: 'error', info: null, error: err?.message || String(err) }; sendToRenderer('update-error', err?.message || String(err)); }); @@ -252,6 +304,68 @@ app.whenReady().then(async () => { try { app.dock.setIcon(iconPath); } catch (_) {} } + session.defaultSession.setPermissionRequestHandler((_wc, permission, callback) => { + const allowed = [ + 'media', 'mediaKeySystem', 'protected-media-identifier', + 'geolocation', 'notifications', 'midi', 'midiSysex', + 'clipboard-read', 'clipboard-sanitized-write', + 'pointerLock', 'fullscreen', 'idle-detection', + ]; + console.log('Permission request:', permission, '->', allowed.includes(permission) ? 'granted' : 'denied'); + callback(allowed.includes(permission)); + }); + session.defaultSession.setPermissionCheckHandler((_wc, permission) => { + const allowed = [ + 'media', 'mediaKeySystem', 'protected-media-identifier', + 'clipboard-read', 'clipboard-sanitized-write', + 'pointerLock', 'fullscreen', 'idle-detection', + ]; + return allowed.includes(permission); + }); + + // Read-only logging for DRM license requests — no modifying interceptors + // so the network stack can set Content-Type and other headers normally. + session.defaultSession.webRequest.onSendHeaders( + { urls: ['*://*/*widevine*license*'] }, + (details) => { + console.log(`[drm-req] ${details.method} ${details.url}`); + for (const [k, v] of Object.entries(details.requestHeaders || {})) { + if (/content-type|origin|referer|auth|accept/i.test(k)) { + console.log(`[drm-req] ${k}: ${v}`); + } + } + }, + ); + session.defaultSession.webRequest.onCompleted( + { urls: ['*://*/*widevine*', '*://*/*license*'] }, + (details) => { + console.log(`[drm-net] ${details.method} ${details.url} → ${details.statusCode}`); + }, + ); + session.defaultSession.webRequest.onErrorOccurred( + { urls: ['*://*/*widevine*', '*://*/*license*'] }, + (details) => { + console.log(`[drm-net] FAILED ${details.method} ${details.url} → ${details.error}`); + }, + ); + + // Wait for the Widevine CDM to be downloaded/ready (CastLabs Component + // Updater Service). On first launch this downloads the CDM; subsequent + // launches use the cached version. + if (components && typeof components.whenReady === 'function') { + try { + await components.whenReady(); + console.log('Widevine CDM ready'); + if (typeof components.status === 'function') { + console.log('CDM component status:', JSON.stringify(components.status())); + } + } catch (err) { + console.warn('Widevine CDM not available:', err.message); + } + } else { + console.log('CastLabs components API not available — using standard Electron (no DRM)'); + } + try { if (isDev) { backendPort = parseInt(process.env.OPENSWARM_PORT || '8324', 10); @@ -262,6 +376,13 @@ app.whenReady().then(async () => { createWindow(); if (!isDev) { setupAutoUpdater(); + mainWindow.webContents.on('did-finish-load', () => { + if (cachedUpdateStatus.status === 'available') { + sendToRenderer('update-available', cachedUpdateStatus.info); + } else if (cachedUpdateStatus.status === 'downloaded') { + sendToRenderer('update-downloaded', cachedUpdateStatus.info); + } + }); } } catch (err) { console.error('Failed to start:', err); @@ -269,6 +390,79 @@ app.whenReady().then(async () => { } }); +app.on('web-contents-created', (_event, contents) => { + contents.setWindowOpenHandler(({ url, disposition }) => { + if (disposition === 'foreground-tab' || disposition === 'background-tab') { + if (mainWindow && !mainWindow.isDestroyed()) { + mainWindow.webContents.send('webview-new-window', url, contents.id); + } + return { action: 'deny' }; + } + + return { + action: 'allow', + overrideBrowserWindowOptions: { + parent: mainWindow || undefined, + }, + }; + }); + + contents.on('did-create-window', (childWindow) => { + if (mainWindow && !mainWindow.isDestroyed() && !childWindow.isDestroyed()) { + childWindow.setParentWindow(mainWindow); + } + }); + + if (contents.getType() === 'webview') { + contents.on('console-message', (_e, level, message, line, sourceId) => { + if (message.includes('widevine') || message.includes('drm') || + message.includes('license') || message.includes('MediaKeySession') || + message.includes('EME') || message.includes('[drm-diag]') || level >= 2) { + const tag = ['LOG', 'INFO', 'WARN', 'ERROR'][level] || 'LOG'; + const src = sourceId ? sourceId.split('/').pop() : ''; + console.log(`[webview:${tag}] ${message}${src ? ` (${src}:${line})` : ''}`); + } + }); + + contents.on('dom-ready', () => { + const url = contents.getURL(); + if (url.includes('spotify')) { + contents.executeJavaScript(` + (function() { + const origFetch = window.fetch; + window.fetch = async function(...args) { + const resp = await origFetch.apply(this, args); + const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || ''; + if (url.includes('widevine-license') && !resp.ok) { + const clone = resp.clone(); + try { + const text = await clone.text(); + console.log('[drm-diag] License response ' + resp.status + ': ' + text.substring(0, 500)); + } catch(e) {} + } + return resp; + }; + + // Check EME availability + if (navigator.requestMediaKeySystemAccess) { + navigator.requestMediaKeySystemAccess('com.widevine.alpha', [{ + initDataTypes: ['cenc'], + audioCapabilities: [{contentType: 'audio/mp4; codecs="mp4a.40.2"'}], + }]).then(function(access) { + console.log('[drm-diag] Widevine EME access: ' + access.keySystem); + }).catch(function(err) { + console.log('[drm-diag] Widevine EME FAILED: ' + err.message); + }); + } else { + console.log('[drm-diag] EME API not available'); + } + })(); + `).catch(() => {}); + } + }); + } +}); + app.on('window-all-closed', () => { if (!isDev) killBackend(); app.quit(); @@ -286,9 +480,14 @@ app.on('activate', () => { ipcMain.handle('get-backend-port', () => backendPort); ipcMain.handle('get-app-version', () => app.getVersion()); +ipcMain.handle('get-webview-preload-path', () => { + return `file://${path.join(__dirname, 'webview-preload.js')}`; +}); + +ipcMain.handle('get-update-status', () => cachedUpdateStatus); ipcMain.handle('check-for-updates', async () => { - if (!isPackaged) { + if (!autoUpdater || !isPackaged) { sendToRenderer('update-error', 'Update check is only available in the packaged app.'); return { success: false, error: 'Not packaged' }; } @@ -305,6 +504,7 @@ ipcMain.handle('check-for-updates', async () => { }); ipcMain.handle('download-update', async () => { + if (!autoUpdater) return { success: false, error: 'Updater not available' }; try { await autoUpdater.downloadUpdate(); return { success: true }; @@ -314,6 +514,7 @@ ipcMain.handle('download-update', async () => { }); ipcMain.handle('install-update', () => { + if (!autoUpdater) return; autoUpdater.quitAndInstall(false, true); }); diff --git a/electron/package-lock.json b/electron/package-lock.json index 3358b1a3..b3422f6d 100644 --- a/electron/package-lock.json +++ b/electron/package-lock.json @@ -1,19 +1,20 @@ { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.11", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.11", + "hasInstallScript": true, "dependencies": { "electron-updater": "^6.3.0", "get-port": "^5.1.1" }, "devDependencies": { "@electron/notarize": "^3.1.1", - "electron": "^33.0.0", + "electron": "castlabs/electron-releases#v33.4.11+wvcus", "electron-builder": "^25.1.0" } }, @@ -2206,9 +2207,8 @@ } }, "node_modules/electron": { - "version": "33.4.11", - "resolved": "https://registry.npmjs.org/electron/-/electron-33.4.11.tgz", - "integrity": "sha512-xmdAs5QWRkInC7TpXGNvzo/7exojubk+72jn1oJL7keNeIlw7xNglf8TGtJtkR4rWC5FJq0oXiIXPS9BcK2Irg==", + "version": "33.4.11+wvcus", + "resolved": "git+ssh://git@github.com/castlabs/electron-releases.git#d1cf58c11ec0a8a04f307ed362d7efde2816778d", "dev": true, "hasInstallScript": true, "license": "MIT", @@ -3953,9 +3953,9 @@ } }, "node_modules/node-abi": { - "version": "3.88.0", - "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.88.0.tgz", - "integrity": "sha512-At6b4UqIEVudaqPsXjmUO1r/N5BUr4yhDGs5PkBE8/oG5+TfLPhFechiskFsnT6Ql0VfUXbalUUCbfXxtj7K+w==", + "version": "3.89.0", + "resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz", + "integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==", "dev": true, "license": "MIT", "dependencies": { @@ -4605,9 +4605,9 @@ } }, "node_modules/sax": { - "version": "1.5.0", - "resolved": "https://registry.npmjs.org/sax/-/sax-1.5.0.tgz", - "integrity": "sha512-21IYA3Q5cQf089Z6tgaUTr7lDAyzoTPx5HRtbhsME8Udispad8dC/+sziTNugOEx54ilvatQ9YCzl4KQLPcRHA==", + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz", + "integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==", "license": "BlueOak-1.0.0", "engines": { "node": ">=11.0.0" diff --git a/electron/package.json b/electron/package.json index 48ee49a7..32c3c37d 100644 --- a/electron/package.json +++ b/electron/package.json @@ -1,11 +1,13 @@ { "name": "openswarm", - "version": "1.0.3", + "version": "1.0.12", "description": "OpenSwarm — AI Agent Orchestrator", "main": "main.js", "scripts": { "start": "electron .", "dev": "ELECTRON_DEV=1 electron .", + "postinstall": "bash scripts/sign-vmp.sh", + "sign-vmp": "bash scripts/sign-vmp.sh", "dist": "electron-builder --mac --publish never", "dist:publish": "electron-builder --mac --publish always", "dist:all": "electron-builder --mac --win --linux" @@ -16,12 +18,15 @@ }, "devDependencies": { "@electron/notarize": "^3.1.1", - "electron": "^33.0.0", + "electron": "castlabs/electron-releases#v33.4.11+wvcus", "electron-builder": "^25.1.0" }, "build": { "appId": "com.clusterlabs.openswarm", "productName": "OpenSwarm", + "electronDownload": { + "mirror": "https://github.com/castlabs/electron-releases/releases/download/v" + }, "directories": { "output": "dist" }, @@ -38,6 +43,7 @@ "entitlementsInherit": "build/entitlements.mac.plist" }, "dmg": { + "artifactName": "OpenSwarm-${arch}.${ext}", "title": "OpenSwarm", "contents": [ { @@ -54,34 +60,24 @@ }, "extraResources": [ { - "from": "../frontend/dist", + "from": "build-staging/frontend", "to": "frontend", "filter": [ "**/*" ] }, { - "from": "../backend", + "from": "build-staging/backend", "to": "backend", "filter": [ - "**/*", - "!__pycache__/**", - "!**/__pycache__/**", - "!.venv/**", - "!*.pyc" + "**/*" ] }, { - "from": "../debugger", + "from": "build-staging/debugger", "to": "debugger", "filter": [ - "**/*", - "!__pycache__/**", - "!**/__pycache__/**", - "!*.pyc", - "!.venv/**", - "!**/.venv/**", - "!**/node_modules/**" + "**/*" ] }, { diff --git a/electron/preload.js b/electron/preload.js index 2f9037be..534c540f 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -2,15 +2,18 @@ const { contextBridge, ipcRenderer } = require('electron'); (async () => { const port = await ipcRenderer.invoke('get-backend-port'); + const webviewPreloadPath = await ipcRenderer.invoke('get-webview-preload-path'); contextBridge.exposeInMainWorld('__OPENSWARM_PORT__', port); contextBridge.exposeInMainWorld('openswarm', { getBackendPort: () => port, + getWebviewPreloadPath: () => webviewPreloadPath, getAppVersion: () => ipcRenderer.invoke('get-app-version'), openExternal: (url) => ipcRenderer.invoke('open-external', url), capturePage: (rect) => ipcRenderer.invoke('capture-page', rect), + getUpdateStatus: () => ipcRenderer.invoke('get-update-status'), checkForUpdates: () => ipcRenderer.invoke('check-for-updates'), downloadUpdate: () => ipcRenderer.invoke('download-update'), installUpdate: () => ipcRenderer.invoke('install-update'), @@ -40,5 +43,11 @@ const { contextBridge, ipcRenderer } = require('electron'); ipcRenderer.on('update-error', listener); return () => ipcRenderer.removeListener('update-error', listener); }, + + onWebviewNewWindow: (cb) => { + const listener = (_event, url, webContentsId) => cb(url, webContentsId); + ipcRenderer.on('webview-new-window', listener); + return () => ipcRenderer.removeListener('webview-new-window', listener); + }, }); })(); diff --git a/electron/scripts/sign-vmp.sh b/electron/scripts/sign-vmp.sh new file mode 100755 index 00000000..ba5eb833 --- /dev/null +++ b/electron/scripts/sign-vmp.sh @@ -0,0 +1,61 @@ +#!/bin/bash +# Signs the CastLabs Electron binary with a production VMP certificate via EVS, +# then repairs macOS framework symlinks that npm/signing may strip. +# +# First-time setup (one-time): +# pip3 install --user castlabs-evs +# python3 -m castlabs_evs.account signup +# +# After signup, this script runs automatically. + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +ELECTRON_DIR="$SCRIPT_DIR/../node_modules/electron/dist" +FW_BASE="$ELECTRON_DIR/Electron.app/Contents/Frameworks" + +fix_framework_symlinks() { + [ -d "$FW_BASE" ] || return 0 + for fw in "$FW_BASE"/*.framework; do + [ -d "$fw/Versions/A" ] || continue + local name + name=$(basename "$fw" .framework) + cd "$fw" + (cd Versions && ln -sf A Current 2>/dev/null) + ln -sf "Versions/Current/$name" "$name" 2>/dev/null + [ -d "Versions/A/Resources" ] && ln -sf Versions/Current/Resources Resources 2>/dev/null + [ -d "Versions/A/Libraries" ] && ln -sf Versions/Current/Libraries Libraries 2>/dev/null + [ -d "Versions/A/Helpers" ] && ln -sf Versions/Current/Helpers Helpers 2>/dev/null + done +} + +if [ ! -d "$ELECTRON_DIR" ]; then + echo "[vmp] Electron dist not found at $ELECTRON_DIR — skipping VMP signing" + fix_framework_symlinks + exit 0 +fi + +# Always fix symlinks first (npm git installs strip them) +fix_framework_symlinks + +if ! python3 -c "import castlabs_evs" 2>/dev/null; then + echo "[vmp] castlabs-evs not installed. Install with: pip3 install --user castlabs-evs" + echo "[vmp] Skipping VMP signing — DRM playback will be limited" + exit 0 +fi + +VERIFY_OUTPUT=$(python3 -m castlabs_evs.vmp verify-pkg "$ELECTRON_DIR" 2>&1) +if echo "$VERIFY_OUTPUT" | grep -q "Signature is valid" && ! echo "$VERIFY_OUTPUT" | grep -q "development only"; then + echo "[vmp] Electron already has a valid production VMP signature" + exit 0 +fi + +echo "[vmp] Signing Electron with production VMP certificate..." +if python3 -m castlabs_evs.vmp sign-pkg "$ELECTRON_DIR" 2>&1; then + echo "[vmp] VMP signing successful — full DRM playback enabled" + # Re-fix symlinks in case signing modified the bundle + fix_framework_symlinks +else + echo "[vmp] VMP signing failed — you may need to run: python3 -m castlabs_evs.account signup" + echo "[vmp] DRM playback will be limited to previews until signed" +fi + +exit 0 diff --git a/electron/webview-preload.js b/electron/webview-preload.js new file mode 100644 index 00000000..95a93d66 --- /dev/null +++ b/electron/webview-preload.js @@ -0,0 +1,87 @@ +/** + * Webview preload script — patches browser fingerprinting so sites like + * Spotify/Netflix don't detect an Electron shell and disable features. + * Loaded via the webview's `preload` attribute before any page script runs. + */ + +'use strict'; + +// Hide webdriver flag +Object.defineProperty(navigator, 'webdriver', { + get: () => false, + configurable: true, +}); + +// Spoof navigator.plugins (Chrome has a few built-in ones) +const fakePlugins = { + 0: { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' }, + 1: { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' }, + 2: { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' }, + length: 3, + item: (i) => fakePlugins[i] || null, + namedItem: (name) => { + for (let i = 0; i < fakePlugins.length; i++) { + if (fakePlugins[i].name === name) return fakePlugins[i]; + } + return null; + }, + refresh: () => {}, + [Symbol.iterator]: function* () { + for (let i = 0; i < this.length; i++) yield this[i]; + }, +}; +try { + Object.defineProperty(navigator, 'plugins', { + get: () => fakePlugins, + configurable: true, + }); +} catch (_) {} + +// Ensure window.chrome exists (sites test for it) +if (!window.chrome) { + window.chrome = {}; +} +if (!window.chrome.runtime) { + window.chrome.runtime = { + connect: () => {}, + sendMessage: () => {}, + onMessage: { addListener: () => {}, removeListener: () => {} }, + }; +} + +// Ensure navigator.languages has sensible values +try { + Object.defineProperty(navigator, 'languages', { + get: () => ['en-US', 'en'], + configurable: true, + }); +} catch (_) {} + +// Patch permissions.query to report 'granted' for common permissions +const originalQuery = navigator.permissions?.query?.bind(navigator.permissions); +if (originalQuery) { + navigator.permissions.query = (params) => { + if (params.name === 'notifications') { + return Promise.resolve({ state: 'granted', onchange: null }); + } + return originalQuery(params).catch(() => + Promise.resolve({ state: 'prompt', onchange: null }) + ); + }; +} + +// Prevent iframe detection heuristics +try { + Object.defineProperty(document, 'hidden', { + get: () => false, + configurable: true, + }); + Object.defineProperty(document, 'visibilityState', { + get: () => 'visible', + configurable: true, + }); +} catch (_) {} + +// Fix console.debug detection (some sites use it as a breakpoint detector) +const noop = () => {}; +if (!window.console.debug) window.console.debug = noop; diff --git a/face.html b/face.html new file mode 100644 index 00000000..23f44491 --- /dev/null +++ b/face.html @@ -0,0 +1,334 @@ + + + + + +Pixel Face + + + + + + + diff --git a/frontend/src/app/Main.tsx b/frontend/src/app/Main.tsx index 7628f5f0..81c973d1 100644 --- a/frontend/src/app/Main.tsx +++ b/frontend/src/app/Main.tsx @@ -3,8 +3,9 @@ import { Provider } from 'react-redux'; import { HashRouter, Routes, Route } from 'react-router-dom'; import { ThemeProvider as MuiThemeProvider, createTheme, CssBaseline } from '@mui/material'; import { store } from '../shared/state/store'; -import { useAppDispatch } from '@/shared/hooks'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { fetchSettings } from '@/shared/state/settingsSlice'; +import { fetchModels } from '@/shared/state/modelsSlice'; import { setAppVersion, setUpdateAvailable, @@ -22,9 +23,13 @@ import Tools from './pages/Tools/Tools'; import Modes from './pages/Modes/Modes'; import Views from './pages/Views/Views'; import Customization from './pages/Customization/Customization'; +import Channels from './pages/Channels/Channels'; +import Analytics from './pages/Analytics/Analytics'; +import AnalyticsOptIn from './components/AnalyticsOptIn'; import { useKeyboardShortcuts } from '@/shared/hooks/useKeyboardShortcuts'; import KeyboardShortcutsHelp from './components/KeyboardShortcutsHelp'; import { ThemeProvider, useThemeMode, useClaudeTokens } from '@/shared/styles/ThemeContext'; +import OnboardingModal from './components/OnboardingModal'; import { ClaudeTokens } from '@/shared/styles/claudeTokens'; function buildMuiTheme(c: ClaudeTokens, mode: 'light' | 'dark') { @@ -155,9 +160,16 @@ const ShortcutsProvider: React.FC<{ children: React.ReactNode }> = ({ children } const SettingsLoader: React.FC<{ children: React.ReactNode }> = ({ children }) => { const dispatch = useAppDispatch(); + const { setMode: setThemeMode } = useThemeMode(); + const theme = useAppSelector((s) => s.settings.data.theme); + const loaded = useAppSelector((s) => s.settings.loaded); useEffect(() => { dispatch(fetchSettings()); + dispatch(fetchModels()); }, [dispatch]); + useEffect(() => { + if (loaded) setThemeMode(theme as 'light' | 'dark'); + }, [loaded, theme, setThemeMode]); return <>{children}; }; @@ -170,6 +182,21 @@ const UpdateListener: React.FC<{ children: React.ReactNode }> = ({ children }) = api.getAppVersion().then((v: string) => dispatch(setAppVersion(v))); + api.getUpdateStatus?.().then((cached) => { + if (!cached) return; + if (cached.status === 'available' && cached.info?.version) { + dispatch(setUpdateAvailable(cached.info.version)); + } else if (cached.status === 'not-available') { + dispatch(setUpdateNotAvailable()); + } else if (cached.status === 'downloading' && cached.info?.percent != null) { + dispatch(setDownloading(cached.info.percent)); + } else if (cached.status === 'downloaded') { + dispatch(setUpdateDownloaded()); + } else if (cached.status === 'error' && cached.error) { + dispatch(setUpdateError(cached.error)); + } + }); + const cleanups = [ api.onUpdateAvailable?.((info: OpenSwarmUpdateInfo) => dispatch(setUpdateAvailable(info.version))), api.onUpdateNotAvailable?.(() => dispatch(setUpdateNotAvailable())), @@ -207,8 +234,12 @@ const ThemedApp: React.FC = () => { } /> } /> } /> + } /> + } /> + + diff --git a/frontend/src/app/components/AnalyticsOptIn.tsx b/frontend/src/app/components/AnalyticsOptIn.tsx new file mode 100644 index 00000000..b96701dd --- /dev/null +++ b/frontend/src/app/components/AnalyticsOptIn.tsx @@ -0,0 +1,84 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import Button from '@mui/material/Button'; +import Paper from '@mui/material/Paper'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; +import { updateSettings } from '@/shared/state/settingsSlice'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; + +const AnalyticsOptIn: React.FC = () => { + const c = useClaudeTokens(); + const dispatch = useAppDispatch(); + const settings = useAppSelector((s) => s.settings.data); + const loaded = useAppSelector((s) => s.settings.loaded); + + if (!loaded || settings.analytics_opt_in !== null) return null; + + const handleChoice = (optIn: boolean) => { + dispatch(updateSettings({ ...settings, analytics_opt_in: optIn })); + }; + + return ( + + + + Help improve OpenSwarm + + + Share anonymous usage statistics like session counts, feature usage, and model preferences. + No conversations, file paths, or personal information — ever. + You can change this anytime in Settings. + + + + + + + + ); +}; + +export default AnalyticsOptIn; diff --git a/frontend/src/app/components/DynamicIsland.tsx b/frontend/src/app/components/DynamicIsland.tsx new file mode 100644 index 00000000..76149127 --- /dev/null +++ b/frontend/src/app/components/DynamicIsland.tsx @@ -0,0 +1,993 @@ +import React, { useMemo, useCallback, useState, useEffect, useRef } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import IconButton from '@mui/material/IconButton'; +import Tooltip from '@mui/material/Tooltip'; +import Collapse from '@mui/material/Collapse'; +import SearchIcon from '@mui/icons-material/Search'; +import StopCircleOutlinedIcon from '@mui/icons-material/StopCircleOutlined'; +import CloseIcon from '@mui/icons-material/Close'; +import CheckIcon from '@mui/icons-material/Check'; +import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; +import ExpandLessIcon from '@mui/icons-material/ExpandLess'; +import { motion, AnimatePresence } from 'framer-motion'; +import { useNavigate } from 'react-router-dom'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; +import { + handleApproval, + stopAgent, + dismissAgentNotification, + dismissAllFinishedNotifications, + ApprovalRequest, + AgentSession, + HistorySession, +} from '@/shared/state/agentsSlice'; +import { setPendingFocusAgentId } from '@/shared/state/tempStateSlice'; +import ApprovalBar, { BatchApprovalBar, parseMcpToolName, useMcpToolMeta, getToolIcon } from '@/app/pages/AgentChat/ApprovalBar'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; + +// --------------------------------------------------------------------------- +// Types +// --------------------------------------------------------------------------- + +type IslandState = 'idle' | 'compact' | 'compact-actionable' | 'expanded'; + +interface SessionApprovalGroup { + sessionId: string; + sessionName: string; + approvals: ApprovalRequest[]; +} + +type TrackedAgent = { + id: string; + name: string; + status: AgentSession['status'] | string; + dashboardId?: string; +}; + +const STATUS_CONFIG: Record = { + running: { label: 'Running', tokenKey: 'success' }, + waiting_approval: { label: 'Waiting', tokenKey: 'warning' }, + completed: { label: 'Done', tokenKey: 'success' }, + error: { label: 'Error', tokenKey: 'error' }, + stopped: { label: 'Stopped', tokenKey: 'info' }, +}; + +// --------------------------------------------------------------------------- +// Spring configs +// --------------------------------------------------------------------------- + +const SPRING_LAYOUT = { type: 'spring' as const, stiffness: 400, damping: 30 }; +const SPRING_BOUNCE = { type: 'spring' as const, stiffness: 500, damping: 25 }; + +// --------------------------------------------------------------------------- +// Sub-components +// --------------------------------------------------------------------------- + +const StatusDot: React.FC<{ status: string; c: ReturnType }> = ({ status, c }) => { + const cfg = STATUS_CONFIG[status]; + const color = cfg?.tokenKey ? (c.status as any)[cfg.tokenKey] : c.text.ghost; + const isActive = status === 'running'; + return ( + + ); +}; + +const AgentStatusRow: React.FC<{ + agent: TrackedAgent; + c: ReturnType; + onStop: (id: string) => void; + onDismiss: (id: string) => void; + onNavigate: (dashboardId: string, agentId: string) => void; +}> = ({ agent, c, onStop, onDismiss, onNavigate }) => { + const isActive = agent.status === 'running' || agent.status === 'waiting_approval'; + const cfg = STATUS_CONFIG[agent.status] ?? { label: agent.status }; + + return ( + agent.dashboardId && onNavigate(agent.dashboardId, agent.id)} + sx={{ + display: 'flex', + alignItems: 'center', + gap: 1, + px: 2, + py: 0.75, + cursor: agent.dashboardId ? 'pointer' : 'default', + '&:hover': { bgcolor: c.border.subtle }, + transition: 'background-color 0.15s', + minHeight: 34, + }} + > + + + {agent.name} + + + {cfg.label} + + {isActive ? ( + + { e.stopPropagation(); onStop(agent.id); }} + sx={{ p: 0.25, color: c.text.ghost, '&:hover': { color: c.status.error, bgcolor: c.border.subtle } }} + > + + + + ) : ( + + { e.stopPropagation(); onDismiss(agent.id); }} + sx={{ p: 0.25, color: c.text.ghost, '&:hover': { bgcolor: c.border.subtle } }} + > + + + + )} + + ); +}; + +// --------------------------------------------------------------------------- +// Compact activity indicator — subtle breathing dot +// --------------------------------------------------------------------------- + +const ActivityIndicator: React.FC<{ c: ReturnType }> = ({ c }) => ( + +); + +// --------------------------------------------------------------------------- +// Main component +// --------------------------------------------------------------------------- + +const DynamicIsland: React.FC = () => { + const c = useClaudeTokens(); + const dispatch = useAppDispatch(); + const navigate = useNavigate(); + const islandRef = useRef(null); + + const sessions = useAppSelector((state) => state.agents.sessions); + const history = useAppSelector((state) => state.agents.history); + const trackedIds = useAppSelector((state) => state.agents.trackedNotificationIds); + + const [userExpanded, setUserExpanded] = useState(false); + + // ---- Derived data ---- + + const groups: SessionApprovalGroup[] = useMemo(() => { + const result: SessionApprovalGroup[] = []; + for (const [sessionId, session] of Object.entries(sessions)) { + if (session.pending_approvals?.length > 0) { + result.push({ + sessionId, + sessionName: session.name || 'Agent', + approvals: session.pending_approvals, + }); + } + } + return result; + }, [sessions]); + + const totalApprovals = useMemo( + () => groups.reduce((sum, g) => sum + g.approvals.length, 0), + [groups], + ); + + const trackedAgents: TrackedAgent[] = useMemo(() => { + const agents = trackedIds + .map((id): TrackedAgent | null => { + const session = sessions[id]; + if (session && session.status !== 'draft') { + return { id, name: session.name, status: session.status, dashboardId: session.dashboard_id }; + } + const hist: HistorySession | undefined = history[id]; + if (hist) { + return { id, name: hist.name, status: hist.status, dashboardId: hist.dashboard_id }; + } + return null; + }) + .filter((a): a is TrackedAgent => a !== null); + + const trackedIdSet = new Set(trackedIds); + for (const g of groups) { + if (!trackedIdSet.has(g.sessionId)) { + const session = sessions[g.sessionId]; + if (session && session.status !== 'draft') { + agents.push({ id: g.sessionId, name: session.name, status: session.status, dashboardId: session.dashboard_id }); + } + } + } + + return agents; + }, [trackedIds, sessions, history, groups]); + + const activeAgents = useMemo( + () => trackedAgents.filter((a) => a.status === 'running' || a.status === 'waiting_approval'), + [trackedAgents], + ); + const finishedAgents = useMemo( + () => trackedAgents.filter((a) => a.status !== 'running' && a.status !== 'waiting_approval'), + [trackedAgents], + ); + + const hasApprovals = totalApprovals > 0; + const hasAgents = trackedAgents.length > 0; + + const hasOnlyQuestionApprovals = useMemo(() => { + if (!hasApprovals) return false; + const allApprovals = groups.flatMap((g) => g.approvals); + return allApprovals.every((a) => a.tool_name === 'AskUserQuestion'); + }, [hasApprovals, groups]); + + const nonQuestionApprovalCount = useMemo( + () => groups.reduce((sum, g) => sum + g.approvals.filter((a) => a.tool_name !== 'AskUserQuestion').length, 0), + [groups], + ); + + const oldestNonQuestionApproval = useMemo(() => { + const all = groups + .flatMap((g) => g.approvals) + .filter((a) => a.tool_name !== 'AskUserQuestion'); + if (all.length === 0) return null; + return all.reduce((oldest, a) => + a.created_at < oldest.created_at ? a : oldest, + ); + }, [groups]); + + // ---- Island state machine ---- + + const islandState: IslandState = useMemo(() => { + if (userExpanded && (hasAgents || hasApprovals)) return 'expanded'; + if (hasApprovals && hasOnlyQuestionApprovals) return 'expanded'; + if (hasApprovals) return 'compact-actionable'; + if (hasAgents) return 'compact'; + return 'idle'; + }, [hasApprovals, hasOnlyQuestionApprovals, userExpanded, hasAgents]); + + useEffect(() => { + if (!hasAgents && !hasApprovals) { + setUserExpanded(false); + } + }, [hasAgents, hasApprovals]); + + // ---- Click outside to collapse ---- + + useEffect(() => { + if (islandState !== 'expanded') return; + const handler = (e: MouseEvent) => { + if (islandRef.current && !islandRef.current.contains(e.target as Node)) { + setUserExpanded(false); + } + }; + document.addEventListener('mousedown', handler); + return () => document.removeEventListener('mousedown', handler); + }, [islandState]); + + // ---- Callbacks ---- + + const onApprove = useCallback( + (requestId: string, updatedInput?: Record) => { + dispatch(handleApproval({ requestId, behavior: 'allow', updatedInput })); + }, + [dispatch], + ); + + const onDeny = useCallback( + (requestId: string, message?: string) => { + dispatch(handleApproval({ requestId, behavior: 'deny', message })); + }, + [dispatch], + ); + + const onStopAgent = useCallback( + (sessionId: string) => dispatch(stopAgent({ sessionId })), + [dispatch], + ); + + const onDismissAgent = useCallback( + (sessionId: string) => dispatch(dismissAgentNotification(sessionId)), + [dispatch], + ); + + const onNavigateToDashboard = useCallback( + (dashboardId: string, agentId: string) => { + dispatch(setPendingFocusAgentId(agentId)); + navigate(`/dashboard/${dashboardId}`); + }, + [navigate, dispatch], + ); + + const onApproveAllNonQuestion = useCallback(() => { + for (const g of groups) { + for (const req of g.approvals) { + if (req.tool_name !== 'AskUserQuestion') { + dispatch(handleApproval({ requestId: req.id, behavior: 'allow' })); + } + } + } + }, [dispatch, groups]); + + const onDenyAllNonQuestion = useCallback(() => { + for (const g of groups) { + for (const req of g.approvals) { + if (req.tool_name !== 'AskUserQuestion') { + dispatch(handleApproval({ requestId: req.id, behavior: 'deny' })); + } + } + } + }, [dispatch, groups]); + + const onClearAllFinished = useCallback(() => { + dispatch(dismissAllFinishedNotifications()); + }, [dispatch]); + + const handleIslandClick = useCallback(() => { + if (islandState === 'compact' || islandState === 'compact-actionable') { + setUserExpanded(true); + } else if (islandState === 'expanded') { + setUserExpanded(false); + } + }, [islandState]); + + // ---- Styling — uses the same neutral palette as the rest of the UI ---- + + const islandWidth = islandState === 'idle' + ? 200 + : islandState === 'compact' + ? 210 + : islandState === 'compact-actionable' + ? 310 + : 400; + + const islandBorderRadius = islandState === 'expanded' ? 14 : 50; + + const shadow = islandState === 'idle' + ? 'none' + : islandState === 'compact' + ? c.shadow.sm + : c.shadow.md; + + // ---- Compact summary text ---- + + const compactText = useMemo(() => { + const parts: string[] = []; + if (activeAgents.length > 0) { + parts.push(`${activeAgents.length} running`); + } + if (finishedAgents.length > 0) { + parts.push(`${finishedAgents.length} done`); + } + return parts.join(' · ') || 'Agents'; + }, [activeAgents.length, finishedAgents.length]); + + const glowKeyframes = useMemo(() => ` + @keyframes approvalGlow { + 0%, 100% { box-shadow: 0 0 6px 1px ${c.status.warning}30; } + 50% { box-shadow: 0 0 12px 3px ${c.status.warning}60; } + } + `, [c.status.warning]); + + // ---- Render ---- + + return ( + <> + {islandState === 'compact-actionable' && } + + + + {islandState === 'idle' && ( + + )} + {islandState === 'compact' && ( + + )} + {islandState === 'compact-actionable' && oldestNonQuestionApproval && ( + setUserExpanded(true)} + /> + )} + {islandState === 'expanded' && ( + setUserExpanded(false)} + /> + )} + + + + + ); +}; + +// --------------------------------------------------------------------------- +// Idle pill — disabled search bar +// --------------------------------------------------------------------------- + +const IdlePill: React.FC<{ c: ReturnType }> = ({ c }) => ( + + + + + + Search... + + + + +); + +// --------------------------------------------------------------------------- +// Compact pill +// --------------------------------------------------------------------------- + +const CompactPill: React.FC<{ + c: ReturnType; + text: string; + activeCount: number; + hasApprovals: boolean; +}> = ({ c, text, activeCount, hasApprovals }) => ( + + + + + {text} + + {hasApprovals && ( + + )} + + +); + +// --------------------------------------------------------------------------- +// Compact-actionable pill — single approval with icon + name + approve/deny +// --------------------------------------------------------------------------- + +const CompactActionablePill: React.FC<{ + c: ReturnType; + request: ApprovalRequest; + remainingCount: number; + onApprove: (requestId: string) => void; + onDeny: (requestId: string) => void; + onExpand: () => void; +}> = ({ c, request, remainingCount, onApprove, onDeny, onExpand }) => { + const parsed = useMemo(() => parseMcpToolName(request.tool_name), [request.tool_name]); + const meta = useMcpToolMeta(parsed); + + const icon = parsed.isMcp + ? (meta.integration?.icon || null) + : getToolIcon(request.tool_name); + + return ( + + + + {icon} + + + {parsed.displayName} + + {remainingCount > 1 && ( + + +{remainingCount - 1} + + )} + + { e.stopPropagation(); onApprove(request.id); }} + sx={{ + p: 0, + width: 18, + height: 18, + color: '#fff', + bgcolor: c.status.success, + '&:hover': { bgcolor: c.status.success, filter: 'brightness(0.85)' }, + }} + > + + + + + { e.stopPropagation(); onDeny(request.id); }} + sx={{ + p: 0, + width: 18, + height: 18, + color: c.status.error, + border: `1px solid ${c.status.error}`, + '&:hover': { bgcolor: `${c.status.error}0a` }, + }} + > + + + + + { e.stopPropagation(); onExpand(); }} + sx={{ p: 0.25, color: c.text.ghost, '&:hover': { color: c.text.tertiary } }} + > + + + + + + ); +}; + +// --------------------------------------------------------------------------- +// Expanded card +// --------------------------------------------------------------------------- + +const ExpandedCard: React.FC<{ + c: ReturnType; + groups: SessionApprovalGroup[]; + totalApprovals: number; + activeAgents: TrackedAgent[]; + finishedAgents: TrackedAgent[]; + hasApprovals: boolean; + hasAgents: boolean; + onApprove: (requestId: string, updatedInput?: Record) => void; + onDeny: (requestId: string, message?: string) => void; + onStopAgent: (id: string) => void; + onDismissAgent: (id: string) => void; + onNavigateToDashboard: (dashboardId: string, agentId: string) => void; + onClearAllFinished: () => void; + onCollapse: () => void; +}> = ({ + c, groups, totalApprovals, + activeAgents, finishedAgents, hasApprovals, hasAgents, + onApprove, onDeny, onStopAgent, onDismissAgent, onNavigateToDashboard, onClearAllFinished, onCollapse, +}) => { + const [completedExpanded, setCompletedExpanded] = useState(false); + const headerTitle = hasApprovals && !hasAgents + ? 'Approval Required' + : hasAgents && !hasApprovals + ? 'Agents' + : 'Notifications'; + + const badgeCount = totalApprovals + activeAgents.length; + + return ( + + {/* Header */} + + + {headerTitle} + + {badgeCount > 0 && ( + + {badgeCount} + + )} + {!hasApprovals && ( + { e.stopPropagation(); onCollapse(); }} + sx={{ p: 0.25, color: c.text.ghost, '&:hover': { color: c.text.tertiary } }} + > + + + )} + + + {/* Content */} + + {hasApprovals && ( + + {hasAgents && ( + + Approvals + + )} + {groups.map((group) => ( + + {groups.length > 1 && ( + + {group.sessionName} + + )} + {group.approvals.length > 1 ? ( + + ) : ( + group.approvals.map((req) => ( + + )) + )} + + ))} + + )} + + {hasApprovals && hasAgents && ( + + )} + + {hasAgents && ( + + {hasApprovals && ( + + Agents + + )} + {activeAgents.map((agent) => ( + + ))} + {finishedAgents.length > 0 && ( + <> + {activeAgents.length > 0 && ( + + )} + setCompletedExpanded((v) => !v)} + sx={{ + display: 'flex', + alignItems: 'center', + gap: 1, + px: 2, + py: 0.5, + cursor: 'pointer', + userSelect: 'none', + '&:hover': { bgcolor: c.border.subtle }, + transition: 'background-color 0.15s', + }} + > + + Completed ({finishedAgents.length}) + + { e.stopPropagation(); onClearAllFinished(); }} + sx={{ + fontSize: '0.58rem', + fontWeight: 600, + color: c.text.ghost, + cursor: 'pointer', + '&:hover': { color: c.text.secondary }, + transition: 'color 0.15s', + }} + > + Clear all + + + {completedExpanded + ? + : } + + + + {finishedAgents.map((agent) => ( + + ))} + + + )} + + )} + + + ); +}; + +export default DynamicIsland; diff --git a/frontend/src/app/components/ElementSelectionContext.tsx b/frontend/src/app/components/ElementSelectionContext.tsx index d42e5d1f..5db25ba1 100644 --- a/frontend/src/app/components/ElementSelectionContext.tsx +++ b/frontend/src/app/components/ElementSelectionContext.tsx @@ -1,4 +1,4 @@ -import React, { createContext, useContext, useState, useRef, useCallback, RefObject } from 'react'; +import React, { createContext, useContext, useState, useRef, useCallback, useMemo, RefObject } from 'react'; export interface SelectedElement { id: string; @@ -20,11 +20,17 @@ interface ElementSelectionContextValue { setSelectMode: (active: boolean) => void; excludeSelectId: string | null; setExcludeSelectId: (id: string | null) => void; + activeOwnerId: string | null; + setActiveOwnerId: (id: string | null) => void; selectedElements: SelectedElement[]; addSelectedElement: (el: SelectedElement) => void; updateSelectedElement: (id: string, patch: Partial) => void; removeSelectedElement: (id: string) => void; clearSelectedElements: () => void; + elementsByOwner: Record; + addElementForOwner: (ownerId: string, el: SelectedElement) => void; + removeOwnerElement: (ownerId: string, elementId: string) => void; + clearOwnerElements: (ownerId: string) => void; iframeRef: RefObject; } @@ -37,9 +43,18 @@ export function useElementSelection() { export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => { const [selectMode, setSelectMode] = useState(false); const [excludeSelectId, setExcludeSelectId] = useState(null); - const [selectedElements, setSelectedElements] = useState([]); + const [activeOwnerId, setActiveOwnerId] = useState(null); + const [elementsByOwner, setElementsByOwner] = useState>({}); const iframeRef = useRef(null); + const activeOwnerIdRef = useRef(activeOwnerId); + activeOwnerIdRef.current = activeOwnerId; + + const selectedElements = useMemo( + () => (activeOwnerId ? elementsByOwner[activeOwnerId] ?? [] : []), + [activeOwnerId, elementsByOwner], + ); + const toggleSelectMode = useCallback(() => { setSelectMode((prev) => { if (prev) setExcludeSelectId(null); @@ -48,22 +63,65 @@ export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = }, []); const addSelectedElement = useCallback((el: SelectedElement) => { - setSelectedElements((prev) => { - if (prev.some((e) => e.id === el.id)) return prev; - return [...prev, el]; + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + const existing = prev[ownerId] ?? []; + if (existing.some((e) => e.id === el.id)) return prev; + return { ...prev, [ownerId]: [...existing, el] }; }); }, []); const updateSelectedElement = useCallback((id: string, patch: Partial) => { - setSelectedElements((prev) => prev.map((e) => e.id === id ? { ...e, ...patch } : e)); + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + const existing = prev[ownerId]; + if (!existing) return prev; + return { ...prev, [ownerId]: existing.map((e) => (e.id === id ? { ...e, ...patch } : e)) }; + }); }, []); const removeSelectedElement = useCallback((id: string) => { - setSelectedElements((prev) => prev.filter((e) => e.id !== id)); + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + const existing = prev[ownerId]; + if (!existing) return prev; + return { ...prev, [ownerId]: existing.filter((e) => e.id !== id) }; + }); }, []); const clearSelectedElements = useCallback(() => { - setSelectedElements([]); + const ownerId = activeOwnerIdRef.current; + if (!ownerId) return; + setElementsByOwner((prev) => { + if (!prev[ownerId]?.length) return prev; + return { ...prev, [ownerId]: [] }; + }); + }, []); + + const addElementForOwner = useCallback((ownerId: string, el: SelectedElement) => { + setElementsByOwner((prev) => { + const existing = prev[ownerId] ?? []; + if (existing.some((e) => e.semanticData?.selectId === el.semanticData?.selectId)) return prev; + return { ...prev, [ownerId]: [...existing, el] }; + }); + }, []); + + const removeOwnerElement = useCallback((ownerId: string, elementId: string) => { + setElementsByOwner((prev) => { + const existing = prev[ownerId]; + if (!existing) return prev; + return { ...prev, [ownerId]: existing.filter((e) => e.id !== elementId) }; + }); + }, []); + + const clearOwnerElements = useCallback((ownerId: string) => { + setElementsByOwner((prev) => { + if (!prev[ownerId]?.length) return prev; + return { ...prev, [ownerId]: [] }; + }); }, []); return ( @@ -74,11 +132,17 @@ export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = setSelectMode, excludeSelectId, setExcludeSelectId, + activeOwnerId, + setActiveOwnerId, selectedElements, addSelectedElement, updateSelectedElement, removeSelectedElement, clearSelectedElements, + elementsByOwner, + addElementForOwner, + removeOwnerElement, + clearOwnerElements, iframeRef, }} > diff --git a/frontend/src/app/components/GlobalApprovalOverlay.tsx b/frontend/src/app/components/GlobalApprovalOverlay.tsx deleted file mode 100644 index 4405e7bf..00000000 --- a/frontend/src/app/components/GlobalApprovalOverlay.tsx +++ /dev/null @@ -1,203 +0,0 @@ -import React, { useMemo, useCallback, useState, useEffect } from 'react'; -import Box from '@mui/material/Box'; -import Typography from '@mui/material/Typography'; -import Chip from '@mui/material/Chip'; -import IconButton from '@mui/material/IconButton'; -import ExpandLessIcon from '@mui/icons-material/ExpandLess'; -import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; -import NotificationsActiveIcon from '@mui/icons-material/NotificationsActive'; -import { useAppDispatch, useAppSelector } from '@/shared/hooks'; -import { handleApproval, ApprovalRequest } from '@/shared/state/agentsSlice'; -import ApprovalBar, { BatchApprovalBar } from '@/app/pages/AgentChat/ApprovalBar'; -import { useClaudeTokens } from '@/shared/styles/ThemeContext'; - -interface SessionApprovalGroup { - sessionId: string; - sessionName: string; - approvals: ApprovalRequest[]; -} - -const GlobalApprovalOverlay: React.FC = () => { - const c = useClaudeTokens(); - const dispatch = useAppDispatch(); - const sessions = useAppSelector((state) => state.agents.sessions); - const [collapsed, setCollapsed] = useState(false); - - const groups: SessionApprovalGroup[] = useMemo(() => { - const result: SessionApprovalGroup[] = []; - for (const [sessionId, session] of Object.entries(sessions)) { - if (session.pending_approvals.length > 0) { - result.push({ - sessionId, - sessionName: session.name || 'Agent', - approvals: session.pending_approvals, - }); - } - } - return result; - }, [sessions]); - - const totalApprovals = useMemo( - () => groups.reduce((sum, g) => sum + g.approvals.length, 0), - [groups], - ); - - useEffect(() => { - if (totalApprovals > 0) { - setCollapsed(false); - } - }, [totalApprovals]); - - const onApprove = useCallback( - (requestId: string, updatedInput?: Record) => { - dispatch(handleApproval({ requestId, behavior: 'allow', updatedInput })); - }, - [dispatch], - ); - - const onDeny = useCallback( - (requestId: string, message?: string) => { - dispatch(handleApproval({ requestId, behavior: 'deny', message })); - }, - [dispatch], - ); - - if (totalApprovals === 0) return null; - - return ( - - {/* Header */} - setCollapsed((v) => !v)} - sx={{ - display: 'flex', - alignItems: 'center', - gap: 1, - px: 2, - py: 1.25, - bgcolor: c.status.warningBg, - borderBottom: collapsed ? 'none' : `1px solid ${c.status.warning}20`, - cursor: 'pointer', - userSelect: 'none', - '&:hover': { bgcolor: `${c.status.warning}18` }, - transition: 'background-color 0.15s', - }} - > - - - Approval Required - - - - {collapsed ? : } - - - - {/* Content */} - {!collapsed && ( - - {groups.map((group) => ( - - {groups.length > 1 && ( - - {group.sessionName} - - )} - {group.approvals.length > 1 ? ( - - ) : ( - group.approvals.map((req) => ( - - )) - )} - - ))} - - )} - - ); -}; - -export default GlobalApprovalOverlay; diff --git a/frontend/src/app/components/KeyboardShortcutsHelp.tsx b/frontend/src/app/components/KeyboardShortcutsHelp.tsx index 1447a635..23e9aa29 100644 --- a/frontend/src/app/components/KeyboardShortcutsHelp.tsx +++ b/frontend/src/app/components/KeyboardShortcutsHelp.tsx @@ -11,6 +11,7 @@ const shortcuts = [ { key: 't', description: 'Go to Templates' }, { key: '1-9', description: 'Open agent by position' }, { key: '⌘M', description: 'Add App' }, + { key: '⌘N', description: 'New Browser' }, { key: '⌘O', description: 'History' }, { key: 'Shift+A', description: 'Approve all pending' }, { key: 'Shift+D', description: 'Deny all pending' }, diff --git a/frontend/src/app/components/Layout/AppShell.tsx b/frontend/src/app/components/Layout/AppShell.tsx index 2d4edc02..59776b65 100644 --- a/frontend/src/app/components/Layout/AppShell.tsx +++ b/frontend/src/app/components/Layout/AppShell.tsx @@ -27,17 +27,24 @@ import ViewSidebarOutlinedIcon from '@mui/icons-material/ViewSidebarOutlined'; import ArrowBackOutlinedIcon from '@mui/icons-material/ArrowBackOutlined'; import ArrowForwardOutlinedIcon from '@mui/icons-material/ArrowForwardOutlined'; import RestartAltIcon from '@mui/icons-material/RestartAlt'; +import SystemUpdateAltIcon from '@mui/icons-material/SystemUpdateAlt'; +import CloseIcon from '@mui/icons-material/Close'; +import LinearProgress from '@mui/material/LinearProgress'; import Settings from '@/app/pages/Settings/Settings'; -import GlobalApprovalOverlay from '@/app/components/GlobalApprovalOverlay'; +import DynamicIsland from '@/app/components/DynamicIsland'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { fetchDashboards, createDashboard, renameDashboard } from '@/shared/state/dashboardsSlice'; +import { addBrowserCard, addBrowserTab } from '@/shared/state/dashboardLayoutSlice'; +import { setPendingBrowserUrl } from '@/shared/state/tempStateSlice'; import { fetchOutputs } from '@/shared/state/outputsSlice'; +import { findBrowserByWebContentsId } from '@/shared/browserRegistry'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; const SIDEBAR_MIN = 160; const SIDEBAR_MAX = 400; const SIDEBAR_DEFAULT = 220; const SIDEBAR_WIDTH_KEY = 'openswarm-sidebar-width'; +const UPDATE_DISMISS_KEY = 'openswarm-update-dismissed'; const CUSTOMIZATION_ITEMS = [ { label: 'Prompts', path: '/templates', icon: }, @@ -75,10 +82,34 @@ const AppShell: React.FC = () => { const updateStatus = useAppSelector((state) => state.update.status); const availableVersion = useAppSelector((state) => state.update.availableVersion); - const [updateBannerDismissed, setUpdateBannerDismissed] = useState(false); + const downloadPercent = useAppSelector((state) => state.update.downloadPercent); - const showUpdateDot = updateStatus === 'available' || updateStatus === 'downloaded'; - const showUpdateBanner = updateStatus === 'downloaded' && !updateBannerDismissed; + const [dismissedVersion, setDismissedVersion] = useState(() => { + try { return localStorage.getItem(UPDATE_DISMISS_KEY); } catch { return null; } + }); + const [snackbarDismissed, setSnackbarDismissed] = useState(false); + + const bannerDismissedForVersion = availableVersion != null && dismissedVersion === availableVersion; + const isUpdateActionable = updateStatus === 'available' || updateStatus === 'downloaded' || updateStatus === 'downloading'; + + const showUpdateDot = (updateStatus === 'available' || updateStatus === 'downloaded') && !bannerDismissedForVersion; + const showUpdateBanner = isUpdateActionable && !bannerDismissedForVersion; + const showUpdateSnackbar = (updateStatus === 'available' || updateStatus === 'downloaded') && !bannerDismissedForVersion && !snackbarDismissed; + + const handleDismissBanner = useCallback(() => { + if (availableVersion) { + try { localStorage.setItem(UPDATE_DISMISS_KEY, availableVersion); } catch {} + setDismissedVersion(availableVersion); + } + }, [availableVersion]); + + const handleDownloadUpdate = useCallback(async () => { + try { await (window as any).openswarm?.downloadUpdate(); } catch {} + }, []); + + const handleInstallUpdate = useCallback(() => { + (window as any).openswarm?.installUpdate(); + }, []); const dashboardItems = useAppSelector((state) => state.dashboards.items); const dashboardList = Object.values(dashboardItems).sort( @@ -95,6 +126,75 @@ const AppShell: React.FC = () => { dispatch(fetchOutputs()); }, [dispatch]); + const openUrlInBrowser = useCallback((url: string, webContentsId?: number) => { + const dashMatch = location.pathname.match(/^\/dashboard\/(.+)/); + if (dashMatch) { + if (webContentsId != null) { + const browserId = findBrowserByWebContentsId(webContentsId); + if (browserId) { + dispatch(addBrowserTab({ browserId, url, makeActive: true })); + return; + } + } + dispatch(addBrowserCard({ url })); + } else { + dispatch(setPendingBrowserUrl(url)); + const lastId = (window as any).__openswarm_last_dashboard_id as string | undefined; + const firstDashboard = dashboardList[0]; + const targetId = lastId || firstDashboard?.id; + if (targetId) { + navigate(`/dashboard/${targetId}`); + } else { + dispatch(createDashboard('Untitled Dashboard')).then((result: any) => { + if (createDashboard.fulfilled.match(result)) { + navigate(`/dashboard/${result.payload.id}`); + } + }); + } + } + }, [location.pathname, dashboardList, dispatch, navigate]); + + useEffect(() => { + let lastUrl = ''; + let lastTime = 0; + + const handleClick = (e: MouseEvent) => { + const anchor = (e.target as HTMLElement)?.closest?.('a'); + if (!anchor) return; + const href = anchor.getAttribute('href'); + if (!href) return; + if (!/^https?:\/\//i.test(href)) return; + if (href.startsWith('http://localhost:')) return; + + e.preventDefault(); + e.stopPropagation(); + + const now = Date.now(); + if (href === lastUrl && now - lastTime < 1000) return; + lastUrl = href; + lastTime = now; + + openUrlInBrowser(href); + }; + + document.addEventListener('click', handleClick, true); + return () => document.removeEventListener('click', handleClick, true); + }, [openUrlInBrowser]); + + useEffect(() => { + const w = window as any; + if (!w.openswarm?.onWebviewNewWindow) return; + let lastUrl = ''; + let lastTime = 0; + return w.openswarm.onWebviewNewWindow((url: string, webContentsId: number) => { + const now = Date.now(); + if (url === lastUrl && now - lastTime < 1000) return; + lastUrl = url; + lastTime = now; + openUrlInBrowser(url, webContentsId); + }); + }, [openUrlInBrowser]); + useEffect(() => { try { localStorage.setItem(SIDEBAR_WIDTH_KEY, String(sidebarWidth)); } catch {} }, [sidebarWidth]); @@ -196,6 +296,8 @@ const AppShell: React.FC = () => { borderBottom: `0.5px solid ${c.border.medium}`, display: 'flex', alignItems: 'center', + position: 'relative', + overflow: 'visible', WebkitAppRegion: 'drag', userSelect: 'none', pl: '78px', @@ -248,6 +350,8 @@ const AppShell: React.FC = () => { + + { component="img" src="./logo.png" alt="OpenSwarm" - sx={{ width: 18, height: 18, borderRadius: 0.5, opacity: 0.7 }} + sx={{ width: 16, height: 16, borderRadius: 0.5, opacity: 0.6 }} /> { + {showUpdateBanner && ( + + + + {updateStatus === 'available' && `OpenSwarm ${availableVersion} is available`} + {updateStatus === 'downloading' && `Downloading OpenSwarm ${availableVersion}…`} + {updateStatus === 'downloaded' && `OpenSwarm ${availableVersion} is ready to install`} + + {updateStatus === 'downloading' && ( + + )} + {updateStatus === 'downloading' && ( + + {Math.round(downloadPercent)}% + + )} + {updateStatus === 'available' && ( + + )} + {updateStatus === 'downloaded' && ( + + )} + + + + + )} + {!sidebarCollapsed && ( <> @@ -732,39 +928,62 @@ const AppShell: React.FC = () => { - setSnackbarDismissed(true)} anchorOrigin={{ vertical: 'bottom', horizontal: 'center' }} > } + icon={updateStatus === 'downloaded' + ? + : + } action={ - + {updateStatus === 'available' && ( + + )} + {updateStatus === 'downloaded' && ( + + )} } sx={{ @@ -775,7 +994,8 @@ const AppShell: React.FC = () => { '& .MuiAlert-icon': { color: c.accent.primary }, }} > - OpenSwarm {availableVersion} downloaded — restart to update + {updateStatus === 'available' && `OpenSwarm ${availableVersion} is available`} + {updateStatus === 'downloaded' && `OpenSwarm ${availableVersion} downloaded — restart to update`} diff --git a/frontend/src/app/components/OnboardingModal.tsx b/frontend/src/app/components/OnboardingModal.tsx new file mode 100644 index 00000000..d046d3ac --- /dev/null +++ b/frontend/src/app/components/OnboardingModal.tsx @@ -0,0 +1,222 @@ +import React, { useState, useEffect } from 'react'; +import { Box, Typography, Modal, Button } from '@mui/material'; +import { useAppSelector } from '@/shared/hooks'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { API_BASE } from '@/shared/config'; + +const SUBSCRIPTION_PROVIDERS = [ + { id: 'claude', name: 'Claude', desc: 'Sonnet, Opus, Haiku', color: '#E8927A', preview: false }, + { id: 'gemini-cli', name: 'Gemini', desc: 'Gemini 2.5 Pro & Flash', color: '#4285F4', preview: true }, + { id: 'codex', name: 'ChatGPT', desc: 'GPT-5.4, o3, o4-mini', color: '#74AA9C', preview: true }, + { id: 'github', name: 'GitHub Copilot', desc: 'Claude + GPT models', color: '#8B949E', preview: true }, +]; + +const OnboardingModal: React.FC = () => { + const c = useClaudeTokens(); + const settings = useAppSelector((s) => s.settings); + const [open, setOpen] = useState(false); + const [dismissed, setDismissed] = useState(false); + const [connecting, setConnecting] = useState(null); + const [nineRouterStatus, setNineRouterStatus] = useState(null); + + // Check if user has any credentials configured + const hasAnyKey = !!( + settings.anthropic_api_key || + settings.openai_api_key || + settings.google_api_key || + settings.openrouter_api_key + ); + + // Check 9Router subscription status + useEffect(() => { + fetch(`${API_BASE}/agents/subscriptions/status`) + .then((r) => r.json()) + .then(setNineRouterStatus) + .catch(() => setNineRouterStatus(null)); + }, []); + + const hasSubscription = (() => { + if (!nineRouterStatus?.running) return false; + const connections = nineRouterStatus?.providers?.connections || []; + return connections.some((p: any) => p.isActive); + })(); + + // Show modal if no keys AND no subscriptions AND not dismissed + useEffect(() => { + if (!hasAnyKey && !hasSubscription && !dismissed && nineRouterStatus !== null) { + setOpen(true); + } else { + setOpen(false); + } + }, [hasAnyKey, hasSubscription, dismissed, nineRouterStatus]); + + const handleConnect = async (providerId: string) => { + setConnecting(providerId); + try { + const r = await fetch(`${API_BASE}/agents/subscriptions/connect`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ provider: providerId }), + }); + const data = await r.json(); + + if (data.flow === 'device_code') { + const verifyUrl = data.verification_uri; + if (verifyUrl) window.open(verifyUrl, '_blank'); + // Poll for completion + const timer = setInterval(async () => { + try { + const pr = await fetch(`${API_BASE}/agents/subscriptions/poll`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ provider: providerId, device_code: data.device_code, code_verifier: data.code_verifier, extra_data: data.extra_data }), + }); + const pd = await pr.json(); + if (pd.success) { + clearInterval(timer); + setConnecting(null); + setOpen(false); + } + } catch {} + }, 5000); + setTimeout(() => { clearInterval(timer); setConnecting(null); }, 300000); + } else if (data.flow === 'authorization_code') { + // Open auth URL as popup — window.opener lets callback page postMessage back + const popup = window.open(data.auth_url, 'oauth_connect', 'width=600,height=700'); + + // Listen for postMessage from 9Router's callback page + // 9Router sends: { type: "oauth_callback", data: { code, state, ... } } + const msgHandler = async (event: MessageEvent) => { + const d = event.data; + const callbackData = d?.type === 'oauth_callback' ? d.data : d; + if (callbackData?.code) { + window.removeEventListener('message', msgHandler); + clearInterval(statusPoller); + if (popup && !popup.closed) popup.close(); + try { + await fetch(`${API_BASE}/agents/subscriptions/exchange`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + provider: providerId, code: callbackData.code, + redirect_uri: data.redirect_uri, code_verifier: data.code_verifier, + state: callbackData.state || data.state, + }), + }); + } catch {} + setConnecting(null); + setOpen(false); + } + }; + window.addEventListener('message', msgHandler); + + // Also poll status as fallback (in case postMessage doesn't work in Electron) + const statusPoller = setInterval(async () => { + try { + const sr = await fetch(`${API_BASE}/agents/subscriptions/status`); + const sd = await sr.json(); + const conns = sd.providers?.connections || []; + if (conns.some((p: any) => p.provider === providerId && p.isActive)) { + clearInterval(statusPoller); + window.removeEventListener('message', msgHandler); + setConnecting(null); + setOpen(false); + } + } catch {} + }, 2000); + setTimeout(() => { clearInterval(statusPoller); window.removeEventListener('message', msgHandler); setConnecting(null); }, 300000); + } + } catch { + setConnecting(null); + } + }; + + const handleApiKey = () => { + setDismissed(true); + setOpen(false); + // User will manually go to Settings → Models to add API keys + }; + + const handleSkip = () => { + setDismissed(true); + setOpen(false); + }; + + if (!open) return null; + + return ( + + + + Welcome to OpenSwarm + + + Connect an AI model to get started + + + {/* Subscription options */} + + Use your existing subscription + + + {SUBSCRIPTION_PROVIDERS.map((p) => ( + !p.preview && !connecting && handleConnect(p.id)} + sx={{ + display: 'flex', alignItems: 'center', justifyContent: 'space-between', + p: 1.5, borderRadius: `${c.radius.md}px`, border: `1px solid ${c.border.subtle}`, + cursor: p.preview ? 'default' : connecting ? 'wait' : 'pointer', + opacity: p.preview ? 0.5 : 1, + transition: 'border-color 0.15s, background 0.15s', + ...(!p.preview && { '&:hover': { borderColor: c.border.medium, bgcolor: `${c.accent.primary}05` } }), + }} + > + + {p.name} + {p.desc} + + + {p.preview ? 'Coming soon' : connecting === p.id ? 'Connecting...' : 'Connect →'} + + + ))} + + + {/* API key option */} + + Or use an API key + + + + I have an API key + + + Go to Settings → Models to enter your key + + + + {/* Skip */} + + + + ); +}; + +export default OnboardingModal; diff --git a/frontend/src/app/components/TalkModeOverlay.tsx b/frontend/src/app/components/TalkModeOverlay.tsx new file mode 100644 index 00000000..808e6d69 --- /dev/null +++ b/frontend/src/app/components/TalkModeOverlay.tsx @@ -0,0 +1,526 @@ +import React, { useEffect, useRef, useState, useCallback } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import IconButton from '@mui/material/IconButton'; +import MicIcon from '@mui/icons-material/Mic'; +import MicOffIcon from '@mui/icons-material/MicOff'; +import VolumeUpIcon from '@mui/icons-material/VolumeUp'; +import CloseIcon from '@mui/icons-material/Close'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { WS_BASE } from '@/shared/config'; + +type FaceState = 'idle' | 'happy' | 'thinking' | 'talking' | 'surprised' | 'sleeping' | 'angry' | 'love'; +type TalkStatus = 'idle' | 'listening' | 'processing' | 'speaking'; + +interface Props { + open: boolean; + onClose: () => void; + sessionId?: string; +} + +// ─── Pixel Face Canvas Renderer ────────────────────────────────── +// Ported from face.html — all the draw logic in one hook. + +function usePixelFace( + canvasRef: React.RefObject, + faceState: FaceState, + size: number, +) { + const stateRef = useRef('idle'); + const animRef = useRef(0); + + useEffect(() => { + stateRef.current = faceState; + }, [faceState]); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) return; + + const ctx = canvas.getContext('2d'); + if (!ctx) return; + + const PX = Math.max(4, Math.floor(size / 30)); + const COLS = Math.ceil(size / PX); + const ROWS = Math.ceil(size / PX); + canvas.width = COLS * PX; + canvas.height = ROWS * PX; + + const BG = '#E8927A'; + const EYE = '#1E1E1E'; + const MOUTH = '#1E1E1E'; + + let breath = 0, talk = 0, think = 0, sleepZ = 0, heartP = 0; + let eyeH = 3, eyeHTarget = 3; + let mouthW = 2, mouthWTarget = 2; + let mouthH = 2, mouthHTarget = 2; + let eyeOffX = 0, eyeOffXTarget = 0; + let eyeOffY = 0, eyeOffYTarget = 0; + let blinkOpen = true, blinkCD = 120 + Math.random() * 200; + let doubleBlink = false; + let idleSinceInput = 0, sleepTransitioned = false; + let idleAction = 'none', idleActionTimer = 0; + let idleGlanceX = 0, idleGlanceY = 0; + + const lerp = (a: number, b: number, t: number) => a + (b - a) * t; + + const px = (col: number, row: number, color: string) => { + ctx.fillStyle = color; + ctx.fillRect(col * PX, row * PX, PX, PX); + }; + + const pxRect = (x: number, y: number, w: number, h: number, color: string) => { + for (let r = 0; r < Math.round(h); r++) + for (let c = 0; c < Math.round(w); c++) + px(Math.round(x) + c, Math.round(y) + r, color); + }; + + function draw() { + const state = stateRef.current; + + breath += 0.025; + talk += 0.3; + think += 0.025; + sleepZ += 0.012; + heartP += 0.05; + idleSinceInput++; + + if (idleSinceInput > 2700 && state === 'idle' && !sleepTransitioned) { + stateRef.current = 'sleeping'; + sleepTransitioned = true; + } + + if (state === 'idle') { + idleActionTimer--; + blinkCD--; + if (blinkCD <= 0 && blinkOpen) { blinkOpen = false; blinkCD = 6; doubleBlink = Math.random() < 0.3; } + else if (!blinkOpen && blinkCD <= 0) { blinkOpen = true; blinkCD = doubleBlink ? 8 : 100 + Math.random() * 280; doubleBlink = false; } + + if (idleActionTimer <= 0) { + const roll = Math.random(); + if (roll < 0.3) { idleAction = 'glance'; idleGlanceX = Math.floor(Math.random() * 5) - 2; idleGlanceY = (Math.random() - 0.5) * 1.2; idleActionTimer = 50 + Math.random() * 120; } + else if (roll < 0.45) { idleAction = 'scan'; idleActionTimer = 180; } + else if (roll < 0.55) { idleAction = 'squint'; idleActionTimer = 35 + Math.random() * 40; } + else if (roll < 0.65) { idleAction = 'lookup'; idleActionTimer = 50 + Math.random() * 70; } + else { idleAction = 'none'; idleGlanceX = 0; idleGlanceY = 0; idleActionTimer = 60 + Math.random() * 200; } + } + } + + if (state === 'talking' || state === 'thinking') { + blinkCD--; + if (blinkCD <= 0 && blinkOpen) { blinkOpen = false; blinkCD = 6; } + else if (!blinkOpen && blinkCD <= 0) { blinkOpen = true; blinkCD = 120 + Math.random() * 250; } + } + if (state !== 'idle' && state !== 'talking' && state !== 'thinking') blinkOpen = true; + + const b = Math.sin(breath) * 0.3; + switch (state) { + case 'idle': { + eyeHTarget = blinkOpen ? 3 : 0; mouthWTarget = 2; mouthHTarget = 2; + let gx = 0, gy = b; + if (idleAction === 'glance') { gx = idleGlanceX; gy = idleGlanceY + b; } + else if (idleAction === 'scan') { const t = 1 - (idleActionTimer / 180); gx = Math.sin(t * Math.PI * 2) * 2.5; } + else if (idleAction === 'squint') { eyeHTarget = blinkOpen ? 2 : 0; gy = b + 0.3; } + else if (idleAction === 'lookup') { gy = -1.2 + b; } + eyeOffXTarget = gx; eyeOffYTarget = gy; break; + } + case 'happy': eyeHTarget = 1; mouthWTarget = 6; mouthHTarget = 1; eyeOffXTarget = 0; eyeOffYTarget = b; break; + case 'thinking': eyeHTarget = blinkOpen ? 3 : 0; mouthWTarget = 2; mouthHTarget = 2; eyeOffXTarget = 2; eyeOffYTarget = b; break; + case 'talking': { eyeHTarget = blinkOpen ? 3 : 0; const open = Math.round(Math.abs(Math.sin(talk)) * 2 + 1); mouthWTarget = 4; mouthHTarget = open; eyeOffXTarget = 0; eyeOffYTarget = b; break; } + case 'surprised': eyeHTarget = 4; mouthWTarget = 3; mouthHTarget = 3; eyeOffXTarget = 0; eyeOffYTarget = b; break; + case 'sleeping': eyeHTarget = 1; mouthWTarget = 2; mouthHTarget = 1; eyeOffXTarget = 0; eyeOffYTarget = b * 2; break; + case 'angry': eyeHTarget = 2; mouthWTarget = 6; mouthHTarget = 1; eyeOffXTarget = 0; eyeOffYTarget = b * 0.3; break; + case 'love': eyeHTarget = 3; mouthWTarget = 2; mouthHTarget = 2; eyeOffXTarget = 0; eyeOffYTarget = b; break; + } + + eyeH = lerp(eyeH, eyeHTarget, 0.18); + mouthW = lerp(mouthW, mouthWTarget, 0.15); + mouthH = lerp(mouthH, mouthHTarget, 0.2); + eyeOffX = lerp(eyeOffX, eyeOffXTarget, 0.1); + eyeOffY = lerp(eyeOffY, eyeOffYTarget, 0.15); + + // Draw + ctx.fillStyle = BG; + ctx.fillRect(0, 0, canvas.width, canvas.height); + + const cx = Math.floor(COLS / 2); + const cy = Math.floor(ROWS / 2); + const eyeSpread = 5, eyeW = 3; + const eh = Math.max(1, Math.round(eyeH)); + const eOffX = Math.round(eyeOffX), eOffY = Math.round(eyeOffY); + const eyeBaseY = cy - 2; + const blinkOff = Math.round((3 - eh) / 2); + + if (state === 'love') { + const pulse = Math.sin(heartP) > 0 ? '#CC2244' : '#BB1E3E'; + const heart = (hx: number, hy: number) => { + px(hx - 1, hy, pulse); px(hx + 1, hy, pulse); + px(hx - 2, hy + 1, pulse); px(hx - 1, hy + 1, pulse); px(hx, hy + 1, pulse); px(hx + 1, hy + 1, pulse); px(hx + 2, hy + 1, pulse); + px(hx - 1, hy + 2, pulse); px(hx, hy + 2, pulse); px(hx + 1, hy + 2, pulse); + px(hx, hy + 3, pulse); + }; + heart(cx - eyeSpread + eOffX, eyeBaseY + eOffY); + heart(cx + eyeSpread + eOffX, eyeBaseY + eOffY); + } else { + pxRect(cx - eyeSpread - 1 + eOffX, eyeBaseY + blinkOff + eOffY, eyeW, eh, EYE); + pxRect(cx + eyeSpread - 1 + eOffX, eyeBaseY + blinkOff + eOffY, eyeW, eh, EYE); + } + + if (state === 'angry') { + const lx = cx - eyeSpread - 1 + eOffX, ly = eyeBaseY + blinkOff + eOffY - 2; + px(lx, ly + 1, EYE); px(lx + 1, ly, EYE); px(lx + 2, ly, EYE); + const rx = cx + eyeSpread - 1 + eOffX; + px(rx + 2, ly + 1, EYE); px(rx + 1, ly, EYE); px(rx, ly, EYE); + } + + const mw = Math.max(1, Math.round(mouthW)), mh = Math.max(1, Math.round(mouthH)); + const mouthY = cy + 4 + Math.round(eyeOffY); + + if (state === 'happy') { + pxRect(cx - Math.floor(mw / 2), mouthY, mw, 1, MOUTH); + px(cx - Math.floor(mw / 2), mouthY - 1, MOUTH); + px(cx - Math.floor(mw / 2) + mw - 1, mouthY - 1, MOUTH); + } else if (state === 'angry') { + pxRect(cx - Math.floor(mw / 2), mouthY, mw, 1, MOUTH); + px(cx - Math.floor(mw / 2), mouthY + 1, MOUTH); + px(cx - Math.floor(mw / 2) + mw - 1, mouthY + 1, MOUTH); + } else { + pxRect(cx - Math.floor(mw / 2), mouthY, mw, mh, MOUTH); + } + + if (state === 'sleeping') { + const zFrame = Math.floor(sleepZ * 60) % 90; + const zy = Math.round(cy - 5 - (zFrame / 90) * 4); + const zx = cx + eyeSpread + 3; + if (zy >= 1 && zFrame < 70) { + px(zx, zy, '#5577CC'); px(zx + 1, zy, '#5577CC'); + px(zx + 1, zy + 1, '#5577CC'); + px(zx, zy + 2, '#5577CC'); px(zx + 1, zy + 2, '#5577CC'); + } + } + + if (state === 'thinking') { + const phase = Math.floor(think * 10) % 4; + const dx = cx + eyeSpread + 3, dy = cy - 5; + if (phase >= 1) px(dx, dy, '#7799DD'); + if (phase >= 2) px(dx + 1, dy - 1, '#7799DD'); + if (phase >= 3) px(dx + 2, dy - 2, '#7799DD'); + } + + animRef.current = requestAnimationFrame(draw); + } + + animRef.current = requestAnimationFrame(draw); + return () => cancelAnimationFrame(animRef.current); + }, [canvasRef, size]); +} + +// ─── Status Labels ─────────────────────────────────────────────── + +const STATUS_LABELS: Record = { + idle: 'Tap to speak', + listening: 'Listening...', + processing: 'Thinking...', + speaking: 'Speaking...', +}; + +// ─── Main Component ───────────────────────────────────────────── + +const TalkModeOverlay: React.FC = ({ open, onClose, sessionId }) => { + const c = useClaudeTokens(); + const canvasRef = useRef(null); + const [faceState, setFaceState] = useState('idle'); + const [talkStatus, setTalkStatus] = useState('idle'); + const [transcript, setTranscript] = useState(''); + const [agentResponse, setAgentResponse] = useState(''); + const wsRef = useRef(null); + const mediaRecorderRef = useRef(null); + const silenceTimerRef = useRef(0); + const audioContextRef = useRef(null); + + usePixelFace(canvasRef, faceState, 240); + + // Map talk status to face state + useEffect(() => { + switch (talkStatus) { + case 'idle': setFaceState('idle'); break; + case 'listening': setFaceState('idle'); break; + case 'processing': setFaceState('thinking'); break; + case 'speaking': setFaceState('talking'); break; + } + }, [talkStatus]); + + // WebSocket connection for talk mode + useEffect(() => { + if (!open || !sessionId) return; + + const ws = new WebSocket(`${WS_BASE}/ws/talk/${sessionId}`); + wsRef.current = ws; + + ws.onopen = () => { + ws.send(JSON.stringify({ type: 'config', stt: {}, tts: {} })); + }; + + ws.onmessage = (event) => { + const msg = JSON.parse(event.data); + + switch (msg.type) { + case 'status': + if (msg.status === 'listening') setTalkStatus('idle'); + else if (msg.status === 'processing') setTalkStatus('processing'); + else if (msg.status === 'speaking') setTalkStatus('speaking'); + break; + + case 'transcript': + setTranscript(msg.text); + break; + + case 'agent_response': + setAgentResponse(msg.text); + setFaceState('happy'); + setTimeout(() => setFaceState('idle'), 2000); + break; + + case 'audio': { + const audioData = atob(msg.data); + const audioArray = new Uint8Array(audioData.length); + for (let i = 0; i < audioData.length; i++) audioArray[i] = audioData.charCodeAt(i); + + if (!audioContextRef.current) audioContextRef.current = new AudioContext(); + const audioCtx = audioContextRef.current; + audioCtx.decodeAudioData(audioArray.buffer.slice(0), (buffer) => { + const source = audioCtx.createBufferSource(); + source.buffer = buffer; + source.connect(audioCtx.destination); + source.onended = () => setTalkStatus('idle'); + source.start(0); + setTalkStatus('speaking'); + }); + break; + } + } + }; + + ws.onerror = () => setFaceState('angry'); + ws.onclose = () => {}; + + return () => { + ws.close(); + wsRef.current = null; + }; + }, [open, sessionId]); + + // Keyboard: Esc to close + useEffect(() => { + if (!open) return; + const handler = (e: KeyboardEvent) => { + if (e.key === 'Escape') onClose(); + }; + window.addEventListener('keydown', handler); + return () => window.removeEventListener('keydown', handler); + }, [open, onClose]); + + const startRecording = useCallback(async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const recorder = new MediaRecorder(stream, { mimeType: 'audio/webm' }); + mediaRecorderRef.current = recorder; + + const chunks: Blob[] = []; + recorder.ondataavailable = (e) => { + if (e.data.size > 0) chunks.push(e.data); + }; + + recorder.onstop = async () => { + stream.getTracks().forEach((t) => t.stop()); + const blob = new Blob(chunks, { type: 'audio/webm' }); + const reader = new FileReader(); + reader.onload = () => { + const base64 = (reader.result as string).split(',')[1]; + if (wsRef.current?.readyState === WebSocket.OPEN) { + wsRef.current.send(JSON.stringify({ type: 'audio', data: base64, format: 'webm' })); + wsRef.current.send(JSON.stringify({ type: 'end_utterance', format: 'webm' })); + } + }; + reader.readAsDataURL(blob); + setTalkStatus('processing'); + }; + + recorder.start(); + setTalkStatus('listening'); + setTranscript(''); + setAgentResponse(''); + + // Auto-stop after silence (simple timeout approach) + silenceTimerRef.current = window.setTimeout(() => { + if (mediaRecorderRef.current?.state === 'recording') { + mediaRecorderRef.current.stop(); + } + }, 5000); + } catch { + setFaceState('angry'); + } + }, []); + + const stopRecording = useCallback(() => { + window.clearTimeout(silenceTimerRef.current); + if (mediaRecorderRef.current?.state === 'recording') { + mediaRecorderRef.current.stop(); + } + }, []); + + const handleMicClick = useCallback(() => { + if (talkStatus === 'listening') { + stopRecording(); + } else if (talkStatus === 'idle') { + startRecording(); + } + }, [talkStatus, startRecording, stopRecording]); + + if (!open) return null; + + const micBg = + talkStatus === 'listening' ? c.accent.primary : + talkStatus === 'speaking' ? '#4caf50' : + c.bg.elevated; + + return ( + { if (e.target === e.currentTarget) onClose(); }} + > + {/* Close button */} + + + + + {/* Face canvas */} + + + + + {/* Status label */} + + {STATUS_LABELS[talkStatus]} + + + {/* Transcript area */} + + {transcript && ( + + "{transcript}" + + )} + + {agentResponse && ( + + {agentResponse} + + )} + + + {/* Mic button */} + + {talkStatus === 'listening' ? : + talkStatus === 'speaking' ? : + } + + + {/* Hint */} + + esc to close + + + ); +}; + +export default TalkModeOverlay; diff --git a/frontend/src/app/components/useDomElementSelector.ts b/frontend/src/app/components/useDomElementSelector.ts index b4535e19..8115769b 100644 --- a/frontend/src/app/components/useDomElementSelector.ts +++ b/frontend/src/app/components/useDomElementSelector.ts @@ -120,6 +120,7 @@ export function useDomElementSelector(): DomSelectorState { const dragOriginRef = useRef<{ x: number; y: number } | null>(null); const isDraggingRef = useRef(false); const dragBoundsRef = useRef<{ left: number; top: number; right: number; bottom: number } | null>(null); + const preDragFocusRef = useRef(null); const excludeIdRef = useRef(null); useEffect(() => { @@ -245,8 +246,11 @@ export function useDomElementSelector(): DomSelectorState { if (e.button !== 0) return; if (e.metaKey || e.ctrlKey) return; const target = e.target as Element; - // Only start drag on "empty" canvas areas (not on selectable elements) - if (target && findSelectableAncestor(target, excludeIdRef.current)) return; + if (target && findSelectableAncestor(target, excludeIdRef.current)) { + e.preventDefault(); + return; + } + preDragFocusRef.current = document.activeElement instanceof HTMLElement ? document.activeElement : null; dragOriginRef.current = { x: e.clientX, y: e.clientY }; isDraggingRef.current = false; }, []); @@ -291,12 +295,17 @@ export function useDomElementSelector(): DomSelectorState { }); } + const wasDragging = isDraggingRef.current; dragOriginRef.current = null; isDraggingRef.current = false; dragBoundsRef.current = null; setDragRect(EMPTY_DRAG); setDragPreview([]); if (dragPreviewRafRef.current) cancelAnimationFrame(dragPreviewRafRef.current); + if (wasDragging && preDragFocusRef.current) { + preDragFocusRef.current.focus(); + } + preDragFocusRef.current = null; }, [ctx]); const handleClick = useCallback((e: MouseEvent) => { @@ -327,6 +336,7 @@ export function useDomElementSelector(): DomSelectorState { dragOriginRef.current = null; dragBoundsRef.current = null; isDraggingRef.current = false; + preDragFocusRef.current = null; return; } @@ -353,6 +363,7 @@ export function useDomElementSelector(): DomSelectorState { dragOriginRef.current = null; dragBoundsRef.current = null; isDraggingRef.current = false; + preDragFocusRef.current = null; }; }, [ctx?.selectMode, handleMouseMove, handleMouseDown, handleMouseUp, handleClick]); diff --git a/frontend/src/app/pages/AgentChat/AgentChat.tsx b/frontend/src/app/pages/AgentChat/AgentChat.tsx index 0c5c43c5..28d823db 100644 --- a/frontend/src/app/pages/AgentChat/AgentChat.tsx +++ b/frontend/src/app/pages/AgentChat/AgentChat.tsx @@ -5,8 +5,16 @@ import Typography from '@mui/material/Typography'; import Chip from '@mui/material/Chip'; import IconButton from '@mui/material/IconButton'; import Tooltip from '@mui/material/Tooltip'; +import TextField from '@mui/material/TextField'; +import ClickAwayListener from '@mui/material/ClickAwayListener'; import CloseIcon from '@mui/icons-material/Close'; import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown'; +import KeyboardArrowUpIcon from '@mui/icons-material/KeyboardArrowUp'; +import PlayArrowIcon from '@mui/icons-material/PlayArrow'; +import EditOutlinedIcon from '@mui/icons-material/EditOutlined'; +import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline'; +import CheckIcon from '@mui/icons-material/Check'; +import DragIndicatorIcon from '@mui/icons-material/DragIndicator'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { sendMessage as sendMessageThunk, @@ -17,6 +25,9 @@ import { handleApproval, editMessage, switchBranch, + duplicateSession, + setActiveSession, + updateSessionProvider, updateSessionModel, updateSessionMode, fetchSession, @@ -25,19 +36,19 @@ import { import { fetchModes } from '@/shared/state/modesSlice'; import { createSessionWs } from '@/shared/ws/WebSocketManager'; import MessageBubble from './MessageBubble'; +import MessageActionBar from './MessageActionBar'; import ToolCallBubble, { ToolPair } from './ToolCallBubble'; import ToolGroupBubble, { RenderItem, ToolGroup, isToolGroup, isToolPair } from './ToolGroupBubble'; import ApprovalBar, { BatchApprovalBar } from './ApprovalBar'; import ChatInput, { ChatInputHandle } from './ChatInput'; import { ContextPath } from '@/app/components/DirectoryBrowser'; -import BranchNavigator from './BranchNavigator'; import DiffViewer from './DiffViewer'; -import { setGlowingBrowserCards, clearGlowingBrowserCards } from '@/shared/state/dashboardLayoutSlice'; +import { setGlowingBrowserCards, fadeGlowingBrowserCards, clearGlowingBrowserCards } from '@/shared/state/dashboardLayoutSlice'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; -const CONTEXT_WINDOWS: Record = { - sonnet: 200_000, - opus: 200_000, +const CONTEXT_WINDOWS_DEFAULT: Record = { + sonnet: 1_000_000, + opus: 1_000_000, haiku: 200_000, }; @@ -91,14 +102,27 @@ const ThinkingBubble: React.FC = () => { ); }; +interface QueuedMessage { + prompt: string; + images?: Array<{ data: string; media_type: string }>; + contextPaths?: Array<{ path: string; type: 'file' | 'directory' }>; + forcedTools?: string[]; + attachedSkills?: Array<{ id: string; name: string; content: string }>; + selectedBrowserIds?: string[]; +} + interface AgentChatProps { sessionId?: string; onClose?: () => void; embedded?: boolean; + autoFocus?: boolean; + isGlowing?: boolean; + onDismissGlow?: () => void; initialContextPaths?: ContextPath[]; + onBranch?: (newSessionId: string) => void; } -const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose, embedded, initialContextPaths }) => { +const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose, embedded, autoFocus, isGlowing, onDismissGlow, initialContextPaths, onBranch }) => { const c = useClaudeTokens(); const STATUS_STYLES: Record = { running: { color: c.status.success, bg: c.status.successBg }, @@ -112,15 +136,26 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const dispatch = useAppDispatch(); const session = useAppSelector((state) => (id ? state.agents.sessions[id] : undefined)); const modesMap = useAppSelector((state) => state.modes.items); + const modelsByProvider = useAppSelector((state) => state.models.byProvider); const scrollContainerRef = useRef(null); const chatInputRef = useRef(null); const isAtBottomRef = useRef(true); const [showScrollButton, setShowScrollButton] = useState(false); + const [showResumeBubble, setShowResumeBubble] = useState(false); + const [awaitingResponse, setAwaitingResponse] = useState(false); const [mode, setMode] = useState('agent'); const [model, setModel] = useState('sonnet'); + const [provider, setProvider] = useState('anthropic'); const wsRef = useRef | null>(null); const initialContextApplied = useRef(false); + const messageQueueRef = useRef([]); + const [queueLength, setQueueLength] = useState(0); + const [queueExpanded, setQueueExpanded] = useState(false); + const [editingQueueIdx, setEditingQueueIdx] = useState(null); + const [editingQueueText, setEditingQueueText] = useState(''); + const [dragIdx, setDragIdx] = useState(null); + const [dropTargetIdx, setDropTargetIdx] = useState(null); const isDraft = session?.status === 'draft'; @@ -153,17 +188,75 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose if (session) setModel(session.model); }, [session?.model]); + useEffect(() => { + if (session?.provider) setProvider(session.provider); + }, [session?.provider]); + useEffect(() => { if (Object.keys(modesMap).length === 0) dispatch(fetchModes()); }, [dispatch, modesMap]); + const dispatchMessage = useCallback((msg: QueuedMessage) => { + if (!id) return; + setShowResumeBubble(false); + setAwaitingResponse(true); + if (isDraft) { + const config: Record = { provider, model, mode }; + if (session?.system_prompt) config.system_prompt = session.system_prompt; + if (session?.target_directory) config.target_directory = session.target_directory; + dispatch( + launchAndSendFirstMessage({ draftId: id, config, prompt: msg.prompt, mode, model, provider, images: msg.images, contextPaths: msg.contextPaths, forcedTools: msg.forcedTools, attachedSkills: msg.attachedSkills, selectedBrowserIds: msg.selectedBrowserIds }) + ).then((action) => { + if (launchAndSendFirstMessage.fulfilled.match(action)) { + const realId = action.payload.session.id; + dispatch(generateTitle({ sessionId: realId, prompt: msg.prompt })); + if (msg.selectedBrowserIds?.length) { + dispatch(setGlowingBrowserCards({ browserIds: msg.selectedBrowserIds, sessionId: realId, label: 'Use Browser' })); + } + } + }); + } else { + if (msg.selectedBrowserIds?.length) { + dispatch(setGlowingBrowserCards({ browserIds: msg.selectedBrowserIds, sessionId: id, label: 'Use Browser' })); + } + dispatch(sendMessageThunk({ sessionId: id, prompt: msg.prompt, mode, model, provider, images: msg.images, contextPaths: msg.contextPaths, forcedTools: msg.forcedTools, attachedSkills: msg.attachedSkills, selectedBrowserIds: msg.selectedBrowserIds })) + .then((action) => { + if (sendMessageThunk.rejected.match(action)) { + setAwaitingResponse(false); + } + }); + } + }, [id, isDraft, mode, model, provider, session?.system_prompt, session?.target_directory, dispatch]); + + const agentBusy = awaitingResponse || (!isDraft && (session?.status === 'running' || session?.status === 'waiting_approval')); + const prevStatusRef = useRef(session?.status); useEffect(() => { const prev = prevStatusRef.current; const curr = session?.status; prevStatusRef.current = curr; - if (prev === 'running' && (curr === 'completed' || curr === 'stopped' || curr === 'error')) { - if (id) dispatch(clearGlowingBrowserCards(id)); + let didDispatchQueued = false; + + const wasActive = prev === 'running' || prev === 'waiting_approval'; + const isTerminal = curr === 'completed' || curr === 'stopped' || curr === 'error'; + + if (wasActive && isTerminal) { + if (id) { + dispatch(fadeGlowingBrowserCards(id)); + setTimeout(() => dispatch(clearGlowingBrowserCards(id)), 2800); + } + + const nextQueued = messageQueueRef.current.shift(); + if (nextQueued) { + setQueueLength(messageQueueRef.current.length); + dispatchMessage(nextQueued); + didDispatchQueued = true; + } else { + if (curr === 'stopped') { + setShowResumeBubble(true); + } + } + const currentMode = modesMap[mode]; if (currentMode?.default_next_mode && modesMap[currentMode.default_next_mode]) { setMode(currentMode.default_next_mode); @@ -172,7 +265,13 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose } } } - }, [session?.status, mode, modesMap, id, isDraft, dispatch]); + if (curr === 'running') { + setShowResumeBubble(false); + } + if (curr !== 'draft' && !didDispatchQueued) { + setAwaitingResponse(false); + } + }, [session?.status, mode, modesMap, id, isDraft, dispatch, dispatchMessage]); const SCROLL_THRESHOLD = 50; @@ -201,27 +300,13 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const handleSend = (prompt: string, images?: Array<{ data: string; media_type: string }>, contextPaths?: Array<{ path: string; type: 'file' | 'directory' }>, forcedTools?: string[], attachedSkills?: Array<{ id: string; name: string; content: string }>, selectedBrowserIds?: string[]) => { if (!id) return; - if (isDraft) { - const config: Record = { model, mode }; - if (session?.system_prompt) config.system_prompt = session.system_prompt; - if (session?.target_directory) config.target_directory = session.target_directory; - dispatch( - launchAndSendFirstMessage({ draftId: id, config, prompt, mode, model, images, contextPaths, forcedTools, attachedSkills }) - ).then((action) => { - if (launchAndSendFirstMessage.fulfilled.match(action)) { - const realId = action.payload.session.id; - dispatch(generateTitle({ sessionId: realId, prompt })); - if (selectedBrowserIds?.length) { - dispatch(setGlowingBrowserCards({ browserIds: selectedBrowserIds, sessionId: realId })); - } - } - }); - } else { - if (selectedBrowserIds?.length) { - dispatch(setGlowingBrowserCards({ browserIds: selectedBrowserIds, sessionId: id })); - } - dispatch(sendMessageThunk({ sessionId: id, prompt, mode, model, images, contextPaths, forcedTools, attachedSkills })); + const msg: QueuedMessage = { prompt, images, contextPaths, forcedTools, attachedSkills, selectedBrowserIds }; + if (agentBusy) { + messageQueueRef.current.push(msg); + setQueueLength(messageQueueRef.current.length); + return; } + dispatchMessage(msg); }; const handleModeChange = useCallback((newMode: string) => { @@ -229,6 +314,11 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose if (id && !isDraft) dispatch(updateSessionMode({ sessionId: id, mode: newMode })); }, [id, isDraft, dispatch]); + const handleProviderChange = useCallback((newProvider: string) => { + setProvider(newProvider); + if (id && !isDraft) dispatch(updateSessionProvider({ sessionId: id, provider: newProvider })); + }, [id, isDraft, dispatch]); + const handleModelChange = useCallback((newModel: string) => { setModel(newModel); if (id && !isDraft) dispatch(updateSessionModel({ sessionId: id, model: newModel })); @@ -247,14 +337,34 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose dispatch(stopAgent({ sessionId: id })); }; - const handleEdit = useCallback( + const handleResume = useCallback(() => { + if (!id) return; + setShowResumeBubble(false); + dispatch(sendMessageThunk({ + sessionId: id, + prompt: "Continue where you left off. Start you're response EXACTLY with 'Sorry, let me pick up where I left off", + mode, + model, + provider, + hidden: true, + })); + }, [id, mode, model, provider, dispatch]); + + const [editingMessageId, setEditingMessageId] = useState(null); + + const handleSaveEdit = useCallback( (messageId: string, newContent: string) => { if (!id) return; dispatch(editMessage({ sessionId: id, messageId, content: newContent })); + setEditingMessageId(null); }, [id, dispatch] ); + const handleCancelEdit = useCallback(() => { + setEditingMessageId(null); + }, []); + const activeBranchMessages = useMemo(() => { if (!session) return []; const branchId = session.active_branch_id || 'main'; @@ -264,16 +374,81 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose return session.messages.filter((m) => m.branch_id === 'main' || m.branch_id === branchId); } - const forkIdx = session.messages.findIndex((m) => m.id === branch.fork_point_message_id); - const preMessages = session.messages - .slice(0, forkIdx) - .filter((m) => m.branch_id === (branch.parent_branch_id || 'main')); - const branchMessages = session.messages.filter((m) => m.branch_id === branchId); - return [...preMessages, ...branchMessages]; + const segments: Array<{ branchId: string; upToMessageId?: string }> = []; + let cur = branch; + let curId = branchId; + while (cur && cur.fork_point_message_id) { + segments.unshift({ branchId: curId, upToMessageId: cur.fork_point_message_id }); + curId = cur.parent_branch_id || 'main'; + cur = session.branches?.[curId]; + } + segments.unshift({ branchId: curId }); + + const result: typeof session.messages = []; + for (let i = 0; i < segments.length; i++) { + const seg = segments[i]; + const nextForkMsgId = seg.upToMessageId; + if (nextForkMsgId) { + const forkIdx = session.messages.findIndex((m) => m.id === nextForkMsgId); + const pre = session.messages + .slice(0, forkIdx) + .filter((m) => m.branch_id === seg.branchId); + result.push(...pre); + } else if (i < segments.length - 1) { + const nextFork = segments[i + 1].upToMessageId; + const forkIdx = nextFork + ? session.messages.findIndex((m) => m.id === nextFork) + : session.messages.length; + result.push( + ...session.messages.slice(0, forkIdx).filter((m) => m.branch_id === seg.branchId) + ); + } else { + result.push(...session.messages.filter((m) => m.branch_id === seg.branchId)); + } + } + const leafMsgs = session.messages.filter((m) => m.branch_id === branchId); + if (!result.some((m) => m.branch_id === branchId)) { + result.push(...leafMsgs); + } + return result; }, [session?.messages, session?.active_branch_id, session?.branches]); + const handleRegenerate = useCallback( + (assistantMsg: AgentMessage) => { + if (!id) return; + const idx = activeBranchMessages.findIndex((m) => m.id === assistantMsg.id); + for (let i = idx - 1; i >= 0; i--) { + if (activeBranchMessages[i].role === 'user') { + const userMsg = activeBranchMessages[i]; + const content = typeof userMsg.content === 'string' ? userMsg.content : JSON.stringify(userMsg.content); + dispatch(editMessage({ sessionId: id, messageId: userMsg.id, content })); + break; + } + } + }, + [id, activeBranchMessages, dispatch] + ); + + const handleBranchChat = useCallback(async (upToMessageId: string) => { + if (!id) return; + const dashId = session?.dashboard_id; + const action = await dispatch(duplicateSession({ sessionId: id, dashboardId: dashId, upToMessageId })); + if (duplicateSession.fulfilled.match(action)) { + if (onBranch) { + onBranch(action.payload.id); + } else { + dispatch(setActiveSession(action.payload.id)); + } + } + }, [id, dispatch, onBranch, session?.dashboard_id]); + const contextEstimate = useMemo(() => { - const limit = CONTEXT_WINDOWS[model] || 200_000; + // Look up context window from dynamic models first, then fall back to defaults + let limit = CONTEXT_WINDOWS_DEFAULT[model] || 200_000; + for (const models of Object.values(modelsByProvider)) { + const found = models.find((m: any) => m.value === model); + if (found?.context_window) { limit = found.context_window; break; } + } let totalChars = 0; if (session?.system_prompt) totalChars += session.system_prompt.length; for (const msg of activeBranchMessages) { @@ -284,7 +459,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose } const used = Math.round(totalChars / 4); return { used, limit }; - }, [activeBranchMessages, session?.system_prompt, session?.streamingMessage?.content, model]); + }, [activeBranchMessages, session?.system_prompt, session?.streamingMessage?.content, model, modelsByProvider]); const sessionRunning = session?.status === 'running' || session?.status === 'waiting_approval'; @@ -373,13 +548,33 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose items.push(...outputItems); } else { - items.push(msg); + if (!msg.hidden) { + items.push(msg); + } i++; } } return items; }, [activeBranchMessages]); + const lastAssistantIdsInTurn = useMemo(() => { + const ids = new Set(); + let lastAssistantId: string | null = null; + for (const item of renderItems) { + if (!isToolGroup(item) && !isToolPair(item)) { + const msg = item as AgentMessage; + if (msg.role === 'assistant') { + lastAssistantId = msg.id; + } else if (msg.role === 'user') { + if (lastAssistantId) ids.add(lastAssistantId); + lastAssistantId = null; + } + } + } + if (lastAssistantId) ids.add(lastAssistantId); + return ids; + }, [renderItems]); + const groupMetaRequestedRef = useRef>(new Set()); const groupMetaRefinedRef = useRef>(new Set()); @@ -427,11 +622,33 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const getSiblingBranches = useCallback( (messageId: string): string[] => { if (!session?.branches) return []; - return Object.values(session.branches) + + const directForks = Object.values(session.branches) .filter((b) => b.fork_point_message_id === messageId) .map((b) => b.id); + if (directForks.length > 0) { + const originalMsg = session.messages.find((m) => m.id === messageId); + const parentBranchId = originalMsg?.branch_id || 'main'; + return [parentBranchId, ...directForks]; + } + + const msg = session.messages.find((m) => m.id === messageId); + if (!msg || msg.role !== 'user') return []; + const msgBranch = session.branches[msg.branch_id]; + if (!msgBranch?.fork_point_message_id) return []; + const branchUserMsgs = session.messages.filter( + (m) => m.branch_id === msg.branch_id && m.role === 'user' + ); + if (branchUserMsgs.length === 0 || branchUserMsgs[0].id !== messageId) return []; + + const forkPointId = msgBranch.fork_point_message_id; + const siblingBranches = Object.values(session.branches) + .filter((b) => b.fork_point_message_id === forkPointId) + .map((b) => b.id); + const parentBranchId = msgBranch.parent_branch_id || 'main'; + return [parentBranchId, ...siblingBranches]; }, - [session?.branches] + [session?.branches, session?.messages] ); if (!session) { @@ -527,37 +744,55 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose {renderItems.map((item) => { if (isToolGroup(item)) { const groupMeta = session.tool_group_meta?.[item.id]; - return ; + return ; } if (isToolPair(item)) { const isPending = item.result === null && sessionRunning; - return ; + return ; } const msg = item; + const isEditing = editingMessageId === msg.id; const siblings = getSiblingBranches(msg.id); const hasBranches = siblings.length > 0; const currentBranchIdx = hasBranches ? siblings.indexOf(session.active_branch_id || 'main') : 0; + const rawText = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content); return ( - - - {hasBranches && ( - { - const prevBranch = siblings[Math.max(0, currentBranchIdx - 1)]; - if (prevBranch && id) dispatch(switchBranch({ sessionId: id, branchId: prevBranch })); - }} - onNext={() => { - const nextBranch = siblings[Math.min(siblings.length - 1, currentBranchIdx + 1)]; - if (nextBranch && id) dispatch(switchBranch({ sessionId: id, branchId: nextBranch })); - }} + + + {!isEditing && (msg.role === 'user' || (msg.role === 'assistant' && lastAssistantIdsInTurn.has(msg.id))) && ( + navigator.clipboard.writeText(rawText)} + onEdit={msg.role === 'user' ? () => setEditingMessageId(msg.id) : undefined} + onRegenerate={msg.role === 'assistant' ? () => handleRegenerate(msg) : undefined} + onBranch={msg.role === 'assistant' ? () => handleBranchChat(msg.id) : undefined} + branchNav={ + hasBranches + ? { + currentIndex: Math.max(0, currentBranchIdx), + totalBranches: siblings.length, + onPrevious: () => { + const prevBranch = siblings[Math.max(0, currentBranchIdx - 1)]; + if (prevBranch && id) dispatch(switchBranch({ sessionId: id, branchId: prevBranch })); + }, + onNext: () => { + const nextBranch = siblings[Math.min(siblings.length - 1, currentBranchIdx + 1)]; + if (nextBranch && id) dispatch(switchBranch({ sessionId: id, branchId: nextBranch })); + }, + } + : undefined + } /> )} - + ); })} {session.streamingMessage && ( @@ -566,6 +801,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose key={`streaming-${session.streamingMessage.id}`} isStreaming isPending + sessionId={session.id} call={{ id: session.streamingMessage.id, role: 'tool_call', @@ -590,9 +826,37 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose /> ) )} - {session.status === 'running' && !session.streamingMessage && ( + {(awaitingResponse || (session.status === 'running' && !session.streamingMessage)) && ( )} + {showResumeBubble && session.status === 'stopped' && ( + + + + + Resume Agent Response + + + + )} {showScrollButton && ( @@ -627,19 +891,264 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose )) )} - + {isGlowing ? ( + { e.stopPropagation(); onDismissGlow?.(); }} + sx={{ + mx: 1.5, + mb: 1.5, + py: 1.25, + display: 'flex', + alignItems: 'center', + justifyContent: 'center', + borderRadius: 2.5, + cursor: 'pointer', + fontWeight: 600, + fontSize: '0.85rem', + color: c.accent.primary, + border: `1.5px solid ${c.accent.primary}`, + background: `${c.accent.primary}08`, + boxShadow: `0 0 12px ${c.accent.primary}25, inset 0 0 12px ${c.accent.primary}08`, + animation: 'continue-chat-glow 2s ease-in-out infinite', + transition: 'background 0.15s, box-shadow 0.15s', + '@keyframes continue-chat-glow': { + '0%, 100%': { + boxShadow: `0 0 12px ${c.accent.primary}25, inset 0 0 12px ${c.accent.primary}08`, + }, + '50%': { + boxShadow: `0 0 20px ${c.accent.primary}40, inset 0 0 20px ${c.accent.primary}15`, + }, + }, + '&:hover': { + background: `${c.accent.primary}14`, + boxShadow: `0 0 24px ${c.accent.primary}50, inset 0 0 20px ${c.accent.primary}18`, + }, + }} + > + Continue chat + + ) : ( + { if (queueExpanded) { setQueueExpanded(false); setEditingQueueIdx(null); } }}> + + {queueLength > 0 && ( + + { setQueueExpanded((v) => !v); setEditingQueueIdx(null); }} + sx={{ + display: 'inline-flex', + alignItems: 'center', + gap: 0.5, + px: 1.25, + py: 0.25, + borderRadius: '8px 8px 0 0', + bgcolor: c.bg.surface, + border: `1px solid ${c.border.subtle}`, + borderBottom: 'none', + cursor: 'pointer', + userSelect: 'none', + '&:hover': { bgcolor: c.bg.secondary }, + transition: 'background 0.12s', + }} + > + {queueExpanded + ? + : + } + + {queueLength} queued + + + { e.stopPropagation(); messageQueueRef.current = []; setQueueLength(0); setQueueExpanded(false); setEditingQueueIdx(null); }} + sx={{ p: 0.15, color: c.text.tertiary, '&:hover': { color: c.status.error } }} + > + + + + + + {queueExpanded && ( + + {messageQueueRef.current.map((msg, idx) => ( + { + setDragIdx(idx); + e.dataTransfer.effectAllowed = 'move'; + }} + onDragOver={(e) => { + e.preventDefault(); + e.dataTransfer.dropEffect = 'move'; + if (dragIdx !== null && dragIdx !== idx) setDropTargetIdx(idx); + }} + onDragLeave={() => { if (dropTargetIdx === idx) setDropTargetIdx(null); }} + onDrop={(e) => { + e.preventDefault(); + if (dragIdx !== null && dragIdx !== idx) { + const q = messageQueueRef.current; + const [item] = q.splice(dragIdx, 1); + q.splice(idx, 0, item); + setQueueLength(q.length); + } + setDragIdx(null); + setDropTargetIdx(null); + }} + onDragEnd={() => { setDragIdx(null); setDropTargetIdx(null); }} + sx={{ + display: 'flex', + alignItems: 'flex-start', + gap: 0.75, + px: 1.5, + py: 1, + borderBottom: idx < queueLength - 1 ? `1px solid ${c.border.subtle}` : 'none', + '&:hover': { bgcolor: c.bg.secondary }, + transition: 'background 0.1s, opacity 0.15s', + ...(dragIdx === idx ? { opacity: 0.35 } : {}), + ...(dropTargetIdx === idx && dragIdx !== null && dragIdx !== idx + ? { borderTop: `2px solid ${c.accent.primary}` } + : {}), + }} + > + + + + {editingQueueIdx === idx ? ( + + setEditingQueueText(e.target.value)} + autoFocus + onKeyDown={(e) => { + if (e.key === 'Enter' && !e.shiftKey) { + e.preventDefault(); + const trimmed = editingQueueText.trim(); + if (trimmed) { + messageQueueRef.current[idx] = { ...messageQueueRef.current[idx], prompt: trimmed }; + setQueueLength(messageQueueRef.current.length); + } + setEditingQueueIdx(null); + } + if (e.key === 'Escape') setEditingQueueIdx(null); + }} + sx={{ + '& .MuiOutlinedInput-root': { + fontSize: '0.78rem', + color: c.text.primary, + '& fieldset': { borderColor: c.border.medium }, + '&.Mui-focused fieldset': { borderColor: c.accent.primary }, + }, + }} + /> + { + const trimmed = editingQueueText.trim(); + if (trimmed) { + messageQueueRef.current[idx] = { ...messageQueueRef.current[idx], prompt: trimmed }; + setQueueLength(messageQueueRef.current.length); + } + setEditingQueueIdx(null); + }} + sx={{ p: 0.25, color: c.accent.primary, mt: 0.25 }} + > + + + + ) : ( + + {msg.prompt} + + )} + {editingQueueIdx !== idx && ( + + + { setEditingQueueIdx(idx); setEditingQueueText(msg.prompt); }} + sx={{ p: 0.25, color: c.text.tertiary, '&:hover': { color: c.text.primary } }} + > + + + + + { + messageQueueRef.current.splice(idx, 1); + setQueueLength(messageQueueRef.current.length); + if (messageQueueRef.current.length === 0) setQueueExpanded(false); + }} + sx={{ p: 0.25, color: c.text.tertiary, '&:hover': { color: c.status.error } }} + > + + + + + )} + + ))} + + )} + + )} + + + + )} ); diff --git a/frontend/src/app/pages/AgentChat/ApprovalBar.tsx b/frontend/src/app/pages/AgentChat/ApprovalBar.tsx index 5cb1ba71..6e703f03 100644 --- a/frontend/src/app/pages/AgentChat/ApprovalBar.tsx +++ b/frontend/src/app/pages/AgentChat/ApprovalBar.tsx @@ -59,14 +59,14 @@ const INTEGRATION_META: Record = { // MCP tool name parser // --------------------------------------------------------------------------- -interface ParsedTool { +export interface ParsedTool { isMcp: boolean; serverSlug: string; actionName: string; displayName: string; } -function parseMcpToolName(rawName: string): ParsedTool { +export function parseMcpToolName(rawName: string): ParsedTool { const m = rawName.match(/^mcp__([^_]+(?:-[^_]+)*)__(.+)$/); if (!m) { return { isMcp: false, serverSlug: '', actionName: rawName, displayName: rawName }; @@ -93,7 +93,7 @@ interface McpToolMeta { serverLabel: string; } -function useMcpToolMeta(parsed: ParsedTool): McpToolMeta { +export function useMcpToolMeta(parsed: ParsedTool): McpToolMeta { const toolItems = useAppSelector((s) => s.tools.items); return useMemo(() => { @@ -185,7 +185,7 @@ interface Props { onDeny: (requestId: string, message?: string) => void; } -function getToolIcon(toolName: string) { +export function getToolIcon(toolName: string) { switch (toolName) { case 'Bash': return ; case 'Read': return ; diff --git a/frontend/src/app/pages/AgentChat/BranchNavigator.tsx b/frontend/src/app/pages/AgentChat/BranchNavigator.tsx index 4dbabb95..e506f8c5 100644 --- a/frontend/src/app/pages/AgentChat/BranchNavigator.tsx +++ b/frontend/src/app/pages/AgentChat/BranchNavigator.tsx @@ -21,31 +21,38 @@ const BranchNavigator: React.FC = ({ currentIndex, totalBranches, onPrevi - - - - - {currentIndex + 1}/{totalBranches} - - - - + + + + + {currentIndex + 1} / {totalBranches} + + + + + ); }; diff --git a/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx b/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx new file mode 100644 index 00000000..010c1782 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/BrowserAgentInlineFeed.tsx @@ -0,0 +1,403 @@ +import React, { useEffect, useRef, useMemo } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined'; +import CheckCircleOutlineIcon from '@mui/icons-material/CheckCircleOutline'; +import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline'; +import LanguageIcon from '@mui/icons-material/Language'; +import OpenInNewIcon from '@mui/icons-material/OpenInNew'; +import TouchAppOutlinedIcon from '@mui/icons-material/TouchAppOutlined'; +import KeyboardOutlinedIcon from '@mui/icons-material/KeyboardOutlined'; +import CameraAltOutlinedIcon from '@mui/icons-material/CameraAltOutlined'; +import ArticleOutlinedIcon from '@mui/icons-material/ArticleOutlined'; +import AccountTreeOutlinedIcon from '@mui/icons-material/AccountTreeOutlined'; +import CodeOutlinedIcon from '@mui/icons-material/CodeOutlined'; +import BuildOutlinedIcon from '@mui/icons-material/BuildOutlined'; +import { createSelector } from '@reduxjs/toolkit'; +import { useAppSelector, useAppDispatch } from '@/shared/hooks'; +import { AgentMessage, AgentSession, fetchBrowserAgentChildren } from '@/shared/state/agentsSlice'; +import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; +import type { RootState } from '@/shared/state/store'; + +interface Props { + parentSessionId: string; + browserId?: string; +} + +interface FeedEntry { + type: 'thought' | 'action' | 'result' | 'system'; + text: string; + actionTool?: string; + sessionLabel?: string; +} + +function formatMessage(msg: AgentMessage): FeedEntry | null { + if (msg.role === 'user') return null; + + if (msg.role === 'assistant' && typeof msg.content === 'string') { + const trimmed = msg.content.trim(); + if (!trimmed) return null; + return { type: 'thought', text: trimmed }; + } + + if (msg.role === 'tool_call') { + const content = + typeof msg.content === 'string' + ? (() => { try { return JSON.parse(msg.content); } catch { return {}; } })() + : msg.content; + const tool = content?.tool || content?.name || '?'; + const input = content?.input || {}; + let brief = ''; + switch (tool) { + case 'BrowserNavigate': + brief = `Navigate → ${input.url || '...'}`; + break; + case 'BrowserClick': + brief = `Click ${input.selector || '...'}`; + break; + case 'BrowserType': { + const txt = (input.text || '').slice(0, 40); + const ellipsis = (input.text || '').length > 40 ? '…' : ''; + brief = `Type "${txt}${ellipsis}" into ${input.selector || '...'}`; + break; + } + case 'BrowserScreenshot': + brief = 'Screenshot'; + break; + case 'BrowserGetText': + brief = 'Read page text'; + break; + case 'BrowserGetElements': + brief = `Inspect elements${input.selector ? ` (${input.selector})` : ''}`; + break; + case 'BrowserEvaluate': + brief = `Evaluate JS`; + break; + default: + brief = `${tool}(${JSON.stringify(input).slice(0, 60)})`; + } + return { type: 'action', text: brief, actionTool: tool }; + } + + if (msg.role === 'tool_result') { + const content = + typeof msg.content === 'string' + ? (() => { try { return JSON.parse(msg.content); } catch { return { text: msg.content }; } })() + : msg.content; + const toolName = content?.tool_name || ''; + const elapsed = content?.elapsed_ms; + const text = content?.text || ''; + + if (toolName === 'BrowserScreenshot') { + return { type: 'result', text: `Screenshot captured${elapsed ? ` (${elapsed}ms)` : ''}` }; + } + const preview = text.length > 120 ? text.slice(0, 120) + '…' : text; + return { type: 'result', text: `${preview}${elapsed ? ` (${elapsed}ms)` : ''}` }; + } + + if (msg.role === 'system') { + return { type: 'system', text: typeof msg.content === 'string' ? msg.content : '' }; + } + + return null; +} + +type SvgIconComponent = typeof OpenInNewIcon; + +function getActionIcon(tool?: string): SvgIconComponent { + switch (tool) { + case 'BrowserNavigate': return OpenInNewIcon; + case 'BrowserClick': return TouchAppOutlinedIcon; + case 'BrowserType': return KeyboardOutlinedIcon; + case 'BrowserScreenshot': return CameraAltOutlinedIcon; + case 'BrowserGetText': return ArticleOutlinedIcon; + case 'BrowserGetElements': return AccountTreeOutlinedIcon; + case 'BrowserEvaluate': return CodeOutlinedIcon; + default: return BuildOutlinedIcon; + } +} + +interface FeedColors { + thought: string; + thoughtIcon: string; + result: string; + error: string; + errorIcon: string; + scrollThumb: string; +} + +const darkFeedColors: FeedColors = { + thought: '#a0aab8', + thoughtIcon: '#555b6e', + result: '#555b6e', + error: '#ff8787', + errorIcon: '#ff8787', + scrollThumb: '#2a2d3e', +}; + +const lightFeedColors: FeedColors = { + thought: '#555550', + thoughtIcon: '#9e9c95', + result: '#9e9c95', + error: '#c03030', + errorIcon: '#c03030', + scrollThumb: '#ccc9c0', +}; + +const selectBrowserSessions = createSelector( + [(state: RootState) => state.agents.sessions, + (_: RootState, parentSessionId: string) => parentSessionId, + (_: RootState, __: string, browserId?: string) => browserId], + (sessions, parentSessionId, browserId) => + Object.values(sessions).filter( + (s): s is AgentSession => + s.mode === 'browser-agent' && + s.parent_session_id === parentSessionId && + (!browserId || s.browser_id === browserId), + ), +); + +const BrowserAgentInlineFeed: React.FC = ({ parentSessionId, browserId }) => { + const c = useClaudeTokens(); + const dispatch = useAppDispatch(); + const { mode } = useThemeMode(); + const fc = mode === 'dark' ? darkFeedColors : lightFeedColors; + const scrollRef = useRef(null); + const fetchedForSession = useRef(null); + + const browserSessions = useAppSelector((state) => + selectBrowserSessions(state, parentSessionId, browserId), + ); + + useEffect(() => { + if (browserSessions.length === 0 && fetchedForSession.current !== parentSessionId) { + fetchedForSession.current = parentSessionId; + dispatch(fetchBrowserAgentChildren(parentSessionId)) + .unwrap() + .catch(() => { fetchedForSession.current = null; }); + } + }, [browserSessions.length, parentSessionId, dispatch]); + + const sessionsWithEntries = useMemo(() => { + return browserSessions.map((session) => { + const entries: FeedEntry[] = []; + for (const msg of session.messages) { + const entry = formatMessage(msg); + if (entry) entries.push(entry); + } + if (session.streamingMessage?.role === 'assistant' && session.streamingMessage.content) { + entries.push({ type: 'thought', text: session.streamingMessage.content }); + } + return { session, entries }; + }); + }, [browserSessions]); + + const totalMessages = browserSessions.reduce( + (n, s) => n + s.messages.length + (s.streamingMessage ? 1 : 0), + 0, + ); + + useEffect(() => { + if (scrollRef.current) { + scrollRef.current.scrollTop = scrollRef.current.scrollHeight; + } + }, [totalMessages]); + + if (browserSessions.length === 0) return null; + + const showLabels = sessionsWithEntries.length > 1; + const accentColor = c.accent.primary; + + return ( + + {sessionsWithEntries.map(({ session, entries }, si) => ( + + {showLabels && ( + 0 ? 1 : 0, mb: 0.25 }}> + + + {session.browser_id || `Browser ${si + 1}`} + + + + )} + + {!showLabels && entries.length === 0 && session.status === 'running' && ( + + Starting browser agent... + + )} + + {entries.map((entry, i) => ( + + ))} + + {!showLabels && session.status === 'running' && entries.length > 0 && ( + + + + )} + + ))} + + ); +}; + +const EntryRow: React.FC<{ entry: FeedEntry; accentColor: string; fc: FeedColors }> = ({ entry, accentColor, fc }) => { + const c = useClaudeTokens(); + + if (entry.type === 'thought') { + return ( + + + + {entry.text} + + + ); + } + + if (entry.type === 'action') { + const ActionIcon = getActionIcon(entry.actionTool); + return ( + + + + {entry.text} + + + ); + } + + if (entry.type === 'result') { + return ( + + + ↳ {entry.text} + + + ); + } + + if (entry.type === 'system') { + return ( + + + + {entry.text} + + + ); + } + + return null; +}; + +const SessionStatusChip: React.FC<{ status: string }> = ({ status }) => { + const c = useClaudeTokens(); + if (status === 'running') { + return ( + + ); + } + if (status === 'completed') { + return ; + } + if (status === 'error') { + return ; + } + return null; +}; + +export default React.memo(BrowserAgentInlineFeed); diff --git a/frontend/src/app/pages/AgentChat/ChatInput.tsx b/frontend/src/app/pages/AgentChat/ChatInput.tsx index ff028b5d..fd0f058e 100644 --- a/frontend/src/app/pages/AgentChat/ChatInput.tsx +++ b/frontend/src/app/pages/AgentChat/ChatInput.tsx @@ -26,6 +26,7 @@ import AttachFileIcon from '@mui/icons-material/AttachFile'; import AdsClickIcon from '@mui/icons-material/AdsClick'; import CommandPicker, { CommandPickerItem, getToolGroupIcon } from '@/app/components/CommandPicker'; import { useElementSelection, SelectedElement } from '@/app/components/ElementSelectionContext'; +import { getClipboardCards, clearClipboard } from '@/shared/dashboardClipboard'; import { getWebview } from '@/shared/browserRegistry'; import { API_BASE } from '@/shared/config'; import { ContextPath } from '@/app/components/DirectoryBrowser'; @@ -66,6 +67,8 @@ interface Props { onModeChange: (mode: string) => void; model: string; onModelChange: (model: string) => void; + provider?: string; + onProviderChange?: (provider: string) => void; isRunning?: boolean; onStop?: () => void; autoRunMode?: boolean; @@ -73,6 +76,7 @@ interface Props { embedded?: boolean; autoFocus?: boolean; sessionId?: string; + queueLength?: number; } export interface ChatInputHandle { @@ -90,10 +94,10 @@ const ICON_MAP: Record = { const FALLBACK_MODE_BASE = { label: 'Agent', icon: ICON_MAP.smart_toy }; -const MODEL_OPTIONS = [ - { value: 'sonnet', label: 'Sonnet', version: '4.6' }, - { value: 'opus', label: 'Opus', version: '4.6' }, - { value: 'haiku', label: 'Haiku', version: '3.5' }, +const FALLBACK_MODELS = [ + { value: 'sonnet', label: 'Claude Sonnet 4.6', context_window: 1_000_000 }, + { value: 'opus', label: 'Claude Opus 4.6', context_window: 1_000_000 }, + { value: 'haiku', label: 'Claude Haiku 4.5', context_window: 200_000 }, ]; function formatTokenCount(n: number): string { @@ -130,7 +134,7 @@ const ContextRing: React.FC<{ used: number; limit: number; accentColor: string; ); }; -const ChatInput = forwardRef(({ onSend, disabled, mode, onModeChange, model, onModelChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus, sessionId }, ref) => { +const ChatInput = forwardRef(({ onSend, disabled, mode, onModeChange, model, onModelChange, provider, onProviderChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus, sessionId, queueLength = 0 }, ref) => { const c = useClaudeTokens(); const editorRef = useRef(null); const containerRef = useRef(null); @@ -138,6 +142,9 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, const dispatch = useAppDispatch(); const elementSelection = useElementSelection(); + const fallbackOwnerIdRef = useRef(`input-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`); + const ownerId = sessionId || fallbackOwnerIdRef.current; + useEffect(() => { if (autoFocus) editorRef.current?.focus(); }, [autoFocus]); @@ -153,6 +160,24 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, const skills = useAppSelector((state) => state.skills.items); const modesMap = useAppSelector((state) => state.modes.items); const modesArr = useMemo(() => Object.values(modesMap), [modesMap]); + const modelsByProvider = useAppSelector((state) => state.models.byProvider); + const modelsLoaded = useAppSelector((state) => state.models.loaded); + + // Build flat model list with provider grouping + const allModelOptions = useMemo(() => { + if (!modelsLoaded || Object.keys(modelsByProvider).length === 0) { + return { flat: FALLBACK_MODELS.map(m => ({ ...m, provider: 'Anthropic' })), grouped: { Anthropic: FALLBACK_MODELS } }; + } + const flat: Array<{ value: string; label: string; context_window: number; provider: string }> = []; + const grouped: Record> = {}; + for (const [prov, models] of Object.entries(modelsByProvider)) { + grouped[prov] = models.map(m => ({ value: m.value, label: m.label, context_window: m.context_window ?? 200_000 })); + for (const m of models) { + flat.push({ value: m.value, label: m.label, context_window: m.context_window ?? 200_000, provider: prov }); + } + } + return { flat, grouped }; + }, [modelsByProvider, modelsLoaded]); useEffect(() => { if (modesArr.length === 0) dispatch(fetchModes()); @@ -281,7 +306,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, let trimmed = serialized.trim(); if (!trimmed) return; - const selectedEls = elementSelection?.selectedElements ?? []; + const selectedEls = elementSelection?.elementsByOwner?.[ownerId] ?? []; let allImages = images.length > 0 ? images.map(({ data, media_type }) => ({ data, media_type })) : []; @@ -298,7 +323,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, lines.push(`${i + 1}. [Browser Card] ${title}`); lines.push(` browser_id: ${el.semanticData.selectId}`); if (url) lines.push(` URL: ${url}`); - lines.push(` (Use BrowserAgent with this browser_id to interact with it)`); + lines.push(` (Use BrowserAgent with this browser_id to interact with it, or CreateBrowserAgent for a new browser)`); } else if (el.semanticType && el.semanticData) { const typeLabel = { 'agent-card': 'Agent Card', @@ -317,6 +342,9 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, .map(([k, v]) => `${k}: ${typeof v === 'string' ? v : JSON.stringify(v)}`) .join(', '); if (metaStr) lines.push(` ${metaStr}`); + if (el.semanticType === 'agent-card' && selectId) { + lines.push(` (Use InvokeAgent with session_id "${selectId}" to query this agent with full conversation context)`); + } } else { const styleStr = Object.entries(el.computedStyles) .map(([k, v]) => `${k}: ${v}`) @@ -359,8 +387,8 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, setForcedTools([]); setAttachedSkills({}); setHasContent(false); - elementSelection?.clearSelectedElements(); - }, [disabled, images, contextPaths, forcedTools, onSend, elementSelection]); + elementSelection?.clearOwnerElements(ownerId); + }, [disabled, images, contextPaths, forcedTools, onSend, elementSelection, ownerId]); const detectTrigger = useCallback(() => { const result = detectEditorTrigger(); @@ -469,6 +497,41 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, }; const handlePaste = useCallback((e: React.ClipboardEvent) => { + const copied = getClipboardCards(); + if (copied.length > 0 && elementSelection) { + e.preventDefault(); + for (const card of copied) { + const semanticTypeMap: Record = { + agent: 'agent-card', + view: 'view-card', + browser: 'browser-card', + }; + const semanticType = semanticTypeMap[card.type]; + if (!semanticType) continue; + const labelMap: Record = { + 'agent-card': 'Agent', + 'view-card': 'View', + 'browser-card': 'Browser', + }; + const semanticLabel = (labelMap[semanticType] || semanticType) + ': ' + card.name; + const el: SelectedElement = { + id: `sel-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`, + selectorPath: `[data-select-type="${semanticType}"][data-select-id="${card.id}"]`, + tagName: 'DIV', + className: '', + outerHTML: '', + computedStyles: {}, + boundingRect: { x: 0, y: 0, width: 0, height: 0 }, + semanticType, + semanticLabel, + semanticData: { ...card.meta, selectId: card.id }, + }; + elementSelection.addElementForOwner(ownerId, el); + } + clearClipboard(); + return; + } + const items = e.clipboardData?.items; if (!items) return; const imageFiles: File[] = []; @@ -486,7 +549,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, e.preventDefault(); const plain = e.clipboardData.getData('text/plain'); if (plain) document.execCommand('insertText', false, plain); - }, [addImageFiles]); + }, [addImageFiles, elementSelection, ownerId]); const handleDragOver = useCallback((e: React.DragEvent) => { e.preventDefault(); @@ -523,7 +586,8 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, bgcolor: c.bg.surface, border: `1px solid ${c.border.subtle}`, borderRadius: '10px', - minWidth: 140, + minWidth: 180, + maxHeight: 400, boxShadow: c.shadow.lg, '& .MuiMenuItem-root': { fontSize: '0.8rem', @@ -535,7 +599,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, }, }; - const selectedElements = elementSelection?.selectedElements ?? []; + const selectedElements = elementSelection?.elementsByOwner?.[ownerId] ?? []; const hasAttachments = images.length > 0 || contextPaths.length > 0 || forcedTools.length > 0 || selectedElements.length > 0; return ( @@ -783,7 +847,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, icon={} label={chipLabel} size="small" - onDelete={() => elementSelection?.removeSelectedElement(el.id)} + onDelete={() => elementSelection?.removeOwnerElement(ownerId, el.id)} sx={{ bgcolor: 'rgba(59, 130, 246, 0.1)', color: '#3b82f6', @@ -849,7 +913,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, userSelect: 'none', }} > - {disabled ? 'Agent is working...' : autoRunMode ? 'Describe what data to generate…' : `${modeConf.label}, @ for context, / for commands`} + {disabled ? 'Agent is working...' : autoRunMode ? 'Describe what data to generate…' : isRunning ? (queueLength > 0 ? `${queueLength} queued — type another or wait…` : 'Agent is working — messages will queue…') : `${modeConf.label}, @ for context, / for commands`} )} @@ -936,7 +1000,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, }} > - {(() => { const m = MODEL_OPTIONS.find((m) => m.value === model); return m ? `${m.label} ${m.version}` : model; })()} + {(() => { const m = allModelOptions.flat.find((m) => m.value === model); return m ? m.label : model; })()} @@ -949,21 +1013,45 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} slotProps={{ paper: menuPaperProps }} > - {MODEL_OPTIONS.map((opt) => ( - { - onModelChange(opt.value); - setModelAnchor(null); - }} - > - - - ))} + {Object.entries(allModelOptions.grouped).map(([prov, models]) => [ + + + {prov} + + , + ...models.map((opt) => ( + { + onModelChange(opt.value); + if (onProviderChange) { + // Derive API-level provider key from the display group name + const provLower = prov.toLowerCase(); + const providerMap: Record = { + anthropic: 'anthropic', + openai: 'openai', + google: 'gemini', + // OpenRouter-backed providers + xai: 'openrouter', + meta: 'openrouter', + deepseek: 'openrouter', + mistral: 'openrouter', + qwen: 'openrouter', + cohere: 'openrouter', + }; + onProviderChange(providerMap[provLower] || provLower); + } + setModelAnchor(null); + }} + > + + + )), + ]).flat()} @@ -977,40 +1065,54 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, /> )} - {elementSelection && !autoRunMode && ( - - { - if (!elementSelection.selectMode && sessionId) { - elementSelection.setExcludeSelectId(sessionId); - } - elementSelection.toggleSelectMode(); - }} - sx={{ - p: 0.5, - ...(elementSelection.selectMode - ? { - bgcolor: '#3b82f6', - color: '#fff', - '&:hover': { bgcolor: '#2563eb' }, - animation: 'selectBtnPulse 2s ease-in-out infinite', - '@keyframes selectBtnPulse': { - '0%, 100%': { boxShadow: '0 0 0 0 rgba(59,130,246,0.4)' }, - '50%': { boxShadow: '0 0 0 4px rgba(59,130,246,0.1)' }, - }, + {elementSelection && !autoRunMode && (() => { + const isMySelectMode = elementSelection.selectMode && elementSelection.activeOwnerId === ownerId; + return ( + + e.preventDefault()} + onClick={() => { + if (isMySelectMode) { + elementSelection.setSelectMode(false); + } else { + if (elementSelection.activeOwnerId !== ownerId) { + elementSelection.clearOwnerElements(ownerId); } - : { - color: c.text.tertiary, - '&:hover': { color: c.text.secondary, bgcolor: 'rgba(0,0,0,0.04)' }, - }), - transition: 'background-color 0.15s, color 0.15s', - }} - > - - - - )} + elementSelection.setActiveOwnerId(ownerId); + if (sessionId) { + elementSelection.setExcludeSelectId(sessionId); + } else { + elementSelection.setExcludeSelectId(null); + } + elementSelection.setSelectMode(true); + } + }} + sx={{ + p: 0.5, + ...(isMySelectMode + ? { + bgcolor: '#3b82f6', + color: '#fff', + '&:hover': { bgcolor: '#2563eb' }, + animation: 'selectBtnPulse 2s ease-in-out infinite', + '@keyframes selectBtnPulse': { + '0%, 100%': { boxShadow: '0 0 0 0 rgba(59,130,246,0.4)' }, + '50%': { boxShadow: '0 0 0 4px rgba(59,130,246,0.1)' }, + }, + } + : { + color: c.text.tertiary, + '&:hover': { color: c.text.secondary, bgcolor: 'rgba(0,0,0,0.04)' }, + }), + transition: 'background-color 0.15s, color 0.15s', + }} + > + + + + ); + })()} (({ onSend, disabled, mode, - {!autoRunMode && (isRunning ? ( - - - - - - ) : hasContent ? ( - - - - - - ) : ( - - - - - - - - ))} + {!autoRunMode && ( + + {hasContent && ( + + + + + + )} + {isRunning ? ( + + + + + + ) : !hasContent ? ( + + + + + + + + ) : null} + + )} {selectedTemplate && ( diff --git a/frontend/src/app/pages/AgentChat/MessageActionBar.tsx b/frontend/src/app/pages/AgentChat/MessageActionBar.tsx new file mode 100644 index 00000000..5ff19921 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/MessageActionBar.tsx @@ -0,0 +1,153 @@ +import React, { useState } from 'react'; +import Box from '@mui/material/Box'; +import IconButton from '@mui/material/IconButton'; +import Typography from '@mui/material/Typography'; +import Tooltip from '@mui/material/Tooltip'; +import ContentCopyIcon from '@mui/icons-material/ContentCopy'; +import CheckIcon from '@mui/icons-material/Check'; +import EditIcon from '@mui/icons-material/Edit'; +import BookmarkBorderIcon from '@mui/icons-material/BookmarkBorder'; +import ReplayIcon from '@mui/icons-material/Replay'; +import CallSplitIcon from '@mui/icons-material/CallSplit'; +import ChevronLeftIcon from '@mui/icons-material/ChevronLeft'; +import ChevronRightIcon from '@mui/icons-material/ChevronRight'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; + +interface BranchNavProps { + currentIndex: number; + totalBranches: number; + onPrevious: () => void; + onNext: () => void; +} + +interface Props { + role: 'user' | 'assistant'; + onCopy: () => void; + onEdit?: () => void; + onRegenerate?: () => void; + onBranch?: () => void; + branchNav?: BranchNavProps; +} + +const btnSx = (c: ReturnType) => ({ + color: c.text.tertiary, + p: 0.4, + '&:hover': { color: c.text.secondary, bgcolor: 'transparent' }, + '&.Mui-disabled': { color: c.border.medium }, +}); + +const MessageActionBar: React.FC = ({ + role, + onCopy, + onEdit, + onRegenerate, + onBranch, + branchNav, +}) => { + const c = useClaudeTokens(); + const [copied, setCopied] = useState(false); + + const handleCopy = () => { + onCopy(); + setCopied(true); + setTimeout(() => setCopied(false), 1500); + }; + + const isUser = role === 'user'; + + return ( + + {isUser ? ( + <> + + + + + + + + + + {copied ? : } + + + {onEdit && ( + + + + + + )} + {branchNav && branchNav.totalBranches > 1 && ( + + + + + + {branchNav.currentIndex + 1} / {branchNav.totalBranches} + + + + + + )} + + ) : ( + <> + + + {copied ? : } + + + {onRegenerate && ( + + + + + + )} + {onBranch && ( + + + + + + )} + + )} + + ); +}; + +export default MessageActionBar; diff --git a/frontend/src/app/pages/AgentChat/MessageBubble.tsx b/frontend/src/app/pages/AgentChat/MessageBubble.tsx index 5fbd998e..11cfb126 100644 --- a/frontend/src/app/pages/AgentChat/MessageBubble.tsx +++ b/frontend/src/app/pages/AgentChat/MessageBubble.tsx @@ -8,7 +8,6 @@ import Chip from '@mui/material/Chip'; import Tooltip from '@mui/material/Tooltip'; import Collapse from '@mui/material/Collapse'; import Modal from '@mui/material/Modal'; -import EditIcon from '@mui/icons-material/Edit'; import CloseIcon from '@mui/icons-material/Close'; import AdsClickIcon from '@mui/icons-material/AdsClick'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; @@ -391,13 +390,14 @@ const MessageImageThumbnails: React.FC<{ interface Props { message: AgentMessage; - onEdit?: (messageId: string, newContent: string) => void; + editing?: boolean; + onSaveEdit?: (messageId: string, newContent: string) => void; + onCancelEdit?: () => void; isStreaming?: boolean; } -const MessageBubble: React.FC = React.memo(({ message, onEdit, isStreaming }) => { +const MessageBubble: React.FC = React.memo(({ message, editing = false, onSaveEdit, onCancelEdit, isStreaming }) => { const c = useClaudeTokens(); - const [editing, setEditing] = useState(false); const [editText, setEditText] = useState(''); const { role, content } = message; @@ -440,23 +440,22 @@ const MessageBubble: React.FC = React.memo(({ message, onEdit, isStreamin ? parseElementContext(rawText) : { userMessage: rawText, elements: [] }; - const handleStartEdit = () => { - setEditText(rawText); - setEditing(true); - }; + React.useEffect(() => { + if (editing) setEditText(rawText); + }, [editing, rawText]); const handleCancelEdit = () => { - setEditing(false); setEditText(''); + onCancelEdit?.(); }; const handleSaveEdit = () => { const trimmed = editText.trim(); - if (trimmed && trimmed !== rawText && onEdit) { - onEdit(message.id, trimmed); + if (trimmed && trimmed !== rawText && onSaveEdit) { + onSaveEdit(message.id, trimmed); } - setEditing(false); setEditText(''); + onCancelEdit?.(); }; const truncatedContent = typeof content === 'string' @@ -472,26 +471,8 @@ const MessageBubble: React.FC = React.memo(({ message, onEdit, isStreamin display: 'flex', justifyContent: isUser ? 'flex-end' : 'flex-start', my: 0.75, - '&:hover .edit-btn': { opacity: 1 }, }} > - {isUser && onEdit && !editing && ( - - - - )} = React.memo(({ message, onEdit, isStreamin '& a': { color: c.accent.primary }, }} > - {rawText} + ( + {children} + ), + }} + >{rawText} {isStreaming && } )} diff --git a/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx b/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx index f058b69e..1b2bf7ef 100644 --- a/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx +++ b/frontend/src/app/pages/AgentChat/ToolCallBubble.tsx @@ -1,8 +1,9 @@ -import React, { useState, useEffect, useCallback, useMemo } from 'react'; +import React, { useState, useEffect, useCallback, useMemo, useRef } from 'react'; import Box from '@mui/material/Box'; import Typography from '@mui/material/Typography'; import Collapse from '@mui/material/Collapse'; import IconButton from '@mui/material/IconButton'; +import Tooltip from '@mui/material/Tooltip'; import ExpandMoreIcon from '@mui/icons-material/ExpandMore'; import ExpandLessIcon from '@mui/icons-material/ExpandLess'; import TerminalIcon from '@mui/icons-material/Terminal'; @@ -15,10 +16,14 @@ import FolderIcon from '@mui/icons-material/Folder'; import AttachFileIcon from '@mui/icons-material/AttachFile'; import SearchIcon from '@mui/icons-material/Search'; import SendIcon from '@mui/icons-material/Send'; +import CallSplitIcon from '@mui/icons-material/CallSplit'; import ReactMarkdown from 'react-markdown'; import remarkGfm from 'remark-gfm'; -import { AgentMessage } from '@/shared/state/agentsSlice'; +import { AgentMessage, expandSession, collapseSession, fetchSession } from '@/shared/state/agentsSlice'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; +import { placeCard, removeCard, setGlowingAgentCard, clearGlowingAgentCard, DEFAULT_CARD_W, DEFAULT_CARD_H, EXPANDED_CARD_MIN_H, GRID_GAP } from '@/shared/state/dashboardLayoutSlice'; import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; +import BrowserAgentInlineFeed from './BrowserAgentInlineFeed'; const GoogleServiceIcon: React.FC<{ service: string; size?: number }> = ({ service, size = 14 }) => { if (service === 'gmail') { @@ -68,7 +73,13 @@ export interface ToolPair { result: AgentMessage | null; } -const pulsingKeyframes = ` +let toolCallKeyframesInjected = false; +function ensureToolCallKeyframes() { + if (toolCallKeyframesInjected) return; + toolCallKeyframesInjected = true; + const style = document.createElement('style'); + style.setAttribute('data-tool-call-keyframes', ''); + style.textContent = ` @keyframes tool-pulse { 0%, 100% { opacity: 1; } 50% { opacity: 0.4; } @@ -77,14 +88,13 @@ const pulsingKeyframes = ` 0%, 100% { box-shadow: 0 0 0 0 rgba(var(--glow-rgb), 0); } 50% { box-shadow: 0 0 10px 2px rgba(var(--glow-rgb), 0.12); } } -`; - -const streamingCursorKeyframes = ` @keyframes blink-cursor { 0%, 100% { opacity: 1; } 50% { opacity: 0; } } `; + document.head.appendChild(style); +} const ElapsedTimer: React.FC<{ startTime: string }> = ({ startTime }) => { const c = useClaudeTokens(); @@ -466,6 +476,7 @@ interface ToolCallBubbleProps { isPending?: boolean; isStreaming?: boolean; mcpCompact?: boolean; + sessionId?: string; } interface TermColors { @@ -929,7 +940,7 @@ const GmailCard: React.FC<{ data: Record; action: string; hideSubje }}> {children} }} + components={{ a: ({ children, ...props }) => {children} }} > {email.bodyPreview || email.snippet} @@ -1169,18 +1180,92 @@ const McpResultCard: React.FC<{ parsed: ParsedMcpResult; compact?: boolean }> = return ; }; +function isBrowserAgentTool(name: string): boolean { + if (name === 'CreateBrowserAgent' || name === 'BrowserAgent' || name === 'BrowserAgents') return true; + const mcp = parseMcpToolName(name); + return mcp.isMcp && mcp.serverSlug === 'openswarm-browser-agent'; +} + +function isInvokeAgentTool(name: string): boolean { + if (name === 'InvokeAgent') return true; + const mcp = parseMcpToolName(name); + return mcp.isMcp && mcp.serverSlug === 'openswarm-invoke-agent'; +} + +function isCreateAgentTool(name: string): boolean { + return name === 'Agent'; +} + +function parseInvokedSessionId(rawText: string): string | null { + const match = rawText.match(/\(forked session:\s*([a-f0-9]+)\)/); + return match ? match[1] : null; +} + +interface InvokeAgentParsed { + agentName: string; + sessionId: string | null; + cost: string | null; + response: string; +} + +function parseCreateAgentResult(rawText: string): string { + if (!rawText) return ''; + try { + const parsed = JSON.parse(rawText); + if (typeof parsed === 'string') return parsed; + if (typeof parsed === 'object' && parsed !== null) { + if (parsed.text) return parsed.text; + if (parsed.content) return typeof parsed.content === 'string' ? parsed.content : JSON.stringify(parsed.content); + if (parsed.result) return typeof parsed.result === 'string' ? parsed.result : JSON.stringify(parsed.result); + } + } catch {} + return rawText; +} + +function parseInvokeAgentResult(rawText: string): InvokeAgentParsed | null { + const headerMatch = rawText.match( + /\*\*Invoked Agent Result\*\*(?:\s*—\s*(.+?))?\s*\(forked session:\s*([a-f0-9]+)\)/, + ); + if (!headerMatch) return null; + + const agentName = headerMatch[1]?.trim() || 'Agent'; + const sessionId = headerMatch[2]; + + const costMatch = rawText.match(/\*Cost:\s*\$([0-9.]+)\*/); + const cost = costMatch ? costMatch[1] : null; + + const bodyStart = rawText.indexOf('\n\n'); + let response = bodyStart >= 0 ? rawText.slice(bodyStart + 2).trim() : ''; + if (response.startsWith('*Cost:')) { + const afterCost = response.indexOf('\n'); + response = afterCost >= 0 ? response.slice(afterCost + 1).trim() : ''; + } + + return { agentName, sessionId, cost, response }; +} + const ToolCallBubble: React.FC = React.memo( - ({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false }) => { + ({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false, sessionId }) => { + ensureToolCallKeyframes(); + const c = useClaudeTokens(); const tc = useTermColors(); + const dispatch = useAppDispatch(); + const cards = useAppSelector((s) => s.dashboardLayout.cards); const [expanded, setExpanded] = useState(false); + const bubbleRef = useRef(null); const { toolName, input, isDenied } = getToolData(call); const mcpInfo = useMemo(() => parseMcpToolName(toolName), [toolName]); const inputSummary = getInputSummary(toolName, input); const formattedInput = useMemo(() => formatInputDisplay(toolName, input), [toolName, input]); const showTimer = isPending && !isDenied && !isStreaming; - const showBody = expanded || isStreaming; + + const isBrowserAgent = isBrowserAgentTool(toolName); + const isInvokeAgent = isInvokeAgentTool(toolName); + const isCreateAgent = isCreateAgentTool(toolName); + const browserAgentAutoExpand = isBrowserAgent && isPending && !isStreaming; + const showBody = expanded || isStreaming || browserAgentAutoExpand; const resultContent = result?.content; const hasStructuredResult = @@ -1206,6 +1291,95 @@ const ToolCallBubble: React.FC = React.memo( (parsedResult?.type === 'bash' && parsedResult.exitCode !== null && parsedResult.exitCode !== 0) || (parsedResult?.type === 'text' && parsedResult.isError); + const invokedSessionId = useMemo( + () => (isInvokeAgent && result ? parseInvokedSessionId(resultRawText) : null), + [isInvokeAgent, result, resultRawText], + ); + + const invokeAgentParsed = useMemo( + () => (isInvokeAgent && result ? parseInvokeAgentResult(resultRawText) : null), + [isInvokeAgent, result, resultRawText], + ); + + const createAgentResponse = useMemo( + () => (isCreateAgent && result ? parseCreateAgentResult(resultRawText) : ''), + [isCreateAgent, result, resultRawText], + ); + + const createAgentSessionId: string | null = useMemo( + () => (isCreateAgent && hasStructuredResult && resultContent?.sub_session_id) ? resultContent.sub_session_id : null, + [isCreateAgent, hasStructuredResult, resultContent], + ); + + const revealTargetSessionId = invokedSessionId || createAgentSessionId; + + const sessions = useAppSelector((s) => s.agents.sessions); + + const handleRevealAgent = useCallback( + (e: React.MouseEvent) => { + e.stopPropagation(); + if (!revealTargetSessionId || !sessionId) return; + + if (cards[revealTargetSessionId]) { + dispatch(collapseSession(revealTargetSessionId)); + dispatch(removeCard(revealTargetSessionId)); + setTimeout(() => { + dispatch(clearGlowingAgentCard(revealTargetSessionId)); + }, 500); + return; + } + + let sourceYRatio: number | undefined; + if (bubbleRef.current) { + const bubbleEl = bubbleRef.current; + const cardEl = bubbleEl.closest('[data-select-type="agent-card"]') as HTMLElement | null; + if (cardEl) { + const cardRect = cardEl.getBoundingClientRect(); + const bubbleRect = bubbleEl.getBoundingClientRect(); + const bubbleCenterY = bubbleRect.top + bubbleRect.height / 2; + const ratio = (bubbleCenterY - cardRect.top) / cardRect.height; + sourceYRatio = Math.max(0, Math.min(1, ratio)); + } + } + + const doPlace = () => { + const parentCard = cards[sessionId]; + const targetX = parentCard + ? parentCard.x + parentCard.width + GRID_GAP * 12 + : 40; + let targetY = parentCard ? parentCard.y : 100; + if (parentCard) { + const columnCards = Object.values(cards).filter( + (c) => Math.abs(c.x - targetX) < 50 && c.session_id !== revealTargetSessionId, + ); + if (columnCards.length > 0) { + const lowestBottom = Math.max( + ...columnCards.map((c) => c.y + Math.max(EXPANDED_CARD_MIN_H, c.height)), + ); + targetY = lowestBottom + GRID_GAP; + } + } + dispatch(placeCard({ + sessionId: revealTargetSessionId, + x: targetX, + y: targetY, + width: DEFAULT_CARD_W, + height: DEFAULT_CARD_H, + })); + dispatch(expandSession(revealTargetSessionId)); + const label = isCreateAgent ? 'Create Agent' : isInvokeAgent ? 'Invoke Agent' : 'Agent'; + dispatch(setGlowingAgentCard({ sessionId: revealTargetSessionId, sourceId: sessionId, sourceYRatio, label })); + }; + + if (!sessions[revealTargetSessionId]) { + dispatch(fetchSession(revealTargetSessionId)).then(doPlace); + } else { + doPlace(); + } + }, + [revealTargetSessionId, sessionId, cards, sessions, dispatch], + ); + const toggle = useCallback(() => { if (!isStreaming) setExpanded((v) => !v); }, [isStreaming]); @@ -1233,10 +1407,415 @@ const ToolCallBubble: React.FC = React.memo( 'data-select-meta': JSON.stringify({ tool: toolName, inputSummary }), }; + if (isInvokeAgent) { + const agentName = invokeAgentParsed?.agentName || input?.session_id || 'Agent'; + const responsePreview = invokeAgentParsed?.response || ''; + const costLabel = invokeAgentParsed?.cost ? `$${invokeAgentParsed.cost}` : null; + const hasResponse = !!invokeAgentParsed; + + return ( + + + {/* Header */} + + + + InvokeAgent + + + + {agentName} + + + + {!hasResponse && !showTimer && } + + {hasResponse && responsePreview && !expanded && ( + + {responsePreview.slice(0, 100)}{responsePreview.length > 100 ? '…' : ''} + + )} + {expanded && } + + {isDenied && ( + + + denied + + )} + + {hasResponse && !isDenied && ( + + {isError ? ( + + ) : ( + + )} + {resultElapsedMs != null && ( + + {formatElapsed(resultElapsedMs)} + + )} + {costLabel && ( + + {costLabel} + + )} + + )} + + {showTimer && } + + {invokedSessionId && ( + + + + + + )} + + {hasResponse && ( + + {expanded ? : } + + )} + + + {/* Expanded body — markdown rendered, not terminal */} + + + ( + {children} + ), + }} + > + {responsePreview} + + + + + + ); + } + + if (isCreateAgent) { + const taskPrompt = input?.prompt || input?.task || input?.message || ''; + const taskLabel = taskPrompt + ? taskPrompt.length > 40 ? taskPrompt.slice(0, 40) + '…' : taskPrompt + : 'Sub-agent'; + const hasResponse = !!createAgentResponse; + + return ( + + + + + + CreateAgent + + + + {taskLabel} + + + + {!hasResponse && !showTimer && } + + {hasResponse && createAgentResponse && !expanded && ( + + {createAgentResponse.slice(0, 100)}{createAgentResponse.length > 100 ? '…' : ''} + + )} + {expanded && } + + {isDenied && ( + + + denied + + )} + + {hasResponse && !isDenied && ( + + {isError ? ( + + ) : ( + + )} + {resultElapsedMs != null && ( + + {formatElapsed(resultElapsedMs)} + + )} + + )} + + {showTimer && } + + {createAgentSessionId && ( + + + + + + )} + + {hasResponse && ( + + {expanded ? : } + + )} + + + + + ( + {children} + ), + }} + > + {createAgentResponse} + + + + + + ); + } + if (mcpCompact && mcpInfo.isMcp) { return ( - = React.memo( '&::-webkit-scrollbar-thumb': { background: tc.SCROLLBAR_THUMB, borderRadius: 3 }, }} > + {isBrowserAgent && sessionId && ( + + )} {parsedResult && parsedResult.type === 'mcp' ? ( ) : parsedResult ? ( @@ -1330,7 +1915,7 @@ const ToolCallBubble: React.FC = React.memo( {parsedResult.type === 'text' ? parsedResult.content : ''} ) : null} - {!parsedResult && isPending && !isStreaming && ( + {!parsedResult && isPending && !isStreaming && !isBrowserAgent && ( @@ -1343,8 +1928,6 @@ const ToolCallBubble: React.FC = React.memo( return ( - - {isStreaming && } = React.memo( )} + {/* Browser agent inline feed */} + {isBrowserAgent && sessionId && ( + + )} + {/* Output */} {parsedResult && parsedResult.type === 'mcp' ? ( @@ -1566,8 +2157,8 @@ const ToolCallBubble: React.FC = React.memo( ) : null} - {/* Pending indicator when waiting for result */} - {!parsedResult && isPending && !isStreaming && ( + {/* Pending indicator when waiting for result (skip for browser agent — feed replaces it) */} + {!parsedResult && isPending && !isStreaming && !isBrowserAgent && ( = React.memo(({ group, isSessionRunning = false, meta }) => { +const ToolGroupBubble: React.FC = React.memo(({ group, isSessionRunning = false, meta, sessionId }) => { const c = useClaudeTokens(); const isMcp = !!group.mcpServer; const [expanded, setExpanded] = useState(isMcp); @@ -186,6 +187,7 @@ const ToolGroupBubble: React.FC = React.memo(({ group, isSessionRunning = result={pair.result} isPending={pair.result === null && isSessionRunning} mcpCompact + sessionId={sessionId} /> ))} diff --git a/frontend/src/app/pages/Analytics/Analytics.tsx b/frontend/src/app/pages/Analytics/Analytics.tsx new file mode 100644 index 00000000..1c10703e --- /dev/null +++ b/frontend/src/app/pages/Analytics/Analytics.tsx @@ -0,0 +1,63 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import Paper from '@mui/material/Paper'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; + +const Analytics: React.FC = () => { + const c = useClaudeTokens(); + + return ( + + + + Analytics + + + + + + + + + + + + Analytics powered by PostHog + + + Usage data is automatically collected — sessions, costs, tool usage, model distribution, and task categories. + All data is anonymous and can be disabled in Settings. + + + + {[ + { label: 'Sessions & Usage', desc: 'How often agents are launched, session duration, completion rates' }, + { label: 'Cost Tracking', desc: 'Spend by model, provider, and time period' }, + { label: 'Task Categories', desc: 'What users do — coding, email, research, social, browsing' }, + { label: 'Model Distribution', desc: 'Which models and providers are most popular' }, + { label: 'Tool Usage', desc: 'Most used MCP tools, execution times, approval rates' }, + { label: 'Retention & Funnels', desc: 'User engagement, feature adoption, onboarding flow' }, + ].map((item) => ( + + + {item.label} + + + {item.desc} + + + ))} + + + + + ); +}; + +export default Analytics; diff --git a/frontend/src/app/pages/Analytics/PixelChart.tsx b/frontend/src/app/pages/Analytics/PixelChart.tsx new file mode 100644 index 00000000..263b7a2d --- /dev/null +++ b/frontend/src/app/pages/Analytics/PixelChart.tsx @@ -0,0 +1,406 @@ +import React, { useRef, useEffect, useCallback } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; + +const PALETTES = { + salmon: ['#C46B57', '#D4795F', '#E8927A', '#F0A088', '#F5B49E'], + blue: ['#445588', '#5577AA', '#6688BB', '#7799CC', '#88AADD'], + coral: ['#993344', '#AA3D4E', '#BB4455', '#CC5566', '#DD6677'], + green: ['#447755', '#558866', '#669977', '#77AA88', '#88BB99'], + purple: ['#665588', '#7766AA', '#8877BB', '#9988CC', '#AA99DD'], +} as const; + +type PaletteKey = keyof typeof PALETTES; + +interface PixelChartProps { + data: { label: string; value: number }[]; + palette?: PaletteKey; + height?: number; + pixelSize?: number; + formatValue?: (v: number) => string; + glow?: boolean; + showXLabels?: boolean; + showYScale?: boolean; + mode?: 'bar' | 'area'; // 'area' draws a filled line chart instead of bars +} + +const PixelChart: React.FC = ({ + data, + palette = 'salmon', + height = 140, + pixelSize = 6, + formatValue, + glow = true, + showXLabels = true, + showYScale = true, + mode = 'bar', +}) => { + const canvasRef = useRef(null); + const containerRef = useRef(null); + const animRef = useRef(0); + const progressRef = useRef(0); + const hoverIdxRef = useRef(-1); + const tooltipRef = useRef(null); + const c = useClaudeTokens(); + const colors = PALETTES[palette]; + + const maxVal = Math.max(...data.map((d) => d.value), 0.001); + + // Compute nice Y-axis ticks + const yTicks = (() => { + if (maxVal <= 0) return [0]; + const rawStep = maxVal / 3; + const magnitude = Math.pow(10, Math.floor(Math.log10(rawStep))); + const normalised = rawStep / magnitude; + let niceStep: number; + if (normalised <= 1) niceStep = magnitude; + else if (normalised <= 2) niceStep = 2 * magnitude; + else if (normalised <= 5) niceStep = 5 * magnitude; + else niceStep = 10 * magnitude; + const ticks: number[] = []; + for (let v = 0; v <= maxVal * 1.1; v += niceStep) { + ticks.push(v); + } + if (ticks.length < 2) ticks.push(niceStep); + return ticks; + })(); + + // X-axis labels: show first, last, and up to 3 evenly spaced + const xLabels = (() => { + if (data.length <= 1) return data.map((d, i) => ({ idx: i, label: d.label })); + if (data.length <= 5) return data.map((d, i) => ({ idx: i, label: d.label })); + const result: { idx: number; label: string }[] = []; + result.push({ idx: 0, label: data[0].label }); + const step = Math.floor(data.length / 4); + for (let i = 1; i <= 3; i++) { + const idx = Math.min(i * step, data.length - 2); + if (idx > 0 && idx < data.length - 1) { + result.push({ idx, label: data[idx].label }); + } + } + result.push({ idx: data.length - 1, label: data[data.length - 1].label }); + return result; + })(); + + const Y_LABEL_WIDTH = showYScale ? 80 : 0; + + const draw = useCallback(() => { + const canvas = canvasRef.current; + const container = containerRef.current; + if (!canvas || !container || data.length === 0) return; + + const dpr = window.devicePixelRatio || 1; + const totalW = container.clientWidth; + const chartW = totalW - Y_LABEL_WIDTH; + const h = height; + canvas.width = totalW * dpr; + canvas.height = h * dpr; + canvas.style.width = `${totalW}px`; + canvas.style.height = `${h}px`; + + const ctx = canvas.getContext('2d'); + if (!ctx) return; + ctx.scale(dpr, dpr); + + const px = pixelSize; + const gridCols = Math.floor(chartW / px); + const gridRows = Math.floor(h / px); + const effectiveMax = yTicks[yTicks.length - 1] || maxVal; + + ctx.clearRect(0, 0, totalW, h); + + // Y-axis labels and horizontal grid lines + if (showYScale) { + ctx.font = '10px monospace'; + ctx.textAlign = 'right'; + ctx.textBaseline = 'middle'; + + for (const tick of yTicks) { + const yNorm = effectiveMax > 0 ? tick / effectiveMax : 0; + const yPx = h - yNorm * (h - px); + + // Grid line + ctx.strokeStyle = c.border.subtle; + ctx.lineWidth = 0.5; + ctx.setLineDash([2, 4]); + ctx.beginPath(); + ctx.moveTo(Y_LABEL_WIDTH, yPx); + ctx.lineTo(totalW, yPx); + ctx.stroke(); + ctx.setLineDash([]); + + // Label + const label = formatValue ? formatValue(tick) : (tick % 1 === 0 ? String(tick) : tick.toFixed(1)); + ctx.fillStyle = c.text.ghost; + ctx.fillText(label, Y_LABEL_WIDTH - 8, yPx); + } + } + + // Subtle grid dots in chart area + ctx.fillStyle = c.border.subtle; + for (let gy = 0; gy < gridRows; gy += 5) { + for (let gx = 0; gx < gridCols; gx += 5) { + ctx.fillRect(Y_LABEL_WIDTH + gx * px, gy * px, 1, 1); + } + } + + const progress = Math.min(progressRef.current, 1); + const hoverIdx = hoverIdxRef.current; + + if (mode === 'area') { + // ── Area / line chart mode ── + // Draw a smooth filled area under a line + const usableH = h - px * 2; + const points: { x: number; y: number }[] = []; + + for (let i = 0; i < data.length; i++) { + const val = data[i].value; + const norm = effectiveMax > 0 ? val / effectiveMax : 0; + const x = Y_LABEL_WIDTH + (i / Math.max(data.length - 1, 1)) * chartW; + const y = h - px - norm * usableH * progress; + points.push({ x, y }); + } + + if (points.length > 0) { + // Filled area with gradient + const gradient = ctx.createLinearGradient(0, 0, 0, h); + gradient.addColorStop(0, colors[colors.length - 1] + '60'); + gradient.addColorStop(0.5, colors[Math.floor(colors.length / 2)] + '30'); + gradient.addColorStop(1, colors[0] + '08'); + + ctx.beginPath(); + ctx.moveTo(points[0].x, h); + // Smooth curve through points + for (let i = 0; i < points.length; i++) { + if (i === 0) { + ctx.lineTo(points[i].x, points[i].y); + } else { + // Cubic bezier for smoothing + const prev = points[i - 1]; + const curr = points[i]; + const cpx = (prev.x + curr.x) / 2; + ctx.bezierCurveTo(cpx, prev.y, cpx, curr.y, curr.x, curr.y); + } + } + ctx.lineTo(points[points.length - 1].x, h); + ctx.closePath(); + ctx.fillStyle = gradient; + ctx.fill(); + + // Line on top + ctx.beginPath(); + for (let i = 0; i < points.length; i++) { + if (i === 0) { + ctx.moveTo(points[i].x, points[i].y); + } else { + const prev = points[i - 1]; + const curr = points[i]; + const cpx = (prev.x + curr.x) / 2; + ctx.bezierCurveTo(cpx, prev.y, cpx, curr.y, curr.x, curr.y); + } + } + ctx.strokeStyle = colors[colors.length - 1]; + ctx.lineWidth = 2; + ctx.stroke(); + + // Glow on line + if (glow) { + ctx.shadowColor = colors[colors.length - 1]; + ctx.shadowBlur = 8; + ctx.stroke(); + ctx.shadowBlur = 0; + } + + // Data point dots + for (let i = 0; i < points.length; i++) { + if (data[i].value > 0) { + const isHov = i === hoverIdx; + ctx.beginPath(); + ctx.arc(points[i].x, points[i].y, isHov ? 4 : 2.5, 0, Math.PI * 2); + ctx.fillStyle = isHov ? colors[colors.length - 1] : colors[Math.floor(colors.length / 2)]; + ctx.fill(); + if (isHov) { + ctx.strokeStyle = colors[colors.length - 1]; + ctx.lineWidth = 1.5; + ctx.stroke(); + } + } + } + + // Pixel scatter in the filled area for the pixel art feel + for (let i = 0; i < points.length - 1; i++) { + const p1 = points[i]; + const p2 = points[i + 1]; + const steps = Math.ceil((p2.x - p1.x) / px); + for (let s = 0; s < steps; s++) { + const t = s / steps; + const x = p1.x + t * (p2.x - p1.x); + const lineY = p1.y + t * (p2.y - p1.y); + // Scatter pixels below the line + for (let py = lineY + px * 2; py < h - px; py += px * 2) { + if (Math.random() > 0.65) { + const depth = (py - lineY) / (h - lineY); + const ci = Math.max(0, Math.floor((1 - depth) * (colors.length - 1))); + ctx.globalAlpha = 0.15 + (1 - depth) * 0.2; + ctx.fillStyle = colors[ci]; + ctx.fillRect(Math.floor(x / px) * px, Math.floor(py / px) * px, px - 1, px - 1); + } + } + } + } + ctx.globalAlpha = 1; + } + } else { + // ── Bar chart mode (original) ── + const barSlots = data.length; + const totalBarPx = Math.max(1, Math.floor(gridCols / barSlots)); + const barW = Math.max(1, totalBarPx - 1); + + for (let i = 0; i < data.length; i++) { + const val = data[i].value; + const normalised = effectiveMax > 0 ? val / effectiveMax : 0; + const usableRows = gridRows - 2; + const targetH = Math.max(normalised > 0 ? 1 : 0, Math.round(normalised * usableRows)); + const barH = Math.round(targetH * progress); + const barX = i * totalBarPx; + const isHovered = i === hoverIdx; + + for (let row = 0; row < barH; row++) { + const y = gridRows - 1 - row; + const colorIdx = Math.min(colors.length - 1, Math.floor((row / Math.max(barH - 1, 1)) * (colors.length - 1))); + const baseColor = isHovered ? colors[Math.min(colorIdx + 1, colors.length - 1)] : colors[colorIdx]; + + for (let col = 0; col < barW; col++) { + ctx.fillStyle = baseColor; + ctx.fillRect(Y_LABEL_WIDTH + (barX + col) * px, y * px, px - 1, px - 1); + } + } + + if (glow && barH > 0) { + const topY = (gridRows - 1 - barH + 1) * px; + ctx.shadowColor = colors[colors.length - 1]; + ctx.shadowBlur = 6; + ctx.fillStyle = colors[colors.length - 1]; + for (let col = 0; col < barW; col++) { + ctx.fillRect(Y_LABEL_WIDTH + (barX + col) * px, topY, px - 1, px - 1); + } + ctx.shadowBlur = 0; + } + } + } + }, [data, height, pixelSize, c, colors, glow, maxVal, yTicks, showYScale, Y_LABEL_WIDTH, formatValue, mode]); + + useEffect(() => { + progressRef.current = 0; + let start: number | null = null; + const animate = (ts: number) => { + if (!start) start = ts; + progressRef.current = Math.min(1, (ts - start) / 600); + draw(); + if (progressRef.current < 1) animRef.current = requestAnimationFrame(animate); + }; + animRef.current = requestAnimationFrame(animate); + return () => cancelAnimationFrame(animRef.current); + }, [data, draw]); + + useEffect(() => { + const handleResize = () => draw(); + window.addEventListener('resize', handleResize); + return () => window.removeEventListener('resize', handleResize); + }, [draw]); + + const handleMouseMove = useCallback( + (e: React.MouseEvent) => { + const canvas = canvasRef.current; + const tooltip = tooltipRef.current; + if (!canvas || !tooltip || data.length === 0) return; + + const rect = canvas.getBoundingClientRect(); + const mx = e.clientX - rect.left - Y_LABEL_WIDTH; + if (mx < 0) { hoverIdxRef.current = -1; tooltip.style.opacity = '0'; draw(); return; } + + const chartW = rect.width - Y_LABEL_WIDTH; + const gridCols = Math.floor(chartW / pixelSize); + const totalBarPx = Math.max(1, Math.floor(gridCols / data.length)); + const idx = Math.floor(mx / (totalBarPx * pixelSize)); + + if (idx >= 0 && idx < data.length) { + hoverIdxRef.current = idx; + const d = data[idx]; + const valStr = formatValue ? formatValue(d.value) : d.value.toFixed(2); + tooltip.textContent = `${d.label}: ${valStr}`; + tooltip.style.opacity = '1'; + tooltip.style.left = `${e.clientX - rect.left}px`; + tooltip.style.top = `${e.clientY - rect.top - 28}px`; + } else { + hoverIdxRef.current = -1; + tooltip.style.opacity = '0'; + } + draw(); + }, + [data, pixelSize, draw, formatValue, Y_LABEL_WIDTH], + ); + + const handleMouseLeave = useCallback(() => { + hoverIdxRef.current = -1; + if (tooltipRef.current) tooltipRef.current.style.opacity = '0'; + draw(); + }, [draw]); + + return ( + + + {/* X-axis labels */} + {showXLabels && data.length > 0 && ( + + {xLabels.map((xl) => ( + + {xl.label} + + ))} + + )} + {/* Tooltip */} + + + ); +}; + +export default PixelChart; diff --git a/frontend/src/app/pages/Channels/Channels.tsx b/frontend/src/app/pages/Channels/Channels.tsx new file mode 100644 index 00000000..77686f39 --- /dev/null +++ b/frontend/src/app/pages/Channels/Channels.tsx @@ -0,0 +1,681 @@ +import React, { useEffect, useState } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import Button from '@mui/material/Button'; +import IconButton from '@mui/material/IconButton'; +import TextField from '@mui/material/TextField'; +import Select from '@mui/material/Select'; +import MenuItem from '@mui/material/MenuItem'; +import Chip from '@mui/material/Chip'; +import Dialog from '@mui/material/Dialog'; +import DialogTitle from '@mui/material/DialogTitle'; +import DialogContent from '@mui/material/DialogContent'; +import DialogActions from '@mui/material/DialogActions'; +import FormControl from '@mui/material/FormControl'; +import InputLabel from '@mui/material/InputLabel'; +import Switch from '@mui/material/Switch'; +import FormControlLabel from '@mui/material/FormControlLabel'; +import Tabs from '@mui/material/Tabs'; +import Tab from '@mui/material/Tab'; +import Tooltip from '@mui/material/Tooltip'; +import AddIcon from '@mui/icons-material/Add'; +import DeleteIcon from '@mui/icons-material/Delete'; +import PhoneIcon from '@mui/icons-material/Phone'; +import SmsIcon from '@mui/icons-material/Sms'; +import WhatsAppIcon from '@mui/icons-material/WhatsApp'; +import PlayArrowIcon from '@mui/icons-material/PlayArrow'; +import StopIcon from '@mui/icons-material/Stop'; +import SendIcon from '@mui/icons-material/Send'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; +import { + fetchChannels, + createChannel, + updateChannel, + deleteChannel, + enableChannel, + disableChannel, + testChannel, + fetchConversations, + ChannelConfig, +} from '@/shared/state/channelsSlice'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; + +const CHANNEL_TYPE_ICONS: Record = { + sms: , + whatsapp: , + voice: , +}; + +const STATUS_COLORS: Record = { + active: '#4caf50', + inactive: '#9e9e9e', + error: '#f44336', +}; + +const Channels: React.FC = () => { + const c = useClaudeTokens(); + const dispatch = useAppDispatch(); + const channels = useAppSelector((s) => s.channels.items); + const conversations = useAppSelector((s) => s.channels.conversations); + const channelList = Object.values(channels).sort( + (a, b) => new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(), + ); + + const [selectedId, setSelectedId] = useState(null); + const [createOpen, setCreateOpen] = useState(false); + const [tab, setTab] = useState(0); + + // Create form + const [newName, setNewName] = useState(''); + const [newType, setNewType] = useState<'sms' | 'whatsapp' | 'voice'>('sms'); + const [newProvider, setNewProvider] = useState<'twilio' | 'telnyx'>('twilio'); + const [newPhone, setNewPhone] = useState(''); + const [newAccountSid, setNewAccountSid] = useState(''); + const [newAuthToken, setNewAuthToken] = useState(''); + + // Test + const [testNumber, setTestNumber] = useState(''); + const [testResult, setTestResult] = useState(''); + + useEffect(() => { + dispatch(fetchChannels()); + }, [dispatch]); + + const selected = selectedId ? channels[selectedId] : null; + + useEffect(() => { + if (selectedId) dispatch(fetchConversations(selectedId)); + }, [selectedId, dispatch]); + + const handleCreate = async () => { + const result = await dispatch( + createChannel({ + name: newName, + channel_type: newType, + provider: newProvider, + phone_number: newPhone, + credentials: { + account_sid: newAccountSid, + auth_token: newAuthToken, + }, + }), + ); + if (createChannel.fulfilled.match(result)) { + setSelectedId(result.payload.id); + setCreateOpen(false); + setNewName(''); + setNewPhone(''); + setNewAccountSid(''); + setNewAuthToken(''); + } + }; + + const handleDelete = async (id: string) => { + await dispatch(deleteChannel(id)); + if (selectedId === id) setSelectedId(null); + }; + + const handleTest = async () => { + if (!selectedId || !testNumber) return; + try { + const result = await dispatch(testChannel({ id: selectedId, to_number: testNumber })); + if (testChannel.fulfilled.match(result)) { + setTestResult('Test sent successfully!'); + } else { + setTestResult('Test failed'); + } + } catch { + setTestResult('Test failed'); + } + }; + + const convList = Object.values(conversations).filter( + (cv) => cv.channel_id === selectedId, + ); + + return ( + + {/* Left: Channel list */} + + + + Channels + + + setCreateOpen(true)} + sx={{ color: c.accent.primary }} + > + + + + + + + + Connect SMS, WhatsApp, or voice calls to your agents. + + + + + {channelList.length === 0 && ( + + + No channels configured yet + + + )} + {channelList.map((ch) => ( + setSelectedId(ch.id)} + sx={{ + p: 1.5, + mb: 0.5, + borderRadius: 2, + cursor: 'pointer', + bgcolor: selectedId === ch.id ? `${c.accent.primary}14` : 'transparent', + border: selectedId === ch.id ? `1px solid ${c.accent.primary}40` : '1px solid transparent', + '&:hover': { bgcolor: `${c.text.tertiary}0A` }, + transition: 'all 0.15s', + }} + > + + {CHANNEL_TYPE_ICONS[ch.channel_type]} + + + {ch.name} + + + {ch.phone_number} · {ch.provider} + + + + + + ))} + + + + {/* Right: Detail panel */} + + {!selected ? ( + + Select a channel or create a new one + + ) : ( + <> + + {CHANNEL_TYPE_ICONS[selected.channel_type]} + + + {selected.name} + + + + + + + + + {selected.enabled ? ( + + ) : ( + + )} + handleDelete(selected.id)} sx={{ color: c.status.error }}> + + + + + + setTab(v)} sx={{ mb: 2, borderBottom: `1px solid ${c.border.subtle}` }}> + + + + {selected.channel_type === 'voice' && } + + + + + {/* Connection Tab */} + {tab === 0 && ( + + + dispatch(updateChannel({ id: selected.id, phone_number: e.target.value })) + } + /> + + Provider + + + + dispatch( + updateChannel({ + id: selected.id, + credentials: { + ...selected.credentials, + [selected.provider === 'twilio' ? 'account_sid' : 'api_key']: e.target.value, + }, + }), + ) + } + /> + + dispatch( + updateChannel({ + id: selected.id, + credentials: { + ...selected.credentials, + [selected.provider === 'twilio' ? 'auth_token' : 'public_key']: e.target.value, + }, + }), + ) + } + /> + + Webhook URL for Twilio: {window.location.origin.replace(/:\d+$/, ':8324')}/api/channels/webhooks/twilio/{selected.channel_type}?channel_id={selected.id} + + + )} + + {/* Agent Tab */} + {tab === 1 && ( + + + Mode + + + + Model + + + + dispatch( + updateChannel({ + id: selected.id, + agent_config: { ...selected.agent_config, system_prompt: e.target.value || undefined }, + }), + ) + } + /> + + )} + + {/* Security Tab */} + {tab === 2 && ( + + + dispatch( + updateChannel({ + id: selected.id, + security: { ...selected.security, verify_signatures: e.target.checked }, + }), + ) + } + /> + } + label="Verify webhook signatures" + /> + + dispatch( + updateChannel({ + id: selected.id, + security: { + ...selected.security, + allowlist: e.target.value.split('\n').filter(Boolean), + }, + }), + ) + } + /> + + dispatch( + updateChannel({ + id: selected.id, + security: { + ...selected.security, + rate_limit_per_minute: parseInt(e.target.value) || 10, + }, + }), + ) + } + /> + + )} + + {/* Voice Tab */} + {tab === 3 && selected.channel_type === 'voice' && ( + + + Call Mode + + + + dispatch( + updateChannel({ + id: selected.id, + voice_config: { + ...(selected.voice_config || {}), + greeting_message: e.target.value, + } as any, + }), + ) + } + /> + + dispatch( + updateChannel({ + id: selected.id, + voice_config: { + ...(selected.voice_config || {}), + voice: e.target.value, + } as any, + }), + ) + } + /> + + )} + + {/* Conversations Tab */} + {tab === (selected.channel_type === 'voice' ? 4 : 3) && ( + + {convList.length === 0 ? ( + No conversations yet + ) : ( + convList.map((conv) => ( + + + {conv.phone_number} + + + {conv.messages.length} messages · {conv.status} + + + {conv.messages.slice(-5).map((msg) => ( + + + {msg.direction === 'inbound' ? 'Received' : 'Sent'} ·{' '} + {new Date(msg.timestamp).toLocaleTimeString()} + + {msg.content} + + ))} + + + )) + )} + + )} + + {/* Test Tab */} + {tab === (selected.channel_type === 'voice' ? 5 : 4) && ( + + setTestNumber(e.target.value)} + /> + + {testResult && ( + + {testResult} + + )} + + )} + + )} + + + {/* Create Dialog */} + setCreateOpen(false)} maxWidth="sm" fullWidth> + New Channel + + setNewName(e.target.value)} + size="small" + placeholder="My SMS Channel" + /> + + Type + + + + Provider + + + setNewPhone(e.target.value)} + size="small" + placeholder="+1234567890" + /> + setNewAccountSid(e.target.value)} + size="small" + type="password" + /> + setNewAuthToken(e.target.value)} + size="small" + type="password" + /> + + + + + + + + ); +}; + +export default Channels; diff --git a/frontend/src/app/pages/Dashboard/AgentCard.tsx b/frontend/src/app/pages/Dashboard/AgentCard.tsx index 31c2e07f..a88a44f3 100644 --- a/frontend/src/app/pages/Dashboard/AgentCard.tsx +++ b/frontend/src/app/pages/Dashboard/AgentCard.tsx @@ -22,12 +22,16 @@ import { import { setCardPosition, setCardSize, + fadeGlowingAgentCard, + clearGlowingAgentCard, + removeCard, } from '@/shared/state/dashboardLayoutSlice'; -import { useAppDispatch } from '@/shared/hooks'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { QuestionForm } from '@/app/pages/AgentChat/ApprovalBar'; import AgentChat from '@/app/pages/AgentChat/AgentChat'; import { parseMcpToolName, getMcpShortAction } from '@/app/pages/AgentChat/ToolCallBubble'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { useOverlayScrollPassthrough } from './useOverlayScrollPassthrough'; // --------------------------------------------------------------------------- // Helper components & functions (unchanged) @@ -168,7 +172,8 @@ interface Props { cardWidth: number; cardHeight: number; zoom?: number; - spawnFrom?: { x: number; y: number }; + spawnFrom?: { x: number; y: number; type?: 'branch' }; + exitTarget?: { x: number; y: number }; isSelected?: boolean; isHighlighted?: boolean; multiDragDelta?: { dx: number; dy: number } | null; @@ -176,6 +181,12 @@ interface Props { onDragStart?: (id: string, type: 'agent' | 'view') => void; onDragMove?: (dx: number, dy: number) => void; onDragEnd?: (dx: number, dy: number, didDrag: boolean) => void; + onBranch?: (sourceSessionId: string, newSessionId: string) => void; + onMeasuredHeight?: (sessionId: string, height: number) => void; + snapColumn?: { x: number; width: number }; + autoFocusInput?: boolean; + cardZOrder?: number; + onBringToFront?: (id: string, type: 'agent' | 'view' | 'browser') => void; } const MIN_W = 480; @@ -183,13 +194,54 @@ const MIN_H = 120; const EXPANDED_OVERLAY_H = 620; const SPAWN_SPRING = { type: 'spring' as const, stiffness: 400, damping: 28, mass: 0.6 }; +const BRANCH_SPRING = { type: 'spring' as const, stiffness: 300, damping: 26, mass: 0.8 }; +const EXIT_SPRING = { type: 'spring' as const, stiffness: 350, damping: 30, mass: 0.7 }; +const GLOW_FADE_MS = 2500; + +const SNAP_THRESHOLD = 60; const AgentCard: React.FC = ({ - session, expanded, cardX, cardY, cardWidth, cardHeight, zoom = 1, spawnFrom, + session, expanded, cardX, cardY, cardWidth, cardHeight, zoom = 1, spawnFrom, exitTarget, isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd, + onBranch, onMeasuredHeight, snapColumn, autoFocusInput, cardZOrder = 0, onBringToFront, }) => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); + + const cardBoxRef = useRef(null); + useEffect(() => { + const el = cardBoxRef.current; + if (!el || !onMeasuredHeight) return; + const ro = new ResizeObserver((entries) => { + for (const entry of entries) { + onMeasuredHeight(session.id, entry.contentRect.height); + } + }); + ro.observe(el); + return () => ro.disconnect(); + }, [session.id, onMeasuredHeight]); + + // ---- Glow state (for branched cards) ---- + const glowEntry = useAppSelector((s) => s.dashboardLayout.glowingAgentCards[session.id]); + const isGlowingRedux = !!glowEntry; + const glowFading = glowEntry?.fading ?? false; + const glowFadeTimer = useRef | null>(null); + + const dismissGlow = useCallback(() => { + if (!isGlowingRedux || glowFading) return; + dispatch(fadeGlowingAgentCard(session.id)); + glowFadeTimer.current = setTimeout(() => { + dispatch(clearGlowingAgentCard(session.id)); + }, GLOW_FADE_MS + 300); + }, [isGlowingRedux, glowFading, dispatch, session.id]); + + useEffect(() => () => { + if (glowFadeTimer.current) clearTimeout(glowFadeTimer.current); + }, []); + + const accentColor = c.accent.primary; + const accentHover = c.accent.hover; const STATUS_COLORS: Record = { running: { color: c.status.success, bg: c.status.successBg }, @@ -212,6 +264,7 @@ const AgentCard: React.FC = ({ const justDraggedRef = useRef(false); const handleDragPointerDown = useCallback((e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY }; @@ -241,11 +294,15 @@ const AgentCard: React.FC = ({ const dx = (e.clientX - dragState.current.startX) / zoom; const dy = (e.clientY - dragState.current.startY) / zoom; if (didDrag.current) { - dispatch(setCardPosition({ - sessionId: session.id, - x: dragState.current.origX + dx, - y: dragState.current.origY + dy, - })); + let finalX = dragState.current.origX + dx; + const finalY = dragState.current.origY + dy; + + if (snapColumn && Math.abs(finalX - snapColumn.x) < SNAP_THRESHOLD) { + finalX = snapColumn.x; + dispatch(setCardSize({ sessionId: session.id, width: snapColumn.width, height: cardHeight })); + } + + dispatch(setCardPosition({ sessionId: session.id, x: finalX, y: finalY })); justDraggedRef.current = true; requestAnimationFrame(() => { justDraggedRef.current = false; }); } @@ -255,7 +312,7 @@ const AgentCard: React.FC = ({ setLocalDragPos(null); setIsDragging(false); (e.currentTarget as HTMLElement).releasePointerCapture(e.pointerId); - }, [zoom, dispatch, session.id, onDragEnd]); + }, [zoom, dispatch, session.id, onDragEnd, snapColumn, cardHeight]); // ---- Unified edge / corner resize ---- const resizeRef = useRef<{ @@ -272,6 +329,7 @@ const AgentCard: React.FC = ({ const handleResizeDown = useCallback( (dir: ResizeDir) => (e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); const effectiveW = Math.max(cardWidth, MIN_W); @@ -337,14 +395,17 @@ const AgentCard: React.FC = ({ const handleRemove = (e: React.MouseEvent) => { e.stopPropagation(); e.preventDefault(); - dispatch(closeSession({ sessionId: session.id })); + dispatch(collapseSession(session.id)); + dispatch(removeCard(session.id)); + if (glowEntry) { + setTimeout(() => { + dispatch(clearGlowingAgentCard(session.id)); + }, 500); + } else { + dispatch(closeSession({ sessionId: session.id })); + } }; - const handleCollapse = (e: React.MouseEvent) => { - e.stopPropagation(); - e.preventDefault(); - dispatch(collapseSession(session.id)); - }; useEffect(() => { if (session.status === 'running' || session.status === 'waiting_approval') { @@ -376,33 +437,54 @@ const AgentCard: React.FC = ({ const activeW = localResize?.w ?? cardWidth; const activeH = localResize?.h ?? cardHeight; + const isBranchSpawn = spawnFrom?.type === 'branch'; + const spawnInitial = spawnFrom + ? isBranchSpawn + ? { opacity: 0.5, scale: 0.92, left: spawnFrom.x, top: spawnFrom.y } + : { opacity: 0, scale: 0.3, left: spawnFrom.x, top: spawnFrom.y } + : false; + const spawnTransition = noTransition || !spawnFrom + ? { duration: 0 } + : isBranchSpawn + ? { left: BRANCH_SPRING, top: BRANCH_SPRING, scale: BRANCH_SPRING, opacity: { duration: 0.25 } } + : { left: SPAWN_SPRING, top: SPAWN_SPRING, scale: SPAWN_SPRING, opacity: { duration: 0.12 } }; + + const exitAnimation = exitTarget + ? { + opacity: 0, + scale: 0.3, + left: exitTarget.x, + top: exitTarget.y, + transition: { left: EXIT_SPRING, top: EXIT_SPRING, scale: EXIT_SPRING, opacity: { duration: 0.2 } }, + } + : { opacity: 0, scale: 0.85, transition: { duration: 0.2 } }; + return ( onBringToFront?.(session.id, 'agent')} style={{ position: 'absolute', - zIndex: isDragging || isResizing ? 999 : expanded ? 100 : 'auto', + zIndex: isDragging || isResizing ? 999999 : cardZOrder, }} > { if (justDraggedRef.current) return; + if (!isSelected && !e.shiftKey) { + dispatch(toggleExpandSession(session.id)); + } onCardSelect?.(session.id, 'agent', e.shiftKey); }} - onDoubleClick={() => dispatch(toggleExpandSession(session.id))} sx={{ position: 'relative', width: localResize ? activeW : Math.max(cardWidth, MIN_W), @@ -410,26 +492,34 @@ const AgentCard: React.FC = ({ bgcolor: c.bg.surface, border: isHighlighted ? `2px solid ${c.accent.primary}` - : isSelected - ? '2px solid #3b82f6' - : hasPending && !expanded - ? `1px solid ${c.status.warning}` - : expanded - ? `1px solid ${c.border.strong}` - : `1px solid ${c.border.subtle}`, + : (isGlowingRedux && !glowFading) + ? `2px solid ${accentColor}` + : isSelected + ? '2px solid #3b82f6' + : hasPending && !expanded + ? `1px solid ${c.status.warning}` + : expanded + ? `1px solid ${c.border.strong}` + : `1px solid ${c.border.subtle}`, borderRadius: 3, p: 2, cursor: expanded ? 'default' : 'pointer', - transition: noTransition ? 'none' : c.transition, + transition: noTransition + ? 'none' + : glowFading + ? `border ${GLOW_FADE_MS}ms ease-out, box-shadow ${GLOW_FADE_MS}ms ease-out` + : c.transition, boxShadow: isHighlighted ? `0 0 0 3px ${c.accent.primary}50, 0 0 20px ${c.accent.primary}35, 0 0 40px ${c.accent.primary}15` - : isDragging - ? c.shadow.lg - : isSelected - ? `0 0 0 1px #3b82f6, ${c.shadow.md}` - : expanded - ? c.shadow.md - : c.shadow.sm, + : (isGlowingRedux && !glowFading) + ? `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15` + : isDragging + ? c.shadow.lg + : isSelected + ? `0 0 0 1px #3b82f6, ${c.shadow.md}` + : expanded + ? c.shadow.md + : c.shadow.sm, display: 'flex', flexDirection: 'column', overflow: 'hidden', @@ -452,9 +542,19 @@ const AgentCard: React.FC = ({ boxShadow: c.shadow.sm, }, }, - zIndex: 50, }), - ...(!isHighlighted && !expanded && !isDragging && !isSelected && { + ...(!isHighlighted && isGlowingRedux && !glowFading && { + animation: 'agent-card-glow-pulse 2s ease-in-out infinite', + '@keyframes agent-card-glow-pulse': { + '0%, 100%': { + boxShadow: `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15`, + }, + '50%': { + boxShadow: `0 0 0 3px ${accentColor}60, 0 0 28px ${accentColor}45, 0 0 56px ${accentColor}25`, + }, + }, + }), + ...(!isHighlighted && !(isGlowingRedux && !glowFading) && !expanded && !isDragging && !isSelected && { '&:hover': { boxShadow: c.shadow.md, borderColor: hasPending ? c.status.warning : c.border.strong, @@ -462,6 +562,82 @@ const AgentCard: React.FC = ({ }), }} > + {/* Glow overlays for branched cards */} + {isGlowingRedux && ( + + {/* Rotating conic gradient border */} + + {/* Top edge shimmer */} + + {/* Inner shadow overlay */} + + + )} + {/* Resize handles: 4 edges + 4 corners */} {HANDLE_DEFS.map(({ dir, sx }) => ( = ({ /> ))} - {/* Selection overlay – blocks content interaction while selected, enabling drag from anywhere */} + {/* Selection overlay – blocks click interaction while selected, enabling drag from anywhere */} {isSelected && ( = ({ if (justDraggedRef.current) return; onCardSelect?.(session.id, 'agent', e.shiftKey); }} - onDoubleClick={() => dispatch(toggleExpandSession(session.id))} sx={{ position: 'absolute', inset: 0, @@ -571,37 +747,20 @@ const AgentCard: React.FC = ({ onPointerDown={(e) => e.stopPropagation()} sx={{ display: 'flex', alignItems: 'center', gap: 0.5, flexShrink: 0, ml: 0.5 }} > - {expanded ? ( - - e.stopPropagation()} - sx={{ - color: c.text.ghost, - p: 0.5, - '&:hover': { color: c.text.secondary, bgcolor: c.bg.secondary }, - }} - > - - - - ) : ( - - e.stopPropagation()} - sx={{ - color: c.text.ghost, - p: 0.5, - '&:hover': { color: c.status.error, bgcolor: `${c.status.errorBg}` }, - }} - > - - - - )} + + e.stopPropagation()} + sx={{ + color: c.text.ghost, + p: 0.5, + '&:hover': { color: c.status.error, bgcolor: `${c.status.errorBg}` }, + }} + > + + + @@ -649,6 +808,10 @@ const AgentCard: React.FC = ({ sessionId={session.id} onClose={() => dispatch(collapseSession(session.id))} embedded + autoFocus={autoFocusInput} + isGlowing={isGlowingRedux && !glowFading} + onDismissGlow={dismissGlow} + onBranch={onBranch ? (newId: string) => onBranch(session.id, newId) : undefined} /> )} diff --git a/frontend/src/app/pages/Dashboard/BrowserAgentOverlay.tsx b/frontend/src/app/pages/Dashboard/BrowserAgentOverlay.tsx index e7a02858..c780cfa6 100644 --- a/frontend/src/app/pages/Dashboard/BrowserAgentOverlay.tsx +++ b/frontend/src/app/pages/Dashboard/BrowserAgentOverlay.tsx @@ -58,12 +58,24 @@ const BrowserAgentOverlay: React.FC = ({ session, browserWidth, browserHe const [expanded, setExpanded] = useState(false); const [confirmStop, setConfirmStop] = useState(false); const [fadeOut, setFadeOut] = useState(false); + const [hidden, setHidden] = useState(false); const confirmTimer = useRef | null>(null); const fadeTimer = useRef | null>(null); + const hideTimer = useRef | null>(null); const isRunning = session.status === 'running'; const isDone = session.status === 'completed' || session.status === 'error' || session.status === 'stopped'; + const prevSessionId = useRef(session.id); + useEffect(() => { + if (session.id !== prevSessionId.current) { + prevSessionId.current = session.id; + setFadeOut(false); + setHidden(false); + setConfirmStop(false); + } + }, [session.id]); + useEffect(() => { if (isDone) { fadeTimer.current = setTimeout(() => setFadeOut(true), 2000); @@ -71,6 +83,13 @@ const BrowserAgentOverlay: React.FC = ({ session, browserWidth, browserHe return () => { if (fadeTimer.current) clearTimeout(fadeTimer.current); }; }, [isDone]); + useEffect(() => { + if (fadeOut) { + hideTimer.current = setTimeout(() => setHidden(true), 400); + } + return () => { if (hideTimer.current) clearTimeout(hideTimer.current); }; + }, [fadeOut]); + useEffect(() => { if (scrollRef.current) { scrollRef.current.scrollTop = scrollRef.current.scrollHeight; @@ -111,7 +130,7 @@ const BrowserAgentOverlay: React.FC = ({ session, browserWidth, browserHe const panelW = expanded ? expandedW : collapsedW; const panelH = expanded ? expandedH : collapsedH; - if (fadeOut) return null; + if (hidden) return null; return ( = ({ session, browserWidth, browserHe flexDirection: 'column', overflow: 'hidden', transition: 'width 0.25s ease, height 0.25s ease, opacity 0.4s ease', - opacity: isDone && !fadeOut ? 0.7 : 1, + opacity: fadeOut ? 0 : isDone ? 0.7 : 1, animation: 'overlay-enter 0.3s ease-out', '@keyframes overlay-enter': { '0%': { opacity: 0, transform: 'translateY(8px) scale(0.95)' }, diff --git a/frontend/src/app/pages/Dashboard/BrowserCard.tsx b/frontend/src/app/pages/Dashboard/BrowserCard.tsx index d7772294..a50c7d31 100644 --- a/frontend/src/app/pages/Dashboard/BrowserCard.tsx +++ b/frontend/src/app/pages/Dashboard/BrowserCard.tsx @@ -40,6 +40,7 @@ import { useBrowserActivity } from '@/shared/useBrowserActivity'; import { getActionLabel } from '@/shared/browserCommandHandler'; import { resolveInput, isGoogleSearch } from '@/shared/resolveUrl'; import BrowserAgentOverlay from './BrowserAgentOverlay'; +import { useOverlayScrollPassthrough } from './useOverlayScrollPassthrough'; import { useElementSelection } from '@/app/components/ElementSelectionContext'; type ResizeDir = 'n' | 's' | 'e' | 'w' | 'ne' | 'nw' | 'se' | 'sw'; @@ -67,6 +68,14 @@ const HANDLE_DEFS: { dir: ResizeDir; sx: Record }[] = [ const isElectron = navigator.userAgent.includes('Electron'); +const chromeUserAgent = navigator.userAgent + .replace(/\s*Electron\/\S+/, '') + .replace(/\s*OpenSwarm\/\S+/, ''); + +const webviewPreloadPath: string | undefined = isElectron + ? (window as any).openswarm?.getWebviewPreloadPath?.() + : undefined; + type WebviewElement = BrowserWebview; interface TabLocalState { @@ -84,6 +93,7 @@ interface Props { cardWidth: number; cardHeight: number; zoom?: number; + cmdHeld?: boolean; isSelected?: boolean; isHighlighted?: boolean; multiDragDelta?: { dx: number; dy: number } | null; @@ -91,29 +101,35 @@ interface Props { onDragStart?: (id: string, type: 'agent' | 'view' | 'browser') => void; onDragMove?: (dx: number, dy: number) => void; onDragEnd?: (dx: number, dy: number, didDrag: boolean) => void; + cardZOrder?: number; + onBringToFront?: (id: string, type: 'agent' | 'view' | 'browser') => void; } const BrowserCard: React.FC = ({ - browserId, tabs, activeTabId, cardX, cardY, cardWidth, cardHeight, zoom = 1, + browserId, tabs, activeTabId, cardX, cardY, cardWidth, cardHeight, zoom = 1, cmdHeld = false, isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd, + cardZOrder = 0, onBringToFront, }) => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage); const elementSelectionCtx = useElementSelection(); const isElementSelectMode = elementSelectionCtx?.selectMode ?? false; const browserAgentSession = useAppSelector((state) => { const sessions = state.agents.sessions; - return Object.values(sessions).find( + const matches = Object.values(sessions).filter( (s) => s.browser_id === browserId && s.mode === 'browser-agent' - && (s.status === 'running' || s.status === 'completed' || s.status === 'error'), - ) ?? null; + && (s.status === 'running' || s.status === 'completed' || s.status === 'error' || s.status === 'stopped'), + ); + return matches.find((s) => s.status === 'running') ?? matches[matches.length - 1] ?? null; }); const activity = useBrowserActivity(browserId); - const agentActive = activity.active; + const agentRunning = browserAgentSession?.status === 'running'; + const agentActive = activity.active || agentRunning; const agentAction = activity.action; const lastAction = activity.lastAction; @@ -192,10 +208,6 @@ const BrowserCard: React.FC = ({ onTitleUpdate(); }; - const onNewWindow = (e: any) => { - if (e.url) dispatch(addBrowserTab({ browserId, url: e.url, makeActive: true })); - }; - const onFaviconUpdate = (e: any) => { const favicons = e.favicons || (e.detail && e.detail.favicons); if (favicons?.[0]) { @@ -208,7 +220,6 @@ const BrowserCard: React.FC = ({ wv.addEventListener('page-title-updated', onTitleUpdate); wv.addEventListener('did-start-loading', onLoadStart); wv.addEventListener('did-stop-loading', onLoadStop); - wv.addEventListener('new-window', onNewWindow); wv.addEventListener('page-favicon-updated', onFaviconUpdate); cleanups.push(() => { @@ -218,7 +229,6 @@ const BrowserCard: React.FC = ({ wv.removeEventListener('page-title-updated', onTitleUpdate); wv.removeEventListener('did-start-loading', onLoadStart); wv.removeEventListener('did-stop-loading', onLoadStop); - wv.removeEventListener('new-window', onNewWindow); wv.removeEventListener('page-favicon-updated', onFaviconUpdate); }); } @@ -371,6 +381,7 @@ const BrowserCard: React.FC = ({ const justDraggedRef = useRef(false); const handleDragPointerDown = useCallback((e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY }; @@ -426,6 +437,7 @@ const BrowserCard: React.FC = ({ const handleResizeDown = useCallback( (dir: ResizeDir) => (e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); resizeRef.current = { @@ -491,20 +503,13 @@ const BrowserCard: React.FC = ({ const accentColor = c.accent.primary; const accentHover = c.accent.hover; + const accentRgb = accentColor.replace('#', '').match(/.{2}/g)?.map(h => parseInt(h, 16)).join(',') || '189,100,57'; // ---- Glow state ---- const glowingBrowserCards = useAppSelector((s) => s.dashboardLayout.glowingBrowserCards); const isGlowingFromRedux = !!glowingBrowserCards[browserId]; - const [hasBeenTouched, setHasBeenTouched] = useState(false); - useEffect(() => { - if (isGlowingFromRedux && agentActive) setHasBeenTouched(true); - }, [isGlowingFromRedux, agentActive]); - useEffect(() => { - if (!isGlowingFromRedux) setHasBeenTouched(false); - }, [isGlowingFromRedux]); - - const showGlow = isGlowingFromRedux && hasBeenTouched; + const showGlow = isGlowingFromRedux; const agentBorder = isHighlighted ? `2px solid ${c.accent.primary}` @@ -535,6 +540,7 @@ const BrowserCard: React.FC = ({ data-select-type="browser-card" data-select-id={browserId} data-select-meta={JSON.stringify({ name: activeTitle || 'Browser', url: activeUrl })} + onPointerDownCapture={() => onBringToFront?.(browserId, 'browser')} onClick={(e: React.MouseEvent) => { if (justDraggedRef.current) return; onCardSelect?.(browserId, 'browser', e.shiftKey); @@ -552,7 +558,7 @@ const BrowserCard: React.FC = ({ overflow: 'hidden', display: 'flex', flexDirection: 'column', - zIndex: isHighlighted ? 50 : (isDragging || isResizing) ? 100 : (agentActive || showGlow) ? 50 : 1, + zIndex: (isDragging || isResizing) ? 999999 : cardZOrder, transition: noTransition ? 'none' : 'box-shadow 0.4s ease, border 0.3s ease', '&:hover .resize-handle': { opacity: 1 }, ...(isHighlighted && { @@ -576,8 +582,8 @@ const BrowserCard: React.FC = ({ }, }), ...(!isHighlighted && (agentActive || showGlow) && { - animation: 'agent-glow-pulse 2s ease-in-out infinite', - '@keyframes agent-glow-pulse': { + animation: `agent-glow-${browserId} 2s ease-in-out infinite`, + [`@keyframes agent-glow-${browserId}`]: { '0%, 100%': { boxShadow: `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15${innerGlow}`, }, @@ -588,9 +594,10 @@ const BrowserCard: React.FC = ({ }), }} > - {/* Selection overlay – blocks content interaction while selected, enabling drag from anywhere */} + {/* Selection overlay – blocks click interaction while selected, enabling drag from anywhere */} {isSelected && ( = ({ {isElementSelectMode && ( )} + {cmdHeld && !isSelected && ( + + )} {isElectron ? ( tabs.map((tab) => ( = ({ data-tab-id={tab.id} src="about:blank" allowpopups="true" + useragent={chromeUserAgent} + {...(webviewPreloadPath ? { preload: webviewPreloadPath } : {})} + webpreferences="plugins=yes, autoplayPolicy=no-user-gesture-required" style={{ position: 'absolute', top: 0, @@ -1035,6 +1048,7 @@ const BrowserCard: React.FC = ({ {/* Camera flash — screenshot */} {(agentAction === 'screenshot' || lastAction === 'screenshot') && ( = ({ pointerEvents: 'none', background: `linear-gradient(180deg, transparent, ${accentColor}90, transparent)`, boxShadow: `0 0 12px ${accentColor}60`, - animation: 'scan-sweep 1.5s ease-in-out infinite', + animation: 'scan-sweep 1.5s ease-in-out infinite alternate', '@keyframes scan-sweep': { '0%': { top: '0%' }, - '100%': { top: '100%' }, + '100%': { top: 'calc(100% - 3px)' }, }, }} /> @@ -1074,10 +1088,11 @@ const BrowserCard: React.FC = ({ {/* Click ripple */} {(agentAction === 'click' || lastAction === 'click') && ( = ({ )} - {/* Orange inner shadow overlay for selection / streaming glow */} + {/* Accent inner shadow overlay for selection / streaming glow */} {showGlow && !agentActive && ( = ({ zIndex: 14, pointerEvents: 'none', borderRadius: 'inherit', - boxShadow: 'inset 0 0 40px rgba(255,140,0,0.35), inset 0 0 80px rgba(255,100,0,0.15)', - animation: 'orange-glow-pulse 2s ease-in-out infinite', - '@keyframes orange-glow-pulse': { + boxShadow: `inset 0 0 40px rgba(${accentRgb},0.35), inset 0 0 80px rgba(${accentRgb},0.15)`, + animation: `accent-glow-${browserId} 2s ease-in-out infinite`, + [`@keyframes accent-glow-${browserId}`]: { '0%, 100%': { - boxShadow: 'inset 0 0 40px rgba(255,140,0,0.35), inset 0 0 80px rgba(255,100,0,0.15)', + boxShadow: `inset 0 0 40px rgba(${accentRgb},0.35), inset 0 0 80px rgba(${accentRgb},0.15)`, }, '50%': { - boxShadow: 'inset 0 0 50px rgba(255,140,0,0.45), inset 0 0 100px rgba(255,100,0,0.22)', + boxShadow: `inset 0 0 50px rgba(${accentRgb},0.45), inset 0 0 100px rgba(${accentRgb},0.22)`, }, }, }} diff --git a/frontend/src/app/pages/Dashboard/Dashboard.tsx b/frontend/src/app/pages/Dashboard/Dashboard.tsx index 2aa9393b..d20b6b97 100644 --- a/frontend/src/app/pages/Dashboard/Dashboard.tsx +++ b/frontend/src/app/pages/Dashboard/Dashboard.tsx @@ -1,4 +1,5 @@ -import React, { useEffect, useCallback, useRef, useState } from 'react'; +import React, { useEffect, useCallback, useRef, useState, useMemo } from 'react'; +import { AnimatePresence, motion } from 'framer-motion'; import { useParams } from 'react-router-dom'; import Box from '@mui/material/Box'; import Typography from '@mui/material/Typography'; @@ -9,10 +10,14 @@ import { fetchSessions, fetchHistory, collapseSession, + closeSession, + duplicateSession, + expandSession, launchAndSendFirstMessage, generateTitle, resumeSession, setExpandedSessionIds, + toggleExpandSession, } from '@/shared/state/agentsSlice'; import type { AgentConfig } from '@/shared/state/agentsSlice'; import { @@ -25,12 +30,25 @@ import { moveCards, resetLayout, setGlowingBrowserCards, + removeViewCard, + removeBrowserCard, + pasteBrowserCard, + placeCard, + setCardPosition, + removeCard, + bringToFront, + setGlowingAgentCard, + clearGlowingAgentCard, + DEFAULT_CARD_W, + DEFAULT_CARD_H, EXPANDED_CARD_MIN_H, + GRID_GAP, } from '@/shared/state/dashboardLayoutSlice'; import { fetchOutputs } from '@/shared/state/outputsSlice'; import { generateDashboardName, updateDashboardThumbnail } from '@/shared/state/dashboardsSlice'; import { dashboardWs } from '@/shared/ws/WebSocketManager'; import { initBrowserCommandHandler } from '@/shared/browserCommandHandler'; +import { clearPendingBrowserUrl, clearPendingFocusAgentId } from '@/shared/state/tempStateSlice'; import AgentCard from './AgentCard'; import DashboardViewCard from './DashboardViewCard'; import BrowserCard from './BrowserCard'; @@ -45,6 +63,7 @@ import type { ContextPath } from '@/app/components/DirectoryBrowser'; import { ElementSelectionProvider, useElementSelection } from '@/app/components/ElementSelectionContext'; import { useDomElementSelector } from '@/app/components/useDomElementSelector'; import SelectionOverlay from '@/app/components/SelectionOverlay'; +import { setClipboardCards, getClipboardCards, type ClipboardCard } from '@/shared/dashboardClipboard'; const SELECT_ATTR = 'data-select-type'; @@ -81,7 +100,11 @@ const DashboardInner: React.FC = () => { const zoomSensitivity = useAppSelector((state) => state.settings.data.zoom_sensitivity); const newAgentShortcut = useAppSelector((state) => state.settings.data.new_agent_shortcut); const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage); + const expandNewChats = useAppSelector((state) => state.settings.data.expand_new_chats_in_dashboard); + const autoRevealSubAgents = useAppSelector((state) => state.settings.data.auto_reveal_sub_agents); const outputs = useAppSelector((state) => state.outputs.items); + const glowingAgentCards = useAppSelector((state) => state.dashboardLayout.glowingAgentCards); + const glowingBrowserCards = useAppSelector((state) => state.dashboardLayout.glowingBrowserCards); const sessionList = Object.values(sessions); const canvas = useCanvasControls(zoomSensitivity); @@ -96,6 +119,8 @@ const DashboardInner: React.FC = () => { const [toolbarOpen, setToolbarOpen] = useState(false); const [highlightedCardId, setHighlightedCardId] = useState(null); const highlightTimerRef = useRef | null>(null); + const [autoFocusSessionId, setAutoFocusSessionId] = useState(null); + const [pendingSelectSessionId, setPendingSelectSessionId] = useState(null); const handleHighlightCard = useCallback((cardId: string) => { if (highlightTimerRef.current) clearTimeout(highlightTimerRef.current); @@ -106,7 +131,35 @@ const DashboardInner: React.FC = () => { }, 2000); }, []); - const spawnOriginsRef = useRef>({}); + useEffect(() => { + if (autoFocusSessionId) { + const timer = setTimeout(() => setAutoFocusSessionId(null), 1500); + return () => clearTimeout(timer); + } + }, [autoFocusSessionId]); + + useEffect(() => { + if (!pendingSelectSessionId) return; + if (!cards[pendingSelectSessionId]) return; + setPendingSelectSessionId(null); + selection.selectCard(pendingSelectSessionId, 'agent', false); + }, [pendingSelectSessionId, cards, selection]); + + const spawnOriginsRef = useRef>({}); + const measuredHeightsRef = useRef>({}); + const [measuredHeightsTick, setMeasuredHeightsTick] = useState(0); + const handleMeasuredHeight = useCallback((sessionId: string, height: number) => { + if (measuredHeightsRef.current[sessionId] !== height) { + measuredHeightsRef.current[sessionId] = height; + setMeasuredHeightsTick((t) => t + 1); + } + }, []); + const revealSpawnedRef = useRef(new Set()); + useEffect(() => { + revealSpawnedRef.current.forEach((id) => { + if (!cards[id]) revealSpawnedRef.current.delete(id); + }); + }, [cards]); const hasFittedRef = useRef(false); const restoredExpandedRef = useRef(false); const canvasStateRef = useRef({ panX: canvas.panX, panY: canvas.panY, zoom: canvas.zoom }); @@ -114,16 +167,16 @@ const DashboardInner: React.FC = () => { // ---- Multi-drag coordination ---- const [multiDragDelta, setMultiDragDelta] = useState<{ dx: number; dy: number } | null>(null); + const [liveDragInfo, setLiveDragInfo] = useState<{ cardId: string; dx: number; dy: number } | null>(null); const activeDragCardRef = useRef(null); const isMultiDragRef = useRef(false); const handleCardDragStart = useCallback((id: string, _type: CardType) => { + activeDragCardRef.current = id; if (selection.isSelected(id)) { - activeDragCardRef.current = id; isMultiDragRef.current = true; } else { selection.deselectAll(); - activeDragCardRef.current = null; isMultiDragRef.current = false; } }, [selection]); @@ -132,6 +185,9 @@ const DashboardInner: React.FC = () => { if (isMultiDragRef.current) { setMultiDragDelta({ dx, dy }); } + if (activeDragCardRef.current) { + setLiveDragInfo({ cardId: activeDragCardRef.current, dx, dy }); + } }, []); const handleCardDragEnd = useCallback((dx: number, dy: number, didDrag: boolean) => { @@ -145,23 +201,34 @@ const DashboardInner: React.FC = () => { activeDragCardRef.current = null; isMultiDragRef.current = false; setMultiDragDelta(null); + setLiveDragInfo(null); }, [selection, dispatch]); const handleCardSelect = useCallback((id: string, type: CardType, shiftKey: boolean) => { selection.selectCard(id, type, shiftKey); }, [selection]); + const handleBringToFront = useCallback((id: string, type: CardType) => { + dispatch(bringToFront({ id, type })); + }, [dispatch]); + // ---- Viewport event handlers (compose pan + marquee) ---- const handleViewportMouseDown = useCallback((e: React.MouseEvent) => { if (e.button === 1) { canvas.handlers.onMouseDown(e); return; } + + if (e.button === 2) { + e.preventDefault(); + canvas.handlers.onMouseDown(e); + return; + } + if (e.button !== 0) return; if (isCardTarget(e.target, e.currentTarget)) return; if (isElementSelectMode) { - // Cmd/Ctrl held → allow panning even in element select mode if (e.metaKey || e.ctrlKey) { canvas.handlers.onMouseDown(e); } @@ -169,10 +236,10 @@ const DashboardInner: React.FC = () => { } if (e.metaKey || e.ctrlKey || canvas.spaceHeld) { - selection.handleCanvasMouseDown(e.nativeEvent); - } else { selection.deselectAll(); canvas.handlers.onMouseDown(e); + } else { + selection.handleCanvasMouseDown(e.nativeEvent); } }, [canvas.handlers, canvas.spaceHeld, selection, isElementSelectMode]); @@ -200,6 +267,20 @@ const DashboardInner: React.FC = () => { return () => { cleanupBrowserHandler(); dashboardWs.disconnect(); }; }, [dispatch, dashboardId]); + const pendingBrowserUrl = useAppSelector((state) => state.tempState.pendingBrowserUrl); + const pendingFocusAgentId = useAppSelector((state) => state.tempState.pendingFocusAgentId); + + useEffect(() => { + if (!dashboardId) return; + (window as any).__openswarm_last_dashboard_id = dashboardId; + }, [dashboardId]); + + useEffect(() => { + if (!pendingBrowserUrl || !layoutInitialized) return; + dispatch(addBrowserCard({ url: pendingBrowserUrl, expandedSessionIds })); + dispatch(clearPendingBrowserUrl()); + }, [pendingBrowserUrl, layoutInitialized, dispatch, expandedSessionIds]); + // Capture a thumbnail screenshot of the dashboard. // Uses Electron's native capturePage for pixel-perfect results. // Captures current viewport as-is (no DOM mutation) to avoid visual flashes. @@ -247,10 +328,25 @@ const DashboardInner: React.FC = () => { useEffect(() => { if (!layoutInitialized || hasFittedRef.current) return; + if (pendingFocusAgentId) return; hasFittedRef.current = true; const timer = setTimeout(() => canvas.actions.fitToView(), 150); return () => clearTimeout(timer); - }, [layoutInitialized, canvas.actions]); + }, [layoutInitialized, canvas.actions, pendingFocusAgentId]); + + useEffect(() => { + if (!pendingFocusAgentId || !layoutInitialized) return; + const agentId = pendingFocusAgentId; + dispatch(clearPendingFocusAgentId()); + hasFittedRef.current = true; + setTimeout(() => { + const card = store.getState().dashboardLayout.cards[agentId]; + if (card) { + canvas.actions.fitToCards([{ x: card.x, y: card.y, width: card.width, height: card.height }], 1.0, true); + handleHighlightCard(agentId); + } + }, 350); + }, [pendingFocusAgentId, layoutInitialized, dispatch, canvas.actions, handleHighlightCard]); useEffect(() => { if (!layoutInitialized || restoredExpandedRef.current) return; @@ -263,7 +359,7 @@ const DashboardInner: React.FC = () => { useEffect(() => { if (!layoutInitialized) return; const dashboardSessionIds = Object.values(sessions) - .filter((s) => s.dashboard_id === dashboardId && s.mode !== 'browser-agent') + .filter((s) => s.dashboard_id === dashboardId && s.mode !== 'browser-agent' && s.mode !== 'invoked-agent' && s.mode !== 'sub-agent') .map((s) => s.id); const liveIds = dashboardSessionIds.sort().join(','); if (liveIds === prevSessionIdsRef.current) return; @@ -271,6 +367,100 @@ const DashboardInner: React.FC = () => { dispatch(reconcileSessions({ sessionIds: dashboardSessionIds, expandedSessionIds })); }, [sessions, layoutInitialized, dispatch, dashboardId, expandedSessionIds]); + // ---- Auto-reveal / collapse / unreveal sub-agent cards ---- + const autoRevealedRef = useRef(new Set()); + const prevSubStatusRef = useRef>({}); + const prevParentStatusRef = useRef>({}); + + useEffect(() => { + if (!layoutInitialized || !autoRevealSubAgents) return; + + const subSessions = Object.values(sessions).filter( + (s) => (s.mode === 'sub-agent' || s.mode === 'invoked-agent') && s.parent_session_id, + ); + + // 1) Auto-reveal newly spawned sub-agents (skip already-terminal ones on load) + for (const sub of subSessions) { + if (autoRevealedRef.current.has(sub.id)) continue; + if (cards[sub.id]) { + autoRevealedRef.current.add(sub.id); + continue; + } + const parentCard = cards[sub.parent_session_id!]; + if (!parentCard) continue; + + const isTerminal = sub.status === 'completed' || sub.status === 'error' || sub.status === 'stopped'; + const parentSession = sessions[sub.parent_session_id!]; + const parentTerminal = parentSession && + (parentSession.status === 'completed' || parentSession.status === 'error' || parentSession.status === 'stopped'); + if (isTerminal && parentTerminal) { + autoRevealedRef.current.add(sub.id); + continue; + } + + autoRevealedRef.current.add(sub.id); + + const targetX = parentCard.x + parentCard.width + GRID_GAP * 12; + let targetY = parentCard.y; + const columnCards = Object.values(cards).filter( + (c) => Math.abs(c.x - targetX) < 50 && c.session_id !== sub.id, + ); + if (columnCards.length > 0) { + const lowestBottom = Math.max( + ...columnCards.map((c) => c.y + Math.max(EXPANDED_CARD_MIN_H, c.height)), + ); + targetY = lowestBottom + GRID_GAP; + } + + dispatch(placeCard({ sessionId: sub.id, x: targetX, y: targetY, width: DEFAULT_CARD_W, height: DEFAULT_CARD_H })); + dispatch(expandSession(sub.id)); + const label = sub.mode === 'sub-agent' ? 'Create Agent' : 'Invoke Agent'; + dispatch(setGlowingAgentCard({ sessionId: sub.id, sourceId: sub.parent_session_id!, label })); + + if (sub.status === 'completed' || sub.status === 'error' || sub.status === 'stopped') { + const subId = sub.id; + setTimeout(() => dispatch(collapseSession(subId)), 2000); + } + } + + // 2) Auto-collapse sub-agents when they complete + const TERMINAL = new Set(['completed', 'error', 'stopped']); + for (const sub of subSessions) { + const prev = prevSubStatusRef.current[sub.id]; + if (prev !== sub.status && TERMINAL.has(sub.status) && cards[sub.id]) { + dispatch(collapseSession(sub.id)); + } + } + const newSubStatuses: Record = {}; + for (const sub of subSessions) { newSubStatuses[sub.id] = sub.status; } + prevSubStatusRef.current = newSubStatuses; + + // 3) Unreveal all sub-agent cards when parent finishes output + const parentIds = new Set(subSessions.map((s) => s.parent_session_id!)); + for (const pid of parentIds) { + const parent = sessions[pid]; + if (!parent) continue; + const prev = prevParentStatusRef.current[pid]; + if (prev !== parent.status && TERMINAL.has(parent.status)) { + const children = subSessions.filter((s) => s.parent_session_id === pid); + for (const child of children) { + if (!cards[child.id]) continue; + dispatch(collapseSession(child.id)); + dispatch(removeCard(child.id)); + setTimeout(() => { + dispatch(clearGlowingAgentCard(child.id)); + }, 500); + } + } + } + const newParentStatuses: Record = {}; + for (const pid of parentIds) { + const parent = sessions[pid]; + if (parent) newParentStatuses[pid] = parent.status; + } + prevParentStatusRef.current = newParentStatuses; + }, [sessions, cards, layoutInitialized, autoRevealSubAgents, dispatch]); + const skipInitialSave = useRef(true); const saveTimerRef = useRef | null>(null); const pendingSaveRef = useRef[0] | null>(null); @@ -326,6 +516,187 @@ const DashboardInner: React.FC = () => { return () => window.removeEventListener('keydown', handleShortcut); }, [newAgentShortcut]); + useEffect(() => { + const handleEnter = (e: KeyboardEvent) => { + if (e.key !== 'Enter') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + if (selection.selectedIds.size !== 1) return; + const [id, type] = selection.selectedIds.entries().next().value!; + if (type !== 'agent') return; + e.preventDefault(); + dispatch(toggleExpandSession(id)); + }; + window.addEventListener('keydown', handleEnter); + return () => window.removeEventListener('keydown', handleEnter); + }, [selection.selectedIds, dispatch]); + + useEffect(() => { + const handleDelete = (e: KeyboardEvent) => { + if (e.key !== 'Backspace' && e.key !== 'Delete') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + if (selection.selectedIds.size === 0) return; + e.preventDefault(); + for (const [id, type] of selection.selectedIds) { + if (type === 'agent') { + dispatch(closeSession({ sessionId: id })); + } else if (type === 'view') { + dispatch(removeViewCard(id)); + } else if (type === 'browser') { + dispatch(removeBrowserCard(id)); + } + } + selection.deselectAll(); + }; + window.addEventListener('keydown', handleDelete); + return () => window.removeEventListener('keydown', handleDelete); + }, [selection, dispatch]); + + useEffect(() => { + const handleCopy = (e: KeyboardEvent) => { + if (!(e.metaKey || e.ctrlKey) || e.key.toLowerCase() !== 'c') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + if (selection.selectedIds.size === 0) return; + + e.preventDefault(); + const copied: ClipboardCard[] = []; + const names: string[] = []; + for (const [id, type] of selection.selectedIds) { + if (type === 'agent') { + const session = sessions[id]; + const card = cards[id]; + if (!session || !card) continue; + copied.push({ + type, id, name: session.name || id, + meta: { name: session.name, status: session.status, model: session.model, mode: session.mode }, + x: card.x, y: card.y, width: card.width, height: card.height, + expanded: expandedSessionIds.includes(id), + }); + names.push(session.name || id); + } else if (type === 'view') { + const output = outputs[id]; + const vc = viewCards[id]; + if (!output || !vc) continue; + copied.push({ + type, id, name: output.name, + meta: { name: output.name, description: output.description }, + x: vc.x, y: vc.y, width: vc.width, height: vc.height, + }); + names.push(output.name); + } else if (type === 'browser') { + const bc = browserCards[id]; + if (!bc) continue; + const activeTab = bc.tabs.find((t) => t.id === bc.activeTabId); + const title = activeTab?.title || 'Browser'; + copied.push({ + type, id, name: title, + meta: { name: title, url: activeTab?.url || bc.url, tabs: bc.tabs }, + x: bc.x, y: bc.y, width: bc.width, height: bc.height, + }); + names.push(title); + } + } + setClipboardCards(copied); + navigator.clipboard.writeText(names.join(', ')).catch(() => {}); + }; + window.addEventListener('keydown', handleCopy); + return () => window.removeEventListener('keydown', handleCopy); + }, [selection.selectedIds, sessions, cards, viewCards, browserCards, outputs, expandedSessionIds]); + + useEffect(() => { + const PASTE_OFFSET = 40; + const handlePaste = async (e: KeyboardEvent) => { + if (!(e.metaKey || e.ctrlKey) || e.key.toLowerCase() !== 'v') return; + const tag = (e.target as HTMLElement)?.tagName; + if (tag === 'INPUT' || tag === 'TEXTAREA' || (e.target as HTMLElement)?.isContentEditable) return; + + const copied = getClipboardCards(); + if (copied.length === 0) return; + e.preventDefault(); + + selection.deselectAll(); + const newSelection = new Map(); + + for (const card of copied) { + const px = card.x + PASTE_OFFSET; + const py = card.y - PASTE_OFFSET; + + if (card.type === 'agent') { + const action = await dispatch(duplicateSession({ sessionId: card.id, dashboardId })); + if (duplicateSession.fulfilled.match(action)) { + const newId = action.payload.id; + dispatch(placeCard({ sessionId: newId, x: px, y: py, width: card.width, height: card.height })); + if (card.expanded) { + dispatch(expandSession(newId)); + } + newSelection.set(newId, 'agent'); + } + } else if (card.type === 'view') { + dispatch(addViewCard({ outputId: card.id, expandedSessionIds, x: px, y: py, width: card.width, height: card.height })); + newSelection.set(card.id, 'view'); + } else if (card.type === 'browser') { + const browserId = `browser-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`; + dispatch(pasteBrowserCard({ + id: browserId, tabs: card.meta.tabs || [], url: card.meta.url || '', + x: px, y: py, width: card.width, height: card.height, + })); + newSelection.set(browserId, 'browser'); + } + } + + if (newSelection.size > 0) { + for (const [id, type] of newSelection) { + selection.selectCard(id, type, true); + } + } + }; + window.addEventListener('keydown', handlePaste); + return () => window.removeEventListener('keydown', handlePaste); + }, [dispatch, dashboardId, expandedSessionIds, selection]); + + const handleBranchFromCard = useCallback( + (sourceSessionId: string, newSessionId: string) => { + const sourceCard = cards[sourceSessionId]; + if (!sourceCard) return; + + const targetX = sourceCard.x + sourceCard.width + GRID_GAP * 12; + let targetY = sourceCard.y; + + const columnCards = Object.values(cards).filter( + (c) => Math.abs(c.x - targetX) < 50 && c.session_id !== newSessionId, + ); + if (columnCards.length > 0) { + const lowestBottom = Math.max( + ...columnCards.map((c) => c.y + Math.max(EXPANDED_CARD_MIN_H, c.height)), + ); + targetY = lowestBottom + GRID_GAP; + } + + spawnOriginsRef.current[newSessionId] = { + x: sourceCard.x, + y: sourceCard.y, + type: 'branch' as const, + }; + + dispatch(placeCard({ + sessionId: newSessionId, + x: targetX, + y: targetY, + width: DEFAULT_CARD_W, + height: DEFAULT_CARD_H, + })); + + if (expandedSessionIds.includes(sourceSessionId)) { + dispatch(expandSession(newSessionId)); + } + + dispatch(setGlowingAgentCard({ sessionId: newSessionId, sourceId: sourceSessionId, label: 'Branch' })); + }, + [cards, dispatch, expandedSessionIds], + ); + const handleNewAgent = useCallback(() => { setToolbarOpen(true); }, []); @@ -376,18 +747,45 @@ const DashboardInner: React.FC = () => { contextPaths: contextPaths?.map((cp) => ({ path: cp.path, type: cp.type })), forcedTools, attachedSkills, - expand: false, + expand: expandNewChats, }), ).then((action) => { if (launchAndSendFirstMessage.fulfilled.match(action)) { const realId = action.payload.session.id; dispatch(generateTitle({ sessionId: realId, prompt })); if (selectedBrowserIds?.length) { - dispatch(setGlowingBrowserCards({ browserIds: selectedBrowserIds, sessionId: realId })); + dispatch(setGlowingBrowserCards({ browserIds: selectedBrowserIds, sessionId: realId, label: 'Use Browser' })); + + if (selectedBrowserIds.length === 1) { + const bc = store.getState().dashboardLayout.browserCards[selectedBrowserIds[0]]; + if (bc) { + dispatch(setCardPosition({ + sessionId: realId, + x: bc.x - DEFAULT_CARD_W - GRID_GAP * 12, + y: bc.y, + })); + } + } } spawnOriginsRef.current[realId] = spawnOriginsRef.current[draftId]; delete spawnOriginsRef.current[draftId]; + if (expandNewChats) { + setAutoFocusSessionId(realId); + } else { + setPendingSelectSessionId(realId); + } + + setTimeout(() => { + const card = store.getState().dashboardLayout.cards[realId]; + if (card) { + const isExp = store.getState().agents.expandedSessionIds.includes(realId); + const height = isExp ? Math.max(EXPANDED_CARD_MIN_H, card.height) : card.height; + canvas.actions.fitToCards([{ x: card.x, y: card.y, width: card.width, height }], 1.0, true); + handleHighlightCard(realId); + } + }, 200); + if (dashboardId) { const currentSessions = store.getState().agents.sessions; const agentCount = Object.values(currentSessions).filter( @@ -408,24 +806,49 @@ const DashboardInner: React.FC = () => { } }); }, - [canvas.viewportRef, dispatch, dashboardId], + [canvas.viewportRef, canvas.actions, dispatch, dashboardId, expandNewChats, handleHighlightCard], ); const handleAddView = useCallback((outputId: string) => { dispatch(addViewCard({ outputId, expandedSessionIds })); - }, [dispatch, expandedSessionIds]); + setTimeout(() => { + const card = store.getState().dashboardLayout.viewCards[outputId]; + if (card) { + canvas.actions.fitToCards([{ x: card.x, y: card.y, width: card.width, height: card.height }], 1.0, true); + handleHighlightCard(outputId); + } + }, 200); + }, [dispatch, expandedSessionIds, canvas.actions, handleHighlightCard]); const handleAddBrowser = useCallback(() => { + const prevIds = new Set(Object.keys(store.getState().dashboardLayout.browserCards)); dispatch(addBrowserCard({ url: browserHomepage, expandedSessionIds })); - }, [dispatch, browserHomepage, expandedSessionIds]); + setTimeout(() => { + const allBrowserCards = store.getState().dashboardLayout.browserCards; + const newId = Object.keys(allBrowserCards).find((id) => !prevIds.has(id)); + if (newId) { + const card = allBrowserCards[newId]; + canvas.actions.fitToCards([{ x: card.x, y: card.y, width: card.width, height: card.height }], 1.0, true); + handleHighlightCard(newId); + } + }, 200); + }, [dispatch, browserHomepage, expandedSessionIds, canvas.actions, handleHighlightCard]); const handleHistoryResume = useCallback((sessionId: string) => { dispatch(resumeSession({ sessionId })).then((action) => { if (resumeSession.fulfilled.match(action)) { - dispatch(collapseSession(sessionId)); + dispatch(expandSession(sessionId)); + setAutoFocusSessionId(sessionId); + setTimeout(() => { + const card = store.getState().dashboardLayout.cards[sessionId]; + if (card) { + canvas.actions.fitToCards([{ x: card.x, y: card.y, width: card.width, height: card.height }], 1.0, true); + handleHighlightCard(sessionId); + } + }, 200); } }); - }, [dispatch]); + }, [dispatch, canvas.actions, handleHighlightCard, setAutoFocusSessionId]); const handleTidy = useCallback(() => { const currentExpanded = store.getState().agents.expandedSessionIds; @@ -444,6 +867,233 @@ const DashboardInner: React.FC = () => { canvas.actions.fitToCards(allRects); }, [dispatch, canvas.actions]); + useEffect(() => { + const DRIFT_THRESHOLD = 60; + + // Group tethered sub-agent cards by source, only including those still in the spawn column + const sourceToSiblings = new Map(); + for (const [id, glow] of Object.entries(glowingAgentCards)) { + const card = cards[id]; + if (!card) continue; + const sourceCard = cards[glow.sourceId]; + if (!sourceCard) continue; + const expectedX = sourceCard.x + sourceCard.width + GRID_GAP * 12; + if (Math.abs(card.x - expectedX) > DRIFT_THRESHOLD) continue; + const list = sourceToSiblings.get(glow.sourceId) ?? []; + list.push(id); + sourceToSiblings.set(glow.sourceId, list); + } + + for (const siblings of sourceToSiblings.values()) { + if (siblings.length < 2) continue; + siblings.sort((a, b) => cards[a].y - cards[b].y); + + let cursor = cards[siblings[0]].y; + for (const id of siblings) { + const card = cards[id]; + const dy = cursor - card.y; + if (Math.abs(dy) > 1) { + dispatch(moveCards({ items: [{ id, type: 'agent' as const }], dx: 0, dy })); + } + const isExpanded = expandedSessionIds.includes(id); + const h = isExpanded + ? Math.max(EXPANDED_CARD_MIN_H, card.height) + : (measuredHeightsRef.current[id] ?? card.height); + cursor += h + GRID_GAP * 2; + } + } + // measuredHeightsTick in deps ensures we re-run once ResizeObserver reports + // the new height after a collapse (avoids stale-height no-ops) + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [expandedSessionIds, glowingAgentCards, cards, dispatch, measuredHeightsTick]); + + useEffect(() => { + const DRIFT_THRESHOLD = 60; + + const sourceToSiblings = new Map(); + for (const [browserId, glow] of Object.entries(glowingBrowserCards)) { + const bc = browserCards[browserId]; + if (!bc) continue; + const sourceCard = cards[glow.sourceId]; + if (!sourceCard) continue; + const expectedX = sourceCard.x + sourceCard.width + GRID_GAP * 12; + if (Math.abs(bc.x - expectedX) > DRIFT_THRESHOLD) continue; + const list = sourceToSiblings.get(glow.sourceId) ?? []; + list.push(browserId); + sourceToSiblings.set(glow.sourceId, list); + } + + for (const siblings of sourceToSiblings.values()) { + if (siblings.length < 2) continue; + siblings.sort((a, b) => browserCards[a].y - browserCards[b].y); + + let cursor = browserCards[siblings[0]].y; + for (const id of siblings) { + const bc = browserCards[id]; + const dy = cursor - bc.y; + if (Math.abs(dy) > 1) { + dispatch(moveCards({ items: [{ id, type: 'browser' as const }], dx: 0, dy })); + } + cursor += bc.height + GRID_GAP * 2; + } + } + }, [glowingBrowserCards, browserCards, cards, dispatch]); + + const TETHER_FADE_MS = 2500; + + const tethers = useMemo(() => { + const ELBOW_RADIUS = 16; + + function elbowPath(x1: number, y1: number, x2: number, y2: number): string { + const dx = x2 - x1; + const dy = y2 - y1; + const midX = x1 + dx / 2; + const r = (Math.abs(dy) < 1 || Math.abs(dx) < ELBOW_RADIUS * 2) + ? 0 + : Math.min(ELBOW_RADIUS, Math.abs(dy) / 2, Math.abs(dx) / 4); + const sy = dy >= 0 ? 1 : -1; + const sx = dx >= 0 ? 1 : -1; + + return [ + `M ${x1},${y1}`, + `H ${midX - sx * r}`, + `Q ${midX},${y1} ${midX},${y1 + sy * r}`, + `V ${y2 - sy * r}`, + `Q ${midX},${y2} ${midX + sx * r},${y2}`, + `H ${x2}`, + ].join(' '); + } + + const agentTethers = Object.entries(glowingAgentCards).map(([copyId, { sourceId, fading, sourceYRatio, label }]) => { + const src = cards[sourceId]; + const dst = cards[copyId]; + if (!src || !dst) return null; + + let srcX = src.x, srcY = src.y; + let dstX = dst.x, dstY = dst.y; + if (liveDragInfo) { + if (liveDragInfo.cardId === sourceId) { srcX += liveDragInfo.dx; srcY += liveDragInfo.dy; } + if (liveDragInfo.cardId === copyId) { dstX += liveDragInfo.dx; dstY += liveDragInfo.dy; } + } + + const srcMeasured = measuredHeightsRef.current[sourceId]; + const srcH = srcMeasured ?? (expandedSessionIds.includes(sourceId) + ? Math.max(EXPANDED_CARD_MIN_H, src.height) + : src.height); + const dstMeasured = measuredHeightsRef.current[copyId]; + const dstH = dstMeasured ?? (expandedSessionIds.includes(copyId) + ? Math.max(EXPANDED_CARD_MIN_H, dst.height) + : dst.height); + + const x1 = srcX + src.width; + const y1 = srcY + srcH * 0.54; + const x2 = dstX; + const y2 = dstY + dstH * (expandedSessionIds.includes(copyId) ? 0.54 : 0.79); + const midX = x1 + (x2 - x1) / 2; + const labelX = midX + (x2 - midX) * 0.15; + const labelY = y2; + + return { + key: copyId, + path: elbowPath(x1, y1, x2, y2), + labelX, + labelY, + label: label || '', + fading, + }; + }).filter(Boolean) as Array<{ key: string; path: string; labelX: number; labelY: number; label: string; fading: boolean }>; + + const browserTethers = Object.entries(glowingBrowserCards).map(([browserId, { sourceId, fading, label }]) => { + const src = cards[sourceId]; + const dst = browserCards[browserId]; + if (!src || !dst) return null; + + let srcX = src.x, srcY = src.y; + let dstX = dst.x, dstY = dst.y; + if (liveDragInfo) { + if (liveDragInfo.cardId === sourceId) { srcX += liveDragInfo.dx; srcY += liveDragInfo.dy; } + if (liveDragInfo.cardId === browserId) { dstX += liveDragInfo.dx; dstY += liveDragInfo.dy; } + } + + const srcMeasured = measuredHeightsRef.current[sourceId]; + const srcH = srcMeasured ?? (expandedSessionIds.includes(sourceId) + ? Math.max(EXPANDED_CARD_MIN_H, src.height) + : src.height); + const dstH = dst.height; + + const srcCx = srcX + src.width / 2; + const dstCx = dstX + dst.width / 2; + + type Anchor = { x: number; y: number; side: 'left' | 'right' | 'top' | 'bottom' }; + const srcAnchors: Anchor[] = [ + { x: srcX + src.width, y: srcY + srcH * 0.54, side: 'right' }, + { x: srcX, y: srcY + srcH * 0.54, side: 'left' }, + { x: srcCx, y: srcY, side: 'top' }, + { x: srcCx, y: srcY + srcH, side: 'bottom' }, + ]; + const dstAnchors: Anchor[] = [ + { x: dstX, y: dstY + dstH * 0.54, side: 'left' }, + { x: dstX + dst.width, y: dstY + dstH * 0.54, side: 'right' }, + { x: dstCx, y: dstY, side: 'top' }, + { x: dstCx, y: dstY + dstH, side: 'bottom' }, + ]; + + let bestSrc = srcAnchors[0], bestDst = dstAnchors[0]; + let bestDist = Infinity; + for (const sa of srcAnchors) { + for (const da of dstAnchors) { + const d = Math.hypot(sa.x - da.x, sa.y - da.y); + if (d < bestDist) { bestDist = d; bestSrc = sa; bestDst = da; } + } + } + + const x1 = bestSrc.x, y1 = bestSrc.y; + const x2 = bestDst.x, y2 = bestDst.y; + + const isVertical = (bestSrc.side === 'top' || bestSrc.side === 'bottom') + && (bestDst.side === 'top' || bestDst.side === 'bottom'); + + let pathD: string; + if (isVertical) { + const dx = x2 - x1; + const dy = y2 - y1; + const midY = y1 + dy / 2; + const r = (Math.abs(dx) < 1 || Math.abs(dy) < ELBOW_RADIUS * 2) + ? 0 + : Math.min(ELBOW_RADIUS, Math.abs(dx) / 2, Math.abs(dy) / 4); + const sx = dx >= 0 ? 1 : -1; + const sy = dy >= 0 ? 1 : -1; + pathD = [ + `M ${x1},${y1}`, + `V ${midY - sy * r}`, + `Q ${x1},${midY} ${x1 + sx * r},${midY}`, + `H ${x2 - sx * r}`, + `Q ${x2},${midY} ${x2},${midY + sy * r}`, + `V ${y2}`, + ].join(' '); + } else { + pathD = elbowPath(x1, y1, x2, y2); + } + + const midX = x1 + (x2 - x1) / 2; + const midY = y1 + (y2 - y1) / 2; + const labelX = isVertical ? midX : midX + (x2 - midX) * 0.15; + const labelY = isVertical ? midY + (y2 - midY) * 0.15 : y2; + + return { + key: `browser-${browserId}`, + path: pathD, + labelX, + labelY, + label: label || '', + fading, + }; + }).filter(Boolean) as Array<{ key: string; path: string; labelX: number; labelY: number; label: string; fading: boolean }>; + + return [...agentTethers, ...browserTethers]; + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [glowingAgentCards, glowingBrowserCards, cards, browserCards, expandedSessionIds, liveDragInfo, measuredHeightsTick]); + const dotSize = Math.max(1, 1.5 * canvas.zoom); const dotSpacing = 24 * canvas.zoom; @@ -486,6 +1136,7 @@ const DashboardInner: React.FC = () => { onMouseDown={handleViewportMouseDown} onMouseMove={handleViewportMouseMove} onMouseUp={handleViewportMouseUp} + onContextMenu={(e) => e.preventDefault()} sx={{ position: 'absolute', inset: 0, @@ -493,7 +1144,7 @@ const DashboardInner: React.FC = () => { cursor: canvas.isPanning ? 'grabbing' : (canvas.spaceHeld || canvas.cmdHeld) - ? 'crosshair' + ? 'grab' : selection.marquee ? 'crosshair' : 'default', @@ -540,11 +1191,168 @@ const DashboardInner: React.FC = () => { position: 'relative', }} > + {/* Tether lines between branched cards */} + {tethers.length > 0 && ( + + + + + + + + + + + + + + + {tethers.map((t) => ( + + + + + {t.label && ( + + + + {t.label} + + + )} + + ))} + + )} + {Object.values(cards).map((card) => { const session = sessions[card.session_id]; if (!session) return null; - const origin = spawnOriginsRef.current[session.id]; - if (origin) delete spawnOriginsRef.current[session.id]; + + let origin = spawnOriginsRef.current[session.id]; + if (origin) { + delete spawnOriginsRef.current[session.id]; + } else { + const glow = glowingAgentCards[session.id]; + if (glow && !revealSpawnedRef.current.has(session.id)) { + revealSpawnedRef.current.add(session.id); + const srcCard = cards[glow.sourceId]; + if (srcCard) { + const srcH = measuredHeightsRef.current[glow.sourceId] + ?? (expandedSessionIds.includes(glow.sourceId) + ? Math.max(EXPANDED_CARD_MIN_H, srcCard.height) + : srcCard.height); + origin = { + x: srcCard.x + srcCard.width, + y: srcCard.y + srcH / 2, + type: 'branch' as const, + }; + } + } + } + + let exitTarget: { x: number; y: number } | undefined; + const glow = glowingAgentCards[session.id]; + if (glow) { + const srcCard = cards[glow.sourceId]; + if (srcCard) { + const srcH = measuredHeightsRef.current[glow.sourceId] + ?? (expandedSessionIds.includes(glow.sourceId) + ? Math.max(EXPANDED_CARD_MIN_H, srcCard.height) + : srcCard.height); + exitTarget = { + x: srcCard.x + srcCard.width, + y: srcCard.y + srcH / 2, + }; + } + } + + let snapColumn: { x: number; width: number } | undefined; + if (glow) { + const srcCard = cards[glow.sourceId]; + if (srcCard) { + snapColumn = { + x: srcCard.x + srcCard.width + GRID_GAP * 12, + width: DEFAULT_CARD_W, + }; + } + } + return ( { cardY={card.y} cardWidth={card.width} cardHeight={card.height} + cardZOrder={card.zOrder ?? 0} zoom={canvas.zoom} spawnFrom={origin} + exitTarget={exitTarget} isSelected={selection.isSelected(session.id)} isHighlighted={highlightedCardId === session.id} multiDragDelta={multiDragDelta} @@ -563,9 +1373,15 @@ const DashboardInner: React.FC = () => { onDragStart={handleCardDragStart} onDragMove={handleCardDragMove} onDragEnd={handleCardDragEnd} + onBranch={handleBranchFromCard} + onMeasuredHeight={handleMeasuredHeight} + snapColumn={snapColumn} + autoFocusInput={autoFocusSessionId === session.id} + onBringToFront={handleBringToFront} /> ); })} + {Object.values(viewCards).map((vc) => { const output = outputs[vc.output_id]; if (!output) return null; @@ -577,7 +1393,9 @@ const DashboardInner: React.FC = () => { cardY={vc.y} cardWidth={vc.width} cardHeight={vc.height} + cardZOrder={vc.zOrder ?? 0} zoom={canvas.zoom} + cmdHeld={canvas.cmdHeld} isSelected={selection.isSelected(vc.output_id)} isHighlighted={highlightedCardId === vc.output_id} multiDragDelta={multiDragDelta} @@ -585,6 +1403,7 @@ const DashboardInner: React.FC = () => { onDragStart={handleCardDragStart} onDragMove={handleCardDragMove} onDragEnd={handleCardDragEnd} + onBringToFront={handleBringToFront} /> ); })} @@ -598,7 +1417,9 @@ const DashboardInner: React.FC = () => { cardY={bc.y} cardWidth={bc.width} cardHeight={bc.height} + cardZOrder={bc.zOrder ?? 0} zoom={canvas.zoom} + cmdHeld={canvas.cmdHeld} isSelected={selection.isSelected(bc.browser_id)} isHighlighted={highlightedCardId === bc.browser_id} multiDragDelta={multiDragDelta} @@ -606,6 +1427,7 @@ const DashboardInner: React.FC = () => { onDragStart={handleCardDragStart} onDragMove={handleCardDragMove} onDragEnd={handleCardDragEnd} + onBringToFront={handleBringToFront} /> ))} {/* Marquee selection rectangle */} diff --git a/frontend/src/app/pages/Dashboard/DashboardHeader.tsx b/frontend/src/app/pages/Dashboard/DashboardHeader.tsx index 417b6315..b6429b9f 100644 --- a/frontend/src/app/pages/Dashboard/DashboardHeader.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardHeader.tsx @@ -92,7 +92,7 @@ const DashboardHeader: React.FC = ({ const handleFocus = useCallback( (cardId: string, card: { x: number; y: number; width: number; height: number }) => { - canvasActions.fitToCards([card], 1.0); + canvasActions.fitToCards([card], 1.0, true); onHighlightCard?.(cardId); setExpanded(false); }, diff --git a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx index 38b000b0..0b092864 100644 --- a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx @@ -42,6 +42,7 @@ interface Props { dashboardId?: string; } +const TOOLBAR_OWNER_ID = '__toolbar__'; const BTN = 40; const WarmTooltip = styled( @@ -90,8 +91,19 @@ const DashboardToolbar = React.forwardRef( const searchInputRef = useRef(null); const historyInputRef = useRef(null); const historyListRef = useRef(null); - const [mode, setMode] = useState('agent'); - const [model, setModel] = useState('sonnet'); + const defaultMode = useAppSelector((s) => s.settings.data.default_mode); + const defaultModel = useAppSelector((s) => s.settings.data.default_model); + const [mode, setMode] = useState(defaultMode || 'agent'); + const [model, setModel] = useState(defaultModel || 'sonnet'); + const [provider, setProvider] = useState('anthropic'); + const settingsApplied = useRef(false); + useEffect(() => { + if (!settingsApplied.current) { + setMode(defaultMode || 'agent'); + setModel(defaultModel || 'sonnet'); + settingsApplied.current = true; + } + }, [defaultMode, defaultModel]); const [viewPickerOpen, setViewPickerOpen] = useState(false); const [viewSearch, setViewSearch] = useState(''); const [historyOpen, setHistoryOpen] = useState(false); @@ -204,13 +216,23 @@ const DashboardToolbar = React.forwardRef( const isExpanded = inputOpen || viewPickerOpen || historyOpen; + const autoSelectOnNew = useAppSelector((s) => s.settings.data.auto_select_mode_on_new_agent); const prevInputOpenRef = useRef(inputOpen); useEffect(() => { - if (prevInputOpenRef.current && !inputOpen && elementSelection?.selectMode) { - elementSelection.setSelectMode(false); + if (prevInputOpenRef.current && !inputOpen && elementSelection) { + elementSelection.clearOwnerElements(TOOLBAR_OWNER_ID); + if (elementSelection.selectMode && elementSelection.activeOwnerId === TOOLBAR_OWNER_ID) { + elementSelection.setSelectMode(false); + } + } + if (!prevInputOpenRef.current && inputOpen && autoSelectOnNew && elementSelection) { + elementSelection.clearOwnerElements(TOOLBAR_OWNER_ID); + elementSelection.setActiveOwnerId(TOOLBAR_OWNER_ID); + elementSelection.setExcludeSelectId(null); + elementSelection.setSelectMode(true); } prevInputOpenRef.current = inputOpen; - }, [inputOpen, elementSelection]); + }, [inputOpen, elementSelection, autoSelectOnNew]); useEffect(() => { if (!isExpanded) return; @@ -226,21 +248,44 @@ const DashboardToolbar = React.forwardRef( useEffect(() => { if (!isExpanded) return; - const handleClick = (e: MouseEvent) => { - if (elementSelection?.selectMode) return; + let downPos: { x: number; y: number; target: Node } | null = null; + const DRAG_THRESHOLD = 5; + + const handleDown = (e: MouseEvent) => { const target = e.target as Node; if (containerRef.current && !containerRef.current.contains(target)) { - const el = target instanceof Element ? target : target.parentElement; - if (el?.closest('[role="dialog"], [role="presentation"], .MuiModal-root, .MuiPopover-root')) { - return; - } - handleDismiss(); + downPos = { x: e.clientX, y: e.clientY, target }; + } else { + downPos = null; } }; - const t = setTimeout(() => document.addEventListener('mousedown', handleClick), 50); + + const handleUp = (e: MouseEvent) => { + if (!downPos) return; + const dx = e.clientX - downPos.x; + const dy = e.clientY - downPos.y; + const target = downPos.target; + downPos = null; + if (Math.abs(dx) > DRAG_THRESHOLD || Math.abs(dy) > DRAG_THRESHOLD) return; + + const el = target instanceof Element ? target : (target as Node).parentElement; + if (el?.closest('[role="dialog"], [role="presentation"], .MuiModal-root, .MuiPopover-root')) { + return; + } + if (elementSelection?.selectMode && el?.closest('[data-select-type]')) { + return; + } + handleDismiss(); + }; + + const t = setTimeout(() => { + document.addEventListener('mousedown', handleDown, true); + document.addEventListener('mouseup', handleUp, true); + }, 50); return () => { clearTimeout(t); - document.removeEventListener('mousedown', handleClick); + document.removeEventListener('mousedown', handleDown, true); + document.removeEventListener('mouseup', handleUp, true); }; }, [isExpanded, handleDismiss, elementSelection?.selectMode]); @@ -266,10 +311,14 @@ const DashboardToolbar = React.forwardRef( e.preventDefault(); handleOpenHistory(); } + if (e.metaKey && e.key.toLowerCase() === 'n' && !e.ctrlKey && !e.shiftKey && !e.altKey) { + e.preventDefault(); + onAddBrowser(); + } }; window.addEventListener('keydown', handleKey); return () => window.removeEventListener('keydown', handleKey); - }, [handleOpenViewPicker, handleOpenHistory]); + }, [handleOpenViewPicker, handleOpenHistory, onAddBrowser]); useEffect(() => { if (!historyOpen) return; @@ -317,8 +366,11 @@ const DashboardToolbar = React.forwardRef( onModeChange={setMode} model={model} onModelChange={setModel} + provider={provider} + onProviderChange={setProvider} embedded autoFocus + sessionId={TOOLBAR_OWNER_ID} /> ) : historyOpen ? ( @@ -596,6 +648,39 @@ const DashboardToolbar = React.forwardRef( + + Browser ⌘N + + } + > + + + + + ( - - Browser - - } - > - - - - - {placeholderItems.map(({ icon: PlaceholderIcon, label, sub }) => ( void; onDragMove?: (dx: number, dy: number) => void; onDragEnd?: (dx: number, dy: number, didDrag: boolean) => void; + cardZOrder?: number; + onBringToFront?: (id: string, type: 'agent' | 'view' | 'browser') => void; } const DashboardViewCard: React.FC = ({ - output, cardX, cardY, cardWidth, cardHeight, zoom = 1, + output, cardX, cardY, cardWidth, cardHeight, zoom = 1, cmdHeld = false, isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd, + cardZOrder = 0, onBringToFront, }) => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); + const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); const previewRef = useRef(null); const [inputData, setInputData] = useState>(() => getDefault(output.input_schema)); @@ -77,6 +83,7 @@ const DashboardViewCard: React.FC = ({ const justDraggedRef = useRef(false); const handleDragPointerDown = useCallback((e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY }; @@ -132,6 +139,7 @@ const DashboardViewCard: React.FC = ({ const handleResizeDown = useCallback( (dir: ResizeDir) => (e: React.PointerEvent) => { + if (e.button !== 0) return; e.preventDefault(); e.stopPropagation(); resizeRef.current = { @@ -254,6 +262,7 @@ const DashboardViewCard: React.FC = ({ data-select-type="view-card" data-select-id={output.id} data-select-meta={JSON.stringify({ name: output.name, description: output.description })} + onPointerDownCapture={() => onBringToFront?.(output.id, 'view')} onClick={(e: React.MouseEvent) => { if (justDraggedRef.current) return; onCardSelect?.(output.id, 'view', e.shiftKey); @@ -279,7 +288,7 @@ const DashboardViewCard: React.FC = ({ overflow: 'hidden', display: 'flex', flexDirection: 'column', - zIndex: isHighlighted ? 50 : (isDragging || isResizing) ? 100 : 1, + zIndex: (isDragging || isResizing) ? 999999 : cardZOrder, transition: noTransition ? 'none' : 'box-shadow 0.2s', '&:hover .resize-handle': { opacity: 1 }, ...(isHighlighted && { @@ -304,9 +313,10 @@ const DashboardViewCard: React.FC = ({ }), }} > - {/* Selection overlay – blocks content interaction while selected, enabling drag from anywhere */} + {/* Selection overlay – blocks click interaction while selected, enabling drag from anywhere */} {isSelected && ( = ({ {/* Preview body */} + {cmdHeld && !isSelected && ( + + )} (null); // Wheel zoom centered on cursor useEffect(() => { @@ -207,6 +208,10 @@ export function useCanvasControls(zoomSensitivity: number = 50) { return () => window.removeEventListener('mouseup', onUp); }, []); + useEffect(() => { + return () => { if (animFrameRef.current) cancelAnimationFrame(animFrameRef.current); }; + }, []); + const zoomIn = useCallback(() => { setState((prev) => { const newZoom = clamp(prev.zoom * ZOOM_IN_FACTOR, MIN_ZOOM, MAX_ZOOM); @@ -276,7 +281,9 @@ export function useCanvasControls(zoomSensitivity: number = 50) { }); }, []); - const fitToCards = useCallback((cardRects: Array<{ x: number; y: number; width: number; height: number }>, maxZoom?: number) => { + const fitToCards = useCallback((cardRects: Array<{ x: number; y: number; width: number; height: number }>, maxZoom?: number, animate?: boolean) => { + if (animFrameRef.current) { cancelAnimationFrame(animFrameRef.current); animFrameRef.current = null; } + const viewport = viewportRef.current; if (!viewport || cardRects.length === 0) { setState({ panX: 0, panY: 0, zoom: 1 }); @@ -303,11 +310,34 @@ export function useCanvasControls(zoomSensitivity: number = 50) { const availW = vRect.width - FIT_PADDING * 2; const availH = vRect.height - FIT_PADDING * 2; const ceiling = maxZoom ?? MAX_ZOOM; - const newZoom = clamp(Math.min(availW / contentWidth, availH / contentHeight), MIN_ZOOM, ceiling); - const newPanX = (vRect.width - contentWidth * newZoom) / 2 - minX * newZoom; - const newPanY = (vRect.height - contentHeight * newZoom) / 2 - minY * newZoom; + const targetZoom = clamp(Math.min(availW / contentWidth, availH / contentHeight), MIN_ZOOM, ceiling); + const targetPanX = (vRect.width - contentWidth * targetZoom) / 2 - minX * targetZoom; + const targetPanY = (vRect.height - contentHeight * targetZoom) / 2 - minY * targetZoom; - setState({ panX: newPanX, panY: newPanY, zoom: newZoom }); + if (!animate) { + setState({ panX: targetPanX, panY: targetPanY, zoom: targetZoom }); + return; + } + + const start = { ...stateRef.current }; + const startTime = performance.now(); + const duration = 320; + + const step = (now: number) => { + const t = Math.min((now - startTime) / duration, 1); + const ease = 1 - Math.pow(1 - t, 3); + setState({ + panX: start.panX + (targetPanX - start.panX) * ease, + panY: start.panY + (targetPanY - start.panY) * ease, + zoom: start.zoom + (targetZoom - start.zoom) * ease, + }); + if (t < 1) { + animFrameRef.current = requestAnimationFrame(step); + } else { + animFrameRef.current = null; + } + }; + animFrameRef.current = requestAnimationFrame(step); }, []); const handlers = useMemo(() => ({ diff --git a/frontend/src/app/pages/Dashboard/useDashboardSelection.ts b/frontend/src/app/pages/Dashboard/useDashboardSelection.ts index 6344d975..aa84f6e2 100644 --- a/frontend/src/app/pages/Dashboard/useDashboardSelection.ts +++ b/frontend/src/app/pages/Dashboard/useDashboardSelection.ts @@ -79,7 +79,10 @@ export function useDashboardSelection( } return next; } - return new Map([[id, type]]); + if (prev.has(id)) { + return new Map(); + } + return prev; }); }, [], @@ -152,7 +155,7 @@ export function useDashboardSelection( const handleCanvasMouseDown = useCallback( (e: MouseEvent) => { - if (e.button !== 0) return; + if (e.button !== 0 && e.button !== 2) return; marqueeOriginRef.current = { screenX: e.clientX, screenY: e.clientY }; isDraggingMarqueeRef.current = false; diff --git a/frontend/src/app/pages/Dashboard/useOverlayScrollPassthrough.ts b/frontend/src/app/pages/Dashboard/useOverlayScrollPassthrough.ts new file mode 100644 index 00000000..9107b8f5 --- /dev/null +++ b/frontend/src/app/pages/Dashboard/useOverlayScrollPassthrough.ts @@ -0,0 +1,80 @@ +import { useRef, useEffect } from 'react'; + +/** + * Attaches a native wheel listener to an overlay element that forwards scroll + * events to whatever scrollable content sits beneath it, while still letting + * the overlay capture pointer events (click / drag). Pinch-zoom (ctrl/meta + + * wheel) is left alone so the canvas zoom still works. + * + * Handles two cases: + * 1. Regular DOM scrollable containers — uses `scrollBy` directly. + * 2. Electron `` elements — executes JS inside the webview to scroll + * the element at the cursor position. + */ +export function useOverlayScrollPassthrough(active: boolean) { + const ref = useRef(null); + + useEffect(() => { + const el = ref.current; + if (!active || !el) return; + + const handleWheel = (e: WheelEvent) => { + if (e.ctrlKey || e.metaKey) return; + + el.style.pointerEvents = 'none'; + const underneath = document.elementFromPoint(e.clientX, e.clientY); + el.style.pointerEvents = ''; + + let dx = e.deltaX; + let dy = e.deltaY; + if (e.deltaMode === 1) { + dx *= 20; + dy *= 20; + } + + let node = underneath as HTMLElement | null; + while (node) { + if (node.tagName === 'WEBVIEW') { + e.stopPropagation(); + e.preventDefault(); + const rect = node.getBoundingClientRect(); + const relX = Math.round(e.clientX - rect.left); + const relY = Math.round(e.clientY - rect.top); + (node as any).executeJavaScript?.( + `(function(){` + + `var el=document.elementFromPoint(${relX},${relY});` + + `while(el){` + + `var s=getComputedStyle(el);` + + `if((s.overflowY==='auto'||s.overflowY==='scroll')&&el.scrollHeight>el.clientHeight){el.scrollBy(${dx},${dy});return}` + + `if((s.overflowX==='auto'||s.overflowX==='scroll')&&el.scrollWidth>el.clientWidth){el.scrollBy(${dx},${dy});return}` + + `el=el.parentElement}` + + `window.scrollBy(${dx},${dy})` + + `})()` + ).catch(() => {}); + return; + } + + const cs = getComputedStyle(node); + const canScrollY = + node.scrollHeight > node.clientHeight && + (cs.overflowY === 'auto' || cs.overflowY === 'scroll'); + const canScrollX = + node.scrollWidth > node.clientWidth && + (cs.overflowX === 'auto' || cs.overflowX === 'scroll'); + + if (canScrollY || canScrollX) { + e.stopPropagation(); + e.preventDefault(); + node.scrollBy(dx, dy); + return; + } + node = node.parentElement; + } + }; + + el.addEventListener('wheel', handleWheel, { passive: false }); + return () => el.removeEventListener('wheel', handleWheel); + }, [active]); + + return ref; +} diff --git a/frontend/src/app/pages/Modes/Modes.tsx b/frontend/src/app/pages/Modes/Modes.tsx index e7e75627..2b024b81 100644 --- a/frontend/src/app/pages/Modes/Modes.tsx +++ b/frontend/src/app/pages/Modes/Modes.tsx @@ -27,6 +27,7 @@ import DeleteIcon from '@mui/icons-material/Delete'; import TuneIcon from '@mui/icons-material/Tune'; import LockIcon from '@mui/icons-material/Lock'; import ArrowForwardIcon from '@mui/icons-material/ArrowForward'; +import RestoreIcon from '@mui/icons-material/Restore'; import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined'; import QuestionAnswerOutlinedIcon from '@mui/icons-material/QuestionAnswerOutlined'; import MapOutlinedIcon from '@mui/icons-material/MapOutlined'; @@ -37,6 +38,7 @@ import { createMode, updateMode, deleteMode, + resetMode, Mode, } from '@/shared/state/modesSlice'; import { fetchBuiltinTools, fetchTools } from '@/shared/state/toolsSlice'; @@ -105,7 +107,7 @@ const ALL_BUILTIN_TOOL_NAMES = ['Read', 'Edit', 'Write', 'Bash', 'Glob', 'Grep', const Modes: React.FC = () => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); - const { items, loading } = useAppSelector((s) => s.modes); + const { items, builtinDefaults, loading } = useAppSelector((s) => s.modes); const toolItems = useAppSelector((s) => s.tools.items); const modes = useMemo(() => Object.values(items), [items]); @@ -174,6 +176,45 @@ const Modes: React.FC = () => { await dispatch(deleteMode(id)); }; + const editingIsBuiltin = editingId ? items[editingId]?.is_builtin ?? false : false; + + const hasDiverged = useMemo(() => { + if (!editingId || !editingIsBuiltin) return false; + const defaults = builtinDefaults[editingId]; + if (!defaults) return false; + const current = items[editingId]; + if (!current) return false; + return ( + current.name !== defaults.name || + current.description !== defaults.description || + (current.system_prompt ?? '') !== (defaults.system_prompt ?? '') || + JSON.stringify(current.tools) !== JSON.stringify(defaults.tools) || + (current.default_next_mode ?? '') !== (defaults.default_next_mode ?? '') || + current.icon !== defaults.icon || + current.color !== defaults.color || + (current.default_folder ?? '') !== (defaults.default_folder ?? '') + ); + }, [editingId, editingIsBuiltin, items, builtinDefaults]); + + const handleReset = async () => { + if (!editingId) return; + const action = await dispatch(resetMode(editingId)); + if (resetMode.fulfilled.match(action)) { + const m = action.payload; + setForm({ + name: m.name, + description: m.description, + system_prompt: m.system_prompt ?? '', + tools: m.tools ?? [], + toolsEnabled: m.tools !== null, + default_next_mode: m.default_next_mode ?? '', + icon: m.icon, + color: m.color, + default_folder: m.default_folder ?? '', + }); + } + }; + const otherModes = modes.filter((m) => m.id !== editingId); return ( @@ -505,23 +546,46 @@ const Modes: React.FC = () => { - - - + + + {editingIsBuiltin && ( + + + + + + )} + + + + + diff --git a/frontend/src/app/pages/Settings/Settings.tsx b/frontend/src/app/pages/Settings/Settings.tsx index 70a14a8d..2abc91ad 100644 --- a/frontend/src/app/pages/Settings/Settings.tsx +++ b/frontend/src/app/pages/Settings/Settings.tsx @@ -11,6 +11,7 @@ import InputAdornment from '@mui/material/InputAdornment'; import ToggleButton from '@mui/material/ToggleButton'; import ToggleButtonGroup from '@mui/material/ToggleButtonGroup'; import Slider from '@mui/material/Slider'; +import Switch from '@mui/material/Switch'; import Snackbar from '@mui/material/Snackbar'; import Alert from '@mui/material/Alert'; import Tab from '@mui/material/Tab'; @@ -39,11 +40,495 @@ import Collapse from '@mui/material/Collapse'; import OpenInNewIcon from '@mui/icons-material/OpenInNew'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { updateSettings, closeSettingsModal, resetSystemPrompt, AppSettings, DEFAULT_SYSTEM_PROMPT } from '@/shared/state/settingsSlice'; +import { fetchModels } from '@/shared/state/modelsSlice'; import { setChecking, setUpdateError } from '@/shared/state/updateSlice'; import { fetchModes } from '@/shared/state/modesSlice'; import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; import DirectoryBrowser from '@/app/components/DirectoryBrowser'; import { CommandsContent } from '@/app/pages/Commands/Commands'; +import { API_BASE } from '@/shared/config'; + +// ── Copilot Auth Button ── +const CopilotAuthButton: React.FC = () => { + const c = useClaudeTokens(); + const [status, setStatus] = useState<'idle' | 'waiting' | 'connected' | 'error'>('idle'); + const [userCode, setUserCode] = useState(''); + const [username, setUsername] = useState(''); + const [error, setError] = useState(''); + + // Check if already connected + useEffect(() => { + fetch(`${API_BASE}/agents/copilot/models`) + .then(r => r.json()) + .then(d => { + if (d.models && d.models.length > 0) setStatus('connected'); + }) + .catch(() => {}); + }, []); + + const startAuth = async () => { + setStatus('waiting'); + setError(''); + try { + const resp = await fetch(`${API_BASE}/agents/copilot/start-auth`, { method: 'POST' }); + const data = await resp.json(); + setUserCode(data.user_code); + window.open(data.verification_uri, '_blank'); + + // Poll for completion + const deviceCode = data.device_code; + const poll = setInterval(async () => { + try { + const r = await fetch(`${API_BASE}/agents/copilot/poll-auth`, { + method: 'POST', + headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ device_code: deviceCode }), + }); + const d = await r.json(); + if (d.status === 'connected') { + clearInterval(poll); + setStatus('connected'); + setUsername(d.username || ''); + } + } catch {} + }, 5000); + + // Timeout after 5 minutes + setTimeout(() => { clearInterval(poll); if (status === 'waiting') { setStatus('error'); setError('Auth timed out'); } }, 300000); + } catch (e: any) { + setStatus('error'); + setError(e.message || 'Failed to start auth'); + } + }; + + const disconnect = async () => { + await fetch(`${API_BASE}/agents/copilot/disconnect`, { method: 'POST' }); + setStatus('idle'); + setUsername(''); + }; + + if (status === 'connected') { + return ( + + + + Connected{username ? ` as @${username}` : ''} + + + Disconnect + + + ); + } + + if (status === 'waiting') { + return ( + + + Enter code {userCode} at github.com/login/device + + Waiting for authorization... + + ); + } + + return ( + + + {error && {error}} + + ); +}; + +// ── Subscription Provider Card ── +const SUBSCRIPTION_PROVIDERS = [ + { id: 'claude', name: 'Claude Pro / Max', desc: 'Sonnet, Opus, Haiku — use your Anthropic subscription', color: '#E8927A', preview: false }, + { id: 'gemini-cli', name: 'Gemini Advanced', desc: 'Gemini 2.5 Pro and Flash — use your Google subscription', color: '#4285F4', preview: true }, + { id: 'codex', name: 'ChatGPT Plus / Pro', desc: 'GPT-5.4, o3, o4-mini — use your OpenAI subscription', color: '#74AA9C', preview: true }, + { id: 'github', name: 'GitHub Copilot', desc: 'Claude + GPT models via your Copilot subscription', color: '#8B949E', preview: true }, +]; + +const SubscriptionCard: React.FC<{ provider: typeof SUBSCRIPTION_PROVIDERS[0]; connected: boolean; onConnect: () => void; onDisconnect: () => void; connecting: boolean; userCode?: string }> = ({ provider, connected, onConnect, onDisconnect, connecting, userCode }) => { + const c = useClaudeTokens(); + const isPreview = (provider as any).preview; + return ( + + + + + + {provider.name} + {provider.desc} + + + {isPreview ? ( + + Coming soon + + ) : connected ? ( + + Disconnect + + ) : connecting && userCode ? ( + + Enter code: + {userCode} + + ) : ( + + )} + + + ); +}; + +const SubscriptionCards: React.FC = () => { + const c = useClaudeTokens(); + const [status, setStatus] = useState(null); + const [connecting, setConnecting] = useState(null); + const [userCode, setUserCode] = useState(''); + const [pollTimer, setPollTimer] = useState(null); + + const fetchStatus = () => { + fetch(`${API_BASE}/agents/subscriptions/status`) + .then(r => r.json()) + .then(setStatus) + .catch(() => setStatus({ running: false, providers: [], models: [] })); + }; + + useEffect(() => { fetchStatus(); }, []); + + const isConnected = (providerId: string) => { + if (!status?.providers) return false; + const connections = status.providers?.connections || (Array.isArray(status.providers) ? status.providers : []); + return connections.some((p: any) => p.provider === providerId && p.isActive); + }; + + const handleConnect = async (providerId: string) => { + setConnecting(providerId); + setUserCode(''); + try { + const r = await fetch(`${API_BASE}/agents/subscriptions/connect`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ provider: providerId }), + }); + const data = await r.json(); + + if (data.flow === 'device_code') { + // Device code flow (GitHub, Qwen, etc.) — show code, poll + const code = data.user_code || ''; + setUserCode(code); + if (data.verification_uri) window.open(data.verification_uri, '_blank'); + + const timer = setInterval(async () => { + try { + const pr = await fetch(`${API_BASE}/agents/subscriptions/poll`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ provider: providerId, device_code: data.device_code, code_verifier: data.code_verifier, extra_data: data.extra_data }), + }); + const pd = await pr.json(); + if (pd.success) { + clearInterval(timer); + setConnecting(null); + setUserCode(''); + fetchStatus(); + } + } catch {} + }, 5000); + setPollTimer(timer); + setTimeout(() => { clearInterval(timer); setConnecting(null); setUserCode(''); }, 300000); + + } else if (data.flow === 'authorization_code') { + // Open auth URL as popup — window.opener lets callback page postMessage back + const popup = window.open(data.auth_url, 'oauth_connect', 'width=600,height=700'); + + const msgHandler = async (event: MessageEvent) => { + const d = event.data; + const callbackData = d?.type === 'oauth_callback' ? d.data : d; + if (callbackData?.code) { + window.removeEventListener('message', msgHandler); + clearInterval(statusPoller); + if (popup && !popup.closed) popup.close(); + try { + await fetch(`${API_BASE}/agents/subscriptions/exchange`, { + method: 'POST', headers: { 'Content-Type': 'application/json' }, + body: JSON.stringify({ + provider: providerId, code: callbackData.code, + redirect_uri: data.redirect_uri, code_verifier: data.code_verifier, + state: callbackData.state || data.state, + }), + }); + } catch {} + setConnecting(null); + fetchStatus(); + } + }; + window.addEventListener('message', msgHandler); + + const statusPoller = setInterval(async () => { + try { + const sr = await fetch(`${API_BASE}/agents/subscriptions/status`); + const sd = await sr.json(); + const connections = sd.providers?.connections || []; + if (connections.some((p: any) => p.provider === providerId && p.isActive)) { + clearInterval(statusPoller); + window.removeEventListener('message', msgHandler); + setConnecting(null); + fetchStatus(); + } + } catch {} + }, 2000); + setPollTimer(statusPoller); + setTimeout(() => { clearInterval(statusPoller); window.removeEventListener('message', msgHandler); setConnecting(null); }, 300000); + + } else { + setConnecting(null); + } + } catch { setConnecting(null); } + }; + + const handleDisconnect = async (providerId: string) => { + // TODO: implement disconnect via 9Router API + fetchStatus(); + }; + + if (!status?.running) { + return ( + + + Starting subscription service... + + + This connects your existing AI subscriptions. If this doesn't load, make sure Node.js is installed. + + + ); + } + + return ( + + {SUBSCRIPTION_PROVIDERS.map(p => ( + handleConnect(p.id)} + onDisconnect={() => handleDisconnect(p.id)} + connecting={connecting === p.id} + userCode={connecting === p.id ? userCode : undefined} + /> + ))} + + ); +}; + +// ── Pixel Bar ── +const PIXEL_SALMON = ['#C46B57', '#D4795F', '#E8927A', '#F0A088', '#F5B49E']; +const PIXEL_BLUE = ['#445588', '#5577AA', '#6688BB', '#7799CC', '#88AADD']; + +const PixelBarOuter: React.FC<{ value: number; max: number; width?: number; palette?: string[]; tokens: any }> = ({ value, max, width = 16, palette = PIXEL_SALMON, tokens: c }) => { + const filled = max > 0 ? Math.max(value > 0 ? 1 : 0, Math.round((value / max) * width)) : 0; + return ( + + {Array.from({ length: width }, (_, i) => ( + + ))} + + ); +}; + +// ── Usage Stats Component ── +const UsageStats: React.FC = () => { + const c = useClaudeTokens(); + const [stats, setStats] = useState(null); + + useEffect(() => { + fetch(`${API_BASE}/analytics/usage-summary`) + .then(r => r.json()) + .then(setStats) + .catch(() => {}); + }, []); + + if (!stats) return null; + + const formatCost = (v: number) => { + if (v === 0) return '$0.00'; + if (v < 0.001) return `$${v.toFixed(6)}`; + if (v < 0.01) return `$${v.toFixed(5)}`; + if (v < 1) return `$${v.toFixed(4)}`; + return `$${v.toFixed(2)}`; + }; + const formatDuration = (s: number) => { + if (s === 0) return '0s'; + if (s < 60) return `${s.toFixed(1)}s`; + if (s < 3600) return `${Math.floor(s / 60)}m ${Math.round(s % 60)}s`; + return `${Math.floor(s / 3600)}h ${Math.floor((s % 3600) / 60)}m`; + }; + const formatTotalTime = (s: number) => { + if (s < 60) return `${s.toFixed(1)}s`; + if (s < 3600) return `${(s / 60).toFixed(1)} min`; + return `${(s / 3600).toFixed(1)} hrs`; + }; + + const cardSx = { + p: 1.5, + borderRadius: `${c.radius.md}px`, + bgcolor: c.bg.elevated, + border: `1px solid ${c.border.subtle}`, + }; + const labelSx = { fontSize: '0.58rem', fontWeight: 700, color: c.text.ghost, textTransform: 'uppercase' as const, letterSpacing: '0.06em', mb: 0.25 }; + const valueSx = { fontSize: '1.05rem', fontWeight: 700, color: c.text.primary, lineHeight: 1.2 }; + const subSx = { fontSize: '0.62rem', color: c.text.tertiary, mt: 0.25 }; + + const modelEntries = Object.entries(stats.models_used || {}).sort((a: any, b: any) => b[1] - a[1]) as [string, number][]; + const providerEntries = Object.entries(stats.providers_used || {}).sort((a: any, b: any) => b[1] - a[1]) as [string, number][]; + const toolEntries = Object.entries(stats.top_tools || {}).slice(0, 10) as [string, number][]; + const maxToolCount = toolEntries.length > 0 ? Math.max(...toolEntries.map(([, c]) => c)) : 1; + const statusEntries = Object.entries(stats.status_breakdown || {}) as [string, string][]; + + // Pixel bar helper that passes tokens + const PixelBar: React.FC<{ value: number; max: number; width?: number; palette?: string[] }> = (props) => ( + + ); + + const totalTime = stats.avg_duration_seconds * stats.total_sessions; + const msgsPerSession = stats.total_sessions > 0 ? (stats.total_messages / stats.total_sessions).toFixed(1) : '0'; + const toolsPerSession = stats.total_sessions > 0 ? (stats.total_tool_calls / stats.total_sessions).toFixed(1) : '0'; + + return ( + + {/* Row 1: Core metrics */} + + + Total Sessions + {stats.total_sessions.toLocaleString()} + + {statusEntries.map(([s, n]) => `${n} ${s}`).join(', ') || 'no sessions'} + + + + Total Cost + {formatCost(stats.total_cost_usd)} + + {formatCost(stats.avg_cost_per_session)} avg per session + + + + Total Messages + {stats.total_messages.toLocaleString()} + + {msgsPerSession} avg per session + + + + Total Tool Calls + {stats.total_tool_calls.toLocaleString()} + + {toolsPerSession} avg per session + + + + + {/* Row 2: Time + efficiency */} + + + Total Run Time + {formatTotalTime(totalTime)} + across all sessions + + + Avg Session + {formatDuration(stats.avg_duration_seconds)} + per session duration + + + Completion Rate + {(stats.completion_rate * 100).toFixed(1)}% + + sessions finished successfully + + + + Providers + {Object.keys(stats.providers_used || {}).length} + + {providerEntries.map(([p]) => p).join(', ') || 'none configured'} + + + + + {/* Model + Provider + Tool breakdown */} + + {/* Models & Providers */} + + Models Used + {modelEntries.length > 0 ? modelEntries.map(([model, count]) => { + const pct = stats.total_sessions > 0 ? ((count / stats.total_sessions) * 100).toFixed(0) : '0'; + return ( + + + {model} + + {count} ({pct}%) + + + + + ); + }) : No sessions yet} + + + {/* Tools */} + + Top Tools + {toolEntries.length > 0 ? toolEntries.map(([tool, count]) => { + const shortName = tool.includes('__') ? tool.split('__').pop() : tool; + const pct = stats.total_tool_calls > 0 ? ((count / stats.total_tool_calls) * 100).toFixed(0) : '0'; + return ( + + + {shortName} + + {count} call{count !== 1 ? 's' : ''} ({pct}%) + + + + + ); + }) : No tool calls yet} + + + + ); +}; const API_KEY_STEPS = [ { @@ -86,7 +571,7 @@ const Settings: React.FC = () => { const downloadPercent = useAppSelector((s) => s.update.downloadPercent); const updateError = useAppSelector((s) => s.update.error); - const [activeTab, setActiveTab] = useState<'general' | 'commands'>('general'); + const [activeTab, setActiveTab] = useState<'general' | 'models' | 'usage' | 'commands'>('general'); const [form, setForm] = useState({ ...settings }); const [showApiKey, setShowApiKey] = useState(false); const [browseOpen, setBrowseOpen] = useState(false); @@ -142,6 +627,7 @@ const Settings: React.FC = () => { if (form.theme !== settings.theme) { setThemeMode(form.theme); } + dispatch(fetchModels()); setSaved(true); }; @@ -164,6 +650,7 @@ const Settings: React.FC = () => { if (form.theme !== settings.theme) { setThemeMode(form.theme); } + dispatch(fetchModels()); setSaved(true); setConfirmDiscard(false); dispatch(closeSettingsModal()); @@ -271,6 +758,8 @@ const Settings: React.FC = () => { }} > + + @@ -495,7 +984,7 @@ const Settings: React.FC = () => { - + New agent shortcut Keyboard shortcut to create an agent. @@ -553,6 +1042,51 @@ const Settings: React.FC = () => { + + + Auto-enable element selection + Automatically enter element selection mode when creating a new agent. + + setForm({ ...form, auto_select_mode_on_new_agent: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + + + + Default agent spawn state in dashboard + When enabled, new agents spawn expanded instead of collapsed. + + setForm({ ...form, expand_new_chats_in_dashboard: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + + + + Auto-reveal sub-agents on dashboard + Automatically show sub-agent cards (from CreateAgent / InvokeAgent) tethered to their parent on the dashboard. + + setForm({ ...form, auto_reveal_sub_agents: e.target.checked })} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {/* ── Browser ── */} Browser @@ -580,127 +1114,20 @@ const Settings: React.FC = () => { - {/* ── API ── */} - API + {/* ── Advanced ── */} + Advanced - - Anthropic API key - - - Stored securely in the local database. - - setShowApiHelp((v) => !v)} - sx={{ - color: c.accent.primary, - fontSize: '0.75rem', - cursor: 'pointer', - display: 'inline-flex', - alignItems: 'center', - gap: 0.4, - whiteSpace: 'nowrap', - userSelect: 'none', - '&:hover': { textDecoration: 'underline' }, - }} - > - {showApiHelp ? 'Hide guide' : 'How do I get a key?'} - + + + Developer mode + Show transport details, environment variables, raw configs, and other technical metadata throughout the app. - - - - {API_KEY_STEPS.map((step, i) => ( - - - {i + 1} - - - - {step.title} - {step.link && ( - { - const w = window as any; - if (w.openswarm?.openExternal) { - w.openswarm.openExternal(step.link); - } else { - window.open(step.link, '_blank', 'noopener'); - } - }} - sx={{ - color: c.accent.primary, - fontSize: '0.75rem', - ml: 0.75, - cursor: 'pointer', - display: 'inline-flex', - alignItems: 'center', - gap: 0.3, - verticalAlign: 'middle', - '&:hover': { textDecoration: 'underline' }, - }} - > - Open - - - )} - - - {step.detail} - - - - ))} - - - - setForm({ ...form, anthropic_api_key: e.target.value || null })} - size="small" - fullWidth - placeholder="sk-ant-..." + setForm({ ...form, dev_mode: e.target.checked })} sx={{ - ...fieldSx, - '& .MuiOutlinedInput-root': { - ...fieldSx['& .MuiOutlinedInput-root'], - fontFamily: c.font.mono, - }, - }} - InputProps={{ - endAdornment: ( - - setShowApiKey(!showApiKey)} - edge="end" - size="small" - sx={{ color: c.text.tertiary }} - > - {showApiKey ? : } - - - ), + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, }} /> @@ -815,6 +1242,188 @@ const Settings: React.FC = () => { + ) : activeTab === 'models' ? ( + + + {/* ── USE EXISTING SUBSCRIPTIONS ── */} + + Use Your Existing Subscriptions + + + + Already paying for Claude, ChatGPT, or Gemini? Connect your subscription — no API key needed, no extra cost. + + + + + {/* ── API KEYS ── */} + + Or Connect With API Keys + + + + Pay per use. Each key is stored locally on your device. + + + {/* OpenRouter — recommended */} + + + OpenRouter + + RECOMMENDED + + + + One key for 300+ models — Llama, DeepSeek, Mistral, Qwen, Grok, and more. Easiest way to get started. + + + setForm({ ...form, openrouter_api_key: e.target.value || null } as any)} + size="small" + fullWidth + placeholder="sk-or-..." + sx={{ ...fieldSx, '& .MuiOutlinedInput-root': { ...fieldSx['& .MuiOutlinedInput-root'], fontFamily: c.font.mono } }} + /> + + Get key + + + + + {/* Anthropic */} + + + Anthropic + {form.anthropic_api_key ? ( + CONNECTED + ) : null} + + Claude Sonnet, Opus, Haiku. + + setForm({ ...form, anthropic_api_key: e.target.value || null })} + size="small" + fullWidth + placeholder="sk-ant-..." + sx={{ ...fieldSx, '& .MuiOutlinedInput-root': { ...fieldSx['& .MuiOutlinedInput-root'], fontFamily: c.font.mono } }} + InputProps={{ + endAdornment: ( + + setShowApiKey(!showApiKey)} edge="end" size="small" sx={{ color: c.text.tertiary }}> + {showApiKey ? : } + + + ), + }} + /> + + Get key + + + + + {/* OpenAI */} + + + OpenAI + {(form as any).openai_api_key ? ( + CONNECTED + ) : null} + + GPT-5.4, o3, o4-mini. + + setForm({ ...form, openai_api_key: e.target.value || null } as any)} + size="small" + fullWidth + placeholder="sk-..." + sx={{ ...fieldSx, '& .MuiOutlinedInput-root': { ...fieldSx['& .MuiOutlinedInput-root'], fontFamily: c.font.mono } }} + /> + + Get key + + + + + {/* Google */} + + + Google + {(form as any).google_api_key ? ( + CONNECTED + ) : null} + + Gemini 2.5 Pro and Flash. + + setForm({ ...form, google_api_key: e.target.value || null } as any)} + size="small" + fullWidth + placeholder="AIza..." + sx={{ ...fieldSx, '& .MuiOutlinedInput-root': { ...fieldSx['& .MuiOutlinedInput-root'], fontFamily: c.font.mono } }} + /> + + Get key + + + + + ) : activeTab === 'usage' ? ( + + + + {/* ── Analytics ── */} + Analytics + + + + Share anonymous usage data + + Help improve OpenSwarm by sharing anonymous statistics like session counts, model usage, and feature adoption. No conversations, file paths, or personal information is ever collected. + + + setForm({ ...form, analytics_opt_in: e.target.checked } as any)} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + ) : ( diff --git a/frontend/src/app/pages/Skills/Skills.tsx b/frontend/src/app/pages/Skills/Skills.tsx index 7424cac5..89b1feaa 100644 --- a/frontend/src/app/pages/Skills/Skills.tsx +++ b/frontend/src/app/pages/Skills/Skills.tsx @@ -579,8 +579,6 @@ const Skills: React.FC = () => { size="small" component="a" href={selectedReg.repositoryUrl} - target="_blank" - rel="noopener" sx={{ color: c.text.tertiary, '&:hover': { color: c.text.primary } }} > diff --git a/frontend/src/app/pages/Tools/Tools.tsx b/frontend/src/app/pages/Tools/Tools.tsx index a6e05224..272b7688 100644 --- a/frontend/src/app/pages/Tools/Tools.tsx +++ b/frontend/src/app/pages/Tools/Tools.tsx @@ -56,6 +56,7 @@ import BlockIcon from '@mui/icons-material/Block'; import VisibilityIcon from '@mui/icons-material/Visibility'; import SecurityIcon from '@mui/icons-material/Security'; import PanToolIcon from '@mui/icons-material/PanTool'; +import CallSplitIcon from '@mui/icons-material/CallSplit'; import RefreshIcon from '@mui/icons-material/Refresh'; import { fetchTools, @@ -74,6 +75,8 @@ import { import { searchRegistry, fetchRegistryStats, + fetchServerDetail, + clearDetail, McpServer, } from '@/shared/state/mcpRegistrySlice'; import { @@ -156,7 +159,7 @@ const INTEGRATIONS: Integration[] = [ }, ]; -const CATEGORY_ORDER = ['filesystem', 'system', 'search', 'interaction', 'planning', 'scheduling']; +const CATEGORY_ORDER = ['filesystem', 'system', 'search', 'interaction', 'agents', 'planning', 'scheduling']; interface ToolForm { name: string; @@ -235,6 +238,7 @@ const ToolSection: React.FC = ({ interaction: { label: 'Interaction', color: '#a855f7', icon: }, planning: { label: 'Planning', color: '#ec4899', icon: }, scheduling: { label: 'Scheduling', color: '#14b8a6', icon: }, + agents: { label: 'Agents', color: '#f97316', icon: }, }; const PermToggle = ({ value, onChange, size = 16 }: { value: string; onChange: (v: string) => void; size?: number }) => ( @@ -343,7 +347,7 @@ const ToolSection: React.FC = ({ return ( - {bt.name} + {bt.display_name || bt.name} {bt.description && {firstSentence(bt.description)}} onPermissionChange(bt.name, v)} size={14} /> @@ -370,7 +374,8 @@ const Tools: React.FC = () => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); const { items, builtinTools, builtinPermissions, loading } = useAppSelector((s) => s.tools); - const { servers: regServers, total: regTotal, loading: regLoading, stats: regStats } = useAppSelector((s) => s.mcpRegistry); + const { servers: regServers, total: regTotal, loading: regLoading, stats: regStats, detail: regDetail, detailLoading: regDetailLoading } = useAppSelector((s) => s.mcpRegistry); + const devMode = useAppSelector((s) => s.settings.data.dev_mode); const outputItems = useAppSelector((s) => s.outputs.items); const outputs = useMemo(() => Object.values(outputItems), [outputItems]); const allTools = Object.values(items); @@ -488,7 +493,8 @@ const Tools: React.FC = () => { if (discoverTools.fulfilled.match(result)) { setSnackbar({ open: true, message: 'Actions discovered successfully' }); } else { - setSnackbar({ open: true, message: 'Discovery failed — is the MCP server running?', severity: 'error' }); + const detail = (result as any).error?.message || 'Discovery failed — is the MCP server running?'; + setSnackbar({ open: true, message: detail, severity: 'error' }); } } finally { setDiscovering(false); @@ -530,8 +536,11 @@ const Tools: React.FC = () => { }; const [expandedServices, setExpandedServices] = useState>({}); + const [expandedSchema, setExpandedSchema] = useState(null); const [viewsSectionOpen, setViewsSectionOpen] = useState(false); + const [browserSectionOpen, setBrowserSectionOpen] = useState(false); + const [browserCollapsed, setBrowserCollapsed] = useState>({ browser_delegation: true, browser_action: true }); const [builtinSectionOpen, setBuiltinSectionOpen] = useState(true); useEffect(() => { @@ -556,8 +565,12 @@ const Tools: React.FC = () => { }; // Built-in tool grouping - const coreTools = useMemo(() => builtinTools.filter((bt) => !bt.deferred), [builtinTools]); - const deferredTools = useMemo(() => builtinTools.filter((bt) => bt.deferred), [builtinTools]); + const BROWSER_CATEGORIES = new Set(['browser_delegation', 'browser_action']); + const coreTools = useMemo(() => builtinTools.filter((bt) => !bt.deferred && !BROWSER_CATEGORIES.has(bt.category)), [builtinTools]); + const deferredTools = useMemo(() => builtinTools.filter((bt) => bt.deferred && !BROWSER_CATEGORIES.has(bt.category)), [builtinTools]); + const browserTools = useMemo(() => builtinTools.filter((bt) => BROWSER_CATEGORIES.has(bt.category)), [builtinTools]); + const browserDelegationTools = useMemo(() => browserTools.filter((bt) => bt.category === 'browser_delegation'), [browserTools]); + const browserActionTools = useMemo(() => browserTools.filter((bt) => bt.category === 'browser_action'), [browserTools]); const groupTools = (list: BuiltinTool[]) => { const g: Record = {}; for (const bt of list) { if (!g[bt.category]) g[bt.category] = []; g[bt.category].push(bt); } @@ -578,10 +591,14 @@ const Tools: React.FC = () => { () => !outputs.every((o) => o.permission === 'deny'), [outputs], ); + const browserSectionEnabled = useMemo( + () => browserTools.length > 0 && !browserTools.every((t) => builtinPermissions[t.name] === 'deny'), + [browserTools, builtinPermissions], + ); const handleSectionEnabledChange = async (tools: BuiltinTool[], enabled: boolean) => { const perms: Record = {}; - for (const t of tools) perms[t.name] = enabled ? 'ask' : 'deny'; + for (const t of tools) perms[t.name] = enabled ? 'always_allow' : 'deny'; await dispatch(updateBuiltinPermissions(perms)); }; @@ -894,7 +911,7 @@ const Tools: React.FC = () => { {builtinSectionOpen ? : } Built-in Action Sets - + @@ -992,6 +1009,156 @@ const Tools: React.FC = () => { )} + {/* Browser */} + {browserTools.length > 0 && ( + + + browserSectionEnabled && setBrowserSectionOpen((v) => !v)} + sx={{ display: 'flex', alignItems: 'center', gap: 2, cursor: browserSectionEnabled ? 'pointer' : 'default' }} + > + + + + + + Browser + + + Browser automation delegation and individual browser actions + + e.stopPropagation()}> + handleSectionEnabledChange(browserTools, checked)} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + {browserSectionEnabled && ( + + + + )} + + + + + + + + Action Permissions + + + + + {/* Delegation group */} + {browserDelegationTools.length > 0 && (() => { + const delegationPolicies = browserDelegationTools.map((t) => builtinPermissions[t.name] || 'always_allow'); + const groupPolicy = delegationPolicies.every((p) => p === 'always_allow') ? 'always_allow' + : delegationPolicies.every((p) => p === 'deny') ? 'deny' + : delegationPolicies.every((p) => p === 'ask') ? 'ask' : 'ask'; + const isOpen = !browserCollapsed.browser_delegation; + return ( + + setBrowserCollapsed((p) => ({ ...p, browser_delegation: !p.browser_delegation }))} + > + + + Delegation + + + e.stopPropagation()}> + handleBuiltinCategoryPermissionChange(browserDelegationTools.map((t) => t.name), 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: groupPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinCategoryPermissionChange(browserDelegationTools.map((t) => t.name), 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: groupPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinCategoryPermissionChange(browserDelegationTools.map((t) => t.name), 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: groupPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + + + {browserDelegationTools.map((bt) => { + const toolPolicy = builtinPermissions[bt.name] || 'always_allow'; + return ( + + + {bt.display_name || bt.name} + {bt.description && {bt.description}} + + e.stopPropagation()}> + handleBuiltinPermissionChange(bt.name, 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: toolPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinPermissionChange(bt.name, 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: toolPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinPermissionChange(bt.name, 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: toolPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + ); + })} + + + + ); + })()} + + {/* Browser Actions group */} + {browserActionTools.length > 0 && (() => { + const actionPolicies = browserActionTools.map((t) => builtinPermissions[t.name] || 'always_allow'); + const groupPolicy = actionPolicies.every((p) => p === 'always_allow') ? 'always_allow' + : actionPolicies.every((p) => p === 'deny') ? 'deny' + : actionPolicies.every((p) => p === 'ask') ? 'ask' : 'ask'; + const isOpen = !browserCollapsed.browser_action; + return ( + + setBrowserCollapsed((p) => ({ ...p, browser_action: !p.browser_action }))} + > + + + Browser Actions + + + e.stopPropagation()}> + handleBuiltinCategoryPermissionChange(browserActionTools.map((t) => t.name), 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: groupPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinCategoryPermissionChange(browserActionTools.map((t) => t.name), 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: groupPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinCategoryPermissionChange(browserActionTools.map((t) => t.name), 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: groupPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: groupPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + + + {browserActionTools.map((bt) => { + const toolPolicy = builtinPermissions[bt.name] || 'always_allow'; + return ( + + + {bt.display_name || bt.name} + {bt.description && {bt.description}} + + e.stopPropagation()}> + handleBuiltinPermissionChange(bt.name, 'always_allow')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'always_allow' ? `${c.status.success}20` : 'transparent', color: toolPolicy === 'always_allow' ? c.status.success : c.text.ghost, '&:hover': { bgcolor: `${c.status.success}15`, color: c.status.success } }}> + handleBuiltinPermissionChange(bt.name, 'ask')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'ask' ? `${c.status.warning}20` : 'transparent', color: toolPolicy === 'ask' ? c.status.warning : c.text.ghost, '&:hover': { bgcolor: `${c.status.warning}15`, color: c.status.warning } }}> + handleBuiltinPermissionChange(bt.name, 'deny')} sx={{ p: 0.4, borderRadius: 1, bgcolor: toolPolicy === 'deny' ? `${c.status.error}20` : 'transparent', color: toolPolicy === 'deny' ? c.status.error : c.text.ghost, '&:hover': { bgcolor: `${c.status.error}15`, color: c.status.error } }}> + + + ); + })} + + + + ); + })()} + + + + + )} + @@ -1029,7 +1196,7 @@ const Tools: React.FC = () => { {ig.name} - } label="docs" size="small" sx={{ bgcolor: c.bg.secondary, color: c.text.ghost, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, fontSize: 10 } }} /> + } label="docs" size="small" sx={{ bgcolor: c.bg.secondary, color: c.text.ghost, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, fontSize: 10 } }} /> {ig.description} @@ -1059,6 +1226,7 @@ const Tools: React.FC = () => { const perms = tool.tool_permissions || {}; const services = perms._services as Record | undefined; const descriptions = (perms._tool_descriptions || {}) as Record; + const schemas = (perms._tool_schemas || {}) as Record; const serviceNames = services ? Object.keys(services) : []; const hasPerms = serviceNames.length > 0; const totalToolCount = serviceNames.reduce((acc, s) => acc + (services![s].read?.length || 0) + (services![s].write?.length || 0), 0); @@ -1131,15 +1299,36 @@ const Tools: React.FC = () => { handleGroupPermissionChange(tool.id, data.read!, v)} size={14} /> - {data.read!.map((name) => ( - - - {toDisplayName(name, serviceName)} - {descriptions[name] && {firstSentence(descriptions[name])}} + {data.read!.map((name) => { + const schemaKey = `${tool.id}:${name}`; + const schema = schemas[name]; + const schemaProps = schema?.properties as Record | undefined; + const schemaRequired = (schema?.required || []) as string[]; + return ( + + devMode && schema && setExpandedSchema((p) => p === schemaKey ? null : schemaKey)}> + + {toDisplayName(name, serviceName)} + {descriptions[name] && {firstSentence(descriptions[name])}} + + handlePermissionChange(tool.id, name, v)} size={14} /> + + {devMode && expandedSchema === schemaKey && schemaProps && ( + + Input Parameters + {Object.entries(schemaProps).map(([pName, pDef]: [string, any]) => ( + + {pName} + {pDef?.type || 'any'} + {schemaRequired.includes(pName) && } + {pDef?.description && {pDef.description}} + + ))} + + )} - handlePermissionChange(tool.id, name, v)} size={14} /> - - ))} + ); + })} )} {(data.write?.length || 0) > 0 && ( @@ -1152,15 +1341,36 @@ const Tools: React.FC = () => { handleGroupPermissionChange(tool.id, data.write!, v)} size={14} /> - {data.write!.map((name) => ( - - - {toDisplayName(name, serviceName)} - {descriptions[name] && {firstSentence(descriptions[name])}} + {data.write!.map((name) => { + const schemaKey = `${tool.id}:${name}`; + const schema = schemas[name]; + const schemaProps = schema?.properties as Record | undefined; + const schemaRequired = (schema?.required || []) as string[]; + return ( + + devMode && schema && setExpandedSchema((p) => p === schemaKey ? null : schemaKey)}> + + {toDisplayName(name, serviceName)} + {descriptions[name] && {firstSentence(descriptions[name])}} + + handlePermissionChange(tool.id, name, v)} size={14} /> + + {devMode && expandedSchema === schemaKey && schemaProps && ( + + Input Parameters + {Object.entries(schemaProps).map(([pName, pDef]: [string, any]) => ( + + {pName} + {pDef?.type || 'any'} + {schemaRequired.includes(pName) && } + {pDef?.description && {pDef.description}} + + ))} + + )} - handlePermissionChange(tool.id, name, v)} size={14} /> - - ))} + ); + })} )} @@ -1202,7 +1412,7 @@ const Tools: React.FC = () => { )} {ig && ( - } label="docs" size="small" sx={{ bgcolor: c.bg.secondary, color: c.text.ghost, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, fontSize: 10 } }} /> + } label="docs" size="small" sx={{ bgcolor: c.bg.secondary, color: c.text.ghost, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, fontSize: 10 } }} /> )} {tool.description && {tool.description}} @@ -1330,6 +1540,46 @@ const Tools: React.FC = () => { ))} )} + + {devMode && isMcp && ( + + + Developer Info + + + + MCP Config + + + {JSON.stringify(tool.mcp_config, null, 2)} + + + + + Auth type: + {tool.auth_type || 'none'} + + + Status: + {tool.auth_status || 'none'} + + {tool.connected_account_email && ( + + Account: + {tool.connected_account_email} + + )} + + {tool.credentials && Object.keys(tool.credentials).length > 0 && ( + + Credentials: + {Object.keys(tool.credentials).map((key) => ( + + ))} + + )} + + )} @@ -1366,17 +1616,24 @@ const Tools: React.FC = () => { MCP Registry {regStats && ( - + <> + + {devMode && regStats.lastUpdated > 0 && ( + + Synced {Math.round((Date.now() / 1000 - regStats.lastUpdated) / 60)}m ago + + )} + )} { {regServers.map((srv) => { const isExpanded = expandedServer === srv.name; + const isInstalled = allTools.some((t) => t.name === (srv.title || cleanServerName(srv.name))); return ( setExpandedServer(isExpanded ? null : srv.name)} + onClick={() => { + const next = isExpanded ? null : srv.name; + setExpandedServer(next); + if (next && devMode) { + dispatch(clearDetail()); + dispatch(fetchServerDetail(srv.name)); + } + }} sx={{ display: 'flex', alignItems: 'center', gap: 1.5, px: 1.5, py: 1, borderRadius: 1.5, cursor: 'pointer', @@ -1504,6 +1769,12 @@ const Tools: React.FC = () => { )} + {devMode && !srv.remoteType && ( + + )} + {isInstalled && ( + } label="Installed" size="small" sx={{ bgcolor: `${c.status.success}15`, color: c.status.success, fontSize: '0.65rem', height: 18, '& .MuiChip-label': { px: 0.4 }, '& .MuiChip-icon': { ml: 0.4, color: c.status.success } }} /> + )} {srv.description} @@ -1533,8 +1804,6 @@ const Tools: React.FC = () => { } label="Website" @@ -1546,8 +1815,6 @@ const Tools: React.FC = () => { } label="Repository" @@ -1558,6 +1825,46 @@ const Tools: React.FC = () => { + {devMode && ( + + {regDetailLoading && expandedServer === srv.name ? ( + + ) : regDetail && regDetail.name === srv.name ? ( + + {(regDetail.keywords?.length > 0 || regDetail.license) && ( + + {regDetail.license && ( + + )} + {regDetail.keywords?.map((kw) => ( + + ))} + + )} + {regDetail.environmentVariables?.length > 0 && ( + + + Required Environment Variables + + {regDetail.environmentVariables.map((ev) => ( + + + {ev.name} + + {ev.description && ( + + {ev.description} + + )} + + ))} + + )} + + ) : null} + + )} +