diff --git a/backend/apps/agents/agent_loop.py b/backend/apps/agents/agent_loop.py new file mode 100644 index 00000000..99b9b449 --- /dev/null +++ b/backend/apps/agents/agent_loop.py @@ -0,0 +1,331 @@ +"""Owned agent loop — replaces claude_agent_sdk's query() function. + +Generalizes the pattern from browser_agent.py (lines 243-334) into a +provider-agnostic, streaming, HITL-aware tool-use loop. +""" + +from __future__ import annotations + +import json +import logging +import time +from typing import Any, Callable, Awaitable +from uuid import uuid4 + +from backend.apps.agents.providers.base import ( + BaseProvider, ContentBlock, ModelResponse, ProviderMessage, + StreamEvent, ToolCall, ToolSchema, +) + +logger = logging.getLogger(__name__) + +# Type aliases for callbacks +ToolExecutor = Callable[[str, dict], Awaitable[list[dict]]] +# hitl_handler(tool_name, tool_input) -> (approved, updated_input_or_None) +HITLHandler = Callable[[str, dict], Awaitable[tuple[bool, dict | None]]] +# ws_emitter(event_type, data) -> None +WSEmitter = Callable[[str, dict], Awaitable[None]] + + +class AgentLoop: + """Provider-agnostic agent loop with streaming and HITL support. + + The loop: + 1. Sends user message to the model + 2. Streams the response (emitting WebSocket events) + 3. If the model requests tool use: + a. For each tool call: check HITL permission → execute → collect result + b. Append tool results → go to step 2 + 4. If the model stops (end_turn/max_tokens): done + """ + + def __init__( + self, + session_id: str, + provider: BaseProvider, + model: str, + system_prompt: str | None, + tools: list[ToolSchema], + tool_executor: ToolExecutor, + hitl_handler: HITLHandler, + ws_emitter: WSEmitter, + max_turns: int | None = None, + cwd: str | None = None, + ): + self.session_id = session_id + self.provider = provider + self.model = model + self.system_prompt = system_prompt + self.tools = tools + self.tool_executor = tool_executor + self.hitl_handler = hitl_handler + self.ws_emitter = ws_emitter + self.max_turns = max_turns + self.cwd = cwd + + # Conversation history in provider-agnostic format + self.messages: list[ProviderMessage] = [] + + # Token tracking + self.total_input_tokens = 0 + self.total_output_tokens = 0 + + async def run(self, user_content: Any) -> None: + """Run the agent loop for a single user turn.""" + # Append user message + user_msg = self.provider.format_user_message(user_content) + self.messages.append(user_msg) + + turn = 0 + while True: + if self.max_turns and turn >= self.max_turns: + logger.info(f"Agent {self.session_id}: max turns ({self.max_turns}) reached") + break + turn += 1 + + # Stream the model response and collect it + response = await self._stream_and_collect() + + # Track usage + self.total_input_tokens += response.usage.get("input_tokens", 0) + self.total_output_tokens += response.usage.get("output_tokens", 0) + + # Append assistant message to conversation history + assistant_msg = self.provider.format_assistant_message(response) + self.messages.append(assistant_msg) + + # If no tool use, we're done + if response.stop_reason != "tool_use": + break + + # Execute tools + tool_results = await self._execute_tools(response) + if not tool_results: + break + + # Append tool results + self.messages.append(ProviderMessage(role="tool_result", content=tool_results)) + + async def _stream_and_collect(self) -> ModelResponse: + """Stream model output, emit WebSocket events, collect full response.""" + collected_content: list[ContentBlock] = [] + collected_usage: dict[str, int] = {} + stop_reason = "end_turn" + + # Track streaming state for WS emissions + stream_text_msg_id: str | None = None + stream_tool_msg_ids: dict[int, str] = {} # block index -> msg_id + block_index_map: dict[int, str] = {} # block index -> msg_id + + # Buffers for collecting content + text_buffers: dict[int, str] = {} + json_buffers: dict[int, str] = {} + tool_names: dict[int, str] = {} + tool_ids: dict[int, str] = {} + block_types: dict[int, str] = {} + + async for event in self.provider.stream_message( + model=self.model, + system=self.system_prompt, + messages=self.messages, + tools=self.tools, + ): + if event.type == "content_block_start": + if event.block_type == "text": + if stream_text_msg_id is None: + stream_text_msg_id = uuid4().hex + await self.ws_emitter("agent:stream_start", { + "message_id": stream_text_msg_id, + "role": "assistant", + }) + block_index_map[event.index] = stream_text_msg_id + block_types[event.index] = "text" + text_buffers[event.index] = "" + + elif event.block_type == "tool_use": + tool_msg_id = uuid4().hex + stream_tool_msg_ids[event.index] = tool_msg_id + block_index_map[event.index] = tool_msg_id + block_types[event.index] = "tool_use" + tool_names[event.index] = event.tool_name + tool_ids[event.index] = event.tool_id + json_buffers[event.index] = "" + + await self.ws_emitter("agent:stream_start", { + "message_id": tool_msg_id, + "role": "tool_call", + "tool_name": event.tool_name, + }) + + elif event.type == "content_block_delta": + msg_id = block_index_map.get(event.index) + if not msg_id: + continue + + if event.delta_type == "text_delta": + text_buffers.setdefault(event.index, "") + text_buffers[event.index] += event.text + await self.ws_emitter("agent:stream_delta", { + "message_id": msg_id, + "delta": event.text, + }) + + elif event.delta_type == "input_json_delta": + json_buffers.setdefault(event.index, "") + json_buffers[event.index] += event.text + await self.ws_emitter("agent:stream_delta", { + "message_id": msg_id, + "delta": event.text, + }) + + elif event.type == "content_block_stop": + msg_id = block_index_map.get(event.index) + bt = block_types.get(event.index, "") + + if bt == "text": + collected_content.append( + ContentBlock(type="text", text=text_buffers.get(event.index, "")) + ) + elif bt == "tool_use": + try: + tool_input = json.loads(json_buffers.get(event.index, "{}")) + except json.JSONDecodeError: + tool_input = {} + collected_content.append(ContentBlock( + type="tool_use", + tool_call=ToolCall( + id=tool_ids.get(event.index, uuid4().hex), + name=tool_names.get(event.index, ""), + input=tool_input, + ), + )) + + # Send stream_end for tool blocks (text block ends at message_stop) + if msg_id and bt == "tool_use": + await self.ws_emitter("agent:stream_end", { + "message_id": msg_id, + }) + + elif event.type == "usage": + # Accumulate token usage from provider stream + for k, v in event.usage.items(): + collected_usage[k] = collected_usage.get(k, 0) + v + + elif event.type == "message_stop": + # Check if any tool calls means stop_reason is tool_use + has_tool_use = any(b.type == "tool_use" for b in collected_content) + if has_tool_use: + stop_reason = "tool_use" + + # End text stream + if stream_text_msg_id: + await self.ws_emitter("agent:stream_end", { + "message_id": stream_text_msg_id, + }) + + # Build and emit the collected messages + await self._emit_collected_messages( + collected_content, stream_text_msg_id, stream_tool_msg_ids, + ) + + return ModelResponse( + content=collected_content, + stop_reason=stop_reason, + usage=collected_usage, + ) + + async def _emit_collected_messages( + self, + content: list[ContentBlock], + text_msg_id: str | None, + tool_msg_ids: dict[int, str], + ) -> None: + """Emit finalized agent:message events for the collected response.""" + from backend.apps.agents.models import Message + + # Emit text message + text_parts = [b.text for b in content if b.type == "text" and b.text] + if text_parts: + msg = Message( + id=text_msg_id or uuid4().hex, + role="assistant", + content="\n".join(text_parts), + ) + await self.ws_emitter("agent:message", { + "message": msg.model_dump(mode="json"), + }) + + # Emit tool call messages + tool_blocks = [b for b in content if b.type == "tool_use" and b.tool_call] + tool_id_list = sorted(tool_msg_ids.items(), key=lambda x: x[0]) + for i, block in enumerate(tool_blocks): + tc = block.tool_call + msg_id = tool_id_list[i][1] if i < len(tool_id_list) else uuid4().hex + msg = Message( + id=msg_id, + role="tool_call", + content={ + "id": tc.id, + "tool": tc.name, + "input": tc.input, + }, + ) + await self.ws_emitter("agent:message", { + "message": msg.model_dump(mode="json"), + }) + + async def _execute_tools(self, response: ModelResponse) -> list[dict]: + """Execute all tool calls from a response, respecting HITL permissions. + + Returns a list of tool result dicts formatted for the provider. + """ + from backend.apps.agents.models import Message + + results = [] + for block in response.content: + if block.type != "tool_use" or not block.tool_call: + continue + + tc = block.tool_call + start_time = time.time() + + # HITL permission check + approved, updated_input = await self.hitl_handler(tc.name, tc.input) + + if not approved: + result_content = [{"type": "text", "text": "Tool use was denied by the user."}] + else: + tool_input = updated_input if updated_input else tc.input + try: + result_content = await self.tool_executor(tc.name, tool_input) + except Exception as e: + logger.warning(f"Tool execution error: {tc.name}: {e}") + result_content = [{"type": "text", "text": f"Error executing {tc.name}: {e}"}] + + elapsed_ms = int((time.time() - start_time) * 1000) + + # Emit tool result to frontend + result_text = "" + for block_item in result_content: + if isinstance(block_item, dict) and block_item.get("type") == "text": + result_text = block_item.get("text", "") + break + + result_msg = Message( + role="tool_result", + content={ + "text": result_text[:2000] if result_text else "Done.", + "tool_name": tc.name, + "elapsed_ms": elapsed_ms, + }, + ) + await self.ws_emitter("agent:message", { + "message": result_msg.model_dump(mode="json"), + }) + + # Format for provider + results.append( + self.provider.format_tool_result(tc.id, result_content) + ) + + return results diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index a8307969..a1e7c869 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -23,10 +23,34 @@ from backend.apps.tools_lib.tools_lib import ( refresh_google_token, ) from backend.config.paths import SESSIONS_DIR +from backend.apps.analytics.collector import record as _analytics logger = logging.getLogger(__name__) -os.environ.setdefault("CLAUDE_CODE_STREAM_CLOSE_TIMEOUT", "3600000") +_TOOL_CATEGORIES: dict[str, str] = { + "gmail": "email", "calendar": "calendar", "drive": "files", + "sheet": "files", "doc": "files", "slide": "files", + "tweet": "social", "twitter": "social", "reddit": "social", + "Bash": "coding", "Edit": "coding", "Write": "coding", "Read": "coding", + "Glob": "coding", "Grep": "coding", + "BrowserAgent": "browsing", "BrowserAgents": "browsing", + "WebSearch": "research", "WebFetch": "research", +} + + +def _infer_task_category(tool_names: list[str]) -> str: + """Infer what the user was doing from the tools used.""" + if not tool_names: + return "chat" + counts: dict[str, int] = {} + for name in tool_names: + for keyword, category in _TOOL_CATEGORIES.items(): + if keyword.lower() in name.lower(): + counts[category] = counts.get(category, 0) + 1 + break + if not counts: + return "other" + return max(counts, key=counts.get) def _save_session(session_id: str, doc_data: dict): @@ -328,6 +352,7 @@ class AgentManager: session = AgentSession( id=session_id, name=config.name, + provider=config.provider, model=config.model, mode=config.mode, system_prompt=config.system_prompt, @@ -337,13 +362,20 @@ class AgentManager: dashboard_id=config.dashboard_id, ) self.sessions[session_id] = session - + + _analytics("session.started", { + "model": session.model, + "provider": session.provider, + "mode": session.mode, + "tool_count": len(tools), + }, session_id=session_id, dashboard_id=config.dashboard_id) + await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, "status": "running", "session": session.model_dump(mode="json"), }) - + return session def _resolve_context_paths(self, context_paths: list | None) -> str: @@ -468,28 +500,20 @@ class AgentManager: return content async def _run_agent_loop(self, session_id: str, prompt: str, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, attached_skills: list | None = None, fork_session: bool = False, selected_browser_ids: list[str] | None = None): - """Run the Claude Agent SDK query loop for a session.""" + """Run the owned agent loop for a session (multi-provider).""" + from backend.apps.agents.agent_loop import AgentLoop + from backend.apps.agents.mcp_client import MCPClientManager + from backend.apps.agents.providers.registry import create_provider, calculate_cost + from backend.apps.settings.credentials import validate_credentials + session = self.sessions.get(session_id) if not session: return - + prompt_content = self._build_prompt_content(prompt, images, context_paths, forced_tools, attached_skills) - try: - from claude_agent_sdk import ( - query, ClaudeAgentOptions, AssistantMessage, ResultMessage, - ) - from claude_agent_sdk.types import ( - HookMatcher, PermissionResultAllow, PermissionResultDeny, - TextBlock, ToolUseBlock, StreamEvent, - ) - except ImportError: - logger.warning("claude_agent_sdk not installed, running in mock mode") - await self._run_mock_agent(session_id, prompt) - return - session.status = "running" - + _builtin_perms = load_builtin_permissions() def _get_effective_policy(tool_name: str) -> str: @@ -539,6 +563,11 @@ class AgentManager: session_id, request_id, tool_name, safe_input ) + _analytics("tool.approval_resolved", { + "tool_name": tool_name, + "decision": decision.get("behavior", "deny"), + }, session_id=session_id) + session.pending_approvals = [ a for a in session.pending_approvals if a.id != request_id ] @@ -549,159 +578,7 @@ class AgentManager: }) return decision - async def can_use_tool(tool_name, input_data, context): - if tool_name != "AskUserQuestion": - policy = _get_effective_policy(tool_name) - if policy == "always_allow": - return PermissionResultAllow(updated_input=input_data) - if policy == "deny": - return PermissionResultDeny(message="Tool denied by permission policy") - - decision = await _request_user_approval(tool_name, input_data) - if decision.get("behavior") == "allow": - return PermissionResultAllow( - updated_input=decision.get("updated_input", input_data) - ) - return PermissionResultDeny( - message=decision.get("message", "User denied this action") - ) - - tool_start_times: dict[str, float] = {} - - async def pre_tool_hook(input_data, tool_use_id, context): - tool_name = input_data.get("tool_name", "") - hook_event = input_data.get("hook_event_name", "PreToolUse") - - if tool_name and tool_name != "AskUserQuestion": - policy = _get_effective_policy(tool_name) - - if policy == "deny": - return { - "hookSpecificOutput": { - "hookEventName": hook_event, - "permissionDecision": "deny", - "permissionDecisionReason": "Tool denied by permission policy", - } - } - - if policy == "ask": - tool_input = input_data.get("tool_input", {}) - decision = await _request_user_approval(tool_name, tool_input) - - if decision.get("behavior") == "allow": - if tool_use_id: - tool_start_times[tool_use_id] = time.time() - return { - "hookSpecificOutput": { - "hookEventName": hook_event, - "permissionDecision": "allow", - } - } - return { - "hookSpecificOutput": { - "hookEventName": hook_event, - "permissionDecision": "deny", - "permissionDecisionReason": decision.get("message", "User denied this action"), - } - } - - if tool_use_id: - tool_start_times[tool_use_id] = time.time() - return {} - - async def post_tool_hook(input_data, tool_use_id, context): - elapsed_ms = None - if tool_use_id and tool_use_id in tool_start_times: - elapsed_ms = int((time.time() - tool_start_times.pop(tool_use_id)) * 1000) - - raw_response = input_data.get("tool_response", "") - - if isinstance(raw_response, list) and raw_response: - text_parts = [ - block.get("text", "") - for block in raw_response - if isinstance(block, dict) and block.get("type") == "text" - ] - if text_parts: - raw_response = "\n".join(text_parts) if len(text_parts) > 1 else text_parts[0] - - if isinstance(raw_response, str): - content = raw_response - else: - try: - import json as _json - content = _json.dumps(raw_response, indent=2, default=str) - except Exception: - content = str(raw_response) - - result_payload = {"text": content} - hook_tool_name = input_data.get("tool_name", "") - if hook_tool_name: - result_payload["tool_name"] = hook_tool_name - if elapsed_ms is not None: - result_payload["elapsed_ms"] = elapsed_ms - - if hook_tool_name == "Agent": - tool_input = input_data.get("tool_input", {}) - agent_prompt = tool_input.get("prompt", tool_input.get("task", "")) - - sub_text = content - sub_cost = 0.0 - sub_tokens = {"input": 0, "output": 0} - sub_model = session.model - if isinstance(raw_response, dict): - blocks = raw_response.get("content") - if isinstance(blocks, list): - parts = [ - b.get("text", "") - for b in blocks - if isinstance(b, dict) and b.get("type") == "text" - ] - if parts: - sub_text = "\n".join(parts) if len(parts) > 1 else parts[0] - elif isinstance(raw_response.get("text"), str): - sub_text = raw_response["text"] - usage = raw_response.get("usage", {}) - if isinstance(usage, dict): - sub_tokens["input"] = usage.get("input_tokens", 0) + usage.get("cache_creation_input_tokens", 0) + usage.get("cache_read_input_tokens", 0) - sub_tokens["output"] = usage.get("output_tokens", 0) - if raw_response.get("model"): - sub_model = raw_response["model"] - - sub_session_id = uuid4().hex - sub_name = agent_prompt[:50] if agent_prompt else "Sub-agent" - sub_session = AgentSession( - id=sub_session_id, - name=sub_name, - status="completed", - model=sub_model, - mode="sub-agent", - cwd=session.cwd, - created_at=datetime.now(), - cost_usd=sub_cost, - tokens=sub_tokens, - messages=[ - Message(role="user", content=agent_prompt, branch_id="main"), - Message(role="assistant", content=sub_text, branch_id="main"), - ], - dashboard_id=session.dashboard_id, - parent_session_id=session_id, - ) - self.sessions[sub_session_id] = sub_session - await ws_manager.broadcast_global("agent:status", { - "session_id": sub_session_id, - "status": sub_session.status, - "session": sub_session.model_dump(mode="json"), - }) - result_payload["sub_session_id"] = sub_session_id - - result_msg = Message(role="tool_result", content=result_payload, branch_id=session.active_branch_id) - session.messages.append(result_msg) - await ws_manager.send_to_session(session_id, "agent:message", { - "session_id": session_id, - "message": result_msg.model_dump(mode="json"), - }) - return {"continue_": True} + mcp_manager: MCPClientManager | None = None try: _, mode_sys_prompt, _ = self._resolve_mode(session.mode) @@ -716,6 +593,13 @@ class AgentManager: skill_block = f"\n{VIEW_BUILDER_SKILL}\n" composed_prompt = f"{composed_prompt}\n\n{skill_block}" if composed_prompt else skill_block + # Validate credentials for the provider + validate_credentials(global_settings, session.provider) + + # Create the provider adapter + provider = create_provider(session.provider, global_settings) + + # Build MCP servers config mcp_servers = await self._build_mcp_servers(session.allowed_tools) _browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents"] @@ -765,214 +649,111 @@ class AgentManager: "type": "stdio", } - effective_allowed = [ - t for t in session.allowed_tools - if t in FULL_TOOLS and _builtin_perms.get(t, "always_allow") == "always_allow" - ] + # Connect MCP servers and collect tool schemas + mcp_manager = MCPClientManager() + await mcp_manager.__aenter__() - effective_disallowed = [ - t for t in FULL_TOOLS - if _builtin_perms.get(t, "always_allow") == "deny" - ] + mcp_tool_schemas = [] + for server_name, server_config in mcp_servers.items(): + tools = await mcp_manager.connect(server_name, server_config) + mcp_tool_schemas.extend(tools) - if mcp_servers: - all_tools_list = load_all_tools() - for name in mcp_servers: - if name == "openswarm-browser-agent": - for bt in _browser_delegation_tools: - policy = _builtin_perms.get(bt, "always_allow") - if policy == "always_allow": - effective_allowed.append(f"mcp__openswarm-browser-agent__{bt}") - elif policy == "deny": - effective_disallowed.append(f"mcp__openswarm-browser-agent__{bt}") - continue + # Collect builtin + MCP tool schemas + from backend.apps.agents.tools.registry import get_all_tool_schemas + all_tool_schemas = list(get_all_tool_schemas()) + list(mcp_tool_schemas) - if name == "openswarm-invoke-agent": - for it in _invoke_agent_tools: - policy = _builtin_perms.get(it, "always_allow") - if policy == "always_allow": - effective_allowed.append(f"mcp__openswarm-invoke-agent__{it}") - elif policy == "deny": - effective_disallowed.append(f"mcp__openswarm-invoke-agent__{it}") - continue + # HITL handler for the agent loop + async def hitl_handler(tool_name: str, tool_input: dict) -> tuple[bool, dict | None]: + policy = _get_effective_policy(tool_name) + if policy == "always_allow": + return True, None + if policy == "deny": + return False, None + # policy == "ask" + decision = await _request_user_approval(tool_name, tool_input) + if decision.get("behavior") == "allow": + return True, decision.get("updated_input") + return False, None - tool_def = next( - (t for t in all_tools_list - if t.mcp_config and t.enabled and _sanitize_server_name(t.name) == name), - None, - ) - if tool_def: - denied = _get_denied_tool_names(tool_def) - known = _get_all_known_tool_names(tool_def) - for tn in known - denied: - policy = tool_def.tool_permissions.get(tn, "ask") - if policy == "always_allow": - effective_allowed.append(f"mcp__{name}__{tn}") - for tn in denied: - effective_disallowed.append(f"mcp__{name}__{tn}") - else: - effective_allowed.append(f"mcp__{name}__*") + # Tool executor — routes to builtins or MCP servers + async def tool_executor(tool_name: str, tool_input: dict) -> list[dict]: + t0 = time.time() - options_kwargs = { - "model": session.model, - "max_buffer_size": 5 * 1024 * 1024, - "permission_mode": "default", - "can_use_tool": can_use_tool, - "hooks": { - "PreToolUse": [HookMatcher(matcher=None, hooks=[pre_tool_hook])], - "PostToolUse": [HookMatcher(matcher=None, hooks=[post_tool_hook])], - }, - "allowed_tools": effective_allowed, - "disallowed_tools": effective_disallowed, - "include_partial_messages": True, - } - if not global_settings.anthropic_api_key: - raise ValueError("Anthropic API key not configured. Set it in Settings.") - options_kwargs["env"] = {"ANTHROPIC_API_KEY": global_settings.anthropic_api_key} - if mcp_servers: - options_kwargs["mcp_servers"] = mcp_servers - if composed_prompt: - options_kwargs["system_prompt"] = composed_prompt - if session.max_turns: - options_kwargs["max_turns"] = session.max_turns + # Check builtin tools first + from backend.apps.agents.tools.registry import get_tool + from backend.apps.agents.tools.base import ToolContext + builtin = get_tool(tool_name) + if builtin: + ctx = ToolContext(cwd=session.cwd or os.path.expanduser("~"), session_id=session_id) + result = await builtin.execute(tool_input, ctx) + elapsed_ms = int((time.time() - t0) * 1000) + _analytics("tool.called", { + "tool_name": tool_name, + "tool_type": "builtin", + "elapsed_ms": elapsed_ms, + "input_summary": str(tool_input)[:200], + }, session_id=session_id) + return result - if session.cwd: - options_kwargs["cwd"] = session.cwd + # Check MCP tools + parsed = mcp_manager.parse_mcp_tool_name(tool_name) + if parsed: + server_name, bare_name = parsed + result = await mcp_manager.call_tool(server_name, bare_name, tool_input) + elapsed_ms = int((time.time() - t0) * 1000) + _analytics("tool.called", { + "tool_name": tool_name, + "tool_short_name": bare_name, + "tool_type": "mcp", + "mcp_server": server_name, + "elapsed_ms": elapsed_ms, + "input_summary": str(tool_input)[:200], + }, session_id=session_id) + return result - if session.sdk_session_id: - options_kwargs["resume"] = session.sdk_session_id - if fork_session: - options_kwargs["fork_session"] = True + return [{"type": "text", "text": f"Unknown tool: {tool_name}"}] - options = ClaudeAgentOptions(**options_kwargs) + # WebSocket emitter — captures messages into session + async def ws_emitter(event_type: str, data: dict) -> None: + data["session_id"] = session_id + await ws_manager.send_to_session(session_id, event_type, data) + # Capture finalized messages into session.messages + if event_type == "agent:message" and "message" in data: + msg_data = data["message"] + try: + msg = Message(**msg_data) + msg.branch_id = session.active_branch_id + session.messages.append(msg) + except Exception: + pass - async def prompt_stream(): - yield { - "type": "user", - "message": {"role": "user", "content": prompt_content}, - } + # Create and run the agent loop + loop = AgentLoop( + session_id=session_id, + provider=provider, + model=session.model, + system_prompt=composed_prompt, + tools=all_tool_schemas, + tool_executor=tool_executor, + hitl_handler=hitl_handler, + ws_emitter=ws_emitter, + max_turns=session.max_turns, + cwd=session.cwd, + ) - stream_text_msg_id = None - stream_tool_msg_ids_ordered = [] - stream_block_index_map = {} + await loop.run(prompt_content) - async for message in query( - prompt=prompt_stream(), - options=options, - ): - if isinstance(message, StreamEvent): - event = message.event - event_type = event.get("type") - - if event_type == "content_block_start": - block = event.get("content_block", {}) - index = event.get("index") - block_type = block.get("type") - - if block_type == "text": - if stream_text_msg_id is None: - stream_text_msg_id = uuid4().hex - await ws_manager.send_to_session(session_id, "agent:stream_start", { - "session_id": session_id, - "message_id": stream_text_msg_id, - "role": "assistant", - }) - stream_block_index_map[index] = stream_text_msg_id - - elif block_type == "tool_use": - tool_msg_id = uuid4().hex - stream_tool_msg_ids_ordered.append(tool_msg_id) - stream_block_index_map[index] = tool_msg_id - await ws_manager.send_to_session(session_id, "agent:stream_start", { - "session_id": session_id, - "message_id": tool_msg_id, - "role": "tool_call", - "tool_name": block.get("name", ""), - }) - - elif event_type == "content_block_delta": - index = event.get("index") - delta = event.get("delta", {}) - delta_type = delta.get("type") - msg_id = stream_block_index_map.get(index) - - if msg_id and delta_type == "text_delta": - await ws_manager.send_to_session(session_id, "agent:stream_delta", { - "session_id": session_id, - "message_id": msg_id, - "delta": delta.get("text", ""), - }) - elif msg_id and delta_type == "input_json_delta": - await ws_manager.send_to_session(session_id, "agent:stream_delta", { - "session_id": session_id, - "message_id": msg_id, - "delta": delta.get("partial_json", ""), - }) - - elif event_type == "content_block_stop": - index = event.get("index") - msg_id = stream_block_index_map.get(index) - if msg_id and msg_id != stream_text_msg_id: - await ws_manager.send_to_session(session_id, "agent:stream_end", { - "session_id": session_id, - "message_id": msg_id, - }) - - elif event_type == "message_stop": - if stream_text_msg_id: - await ws_manager.send_to_session(session_id, "agent:stream_end", { - "session_id": session_id, - "message_id": stream_text_msg_id, - }) - - elif isinstance(message, AssistantMessage): - content_parts = [] - tool_uses = [] - for block in message.content: - if isinstance(block, TextBlock): - content_parts.append(block.text) - elif isinstance(block, ToolUseBlock): - tool_uses.append({ - "id": block.id, - "tool": block.name, - "input": block.input, - }) - - if content_parts: - asst_msg = Message( - id=stream_text_msg_id or uuid4().hex, - role="assistant", - content="\n".join(content_parts), - branch_id=session.active_branch_id, - ) - session.messages.append(asst_msg) - await ws_manager.send_to_session(session_id, "agent:message", { - "session_id": session_id, - "message": asst_msg.model_dump(mode="json"), - }) - - for i, tu in enumerate(tool_uses): - msg_id = stream_tool_msg_ids_ordered[i] if i < len(stream_tool_msg_ids_ordered) else uuid4().hex - tool_msg = Message(id=msg_id, role="tool_call", content=tu, branch_id=session.active_branch_id) - session.messages.append(tool_msg) - await ws_manager.send_to_session(session_id, "agent:message", { - "session_id": session_id, - "message": tool_msg.model_dump(mode="json"), - }) - - stream_text_msg_id = None - stream_tool_msg_ids_ordered = [] - stream_block_index_map = {} - - elif isinstance(message, ResultMessage): - session.sdk_session_id = getattr(message, "session_id", None) - cost = getattr(message, "total_cost_usd", None) - if cost is not None: - session.cost_usd = cost - await ws_manager.send_to_session(session_id, "agent:cost_update", { - "session_id": session_id, - "cost_usd": session.cost_usd, - }) + # Update cost from token usage + session.tokens["input"] += loop.total_input_tokens + session.tokens["output"] += loop.total_output_tokens + session.cost_usd += calculate_cost( + session.provider, session.model, + loop.total_input_tokens, loop.total_output_tokens, + ) + await ws_manager.send_to_session(session_id, "agent:cost_update", { + "session_id": session_id, + "cost_usd": session.cost_usd, + }) session.status = "completed" except asyncio.CancelledError: @@ -980,6 +761,10 @@ class AgentManager: except Exception as e: logger.exception(f"Agent {session_id} error: {e}") session.status = "error" + _analytics("error.occurred", { + "error_category": type(e).__name__, + "error_message": str(e)[:200], + }, session_id=session_id) error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id) session.messages.append(error_msg) await ws_manager.send_to_session(session_id, "agent:message", { @@ -987,7 +772,61 @@ class AgentManager: "message": error_msg.model_dump(mode="json"), }) finally: + if mcp_manager: + try: + await mcp_manager.__aexit__(None, None, None) + except Exception as e: + logger.warning(f"MCP cleanup error: {e}") if session_id in self.sessions: + # Track session completion + duration = 0.0 + if session.created_at: + duration = (datetime.now() - session.created_at).total_seconds() + msg_count = len([m for m in session.messages if m.role in ("user", "assistant")]) + tool_names = [m.content.get("tool", "") for m in session.messages if m.role == "tool_call" and isinstance(m.content, dict)] + task_category = _infer_task_category(tool_names) + + # Collect rich session data for analytics — full messages, no truncation + user_messages = [ + m.content if isinstance(m.content, str) else str(m.content) + for m in session.messages if m.role == "user" + ] + assistant_messages = [ + m.content if isinstance(m.content, str) else str(m.content) + for m in session.messages if m.role == "assistant" + ] + # Unique tools used with call counts + tool_call_counts: dict[str, int] = {} + for tn in tool_names: + short = tn.split("__")[-1] if "__" in tn else tn + tool_call_counts[short] = tool_call_counts.get(short, 0) + 1 + # MCP servers used + mcp_servers_used = list(set( + tn.split("__")[1] for tn in tool_names + if tn.startswith("mcp__") and len(tn.split("__")) >= 3 + )) + + _analytics("session.completed", { + "model": session.model, + "provider": session.provider, + "mode": session.mode, + "cost_usd": session.cost_usd, + "message_count": msg_count, + "duration_seconds": round(duration, 1), + "status": session.status, + "task_category": task_category, + "tool_count": len(tool_names), + # Rich data + "session_title": session.name, + "user_messages": user_messages, + "assistant_messages": assistant_messages, + "first_user_message": user_messages[0] if user_messages else "", + "tools_used": tool_call_counts, + "tools_list": list(tool_call_counts.keys()), + "mcp_servers_used": mcp_servers_used, + "system_prompt_length": len(session.system_prompt or ""), + }, session_id=session_id, dashboard_id=session.dashboard_id) + await ws_manager.send_to_session(session_id, "agent:status", { "session_id": session_id, "status": session.status, @@ -1134,6 +973,7 @@ class AgentManager: prompt: str, mode: str | None = None, model: str | None = None, + provider: str | None = None, images: list | None = None, context_paths: list | None = None, forced_tools: list[str] | None = None, @@ -1145,12 +985,15 @@ class AgentManager: session = self.sessions.get(session_id) if not session: raise ValueError(f"Session {session_id} not found") - + existing = self.tasks.get(session_id) if existing and not existing.done(): return session_changed = False + if provider and provider != session.provider: + session.provider = provider + session_changed = True if model and model != session.model: session.model = model session_changed = True diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 070e29c5..a1e69dcf 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -52,6 +52,7 @@ async def send_message(session_id: str, body: dict): prompt, mode=body.get("mode"), model=body.get("model"), + provider=body.get("provider"), images=body.get("images"), context_paths=body.get("context_paths"), forced_tools=body.get("forced_tools"), @@ -178,3 +179,12 @@ async def resume_session(session_id: str): raise HTTPException(status_code=404, detail=str(e)) return {"session": session.model_dump(mode="json")} + +@agents.router.get("/models") +async def list_models(): + """Return available models grouped by provider, filtered by configured credentials.""" + from backend.apps.agents.providers.registry import get_available_models + from backend.apps.settings.settings import load_settings + settings = load_settings() + return {"models": get_available_models(settings)} + diff --git a/backend/apps/agents/browser_agent.py b/backend/apps/agents/browser_agent.py index 1508797a..643ad242 100644 --- a/backend/apps/agents/browser_agent.py +++ b/backend/apps/agents/browser_agent.py @@ -24,7 +24,7 @@ logger = logging.getLogger(__name__) MODEL_MAP = { "sonnet": "claude-sonnet-4-20250514", "opus": "claude-opus-4-20250514", - "haiku": "claude-haiku-4-20250414", + "haiku": "claude-haiku-4-5-20251001", } BROWSER_TOOLS_SCHEMA = [ diff --git a/backend/apps/agents/mcp_client.py b/backend/apps/agents/mcp_client.py new file mode 100644 index 00000000..303078ab --- /dev/null +++ b/backend/apps/agents/mcp_client.py @@ -0,0 +1,360 @@ +"""Standalone MCP client manager for agent sessions. + +Replaces claude_agent_sdk's internal MCP server management. +One MCPClientManager instance per agent session — manages connections +to stdio/http/sse MCP servers, discovers tools, and routes tool calls. +""" + +from __future__ import annotations + +import asyncio +import json +import logging +from contextlib import AsyncExitStack +from dataclasses import dataclass, field +from typing import Any + +from backend.apps.agents.providers.base import ToolSchema + +logger = logging.getLogger(__name__) + + +@dataclass +class MCPConnection: + """A live connection to an MCP server.""" + server_name: str + session: Any # mcp.ClientSession + tools: list[ToolSchema] = field(default_factory=list) + + +class MCPClientManager: + """Manages connections to MCP servers for a single agent session.""" + + def __init__(self): + self._connections: dict[str, MCPConnection] = {} + self._exit_stack = AsyncExitStack() + self._started = False + + async def __aenter__(self): + await self._exit_stack.__aenter__() + self._started = True + return self + + async def __aexit__(self, *exc): + await self.disconnect_all() + try: + await self._exit_stack.__aexit__(*exc) + except (BaseExceptionGroup, ExceptionGroup, Exception) as e: + # MCP subprocess cleanup errors are non-fatal + logger.warning(f"MCP cleanup error (non-fatal): {e}") + self._started = False + + async def connect(self, server_name: str, config: dict, timeout: float = 30.0) -> list[ToolSchema]: + """Connect to an MCP server and return its available tools. + + The tools are returned with names prefixed as mcp____. + """ + transport = config.get("type", "stdio") + try: + if transport == "stdio": + coro = self._connect_stdio(server_name, config) + elif transport == "sse": + coro = self._connect_sse(server_name, config) + elif transport == "http": + coro = self._connect_http(server_name, config) + else: + logger.warning(f"Unsupported MCP transport: {transport} for {server_name}") + return [] + + conn = await asyncio.wait_for(coro, timeout=timeout) + self._connections[server_name] = conn + logger.info(f"MCP connected: {server_name} ({len(conn.tools)} tools)") + return conn.tools + + except asyncio.TimeoutError: + logger.warning(f"MCP server {server_name} connection timed out after {timeout}s") + return [] + except Exception as e: + logger.warning(f"Failed to connect MCP server {server_name}: {e}") + return [] + + async def _connect_stdio(self, server_name: str, config: dict) -> MCPConnection: + """Connect to a stdio MCP server (spawns a subprocess).""" + from mcp import ClientSession + from mcp.client.stdio import stdio_client, StdioServerParameters + + command = config.get("command", "") + args = config.get("args", []) + env = config.get("env") + + params = StdioServerParameters( + command=command, + args=args, + env=env, + ) + + transport = await self._exit_stack.enter_async_context( + stdio_client(params) + ) + read_stream, write_stream = transport + session = await self._exit_stack.enter_async_context( + ClientSession(read_stream, write_stream) + ) + await session.initialize() + + result = await session.list_tools() + tools = [ + ToolSchema( + name=f"mcp__{server_name}__{t.name}", + description=t.description or "", + input_schema=t.inputSchema if hasattr(t, "inputSchema") else (t.input_schema if hasattr(t, "input_schema") else {}), + ) + for t in result.tools + ] + + return MCPConnection(server_name=server_name, session=session, tools=tools) + + async def _connect_sse(self, server_name: str, config: dict) -> MCPConnection: + """Connect to an SSE MCP server.""" + from mcp import ClientSession + from mcp.client.sse import sse_client + + url = config.get("url", "") + headers = config.get("headers") + + transport = await self._exit_stack.enter_async_context( + sse_client(url=url, headers=headers, timeout=30, sse_read_timeout=300) + ) + read_stream, write_stream = transport + session = await self._exit_stack.enter_async_context( + ClientSession(read_stream, write_stream) + ) + await session.initialize() + + result = await session.list_tools() + tools = [ + ToolSchema( + name=f"mcp__{server_name}__{t.name}", + description=t.description or "", + input_schema=t.inputSchema if hasattr(t, "inputSchema") else (t.input_schema if hasattr(t, "input_schema") else {}), + ) + for t in result.tools + ] + + return MCPConnection(server_name=server_name, session=session, tools=tools) + + async def _connect_http(self, server_name: str, config: dict) -> MCPConnection: + """Connect to a Streamable HTTP MCP server. + + Falls back to SSE if streamable HTTP fails. + """ + url = config.get("url", "") + headers = config.get("headers") + + # Try streamable HTTP first, fall back to SSE + try: + return await self._connect_http_streamable(server_name, url, headers) + except Exception as e: + logger.info(f"Streamable HTTP failed for {server_name}, trying SSE: {e}") + return await self._connect_sse(server_name, config) + + async def _connect_http_streamable( + self, server_name: str, url: str, headers: dict | None, + ) -> MCPConnection: + """Connect via Streamable HTTP (JSON-RPC POST).""" + import httpx + from mcp import ClientSession + + # Use httpx for streamable HTTP — keep client alive in the exit stack + client = await self._exit_stack.enter_async_context( + httpx.AsyncClient(timeout=30.0) + ) + + h = { + "Content-Type": "application/json", + "Accept": "application/json, text/event-stream", + **(headers or {}), + } + + # Initialize + init_resp = await client.post(url, headers=h, json={ + "jsonrpc": "2.0", "id": 1, "method": "initialize", + "params": { + "protocolVersion": "2025-03-26", + "capabilities": {}, + "clientInfo": {"name": "self-swarm", "version": "0.1.0"}, + }, + }) + if init_resp.status_code not in (200, 201): + raise ConnectionError(f"MCP initialize failed: {init_resp.status_code}") + + session_id = init_resp.headers.get("mcp-session-id", "") + if session_id: + h["mcp-session-id"] = session_id + + # Notify initialized + await client.post(url, headers=h, json={ + "jsonrpc": "2.0", "method": "notifications/initialized", + }) + + # List tools + list_resp = await client.post(url, headers=h, json={ + "jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}, + }) + if list_resp.status_code not in (200, 201): + raise ConnectionError(f"MCP tools/list failed: {list_resp.status_code}") + + ct = list_resp.headers.get("content-type", "") + if "text/event-stream" in ct: + data = self._parse_sse_json(list_resp.text) + else: + data = list_resp.json() + + if not data: + raise ConnectionError("Empty response from MCP server") + + tools_list = data.get("result", {}).get("tools", []) + tools = [ + ToolSchema( + name=f"mcp__{server_name}__{t.get('name', '')}", + description=t.get("description", ""), + input_schema=t.get("inputSchema", t.get("input_schema", {})), + ) + for t in tools_list + ] + + # Store the HTTP client info for call_tool + conn = MCPConnection(server_name=server_name, session=None, tools=tools) + conn._http_client = client # type: ignore[attr-defined] + conn._http_url = url # type: ignore[attr-defined] + conn._http_headers = h # type: ignore[attr-defined] + conn._next_id = 3 # type: ignore[attr-defined] + return conn + + @staticmethod + def _parse_sse_json(text: str) -> dict | None: + """Extract JSON from an SSE response body.""" + for line in text.splitlines(): + stripped = line.strip() + if stripped.startswith("data:"): + payload = stripped[len("data:"):].strip() + if payload: + try: + return json.loads(payload) + except json.JSONDecodeError: + continue + try: + return json.loads(text) + except json.JSONDecodeError: + return None + + async def call_tool( + self, server_name: str, tool_name: str, arguments: dict, + ) -> list[dict]: + """Call a tool on a specific MCP server. + + Args: + server_name: The MCP server name (e.g. "google-workspace") + tool_name: The bare tool name (without mcp__prefix) + arguments: Tool input arguments + + Returns: + List of content blocks: [{"type": "text", "text": "..."}] + """ + conn = self._connections.get(server_name) + if not conn: + return [{"type": "text", "text": f"MCP server {server_name} not connected"}] + + try: + if conn.session is not None: + # stdio or SSE — use MCP ClientSession + result = await conn.session.call_tool(tool_name, arguments) + return self._format_mcp_result(result) + elif hasattr(conn, "_http_client"): + # Streamable HTTP — use JSON-RPC + return await self._call_tool_http(conn, tool_name, arguments) + else: + return [{"type": "text", "text": f"No session for MCP server {server_name}"}] + + except Exception as e: + logger.warning(f"MCP tool call failed: {server_name}/{tool_name}: {e}") + return [{"type": "text", "text": f"Error calling {tool_name}: {e}"}] + + async def _call_tool_http( + self, conn: MCPConnection, tool_name: str, arguments: dict, + ) -> list[dict]: + """Call a tool via Streamable HTTP.""" + client = conn._http_client # type: ignore[attr-defined] + url = conn._http_url # type: ignore[attr-defined] + headers = conn._http_headers # type: ignore[attr-defined] + req_id = conn._next_id # type: ignore[attr-defined] + conn._next_id = req_id + 1 # type: ignore[attr-defined] + + resp = await client.post(url, headers=headers, json={ + "jsonrpc": "2.0", + "id": req_id, + "method": "tools/call", + "params": {"name": tool_name, "arguments": arguments}, + }, timeout=300.0) + + ct = resp.headers.get("content-type", "") + if "text/event-stream" in ct: + data = self._parse_sse_json(resp.text) + else: + data = resp.json() + + if not data: + return [{"type": "text", "text": "Empty response from MCP server"}] + + if "error" in data: + return [{"type": "text", "text": f"MCP error: {data['error']}"}] + + result = data.get("result", {}) + content = result.get("content", []) + return content if content else [{"type": "text", "text": json.dumps(result)}] + + @staticmethod + def _format_mcp_result(result: Any) -> list[dict]: + """Convert an MCP CallToolResult to content blocks.""" + if hasattr(result, "content"): + blocks = [] + for item in result.content: + if hasattr(item, "text"): + blocks.append({"type": "text", "text": item.text}) + elif hasattr(item, "data"): + blocks.append({ + "type": "image", + "source": { + "type": "base64", + "media_type": getattr(item, "mimeType", "image/png"), + "data": item.data, + }, + }) + else: + blocks.append({"type": "text", "text": str(item)}) + return blocks if blocks else [{"type": "text", "text": "Done."}] + + return [{"type": "text", "text": str(result)}] + + def get_all_tool_schemas(self) -> list[ToolSchema]: + """Return tool schemas from all connected MCP servers.""" + schemas = [] + for conn in self._connections.values(): + schemas.extend(conn.tools) + return schemas + + def parse_mcp_tool_name(self, full_name: str) -> tuple[str, str] | None: + """Parse mcp____ into (server_name, tool_name). + + Returns None if the name doesn't match the MCP naming convention. + """ + import re + m = re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", full_name) + if m: + return m.group(1), m.group(2) + return None + + async def disconnect_all(self): + """Disconnect all MCP servers. Called on session end.""" + self._connections.clear() + # The AsyncExitStack handles actual cleanup of transports/sessions diff --git a/backend/apps/agents/models.py b/backend/apps/agents/models.py index 948a007c..d5e6ccfe 100644 --- a/backend/apps/agents/models.py +++ b/backend/apps/agents/models.py @@ -5,6 +5,7 @@ from uuid import uuid4 class AgentConfig(BaseModel): name: str = Field(default_factory=lambda: f"Agent-{uuid4().hex[:6]}") + provider: str = "anthropic" model: str = "sonnet" mode: str = "agent" system_prompt: Optional[str] = None @@ -55,6 +56,7 @@ class AgentSession(BaseModel): id: str = Field(default_factory=lambda: uuid4().hex) name: str status: Literal["running", "waiting_approval", "completed", "error", "stopped"] = "running" + provider: str = "anthropic" model: str = "sonnet" mode: str = "agent" sdk_session_id: Optional[str] = None diff --git a/backend/apps/agents/providers/__init__.py b/backend/apps/agents/providers/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/agents/providers/anthropic.py b/backend/apps/agents/providers/anthropic.py new file mode 100644 index 00000000..bd347ead --- /dev/null +++ b/backend/apps/agents/providers/anthropic.py @@ -0,0 +1,260 @@ +"""Anthropic provider adapter using the native Anthropic SDK.""" + +from __future__ import annotations + +import json +import logging +from typing import Any, AsyncIterator + +import anthropic + +from backend.apps.agents.providers.base import ( + BaseProvider, ContentBlock, ModelResponse, ProviderMessage, + StreamEvent, ToolCall, ToolSchema, +) + +logger = logging.getLogger(__name__) + +MODEL_MAP = { + "sonnet": "claude-sonnet-4-6", + "opus": "claude-opus-4-6", + "haiku": "claude-haiku-4-5", +} + + +class AnthropicProvider(BaseProvider): + """Provider adapter for Anthropic's Messages API.""" + + def __init__( + self, + api_key: str | None = None, + auth_token: str | None = None, + base_url: str | None = None, + ): + kwargs: dict[str, Any] = {} + if auth_token: + kwargs["auth_token"] = auth_token + elif api_key: + kwargs["api_key"] = api_key + if base_url: + kwargs["base_url"] = base_url + self.client = anthropic.AsyncAnthropic(**kwargs) + + def get_model_id(self, short_name: str) -> str: + return MODEL_MAP.get(short_name, short_name) + + def clean_tool_schema(self, schema: ToolSchema) -> dict: + return { + "name": schema.name, + "description": schema.description, + "input_schema": schema.input_schema, + } + + def format_tool_result(self, tool_use_id: str, content: list[dict]) -> dict: + return { + "type": "tool_result", + "tool_use_id": tool_use_id, + "content": content, + } + + def format_user_message(self, content: Any) -> ProviderMessage: + return ProviderMessage(role="user", content=content) + + def format_assistant_message(self, response: ModelResponse) -> ProviderMessage: + blocks = [] + for block in response.content: + if block.type == "text": + blocks.append({"type": "text", "text": block.text}) + elif block.type == "tool_use" and block.tool_call: + blocks.append({ + "type": "tool_use", + "id": block.tool_call.id, + "name": block.tool_call.name, + "input": block.tool_call.input, + }) + return ProviderMessage(role="assistant", content=blocks) + + def _build_messages(self, messages: list[ProviderMessage]) -> list[dict]: + """Convert ProviderMessages to Anthropic API format.""" + result = [] + for msg in messages: + if msg.role == "tool_result": + # Tool results: content is a list of tool_result dicts + if isinstance(msg.content, list): + result.append({"role": "user", "content": msg.content}) + else: + result.append({"role": "user", "content": [msg.content]}) + elif msg.role == "assistant": + result.append({"role": "assistant", "content": msg.content}) + elif msg.role == "user": + result.append({"role": "user", "content": msg.content}) + return result + + async def create_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> ModelResponse: + kwargs: dict[str, Any] = { + "model": self.get_model_id(model), + "max_tokens": max_tokens, + "messages": self._build_messages(messages), + } + if system: + kwargs["system"] = system + if tools: + kwargs["tools"] = [self.clean_tool_schema(t) for t in tools] + + resp = await self.client.messages.create(**kwargs) + + content = [] + for block in resp.content: + if block.type == "text": + content.append(ContentBlock(type="text", text=block.text)) + elif block.type == "tool_use": + content.append(ContentBlock( + type="tool_use", + tool_call=ToolCall( + id=block.id, + name=block.name, + input=block.input, + ), + )) + + return ModelResponse( + content=content, + stop_reason="tool_use" if resp.stop_reason == "tool_use" else "end_turn", + usage={ + "input_tokens": resp.usage.input_tokens, + "output_tokens": resp.usage.output_tokens, + }, + ) + + async def stream_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> AsyncIterator[StreamEvent]: + kwargs: dict[str, Any] = { + "model": self.get_model_id(model), + "max_tokens": max_tokens, + "messages": self._build_messages(messages), + } + if system: + kwargs["system"] = system + if tools: + kwargs["tools"] = [self.clean_tool_schema(t) for t in tools] + + # Use create() with stream=True for raw SSE events + kwargs["stream"] = True + raw_stream = await self.client.messages.create(**kwargs) + + current_block_type: dict[int, str] = {} + current_tool_name: dict[int, str] = {} + current_tool_id: dict[int, str] = {} + current_text: dict[int, str] = {} + current_json: dict[int, str] = {} + + async for event in raw_stream: + event_type = getattr(event, "type", "") + + if event_type == "content_block_start": + index = event.index + block = event.content_block + block_type = block.type + current_block_type[index] = block_type + + if block_type == "text": + current_text[index] = "" + yield StreamEvent( + type="content_block_start", + index=index, + block_type="text", + ) + elif block_type == "tool_use": + current_tool_name[index] = block.name + current_tool_id[index] = block.id + current_json[index] = "" + yield StreamEvent( + type="content_block_start", + index=index, + block_type="tool_use", + tool_name=block.name, + tool_id=block.id, + ) + + elif event_type == "content_block_delta": + index = event.index + delta = event.delta + delta_type = delta.type + + if delta_type == "text_delta": + current_text.setdefault(index, "") + current_text[index] += delta.text + yield StreamEvent( + type="content_block_delta", + index=index, + delta_type="text_delta", + text=delta.text, + ) + elif delta_type == "input_json_delta": + current_json.setdefault(index, "") + current_json[index] += delta.partial_json + yield StreamEvent( + type="content_block_delta", + index=index, + delta_type="input_json_delta", + text=delta.partial_json, + ) + + elif event_type == "content_block_stop": + yield StreamEvent(type="content_block_stop", index=event.index) + + elif event_type == "message_delta": + # Extract output token usage from the final delta + usage_data = {} + delta_usage = getattr(event, "usage", None) + if delta_usage: + output_tokens = getattr(delta_usage, "output_tokens", 0) + if output_tokens: + usage_data["output_tokens"] = output_tokens + if usage_data: + yield StreamEvent(type="usage", usage=usage_data) + + elif event_type == "message_start": + # Extract input token usage from the message start + msg = getattr(event, "message", None) + if msg: + msg_usage = getattr(msg, "usage", None) + if msg_usage: + usage_data = {} + input_tokens = getattr(msg_usage, "input_tokens", 0) + output_tokens = getattr(msg_usage, "output_tokens", 0) + if input_tokens: + usage_data["input_tokens"] = input_tokens + if output_tokens: + usage_data["output_tokens"] = output_tokens + if usage_data: + yield StreamEvent(type="usage", usage=usage_data) + + yield StreamEvent(type="message_stop") + + async def stream_and_collect( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> tuple[AsyncIterator[StreamEvent], ModelResponse]: + """Helper: stream events and also return the full collected response. + + Not used directly — the AgentLoop handles collection. + """ + raise NotImplementedError("Use stream_message() directly; AgentLoop collects.") diff --git a/backend/apps/agents/providers/base.py b/backend/apps/agents/providers/base.py new file mode 100644 index 00000000..ef2f746e --- /dev/null +++ b/backend/apps/agents/providers/base.py @@ -0,0 +1,135 @@ +"""Provider-agnostic base classes for multi-model support. + +All provider adapters (Anthropic, OpenAI, Gemini, OpenAI-compatible) +implement BaseProvider, translating their native APIs into these +common data structures. +""" + +from __future__ import annotations + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Any, AsyncIterator + + +@dataclass +class ToolSchema: + """Provider-agnostic tool definition.""" + name: str + description: str + input_schema: dict[str, Any] + + +@dataclass +class ToolCall: + """A tool invocation requested by the model.""" + id: str + name: str + input: dict[str, Any] + + +@dataclass +class ContentBlock: + """A block of content from the model response.""" + type: str # "text" | "tool_use" + text: str = "" + tool_call: ToolCall | None = None + + +@dataclass +class ModelResponse: + """Complete (non-streaming) response from a provider.""" + content: list[ContentBlock] + stop_reason: str # "end_turn" | "tool_use" | "max_tokens" + usage: dict[str, int] = field(default_factory=dict) + + +@dataclass +class StreamEvent: + """A single streaming event, normalized across providers. + + The event types match what the frontend already expects via WebSocket: + content_block_start, content_block_delta, content_block_stop, message_stop. + """ + type: str + index: int = 0 + block_type: str = "" # "text" | "tool_use" + delta_type: str = "" # "text_delta" | "input_json_delta" + text: str = "" + tool_name: str = "" + tool_id: str = "" + usage: dict[str, int] = field(default_factory=dict) + + +@dataclass +class ProviderMessage: + """Provider-agnostic message for conversation history. + + Each provider adapter converts these to/from its native format. + """ + role: str # "user" | "assistant" | "tool_result" + content: Any # str, list[dict], or provider-specific content + + +class BaseProvider(ABC): + """Abstract base for LLM provider adapters.""" + + @abstractmethod + async def stream_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> AsyncIterator[StreamEvent]: + """Stream a model response, yielding normalized StreamEvents.""" + ... + + @abstractmethod + async def create_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> ModelResponse: + """Non-streaming message creation.""" + ... + + @abstractmethod + def format_tool_result( + self, + tool_use_id: str, + content: list[dict], + ) -> dict: + """Format a tool result in this provider's expected message format.""" + ... + + @abstractmethod + def format_user_message(self, content: Any) -> ProviderMessage: + """Wrap user content (str or multimodal blocks) into a ProviderMessage.""" + ... + + @abstractmethod + def format_assistant_message(self, response: ModelResponse) -> ProviderMessage: + """Convert a ModelResponse into a ProviderMessage for conversation history.""" + ... + + @abstractmethod + def get_model_id(self, short_name: str) -> str: + """Resolve a short model name to the full API model ID.""" + ... + + def clean_tool_schema(self, schema: ToolSchema) -> dict: + """Convert a ToolSchema to the provider's native tool format. + + Default: Anthropic-style format. Override for providers that need + different formats or schema cleaning (e.g. Gemini). + """ + return { + "name": schema.name, + "description": schema.description, + "input_schema": schema.input_schema, + } diff --git a/backend/apps/agents/providers/gemini.py b/backend/apps/agents/providers/gemini.py new file mode 100644 index 00000000..7abb6a20 --- /dev/null +++ b/backend/apps/agents/providers/gemini.py @@ -0,0 +1,600 @@ +"""Gemini provider adapter using the new google-genai SDK.""" + +from __future__ import annotations + +import base64 +import json +import logging +from copy import deepcopy +from typing import Any, AsyncIterator +from uuid import uuid4 + +from google import genai +from google.genai import types + +from backend.apps.agents.providers.base import ( + BaseProvider, + ContentBlock, + ModelResponse, + ProviderMessage, + StreamEvent, + ToolCall, + ToolSchema, +) + +logger = logging.getLogger(__name__) + +MODEL_MAP = { + "flash": "gemini-2.5-flash", + "pro": "gemini-2.5-pro", +} + +# JSON Schema keywords that Gemini does not support +_UNSUPPORTED_VALIDATION_KEYS = frozenset({ + "minLength", + "maxLength", + "minimum", + "maximum", + "exclusiveMinimum", + "exclusiveMaximum", + "pattern", + "minItems", + "maxItems", + "uniqueItems", + "minProperties", + "maxProperties", + "multipleOf", + "format", + "const", +}) + +_UNSUPPORTED_STRUCTURAL_KEYS = frozenset({ + "$ref", + "$defs", + "patternProperties", +}) + + +# --------------------------------------------------------------------------- +# Schema cleaning — port of OpenClaw's clean-for-gemini.ts logic +# --------------------------------------------------------------------------- + +def _resolve_ref(ref: str, root_defs: dict[str, Any]) -> dict[str, Any]: + """Attempt to resolve a $ref pointer like '#/$defs/Foo'.""" + if ref.startswith("#/$defs/"): + name = ref[len("#/$defs/"):] + if name in root_defs: + return deepcopy(root_defs[name]) + # Cannot resolve — return empty object + return {"type": "object"} + + +def _clean_schema_node(node: dict[str, Any], root_defs: dict[str, Any]) -> dict[str, Any]: + """Recursively clean a single JSON Schema node for Gemini compatibility.""" + if not isinstance(node, dict): + return node + + # If this node is just a $ref, resolve it first then clean the result + if "$ref" in node and len(node) <= 2: # $ref possibly with description + resolved = _resolve_ref(node["$ref"], root_defs) + # Carry over description if the ref node had one + if "description" in node: + resolved["description"] = node["description"] + return _clean_schema_node(resolved, root_defs) + + result: dict[str, Any] = {} + + for key, value in node.items(): + # Drop unsupported keys + if key in _UNSUPPORTED_VALIDATION_KEYS: + continue + if key in _UNSUPPORTED_STRUCTURAL_KEYS: + continue + + # Handle additionalProperties: drop if boolean, recurse if schema + if key == "additionalProperties": + if isinstance(value, bool): + continue + # It's a schema dict — clean and keep it + result[key] = _clean_schema_node(value, root_defs) + continue + + # Handle anyOf / oneOf: flatten into something Gemini can use + if key in ("anyOf", "oneOf"): + if isinstance(value, list): + flattened = _flatten_union(value, root_defs) + if flattened is not None: + result.update(flattened) + continue + + # Handle allOf: merge all members + if key == "allOf": + if isinstance(value, list): + merged = _merge_all_of(value, root_defs) + result.update(merged) + continue + + # Recurse into properties + if key == "properties" and isinstance(value, dict): + result[key] = { + prop_name: _clean_schema_node(prop_schema, root_defs) + for prop_name, prop_schema in value.items() + } + continue + + # Recurse into items + if key == "items": + if isinstance(value, dict): + result[key] = _clean_schema_node(value, root_defs) + elif isinstance(value, list): + result[key] = [_clean_schema_node(item, root_defs) for item in value] + else: + result[key] = value + continue + + # Keep everything else (type, description, title, default, enum, required, etc.) + result[key] = value + + return result + + +def _flatten_union( + variants: list[dict[str, Any]], + root_defs: dict[str, Any], +) -> dict[str, Any] | None: + """Collapse anyOf/oneOf into a Gemini-compatible schema. + + Strategies (applied in order): + 1. If all variants are literal types (with const or single-value enums), + collapse into a single enum. + 2. If there's a null variant mixed with non-null variants, strip the null + variant and return the remaining schema (nullable). + 3. If only one non-null variant remains after stripping, unwrap it. + 4. Otherwise return the first variant (best-effort). + """ + if not variants: + return None + + # Clean each variant first (resolve refs, etc.) + cleaned = [_clean_schema_node(v, root_defs) for v in variants] + + # Strategy 1: all literal types → collapse to enum + enum_values: list[Any] = [] + all_literals = True + for v in cleaned: + if "const" in v: + enum_values.append(v["const"]) + elif "enum" in v and isinstance(v["enum"], list) and len(v["enum"]) == 1: + enum_values.append(v["enum"][0]) + else: + all_literals = False + break + if all_literals and enum_values: + return {"type": "string", "enum": enum_values} + + # Strategy 2 & 3: strip null variants + non_null = [v for v in cleaned if v.get("type") != "null"] + + if len(non_null) == 0: + # All variants are null + return {"type": "string"} + + if len(non_null) == 1: + # Single non-null variant — unwrap it, mark as nullable + result = non_null[0].copy() + result["nullable"] = True + return result + + # Strategy 4: multiple non-null variants, just use first (best-effort) + return non_null[0] + + +def _merge_all_of( + members: list[dict[str, Any]], + root_defs: dict[str, Any], +) -> dict[str, Any]: + """Merge allOf members into a single cleaned schema.""" + merged: dict[str, Any] = {} + for member in members: + cleaned = _clean_schema_node(member, root_defs) + for key, value in cleaned.items(): + if key == "properties" and key in merged: + merged[key].update(value) + elif key == "required" and key in merged: + existing = set(merged[key]) + existing.update(value) + merged[key] = sorted(existing) + else: + merged[key] = value + return merged + + +def clean_schema_for_gemini(schema: dict[str, Any]) -> dict[str, Any]: + """Top-level entry point: clean a full JSON Schema for Gemini compatibility.""" + root_defs = schema.get("$defs", schema.get("definitions", {})) + return _clean_schema_node(schema, root_defs) + + +# --------------------------------------------------------------------------- +# Provider implementation +# --------------------------------------------------------------------------- + +class GeminiProvider(BaseProvider): + """Provider adapter for Google Gemini via the google-genai SDK.""" + + def __init__(self, api_key: str): + self.client = genai.Client(api_key=api_key) + + def get_model_id(self, short_name: str) -> str: + return MODEL_MAP.get(short_name, short_name) + + def clean_tool_schema(self, schema: ToolSchema) -> dict: + """Convert a ToolSchema to Gemini function declaration format. + + Cleans the JSON Schema of unsupported features before sending. + """ + cleaned_params = clean_schema_for_gemini(schema.input_schema) + return { + "name": schema.name, + "description": schema.description, + "parameters": cleaned_params, + } + + def format_tool_result(self, tool_use_id: str, content: list[dict]) -> dict: + """Format a tool result for Gemini conversation history. + + Gemini uses FunctionResponse parts; we store enough info to reconstruct them. + """ + # Extract text content for the function response + text_parts = [] + for block in content: + if block.get("type") == "text": + text_parts.append(block.get("text", "")) + elif block.get("type") == "image": + text_parts.append("[image]") + else: + text_parts.append(json.dumps(block)) + + return { + "tool_use_id": tool_use_id, + "output": "\n".join(text_parts) if text_parts else "Done.", + } + + def format_user_message(self, content: Any) -> ProviderMessage: + if isinstance(content, str): + return ProviderMessage(role="user", content=content) + return ProviderMessage(role="user", content=content) + + def format_assistant_message(self, response: ModelResponse) -> ProviderMessage: + """Convert a ModelResponse into a ProviderMessage for conversation history.""" + blocks = [] + for block in response.content: + if block.type == "text": + blocks.append({"type": "text", "text": block.text}) + elif block.type == "tool_use" and block.tool_call: + blocks.append({ + "type": "tool_use", + "id": block.tool_call.id, + "name": block.tool_call.name, + "input": block.tool_call.input, + }) + return ProviderMessage(role="assistant", content=blocks) + + # ------------------------------------------------------------------ + # Internal helpers for building Gemini API messages + # ------------------------------------------------------------------ + + def _build_tools(self, tools: list[ToolSchema]) -> list[types.Tool] | None: + """Convert ToolSchemas to Gemini Tool objects.""" + if not tools: + return None + declarations = [] + for t in tools: + cleaned = self.clean_tool_schema(t) + declarations.append(types.FunctionDeclaration( + name=cleaned["name"], + description=cleaned["description"], + parameters=cleaned["parameters"], + )) + return [types.Tool(function_declarations=declarations)] + + def _build_contents( + self, messages: list[ProviderMessage], + ) -> list[types.Content]: + """Convert ProviderMessages into a list of Gemini Content objects. + + Gemini requires: + - Conversation starts with a user message + - Strict alternating user/model turns + We merge consecutive same-role messages to satisfy this. + """ + raw_contents: list[types.Content] = [] + + for msg in messages: + if msg.role == "user": + parts = self._user_content_to_parts(msg.content) + raw_contents.append(types.Content(role="user", parts=parts)) + + elif msg.role == "assistant": + parts = self._assistant_content_to_parts(msg.content) + raw_contents.append(types.Content(role="model", parts=parts)) + + elif msg.role == "tool_result": + # Tool results become user turns with FunctionResponse parts + parts = self._tool_result_to_parts(msg.content) + raw_contents.append(types.Content(role="user", parts=parts)) + + # Ensure conversation starts with user + if raw_contents and raw_contents[0].role != "user": + raw_contents.insert( + 0, + types.Content( + role="user", + parts=[types.Part.from_text("Hello.")], + ), + ) + + # Merge consecutive same-role turns + merged: list[types.Content] = [] + for content in raw_contents: + if merged and merged[-1].role == content.role: + merged[-1].parts.extend(content.parts) + else: + merged.append(content) + + return merged + + def _user_content_to_parts(self, content: Any) -> list[types.Part]: + """Convert user message content to Gemini Part objects.""" + if isinstance(content, str): + return [types.Part.from_text(content)] + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, str): + parts.append(types.Part.from_text(block)) + elif isinstance(block, dict): + if block.get("type") == "text": + parts.append(types.Part.from_text(block.get("text", ""))) + elif block.get("type") == "image": + source = block.get("source", {}) + media_type = source.get("media_type", "image/png") + data = source.get("data", "") + parts.append(types.Part.from_bytes( + data=base64.b64decode(data), + mime_type=media_type, + )) + else: + parts.append(types.Part.from_text(json.dumps(block))) + return parts if parts else [types.Part.from_text("")] + return [types.Part.from_text(str(content))] + + def _assistant_content_to_parts(self, content: Any) -> list[types.Part]: + """Convert assistant message content to Gemini Part objects.""" + if isinstance(content, str): + return [types.Part.from_text(content)] + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + text = block.get("text", "") + if text: + parts.append(types.Part.from_text(text)) + elif block.get("type") == "tool_use": + parts.append(types.Part( + function_call=types.FunctionCall( + name=block.get("name", ""), + args=block.get("input", {}), + ) + )) + return parts if parts else [types.Part.from_text("")] + return [types.Part.from_text(str(content))] + + def _tool_result_to_parts(self, content: Any) -> list[types.Part]: + """Convert tool result content to Gemini FunctionResponse parts.""" + parts = [] + if isinstance(content, list): + for item in content: + if isinstance(item, dict) and "tool_use_id" in item: + # The tool_use_id is the function name in our format_tool_result + func_name = item.get("tool_use_id", "unknown") + output = item.get("output", "Done.") + parts.append(types.Part( + function_response=types.FunctionResponse( + name=func_name, + response={"result": output}, + ) + )) + elif isinstance(content, dict) and "tool_use_id" in content: + func_name = content.get("tool_use_id", "unknown") + output = content.get("output", "Done.") + parts.append(types.Part( + function_response=types.FunctionResponse( + name=func_name, + response={"result": output}, + ) + )) + + return parts if parts else [types.Part.from_text("Tool result unavailable.")] + + # ------------------------------------------------------------------ + # Non-streaming message creation + # ------------------------------------------------------------------ + + async def create_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> ModelResponse: + contents = self._build_contents(messages) + gemini_tools = self._build_tools(tools) + + config = types.GenerateContentConfig( + max_output_tokens=max_tokens, + ) + if system: + config.system_instruction = system + if gemini_tools: + config.tools = gemini_tools + + resp = await self.client.aio.models.generate_content( + model=self.get_model_id(model), + contents=contents, + config=config, + ) + + return self._parse_response(resp) + + def _parse_response(self, resp: Any) -> ModelResponse: + """Parse a Gemini GenerateContentResponse into a ModelResponse.""" + content: list[ContentBlock] = [] + has_tool_calls = False + + if resp.candidates: + candidate = resp.candidates[0] + if candidate.content and candidate.content.parts: + for part in candidate.content.parts: + if part.text is not None: + content.append(ContentBlock(type="text", text=part.text)) + elif part.function_call is not None: + has_tool_calls = True + fc = part.function_call + content.append(ContentBlock( + type="tool_use", + tool_call=ToolCall( + id=uuid4().hex, + name=fc.name, + input=dict(fc.args) if fc.args else {}, + ), + )) + + stop_reason = "tool_use" if has_tool_calls else "end_turn" + + usage = {} + if resp.usage_metadata: + usage = { + "input_tokens": getattr(resp.usage_metadata, "prompt_token_count", 0) or 0, + "output_tokens": getattr(resp.usage_metadata, "candidates_token_count", 0) or 0, + } + + return ModelResponse(content=content, stop_reason=stop_reason, usage=usage) + + # ------------------------------------------------------------------ + # Streaming message creation + # ------------------------------------------------------------------ + + async def stream_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> AsyncIterator[StreamEvent]: + contents = self._build_contents(messages) + gemini_tools = self._build_tools(tools) + + config = types.GenerateContentConfig( + max_output_tokens=max_tokens, + ) + if system: + config.system_instruction = system + if gemini_tools: + config.tools = gemini_tools + + stream = self.client.aio.models.generate_content_stream( + model=self.get_model_id(model), + contents=contents, + config=config, + ) + + # Track streaming state + block_index = 0 + text_block_open = False + tool_blocks: dict[str, int] = {} # tool_name -> block_index (for dedup) + + async for chunk in stream: + if not chunk.candidates: + continue + + candidate = chunk.candidates[0] + if not candidate.content or not candidate.content.parts: + continue + + for part in candidate.content.parts: + if part.text is not None: + text = part.text + if not text_block_open: + text_block_open = True + yield StreamEvent( + type="content_block_start", + index=block_index, + block_type="text", + ) + + yield StreamEvent( + type="content_block_delta", + index=block_index, + delta_type="text_delta", + text=text, + ) + + elif part.function_call is not None: + fc = part.function_call + + # Close text block if open + if text_block_open: + yield StreamEvent( + type="content_block_stop", + index=block_index, + ) + block_index += 1 + text_block_open = False + + tool_id = uuid4().hex + tool_block_idx = block_index + block_index += 1 + + args = dict(fc.args) if fc.args else {} + args_json = json.dumps(args) + + yield StreamEvent( + type="content_block_start", + index=tool_block_idx, + block_type="tool_use", + tool_name=fc.name, + tool_id=tool_id, + ) + yield StreamEvent( + type="content_block_delta", + index=tool_block_idx, + delta_type="input_json_delta", + text=args_json, + ) + yield StreamEvent( + type="content_block_stop", + index=tool_block_idx, + ) + + # Close any remaining open text block + if text_block_open: + yield StreamEvent( + type="content_block_stop", + index=block_index, + ) + + # Emit usage from the last chunk if available + if chunk and hasattr(chunk, 'usage_metadata') and chunk.usage_metadata: + um = chunk.usage_metadata + usage_data = { + "input_tokens": getattr(um, "prompt_token_count", 0) or 0, + "output_tokens": getattr(um, "candidates_token_count", 0) or 0, + } + if any(v > 0 for v in usage_data.values()): + yield StreamEvent(type="usage", usage=usage_data) + + yield StreamEvent(type="message_stop") diff --git a/backend/apps/agents/providers/openai_compat.py b/backend/apps/agents/providers/openai_compat.py new file mode 100644 index 00000000..7b95b718 --- /dev/null +++ b/backend/apps/agents/providers/openai_compat.py @@ -0,0 +1,330 @@ +"""OpenAI-compatible provider adapter. + +Works with ANY endpoint that speaks the OpenAI Chat Completions API: +OpenAI, OpenRouter, Together, Groq, Fireworks, Mistral, Ollama, vLLM, etc. +""" + +from __future__ import annotations + +import json +import logging +from typing import Any, AsyncIterator +from uuid import uuid4 + +from openai import AsyncOpenAI + +from backend.apps.agents.providers.base import ( + BaseProvider, ContentBlock, ModelResponse, ProviderMessage, + StreamEvent, ToolCall, ToolSchema, +) + +logger = logging.getLogger(__name__) + + +class OpenAICompatProvider(BaseProvider): + """Provider adapter for any OpenAI-compatible API endpoint.""" + + def __init__( + self, + api_key: str = "", + base_url: str | None = None, + ): + kwargs: dict[str, Any] = {} + if api_key: + kwargs["api_key"] = api_key + if base_url: + kwargs["base_url"] = base_url + self.client = AsyncOpenAI(**kwargs) + + def get_model_id(self, short_name: str) -> str: + # Pass through — user selects exact model ID + return short_name + + def clean_tool_schema(self, schema: ToolSchema) -> dict: + """Convert to OpenAI function calling format.""" + return { + "type": "function", + "function": { + "name": schema.name, + "description": schema.description, + "parameters": schema.input_schema, + }, + } + + def format_tool_result(self, tool_use_id: str, content: list[dict]) -> dict: + """Format tool result as OpenAI expects.""" + # OpenAI wants a single string for tool results + text_parts = [] + for block in content: + if block.get("type") == "text": + text_parts.append(block.get("text", "")) + elif block.get("type") == "image": + text_parts.append("[image]") + else: + text_parts.append(json.dumps(block)) + return { + "role": "tool", + "tool_call_id": tool_use_id, + "content": "\n".join(text_parts) if text_parts else "Done.", + } + + def format_user_message(self, content: Any) -> ProviderMessage: + """Convert user content to OpenAI format.""" + if isinstance(content, str): + return ProviderMessage(role="user", content=content) + # Multimodal content (text + images) + if isinstance(content, list): + parts = [] + for block in content: + if isinstance(block, dict): + if block.get("type") == "text": + parts.append({"type": "text", "text": block["text"]}) + elif block.get("type") == "image": + source = block.get("source", {}) + media_type = source.get("media_type", "image/png") + data = source.get("data", "") + parts.append({ + "type": "image_url", + "image_url": {"url": f"data:{media_type};base64,{data}"}, + }) + elif isinstance(block, str): + parts.append({"type": "text", "text": block}) + return ProviderMessage(role="user", content=parts) + return ProviderMessage(role="user", content=str(content)) + + def format_assistant_message(self, response: ModelResponse) -> ProviderMessage: + """Convert ModelResponse to OpenAI assistant message format.""" + text_parts = [] + tool_calls = [] + for block in response.content: + if block.type == "text": + text_parts.append(block.text) + elif block.type == "tool_use" and block.tool_call: + tool_calls.append({ + "id": block.tool_call.id, + "type": "function", + "function": { + "name": block.tool_call.name, + "arguments": json.dumps(block.tool_call.input), + }, + }) + msg: dict[str, Any] = {"role": "assistant"} + if text_parts: + msg["content"] = "\n".join(text_parts) + else: + msg["content"] = None + if tool_calls: + msg["tool_calls"] = tool_calls + return ProviderMessage(role="assistant", content=msg) + + def _build_messages( + self, + system: str | None, + messages: list[ProviderMessage], + ) -> list[dict]: + """Convert ProviderMessages to OpenAI API format.""" + result = [] + if system: + result.append({"role": "system", "content": system}) + + for msg in messages: + if msg.role == "assistant": + # Assistant messages are already in OpenAI format from format_assistant_message + if isinstance(msg.content, dict) and "role" in msg.content: + result.append(msg.content) + else: + # Raw content blocks from provider-agnostic format + text_parts = [] + tool_calls = [] + if isinstance(msg.content, list): + for block in msg.content: + if isinstance(block, dict): + if block.get("type") == "text": + text_parts.append(block["text"]) + elif block.get("type") == "tool_use": + tool_calls.append({ + "id": block.get("id", uuid4().hex), + "type": "function", + "function": { + "name": block.get("name", ""), + "arguments": json.dumps(block.get("input", {})), + }, + }) + api_msg: dict[str, Any] = { + "role": "assistant", + "content": "\n".join(text_parts) if text_parts else None, + } + if tool_calls: + api_msg["tool_calls"] = tool_calls + result.append(api_msg) + + elif msg.role == "tool_result": + # Tool results: content is a list of tool result dicts + if isinstance(msg.content, list): + for tr in msg.content: + if isinstance(tr, dict) and "tool_call_id" in tr: + result.append(tr) + elif isinstance(msg.content, dict) and "tool_call_id" in msg.content: + result.append(msg.content) + + elif msg.role == "user": + result.append({"role": "user", "content": msg.content}) + + return result + + async def create_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> ModelResponse: + kwargs: dict[str, Any] = { + "model": self.get_model_id(model), + "max_tokens": max_tokens, + "messages": self._build_messages(system, messages), + } + if tools: + kwargs["tools"] = [self.clean_tool_schema(t) for t in tools] + + resp = await self.client.chat.completions.create(**kwargs) + choice = resp.choices[0] + message = choice.message + + content: list[ContentBlock] = [] + if message.content: + content.append(ContentBlock(type="text", text=message.content)) + + if message.tool_calls: + for tc in message.tool_calls: + try: + args = json.loads(tc.function.arguments) + except json.JSONDecodeError: + args = {} + content.append(ContentBlock( + type="tool_use", + tool_call=ToolCall( + id=tc.id, + name=tc.function.name, + input=args, + ), + )) + + stop = "end_turn" + if choice.finish_reason == "tool_calls": + stop = "tool_use" + elif message.tool_calls: + stop = "tool_use" + + usage_dict = {} + if resp.usage: + usage_dict = { + "input_tokens": resp.usage.prompt_tokens, + "output_tokens": resp.usage.completion_tokens, + } + + return ModelResponse(content=content, stop_reason=stop, usage=usage_dict) + + async def stream_message( + self, + model: str, + system: str | None, + messages: list[ProviderMessage], + tools: list[ToolSchema], + max_tokens: int = 8192, + ) -> AsyncIterator[StreamEvent]: + kwargs: dict[str, Any] = { + "model": self.get_model_id(model), + "max_tokens": max_tokens, + "messages": self._build_messages(system, messages), + "stream": True, + "stream_options": {"include_usage": True}, + } + if tools: + kwargs["tools"] = [self.clean_tool_schema(t) for t in tools] + + stream = await self.client.chat.completions.create(**kwargs) + + # Track streaming state to emit normalized events + text_started = False + text_index = 0 + tool_indices: dict[int, dict] = {} # openai tool_call index -> {name, id, json_buf} + next_block_index = 0 + + async for chunk in stream: + if not chunk.choices: + # Usage-only chunk at the end + if chunk.usage: + yield StreamEvent(type="usage", usage={ + "input_tokens": chunk.usage.prompt_tokens or 0, + "output_tokens": chunk.usage.completion_tokens or 0, + }) + continue + + delta = chunk.choices[0].delta + finish_reason = chunk.choices[0].finish_reason + + # Text content + if delta.content is not None: + if not text_started: + text_started = True + text_index = next_block_index + next_block_index += 1 + yield StreamEvent( + type="content_block_start", + index=text_index, + block_type="text", + ) + yield StreamEvent( + type="content_block_delta", + index=text_index, + delta_type="text_delta", + text=delta.content, + ) + + # Tool calls + if delta.tool_calls: + for tc_delta in delta.tool_calls: + tc_idx = tc_delta.index + if tc_idx not in tool_indices: + # New tool call starting + if text_started: + yield StreamEvent(type="content_block_stop", index=text_index) + text_started = False + + block_idx = next_block_index + next_block_index += 1 + tool_indices[tc_idx] = { + "block_index": block_idx, + "id": tc_delta.id or uuid4().hex, + "name": tc_delta.function.name if tc_delta.function else "", + "json_buf": "", + } + yield StreamEvent( + type="content_block_start", + index=block_idx, + block_type="tool_use", + tool_name=tool_indices[tc_idx]["name"], + tool_id=tool_indices[tc_idx]["id"], + ) + + info = tool_indices[tc_idx] + if tc_delta.function and tc_delta.function.name: + info["name"] = tc_delta.function.name + if tc_delta.function and tc_delta.function.arguments: + info["json_buf"] += tc_delta.function.arguments + yield StreamEvent( + type="content_block_delta", + index=info["block_index"], + delta_type="input_json_delta", + text=tc_delta.function.arguments, + ) + + # Finish + if finish_reason is not None: + if text_started: + yield StreamEvent(type="content_block_stop", index=text_index) + for info in tool_indices.values(): + yield StreamEvent(type="content_block_stop", index=info["block_index"]) + yield StreamEvent(type="message_stop") diff --git a/backend/apps/agents/providers/registry.py b/backend/apps/agents/providers/registry.py new file mode 100644 index 00000000..f342cf05 --- /dev/null +++ b/backend/apps/agents/providers/registry.py @@ -0,0 +1,286 @@ +"""Provider factory and model registry. + +Two-tier system: +1. Built-in providers (Anthropic, OpenAI, Gemini) with curated model lists +2. User-configured custom providers (any OpenAI-compatible endpoint) + - Includes built-in OpenRouter integration for 300+ models +""" + +from __future__ import annotations + +import logging +from typing import Any, TYPE_CHECKING + +from backend.apps.agents.providers.base import BaseProvider + +if TYPE_CHECKING: + from backend.apps.settings.models import AppSettings + +logger = logging.getLogger(__name__) + +# --------------------------------------------------------------------------- +# Tier 1: Built-in models (curated, we know their quirks) +# --------------------------------------------------------------------------- + +BUILTIN_MODELS: dict[str, list[dict[str, Any]]] = { + # ── Native API providers (use direct SDK) ── + "Anthropic": [ + {"value": "sonnet", "label": "Claude Sonnet 4.6", "context_window": 1_000_000, "model_id": "claude-sonnet-4-6", "api": "anthropic"}, + {"value": "opus", "label": "Claude Opus 4.6", "context_window": 1_000_000, "model_id": "claude-opus-4-6", "api": "anthropic"}, + {"value": "haiku", "label": "Claude Haiku 4.5", "context_window": 200_000, "model_id": "claude-haiku-4-5", "api": "anthropic"}, + ], + "OpenAI": [ + {"value": "gpt-5.4", "label": "GPT-5.4", "context_window": 1_000_000, "api": "openai"}, + {"value": "gpt-5.4-mini", "label": "GPT-5.4 Mini", "context_window": 400_000, "api": "openai"}, + {"value": "o3", "label": "o3", "context_window": 200_000, "api": "openai"}, + {"value": "o4-mini", "label": "o4-mini", "context_window": 200_000, "api": "openai"}, + ], + "Google": [ + {"value": "gemini-2.5-pro", "label": "Gemini 2.5 Pro", "context_window": 1_048_576, "api": "gemini"}, + {"value": "gemini-2.5-flash", "label": "Gemini 2.5 Flash", "context_window": 1_048_576, "api": "gemini"}, + ], + # ── Via OpenRouter (need OpenRouter API key) ── + "xAI": [ + {"value": "x-ai/grok-4-0214", "label": "Grok 4", "context_window": 2_000_000, "api": "openrouter"}, + ], + "Meta": [ + {"value": "meta-llama/llama-4-maverick", "label": "Llama 4 Maverick", "context_window": 1_000_000, "api": "openrouter"}, + {"value": "meta-llama/llama-4-scout", "label": "Llama 4 Scout", "context_window": 10_000_000, "api": "openrouter"}, + ], + "DeepSeek": [ + {"value": "deepseek/deepseek-chat-v3-0324", "label": "DeepSeek V3", "context_window": 163_840, "api": "openrouter"}, + {"value": "deepseek/deepseek-r1", "label": "DeepSeek R1", "context_window": 163_840, "api": "openrouter"}, + ], + "Mistral": [ + {"value": "mistralai/mistral-large-2501", "label": "Mistral Large", "context_window": 256_000, "api": "openrouter"}, + {"value": "mistralai/mistral-small-3.1-24b-instruct", "label": "Mistral Small 3.1", "context_window": 128_000, "api": "openrouter"}, + ], + "Qwen": [ + {"value": "qwen/qwen3-coder", "label": "Qwen3 Coder 480B", "context_window": 262_144, "api": "openrouter"}, + {"value": "qwen/qwen3-235b-a22b", "label": "Qwen3 235B", "context_window": 131_072, "api": "openrouter"}, + ], + "Cohere": [ + {"value": "cohere/command-a-03-2025", "label": "Command A", "context_window": 256_000, "api": "openrouter"}, + ], +} + +# --------------------------------------------------------------------------- +# OpenRouter: built-in integration for 300+ models +# --------------------------------------------------------------------------- + +OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1" + + +# --------------------------------------------------------------------------- +# Provider factory +# --------------------------------------------------------------------------- + +def create_provider( + provider_name: str, + settings: AppSettings, + provider_config: dict | None = None, +) -> BaseProvider: + """Create a provider adapter. + + Routes based on the 'api' field in BUILTIN_MODELS: + - "anthropic" → native Anthropic SDK + - "openai" → native OpenAI SDK (direct API) + - "gemini" → native Google GenAI SDK + - "openrouter" → OpenAI-compat via openrouter.ai (Meta, Mistral, DeepSeek, Qwen, xAI, etc.) + Custom providers use OpenAI-compat with user's base_url. + """ + api_type = _get_api_type(provider_name) + + if api_type == "anthropic": + from backend.apps.agents.providers.anthropic import AnthropicProvider + if getattr(settings, "connection_mode", "own_key") == "managed": + return AnthropicProvider( + auth_token=getattr(settings, "openswarm_auth_token", None), + base_url=getattr(settings, "openswarm_proxy_url", None) or "https://api.openswarm.ai", + ) + return AnthropicProvider(api_key=settings.anthropic_api_key) + + if api_type == "openai": + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + return OpenAICompatProvider( + api_key=settings.openai_api_key or "", + base_url="https://api.openai.com/v1", + ) + + if api_type == "gemini": + from backend.apps.agents.providers.gemini import GeminiProvider + return GeminiProvider(api_key=settings.google_api_key or "") + + if api_type == "openrouter": + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + return OpenAICompatProvider( + api_key=getattr(settings, "openrouter_api_key", "") or "", + base_url=OPENROUTER_BASE_URL, + ) + + # Custom provider — look up in settings.custom_providers + if provider_config: + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + return OpenAICompatProvider( + api_key=provider_config.get("api_key", ""), + base_url=provider_config.get("base_url", ""), + ) + + for cp in getattr(settings, "custom_providers", []): + if cp.name == provider_name: + from backend.apps.agents.providers.openai_compat import OpenAICompatProvider + return OpenAICompatProvider( + api_key=cp.api_key, + base_url=cp.base_url, + ) + + raise ValueError(f"Unknown provider: {provider_name}") + + +def _get_api_type(provider_name: str) -> str: + """Get the API type for a provider from BUILTIN_MODELS. + + Accepts both display names ('Anthropic') and lowercase API names ('anthropic'). + """ + # Direct lookup first (display name like 'Anthropic', 'OpenAI', etc.) + models = BUILTIN_MODELS.get(provider_name, []) + if models: + return models[0].get("api", "openrouter") + + # Lowercase API name mapping + _API_NAME_MAP = { + "anthropic": "anthropic", + "openai": "openai", + "gemini": "gemini", + "google": "gemini", + "openrouter": "openrouter", + } + if provider_name.lower() in _API_NAME_MAP: + return _API_NAME_MAP[provider_name.lower()] + + # Case-insensitive lookup into BUILTIN_MODELS + lower = provider_name.lower() + for key, models in BUILTIN_MODELS.items(): + if key.lower() == lower: + return models[0].get("api", "openrouter") + + return "openrouter" + + +def _has_credentials(provider_name: str, settings: AppSettings) -> bool: + """Check if a provider has credentials configured.""" + api_type = _get_api_type(provider_name) + + if api_type == "anthropic": + if getattr(settings, "connection_mode", "own_key") == "managed": + return bool(getattr(settings, "openswarm_auth_token", None)) + return bool(settings.anthropic_api_key) + if api_type == "openai": + return bool(settings.openai_api_key) + if api_type == "gemini": + return bool(getattr(settings, "google_api_key", None)) + if api_type == "openrouter": + return bool(getattr(settings, "openrouter_api_key", None)) + return False + + +def get_available_models(settings: AppSettings) -> dict[str, list[dict]]: + """Return all models — always show everything, mark which have keys configured. + + Like Cursor: show all models upfront, prompt for key when user tries to use one. + Returns: {"provider_name": [{"value": ..., "label": ..., "context_window": ..., "configured": bool}, ...]} + """ + result: dict[str, list[dict]] = {} + + # Built-in providers — always show all + for provider_name, models in BUILTIN_MODELS.items(): + configured = _has_credentials(provider_name, settings) + result[provider_name] = [ + {**m, "configured": configured} + for m in models + ] + + # Custom providers + for cp in getattr(settings, "custom_providers", []): + if cp.models: + result[cp.name] = [ + { + "value": m.get("value", m.get("id", "")), + "label": m.get("label", m.get("value", m.get("id", ""))), + "context_window": m.get("context_window", 128_000), + "configured": True, + } + for m in cp.models + ] + + return result + + +def get_context_window(provider: str, model: str, settings: AppSettings | None = None) -> int: + """Look up context window for any model.""" + # Check built-in models first + for models in BUILTIN_MODELS.values(): + for m in models: + if m["value"] == model: + return m.get("context_window", 128_000) + + # Check custom providers + if settings: + for cp in getattr(settings, "custom_providers", []): + for m in cp.models: + if m.get("value") == model or m.get("id") == model: + return m.get("context_window", 128_000) + + return 128_000 # safe default + + +# --------------------------------------------------------------------------- +# Cost tracking +# --------------------------------------------------------------------------- + +COST_PER_1M_TOKENS: dict[tuple[str, str], tuple[float, float]] = { + # (provider, model): (input_cost_per_1M, output_cost_per_1M) + # Anthropic + ("Anthropic", "sonnet"): (3.0, 15.0), + ("Anthropic", "opus"): (5.0, 25.0), + ("Anthropic", "haiku"): (1.0, 5.0), + # OpenAI + ("OpenAI", "gpt-5.4"): (2.50, 15.0), + ("OpenAI", "gpt-5.4-mini"): (0.75, 3.0), + ("OpenAI", "o3"): (2.0, 8.0), + ("OpenAI", "o4-mini"): (1.10, 4.40), + # Google + ("Google", "gemini-2.5-flash"): (0.15, 0.60), + ("Google", "gemini-2.5-pro"): (1.25, 10.0), + # OpenRouter-backed (approximate) + ("xAI", "x-ai/grok-4-0214"): (3.0, 15.0), + ("Meta", "meta-llama/llama-4-maverick"): (0.50, 0.70), + ("Meta", "meta-llama/llama-4-scout"): (0.15, 0.40), + ("DeepSeek", "deepseek/deepseek-chat-v3-0324"): (0.30, 0.90), + ("DeepSeek", "deepseek/deepseek-r1"): (0.80, 2.40), + ("Mistral", "mistralai/mistral-large-2501"): (2.0, 6.0), + ("Mistral", "mistralai/mistral-small-3.1-24b-instruct"): (0.10, 0.30), + ("Qwen", "qwen/qwen3-coder"): (0.0, 0.0), + ("Qwen", "qwen/qwen3-235b-a22b"): (0.20, 0.70), + ("Cohere", "cohere/command-a-03-2025"): (2.50, 10.0), +} + + +def calculate_cost( + provider: str, model: str, + input_tokens: int, output_tokens: int, +) -> float: + """Calculate cost in USD from token counts.""" + # Direct lookup first + rates = COST_PER_1M_TOKENS.get((provider, model)) + if not rates: + # Case-insensitive provider lookup + lower = provider.lower() + for (p, m), r in COST_PER_1M_TOKENS.items(): + if p.lower() == lower and m == model: + rates = r + break + if not rates: + return 0.0 + input_rate, output_rate = rates + return (input_tokens * input_rate + output_tokens * output_rate) / 1_000_000 diff --git a/backend/apps/agents/tools/__init__.py b/backend/apps/agents/tools/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/agents/tools/base.py b/backend/apps/agents/tools/base.py new file mode 100644 index 00000000..6190a170 --- /dev/null +++ b/backend/apps/agents/tools/base.py @@ -0,0 +1,47 @@ +"""Base classes for builtin tool implementations.""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass +from typing import Any + + +@dataclass +class ToolContext: + """Runtime context passed to every tool execution.""" + cwd: str + session_id: str + + +class BaseTool(ABC): + """Abstract base for all builtin tools. + + Subclasses must set ``name`` and ``description`` as class attributes and + implement ``get_schema`` (JSON Schema for tool input) and ``execute``. + """ + + name: str + description: str + + @abstractmethod + def get_schema(self) -> dict: + """Return JSON Schema for this tool's input parameters.""" + ... + + @abstractmethod + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + """Execute the tool. + + Returns a list of content blocks, e.g. + ``[{"type": "text", "text": "..."}]``. + """ + ... + + def to_tool_schema(self): + """Convert to the provider-agnostic ``ToolSchema`` used everywhere.""" + from backend.apps.agents.providers.base import ToolSchema + + return ToolSchema( + name=self.name, + description=self.description, + input_schema=self.get_schema(), + ) diff --git a/backend/apps/agents/tools/filesystem.py b/backend/apps/agents/tools/filesystem.py new file mode 100644 index 00000000..5ea3c253 --- /dev/null +++ b/backend/apps/agents/tools/filesystem.py @@ -0,0 +1,476 @@ +"""Filesystem tools: Read, Write, Edit, Glob, Grep.""" + +from __future__ import annotations + +import asyncio +import base64 +import mimetypes +import os +import re +from pathlib import Path +from typing import Any + +from backend.apps.agents.tools.base import BaseTool, ToolContext + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + +_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg"} +_MAX_OUTPUT_BYTES = 50 * 1024 # ~50 KB cap for grep output + + +def _resolve(file_path: str, cwd: str) -> Path: + """Resolve *file_path* against *cwd* when it is relative.""" + p = Path(file_path) + if not p.is_absolute(): + p = Path(cwd) / p + return p.resolve() + + +def _text_block(text: str) -> list[dict]: + return [{"type": "text", "text": text}] + + +# ─────────────────────────────────────────────────────────────────────────── +# ReadTool +# ─────────────────────────────────────────────────────────────────────────── + + +class ReadTool(BaseTool): + name = "Read" + description = ( + "Read a file from the filesystem. Returns lines with line numbers " + "(cat -n style). For image files returns base64 content. Supports " + "offset and limit parameters for reading portions of large files." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Absolute or relative path to the file to read.", + }, + "offset": { + "type": "integer", + "description": "1-based line number to start reading from.", + }, + "limit": { + "type": "integer", + "description": "Maximum number of lines to return (default 2000).", + }, + }, + "required": ["file_path"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + file_path = _resolve(input_data["file_path"], context.cwd) + + if not file_path.exists(): + return _text_block(f"Error: file not found: {file_path}") + + if not file_path.is_file(): + return _text_block(f"Error: not a regular file: {file_path}") + + # Binary / image files → base64 + ext = file_path.suffix.lower() + if ext in _IMAGE_EXTENSIONS: + try: + raw = file_path.read_bytes() + b64 = base64.b64encode(raw).decode("ascii") + media = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream" + return [ + { + "type": "image", + "source": { + "type": "base64", + "media_type": media, + "data": b64, + }, + } + ] + except Exception as exc: + return _text_block(f"Error reading image {file_path}: {exc}") + + # Text files + offset = max(input_data.get("offset", 1), 1) + limit = input_data.get("limit", 2000) + if limit <= 0: + limit = 2000 + + try: + with open(file_path, "r", errors="replace") as fh: + lines: list[str] = [] + for lineno, line in enumerate(fh, start=1): + if lineno < offset: + continue + if len(lines) >= limit: + break + # cat -n style: right-justified line number + tab + content + lines.append(f"{lineno:>6}\t{line.rstrip()}") + if not lines: + return _text_block(f"(file is empty or offset beyond end of file: {file_path})") + return _text_block("\n".join(lines)) + except Exception as exc: + return _text_block(f"Error reading {file_path}: {exc}") + + +# ─────────────────────────────────────────────────────────────────────────── +# WriteTool +# ─────────────────────────────────────────────────────────────────────────── + + +class WriteTool(BaseTool): + name = "Write" + description = ( + "Write content to a file. Creates parent directories if they do not " + "exist. Overwrites the file if it already exists." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Absolute or relative path to the file to write.", + }, + "content": { + "type": "string", + "description": "The full content to write to the file.", + }, + }, + "required": ["file_path", "content"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + file_path = _resolve(input_data["file_path"], context.cwd) + content: str = input_data["content"] + + try: + file_path.parent.mkdir(parents=True, exist_ok=True) + file_path.write_text(content, encoding="utf-8") + return _text_block(f"Successfully wrote {len(content)} bytes to {file_path}") + except Exception as exc: + return _text_block(f"Error writing {file_path}: {exc}") + + +# ─────────────────────────────────────────────────────────────────────────── +# EditTool +# ─────────────────────────────────────────────────────────────────────────── + + +class EditTool(BaseTool): + name = "Edit" + description = ( + "Perform exact string replacements in a file. By default the " + "old_string must appear exactly once (not unique → error). Pass " + "replace_all=true to replace every occurrence." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "file_path": { + "type": "string", + "description": "Absolute or relative path to the file to edit.", + }, + "old_string": { + "type": "string", + "description": "The exact text to find in the file.", + }, + "new_string": { + "type": "string", + "description": "The text to replace old_string with.", + }, + "replace_all": { + "type": "boolean", + "description": "If true, replace all occurrences. Default false.", + "default": False, + }, + }, + "required": ["file_path", "old_string", "new_string"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + file_path = _resolve(input_data["file_path"], context.cwd) + old_string: str = input_data["old_string"] + new_string: str = input_data["new_string"] + replace_all: bool = input_data.get("replace_all", False) + + if not file_path.exists(): + return _text_block(f"Error: file not found: {file_path}") + if not file_path.is_file(): + return _text_block(f"Error: not a regular file: {file_path}") + + try: + content = file_path.read_text(encoding="utf-8") + except Exception as exc: + return _text_block(f"Error reading {file_path}: {exc}") + + count = content.count(old_string) + if count == 0: + return _text_block( + f"Error: old_string not found in {file_path}. " + "Make sure the string matches exactly, including whitespace and indentation." + ) + + if not replace_all and count > 1: + return _text_block( + f"Error: old_string appears {count} times in {file_path}. " + "Provide more surrounding context to make the match unique, " + "or set replace_all=true to replace every occurrence." + ) + + if replace_all: + new_content = content.replace(old_string, new_string) + else: + # Replace only the first (and only) occurrence + new_content = content.replace(old_string, new_string, 1) + + try: + file_path.write_text(new_content, encoding="utf-8") + except Exception as exc: + return _text_block(f"Error writing {file_path}: {exc}") + + replacements = count if replace_all else 1 + return _text_block( + f"Successfully edited {file_path} ({replacements} replacement{'s' if replacements != 1 else ''})." + ) + + +# ─────────────────────────────────────────────────────────────────────────── +# GlobTool +# ─────────────────────────────────────────────────────────────────────────── + + +class GlobTool(BaseTool): + name = "Glob" + description = ( + "Fast file pattern matching. Supports glob patterns like '**/*.py'. " + "Returns matching file paths sorted by modification time (newest first)." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Glob pattern to match files (e.g. '**/*.py', 'src/**/*.ts').", + }, + "path": { + "type": "string", + "description": "Directory to search in. Defaults to the working directory.", + }, + }, + "required": ["pattern"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + pattern: str = input_data["pattern"] + base = Path(input_data.get("path") or context.cwd) + + if not base.is_dir(): + return _text_block(f"Error: directory not found: {base}") + + try: + matches: list[Path] = [] + for p in base.glob(pattern): + if p.is_file(): + matches.append(p) + if len(matches) >= 500: + break + + # Sort by modification time, newest first + matches.sort(key=lambda p: p.stat().st_mtime, reverse=True) + + if not matches: + return _text_block(f"No files matched pattern '{pattern}' in {base}") + + result = "\n".join(str(p) for p in matches) + return _text_block(result) + except Exception as exc: + return _text_block(f"Error during glob '{pattern}' in {base}: {exc}") + + +# ─────────────────────────────────────────────────────────────────────────── +# GrepTool +# ─────────────────────────────────────────────────────────────────────────── + + +class GrepTool(BaseTool): + name = "Grep" + description = ( + "Search file contents using regular expressions. Uses ripgrep (rg) " + "when available, otherwise falls back to Python's re module. " + "Supports output modes: files_with_matches, content, count." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "pattern": { + "type": "string", + "description": "Regular expression pattern to search for.", + }, + "path": { + "type": "string", + "description": "File or directory to search in. Defaults to the working directory.", + }, + "glob": { + "type": "string", + "description": "Glob pattern to filter files (e.g. '*.py', '*.{ts,tsx}').", + }, + "output_mode": { + "type": "string", + "enum": ["files_with_matches", "content", "count"], + "description": "Output mode. Default: files_with_matches.", + "default": "files_with_matches", + }, + }, + "required": ["pattern"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + pattern: str = input_data["pattern"] + search_path: str = input_data.get("path") or context.cwd + file_glob: str | None = input_data.get("glob") + output_mode: str = input_data.get("output_mode", "files_with_matches") + + # Try ripgrep first + try: + result = await self._run_rg(pattern, search_path, file_glob, output_mode) + if result is not None: + return result + except FileNotFoundError: + pass # rg not installed, fall through to Python fallback + + # Python fallback + return await self._python_grep(pattern, search_path, file_glob, output_mode) + + async def _run_rg( + self, + pattern: str, + search_path: str, + file_glob: str | None, + output_mode: str, + ) -> list[dict] | None: + """Run ripgrep and return results, or None if rg is not available.""" + cmd = ["rg", "--no-heading", "--color=never"] + + if output_mode == "files_with_matches": + cmd.append("--files-with-matches") + elif output_mode == "count": + cmd.append("--count") + else: + cmd.extend(["--line-number"]) + + if file_glob: + cmd.extend(["--glob", file_glob]) + + cmd.append(pattern) + cmd.append(search_path) + + try: + proc = await asyncio.create_subprocess_exec( + *cmd, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30) + except FileNotFoundError: + raise # re-raise so caller knows rg is missing + except asyncio.TimeoutError: + return _text_block("Error: grep timed out after 30 seconds.") + except Exception as exc: + return _text_block(f"Error running ripgrep: {exc}") + + output = stdout.decode("utf-8", errors="replace") + + if proc.returncode not in (0, 1): + err = stderr.decode("utf-8", errors="replace").strip() + if err: + return _text_block(f"Grep error: {err}") + + if not output.strip(): + return _text_block(f"No matches found for pattern '{pattern}'.") + + # Truncate if too large + if len(output) > _MAX_OUTPUT_BYTES: + output = output[:_MAX_OUTPUT_BYTES] + "\n... (output truncated)" + + return _text_block(output.rstrip()) + + async def _python_grep( + self, + pattern: str, + search_path: str, + file_glob: str | None, + output_mode: str, + ) -> list[dict]: + """Pure-Python grep fallback using the re module.""" + try: + regex = re.compile(pattern) + except re.error as exc: + return _text_block(f"Invalid regex pattern: {exc}") + + base = Path(search_path) + if base.is_file(): + files = [base] + elif base.is_dir(): + glob_pat = file_glob or "**/*" + files = [p for p in base.glob(glob_pat) if p.is_file()] + else: + return _text_block(f"Error: path not found: {search_path}") + + lines_out: list[str] = [] + total_bytes = 0 + truncated = False + + for fp in sorted(files): + try: + text = fp.read_text(encoding="utf-8", errors="replace") + except Exception: + continue + + file_matches: list[tuple[int, str]] = [] + for lineno, line in enumerate(text.splitlines(), start=1): + if regex.search(line): + file_matches.append((lineno, line)) + + if not file_matches: + continue + + if output_mode == "files_with_matches": + entry = str(fp) + elif output_mode == "count": + entry = f"{fp}:{len(file_matches)}" + else: + parts = [f"{fp}:{ln}:{txt}" for ln, txt in file_matches] + entry = "\n".join(parts) + + total_bytes += len(entry) + if total_bytes > _MAX_OUTPUT_BYTES: + truncated = True + break + + lines_out.append(entry) + + if not lines_out: + return _text_block(f"No matches found for pattern '{pattern}'.") + + result = "\n".join(lines_out) + if truncated: + result += "\n... (output truncated)" + + return _text_block(result) diff --git a/backend/apps/agents/tools/registry.py b/backend/apps/agents/tools/registry.py new file mode 100644 index 00000000..91cd6b5b --- /dev/null +++ b/backend/apps/agents/tools/registry.py @@ -0,0 +1,61 @@ +"""Central tool registry. + +Importing this module automatically registers all builtin tools. +""" + +from __future__ import annotations + +from backend.apps.agents.tools.base import BaseTool +from backend.apps.agents.providers.base import ToolSchema + +_TOOLS: dict[str, BaseTool] = {} + + +def register_tool(tool: BaseTool) -> None: + """Register a tool instance by its name.""" + _TOOLS[tool.name] = tool + + +def get_tool(name: str) -> BaseTool | None: + """Look up a registered tool by name. Returns None if not found.""" + return _TOOLS.get(name) + + +def get_all_tools() -> list[BaseTool]: + """Return all registered tool instances.""" + return list(_TOOLS.values()) + + +def get_all_tool_schemas() -> list[ToolSchema]: + """Return provider-agnostic ToolSchema for every registered tool.""" + return [t.to_tool_schema() for t in _TOOLS.values()] + + +def init_tools() -> None: + """Import and register all builtin tools.""" + from backend.apps.agents.tools.filesystem import ( + ReadTool, + WriteTool, + EditTool, + GlobTool, + GrepTool, + ) + from backend.apps.agents.tools.system import BashTool, AskUserQuestionTool + from backend.apps.agents.tools.web import WebSearchTool, WebFetchTool + + for tool_cls in [ + ReadTool, + WriteTool, + EditTool, + GlobTool, + GrepTool, + BashTool, + AskUserQuestionTool, + WebSearchTool, + WebFetchTool, + ]: + register_tool(tool_cls()) + + +# Auto-register on import +init_tools() diff --git a/backend/apps/agents/tools/system.py b/backend/apps/agents/tools/system.py new file mode 100644 index 00000000..3a394328 --- /dev/null +++ b/backend/apps/agents/tools/system.py @@ -0,0 +1,125 @@ +"""System tools: Bash and AskUserQuestion.""" + +from __future__ import annotations + +import asyncio + +from backend.apps.agents.tools.base import BaseTool, ToolContext + +_MAX_OUTPUT_BYTES = 100 * 1024 # ~100 KB cap + + +class BashTool(BaseTool): + name = "Bash" + description = ( + "Execute a shell command and return its output. The command runs in " + "the session's working directory. Supports an optional timeout " + "(default 120 000 ms). Stdout and stderr are captured and returned." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "command": { + "type": "string", + "description": "The shell command to execute.", + }, + "timeout": { + "type": "integer", + "description": "Timeout in milliseconds (default 120000, max 600000).", + "default": 120000, + }, + "description": { + "type": "string", + "description": "Optional human-readable description of what this command does.", + }, + }, + "required": ["command"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + command: str = input_data["command"] + timeout_ms: int = min(input_data.get("timeout", 120000), 600000) + timeout_s: float = timeout_ms / 1000.0 + + try: + proc = await asyncio.create_subprocess_shell( + command, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + cwd=context.cwd, + ) + except Exception as exc: + return [{"type": "text", "text": f"Error starting command: {exc}"}] + + try: + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_s) + except asyncio.TimeoutError: + # Attempt to kill the process + try: + proc.kill() + stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=5) + except Exception: + stdout, stderr = b"", b"" + + partial = self._decode(stdout, stderr) + msg = ( + f"Command timed out after {timeout_ms}ms.\n" + f"Partial output:\n{partial}" + ) + return [{"type": "text", "text": self._truncate(msg)}] + except Exception as exc: + return [{"type": "text", "text": f"Error executing command: {exc}"}] + + output = self._decode(stdout, stderr) + + if proc.returncode != 0: + output = f"Exit code: {proc.returncode}\n{output}" + + if not output.strip(): + output = f"(command completed with exit code {proc.returncode})" + + return [{"type": "text", "text": self._truncate(output)}] + + @staticmethod + def _decode(stdout: bytes, stderr: bytes) -> str: + parts: list[str] = [] + if stdout: + parts.append(stdout.decode("utf-8", errors="replace")) + if stderr: + parts.append(stderr.decode("utf-8", errors="replace")) + return "\n".join(parts) + + @staticmethod + def _truncate(text: str) -> str: + if len(text) > _MAX_OUTPUT_BYTES: + return text[:_MAX_OUTPUT_BYTES] + "\n... (output truncated)" + return text + + +class AskUserQuestionTool(BaseTool): + name = "AskUserQuestion" + description = ( + "Ask the user a clarifying question. The actual blocking/HITL " + "interaction is handled by the agent loop's hitl_handler; this tool " + "simply surfaces the question text." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "question": { + "type": "string", + "description": "The question to ask the user.", + }, + }, + "required": ["question"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + question: str = input_data.get("question", "") + return [{"type": "text", "text": question}] diff --git a/backend/apps/agents/tools/web.py b/backend/apps/agents/tools/web.py new file mode 100644 index 00000000..097881b3 --- /dev/null +++ b/backend/apps/agents/tools/web.py @@ -0,0 +1,206 @@ +"""Web tools: WebSearch and WebFetch.""" + +from __future__ import annotations + +import html +import re +from typing import Any + +import httpx + +from backend.apps.agents.tools.base import BaseTool, ToolContext + +_MAX_OUTPUT_BYTES = 100 * 1024 # ~100 KB +_HTTP_TIMEOUT = 30 # seconds +_USER_AGENT = ( + "Mozilla/5.0 (compatible; SelfSwarmBot/1.0; +https://github.com/openswarm-ai/self-swarm)" +) + + +def _truncate(text: str, limit: int = _MAX_OUTPUT_BYTES) -> str: + if len(text) > limit: + return text[:limit] + "\n... (output truncated)" + return text + + +def _strip_html(raw_html: str) -> str: + """Naive but effective HTML → plain-text conversion.""" + # Remove script/style blocks + text = re.sub(r"<(script|style)[^>]*>.*?", "", raw_html, flags=re.DOTALL | re.IGNORECASE) + # Remove HTML tags + text = re.sub(r"<[^>]+>", " ", text) + # Decode HTML entities + text = html.unescape(text) + # Collapse whitespace + text = re.sub(r"[ \t]+", " ", text) + text = re.sub(r"\n{3,}", "\n\n", text) + return text.strip() + + +# ─────────────────────────────────────────────────────────────────────────── +# WebSearchTool +# ─────────────────────────────────────────────────────────────────────────── + + +class WebSearchTool(BaseTool): + name = "WebSearch" + description = ( + "Search the web using DuckDuckGo and return titles, URLs, and " + "snippets for the top results." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "query": { + "type": "string", + "description": "The search query.", + }, + "num_results": { + "type": "integer", + "description": "Maximum number of results to return (default 5).", + "default": 5, + }, + }, + "required": ["query"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + query: str = input_data["query"] + num_results: int = input_data.get("num_results", 5) + + try: + results = await self._search_ddg(query, num_results) + if not results: + return [{"type": "text", "text": f"No search results found for: {query}"}] + return [{"type": "text", "text": results}] + except Exception as exc: + return [{"type": "text", "text": f"Web search error: {exc}"}] + + @staticmethod + async def _search_ddg(query: str, num_results: int) -> str: + """Query DuckDuckGo HTML endpoint and parse results.""" + async with httpx.AsyncClient( + timeout=_HTTP_TIMEOUT, + follow_redirects=True, + headers={"User-Agent": _USER_AGENT}, + ) as client: + resp = await client.post( + "https://html.duckduckgo.com/html/", + data={"q": query}, + ) + resp.raise_for_status() + + body = resp.text + + # Parse result blocks – DuckDuckGo wraps each result in + #
...
+ result_blocks = re.findall( + r']*class="[^"]*result[^"]*"[^>]*>(.*?)\s*(?=]*class="[^"]*result|$)', + body, + flags=re.DOTALL, + ) + + entries: list[str] = [] + for block in result_blocks: + if len(entries) >= num_results: + break + + # Title + URL + link_match = re.search( + r']*class="[^"]*result__a[^"]*"[^>]*href="([^"]*)"[^>]*>(.*?)', + block, + flags=re.DOTALL, + ) + if not link_match: + continue + + raw_url = html.unescape(link_match.group(1)) + title = _strip_html(link_match.group(2)).strip() + + # Snippet + snippet_match = re.search( + r']*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)', + block, + flags=re.DOTALL, + ) + snippet = _strip_html(snippet_match.group(1)).strip() if snippet_match else "" + + # DuckDuckGo wraps URLs through a redirect; try to extract the real URL + real_url_match = re.search(r"uddg=([^&]+)", raw_url) + if real_url_match: + from urllib.parse import unquote + url = unquote(real_url_match.group(1)) + else: + url = raw_url + + entry = f"[{len(entries) + 1}] {title}\n {url}" + if snippet: + entry += f"\n {snippet}" + entries.append(entry) + + return "\n\n".join(entries) + + +# ─────────────────────────────────────────────────────────────────────────── +# WebFetchTool +# ─────────────────────────────────────────────────────────────────────────── + + +class WebFetchTool(BaseTool): + name = "WebFetch" + description = ( + "Fetch the contents of a URL and return the extracted text. " + "HTML is stripped to plain text. Output is truncated to ~100 KB." + ) + + def get_schema(self) -> dict: + return { + "type": "object", + "properties": { + "url": { + "type": "string", + "description": "The URL to fetch.", + }, + "prompt": { + "type": "string", + "description": "Optional prompt/context describing what information to look for.", + }, + }, + "required": ["url"], + "additionalProperties": False, + } + + async def execute(self, input_data: dict, context: ToolContext) -> list[dict]: + url: str = input_data["url"] + prompt: str | None = input_data.get("prompt") + + try: + async with httpx.AsyncClient( + timeout=_HTTP_TIMEOUT, + follow_redirects=True, + headers={"User-Agent": _USER_AGENT}, + ) as client: + resp = await client.get(url) + resp.raise_for_status() + except httpx.HTTPStatusError as exc: + return [{"type": "text", "text": f"HTTP error {exc.response.status_code} fetching {url}"}] + except Exception as exc: + return [{"type": "text", "text": f"Error fetching {url}: {exc}"}] + + content_type = resp.headers.get("content-type", "") + + if "html" in content_type or resp.text.strip().startswith(" list[dict]: + """Load all persisted session JSON files.""" + results = [] + if not os.path.exists(SESSIONS_DIR): + return results + for fname in os.listdir(SESSIONS_DIR): + if fname.endswith(".json"): + try: + with open(os.path.join(SESSIONS_DIR, fname)) as f: + results.append(json.load(f)) + except Exception: + pass + return results + + +@analytics.router.get("/usage-summary") +async def usage_summary(): + """Compute usage stats from persisted sessions for the Settings page.""" + from backend.apps.agents.agent_manager import agent_manager + + # Combine persisted + active sessions + sessions = _load_all_sessions() + for s in agent_manager.get_all_sessions(): + sessions.append(s.model_dump(mode="json")) + + total_sessions = len(sessions) + total_cost = sum(s.get("cost_usd", 0) for s in sessions) + total_messages = 0 + total_tool_calls = 0 + total_duration = 0.0 + model_counts: Counter = Counter() + provider_counts: Counter = Counter() + tool_counts: Counter = Counter() + status_counts: Counter = Counter() + + for s in sessions: + messages = s.get("messages", []) + user_msgs = [m for m in messages if m.get("role") in ("user", "assistant")] + tool_msgs = [m for m in messages if m.get("role") == "tool_call"] + total_messages += len(user_msgs) + total_tool_calls += len(tool_msgs) + + model_counts[s.get("model", "unknown")] += 1 + provider_counts[s.get("provider", "anthropic")] += 1 + status_counts[s.get("status", "unknown")] += 1 + + # Duration + created = s.get("created_at") + closed = s.get("closed_at") + if created and closed: + try: + from datetime import datetime + fmt = "%Y-%m-%dT%H:%M:%S" + c_str = created[:19] + cl_str = closed[:19] + dur = (datetime.fromisoformat(cl_str) - datetime.fromisoformat(c_str)).total_seconds() + if dur > 0: + total_duration += dur + except Exception: + pass + + # Count individual tools + for m in tool_msgs: + content = m.get("content", {}) + if isinstance(content, dict): + tool_name = content.get("tool", "") + if tool_name: + tool_counts[tool_name] += 1 + + avg_duration = total_duration / total_sessions if total_sessions > 0 else 0 + avg_cost = total_cost / total_sessions if total_sessions > 0 else 0 + completed = status_counts.get("completed", 0) + completion_rate = completed / total_sessions if total_sessions > 0 else 0 + + return { + "total_sessions": total_sessions, + "total_cost_usd": round(total_cost, 4), + "total_messages": total_messages, + "total_tool_calls": total_tool_calls, + "avg_duration_seconds": round(avg_duration, 1), + "avg_cost_per_session": round(avg_cost, 4), + "completion_rate": round(completion_rate, 3), + "models_used": dict(model_counts.most_common(10)), + "providers_used": dict(provider_counts.most_common(10)), + "top_tools": dict(tool_counts.most_common(15)), + "status_breakdown": dict(status_counts), + } + + +@analytics.router.get("/status") +async def analytics_status(): + return {"status": "posthog", "enabled": True} diff --git a/backend/apps/analytics/collector.py b/backend/apps/analytics/collector.py new file mode 100644 index 00000000..dddcedc5 --- /dev/null +++ b/backend/apps/analytics/collector.py @@ -0,0 +1,123 @@ +"""PostHog-only analytics collector. + +All events go directly to PostHog. No local SQLite storage. + +Usage from any module: + from backend.apps.analytics.collector import record + record("session.started", {"model": "opus"}, session_id="abc123") +""" + +import logging +import platform +from uuid import uuid4 + +from posthog import Posthog + +logger = logging.getLogger(__name__) + +POSTHOG_API_KEY = "phc_KdVLvAdjCuHeacFoDm1CM1Gb23XikewRqlX67Mj6TNB" +POSTHOG_HOST = "https://us.i.posthog.com" + +_posthog: Posthog | None = None +_installation_id: str | None = None + + +def init(): + """Initialise PostHog. Called once at app startup.""" + global _posthog + if _posthog is None: + _posthog = Posthog( + project_api_key=POSTHOG_API_KEY, + host=POSTHOG_HOST, + ) + return _posthog + + +def shutdown(): + """Flush and close. Called at app shutdown.""" + global _posthog + if _posthog: + try: + _posthog.shutdown() + except Exception: + pass + _posthog = None + + +def _get_installation_id() -> str: + """Get or create a stable anonymous installation ID.""" + global _installation_id + if _installation_id: + return _installation_id + try: + from backend.apps.settings.settings import load_settings, _save_settings + settings = load_settings() + iid = getattr(settings, "installation_id", None) + if not iid: + iid = uuid4().hex + settings.installation_id = iid + _save_settings(settings) + _installation_id = iid + except Exception: + _installation_id = uuid4().hex + return _installation_id + + +def _is_opted_in() -> bool: + """Check if user has opted in to analytics.""" + try: + from backend.apps.settings.settings import load_settings + return getattr(load_settings(), "analytics_opt_in", True) + except Exception: + return True + + +def record( + event_type: str, + properties: dict | None = None, + session_id: str | None = None, + dashboard_id: str | None = None, +): + """Record an analytics event to PostHog.""" + if not _posthog or not _is_opted_in(): + return + + props = {**(properties or {})} + if session_id: + props["session_id"] = session_id + if dashboard_id: + props["dashboard_id"] = dashboard_id + props["os"] = platform.system() + props["platform"] = platform.platform() + + try: + _posthog.capture( + event_type, + distinct_id=_get_installation_id(), + properties=props, + ) + except Exception as e: + logger.debug(f"PostHog capture failed (non-critical): {e}") + + +def identify(extra_properties: dict | None = None): + """Identify the current installation with properties.""" + if not _posthog or not _is_opted_in(): + return + + try: + _posthog.identify( + _get_installation_id(), + properties={ + "os": platform.system(), + "platform": platform.platform(), + **(extra_properties or {}), + }, + ) + except Exception as e: + logger.debug(f"PostHog identify failed (non-critical): {e}") + + +def get_collector(): + """Backward compat — returns None since we no longer have a local collector.""" + return None diff --git a/backend/apps/analytics/models.py b/backend/apps/analytics/models.py new file mode 100644 index 00000000..a8696dde --- /dev/null +++ b/backend/apps/analytics/models.py @@ -0,0 +1,37 @@ +from pydantic import BaseModel +from typing import Optional + + +class AnalyticsEvent(BaseModel): + id: Optional[int] = None + timestamp: str + event_type: str + properties: dict + session_id: Optional[str] = None + dashboard_id: Optional[str] = None + + +class UsageSummary(BaseModel): + total_sessions: int = 0 + total_cost_usd: float = 0.0 + total_messages: int = 0 + total_tool_calls: int = 0 + avg_session_duration_seconds: float = 0.0 + session_completion_rate: float = 0.0 + approval_rate: float = 0.0 + models_used: dict[str, int] = {} + modes_used: dict[str, int] = {} + top_tools: list[list] = [] + + +class TimeSeriesPoint(BaseModel): + date: str + value: float + + +class ExportPayload(BaseModel): + export_version: str = "1.0" + exported_at: str = "" + app_version: str = "unknown" + period: dict = {} + summary: dict = {} diff --git a/backend/apps/auth/__init__.py b/backend/apps/auth/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/auth/auth.py b/backend/apps/auth/auth.py new file mode 100644 index 00000000..1110fa02 --- /dev/null +++ b/backend/apps/auth/auth.py @@ -0,0 +1,168 @@ +"""Auth SubApp — handles managed-mode authentication with Open Swarm service. + +All endpoints are currently stubbed with mock responses so the full UI flow +works end-to-end without a real proxy server. +""" + +from contextlib import asynccontextmanager +from uuid import uuid4 + +from pydantic import BaseModel +from typing import Optional + +from backend.config.Apps import SubApp +from backend.apps.settings.settings import load_settings, update_settings + + +# ── Models ────────────────────────────────────────────────────────────── + +class LoginRequest(BaseModel): + email: str + password: str + + +class GoogleCallbackRequest(BaseModel): + code: str + redirect_uri: str = "" + + +class LoginResponse(BaseModel): + ok: bool + token: Optional[str] = None + email: Optional[str] = None + proxy_url: Optional[str] = None + error: Optional[str] = None + + +class ValidateResponse(BaseModel): + valid: bool + email: Optional[str] = None + + +class UsageResponse(BaseModel): + used_usd: float + quota_usd: float + reset_date: str + + +# ── SubApp setup ──────────────────────────────────────────────────────── + +@asynccontextmanager +async def _lifespan(): + yield + +auth = SubApp("auth", _lifespan) +router = auth.router + + +# ── Endpoints ─────────────────────────────────────────────────────────── + +@router.post("/login") +async def login(req: LoginRequest) -> LoginResponse: + """Authenticate with email + password. + + TODO: replace with real API call to Open Swarm auth server. + """ + if not req.email or not req.password: + return LoginResponse(ok=False, error="Email and password are required") + + # Stub: generate a mock token for any valid-looking input + mock_token = f"osw_{uuid4().hex}" + proxy_url = "https://api.openswarm.ai" + + # Persist credentials to settings + settings = load_settings() + settings.connection_mode = "managed" + settings.openswarm_auth_token = mock_token + settings.openswarm_proxy_url = proxy_url + settings.openswarm_user_email = req.email + await _save_settings(settings) + + return LoginResponse(ok=True, token=mock_token, email=req.email, proxy_url=proxy_url) + + +@router.post("/google-url") +async def google_auth_url() -> dict: + """Return the Google OAuth authorize URL. + + TODO: replace with real Google OAuth URL construction. + """ + # Stub: return a placeholder URL + return { + "url": "https://accounts.google.com/o/oauth2/v2/auth?client_id=PLACEHOLDER&response_type=code&scope=email+profile&redirect_uri=http://localhost:8324/api/auth/google-callback" + } + + +@router.post("/google-callback") +async def google_callback(req: GoogleCallbackRequest) -> LoginResponse: + """Exchange Google OAuth code for a session token. + + TODO: replace with real OAuth code exchange + Open Swarm auth server call. + """ + if not req.code: + return LoginResponse(ok=False, error="Authorization code is required") + + # Stub: generate a mock token + mock_token = f"osw_{uuid4().hex}" + proxy_url = "https://api.openswarm.ai" + mock_email = "user@gmail.com" + + settings = load_settings() + settings.connection_mode = "managed" + settings.openswarm_auth_token = mock_token + settings.openswarm_proxy_url = proxy_url + settings.openswarm_user_email = mock_email + await _save_settings(settings) + + return LoginResponse(ok=True, token=mock_token, email=mock_email, proxy_url=proxy_url) + + +@router.post("/validate") +async def validate_token() -> ValidateResponse: + """Check if the stored auth token is still valid. + + TODO: replace with real validation call to Open Swarm auth server. + """ + settings = load_settings() + if not settings.openswarm_auth_token: + return ValidateResponse(valid=False) + + # Stub: always return valid + return ValidateResponse(valid=True, email=settings.openswarm_user_email) + + +@router.post("/logout") +async def logout() -> dict: + """Clear managed-mode credentials from settings.""" + settings = load_settings() + settings.connection_mode = "own_key" + settings.openswarm_auth_token = None + settings.openswarm_proxy_url = None + settings.openswarm_user_email = None + await _save_settings(settings) + return {"ok": True} + + +@router.get("/usage") +async def get_usage() -> UsageResponse: + """Fetch usage and quota information for the current managed-mode user. + + TODO: replace with real API call to Open Swarm proxy server. + """ + settings = load_settings() + if not settings.openswarm_auth_token: + return UsageResponse(used_usd=0, quota_usd=0, reset_date="") + + # Stub: return mock usage data + return UsageResponse(used_usd=0, quota_usd=50, reset_date="2026-04-01") + + +# ── Helpers ───────────────────────────────────────────────────────────── + +async def _save_settings(settings): + """Persist settings to disk (reuses the settings module's update logic).""" + import json + from backend.apps.settings.settings import SETTINGS_FILE + + with open(SETTINGS_FILE, "w") as f: + json.dump(settings.model_dump(), f, indent=2) diff --git a/backend/apps/dashboards/dashboards.py b/backend/apps/dashboards/dashboards.py index 29d32af3..a7a05d06 100644 --- a/backend/apps/dashboards/dashboards.py +++ b/backend/apps/dashboards/dashboards.py @@ -123,8 +123,10 @@ async def list_dashboards(): @dashboards.router.post("/create") async def create_dashboard(body: DashboardCreate): + from backend.apps.analytics.collector import record as _analytics dashboard = Dashboard(name=body.name) _save(dashboard) + _analytics("dashboard.created", {}, dashboard_id=dashboard.id) return dashboard.model_dump(mode="json") @@ -151,13 +153,10 @@ async def generate_name(dashboard_id: str): fallback = prompts[0][:40] try: - import anthropic from backend.apps.settings.settings import load_settings + from backend.apps.settings.credentials import get_anthropic_client global_settings = load_settings() - if not global_settings.anthropic_api_key: - raise ValueError("API key not configured") - - client = anthropic.AsyncAnthropic(api_key=global_settings.anthropic_api_key) + client = get_anthropic_client(global_settings) if len(prompts) == 1: system = ( diff --git a/backend/apps/outputs/outputs.py b/backend/apps/outputs/outputs.py index 22b3bdf7..cb860f89 100644 --- a/backend/apps/outputs/outputs.py +++ b/backend/apps/outputs/outputs.py @@ -32,13 +32,11 @@ def _resolve_model(short_name: str) -> str: def _get_anthropic_client(): - """Create an AsyncAnthropic client using the API key from app settings.""" - import anthropic + """Create an AsyncAnthropic client using credentials from app settings.""" + from backend.apps.settings.credentials import get_anthropic_client settings = load_settings() - if not settings.anthropic_api_key: - raise ValueError("Anthropic API key not configured. Set it in Settings.") - return anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key) + return get_anthropic_client(settings) def _validate_against_schema(data: dict, schema: dict) -> str | None: diff --git a/backend/apps/settings/credentials.py b/backend/apps/settings/credentials.py new file mode 100644 index 00000000..985ac117 --- /dev/null +++ b/backend/apps/settings/credentials.py @@ -0,0 +1,111 @@ +"""Centralized credential resolution for LLM API calls. + +Supports multiple providers: Anthropic (native), OpenAI, Gemini, +OpenRouter, and user-configured custom providers. +""" + +from __future__ import annotations + +from typing import TYPE_CHECKING + +if TYPE_CHECKING: + import anthropic + from backend.apps.settings.models import AppSettings + +OPENSWARM_DEFAULT_PROXY_URL = "https://api.openswarm.ai" + + +def validate_credentials(settings: AppSettings, provider: str = "anthropic") -> None: + """Raise ValueError if credentials are missing for the given provider.""" + if provider == "anthropic": + if getattr(settings, "connection_mode", "own_key") == "managed": + if not getattr(settings, "openswarm_auth_token", None): + raise ValueError( + "Open Swarm account not connected. Sign in via Settings → API." + ) + else: + if not settings.anthropic_api_key: + raise ValueError( + "Anthropic API key not configured. Set it in Settings." + ) + elif provider == "openai": + if not settings.openai_api_key: + raise ValueError("OpenAI API key not configured. Set it in Settings.") + elif provider == "gemini": + if not getattr(settings, "google_api_key", None): + raise ValueError("Google API key not configured. Set it in Settings.") + elif provider == "openrouter": + if not getattr(settings, "openrouter_api_key", None): + raise ValueError("OpenRouter API key not configured. Set it in Settings.") + else: + # Custom provider — check if it exists in custom_providers + for cp in getattr(settings, "custom_providers", []): + if cp.name == provider: + return # Custom providers may have empty api_key (e.g. local Ollama) + raise ValueError(f"Provider '{provider}' not found in settings.") + + +def get_provider_credentials(settings: AppSettings, provider: str) -> dict[str, str]: + """Return credential dict for a specific provider.""" + validate_credentials(settings, provider) + + if provider == "anthropic": + if getattr(settings, "connection_mode", "own_key") == "managed": + return { + "auth_token": getattr(settings, "openswarm_auth_token", "") or "", + "base_url": getattr(settings, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL, + } + return {"api_key": settings.anthropic_api_key or ""} + + if provider == "openai": + return {"api_key": settings.openai_api_key or ""} + + if provider == "gemini": + return {"api_key": getattr(settings, "google_api_key", "") or ""} + + if provider == "openrouter": + return {"api_key": getattr(settings, "openrouter_api_key", "") or ""} + + # Custom provider + for cp in getattr(settings, "custom_providers", []): + if cp.name == provider: + return {"api_key": cp.api_key, "base_url": cp.base_url} + + raise ValueError(f"No credentials for provider: {provider}") + + +# --------------------------------------------------------------------------- +# Legacy helpers (kept for backward compat during migration) +# --------------------------------------------------------------------------- + +def get_agent_sdk_env(settings: AppSettings) -> dict[str, str]: + """Return the env dict for ClaudeAgentOptions based on connection mode. + + DEPRECATED: Use create_provider() from providers.registry instead. + """ + validate_credentials(settings, "anthropic") + + if getattr(settings, "connection_mode", "own_key") == "managed": + proxy_url = getattr(settings, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL + return { + "ANTHROPIC_AUTH_TOKEN": getattr(settings, "openswarm_auth_token", ""), + "ANTHROPIC_BASE_URL": proxy_url, + } + + return {"ANTHROPIC_API_KEY": settings.anthropic_api_key} + + +def get_anthropic_client(settings: AppSettings) -> anthropic.AsyncAnthropic: + """Return a configured AsyncAnthropic client based on connection mode.""" + import anthropic + + validate_credentials(settings, "anthropic") + + if getattr(settings, "connection_mode", "own_key") == "managed": + proxy_url = getattr(settings, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL + return anthropic.AsyncAnthropic( + auth_token=getattr(settings, "openswarm_auth_token", None), + base_url=proxy_url, + ) + + return anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key) diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index 34a589df..b6760d8c 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -1,5 +1,5 @@ -from pydantic import BaseModel -from typing import Optional +from pydantic import BaseModel, Field +from typing import Optional, Any DEFAULT_SYSTEM_PROMPT = ( '"Ask the user as many follow ups as needed in order to eliminate any possible ambiguity. ' @@ -28,9 +28,22 @@ class AppSettings(BaseModel): elevenlabs_api_key: Optional[str] = None deepgram_api_key: Optional[str] = None openai_api_key: Optional[str] = None + google_api_key: Optional[str] = None + openrouter_api_key: Optional[str] = None + custom_providers: list["CustomProvider"] = Field(default_factory=list) webhook_base_url: Optional[str] = None # Dashboard / UI preferences auto_select_mode_on_new_agent: bool = False expand_new_chats_in_dashboard: bool = False auto_reveal_sub_agents: bool = True dev_mode: bool = False + # Analytics: opted in by default, user can toggle off + analytics_opt_in: bool = True + installation_id: Optional[str] = None + + +class CustomProvider(BaseModel): + name: str + base_url: str + api_key: str = "" + models: list[dict[str, Any]] = Field(default_factory=list) diff --git a/backend/apps/settings/settings.py b/backend/apps/settings/settings.py index e590d9a4..d0c8ca41 100644 --- a/backend/apps/settings/settings.py +++ b/backend/apps/settings/settings.py @@ -38,6 +38,13 @@ def load_settings() -> AppSettings: return AppSettings() +def _save_settings(settings: AppSettings): + """Persist settings to JSON file.""" + os.makedirs(DATA_DIR, exist_ok=True) + with open(SETTINGS_FILE, "w") as f: + json.dump(settings.model_dump(), f, indent=2) + + @settings.router.get("") async def get_settings(): return load_settings().model_dump() diff --git a/backend/config/paths.py b/backend/config/paths.py index 1da2d323..62259a68 100644 --- a/backend/config/paths.py +++ b/backend/config/paths.py @@ -37,5 +37,6 @@ DASHBOARD_LAYOUT_DIR = os.path.join(DATA_ROOT, "dashboard_layout") BUILTIN_PERMISSIONS_PATH = os.path.join(DATA_ROOT, "builtin_permissions.json") CHANNELS_DIR = os.path.join(DATA_ROOT, "channels") CHANNELS_SESSIONS_DIR = os.path.join(DATA_ROOT, "channels", "sessions") +ANALYTICS_DIR = os.path.join(DATA_ROOT, "analytics") BACKEND_DIR = _BACKEND_DIR diff --git a/backend/main.py b/backend/main.py index 1320084c..9f539480 100644 --- a/backend/main.py +++ b/backend/main.py @@ -20,11 +20,13 @@ from backend.apps.skill_registry.skill_registry import skill_registry from backend.apps.outputs.outputs import outputs from backend.apps.dashboards.dashboards import dashboards from backend.apps.channels.channels import channels +from backend.apps.analytics.analytics import analytics +from backend.apps.auth.auth import auth from fastapi.middleware.cors import CORSMiddleware from fastapi import WebSocket, WebSocketDisconnect import json -main_app = MainApp([health, agents, templates, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, dashboards, channels]) +main_app = MainApp([health, agents, templates, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, dashboards, channels, analytics, auth]) app = main_app.app app.add_middleware( @@ -52,6 +54,7 @@ async def websocket_session(websocket: WebSocket, session_id: str): payload.get("prompt", ""), mode=payload.get("mode"), model=payload.get("model"), + provider=payload.get("provider"), images=payload.get("images"), ) elif event == "agent:approval_response": diff --git a/backend/requirements.txt b/backend/requirements.txt index 6351bcb0..99ef9c1e 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -1,5 +1,8 @@ anthropic -claude-agent-sdk +openai +google-genai +mcp +posthog jsonschema fastapi[standard] pydantic==2.10.5 diff --git a/frontend/src/app/Main.tsx b/frontend/src/app/Main.tsx index 50e051d8..1fbe8af1 100644 --- a/frontend/src/app/Main.tsx +++ b/frontend/src/app/Main.tsx @@ -5,6 +5,7 @@ import { ThemeProvider as MuiThemeProvider, createTheme, CssBaseline } from '@mu import { store } from '../shared/state/store'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { fetchSettings } from '@/shared/state/settingsSlice'; +import { fetchModels } from '@/shared/state/modelsSlice'; import { setAppVersion, setUpdateAvailable, @@ -23,6 +24,8 @@ import Modes from './pages/Modes/Modes'; import Views from './pages/Views/Views'; import Customization from './pages/Customization/Customization'; import Channels from './pages/Channels/Channels'; +import Analytics from './pages/Analytics/Analytics'; +import AnalyticsOptIn from './components/AnalyticsOptIn'; import { useKeyboardShortcuts } from '@/shared/hooks/useKeyboardShortcuts'; import KeyboardShortcutsHelp from './components/KeyboardShortcutsHelp'; import { ThemeProvider, useThemeMode, useClaudeTokens } from '@/shared/styles/ThemeContext'; @@ -161,6 +164,7 @@ const SettingsLoader: React.FC<{ children: React.ReactNode }> = ({ children }) = const loaded = useAppSelector((s) => s.settings.loaded); useEffect(() => { dispatch(fetchSettings()); + dispatch(fetchModels()); }, [dispatch]); useEffect(() => { if (loaded) setThemeMode(theme as 'light' | 'dark'); @@ -230,8 +234,10 @@ const ThemedApp: React.FC = () => { } /> } /> } /> + } /> + diff --git a/frontend/src/app/components/AnalyticsOptIn.tsx b/frontend/src/app/components/AnalyticsOptIn.tsx new file mode 100644 index 00000000..b96701dd --- /dev/null +++ b/frontend/src/app/components/AnalyticsOptIn.tsx @@ -0,0 +1,84 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import Button from '@mui/material/Button'; +import Paper from '@mui/material/Paper'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; +import { updateSettings } from '@/shared/state/settingsSlice'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; + +const AnalyticsOptIn: React.FC = () => { + const c = useClaudeTokens(); + const dispatch = useAppDispatch(); + const settings = useAppSelector((s) => s.settings.data); + const loaded = useAppSelector((s) => s.settings.loaded); + + if (!loaded || settings.analytics_opt_in !== null) return null; + + const handleChoice = (optIn: boolean) => { + dispatch(updateSettings({ ...settings, analytics_opt_in: optIn })); + }; + + return ( + + + + Help improve OpenSwarm + + + Share anonymous usage statistics like session counts, feature usage, and model preferences. + No conversations, file paths, or personal information — ever. + You can change this anytime in Settings. + + + + + + + + ); +}; + +export default AnalyticsOptIn; diff --git a/frontend/src/app/components/Layout/AppShell.tsx b/frontend/src/app/components/Layout/AppShell.tsx index cd3e51a0..95c78d60 100644 --- a/frontend/src/app/components/Layout/AppShell.tsx +++ b/frontend/src/app/components/Layout/AppShell.tsx @@ -24,6 +24,7 @@ import AddIcon from '@mui/icons-material/Add'; import SettingsIcon from '@mui/icons-material/Settings'; import ExtensionIcon from '@mui/icons-material/Extension'; import PhoneIcon from '@mui/icons-material/Phone'; +import BarChartIcon from '@mui/icons-material/BarChart'; import ViewSidebarOutlinedIcon from '@mui/icons-material/ViewSidebarOutlined'; import ArrowBackOutlinedIcon from '@mui/icons-material/ArrowBackOutlined'; import ArrowForwardOutlinedIcon from '@mui/icons-material/ArrowForwardOutlined'; @@ -33,6 +34,7 @@ import CloseIcon from '@mui/icons-material/Close'; import LinearProgress from '@mui/material/LinearProgress'; import Settings from '@/app/pages/Settings/Settings'; import GlobalApprovalOverlay from '@/app/components/GlobalApprovalOverlay'; +import TalkModeOverlay from '@/app/components/TalkModeOverlay'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { fetchDashboards, createDashboard, renameDashboard } from '@/shared/state/dashboardsSlice'; import { addBrowserCard, addBrowserTab } from '@/shared/state/dashboardLayoutSlice'; @@ -69,6 +71,7 @@ const AppShell: React.FC = () => { const [sidebarCollapsed, setSidebarCollapsed] = useState(false); const [renamingDashboardId, setRenamingDashboardId] = useState(null); const [renameValue, setRenameValue] = useState(''); + const [talkModeOpen, setTalkModeOpen] = useState(false); const [sidebarWidth, setSidebarWidth] = useState(() => { try { const stored = localStorage.getItem(SIDEBAR_WIDTH_KEY); @@ -231,6 +234,7 @@ const AppShell: React.FC = () => { const isAppsRoute = location.pathname === '/apps' || location.pathname.startsWith('/apps/'); const isCustomizationRoute = location.pathname === '/customization' || CUSTOMIZATION_PATHS.has(location.pathname); const isChannelsRoute = location.pathname === '/channels'; + const isAnalyticsRoute = location.pathname === '/analytics'; const activeDashboardId = location.pathname.startsWith('/dashboard/') ? location.pathname.split('/dashboard/')[1] : null; @@ -959,6 +963,10 @@ const AppShell: React.FC = () => { + setTalkModeOpen(false)} + /> void; + sessionId?: string; +} + +// ─── Pixel Face Canvas Renderer ────────────────────────────────── +// Ported from face.html — all the draw logic in one hook. + +function usePixelFace( + canvasRef: React.RefObject, + faceState: FaceState, + size: number, +) { + const stateRef = useRef('idle'); + const animRef = useRef(0); + + useEffect(() => { + stateRef.current = faceState; + }, [faceState]); + + useEffect(() => { + const canvas = canvasRef.current; + if (!canvas) return; + + const ctx = canvas.getContext('2d'); + if (!ctx) return; + + const PX = Math.max(4, Math.floor(size / 30)); + const COLS = Math.ceil(size / PX); + const ROWS = Math.ceil(size / PX); + canvas.width = COLS * PX; + canvas.height = ROWS * PX; + + const BG = '#E8927A'; + const EYE = '#1E1E1E'; + const MOUTH = '#1E1E1E'; + + let breath = 0, talk = 0, think = 0, sleepZ = 0, heartP = 0; + let eyeH = 3, eyeHTarget = 3; + let mouthW = 2, mouthWTarget = 2; + let mouthH = 2, mouthHTarget = 2; + let eyeOffX = 0, eyeOffXTarget = 0; + let eyeOffY = 0, eyeOffYTarget = 0; + let blinkOpen = true, blinkCD = 120 + Math.random() * 200; + let doubleBlink = false; + let idleSinceInput = 0, sleepTransitioned = false; + let idleAction = 'none', idleActionTimer = 0; + let idleGlanceX = 0, idleGlanceY = 0; + + const lerp = (a: number, b: number, t: number) => a + (b - a) * t; + + const px = (col: number, row: number, color: string) => { + ctx.fillStyle = color; + ctx.fillRect(col * PX, row * PX, PX, PX); + }; + + const pxRect = (x: number, y: number, w: number, h: number, color: string) => { + for (let r = 0; r < Math.round(h); r++) + for (let c = 0; c < Math.round(w); c++) + px(Math.round(x) + c, Math.round(y) + r, color); + }; + + function draw() { + const state = stateRef.current; + + breath += 0.025; + talk += 0.3; + think += 0.025; + sleepZ += 0.012; + heartP += 0.05; + idleSinceInput++; + + if (idleSinceInput > 2700 && state === 'idle' && !sleepTransitioned) { + stateRef.current = 'sleeping'; + sleepTransitioned = true; + } + + if (state === 'idle') { + idleActionTimer--; + blinkCD--; + if (blinkCD <= 0 && blinkOpen) { blinkOpen = false; blinkCD = 6; doubleBlink = Math.random() < 0.3; } + else if (!blinkOpen && blinkCD <= 0) { blinkOpen = true; blinkCD = doubleBlink ? 8 : 100 + Math.random() * 280; doubleBlink = false; } + + if (idleActionTimer <= 0) { + const roll = Math.random(); + if (roll < 0.3) { idleAction = 'glance'; idleGlanceX = Math.floor(Math.random() * 5) - 2; idleGlanceY = (Math.random() - 0.5) * 1.2; idleActionTimer = 50 + Math.random() * 120; } + else if (roll < 0.45) { idleAction = 'scan'; idleActionTimer = 180; } + else if (roll < 0.55) { idleAction = 'squint'; idleActionTimer = 35 + Math.random() * 40; } + else if (roll < 0.65) { idleAction = 'lookup'; idleActionTimer = 50 + Math.random() * 70; } + else { idleAction = 'none'; idleGlanceX = 0; idleGlanceY = 0; idleActionTimer = 60 + Math.random() * 200; } + } + } + + if (state === 'talking' || state === 'thinking') { + blinkCD--; + if (blinkCD <= 0 && blinkOpen) { blinkOpen = false; blinkCD = 6; } + else if (!blinkOpen && blinkCD <= 0) { blinkOpen = true; blinkCD = 120 + Math.random() * 250; } + } + if (state !== 'idle' && state !== 'talking' && state !== 'thinking') blinkOpen = true; + + const b = Math.sin(breath) * 0.3; + switch (state) { + case 'idle': { + eyeHTarget = blinkOpen ? 3 : 0; mouthWTarget = 2; mouthHTarget = 2; + let gx = 0, gy = b; + if (idleAction === 'glance') { gx = idleGlanceX; gy = idleGlanceY + b; } + else if (idleAction === 'scan') { const t = 1 - (idleActionTimer / 180); gx = Math.sin(t * Math.PI * 2) * 2.5; } + else if (idleAction === 'squint') { eyeHTarget = blinkOpen ? 2 : 0; gy = b + 0.3; } + else if (idleAction === 'lookup') { gy = -1.2 + b; } + eyeOffXTarget = gx; eyeOffYTarget = gy; break; + } + case 'happy': eyeHTarget = 1; mouthWTarget = 6; mouthHTarget = 1; eyeOffXTarget = 0; eyeOffYTarget = b; break; + case 'thinking': eyeHTarget = blinkOpen ? 3 : 0; mouthWTarget = 2; mouthHTarget = 2; eyeOffXTarget = 2; eyeOffYTarget = b; break; + case 'talking': { eyeHTarget = blinkOpen ? 3 : 0; const open = Math.round(Math.abs(Math.sin(talk)) * 2 + 1); mouthWTarget = 4; mouthHTarget = open; eyeOffXTarget = 0; eyeOffYTarget = b; break; } + case 'surprised': eyeHTarget = 4; mouthWTarget = 3; mouthHTarget = 3; eyeOffXTarget = 0; eyeOffYTarget = b; break; + case 'sleeping': eyeHTarget = 1; mouthWTarget = 2; mouthHTarget = 1; eyeOffXTarget = 0; eyeOffYTarget = b * 2; break; + case 'angry': eyeHTarget = 2; mouthWTarget = 6; mouthHTarget = 1; eyeOffXTarget = 0; eyeOffYTarget = b * 0.3; break; + case 'love': eyeHTarget = 3; mouthWTarget = 2; mouthHTarget = 2; eyeOffXTarget = 0; eyeOffYTarget = b; break; + } + + eyeH = lerp(eyeH, eyeHTarget, 0.18); + mouthW = lerp(mouthW, mouthWTarget, 0.15); + mouthH = lerp(mouthH, mouthHTarget, 0.2); + eyeOffX = lerp(eyeOffX, eyeOffXTarget, 0.1); + eyeOffY = lerp(eyeOffY, eyeOffYTarget, 0.15); + + // Draw + ctx.fillStyle = BG; + ctx.fillRect(0, 0, canvas.width, canvas.height); + + const cx = Math.floor(COLS / 2); + const cy = Math.floor(ROWS / 2); + const eyeSpread = 5, eyeW = 3; + const eh = Math.max(1, Math.round(eyeH)); + const eOffX = Math.round(eyeOffX), eOffY = Math.round(eyeOffY); + const eyeBaseY = cy - 2; + const blinkOff = Math.round((3 - eh) / 2); + + if (state === 'love') { + const pulse = Math.sin(heartP) > 0 ? '#CC2244' : '#BB1E3E'; + const heart = (hx: number, hy: number) => { + px(hx - 1, hy, pulse); px(hx + 1, hy, pulse); + px(hx - 2, hy + 1, pulse); px(hx - 1, hy + 1, pulse); px(hx, hy + 1, pulse); px(hx + 1, hy + 1, pulse); px(hx + 2, hy + 1, pulse); + px(hx - 1, hy + 2, pulse); px(hx, hy + 2, pulse); px(hx + 1, hy + 2, pulse); + px(hx, hy + 3, pulse); + }; + heart(cx - eyeSpread + eOffX, eyeBaseY + eOffY); + heart(cx + eyeSpread + eOffX, eyeBaseY + eOffY); + } else { + pxRect(cx - eyeSpread - 1 + eOffX, eyeBaseY + blinkOff + eOffY, eyeW, eh, EYE); + pxRect(cx + eyeSpread - 1 + eOffX, eyeBaseY + blinkOff + eOffY, eyeW, eh, EYE); + } + + if (state === 'angry') { + const lx = cx - eyeSpread - 1 + eOffX, ly = eyeBaseY + blinkOff + eOffY - 2; + px(lx, ly + 1, EYE); px(lx + 1, ly, EYE); px(lx + 2, ly, EYE); + const rx = cx + eyeSpread - 1 + eOffX; + px(rx + 2, ly + 1, EYE); px(rx + 1, ly, EYE); px(rx, ly, EYE); + } + + const mw = Math.max(1, Math.round(mouthW)), mh = Math.max(1, Math.round(mouthH)); + const mouthY = cy + 4 + Math.round(eyeOffY); + + if (state === 'happy') { + pxRect(cx - Math.floor(mw / 2), mouthY, mw, 1, MOUTH); + px(cx - Math.floor(mw / 2), mouthY - 1, MOUTH); + px(cx - Math.floor(mw / 2) + mw - 1, mouthY - 1, MOUTH); + } else if (state === 'angry') { + pxRect(cx - Math.floor(mw / 2), mouthY, mw, 1, MOUTH); + px(cx - Math.floor(mw / 2), mouthY + 1, MOUTH); + px(cx - Math.floor(mw / 2) + mw - 1, mouthY + 1, MOUTH); + } else { + pxRect(cx - Math.floor(mw / 2), mouthY, mw, mh, MOUTH); + } + + if (state === 'sleeping') { + const zFrame = Math.floor(sleepZ * 60) % 90; + const zy = Math.round(cy - 5 - (zFrame / 90) * 4); + const zx = cx + eyeSpread + 3; + if (zy >= 1 && zFrame < 70) { + px(zx, zy, '#5577CC'); px(zx + 1, zy, '#5577CC'); + px(zx + 1, zy + 1, '#5577CC'); + px(zx, zy + 2, '#5577CC'); px(zx + 1, zy + 2, '#5577CC'); + } + } + + if (state === 'thinking') { + const phase = Math.floor(think * 10) % 4; + const dx = cx + eyeSpread + 3, dy = cy - 5; + if (phase >= 1) px(dx, dy, '#7799DD'); + if (phase >= 2) px(dx + 1, dy - 1, '#7799DD'); + if (phase >= 3) px(dx + 2, dy - 2, '#7799DD'); + } + + animRef.current = requestAnimationFrame(draw); + } + + animRef.current = requestAnimationFrame(draw); + return () => cancelAnimationFrame(animRef.current); + }, [canvasRef, size]); +} + +// ─── Status Labels ─────────────────────────────────────────────── + +const STATUS_LABELS: Record = { + idle: 'Tap to speak', + listening: 'Listening...', + processing: 'Thinking...', + speaking: 'Speaking...', +}; + +// ─── Main Component ───────────────────────────────────────────── + +const TalkModeOverlay: React.FC = ({ open, onClose, sessionId }) => { + const c = useClaudeTokens(); + const canvasRef = useRef(null); + const [faceState, setFaceState] = useState('idle'); + const [talkStatus, setTalkStatus] = useState('idle'); + const [transcript, setTranscript] = useState(''); + const [agentResponse, setAgentResponse] = useState(''); + const wsRef = useRef(null); + const mediaRecorderRef = useRef(null); + const silenceTimerRef = useRef(0); + const audioContextRef = useRef(null); + + usePixelFace(canvasRef, faceState, 240); + + // Map talk status to face state + useEffect(() => { + switch (talkStatus) { + case 'idle': setFaceState('idle'); break; + case 'listening': setFaceState('idle'); break; + case 'processing': setFaceState('thinking'); break; + case 'speaking': setFaceState('talking'); break; + } + }, [talkStatus]); + + // WebSocket connection for talk mode + useEffect(() => { + if (!open || !sessionId) return; + + const ws = new WebSocket(`${WS_BASE}/ws/talk/${sessionId}`); + wsRef.current = ws; + + ws.onopen = () => { + ws.send(JSON.stringify({ type: 'config', stt: {}, tts: {} })); + }; + + ws.onmessage = (event) => { + const msg = JSON.parse(event.data); + + switch (msg.type) { + case 'status': + if (msg.status === 'listening') setTalkStatus('idle'); + else if (msg.status === 'processing') setTalkStatus('processing'); + else if (msg.status === 'speaking') setTalkStatus('speaking'); + break; + + case 'transcript': + setTranscript(msg.text); + break; + + case 'agent_response': + setAgentResponse(msg.text); + setFaceState('happy'); + setTimeout(() => setFaceState('idle'), 2000); + break; + + case 'audio': { + const audioData = atob(msg.data); + const audioArray = new Uint8Array(audioData.length); + for (let i = 0; i < audioData.length; i++) audioArray[i] = audioData.charCodeAt(i); + + if (!audioContextRef.current) audioContextRef.current = new AudioContext(); + const audioCtx = audioContextRef.current; + audioCtx.decodeAudioData(audioArray.buffer.slice(0), (buffer) => { + const source = audioCtx.createBufferSource(); + source.buffer = buffer; + source.connect(audioCtx.destination); + source.onended = () => setTalkStatus('idle'); + source.start(0); + setTalkStatus('speaking'); + }); + break; + } + } + }; + + ws.onerror = () => setFaceState('angry'); + ws.onclose = () => {}; + + return () => { + ws.close(); + wsRef.current = null; + }; + }, [open, sessionId]); + + // Keyboard: Esc to close + useEffect(() => { + if (!open) return; + const handler = (e: KeyboardEvent) => { + if (e.key === 'Escape') onClose(); + }; + window.addEventListener('keydown', handler); + return () => window.removeEventListener('keydown', handler); + }, [open, onClose]); + + const startRecording = useCallback(async () => { + try { + const stream = await navigator.mediaDevices.getUserMedia({ audio: true }); + const recorder = new MediaRecorder(stream, { mimeType: 'audio/webm' }); + mediaRecorderRef.current = recorder; + + const chunks: Blob[] = []; + recorder.ondataavailable = (e) => { + if (e.data.size > 0) chunks.push(e.data); + }; + + recorder.onstop = async () => { + stream.getTracks().forEach((t) => t.stop()); + const blob = new Blob(chunks, { type: 'audio/webm' }); + const reader = new FileReader(); + reader.onload = () => { + const base64 = (reader.result as string).split(',')[1]; + if (wsRef.current?.readyState === WebSocket.OPEN) { + wsRef.current.send(JSON.stringify({ type: 'audio', data: base64, format: 'webm' })); + wsRef.current.send(JSON.stringify({ type: 'end_utterance', format: 'webm' })); + } + }; + reader.readAsDataURL(blob); + setTalkStatus('processing'); + }; + + recorder.start(); + setTalkStatus('listening'); + setTranscript(''); + setAgentResponse(''); + + // Auto-stop after silence (simple timeout approach) + silenceTimerRef.current = window.setTimeout(() => { + if (mediaRecorderRef.current?.state === 'recording') { + mediaRecorderRef.current.stop(); + } + }, 5000); + } catch { + setFaceState('angry'); + } + }, []); + + const stopRecording = useCallback(() => { + window.clearTimeout(silenceTimerRef.current); + if (mediaRecorderRef.current?.state === 'recording') { + mediaRecorderRef.current.stop(); + } + }, []); + + const handleMicClick = useCallback(() => { + if (talkStatus === 'listening') { + stopRecording(); + } else if (talkStatus === 'idle') { + startRecording(); + } + }, [talkStatus, startRecording, stopRecording]); + + if (!open) return null; + + const micBg = + talkStatus === 'listening' ? c.accent.primary : + talkStatus === 'speaking' ? '#4caf50' : + c.bg.elevated; + + return ( + { if (e.target === e.currentTarget) onClose(); }} + > + {/* Close button */} + + + + + {/* Face canvas */} + + + + + {/* Status label */} + + {STATUS_LABELS[talkStatus]} + + + {/* Transcript area */} + + {transcript && ( + + "{transcript}" + + )} + + {agentResponse && ( + + {agentResponse} + + )} + + + {/* Mic button */} + + {talkStatus === 'listening' ? : + talkStatus === 'speaking' ? : + } + + + {/* Hint */} + + esc to close + + + ); +}; + +export default TalkModeOverlay; diff --git a/frontend/src/app/pages/AgentChat/AgentChat.tsx b/frontend/src/app/pages/AgentChat/AgentChat.tsx index 6d1764cd..ebf885fd 100644 --- a/frontend/src/app/pages/AgentChat/AgentChat.tsx +++ b/frontend/src/app/pages/AgentChat/AgentChat.tsx @@ -27,6 +27,7 @@ import { switchBranch, duplicateSession, setActiveSession, + updateSessionProvider, updateSessionModel, updateSessionMode, fetchSession, @@ -45,9 +46,9 @@ import DiffViewer from './DiffViewer'; import { setGlowingBrowserCards, clearGlowingBrowserCards } from '@/shared/state/dashboardLayoutSlice'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; -const CONTEXT_WINDOWS: Record = { - sonnet: 200_000, - opus: 200_000, +const CONTEXT_WINDOWS_DEFAULT: Record = { + sonnet: 1_000_000, + opus: 1_000_000, haiku: 200_000, }; @@ -135,6 +136,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const dispatch = useAppDispatch(); const session = useAppSelector((state) => (id ? state.agents.sessions[id] : undefined)); const modesMap = useAppSelector((state) => state.modes.items); + const modelsByProvider = useAppSelector((state) => state.models.byProvider); const scrollContainerRef = useRef(null); const chatInputRef = useRef(null); const isAtBottomRef = useRef(true); @@ -143,6 +145,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose const [awaitingResponse, setAwaitingResponse] = useState(false); const [mode, setMode] = useState('agent'); const [model, setModel] = useState('sonnet'); + const [provider, setProvider] = useState('anthropic'); const wsRef = useRef | null>(null); const initialContextApplied = useRef(false); @@ -185,6 +188,10 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose if (session) setModel(session.model); }, [session?.model]); + useEffect(() => { + if (session?.provider) setProvider(session.provider); + }, [session?.provider]); + useEffect(() => { if (Object.keys(modesMap).length === 0) dispatch(fetchModes()); }, [dispatch, modesMap]); @@ -194,11 +201,11 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose setShowResumeBubble(false); setAwaitingResponse(true); if (isDraft) { - const config: Record = { model, mode }; + const config: Record = { provider, model, mode }; if (session?.system_prompt) config.system_prompt = session.system_prompt; if (session?.target_directory) config.target_directory = session.target_directory; dispatch( - launchAndSendFirstMessage({ draftId: id, config, prompt: msg.prompt, mode, model, images: msg.images, contextPaths: msg.contextPaths, forcedTools: msg.forcedTools, attachedSkills: msg.attachedSkills, selectedBrowserIds: msg.selectedBrowserIds }) + launchAndSendFirstMessage({ draftId: id, config, prompt: msg.prompt, mode, model, provider, images: msg.images, contextPaths: msg.contextPaths, forcedTools: msg.forcedTools, attachedSkills: msg.attachedSkills, selectedBrowserIds: msg.selectedBrowserIds }) ).then((action) => { if (launchAndSendFirstMessage.fulfilled.match(action)) { const realId = action.payload.session.id; @@ -212,14 +219,14 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose if (msg.selectedBrowserIds?.length) { dispatch(setGlowingBrowserCards({ browserIds: msg.selectedBrowserIds, sessionId: id })); } - dispatch(sendMessageThunk({ sessionId: id, prompt: msg.prompt, mode, model, images: msg.images, contextPaths: msg.contextPaths, forcedTools: msg.forcedTools, attachedSkills: msg.attachedSkills, selectedBrowserIds: msg.selectedBrowserIds })) + dispatch(sendMessageThunk({ sessionId: id, prompt: msg.prompt, mode, model, provider, images: msg.images, contextPaths: msg.contextPaths, forcedTools: msg.forcedTools, attachedSkills: msg.attachedSkills, selectedBrowserIds: msg.selectedBrowserIds })) .then((action) => { if (sendMessageThunk.rejected.match(action)) { setAwaitingResponse(false); } }); } - }, [id, isDraft, mode, model, session?.system_prompt, session?.target_directory, dispatch]); + }, [id, isDraft, mode, model, provider, session?.system_prompt, session?.target_directory, dispatch]); const agentBusy = awaitingResponse || (!isDraft && (session?.status === 'running' || session?.status === 'waiting_approval')); @@ -304,6 +311,11 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose if (id && !isDraft) dispatch(updateSessionMode({ sessionId: id, mode: newMode })); }, [id, isDraft, dispatch]); + const handleProviderChange = useCallback((newProvider: string) => { + setProvider(newProvider); + if (id && !isDraft) dispatch(updateSessionProvider({ sessionId: id, provider: newProvider })); + }, [id, isDraft, dispatch]); + const handleModelChange = useCallback((newModel: string) => { setModel(newModel); if (id && !isDraft) dispatch(updateSessionModel({ sessionId: id, model: newModel })); @@ -330,9 +342,10 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose prompt: "Continue where you left off. Start you're response EXACTLY with 'Sorry, let me pick up where I left off", mode, model, + provider, hidden: true, })); - }, [id, mode, model, dispatch]); + }, [id, mode, model, provider, dispatch]); const [editingMessageId, setEditingMessageId] = useState(null); @@ -427,7 +440,12 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose }, [id, dispatch, onBranch, session?.dashboard_id]); const contextEstimate = useMemo(() => { - const limit = CONTEXT_WINDOWS[model] || 200_000; + // Look up context window from dynamic models first, then fall back to defaults + let limit = CONTEXT_WINDOWS_DEFAULT[model] || 200_000; + for (const models of Object.values(modelsByProvider)) { + const found = models.find((m: any) => m.value === model); + if (found?.context_window) { limit = found.context_window; break; } + } let totalChars = 0; if (session?.system_prompt) totalChars += session.system_prompt.length; for (const msg of activeBranchMessages) { @@ -438,7 +456,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose } const used = Math.round(totalChars / 4); return { used, limit }; - }, [activeBranchMessages, session?.system_prompt, session?.streamingMessage?.content, model]); + }, [activeBranchMessages, session?.system_prompt, session?.streamingMessage?.content, model, modelsByProvider]); const sessionRunning = session?.status === 'running' || session?.status === 'waiting_approval'; @@ -1116,6 +1134,8 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose onModeChange={handleModeChange} model={model} onModelChange={handleModelChange} + provider={provider} + onProviderChange={handleProviderChange} isRunning={agentBusy} onStop={handleStop} queueLength={queueLength} diff --git a/frontend/src/app/pages/AgentChat/ChatInput.tsx b/frontend/src/app/pages/AgentChat/ChatInput.tsx index d07e4cb1..fd0f058e 100644 --- a/frontend/src/app/pages/AgentChat/ChatInput.tsx +++ b/frontend/src/app/pages/AgentChat/ChatInput.tsx @@ -67,6 +67,8 @@ interface Props { onModeChange: (mode: string) => void; model: string; onModelChange: (model: string) => void; + provider?: string; + onProviderChange?: (provider: string) => void; isRunning?: boolean; onStop?: () => void; autoRunMode?: boolean; @@ -92,10 +94,10 @@ const ICON_MAP: Record = { const FALLBACK_MODE_BASE = { label: 'Agent', icon: ICON_MAP.smart_toy }; -const MODEL_OPTIONS = [ - { value: 'sonnet', label: 'Sonnet', version: '4.6' }, - { value: 'opus', label: 'Opus', version: '4.6' }, - { value: 'haiku', label: 'Haiku', version: '3.5' }, +const FALLBACK_MODELS = [ + { value: 'sonnet', label: 'Claude Sonnet 4.6', context_window: 1_000_000 }, + { value: 'opus', label: 'Claude Opus 4.6', context_window: 1_000_000 }, + { value: 'haiku', label: 'Claude Haiku 4.5', context_window: 200_000 }, ]; function formatTokenCount(n: number): string { @@ -132,7 +134,7 @@ const ContextRing: React.FC<{ used: number; limit: number; accentColor: string; ); }; -const ChatInput = forwardRef(({ onSend, disabled, mode, onModeChange, model, onModelChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus, sessionId, queueLength = 0 }, ref) => { +const ChatInput = forwardRef(({ onSend, disabled, mode, onModeChange, model, onModelChange, provider, onProviderChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus, sessionId, queueLength = 0 }, ref) => { const c = useClaudeTokens(); const editorRef = useRef(null); const containerRef = useRef(null); @@ -158,6 +160,24 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, const skills = useAppSelector((state) => state.skills.items); const modesMap = useAppSelector((state) => state.modes.items); const modesArr = useMemo(() => Object.values(modesMap), [modesMap]); + const modelsByProvider = useAppSelector((state) => state.models.byProvider); + const modelsLoaded = useAppSelector((state) => state.models.loaded); + + // Build flat model list with provider grouping + const allModelOptions = useMemo(() => { + if (!modelsLoaded || Object.keys(modelsByProvider).length === 0) { + return { flat: FALLBACK_MODELS.map(m => ({ ...m, provider: 'Anthropic' })), grouped: { Anthropic: FALLBACK_MODELS } }; + } + const flat: Array<{ value: string; label: string; context_window: number; provider: string }> = []; + const grouped: Record> = {}; + for (const [prov, models] of Object.entries(modelsByProvider)) { + grouped[prov] = models.map(m => ({ value: m.value, label: m.label, context_window: m.context_window ?? 200_000 })); + for (const m of models) { + flat.push({ value: m.value, label: m.label, context_window: m.context_window ?? 200_000, provider: prov }); + } + } + return { flat, grouped }; + }, [modelsByProvider, modelsLoaded]); useEffect(() => { if (modesArr.length === 0) dispatch(fetchModes()); @@ -566,7 +586,8 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, bgcolor: c.bg.surface, border: `1px solid ${c.border.subtle}`, borderRadius: '10px', - minWidth: 140, + minWidth: 180, + maxHeight: 400, boxShadow: c.shadow.lg, '& .MuiMenuItem-root': { fontSize: '0.8rem', @@ -979,7 +1000,7 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, }} > - {(() => { const m = MODEL_OPTIONS.find((m) => m.value === model); return m ? `${m.label} ${m.version}` : model; })()} + {(() => { const m = allModelOptions.flat.find((m) => m.value === model); return m ? m.label : model; })()} @@ -992,21 +1013,45 @@ const ChatInput = forwardRef(({ onSend, disabled, mode, transformOrigin={{ vertical: 'bottom', horizontal: 'left' }} slotProps={{ paper: menuPaperProps }} > - {MODEL_OPTIONS.map((opt) => ( - { - onModelChange(opt.value); - setModelAnchor(null); - }} - > - - - ))} + {Object.entries(allModelOptions.grouped).map(([prov, models]) => [ + + + {prov} + + , + ...models.map((opt) => ( + { + onModelChange(opt.value); + if (onProviderChange) { + // Derive API-level provider key from the display group name + const provLower = prov.toLowerCase(); + const providerMap: Record = { + anthropic: 'anthropic', + openai: 'openai', + google: 'gemini', + // OpenRouter-backed providers + xai: 'openrouter', + meta: 'openrouter', + deepseek: 'openrouter', + mistral: 'openrouter', + qwen: 'openrouter', + cohere: 'openrouter', + }; + onProviderChange(providerMap[provLower] || provLower); + } + setModelAnchor(null); + }} + > + + + )), + ]).flat()} diff --git a/frontend/src/app/pages/Analytics/Analytics.tsx b/frontend/src/app/pages/Analytics/Analytics.tsx new file mode 100644 index 00000000..1c10703e --- /dev/null +++ b/frontend/src/app/pages/Analytics/Analytics.tsx @@ -0,0 +1,63 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import Paper from '@mui/material/Paper'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; + +const Analytics: React.FC = () => { + const c = useClaudeTokens(); + + return ( + + + + Analytics + + + + + + + + + + + + Analytics powered by PostHog + + + Usage data is automatically collected — sessions, costs, tool usage, model distribution, and task categories. + All data is anonymous and can be disabled in Settings. + + + + {[ + { label: 'Sessions & Usage', desc: 'How often agents are launched, session duration, completion rates' }, + { label: 'Cost Tracking', desc: 'Spend by model, provider, and time period' }, + { label: 'Task Categories', desc: 'What users do — coding, email, research, social, browsing' }, + { label: 'Model Distribution', desc: 'Which models and providers are most popular' }, + { label: 'Tool Usage', desc: 'Most used MCP tools, execution times, approval rates' }, + { label: 'Retention & Funnels', desc: 'User engagement, feature adoption, onboarding flow' }, + ].map((item) => ( + + + {item.label} + + + {item.desc} + + + ))} + + + + + ); +}; + +export default Analytics; diff --git a/frontend/src/app/pages/Analytics/PixelChart.tsx b/frontend/src/app/pages/Analytics/PixelChart.tsx new file mode 100644 index 00000000..263b7a2d --- /dev/null +++ b/frontend/src/app/pages/Analytics/PixelChart.tsx @@ -0,0 +1,406 @@ +import React, { useRef, useEffect, useCallback } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; + +const PALETTES = { + salmon: ['#C46B57', '#D4795F', '#E8927A', '#F0A088', '#F5B49E'], + blue: ['#445588', '#5577AA', '#6688BB', '#7799CC', '#88AADD'], + coral: ['#993344', '#AA3D4E', '#BB4455', '#CC5566', '#DD6677'], + green: ['#447755', '#558866', '#669977', '#77AA88', '#88BB99'], + purple: ['#665588', '#7766AA', '#8877BB', '#9988CC', '#AA99DD'], +} as const; + +type PaletteKey = keyof typeof PALETTES; + +interface PixelChartProps { + data: { label: string; value: number }[]; + palette?: PaletteKey; + height?: number; + pixelSize?: number; + formatValue?: (v: number) => string; + glow?: boolean; + showXLabels?: boolean; + showYScale?: boolean; + mode?: 'bar' | 'area'; // 'area' draws a filled line chart instead of bars +} + +const PixelChart: React.FC = ({ + data, + palette = 'salmon', + height = 140, + pixelSize = 6, + formatValue, + glow = true, + showXLabels = true, + showYScale = true, + mode = 'bar', +}) => { + const canvasRef = useRef(null); + const containerRef = useRef(null); + const animRef = useRef(0); + const progressRef = useRef(0); + const hoverIdxRef = useRef(-1); + const tooltipRef = useRef(null); + const c = useClaudeTokens(); + const colors = PALETTES[palette]; + + const maxVal = Math.max(...data.map((d) => d.value), 0.001); + + // Compute nice Y-axis ticks + const yTicks = (() => { + if (maxVal <= 0) return [0]; + const rawStep = maxVal / 3; + const magnitude = Math.pow(10, Math.floor(Math.log10(rawStep))); + const normalised = rawStep / magnitude; + let niceStep: number; + if (normalised <= 1) niceStep = magnitude; + else if (normalised <= 2) niceStep = 2 * magnitude; + else if (normalised <= 5) niceStep = 5 * magnitude; + else niceStep = 10 * magnitude; + const ticks: number[] = []; + for (let v = 0; v <= maxVal * 1.1; v += niceStep) { + ticks.push(v); + } + if (ticks.length < 2) ticks.push(niceStep); + return ticks; + })(); + + // X-axis labels: show first, last, and up to 3 evenly spaced + const xLabels = (() => { + if (data.length <= 1) return data.map((d, i) => ({ idx: i, label: d.label })); + if (data.length <= 5) return data.map((d, i) => ({ idx: i, label: d.label })); + const result: { idx: number; label: string }[] = []; + result.push({ idx: 0, label: data[0].label }); + const step = Math.floor(data.length / 4); + for (let i = 1; i <= 3; i++) { + const idx = Math.min(i * step, data.length - 2); + if (idx > 0 && idx < data.length - 1) { + result.push({ idx, label: data[idx].label }); + } + } + result.push({ idx: data.length - 1, label: data[data.length - 1].label }); + return result; + })(); + + const Y_LABEL_WIDTH = showYScale ? 80 : 0; + + const draw = useCallback(() => { + const canvas = canvasRef.current; + const container = containerRef.current; + if (!canvas || !container || data.length === 0) return; + + const dpr = window.devicePixelRatio || 1; + const totalW = container.clientWidth; + const chartW = totalW - Y_LABEL_WIDTH; + const h = height; + canvas.width = totalW * dpr; + canvas.height = h * dpr; + canvas.style.width = `${totalW}px`; + canvas.style.height = `${h}px`; + + const ctx = canvas.getContext('2d'); + if (!ctx) return; + ctx.scale(dpr, dpr); + + const px = pixelSize; + const gridCols = Math.floor(chartW / px); + const gridRows = Math.floor(h / px); + const effectiveMax = yTicks[yTicks.length - 1] || maxVal; + + ctx.clearRect(0, 0, totalW, h); + + // Y-axis labels and horizontal grid lines + if (showYScale) { + ctx.font = '10px monospace'; + ctx.textAlign = 'right'; + ctx.textBaseline = 'middle'; + + for (const tick of yTicks) { + const yNorm = effectiveMax > 0 ? tick / effectiveMax : 0; + const yPx = h - yNorm * (h - px); + + // Grid line + ctx.strokeStyle = c.border.subtle; + ctx.lineWidth = 0.5; + ctx.setLineDash([2, 4]); + ctx.beginPath(); + ctx.moveTo(Y_LABEL_WIDTH, yPx); + ctx.lineTo(totalW, yPx); + ctx.stroke(); + ctx.setLineDash([]); + + // Label + const label = formatValue ? formatValue(tick) : (tick % 1 === 0 ? String(tick) : tick.toFixed(1)); + ctx.fillStyle = c.text.ghost; + ctx.fillText(label, Y_LABEL_WIDTH - 8, yPx); + } + } + + // Subtle grid dots in chart area + ctx.fillStyle = c.border.subtle; + for (let gy = 0; gy < gridRows; gy += 5) { + for (let gx = 0; gx < gridCols; gx += 5) { + ctx.fillRect(Y_LABEL_WIDTH + gx * px, gy * px, 1, 1); + } + } + + const progress = Math.min(progressRef.current, 1); + const hoverIdx = hoverIdxRef.current; + + if (mode === 'area') { + // ── Area / line chart mode ── + // Draw a smooth filled area under a line + const usableH = h - px * 2; + const points: { x: number; y: number }[] = []; + + for (let i = 0; i < data.length; i++) { + const val = data[i].value; + const norm = effectiveMax > 0 ? val / effectiveMax : 0; + const x = Y_LABEL_WIDTH + (i / Math.max(data.length - 1, 1)) * chartW; + const y = h - px - norm * usableH * progress; + points.push({ x, y }); + } + + if (points.length > 0) { + // Filled area with gradient + const gradient = ctx.createLinearGradient(0, 0, 0, h); + gradient.addColorStop(0, colors[colors.length - 1] + '60'); + gradient.addColorStop(0.5, colors[Math.floor(colors.length / 2)] + '30'); + gradient.addColorStop(1, colors[0] + '08'); + + ctx.beginPath(); + ctx.moveTo(points[0].x, h); + // Smooth curve through points + for (let i = 0; i < points.length; i++) { + if (i === 0) { + ctx.lineTo(points[i].x, points[i].y); + } else { + // Cubic bezier for smoothing + const prev = points[i - 1]; + const curr = points[i]; + const cpx = (prev.x + curr.x) / 2; + ctx.bezierCurveTo(cpx, prev.y, cpx, curr.y, curr.x, curr.y); + } + } + ctx.lineTo(points[points.length - 1].x, h); + ctx.closePath(); + ctx.fillStyle = gradient; + ctx.fill(); + + // Line on top + ctx.beginPath(); + for (let i = 0; i < points.length; i++) { + if (i === 0) { + ctx.moveTo(points[i].x, points[i].y); + } else { + const prev = points[i - 1]; + const curr = points[i]; + const cpx = (prev.x + curr.x) / 2; + ctx.bezierCurveTo(cpx, prev.y, cpx, curr.y, curr.x, curr.y); + } + } + ctx.strokeStyle = colors[colors.length - 1]; + ctx.lineWidth = 2; + ctx.stroke(); + + // Glow on line + if (glow) { + ctx.shadowColor = colors[colors.length - 1]; + ctx.shadowBlur = 8; + ctx.stroke(); + ctx.shadowBlur = 0; + } + + // Data point dots + for (let i = 0; i < points.length; i++) { + if (data[i].value > 0) { + const isHov = i === hoverIdx; + ctx.beginPath(); + ctx.arc(points[i].x, points[i].y, isHov ? 4 : 2.5, 0, Math.PI * 2); + ctx.fillStyle = isHov ? colors[colors.length - 1] : colors[Math.floor(colors.length / 2)]; + ctx.fill(); + if (isHov) { + ctx.strokeStyle = colors[colors.length - 1]; + ctx.lineWidth = 1.5; + ctx.stroke(); + } + } + } + + // Pixel scatter in the filled area for the pixel art feel + for (let i = 0; i < points.length - 1; i++) { + const p1 = points[i]; + const p2 = points[i + 1]; + const steps = Math.ceil((p2.x - p1.x) / px); + for (let s = 0; s < steps; s++) { + const t = s / steps; + const x = p1.x + t * (p2.x - p1.x); + const lineY = p1.y + t * (p2.y - p1.y); + // Scatter pixels below the line + for (let py = lineY + px * 2; py < h - px; py += px * 2) { + if (Math.random() > 0.65) { + const depth = (py - lineY) / (h - lineY); + const ci = Math.max(0, Math.floor((1 - depth) * (colors.length - 1))); + ctx.globalAlpha = 0.15 + (1 - depth) * 0.2; + ctx.fillStyle = colors[ci]; + ctx.fillRect(Math.floor(x / px) * px, Math.floor(py / px) * px, px - 1, px - 1); + } + } + } + } + ctx.globalAlpha = 1; + } + } else { + // ── Bar chart mode (original) ── + const barSlots = data.length; + const totalBarPx = Math.max(1, Math.floor(gridCols / barSlots)); + const barW = Math.max(1, totalBarPx - 1); + + for (let i = 0; i < data.length; i++) { + const val = data[i].value; + const normalised = effectiveMax > 0 ? val / effectiveMax : 0; + const usableRows = gridRows - 2; + const targetH = Math.max(normalised > 0 ? 1 : 0, Math.round(normalised * usableRows)); + const barH = Math.round(targetH * progress); + const barX = i * totalBarPx; + const isHovered = i === hoverIdx; + + for (let row = 0; row < barH; row++) { + const y = gridRows - 1 - row; + const colorIdx = Math.min(colors.length - 1, Math.floor((row / Math.max(barH - 1, 1)) * (colors.length - 1))); + const baseColor = isHovered ? colors[Math.min(colorIdx + 1, colors.length - 1)] : colors[colorIdx]; + + for (let col = 0; col < barW; col++) { + ctx.fillStyle = baseColor; + ctx.fillRect(Y_LABEL_WIDTH + (barX + col) * px, y * px, px - 1, px - 1); + } + } + + if (glow && barH > 0) { + const topY = (gridRows - 1 - barH + 1) * px; + ctx.shadowColor = colors[colors.length - 1]; + ctx.shadowBlur = 6; + ctx.fillStyle = colors[colors.length - 1]; + for (let col = 0; col < barW; col++) { + ctx.fillRect(Y_LABEL_WIDTH + (barX + col) * px, topY, px - 1, px - 1); + } + ctx.shadowBlur = 0; + } + } + } + }, [data, height, pixelSize, c, colors, glow, maxVal, yTicks, showYScale, Y_LABEL_WIDTH, formatValue, mode]); + + useEffect(() => { + progressRef.current = 0; + let start: number | null = null; + const animate = (ts: number) => { + if (!start) start = ts; + progressRef.current = Math.min(1, (ts - start) / 600); + draw(); + if (progressRef.current < 1) animRef.current = requestAnimationFrame(animate); + }; + animRef.current = requestAnimationFrame(animate); + return () => cancelAnimationFrame(animRef.current); + }, [data, draw]); + + useEffect(() => { + const handleResize = () => draw(); + window.addEventListener('resize', handleResize); + return () => window.removeEventListener('resize', handleResize); + }, [draw]); + + const handleMouseMove = useCallback( + (e: React.MouseEvent) => { + const canvas = canvasRef.current; + const tooltip = tooltipRef.current; + if (!canvas || !tooltip || data.length === 0) return; + + const rect = canvas.getBoundingClientRect(); + const mx = e.clientX - rect.left - Y_LABEL_WIDTH; + if (mx < 0) { hoverIdxRef.current = -1; tooltip.style.opacity = '0'; draw(); return; } + + const chartW = rect.width - Y_LABEL_WIDTH; + const gridCols = Math.floor(chartW / pixelSize); + const totalBarPx = Math.max(1, Math.floor(gridCols / data.length)); + const idx = Math.floor(mx / (totalBarPx * pixelSize)); + + if (idx >= 0 && idx < data.length) { + hoverIdxRef.current = idx; + const d = data[idx]; + const valStr = formatValue ? formatValue(d.value) : d.value.toFixed(2); + tooltip.textContent = `${d.label}: ${valStr}`; + tooltip.style.opacity = '1'; + tooltip.style.left = `${e.clientX - rect.left}px`; + tooltip.style.top = `${e.clientY - rect.top - 28}px`; + } else { + hoverIdxRef.current = -1; + tooltip.style.opacity = '0'; + } + draw(); + }, + [data, pixelSize, draw, formatValue, Y_LABEL_WIDTH], + ); + + const handleMouseLeave = useCallback(() => { + hoverIdxRef.current = -1; + if (tooltipRef.current) tooltipRef.current.style.opacity = '0'; + draw(); + }, [draw]); + + return ( + + + {/* X-axis labels */} + {showXLabels && data.length > 0 && ( + + {xLabels.map((xl) => ( + + {xl.label} + + ))} + + )} + {/* Tooltip */} + + + ); +}; + +export default PixelChart; diff --git a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx index 9dad9329..0b092864 100644 --- a/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx +++ b/frontend/src/app/pages/Dashboard/DashboardToolbar.tsx @@ -95,6 +95,7 @@ const DashboardToolbar = React.forwardRef( const defaultModel = useAppSelector((s) => s.settings.data.default_model); const [mode, setMode] = useState(defaultMode || 'agent'); const [model, setModel] = useState(defaultModel || 'sonnet'); + const [provider, setProvider] = useState('anthropic'); const settingsApplied = useRef(false); useEffect(() => { if (!settingsApplied.current) { @@ -365,6 +366,8 @@ const DashboardToolbar = React.forwardRef( onModeChange={setMode} model={model} onModelChange={setModel} + provider={provider} + onProviderChange={setProvider} embedded autoFocus sessionId={TOOLBAR_OWNER_ID} diff --git a/frontend/src/app/pages/Settings/Settings.tsx b/frontend/src/app/pages/Settings/Settings.tsx index 79f743bc..a888cacb 100644 --- a/frontend/src/app/pages/Settings/Settings.tsx +++ b/frontend/src/app/pages/Settings/Settings.tsx @@ -40,11 +40,203 @@ import Collapse from '@mui/material/Collapse'; import OpenInNewIcon from '@mui/icons-material/OpenInNew'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { updateSettings, closeSettingsModal, resetSystemPrompt, AppSettings, DEFAULT_SYSTEM_PROMPT } from '@/shared/state/settingsSlice'; +import { fetchModels } from '@/shared/state/modelsSlice'; import { setChecking, setUpdateError } from '@/shared/state/updateSlice'; import { fetchModes } from '@/shared/state/modesSlice'; import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext'; import DirectoryBrowser from '@/app/components/DirectoryBrowser'; import { CommandsContent } from '@/app/pages/Commands/Commands'; +import { API_BASE } from '@/shared/config'; + +// ── Pixel Bar ── +const PIXEL_SALMON = ['#C46B57', '#D4795F', '#E8927A', '#F0A088', '#F5B49E']; +const PIXEL_BLUE = ['#445588', '#5577AA', '#6688BB', '#7799CC', '#88AADD']; + +const PixelBarOuter: React.FC<{ value: number; max: number; width?: number; palette?: string[]; tokens: any }> = ({ value, max, width = 16, palette = PIXEL_SALMON, tokens: c }) => { + const filled = max > 0 ? Math.max(value > 0 ? 1 : 0, Math.round((value / max) * width)) : 0; + return ( + + {Array.from({ length: width }, (_, i) => ( + + ))} + + ); +}; + +// ── Usage Stats Component ── +const UsageStats: React.FC = () => { + const c = useClaudeTokens(); + const [stats, setStats] = useState(null); + + useEffect(() => { + fetch(`${API_BASE}/analytics/usage-summary`) + .then(r => r.json()) + .then(setStats) + .catch(() => {}); + }, []); + + if (!stats) return null; + + const formatCost = (v: number) => { + if (v === 0) return '$0.00'; + if (v < 0.001) return `$${v.toFixed(6)}`; + if (v < 0.01) return `$${v.toFixed(5)}`; + if (v < 1) return `$${v.toFixed(4)}`; + return `$${v.toFixed(2)}`; + }; + const formatDuration = (s: number) => { + if (s === 0) return '0s'; + if (s < 60) return `${s.toFixed(1)}s`; + if (s < 3600) return `${Math.floor(s / 60)}m ${Math.round(s % 60)}s`; + return `${Math.floor(s / 3600)}h ${Math.floor((s % 3600) / 60)}m`; + }; + const formatTotalTime = (s: number) => { + if (s < 60) return `${s.toFixed(1)}s`; + if (s < 3600) return `${(s / 60).toFixed(1)} min`; + return `${(s / 3600).toFixed(1)} hrs`; + }; + + const cardSx = { + p: 1.5, + borderRadius: `${c.radius.md}px`, + bgcolor: c.bg.elevated, + border: `1px solid ${c.border.subtle}`, + }; + const labelSx = { fontSize: '0.58rem', fontWeight: 700, color: c.text.ghost, textTransform: 'uppercase' as const, letterSpacing: '0.06em', mb: 0.25 }; + const valueSx = { fontSize: '1.05rem', fontWeight: 700, color: c.text.primary, lineHeight: 1.2 }; + const subSx = { fontSize: '0.62rem', color: c.text.tertiary, mt: 0.25 }; + + const modelEntries = Object.entries(stats.models_used || {}).sort((a: any, b: any) => b[1] - a[1]) as [string, number][]; + const providerEntries = Object.entries(stats.providers_used || {}).sort((a: any, b: any) => b[1] - a[1]) as [string, number][]; + const toolEntries = Object.entries(stats.top_tools || {}).slice(0, 10) as [string, number][]; + const maxToolCount = toolEntries.length > 0 ? Math.max(...toolEntries.map(([, c]) => c)) : 1; + const statusEntries = Object.entries(stats.status_breakdown || {}) as [string, string][]; + + // Pixel bar helper that passes tokens + const PixelBar: React.FC<{ value: number; max: number; width?: number; palette?: string[] }> = (props) => ( + + ); + + const totalTime = stats.avg_duration_seconds * stats.total_sessions; + const msgsPerSession = stats.total_sessions > 0 ? (stats.total_messages / stats.total_sessions).toFixed(1) : '0'; + const toolsPerSession = stats.total_sessions > 0 ? (stats.total_tool_calls / stats.total_sessions).toFixed(1) : '0'; + + return ( + + {/* Row 1: Core metrics */} + + + Total Sessions + {stats.total_sessions.toLocaleString()} + + {statusEntries.map(([s, n]) => `${n} ${s}`).join(', ') || 'no sessions'} + + + + Total Cost + {formatCost(stats.total_cost_usd)} + + {formatCost(stats.avg_cost_per_session)} avg per session + + + + Total Messages + {stats.total_messages.toLocaleString()} + + {msgsPerSession} avg per session + + + + Total Tool Calls + {stats.total_tool_calls.toLocaleString()} + + {toolsPerSession} avg per session + + + + + {/* Row 2: Time + efficiency */} + + + Total Run Time + {formatTotalTime(totalTime)} + across all sessions + + + Avg Session + {formatDuration(stats.avg_duration_seconds)} + per session duration + + + Completion Rate + {(stats.completion_rate * 100).toFixed(1)}% + + sessions finished successfully + + + + Providers + {Object.keys(stats.providers_used || {}).length} + + {providerEntries.map(([p]) => p).join(', ') || 'none configured'} + + + + + {/* Model + Provider + Tool breakdown */} + + {/* Models & Providers */} + + Models Used + {modelEntries.length > 0 ? modelEntries.map(([model, count]) => { + const pct = stats.total_sessions > 0 ? ((count / stats.total_sessions) * 100).toFixed(0) : '0'; + return ( + + + {model} + + {count} ({pct}%) + + + + + ); + }) : No sessions yet} + + + {/* Tools */} + + Top Tools + {toolEntries.length > 0 ? toolEntries.map(([tool, count]) => { + const shortName = tool.includes('__') ? tool.split('__').pop() : tool; + const pct = stats.total_tool_calls > 0 ? ((count / stats.total_tool_calls) * 100).toFixed(0) : '0'; + return ( + + + {shortName} + + {count} call{count !== 1 ? 's' : ''} ({pct}%) + + + + + ); + }) : No tool calls yet} + + + + ); +}; const API_KEY_STEPS = [ { @@ -87,7 +279,7 @@ const Settings: React.FC = () => { const downloadPercent = useAppSelector((s) => s.update.downloadPercent); const updateError = useAppSelector((s) => s.update.error); - const [activeTab, setActiveTab] = useState<'general' | 'commands'>('general'); + const [activeTab, setActiveTab] = useState<'general' | 'models' | 'usage' | 'commands'>('general'); const [form, setForm] = useState({ ...settings }); const [showApiKey, setShowApiKey] = useState(false); const [browseOpen, setBrowseOpen] = useState(false); @@ -143,6 +335,7 @@ const Settings: React.FC = () => { if (form.theme !== settings.theme) { setThemeMode(form.theme); } + dispatch(fetchModels()); setSaved(true); }; @@ -165,6 +358,7 @@ const Settings: React.FC = () => { if (form.theme !== settings.theme) { setThemeMode(form.theme); } + dispatch(fetchModels()); setSaved(true); setConfirmDiscard(false); dispatch(closeSettingsModal()); @@ -272,6 +466,8 @@ const Settings: React.FC = () => { }} > + + @@ -626,125 +822,6 @@ const Settings: React.FC = () => { - {/* ── API ── */} - API - - - Anthropic API key - - - Stored securely in the local database. - - setShowApiHelp((v) => !v)} - sx={{ - color: c.accent.primary, - fontSize: '0.75rem', - cursor: 'pointer', - display: 'inline-flex', - alignItems: 'center', - gap: 0.4, - whiteSpace: 'nowrap', - userSelect: 'none', - '&:hover': { textDecoration: 'underline' }, - }} - > - {showApiHelp ? 'Hide guide' : 'How do I get a key?'} - - - - - - {API_KEY_STEPS.map((step, i) => ( - - - {i + 1} - - - - {step.title} - {step.link && ( - - Open - - - )} - - - {step.detail} - - - - ))} - - - - setForm({ ...form, anthropic_api_key: e.target.value || null })} - size="small" - fullWidth - placeholder="sk-ant-..." - sx={{ - ...fieldSx, - '& .MuiOutlinedInput-root': { - ...fieldSx['& .MuiOutlinedInput-root'], - fontFamily: c.font.mono, - }, - }} - InputProps={{ - endAdornment: ( - - setShowApiKey(!showApiKey)} - edge="end" - size="small" - sx={{ color: c.text.tertiary }} - > - {showApiKey ? : } - - - ), - }} - /> - - {/* ── Advanced ── */} Advanced @@ -873,6 +950,156 @@ const Settings: React.FC = () => { + ) : activeTab === 'models' ? ( + + + Connect your AI model providers. Each key is stored locally on your device. + + + {/* OpenRouter — recommended */} + + + OpenRouter + + RECOMMENDED + + + + One key for 300+ models — Llama, DeepSeek, Mistral, Qwen, Grok, and more. Easiest way to get started. + + + setForm({ ...form, openrouter_api_key: e.target.value || null } as any)} + size="small" + fullWidth + placeholder="sk-or-..." + sx={{ ...fieldSx, '& .MuiOutlinedInput-root': { ...fieldSx['& .MuiOutlinedInput-root'], fontFamily: c.font.mono } }} + /> + + Get key + + + + + {/* Anthropic */} + + Anthropic + Claude Sonnet, Opus, Haiku — direct API access. + + setForm({ ...form, anthropic_api_key: e.target.value || null })} + size="small" + fullWidth + placeholder="sk-ant-..." + sx={{ ...fieldSx, '& .MuiOutlinedInput-root': { ...fieldSx['& .MuiOutlinedInput-root'], fontFamily: c.font.mono } }} + InputProps={{ + endAdornment: ( + + setShowApiKey(!showApiKey)} edge="end" size="small" sx={{ color: c.text.tertiary }}> + {showApiKey ? : } + + + ), + }} + /> + + Get key + + + + + {/* OpenAI */} + + OpenAI + GPT-5.4, o3, o4-mini — direct API access. + + setForm({ ...form, openai_api_key: e.target.value || null } as any)} + size="small" + fullWidth + placeholder="sk-..." + sx={{ ...fieldSx, '& .MuiOutlinedInput-root': { ...fieldSx['& .MuiOutlinedInput-root'], fontFamily: c.font.mono } }} + /> + + Get key + + + + + {/* Google */} + + Google + Gemini 2.5 Pro and Flash — direct API access. + + setForm({ ...form, google_api_key: e.target.value || null } as any)} + size="small" + fullWidth + placeholder="AIza..." + sx={{ ...fieldSx, '& .MuiOutlinedInput-root': { ...fieldSx['& .MuiOutlinedInput-root'], fontFamily: c.font.mono } }} + /> + + Get key + + + + + ) : activeTab === 'usage' ? ( + + + + {/* ── Analytics ── */} + Analytics + + + + Share anonymous usage data + + Help improve OpenSwarm by sharing anonymous statistics like session counts, model usage, and feature adoption. No conversations, file paths, or personal information is ever collected. + + + setForm({ ...form, analytics_opt_in: e.target.checked } as any)} + sx={{ + '& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary }, + '& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary }, + }} + /> + + ) : ( diff --git a/frontend/src/app/pages/Views/ViewEditor.tsx b/frontend/src/app/pages/Views/ViewEditor.tsx index 75ea08c7..64ea271f 100644 --- a/frontend/src/app/pages/Views/ViewEditor.tsx +++ b/frontend/src/app/pages/Views/ViewEditor.tsx @@ -519,6 +519,7 @@ const ViewEditor: React.FC = ({ output, onClose }) => { const [autoRunEnabled, setAutoRunEnabled] = useState(savedAutoRun?.enabled ?? false); const [autoRunMode, setAutoRunMode] = useState(savedAutoRun?.mode ?? 'agent'); const [autoRunModel, setAutoRunModel] = useState(savedAutoRun?.model ?? 'sonnet'); + const [autoRunProvider, setAutoRunProvider] = useState('anthropic'); const [autoRunning, setAutoRunning] = useState(false); const autoRunInputRef = useRef(null); const autoRunInitialized = useRef(false); @@ -1539,6 +1540,8 @@ const ViewEditor: React.FC = ({ output, onClose }) => { onModeChange={setAutoRunMode} model={autoRunModel} onModelChange={setAutoRunModel} + provider={autoRunProvider} + onProviderChange={setAutoRunProvider} />