WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-08-20 19:52:23 +02:00
Code Issues Packages Projects Releases Wiki Activity

[Haik]: Agentic refactor 2. Split Backend God Model

Browse Source
This commit is contained in:
haikdc
2026-03-30 02:12:22 -07:00
parent b054ec41e0
commit 7239f70446
24 changed files with 3273 additions and 4098 deletions
Download Patch File Download Diff File Expand all files Collapse all files
README.md
+2 -2
View File
@@ -1,5 +1,5 @@
<p align="center">
<img src="assets/icon.png" alt="Open Swarm" width="128" height="128">
<img src="readme_assets/icon.png" alt="Open Swarm" width="128" height="128">
</p>
<h1 align="center">Open Swarm</h1>
@@ -22,7 +22,7 @@
<br>
<p align="center">
<img src="assets/screenshot.png" alt="Open Swarm Dashboard" width="900">
<img src="readme_assets/screenshot.png" alt="Open Swarm Dashboard" width="900">
</p>
<br>
backend/apps/agents/agent_loop.py
+657 -282
View File
@@ -1,331 +1,706 @@
"""Owned agent loop — replaces claude_agent_sdk's query() function.
"""Main agent loop — extracted from AgentManager._run_agent_loop.
Generalizes the pattern from browser_agent.py (lines 243-334) into a
provider-agnostic, streaming, HITL-aware tool-use loop.
Handles the Claude Agent SDK query loop, approval hooks, streaming,
mock-agent fallback, and session-completed analytics.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
import sys
import time
from typing import Any, Callable, Awaitable
from datetime import datetime
from uuid import uuid4
from backend.apps.agents.providers.base import (
BaseProvider, ContentBlock, ModelResponse, ProviderMessage,
StreamEvent, ToolCall, ToolSchema,
from backend.apps.agents.models import AgentSession, ApprovalRequest, Message
from backend.apps.agents.ws_manager import ws_manager
from backend.apps.agents.prompt_builder import (
resolve_mode, compose_system_prompt, build_connected_tools_context,
build_outputs_context, build_browser_context, build_prompt_content,
get_pre_selected_browser_ids,
)
from backend.apps.agents.mcp_builder import (
FULL_TOOLS, build_mcp_servers, get_effective_policy, get_all_tool_names,
_get_denied_tool_names, _get_all_known_tool_names, _is_fully_denied,
)
from backend.apps.agents.session_store import save_session
from backend.apps.settings.settings import load_settings
from backend.apps.tools_lib.tools_lib import (
_load_all as load_all_tools,
load_builtin_permissions,
)
from backend.apps.common.mcp_utils import sanitize_server_name as _sanitize_server_name
from backend.apps.analytics.collector import record as _analytics
logger = logging.getLogger(__name__)
# Type aliases for callbacks
ToolExecutor = Callable[[str, dict], Awaitable[list[dict]]]
# hitl_handler(tool_name, tool_input) -> (approved, updated_input_or_None)
HITLHandler = Callable[[str, dict], Awaitable[tuple[bool, dict | None]]]
# ws_emitter(event_type, data) -> None
WSEmitter = Callable[[str, dict], Awaitable[None]]
# ---------------------------------------------------------------------------
# Streaming helpers
# ---------------------------------------------------------------------------
async def stream_text(session_id: str, msg_id: str, text: str, delay: float = 0.03):
await ws_manager.send_to_session(session_id, "agent:stream_start", {
"session_id": session_id, "message_id": msg_id, "role": "assistant",
})
words = text.split(" ")
for i, word in enumerate(words):
chunk = word if i == 0 else " " + word
await ws_manager.send_to_session(session_id, "agent:stream_delta", {
"session_id": session_id, "message_id": msg_id, "delta": chunk,
})
await asyncio.sleep(delay)
await ws_manager.send_to_session(session_id, "agent:stream_end", {
"session_id": session_id, "message_id": msg_id,
})
class AgentLoop:
"""Provider-agnostic agent loop with streaming and HITL support.
async def stream_tool_input(session_id: str, msg_id: str, tool_name: str, input_json: str, delay: float = 0.02):
await ws_manager.send_to_session(session_id, "agent:stream_start", {
"session_id": session_id, "message_id": msg_id, "role": "tool_call", "tool_name": tool_name,
})
chunk_size = 12
for i in range(0, len(input_json), chunk_size):
await ws_manager.send_to_session(session_id, "agent:stream_delta", {
"session_id": session_id, "message_id": msg_id, "delta": input_json[i:i + chunk_size],
})
await asyncio.sleep(delay)
await ws_manager.send_to_session(session_id, "agent:stream_end", {
"session_id": session_id, "message_id": msg_id,
})
The loop:
1. Sends user message to the model
2. Streams the response (emitting WebSocket events)
3. If the model requests tool use:
a. For each tool call: check HITL permission → execute → collect result
b. Append tool results → go to step 2
4. If the model stops (end_turn/max_tokens): done
"""
def __init__(
self,
session_id: str,
provider: BaseProvider,
model: str,
system_prompt: str | None,
tools: list[ToolSchema],
tool_executor: ToolExecutor,
hitl_handler: HITLHandler,
ws_emitter: WSEmitter,
max_turns: int | None = None,
cwd: str | None = None,
):
self.session_id = session_id
self.provider = provider
self.model = model
self.system_prompt = system_prompt
self.tools = tools
self.tool_executor = tool_executor
self.hitl_handler = hitl_handler
self.ws_emitter = ws_emitter
self.max_turns = max_turns
self.cwd = cwd
# ---------------------------------------------------------------------------
# Analytics helper
# ---------------------------------------------------------------------------
# Conversation history in provider-agnostic format
self.messages: list[ProviderMessage] = []
def fire_session_completed(session: AgentSession, sessions_dict: dict[str, AgentSession]):
duration = 0.0
if session.created_at:
end = session.closed_at or datetime.now()
duration = (end - session.created_at).total_seconds()
tool_names = [
m.content.get("tool", "") for m in session.messages
if m.role == "tool_call" and isinstance(m.content, dict)
]
user_messages = [
(m.content if isinstance(m.content, str) else str(m.content))[:200]
for m in session.messages if m.role == "user"
]
_analytics("session.completed", {
"model": session.model,
"provider": getattr(session, "provider", "anthropic"),
"mode": session.mode,
"cost_usd": session.cost_usd,
"message_count": len([m for m in session.messages if m.role in ("user", "assistant")]),
"duration_seconds": round(duration, 1),
"status": session.status,
"tool_count": len(tool_names),
"tools_list": list(set(tool_names)),
"session_title": session.name,
"first_user_message": user_messages[0] if user_messages else "",
"input_tokens": session.tokens.get("input", 0),
"output_tokens": session.tokens.get("output", 0),
"is_sub_agent": session.parent_session_id is not None,
"parent_session_id": session.parent_session_id,
"sub_agent_count": len([s for s in sessions_dict.values() if s.parent_session_id == session.id]),
"branch_count": len(session.branches),
}, session_id=session.id, dashboard_id=session.dashboard_id)
# Token tracking
self.total_input_tokens = 0
self.total_output_tokens = 0
async def run(self, user_content: Any) -> None:
"""Run the agent loop for a single user turn."""
# Append user message
user_msg = self.provider.format_user_message(user_content)
self.messages.append(user_msg)
# ---------------------------------------------------------------------------
# Mock agent
# ---------------------------------------------------------------------------
turn = 0
while True:
if self.max_turns and turn >= self.max_turns:
logger.info(f"Agent {self.session_id}: max turns ({self.max_turns}) reached")
break
turn += 1
async def run_mock_agent(session_id: str, prompt: str, sessions: dict[str, AgentSession]):
session = sessions.get(session_id)
if not session:
return
# Stream the model response and collect it
response = await self._stream_and_collect()
await asyncio.sleep(1)
# Track usage
self.total_input_tokens += response.usage.get("input_tokens", 0)
self.total_output_tokens += response.usage.get("output_tokens", 0)
request_id = uuid4().hex
approval_req = ApprovalRequest(
id=request_id, session_id=session_id, tool_name="Bash",
tool_input={"command": f"echo 'Processing: {prompt}'", "description": "Echo the user prompt"},
)
session.pending_approvals.append(approval_req)
session.status = "waiting_approval"
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id, "status": "waiting_approval",
})
# Append assistant message to conversation history
assistant_msg = self.provider.format_assistant_message(response)
self.messages.append(assistant_msg)
decision = await ws_manager.send_approval_request(
session_id, request_id, "Bash",
{"command": f"echo 'Processing: {prompt}'", "description": "Echo the user prompt"},
)
# If no tool use, we're done
if response.stop_reason != "tool_use":
break
session.pending_approvals = [a for a in session.pending_approvals if a.id != request_id]
session.status = "running"
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id, "status": "running",
})
# Execute tools
tool_results = await self._execute_tools(response)
if not tool_results:
break
tool_input_content = {"tool": "Bash", "input": {"command": f"echo 'Processing: {prompt}'"}, "approved": decision.get("behavior") == "allow"}
tool_msg_id = uuid4().hex
await stream_tool_input(session_id, tool_msg_id, "Bash", json.dumps(tool_input_content["input"], indent=2))
tool_msg = Message(id=tool_msg_id, role="tool_call", content=tool_input_content, branch_id=session.active_branch_id)
session.messages.append(tool_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": tool_msg.model_dump(mode="json"),
})
# Append tool results
self.messages.append(ProviderMessage(role="tool_result", content=tool_results))
await asyncio.sleep(1)
async def _stream_and_collect(self) -> ModelResponse:
"""Stream model output, emit WebSocket events, collect full response."""
collected_content: list[ContentBlock] = []
collected_usage: dict[str, int] = {}
stop_reason = "end_turn"
if decision.get("behavior") == "allow":
tool_result = Message(role="tool_result", content=f"Processing: {prompt}", branch_id=session.active_branch_id)
session.messages.append(tool_result)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": tool_result.model_dump(mode="json"),
})
# Track streaming state for WS emissions
stream_text_msg_id: str | None = None
stream_tool_msg_ids: dict[int, str] = {} # block index -> msg_id
block_index_map: dict[int, str] = {} # block index -> msg_id
await asyncio.sleep(1)
# Buffers for collecting content
text_buffers: dict[int, str] = {}
json_buffers: dict[int, str] = {}
tool_names: dict[int, str] = {}
tool_ids: dict[int, str] = {}
block_types: dict[int, str] = {}
asst_text = (
f"I've processed your request: \"{prompt}\"\n\n"
"This is a mock response because `claude-agent-sdk` is not installed. "
"Install it with `pip install claude-agent-sdk` to use real Claude Code instances.\n\n"
f"The agent was configured with:\n- Model: {session.model}\n- Mode: {session.mode}"
)
asst_msg_id = uuid4().hex
await stream_text(session_id, asst_msg_id, asst_text)
async for event in self.provider.stream_message(
model=self.model,
system=self.system_prompt,
messages=self.messages,
tools=self.tools,
):
if event.type == "content_block_start":
if event.block_type == "text":
if stream_text_msg_id is None:
stream_text_msg_id = uuid4().hex
await self.ws_emitter("agent:stream_start", {
"message_id": stream_text_msg_id,
"role": "assistant",
})
block_index_map[event.index] = stream_text_msg_id
block_types[event.index] = "text"
text_buffers[event.index] = ""
asst_msg = Message(id=asst_msg_id, role="assistant", content=asst_text, branch_id=session.active_branch_id)
session.messages.append(asst_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": asst_msg.model_dump(mode="json"),
})
elif event.block_type == "tool_use":
tool_msg_id = uuid4().hex
stream_tool_msg_ids[event.index] = tool_msg_id
block_index_map[event.index] = tool_msg_id
block_types[event.index] = "tool_use"
tool_names[event.index] = event.tool_name
tool_ids[event.index] = event.tool_id
json_buffers[event.index] = ""
session.status = "completed"
session.closed_at = datetime.now()
session.cost_usd = 0.001
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id, "status": "completed",
"session": session.model_dump(mode="json"),
})
await ws_manager.send_to_session(session_id, "agent:cost_update", {
"session_id": session_id, "cost_usd": session.cost_usd,
})
await self.ws_emitter("agent:stream_start", {
"message_id": tool_msg_id,
"role": "tool_call",
"tool_name": event.tool_name,
})
elif event.type == "content_block_delta":
msg_id = block_index_map.get(event.index)
if not msg_id:
# ---------------------------------------------------------------------------
# Main agent loop
# ---------------------------------------------------------------------------
async def run_agent_loop(
sessions: dict[str, AgentSession],
session_id: str,
prompt: str,
images: list | None = None,
context_paths: list | None = None,
forced_tools: list[str] | None = None,
attached_skills: list | None = None,
fork_session: bool = False,
selected_browser_ids: list[str] | None = None,
):
"""Run the Claude Agent SDK query loop for a session."""
session = sessions.get(session_id)
if not session:
return
prompt_content = build_prompt_content(
prompt, images, context_paths, forced_tools, attached_skills,
load_all_tools_fn=load_all_tools,
)
try:
from claude_agent_sdk import (
query, ClaudeAgentOptions, AssistantMessage, ResultMessage,
)
from claude_agent_sdk.types import (
HookMatcher, PermissionResultAllow, PermissionResultDeny,
TextBlock, ToolUseBlock, StreamEvent,
SystemMessage,
)
except ImportError:
logger.warning("claude_agent_sdk not installed, running in mock mode")
await run_mock_agent(session_id, prompt, sessions)
return
session.status = "running"
_builtin_perms = load_builtin_permissions()
async def _request_user_approval(tool_name: str, tool_input) -> dict:
safe_input = tool_input if isinstance(tool_input, dict) else {}
request_id = uuid4().hex
approval_req = ApprovalRequest(
id=request_id, session_id=session_id, tool_name=tool_name, tool_input=safe_input,
)
session.pending_approvals.append(approval_req)
session.status = "waiting_approval"
_analytics("approval.requested", {
"tool_name": tool_name,
"is_first_approval_in_session": len(session.pending_approvals) == 1,
"model": session.model,
}, session_id=session_id, dashboard_id=session.dashboard_id)
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id, "status": "waiting_approval",
})
decision = await ws_manager.send_approval_request(
session_id, request_id, tool_name, safe_input,
)
approval_latency_ms = int((datetime.now() - approval_req.created_at).total_seconds() * 1000)
_analytics("approval.resolved", {
"tool_name": tool_name,
"decision": decision.get("behavior", "unknown"),
"latency_ms": approval_latency_ms,
"input_was_modified": decision.get("updated_input") is not None,
"model": session.model,
}, session_id=session_id, dashboard_id=session.dashboard_id)
session.pending_approvals = [a for a in session.pending_approvals if a.id != request_id]
session.status = "running"
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id, "status": "running",
})
return decision
async def can_use_tool(tool_name, input_data, context):
if tool_name != "AskUserQuestion":
policy = get_effective_policy(tool_name, _builtin_perms)
if policy == "always_allow":
return PermissionResultAllow(updated_input=input_data)
if policy == "deny":
return PermissionResultDeny(message="Tool denied by permission policy")
decision = await _request_user_approval(tool_name, input_data)
if decision.get("behavior") == "allow":
return PermissionResultAllow(updated_input=decision.get("updated_input", input_data))
return PermissionResultDeny(message=decision.get("message", "User denied this action"))
tool_start_times: dict[str, float] = {}
async def pre_tool_hook(input_data, tool_use_id, context):
tool_name = input_data.get("tool_name", "")
hook_event = input_data.get("hook_event_name", "PreToolUse")
if tool_name and tool_name != "AskUserQuestion":
policy = get_effective_policy(tool_name, _builtin_perms)
if policy == "deny":
return {"hookSpecificOutput": {"hookEventName": hook_event, "permissionDecision": "deny", "permissionDecisionReason": "Tool denied by permission policy"}}
if policy == "ask":
tool_input = input_data.get("tool_input", {})
decision = await _request_user_approval(tool_name, tool_input)
if decision.get("behavior") == "allow":
if tool_use_id:
tool_start_times[tool_use_id] = time.time()
return {"hookSpecificOutput": {"hookEventName": hook_event, "permissionDecision": "allow"}}
return {"hookSpecificOutput": {"hookEventName": hook_event, "permissionDecision": "deny", "permissionDecisionReason": decision.get("message", "User denied this action")}}
if tool_use_id:
tool_start_times[tool_use_id] = time.time()
return {}
async def post_tool_hook(input_data, tool_use_id, context):
import re as _re_tool
elapsed_ms = None
if tool_use_id and tool_use_id in tool_start_times:
elapsed_ms = int((time.time() - tool_start_times.pop(tool_use_id)) * 1000)
raw_response = input_data.get("tool_response", "")
hook_tool_name_early = input_data.get("tool_name", "")
if hook_tool_name_early:
_is_mcp = "__" in hook_tool_name_early
_mcp_server = ""
_tool_short = hook_tool_name_early
if _is_mcp:
_mcp_match = _re_tool.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", hook_tool_name_early)
if _mcp_match:
_mcp_server = _mcp_match.group(1)
_tool_short = _mcp_match.group(2)
_analytics("tool.executed", {
"tool_name": hook_tool_name_early, "tool_short_name": _tool_short,
"tool_type": "mcp" if _is_mcp else "builtin", "mcp_server": _mcp_server,
"duration_ms": elapsed_ms,
"success": not (isinstance(raw_response, str) and raw_response.startswith("Error")),
"model": session.model, "provider": session.provider,
}, session_id=session_id, dashboard_id=session.dashboard_id)
if isinstance(raw_response, list) and raw_response:
text_parts = [b.get("text", "") for b in raw_response if isinstance(b, dict) and b.get("type") == "text"]
if text_parts:
raw_response = "\n".join(text_parts) if len(text_parts) > 1 else text_parts[0]
if isinstance(raw_response, str):
content = raw_response
else:
try:
content = json.dumps(raw_response, indent=2, default=str)
except Exception:
content = str(raw_response)
result_payload: dict = {"text": content}
hook_tool_name = input_data.get("tool_name", "")
if hook_tool_name:
result_payload["tool_name"] = hook_tool_name
if elapsed_ms is not None:
result_payload["elapsed_ms"] = elapsed_ms
if hook_tool_name == "Agent":
tool_input = input_data.get("tool_input", {})
agent_prompt = tool_input.get("prompt", tool_input.get("task", ""))
sub_text = content
sub_cost = 0.0
sub_tokens: dict = {"input": 0, "output": 0}
sub_model = session.model
if isinstance(raw_response, dict):
blocks = raw_response.get("content")
if isinstance(blocks, list):
parts = [b.get("text", "") for b in blocks if isinstance(b, dict) and b.get("type") == "text"]
if parts:
sub_text = "\n".join(parts) if len(parts) > 1 else parts[0]
elif isinstance(raw_response.get("text"), str):
sub_text = raw_response["text"]
usage = raw_response.get("usage", {})
if isinstance(usage, dict):
sub_tokens["input"] = usage.get("input_tokens", 0) + usage.get("cache_creation_input_tokens", 0) + usage.get("cache_read_input_tokens", 0)
sub_tokens["output"] = usage.get("output_tokens", 0)
if raw_response.get("model"):
sub_model = raw_response["model"]
sub_session_id = uuid4().hex
sub_name = agent_prompt[:50] if agent_prompt else "Sub-agent"
sub_session = AgentSession(
id=sub_session_id, name=sub_name, status="completed", model=sub_model,
mode="sub-agent", cwd=session.cwd, created_at=datetime.now(),
cost_usd=sub_cost, tokens=sub_tokens,
messages=[
Message(role="user", content=agent_prompt, branch_id="main"),
Message(role="assistant", content=sub_text, branch_id="main"),
],
dashboard_id=session.dashboard_id, parent_session_id=session_id,
)
sessions[sub_session_id] = sub_session
await ws_manager.broadcast_global("agent:status", {
"session_id": sub_session_id, "status": sub_session.status,
"session": sub_session.model_dump(mode="json"),
})
result_payload["sub_session_id"] = sub_session_id
result_msg = Message(role="tool_result", content=result_payload, branch_id=session.active_branch_id)
session.messages.append(result_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": result_msg.model_dump(mode="json"),
})
return {"continue_": True}
try:
_, mode_sys_prompt, _ = resolve_mode(session.mode, get_all_tool_names)
connected_tools_ctx = build_connected_tools_context(
session.allowed_tools, load_all_tools, get_all_tool_names, _is_fully_denied, _get_denied_tool_names,
)
outputs_ctx = build_outputs_context()
browser_ctx = build_browser_context(session.dashboard_id, selected_browser_ids=selected_browser_ids)
global_settings = load_settings()
composed_prompt = compose_system_prompt(
global_settings.default_system_prompt, mode_sys_prompt, session.system_prompt,
connected_tools_ctx, outputs_ctx, browser_ctx,
)
if session.mode == "view-builder":
from backend.apps.outputs.view_builder_templates import VIEW_BUILDER_SKILL
skill_block = f"<app_builder_reference>\n{VIEW_BUILDER_SKILL}\n</app_builder_reference>"
composed_prompt = f"{composed_prompt}\n\n{skill_block}" if composed_prompt else skill_block
mcp_servers = await build_mcp_servers(session.allowed_tools)
_browser_delegation_tools = ["CreateBrowserAgent", "BrowserAgent", "BrowserAgents"]
_browser_all_denied = all(_builtin_perms.get(t, "always_allow") == "deny" for t in _browser_delegation_tools)
if not _browser_all_denied:
browser_agent_server_path = os.path.join(os.path.dirname(__file__), "browser_agent_mcp_server.py")
backend_port = os.environ.get("OPENSWARM_PORT", "8324")
pre_selected_bids = get_pre_selected_browser_ids(session.dashboard_id)
mcp_servers["openswarm-browser-agent"] = {
"command": sys.executable,
"args": [browser_agent_server_path],
"env": {
"OPENSWARM_PORT": backend_port,
"OPENSWARM_AGENT_MODEL": session.model,
"OPENSWARM_DASHBOARD_ID": session.dashboard_id or "",
"OPENSWARM_PRE_SELECTED_BROWSER_IDS": ",".join(pre_selected_bids),
"OPENSWARM_PARENT_SESSION_ID": session.id,
},
"type": "stdio",
}
_invoke_agent_tools = ["InvokeAgent"]
_invoke_all_denied = all(_builtin_perms.get(t, "always_allow") == "deny" for t in _invoke_agent_tools)
if not _invoke_all_denied:
invoke_agent_server_path = os.path.join(os.path.dirname(__file__), "invoke_agent_mcp_server.py")
backend_port = os.environ.get("OPENSWARM_PORT", "8324")
mcp_servers["openswarm-invoke-agent"] = {
"command": sys.executable,
"args": [invoke_agent_server_path],
"env": {
"OPENSWARM_PORT": backend_port,
"OPENSWARM_PARENT_SESSION_ID": session.id,
"OPENSWARM_DASHBOARD_ID": session.dashboard_id or "",
},
"type": "stdio",
}
effective_allowed = [
t for t in session.allowed_tools
if t in FULL_TOOLS and _builtin_perms.get(t, "always_allow") == "always_allow"
]
effective_disallowed = [
t for t in FULL_TOOLS
if _builtin_perms.get(t, "always_allow") == "deny"
]
if mcp_servers:
all_tools_list = load_all_tools()
for name in mcp_servers:
if name == "openswarm-browser-agent":
for bt in _browser_delegation_tools:
policy = _builtin_perms.get(bt, "always_allow")
if policy == "always_allow":
effective_allowed.append(f"mcp__openswarm-browser-agent__{bt}")
elif policy == "deny":
effective_disallowed.append(f"mcp__openswarm-browser-agent__{bt}")
continue
if name == "openswarm-invoke-agent":
for it in _invoke_agent_tools:
policy = _builtin_perms.get(it, "always_allow")
if policy == "always_allow":
effective_allowed.append(f"mcp__openswarm-invoke-agent__{it}")
elif policy == "deny":
effective_disallowed.append(f"mcp__openswarm-invoke-agent__{it}")
continue
tool_def = next(
(t for t in all_tools_list if t.mcp_config and t.enabled and _sanitize_server_name(t.name) == name),
None,
)
if tool_def:
denied = _get_denied_tool_names(tool_def)
known = _get_all_known_tool_names(tool_def)
for tn in known - denied:
policy = tool_def.tool_permissions.get(tn, "ask")
if policy == "always_allow":
effective_allowed.append(f"mcp__{name}__{tn}")
for tn in denied:
effective_disallowed.append(f"mcp__{name}__{tn}")
else:
effective_allowed.append(f"mcp__{name}__*")
if event.delta_type == "text_delta":
text_buffers.setdefault(event.index, "")
text_buffers[event.index] += event.text
await self.ws_emitter("agent:stream_delta", {
"message_id": msg_id,
"delta": event.text,
})
google_allowed = [t for t in effective_allowed if "google-workspace" in t]
reddit_allowed = [t for t in effective_allowed if "reddit" in t]
builtin_allowed = [t for t in effective_allowed if not t.startswith("mcp__")]
logger.info(f"[MCP-DEBUG] effective_allowed: {len(effective_allowed)} total "
f"(builtins={len(builtin_allowed)}, google={len(google_allowed)}, reddit={len(reddit_allowed)})")
if effective_disallowed:
logger.info(f"[MCP-DEBUG] effective_disallowed: {effective_disallowed}")
elif event.delta_type == "input_json_delta":
json_buffers.setdefault(event.index, "")
json_buffers[event.index] += event.text
await self.ws_emitter("agent:stream_delta", {
"message_id": msg_id,
"delta": event.text,
})
options_kwargs: dict = {
"model": session.model,
"max_buffer_size": 5 * 1024 * 1024,
"permission_mode": "default",
"can_use_tool": can_use_tool,
"hooks": {
"PreToolUse": [HookMatcher(matcher=None, hooks=[pre_tool_hook])],
"PostToolUse": [HookMatcher(matcher=None, hooks=[post_tool_hook])],
},
"allowed_tools": effective_allowed,
"disallowed_tools": effective_disallowed,
"include_partial_messages": True,
}
elif event.type == "content_block_stop":
msg_id = block_index_map.get(event.index)
bt = block_types.get(event.index, "")
from backend.apps.nine_router import is_running as _9r_running
if global_settings.anthropic_api_key:
options_kwargs["env"] = {"ANTHROPIC_API_KEY": global_settings.anthropic_api_key}
logger.info("[MCP-DEBUG] Using direct API key")
elif _9r_running():
options_kwargs["env"] = {
"ANTHROPIC_API_KEY": "9router",
"ANTHROPIC_BASE_URL": "http://localhost:20128",
}
options_kwargs["extra_args"] = {"bare": None}
logger.info("[MCP-DEBUG] Using 9Router (bare mode)")
else:
raise ValueError("No AI provider configured. Set an API key or connect a subscription.")
if bt == "text":
collected_content.append(
ContentBlock(type="text", text=text_buffers.get(event.index, ""))
if mcp_servers:
options_kwargs["mcp_servers"] = mcp_servers
mcp_json_len = len(json.dumps({"mcpServers": mcp_servers}))
logger.info(f"[MCP-DEBUG] mcp_servers passed to SDK: {list(mcp_servers.keys())}, JSON length={mcp_json_len}")
if composed_prompt:
options_kwargs["system_prompt"] = composed_prompt
if session.max_turns:
options_kwargs["max_turns"] = session.max_turns
if session.cwd:
options_kwargs["cwd"] = session.cwd
if session.sdk_session_id:
options_kwargs["resume"] = session.sdk_session_id
if fork_session:
options_kwargs["fork_session"] = True
logger.info(f"[MCP-DEBUG] Creating ClaudeAgentOptions with model={session.model}")
options = ClaudeAgentOptions(**options_kwargs)
logger.info("[MCP-DEBUG] ClaudeAgentOptions created. Starting query...")
async def prompt_stream():
yield {"type": "user", "message": {"role": "user", "content": prompt_content}}
stream_text_msg_id = None
stream_tool_msg_ids_ordered: list[str] = []
stream_block_index_map: dict[int, str] = {}
_turn_number = 0
_first_event = True
async for message in query(prompt=prompt_stream(), options=options):
if _first_event:
logger.info(f"[MCP-DEBUG] First event received: {type(message).__name__}")
_first_event = False
if isinstance(message, SystemMessage):
raw = message.__dict__ if hasattr(message, '__dict__') else str(message)
logger.info(f"[MCP-DEBUG] SystemMessage: {raw}")
if isinstance(message, StreamEvent):
event = message.event
event_type = event.get("type")
if event_type == "content_block_start":
block = event.get("content_block", {})
index = event.get("index")
block_type = block.get("type")
if block_type == "text":
if stream_text_msg_id is None:
stream_text_msg_id = uuid4().hex
await ws_manager.send_to_session(session_id, "agent:stream_start", {
"session_id": session_id, "message_id": stream_text_msg_id, "role": "assistant",
})
stream_block_index_map[index] = stream_text_msg_id
elif block_type == "tool_use":
tool_msg_id = uuid4().hex
stream_tool_msg_ids_ordered.append(tool_msg_id)
stream_block_index_map[index] = tool_msg_id
await ws_manager.send_to_session(session_id, "agent:stream_start", {
"session_id": session_id, "message_id": tool_msg_id,
"role": "tool_call", "tool_name": block.get("name", ""),
})
elif event_type == "content_block_delta":
index = event.get("index")
delta = event.get("delta", {})
delta_type = delta.get("type")
msg_id = stream_block_index_map.get(index)
if msg_id and delta_type == "text_delta":
await ws_manager.send_to_session(session_id, "agent:stream_delta", {
"session_id": session_id, "message_id": msg_id, "delta": delta.get("text", ""),
})
elif msg_id and delta_type == "input_json_delta":
await ws_manager.send_to_session(session_id, "agent:stream_delta", {
"session_id": session_id, "message_id": msg_id, "delta": delta.get("partial_json", ""),
})
elif event_type == "content_block_stop":
index = event.get("index")
msg_id = stream_block_index_map.get(index)
if msg_id and msg_id != stream_text_msg_id:
await ws_manager.send_to_session(session_id, "agent:stream_end", {
"session_id": session_id, "message_id": msg_id,
})
elif event_type == "message_stop":
if stream_text_msg_id:
await ws_manager.send_to_session(session_id, "agent:stream_end", {
"session_id": session_id, "message_id": stream_text_msg_id,
})
elif isinstance(message, AssistantMessage):
content_parts = []
tool_uses = []
for block in message.content:
if isinstance(block, TextBlock):
content_parts.append(block.text)
elif isinstance(block, ToolUseBlock):
tool_uses.append({"id": block.id, "tool": block.name, "input": block.input})
if content_parts:
asst_msg = Message(
id=stream_text_msg_id or uuid4().hex,
role="assistant", content="\n".join(content_parts),
branch_id=session.active_branch_id,
)
elif bt == "tool_use":
try:
tool_input = json.loads(json_buffers.get(event.index, "{}"))
except json.JSONDecodeError:
tool_input = {}
collected_content.append(ContentBlock(
type="tool_use",
tool_call=ToolCall(
id=tool_ids.get(event.index, uuid4().hex),
name=tool_names.get(event.index, ""),
input=tool_input,
),
))
# Send stream_end for tool blocks (text block ends at message_stop)
if msg_id and bt == "tool_use":
await self.ws_emitter("agent:stream_end", {
"message_id": msg_id,
session.messages.append(asst_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": asst_msg.model_dump(mode="json"),
})
elif event.type == "usage":
# Accumulate token usage from provider stream
for k, v in event.usage.items():
collected_usage[k] = collected_usage.get(k, 0) + v
elif event.type == "message_stop":
# Check if any tool calls means stop_reason is tool_use
has_tool_use = any(b.type == "tool_use" for b in collected_content)
if has_tool_use:
stop_reason = "tool_use"
# End text stream
if stream_text_msg_id:
await self.ws_emitter("agent:stream_end", {
"message_id": stream_text_msg_id,
for i, tu in enumerate(tool_uses):
mid = stream_tool_msg_ids_ordered[i] if i < len(stream_tool_msg_ids_ordered) else uuid4().hex
tool_msg = Message(id=mid, role="tool_call", content=tu, branch_id=session.active_branch_id)
session.messages.append(tool_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": tool_msg.model_dump(mode="json"),
})
# Build and emit the collected messages
await self._emit_collected_messages(
collected_content, stream_text_msg_id, stream_tool_msg_ids,
)
_turn_number += 1
_analytics("turn.completed", {
"turn_number": _turn_number, "tool_calls_in_turn": len(tool_uses), "model": session.model,
}, session_id=session_id, dashboard_id=session.dashboard_id)
return ModelResponse(
content=collected_content,
stop_reason=stop_reason,
usage=collected_usage,
)
stream_text_msg_id = None
stream_tool_msg_ids_ordered = []
stream_block_index_map = {}
async def _emit_collected_messages(
self,
content: list[ContentBlock],
text_msg_id: str | None,
tool_msg_ids: dict[int, str],
) -> None:
"""Emit finalized agent:message events for the collected response."""
from backend.apps.agents.models import Message
elif isinstance(message, ResultMessage):
session.sdk_session_id = getattr(message, "session_id", None)
cost = getattr(message, "total_cost_usd", None)
if cost is not None:
session.cost_usd = cost
await ws_manager.send_to_session(session_id, "agent:cost_update", {
"session_id": session_id, "cost_usd": session.cost_usd,
})
usage = getattr(message, "usage", None) or {}
if isinstance(usage, dict):
inp = usage.get("input_tokens", 0) or 0
out = usage.get("output_tokens", 0) or 0
cache_create = usage.get("cache_creation_input_tokens", 0) or 0
cache_read = usage.get("cache_read_input_tokens", 0) or 0
session.tokens["input"] = inp + cache_create + cache_read
session.tokens["output"] = out
# Emit text message
text_parts = [b.text for b in content if b.type == "text" and b.text]
if text_parts:
msg = Message(
id=text_msg_id or uuid4().hex,
role="assistant",
content="\n".join(text_parts),
)
await self.ws_emitter("agent:message", {
"message": msg.model_dump(mode="json"),
session.status = "completed"
except asyncio.CancelledError:
session.status = "stopped"
except Exception as e:
logger.exception(f"Agent {session_id} error: {e}")
session.status = "error"
_analytics("session.error", {
"error_type": type(e).__name__, "error_message": str(e)[:500],
"model": session.model, "provider": session.provider, "mode": session.mode,
}, session_id=session_id, dashboard_id=session.dashboard_id)
error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id)
session.messages.append(error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": error_msg.model_dump(mode="json"),
})
finally:
if session_id in sessions:
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id, "status": session.status,
"session": session.model_dump(mode="json"),
})
# Emit tool call messages
tool_blocks = [b for b in content if b.type == "tool_use" and b.tool_call]
tool_id_list = sorted(tool_msg_ids.items(), key=lambda x: x[0])
for i, block in enumerate(tool_blocks):
tc = block.tool_call
msg_id = tool_id_list[i][1] if i < len(tool_id_list) else uuid4().hex
msg = Message(
id=msg_id,
role="tool_call",
content={
"id": tc.id,
"tool": tc.name,
"input": tc.input,
},
)
await self.ws_emitter("agent:message", {
"message": msg.model_dump(mode="json"),
})
async def _execute_tools(self, response: ModelResponse) -> list[dict]:
"""Execute all tool calls from a response, respecting HITL permissions.
Returns a list of tool result dicts formatted for the provider.
"""
from backend.apps.agents.models import Message
results = []
for block in response.content:
if block.type != "tool_use" or not block.tool_call:
continue
tc = block.tool_call
start_time = time.time()
# HITL permission check
approved, updated_input = await self.hitl_handler(tc.name, tc.input)
if not approved:
result_content = [{"type": "text", "text": "Tool use was denied by the user."}]
else:
tool_input = updated_input if updated_input else tc.input
try:
result_content = await self.tool_executor(tc.name, tool_input)
except Exception as e:
logger.warning(f"Tool execution error: {tc.name}: {e}")
result_content = [{"type": "text", "text": f"Error executing {tc.name}: {e}"}]
elapsed_ms = int((time.time() - start_time) * 1000)
# Emit tool result to frontend
result_text = ""
for block_item in result_content:
if isinstance(block_item, dict) and block_item.get("type") == "text":
result_text = block_item.get("text", "")
break
result_msg = Message(
role="tool_result",
content={
"text": result_text[:15000] if result_text else "Done.",
"tool_name": tc.name,
"elapsed_ms": elapsed_ms,
},
)
await self.ws_emitter("agent:message", {
"message": result_msg.model_dump(mode="json"),
})
# Format for provider
results.append(
self.provider.format_tool_result(tc.id, result_content)
)
return results
try:
save_session(session_id, session.model_dump(mode="json"))
except Exception as e:
logger.warning(f"Failed to snapshot session {session_id}: {e}")
backend/apps/agents/agent_manager.py
+185 -1718
View File
File diff suppressed because it is too large Load Diff
backend/apps/agents/browser/__init__.py
+3
View File
@@ -0,0 +1,3 @@
"""Browser sub-agent package."""
from backend.apps.agents.browser.runner import run_browser_agent, run_browser_agents
backend/apps/agents/browser/executor.py
+71
View File
@@ -0,0 +1,71 @@
"""Browser tool execution and approval helpers."""
from __future__ import annotations
import asyncio
import json
import logging
from uuid import uuid4
from backend.apps.agents.models import AgentSession, ApprovalRequest
from backend.apps.agents.ws_manager import ws_manager
from backend.apps.agents.browser.schemas import ACTION_MAP
logger = logging.getLogger(__name__)
async def execute_browser_tool(
tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "",
) -> dict:
action = ACTION_MAP.get(tool_name)
if not action:
return {"error": f"Unknown browser tool: {tool_name}"}
params = {k: v for k, v in tool_input.items()}
request_id = uuid4().hex
result = await ws_manager.send_browser_command(
request_id, action, browser_id, params, tab_id=tab_id,
)
return result
def _format_tool_result(result: dict, tool_name: str) -> list[dict]:
if "error" in result:
return [{"type": "text", "text": f"Error: {result['error']}"}]
if tool_name == "BrowserScreenshot" and result.get("image"):
return [
{
"type": "image",
"source": {"type": "base64", "media_type": "image/png", "data": result["image"]},
},
{"type": "text", "text": f"Screenshot captured. URL: {result.get('url', 'unknown')}"},
]
text = result.get("text", json.dumps(result))
return [{"type": "text", "text": str(text)}]
async def _request_browser_approval(
session: AgentSession, tool_name: str, tool_input: dict,
) -> dict:
request_id = uuid4().hex
approval_req = ApprovalRequest(
id=request_id, session_id=session.id,
tool_name=tool_name, tool_input=tool_input,
)
session.pending_approvals.append(approval_req)
session.status = "waiting_approval"
await ws_manager.send_to_session(session.id, "agent:status", {
"session_id": session.id, "status": "waiting_approval",
})
try:
decision = await asyncio.wait_for(
ws_manager.send_approval_request(session.id, request_id, tool_name, tool_input),
timeout=300.0,
)
except asyncio.TimeoutError:
decision = {"behavior": "deny", "message": "Approval timed out"}
session.pending_approvals = [a for a in session.pending_approvals if a.id != request_id]
session.status = "running"
await ws_manager.send_to_session(session.id, "agent:status", {
"session_id": session.id, "status": "running",
})
return decision
backend/apps/agents/browser/runner.py
+234
View File
@@ -0,0 +1,234 @@
"""Browser agent runner — run_browser_agent and run_browser_agents."""
from __future__ import annotations
import asyncio
import logging
import time
from datetime import datetime
from uuid import uuid4
from backend.apps.agents.models import AgentSession, Message
from backend.apps.agents.ws_manager import ws_manager
from backend.apps.common.model_registry import resolve_model_id
from backend.apps.tools_lib.tools_lib import load_builtin_permissions
from backend.apps.agents.browser.schemas import (
BROWSER_TOOLS_SCHEMA, SYSTEM_PROMPT, MAX_TURNS,
)
from backend.apps.agents.browser.executor import (
execute_browser_tool, _format_tool_result, _request_browser_approval,
)
logger = logging.getLogger(__name__)
async def run_browser_agent(
task: str, browser_id: str, model: str,
dashboard_id: str | None = None, tab_id: str = "",
pre_selected: bool = False, initial_url: str | None = None,
parent_session_id: str | None = None,
) -> dict:
from backend.apps.agents.agent_manager import agent_manager
_browser_perms = load_builtin_permissions()
session_id = uuid4().hex
cancel_event = asyncio.Event()
session = AgentSession(
id=session_id, name="Browser Agent", model=model,
mode="browser-agent", status="running", dashboard_id=dashboard_id,
browser_id=browser_id, system_prompt=SYSTEM_PROMPT,
parent_session_id=parent_session_id,
)
session._cancel_event = cancel_event
agent_manager.sessions[session_id] = session
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id, "status": "running",
"session": session.model_dump(mode="json"),
})
if initial_url:
nav_result = await execute_browser_tool("BrowserNavigate", {"url": initial_url}, browser_id, tab_id)
logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}")
api_model = resolve_model_id(model)
from backend.apps.settings.settings import load_settings
from backend.apps.settings.credentials import get_anthropic_client
client = get_anthropic_client(load_settings())
messages: list[dict] = [{"role": "user", "content": task}]
action_log: list[dict] = []
final_screenshot: str | None = None
user_msg = Message(role="user", content=task)
session.messages.append(user_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": user_msg.model_dump(mode="json"),
})
try:
for turn in range(MAX_TURNS):
if cancel_event.is_set():
break
response = await client.messages.create(
model=api_model, max_tokens=4096, system=SYSTEM_PROMPT,
tools=BROWSER_TOOLS_SCHEMA, messages=messages,
)
assistant_content = []
text_parts = []
tool_uses = []
for block in response.content:
if block.type == "text":
text_parts.append(block.text)
assistant_content.append({"type": "text", "text": block.text})
elif block.type == "tool_use":
tool_uses.append(block)
assistant_content.append({"type": "tool_use", "id": block.id, "name": block.name, "input": block.input})
if text_parts:
asst_msg = Message(role="assistant", content="\n".join(text_parts))
session.messages.append(asst_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": asst_msg.model_dump(mode="json"),
})
for tu in tool_uses:
tool_msg = Message(role="tool_call", content={"id": tu.id, "tool": tu.name, "input": tu.input})
session.messages.append(tool_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": tool_msg.model_dump(mode="json"),
})
messages.append({"role": "assistant", "content": assistant_content})
if response.stop_reason != "tool_use":
break
tool_results = []
cancelled = False
for tu in tool_uses:
if cancel_event.is_set():
cancelled = True
break
policy = _browser_perms.get(tu.name, "always_allow")
if policy == "deny":
denied_text = f"Tool {tu.name} is denied by permission policy."
tool_results.append({"type": "tool_result", "tool_use_id": tu.id, "content": [{"type": "text", "text": denied_text}]})
result_msg = Message(role="tool_result", content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0})
session.messages.append(result_msg)
await ws_manager.send_to_session(session_id, "agent:message", {"session_id": session_id, "message": result_msg.model_dump(mode="json")})
continue
if policy == "ask":
decision = await _request_browser_approval(session, tu.name, tu.input)
if decision.get("behavior") == "deny":
denied_text = decision.get("message") or f"Tool {tu.name} denied by user."
tool_results.append({"type": "tool_result", "tool_use_id": tu.id, "content": [{"type": "text", "text": denied_text}]})
result_msg = Message(role="tool_result", content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0})
session.messages.append(result_msg)
await ws_manager.send_to_session(session_id, "agent:message", {"session_id": session_id, "message": result_msg.model_dump(mode="json")})
continue
start = time.time()
result = await execute_browser_tool(tu.name, tu.input, browser_id, tab_id)
elapsed_ms = int((time.time() - start) * 1000)
action_log.append({"tool": tu.name, "input": tu.input, "result_summary": result.get("text", result.get("error", ""))[:200], "elapsed_ms": elapsed_ms})
if tu.name == "BrowserScreenshot" and result.get("image"):
final_screenshot = result["image"]
content_blocks = _format_tool_result(result, tu.name)
tool_results.append({"type": "tool_result", "tool_use_id": tu.id, "content": content_blocks})
result_text = result.get("text", result.get("error", ""))
result_msg = Message(role="tool_result", content={"text": result_text, "tool_name": tu.name, "elapsed_ms": elapsed_ms})
session.messages.append(result_msg)
await ws_manager.send_to_session(session_id, "agent:message", {"session_id": session_id, "message": result_msg.model_dump(mode="json")})
messages.append({"role": "user", "content": tool_results})
if cancelled:
break
if cancel_event.is_set():
session.status = "stopped"
await ws_manager.send_to_session(session_id, "agent:status", {"session_id": session_id, "status": "stopped", "session": session.model_dump(mode="json")})
return {"session_id": session_id, "browser_id": browser_id, "summary": "Agent was stopped.", "action_log": action_log, "final_screenshot": final_screenshot}
summary_parts = text_parts if text_parts else ["Task completed."]
summary = "\n".join(summary_parts)
if not final_screenshot:
try:
ss_result = await execute_browser_tool("BrowserScreenshot", {}, browser_id, tab_id)
if ss_result.get("image"):
final_screenshot = ss_result["image"]
except Exception:
pass
session.status = "completed"
await ws_manager.send_to_session(session_id, "agent:status", {"session_id": session_id, "status": "completed", "session": session.model_dump(mode="json")})
return {"session_id": session_id, "browser_id": browser_id, "summary": summary, "action_log": action_log, "final_screenshot": final_screenshot}
except Exception as e:
logger.exception(f"Browser agent {session_id} error: {e}")
session.status = "error"
error_msg = Message(role="system", content=f"Error: {str(e)}")
session.messages.append(error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {"session_id": session_id, "message": error_msg.model_dump(mode="json")})
await ws_manager.send_to_session(session_id, "agent:status", {"session_id": session_id, "status": "error", "session": session.model_dump(mode="json")})
return {"session_id": session_id, "browser_id": browser_id, "summary": f"Error: {str(e)}", "action_log": action_log, "final_screenshot": None}
async def _create_browser_card(dashboard_id: str, url: str, parent_session_id: str | None = None) -> str:
from backend.apps.dashboards.dashboards import _load, _save
from backend.apps.dashboards.models import BrowserCardPosition, BrowserTab
dashboard = _load(dashboard_id)
browser_id = f"browser-{uuid4().hex[:8]}"
tab_id = f"tab-{uuid4().hex[:8]}"
tab = BrowserTab(id=tab_id, url=url or "https://www.google.com", title="")
card = BrowserCardPosition(
browser_id=browser_id, url=url or "https://www.google.com",
tabs=[tab], activeTabId=tab_id, x=40, y=100, width=1280, height=800,
)
dashboard.layout.browser_cards[browser_id] = card
dashboard.updated_at = datetime.now()
_save(dashboard)
await ws_manager.broadcast_global("dashboard:browser_card_added", {
"dashboard_id": dashboard_id,
"browser_card": card.model_dump(mode="json"),
"parent_session_id": parent_session_id or "",
})
return browser_id
async def run_browser_agents(
tasks: list[dict], model: str,
dashboard_id: str | None = None,
pre_selected_browser_ids: list[str] | None = None,
parent_session_id: str | None = None,
) -> list[dict]:
from backend.apps.analytics.collector import record as _analytics
_analytics("feature.used", {
"feature": "browser_agent.launched", "task_count": len(tasks), "model": model,
}, dashboard_id=dashboard_id)
pre_selected = set(pre_selected_browser_ids or [])
async def _run_one(task_def: dict) -> dict:
browser_id = task_def.get("browser_id", "")
task_text = task_def.get("task", "")
url = task_def.get("url", "")
if not browser_id and dashboard_id:
browser_id = await _create_browser_card(dashboard_id, url, parent_session_id)
await asyncio.sleep(2.0)
is_pre_selected = browser_id in pre_selected
return await run_browser_agent(
task=task_text, browser_id=browser_id, model=model,
dashboard_id=dashboard_id, pre_selected=is_pre_selected,
initial_url=url if url and browser_id not in pre_selected else None,
parent_session_id=parent_session_id,
)
results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True)
final = []
for r in results:
if isinstance(r, Exception):
final.append({"summary": f"Error: {str(r)}", "action_log": [], "final_screenshot": None})
else:
final.append(r)
return final
backend/apps/agents/browser/schemas.py
+137
View File
@@ -0,0 +1,137 @@
"""Browser tool schemas, constants, and system prompt."""
BROWSER_TOOLS_SCHEMA = [
{
"name": "BrowserScreenshot",
"description": (
"Capture a screenshot of the browser page. Returns the screenshot as a "
"base64-encoded PNG image. Use this to see what is currently displayed."
),
"input_schema": {"type": "object", "properties": {}, "required": []},
},
{
"name": "BrowserGetText",
"description": "Get the visible text content of the browser page. Returns up to 15000 characters.",
"input_schema": {"type": "object", "properties": {}, "required": []},
},
{
"name": "BrowserNavigate",
"description": "Navigate the browser to a URL.",
"input_schema": {
"type": "object",
"properties": {"url": {"type": "string", "description": "The URL to navigate to."}},
"required": ["url"],
},
},
{
"name": "BrowserClick",
"description": "Click an element identified by a CSS selector. Use BrowserGetElements first to discover valid selectors.",
"input_schema": {
"type": "object",
"properties": {"selector": {"type": "string", "description": "CSS selector of the element to click."}},
"required": ["selector"],
},
},
{
"name": "BrowserType",
"description": "Type text into an input element. Clears existing value first.",
"input_schema": {
"type": "object",
"properties": {
"selector": {"type": "string", "description": "CSS selector of the input element."},
"text": {"type": "string", "description": "The text to type."},
},
"required": ["selector", "text"],
},
},
{
"name": "BrowserEvaluate",
"description": "Evaluate a JavaScript expression in the browser page and return the result.",
"input_schema": {
"type": "object",
"properties": {"expression": {"type": "string", "description": "JavaScript expression to evaluate."}},
"required": ["expression"],
},
},
{
"name": "BrowserGetElements",
"description": (
"Get a list of interactive elements on the page with CSS selectors. "
"Call this BEFORE clicking or typing so you know which selectors are valid."
),
"input_schema": {
"type": "object",
"properties": {
"selector": {
"type": "string",
"description": "Optional CSS selector to scope the search (e.g. 'form', '#main'). Defaults to 'body'.",
},
},
"required": [],
},
},
{
"name": "BrowserScroll",
"description": (
"Scroll the page up or down. Automatically finds the correct scrollable "
"container. Returns scroll position info including whether top/bottom has been reached."
),
"input_schema": {
"type": "object",
"properties": {
"direction": {"type": "string", "enum": ["up", "down"], "description": "Scroll direction. Defaults to 'down'."},
"amount": {"type": "number", "description": "Pixels to scroll. Defaults to 500."},
},
"required": [],
},
},
{
"name": "BrowserWait",
"description": (
"Wait for a specified duration. Useful after navigation or actions that "
"trigger page loads. Min 100ms, max 10000ms."
),
"input_schema": {
"type": "object",
"properties": {"milliseconds": {"type": "number", "description": "Duration to wait in milliseconds. Defaults to 1000."}},
"required": [],
},
},
]
ACTION_MAP = {
"BrowserScreenshot": "screenshot",
"BrowserGetText": "get_text",
"BrowserNavigate": "navigate",
"BrowserClick": "click",
"BrowserType": "type",
"BrowserEvaluate": "evaluate",
"BrowserGetElements": "get_elements",
"BrowserScroll": "scroll",
"BrowserWait": "wait",
}
SYSTEM_PROMPT = (
"You are a browser automation agent. You control a single browser tab and "
"execute the task you are given.\n\n"
"Strategy:\n"
"1. Start by taking a screenshot to understand the page.\n"
"2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n"
"3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with "
"window.scrollBy() as many sites use nested scroll containers that BrowserScroll "
"handles automatically.\n"
"4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n"
"5. After performing actions, take a screenshot to verify the result.\n"
"6. If an action fails, try alternative selectors or approaches.\n"
"7. When the task is complete, provide a clear summary of what you accomplished.\n\n"
"Important notes:\n"
"- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n"
"- BrowserScroll returns position info including atTop/atBottom — use this to know when "
"you've reached the end of the page.\n"
"- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n"
"- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n"
"You have access ONLY to browser tools. Do not ask the user questions — "
"complete the task autonomously to the best of your ability."
)
MAX_TURNS = 25
backend/apps/agents/browser_agent.py
+7 -625
View File
@@ -1,627 +1,9 @@
"""
Browser sub-agent runner.
"""Backward-compatible shim — re-exports from the browser sub-package."""
Provides a lightweight Anthropic API tool-use loop that drives browser
interactions directly through ws_manager (no MCP subprocess needed).
Sub-agents appear as visible AgentSession cards on the dashboard.
"""
import asyncio
import json
import logging
import time
from datetime import datetime
from uuid import uuid4
import anthropic
from backend.apps.agents.models import AgentSession, ApprovalRequest, Message
from backend.apps.agents.ws_manager import ws_manager
from backend.apps.common.model_registry import resolve_model_id
from backend.apps.tools_lib.tools_lib import load_builtin_permissions
logger = logging.getLogger(__name__)
BROWSER_TOOLS_SCHEMA = [
{
"name": "BrowserScreenshot",
"description": (
"Capture a screenshot of the browser page. Returns the screenshot as a "
"base64-encoded PNG image. Use this to see what is currently displayed."
),
"input_schema": {
"type": "object",
"properties": {},
"required": [],
},
},
{
"name": "BrowserGetText",
"description": (
"Get the visible text content of the browser page. Returns up to 15000 characters."
),
"input_schema": {
"type": "object",
"properties": {},
"required": [],
},
},
{
"name": "BrowserNavigate",
"description": "Navigate the browser to a URL.",
"input_schema": {
"type": "object",
"properties": {
"url": {"type": "string", "description": "The URL to navigate to."},
},
"required": ["url"],
},
},
{
"name": "BrowserClick",
"description": "Click an element identified by a CSS selector. Use BrowserGetElements first to discover valid selectors.",
"input_schema": {
"type": "object",
"properties": {
"selector": {"type": "string", "description": "CSS selector of the element to click."},
},
"required": ["selector"],
},
},
{
"name": "BrowserType",
"description": "Type text into an input element. Clears existing value first.",
"input_schema": {
"type": "object",
"properties": {
"selector": {"type": "string", "description": "CSS selector of the input element."},
"text": {"type": "string", "description": "The text to type."},
},
"required": ["selector", "text"],
},
},
{
"name": "BrowserEvaluate",
"description": "Evaluate a JavaScript expression in the browser page and return the result.",
"input_schema": {
"type": "object",
"properties": {
"expression": {"type": "string", "description": "JavaScript expression to evaluate."},
},
"required": ["expression"],
},
},
{
"name": "BrowserGetElements",
"description": (
"Get a list of interactive elements on the page with CSS selectors. "
"Call this BEFORE clicking or typing so you know which selectors are valid."
),
"input_schema": {
"type": "object",
"properties": {
"selector": {
"type": "string",
"description": "Optional CSS selector to scope the search (e.g. 'form', '#main'). Defaults to 'body'.",
},
},
"required": [],
},
},
{
"name": "BrowserScroll",
"description": (
"Scroll the page up or down. Automatically finds the correct scrollable "
"container (works on SPAs like Notion, Gmail, etc. that use nested scroll "
"containers instead of window-level scrolling). Returns scroll position info "
"including whether top/bottom has been reached."
),
"input_schema": {
"type": "object",
"properties": {
"direction": {
"type": "string",
"enum": ["up", "down"],
"description": "Scroll direction. Defaults to 'down'.",
},
"amount": {
"type": "number",
"description": "Pixels to scroll. Defaults to 500.",
},
},
"required": [],
},
},
{
"name": "BrowserWait",
"description": (
"Wait for a specified duration. Useful after navigation or actions that "
"trigger page loads, animations, or async content rendering. "
"Min 100ms, max 10000ms."
),
"input_schema": {
"type": "object",
"properties": {
"milliseconds": {
"type": "number",
"description": "Duration to wait in milliseconds. Defaults to 1000.",
},
},
"required": [],
},
},
]
ACTION_MAP = {
"BrowserScreenshot": "screenshot",
"BrowserGetText": "get_text",
"BrowserNavigate": "navigate",
"BrowserClick": "click",
"BrowserType": "type",
"BrowserEvaluate": "evaluate",
"BrowserGetElements": "get_elements",
"BrowserScroll": "scroll",
"BrowserWait": "wait",
}
SYSTEM_PROMPT = (
"You are a browser automation agent. You control a single browser tab and "
"execute the task you are given.\n\n"
"Strategy:\n"
"1. Start by taking a screenshot to understand the page.\n"
"2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n"
"3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with "
"window.scrollBy() as many sites use nested scroll containers that BrowserScroll "
"handles automatically.\n"
"4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n"
"5. After performing actions, take a screenshot to verify the result.\n"
"6. If an action fails, try alternative selectors or approaches.\n"
"7. When the task is complete, provide a clear summary of what you accomplished.\n\n"
"Important notes:\n"
"- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n"
"- BrowserScroll returns position info including atTop/atBottom — use this to know when "
"you've reached the end of the page.\n"
"- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n"
"- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n"
"You have access ONLY to browser tools. Do not ask the user questions — "
"complete the task autonomously to the best of your ability."
from backend.apps.agents.browser.runner import ( # noqa: F401
run_browser_agent,
run_browser_agents,
)
from backend.apps.agents.browser.executor import ( # noqa: F401
execute_browser_tool,
)
MAX_TURNS = 25
async def execute_browser_tool(
tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "",
) -> dict:
"""Execute a browser tool via ws_manager directly (no MCP/HTTP round-trip)."""
action = ACTION_MAP.get(tool_name)
if not action:
return {"error": f"Unknown browser tool: {tool_name}"}
params = {k: v for k, v in tool_input.items()}
request_id = uuid4().hex
result = await ws_manager.send_browser_command(
request_id, action, browser_id, params, tab_id=tab_id,
)
return result
def _format_tool_result(result: dict, tool_name: str) -> list[dict]:
"""Convert a browser command result dict into Anthropic API content blocks."""
if "error" in result:
return [{"type": "text", "text": f"Error: {result['error']}"}]
if tool_name == "BrowserScreenshot" and result.get("image"):
blocks = [
{
"type": "image",
"source": {
"type": "base64",
"media_type": "image/png",
"data": result["image"],
},
},
{"type": "text", "text": f"Screenshot captured. URL: {result.get('url', 'unknown')}"},
]
return blocks
text = result.get("text", json.dumps(result))
return [{"type": "text", "text": str(text)}]
async def _request_browser_approval(
session: AgentSession, tool_name: str, tool_input: dict,
) -> dict:
"""Send an approval request for a browser sub-agent tool and wait for the decision."""
request_id = uuid4().hex
approval_req = ApprovalRequest(
id=request_id,
session_id=session.id,
tool_name=tool_name,
tool_input=tool_input,
)
session.pending_approvals.append(approval_req)
session.status = "waiting_approval"
await ws_manager.send_to_session(session.id, "agent:status", {
"session_id": session.id,
"status": "waiting_approval",
})
try:
decision = await asyncio.wait_for(
ws_manager.send_approval_request(
session.id, request_id, tool_name, tool_input,
),
timeout=300.0,
)
except asyncio.TimeoutError:
decision = {"behavior": "deny", "message": "Approval timed out"}
session.pending_approvals = [
a for a in session.pending_approvals if a.id != request_id
]
session.status = "running"
await ws_manager.send_to_session(session.id, "agent:status", {
"session_id": session.id,
"status": "running",
})
return decision
async def run_browser_agent(
task: str,
browser_id: str,
model: str,
dashboard_id: str | None = None,
tab_id: str = "",
pre_selected: bool = False,
initial_url: str | None = None,
parent_session_id: str | None = None,
) -> dict:
"""Run a browser sub-agent loop for a single browser card.
Creates a visible AgentSession, streams progress via WebSocket,
and returns the full action log + summary + final screenshot.
"""
from backend.apps.agents.agent_manager import agent_manager
_browser_perms = load_builtin_permissions()
session_id = uuid4().hex
cancel_event = asyncio.Event()
session = AgentSession(
id=session_id,
name=f"Browser Agent",
model=model,
mode="browser-agent",
status="running",
dashboard_id=dashboard_id,
browser_id=browser_id,
system_prompt=SYSTEM_PROMPT,
parent_session_id=parent_session_id,
)
session._cancel_event = cancel_event
agent_manager.sessions[session_id] = session
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id,
"status": "running",
"session": session.model_dump(mode="json"),
})
if initial_url:
nav_result = await execute_browser_tool(
"BrowserNavigate", {"url": initial_url}, browser_id, tab_id,
)
logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}")
api_model = resolve_model_id(model)
from backend.apps.settings.settings import load_settings
from backend.apps.settings.credentials import get_anthropic_client
client = get_anthropic_client(load_settings())
messages: list[dict] = [{"role": "user", "content": task}]
action_log: list[dict] = []
final_screenshot: str | None = None
user_msg = Message(role="user", content=task)
session.messages.append(user_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": user_msg.model_dump(mode="json"),
})
try:
for turn in range(MAX_TURNS):
if cancel_event.is_set():
break
response = await client.messages.create(
model=api_model,
max_tokens=4096,
system=SYSTEM_PROMPT,
tools=BROWSER_TOOLS_SCHEMA,
messages=messages,
)
assistant_content = []
text_parts = []
tool_uses = []
for block in response.content:
if block.type == "text":
text_parts.append(block.text)
assistant_content.append({"type": "text", "text": block.text})
elif block.type == "tool_use":
tool_uses.append(block)
assistant_content.append({
"type": "tool_use",
"id": block.id,
"name": block.name,
"input": block.input,
})
if text_parts:
asst_msg = Message(
role="assistant",
content="\n".join(text_parts),
)
session.messages.append(asst_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": asst_msg.model_dump(mode="json"),
})
for tu in tool_uses:
tool_msg = Message(
role="tool_call",
content={"id": tu.id, "tool": tu.name, "input": tu.input},
)
session.messages.append(tool_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": tool_msg.model_dump(mode="json"),
})
messages.append({"role": "assistant", "content": assistant_content})
if response.stop_reason != "tool_use":
break
tool_results = []
cancelled = False
for tu in tool_uses:
if cancel_event.is_set():
cancelled = True
break
policy = _browser_perms.get(tu.name, "always_allow")
if policy == "deny":
denied_text = f"Tool {tu.name} is denied by permission policy."
tool_results.append({
"type": "tool_result",
"tool_use_id": tu.id,
"content": [{"type": "text", "text": denied_text}],
})
result_msg = Message(
role="tool_result",
content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0},
)
session.messages.append(result_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": result_msg.model_dump(mode="json"),
})
continue
if policy == "ask":
decision = await _request_browser_approval(
session, tu.name, tu.input,
)
if decision.get("behavior") == "deny":
denied_text = decision.get("message") or f"Tool {tu.name} denied by user."
tool_results.append({
"type": "tool_result",
"tool_use_id": tu.id,
"content": [{"type": "text", "text": denied_text}],
})
result_msg = Message(
role="tool_result",
content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0},
)
session.messages.append(result_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": result_msg.model_dump(mode="json"),
})
continue
start = time.time()
result = await execute_browser_tool(
tu.name, tu.input, browser_id, tab_id,
)
elapsed_ms = int((time.time() - start) * 1000)
action_log.append({
"tool": tu.name,
"input": tu.input,
"result_summary": result.get("text", result.get("error", ""))[:200],
"elapsed_ms": elapsed_ms,
})
if tu.name == "BrowserScreenshot" and result.get("image"):
final_screenshot = result["image"]
content_blocks = _format_tool_result(result, tu.name)
tool_results.append({
"type": "tool_result",
"tool_use_id": tu.id,
"content": content_blocks,
})
result_text = result.get("text", result.get("error", ""))
result_msg = Message(
role="tool_result",
content={"text": result_text, "tool_name": tu.name, "elapsed_ms": elapsed_ms},
)
session.messages.append(result_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": result_msg.model_dump(mode="json"),
})
messages.append({"role": "user", "content": tool_results})
if cancelled:
break
if cancel_event.is_set():
session.status = "stopped"
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id,
"status": "stopped",
"session": session.model_dump(mode="json"),
})
return {
"session_id": session_id,
"browser_id": browser_id,
"summary": "Agent was stopped.",
"action_log": action_log,
"final_screenshot": final_screenshot,
}
summary_parts = text_parts if text_parts else ["Task completed."]
summary = "\n".join(summary_parts)
if not final_screenshot:
try:
ss_result = await execute_browser_tool(
"BrowserScreenshot", {}, browser_id, tab_id,
)
if ss_result.get("image"):
final_screenshot = ss_result["image"]
except Exception:
pass
session.status = "completed"
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id,
"status": "completed",
"session": session.model_dump(mode="json"),
})
return {
"session_id": session_id,
"browser_id": browser_id,
"summary": summary,
"action_log": action_log,
"final_screenshot": final_screenshot,
}
except Exception as e:
logger.exception(f"Browser agent {session_id} error: {e}")
session.status = "error"
error_msg = Message(role="system", content=f"Error: {str(e)}")
session.messages.append(error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
})
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id,
"status": "error",
"session": session.model_dump(mode="json"),
})
return {
"session_id": session_id,
"browser_id": browser_id,
"summary": f"Error: {str(e)}",
"action_log": action_log,
"final_screenshot": None,
}
async def _create_browser_card(dashboard_id: str, url: str, parent_session_id: str | None = None) -> str:
"""Create a new browser card on the dashboard and return its browser_id."""
from backend.apps.dashboards.dashboards import _load, _save
from backend.apps.dashboards.models import BrowserCardPosition, BrowserTab
dashboard = _load(dashboard_id)
browser_id = f"browser-{uuid4().hex[:8]}"
tab_id = f"tab-{uuid4().hex[:8]}"
tab = BrowserTab(id=tab_id, url=url or "https://www.google.com", title="")
card = BrowserCardPosition(
browser_id=browser_id,
url=url or "https://www.google.com",
tabs=[tab],
activeTabId=tab_id,
x=40,
y=100,
width=1280,
height=800,
)
dashboard.layout.browser_cards[browser_id] = card
dashboard.updated_at = datetime.now()
_save(dashboard)
await ws_manager.broadcast_global("dashboard:browser_card_added", {
"dashboard_id": dashboard_id,
"browser_card": card.model_dump(mode="json"),
"parent_session_id": parent_session_id or "",
})
return browser_id
async def run_browser_agents(
tasks: list[dict],
model: str,
dashboard_id: str | None = None,
pre_selected_browser_ids: list[str] | None = None,
parent_session_id: str | None = None,
) -> list[dict]:
"""Run multiple browser sub-agents in parallel.
Each task dict has: { browser_id (optional), task, url (optional) }
Returns a list of result dicts, one per task.
"""
from backend.apps.analytics.collector import record as _analytics
_analytics("feature.used", {
"feature": "browser_agent.launched",
"task_count": len(tasks),
"model": model,
}, dashboard_id=dashboard_id)
pre_selected = set(pre_selected_browser_ids or [])
async def _run_one(task_def: dict) -> dict:
browser_id = task_def.get("browser_id", "")
task_text = task_def.get("task", "")
url = task_def.get("url", "")
if not browser_id and dashboard_id:
browser_id = await _create_browser_card(dashboard_id, url, parent_session_id)
await asyncio.sleep(2.0)
is_pre_selected = browser_id in pre_selected
return await run_browser_agent(
task=task_text,
browser_id=browser_id,
model=model,
dashboard_id=dashboard_id,
pre_selected=is_pre_selected,
initial_url=url if url and browser_id not in pre_selected else None,
parent_session_id=parent_session_id,
)
results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True)
final = []
for r in results:
if isinstance(r, Exception):
final.append({"summary": f"Error: {str(r)}", "action_log": [], "final_screenshot": None})
else:
final.append(r)
return final
backend/apps/agents/mcp_builder.py
+135
View File
@@ -0,0 +1,135 @@
"""MCP server building, tool-policy resolution, and tool-name helpers.
Extracted from AgentManager to keep each module focused on a single concern.
"""
from __future__ import annotations
import logging
import re as _re
from backend.apps.common.mcp_utils import sanitize_server_name as _sanitize_server_name
from backend.apps.tools_lib.tools_lib import (
_load_all as load_all_tools,
derive_mcp_config,
load_builtin_permissions,
refresh_google_token,
)
logger = logging.getLogger(__name__)
FULL_TOOLS = [
"Read", "Edit", "Write", "Bash", "Glob", "Grep", "AskUserQuestion",
"WebSearch", "WebFetch", "NotebookEdit", "TodoWrite",
"EnterPlanMode", "ExitPlanMode", "EnterWorktree",
"TaskOutput", "TaskStop",
"CronCreate", "CronList", "CronDelete",
"RenderOutput",
"InvokeAgent",
"Agent",
]
def _get_denied_tool_names(tool) -> set[str]:
"""Return the set of MCP sub-tool names whose permission is 'deny'."""
return {
key for key, value in tool.tool_permissions.items()
if not key.startswith("_") and value == "deny"
}
def _get_all_known_tool_names(tool) -> set[str]:
"""Return all known sub-tool names for an MCP tool."""
return set(tool.tool_permissions.get("_tool_descriptions", {}).keys())
def _is_fully_denied(tool) -> bool:
"""True when every known sub-tool on this MCP server is set to 'deny'."""
known = _get_all_known_tool_names(tool)
if not known:
return False
return known <= _get_denied_tool_names(tool)
def get_all_tool_names() -> list[str]:
"""FULL_TOOLS + installed MCP tool identifiers (mcp:<tool_name>).
Builtin tools set to 'deny' and MCP servers whose every sub-tool
is denied are excluded.
"""
builtin_perms = load_builtin_permissions()
builtin_tools = [
t for t in FULL_TOOLS
if builtin_perms.get(t, "always_allow") != "deny"
]
mcp_names = [
f"mcp:{t.name}"
for t in load_all_tools()
if t.mcp_config
and t.enabled
and t.auth_status in ("configured", "connected")
and not _is_fully_denied(t)
]
return builtin_tools + mcp_names
async def build_mcp_servers(allowed_tools: list[str]) -> dict:
"""Build the mcp_servers dict for ClaudeAgentOptions from installed MCP tools."""
mcp_servers: dict = {}
all_tools = load_all_tools()
mcp_tools = [t for t in all_tools if t.mcp_config and t.enabled and t.auth_status in ("configured", "connected")]
logger.info(f"[MCP-DEBUG] Building MCP servers. {len(mcp_tools)} MCP tools found, allowed_tools has {len(allowed_tools)} entries")
for tool in mcp_tools:
tool_ref = f"mcp:{tool.name}"
if tool_ref not in allowed_tools and allowed_tools != get_all_tool_names():
if not any(tool_ref == at for at in allowed_tools):
logger.info(f"[MCP-DEBUG] SKIPPED {tool.name}: '{tool_ref}' not in allowed_tools")
continue
if _is_fully_denied(tool):
logger.info(f"[MCP-DEBUG] SKIPPED {tool.name}: fully denied")
continue
if tool.auth_type == "oauth2" and tool.auth_status == "connected":
refreshed = await refresh_google_token(tool)
logger.info(f"[MCP-DEBUG] {tool.name} token refresh: {'OK' if refreshed else 'FAILED'}")
config = derive_mcp_config(tool)
if config:
server_name = _sanitize_server_name(tool.name)
mcp_servers[server_name] = config
env_keys = list(config.get("env", {}).keys())
logger.info(f"[MCP-DEBUG] ADDED {server_name}: command={config.get('command')}, args={config.get('args')}, env_keys={env_keys}")
else:
logger.warning(f"[MCP-DEBUG] {tool.name}: derive_mcp_config returned None")
logger.info(f"[MCP-DEBUG] Final mcp_servers: {list(mcp_servers.keys())}")
return mcp_servers
def get_effective_policy(
tool_name: str,
builtin_perms: dict[str, str],
) -> str:
"""Return 'always_allow', 'deny', or 'ask' for any tool."""
if tool_name in builtin_perms:
return builtin_perms[tool_name]
bm = _re.match(r"mcp__openswarm-browser-agent__(.+)", tool_name)
if bm:
return builtin_perms.get(bm.group(1), "always_allow")
im = _re.match(r"mcp__openswarm-invoke-agent__(.+)", tool_name)
if im:
return builtin_perms.get(im.group(1), "always_allow")
m = _re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name)
if m:
server_slug, mcp_tool_name = m.group(1), m.group(2)
for t in load_all_tools():
if not t.mcp_config or not t.enabled:
continue
if _sanitize_server_name(t.name) == server_slug:
return t.tool_permissions.get(mcp_tool_name, "ask")
return "always_allow"
backend/apps/agents/prompt_builder.py
+324
View File
@@ -0,0 +1,324 @@
"""Prompt-building helpers extracted from AgentManager.
All functions are stateless — they accept data as parameters instead of
relying on ``self``.
"""
from __future__ import annotations
import json as _json
import logging
import os
from typing import Any
from backend.apps.agents.models import AgentSession
from backend.apps.modes.modes import load_mode
from backend.apps.outputs.outputs import _load_all as load_all_outputs
from backend.apps.common.mcp_utils import sanitize_server_name as _sanitize_server_name
logger = logging.getLogger(__name__)
def resolve_mode(
mode_id: str,
get_all_tool_names_fn,
) -> tuple[list[str], str | None, str | None]:
"""Return (tools, system_prompt, default_folder) from the mode store."""
mode_def = load_mode(mode_id)
if mode_def:
tools = mode_def.tools if mode_def.tools is not None else get_all_tool_names_fn()
return tools, mode_def.system_prompt, mode_def.default_folder
return get_all_tool_names_fn(), None, None
def compose_system_prompt(
default_prompt: str | None,
mode_prompt: str | None,
session_prompt: str | None,
connected_tools_ctx: str | None = None,
outputs_ctx: str | None = None,
browser_ctx: str | None = None,
) -> str | None:
parts = [p for p in (default_prompt, mode_prompt, session_prompt,
connected_tools_ctx, outputs_ctx, browser_ctx) if p]
return "\n\n".join(parts) if parts else None
def build_connected_tools_context(
allowed_tools: list[str],
load_all_tools_fn,
get_all_tool_names_fn,
is_fully_denied_fn,
get_denied_tool_names_fn,
) -> str | None:
all_tools = load_all_tools_fn()
mcp_tools = [t for t in all_tools if t.mcp_config and t.enabled and t.auth_status in ("configured", "connected")]
sections: list[str] = []
for tool in mcp_tools:
tool_ref = f"mcp:{tool.name}"
if tool_ref not in allowed_tools and allowed_tools != get_all_tool_names_fn():
continue
if is_fully_denied_fn(tool):
continue
server_name = _sanitize_server_name(tool.name)
denied = get_denied_tool_names_fn(tool)
tool_descs = {
k: v for k, v in tool.tool_permissions.get("_tool_descriptions", {}).items()
if k not in denied
}
if not tool_descs:
continue
lines = [f"MCP Server: {server_name}"]
lines.append(f" Status: {tool.auth_status}")
if tool.connected_account_email:
lines.append(f" Connected account: {tool.connected_account_email}")
lines.append(
f" IMPORTANT: When calling tools from this server that require an email "
f"parameter (e.g. user_google_email, user_email), always use "
f"\"{tool.connected_account_email}\" automatically — do NOT ask the user."
)
tool_names = list(tool_descs.keys())
if tool_names:
lines.append(f" Available tools ({len(tool_names)}): {', '.join(tool_names)}")
sections.append("\n".join(lines))
not_connected = [
t for t in all_tools
if t.mcp_config and t.enabled
and t.auth_type in ("oauth2", "env_vars")
and t.auth_status != "connected"
]
if not_connected:
nc_lines = ["Tools installed but not yet connected (user needs to authorize in Settings → Tools):"]
for t in not_connected:
nc_lines.append(f" - {t.name}")
sections.append("\n".join(nc_lines))
if not sections:
return None
return (
"<connected_mcp_tools>\n"
"The following MCP tool servers are connected and available. "
"Use them directly when relevant to the user's request.\n\n"
+ "\n\n".join(sections)
+ "\n</connected_mcp_tools>"
)
def build_outputs_context() -> str | None:
all_outputs = load_all_outputs()
if not all_outputs:
return None
sections: list[str] = []
for out in all_outputs:
lines = [f"- **{out.name}** (id: `{out.id}`)"]
if out.description:
lines.append(f" Description: {out.description}")
schema_str = _json.dumps(out.input_schema, indent=2)
lines.append(f" Input schema:\n```json\n{schema_str}\n```")
sections.append("\n".join(lines))
return (
"<available_views>\n"
"The following reusable View artifacts are available. "
"Use the RenderOutput tool to invoke one by providing its output_id "
"and the required input_data matching its schema.\n\n"
+ "\n\n".join(sections)
+ "\n</available_views>"
)
def build_browser_context(
dashboard_id: str | None,
selected_browser_ids: list[str] | None = None,
) -> str | None:
if not dashboard_id:
return None
try:
from backend.apps.dashboards.dashboards import _load as load_dashboard
dashboard = load_dashboard(dashboard_id)
except Exception:
return None
raw = dashboard.model_dump(mode="json")
browser_cards = raw.get("layout", {}).get("browser_cards", {})
lines = [
"<browser_agent_instructions>",
"You have access to browser automation through the CreateBrowserAgent, BrowserAgent, and BrowserAgents tools.",
"",
"- **CreateBrowserAgent(task, url?)**: Create a new browser card and run a task on it. "
"Use this when you need a fresh browser. Optionally provide a starting URL.",
"- **BrowserAgent(browser_id, task)**: Delegate a task to an existing browser card. "
"The browser agent will autonomously navigate, click, type, and interact with the page, then return a summary and screenshot.",
"- **BrowserAgents(tasks)**: Run multiple browser tasks in parallel on existing browser cards. "
"Each task requires a browser_id.",
"",
"You do NOT have direct access to low-level browser tools (click, type, screenshot, etc.). "
"Instead, describe what you want accomplished and the browser agent will handle the details.",
]
if browser_cards and selected_browser_ids:
visible_cards = [
card for card in browser_cards.values()
if card.get("browser_id", "") in selected_browser_ids
]
if visible_cards:
lines.append("")
lines.append("The user selected these browser cards for you to work with:")
for card in visible_cards:
bid = card.get("browser_id", "")
tabs = card.get("tabs", [])
active_tab_id = card.get("activeTabId", "")
active_tab = next((t for t in tabs if t.get("id") == active_tab_id), None)
url = (active_tab or {}).get("url", card.get("url", ""))
title = (active_tab or {}).get("title", "")
lines.append(f"- browser_id: \"{bid}\"")
if title:
lines.append(f" Title: {title}")
if url:
lines.append(f" URL: {url}")
lines.append("</browser_agent_instructions>")
return "\n".join(lines)
def get_pre_selected_browser_ids(dashboard_id: str | None) -> list[str]:
if not dashboard_id:
return []
try:
from backend.apps.dashboards.dashboards import _load as load_dashboard
dashboard = load_dashboard(dashboard_id)
except Exception:
return []
raw = dashboard.model_dump(mode="json")
browser_cards = raw.get("layout", {}).get("browser_cards", {})
return [card.get("browser_id", "") for card in browser_cards.values() if card.get("browser_id")]
def resolve_context_paths(context_paths: list | None) -> str:
if not context_paths:
return ""
sections: list[str] = []
for cp in context_paths:
path = cp.get("path", "")
cp_type = cp.get("type", "file")
if not path or not os.path.exists(path):
sections.append(f"[Context: {path} — not found]")
continue
if cp_type == "file" and os.path.isfile(path):
try:
with open(path, "r", errors="replace") as f:
content = f.read(512_000)
sections.append(f"<context_file path=\"{path}\">\n{content}\n</context_file>")
except Exception as e:
sections.append(f"[Context: {path} — error reading: {e}]")
elif cp_type == "directory" and os.path.isdir(path):
tree_lines = build_dir_tree(path, max_depth=4)
sections.append(f"<context_directory path=\"{path}\">\n{chr(10).join(tree_lines)}\n</context_directory>")
else:
sections.append(f"[Context: {path} — type mismatch]")
return "\n\n".join(sections)
def build_dir_tree(root: str, max_depth: int = 4, prefix: str = "") -> list[str]:
lines: list[str] = []
try:
entries = sorted(os.listdir(root))
except PermissionError:
return [f"{prefix}[permission denied]"]
dirs = [e for e in entries if not e.startswith(".") and os.path.isdir(os.path.join(root, e))]
files = [e for e in entries if not e.startswith(".") and os.path.isfile(os.path.join(root, e))]
for f in files:
lines.append(f"{prefix}{f}")
for d in dirs:
lines.append(f"{prefix}{d}/")
if max_depth > 1:
sub = build_dir_tree(os.path.join(root, d), max_depth - 1, prefix + " ")
lines.extend(sub)
return lines
def resolve_forced_tools(
forced_tools: list[str] | None,
load_all_tools_fn,
) -> str:
if not forced_tools:
return ""
from backend.apps.tools_lib.models import BUILTIN_TOOLS
desc_map: dict[str, str] = {t.name: t.description for t in BUILTIN_TOOLS}
tool_to_server: dict[str, str] = {}
tool_to_email: dict[str, str] = {}
for t in load_all_tools_fn():
if not t.enabled or not t.tool_permissions:
continue
tool_descs = t.tool_permissions.get("_tool_descriptions", {})
server_name = _sanitize_server_name(t.name)
for tn, td in tool_descs.items():
desc_map[tn] = td
tool_to_server[tn] = server_name
if t.connected_account_email:
tool_to_email[tn] = t.connected_account_email
lines: list[str] = []
for name in forced_tools:
desc = desc_map.get(name, "")
line = f"- {name}: {desc}" if desc else f"- {name}"
server = tool_to_server.get(name)
if server:
line += f"\n (MCP server: {server})"
email = tool_to_email.get(name)
if email:
line += f"\n (connected account: {email} — use this for any email parameter)"
lines.append(line)
return (
"<forced_tools>\n"
"The user explicitly requested these tools be used. "
"Prioritize using them to address the user's request.\n"
+ "\n".join(lines)
+ "\n</forced_tools>"
)
def resolve_attached_skills(attached_skills: list | None) -> str:
if not attached_skills:
return ""
sections: list[str] = []
for skill in attached_skills:
name = skill.get("name", "Unknown")
content = skill.get("content", "")
if content:
sections.append(f"[Using skill: {name}]\n\n{content}")
return "\n\n".join(sections)
def build_prompt_content(
prompt: str,
images: list | None = None,
context_paths: list | None = None,
forced_tools: list[str] | None = None,
attached_skills: list | None = None,
load_all_tools_fn=None,
):
context_text = resolve_context_paths(context_paths)
forced_tools_text = resolve_forced_tools(forced_tools, load_all_tools_fn)
skills_text = resolve_attached_skills(attached_skills)
parts = [p for p in (forced_tools_text, context_text, skills_text, prompt) if p]
full_prompt = "\n\n".join(parts)
if not images:
return full_prompt
content: list[dict[str, Any]] = [{"type": "text", "text": full_prompt}]
for img in images:
content.append({
"type": "image",
"source": {
"type": "base64",
"media_type": img.get("media_type", "image/png"),
"data": img["data"],
},
})
return content
backend/apps/agents/session_store.py
+156
View File
@@ -0,0 +1,156 @@
"""Session persistence, history queries, and message-copying helpers.
Uses ``SessionStore`` from ``backend.apps.common.json_store`` for on-disk
JSON CRUD and exposes higher-level helpers consumed by ``AgentManager``.
"""
from __future__ import annotations
import logging
from datetime import datetime
from uuid import uuid4
from backend.apps.agents.models import AgentSession, Message, MessageBranch
from backend.apps.common.json_store import SessionStore
from backend.config.paths import SESSIONS_DIR
logger = logging.getLogger(__name__)
_session_store = SessionStore(SESSIONS_DIR)
save_session = _session_store.save
load_session_data = _session_store.load
delete_session_file = _session_store.delete
load_all_session_data = _session_store.load_all
def build_search_text(session: AgentSession, max_len: int = 5000) -> str:
"""Build a search-indexing string from session name and message content."""
parts = [session.name or ""]
for msg in session.messages:
if msg.role in ("user", "assistant") and isinstance(msg.content, str):
parts.append(msg.content)
text = " ".join(parts)
return text[:max_len]
def get_history(
q: str = "",
limit: int = 20,
offset: int = 0,
dashboard_id: str | None = None,
) -> dict:
"""Return paginated, optionally filtered summaries of closed sessions."""
all_data = load_all_session_data()
all_data.sort(key=lambda pair: pair[1].get("closed_at") or "", reverse=True)
q_lower = q.strip().lower()
history: list[dict] = []
for sid, data in all_data:
if dashboard_id and data.get("dashboard_id") != dashboard_id:
continue
if q_lower:
name = (data.get("name") or "").lower()
search_text = (data.get("search_text") or "").lower()
if q_lower not in name and q_lower not in search_text:
continue
history.append({
"id": data.get("id", sid),
"name": data.get("name", "Untitled"),
"status": data.get("status", "stopped"),
"model": data.get("model", "sonnet"),
"mode": data.get("mode", "agent"),
"created_at": data.get("created_at"),
"closed_at": data.get("closed_at"),
"cost_usd": data.get("cost_usd", 0),
"dashboard_id": data.get("dashboard_id"),
})
total = len(history)
page = history[offset : offset + limit]
return {
"sessions": page,
"total": total,
"has_more": offset + limit < total,
}
async def reconcile_on_startup() -> None:
"""Mark any stale running sessions as stopped."""
for sid, data in load_all_session_data():
if data.get("status") in ("running", "waiting_approval"):
data["status"] = "stopped"
save_session(sid, data)
logger.info(f"Marked stale session {sid} as stopped")
def get_browser_agent_children(
sessions: dict[str, AgentSession],
parent_session_id: str,
) -> list[dict]:
"""Return browser-agent sessions for a parent, from memory or disk."""
results: list[dict] = []
seen: set[str] = set()
for s in sessions.values():
if s.mode == "browser-agent" and s.parent_session_id == parent_session_id:
results.append(s.model_dump(mode="json"))
seen.add(s.id)
for sid, data in load_all_session_data():
if sid in seen:
continue
if data.get("mode") == "browser-agent" and data.get("parent_session_id") == parent_session_id:
results.append(data)
return results
def copy_session_messages(
source: AgentSession,
up_to_message_id: str | None = None,
) -> tuple[list[Message], dict[str, MessageBranch], dict[str, str]]:
"""Deep-copy messages and branches from *source*, returning new IDs.
Returns ``(new_messages, new_branches, old_to_new_msg_id_map)``.
"""
source_messages = list(source.messages)
if up_to_message_id:
cut_idx = next(
(i for i, m in enumerate(source_messages) if m.id == up_to_message_id),
None,
)
if cut_idx is not None:
source_messages = source_messages[: cut_idx + 1]
old_to_new: dict[str, str] = {}
new_messages: list[Message] = []
for msg in source_messages:
new_id = uuid4().hex
old_to_new[msg.id] = new_id
new_messages.append(Message(
id=new_id,
role=msg.role,
content=msg.content,
timestamp=msg.timestamp,
branch_id=msg.branch_id,
parent_id=old_to_new.get(msg.parent_id) if msg.parent_id else None,
context_paths=msg.context_paths,
attached_skills=msg.attached_skills,
forced_tools=msg.forced_tools,
images=msg.images,
))
new_branches: dict[str, MessageBranch] = {}
for bid, branch in source.branches.items():
new_branches[bid] = MessageBranch(
id=bid,
parent_branch_id=branch.parent_branch_id,
fork_point_message_id=(
old_to_new.get(branch.fork_point_message_id)
if branch.fork_point_message_id else None
),
created_at=branch.created_at,
)
return new_messages, new_branches, old_to_new
backend/apps/outputs/ai_generation.py
+223
View File
@@ -0,0 +1,223 @@
"""AI-powered endpoints: vibe-code, auto-run, auto-run-agent."""
from __future__ import annotations
import json
import logging
from backend.apps.outputs.helpers import _validate_against_schema
from backend.apps.outputs.executor import execute_backend_code
from backend.apps.common.model_registry import resolve_model_id as _resolve_model
from backend.apps.outputs.models import (
VibeCodeRequest, AutoRunRequest, AutoRunAgentRequest,
)
logger = logging.getLogger(__name__)
def _get_anthropic_client():
from backend.apps.settings.credentials import get_anthropic_client
from backend.apps.settings.settings import load_settings
return get_anthropic_client(load_settings())
VIBE_CODE_SYSTEM_PROMPT = """\
You are an expert at building self-contained HTML/JS/CSS applications that run in an iframe.
The user will describe what they want, and you will generate:
1. **frontend_code**: A complete HTML document. React 18 is available via esm.sh CDN.
- Use: <script type="importmap">{"imports":{"react":"https://esm.sh/react@18","react-dom/client":"https://esm.sh/react-dom@18/client"}}</script>
- Input data is at window.OUTPUT_INPUT (object), backend result at window.OUTPUT_BACKEND_RESULT.
2. **input_schema**: A JSON Schema object defining the structured input.
3. **backend_code** (optional): Python code where input_data is a global dict and result is a global dict to assign to.
4. **name**: A short name for the view.
5. **description**: A one-sentence description.
6. **message**: A brief explanation of what you did/changed.
Return ONLY valid JSON with these keys. No markdown fences, no extra text.\
"""
async def vibe_code(body: VibeCodeRequest):
from backend.apps.analytics.collector import record as _analytics
_analytics("feature.used", {"feature": "vibe_code.used"})
try:
import anthropic
except ImportError:
return {
"message": "anthropic SDK not installed. Install with: pip install anthropic",
"frontend_code": body.current_frontend_code,
"backend_code": body.current_backend_code,
"input_schema": body.current_schema,
}
context_parts = []
if body.current_frontend_code:
context_parts.append(f"Current frontend code:\n```html\n{body.current_frontend_code}\n```")
if body.current_backend_code:
context_parts.append(f"Current backend code:\n```python\n{body.current_backend_code}\n```")
if body.current_schema:
context_parts.append(f"Current input schema:\n```json\n{body.current_schema}\n```")
if body.name:
context_parts.append(f"Current name: {body.name}")
if body.description:
context_parts.append(f"Current description: {body.description}")
user_message = body.prompt
if context_parts:
user_message = "\n\n".join(context_parts) + "\n\nUser request: " + body.prompt
client = _get_anthropic_client()
try:
resp = await client.messages.create(
model="claude-sonnet-4-20250514", max_tokens=8000,
system=VIBE_CODE_SYSTEM_PROMPT,
messages=[{"role": "user", "content": user_message}],
)
raw = resp.content[0].text.strip()
if raw.startswith("```"):
raw = raw.split("\n", 1)[1] if "\n" in raw else raw[3:]
if raw.endswith("```"):
raw = raw[:-3]
result = json.loads(raw)
return {
"message": result.get("message", "View updated."),
"frontend_code": result.get("frontend_code", body.current_frontend_code),
"backend_code": result.get("backend_code", body.current_backend_code),
"input_schema": result.get("input_schema", body.current_schema),
"name": result.get("name", body.name),
"description": result.get("description", body.description),
}
except json.JSONDecodeError:
return {
"message": "I generated code but couldn't parse the response. Please try again.",
"frontend_code": body.current_frontend_code,
"backend_code": body.current_backend_code,
"input_schema": body.current_schema,
}
except Exception as e:
logger.exception("Vibe code generation failed")
return {
"message": f"Error: {str(e)}",
"frontend_code": body.current_frontend_code,
"backend_code": body.current_backend_code,
"input_schema": body.current_schema,
}
AUTO_RUN_SYSTEM_PROMPT = """\
You generate structured JSON data matching a given schema.
The user provides a prompt describing what data to generate and a JSON Schema.
Return ONLY valid JSON that conforms to the schema. No markdown fences, no extra text, no explanation.
Every required field must be present. Use realistic, meaningful data.\
"""
async def auto_run_output(body: AutoRunRequest):
try:
import anthropic
except ImportError:
return {"error": "anthropic SDK not installed", "input_data": None, "backend_result": None}
schema_str = json.dumps(body.input_schema, indent=2)
user_message = f"Schema:\n```json\n{schema_str}\n```\n\nGenerate data for: {body.prompt}"
api_model = _resolve_model(body.model)
client = _get_anthropic_client()
try:
resp = await client.messages.create(
model=api_model, max_tokens=4000,
system=AUTO_RUN_SYSTEM_PROMPT,
messages=[{"role": "user", "content": user_message}],
)
raw = resp.content[0].text.strip()
if raw.startswith("```"):
raw = raw.split("\n", 1)[1] if "\n" in raw else raw[3:]
if raw.endswith("```"):
raw = raw[:-3]
input_data = json.loads(raw)
validation_err = _validate_against_schema(input_data, body.input_schema)
if validation_err:
return {"input_data": input_data, "backend_result": None, "error": validation_err}
backend_result = None
stdout_text = None
stderr_text = None
error = None
if body.backend_code:
try:
exec_result = await execute_backend_code(body.backend_code, input_data)
backend_result = exec_result.result
stdout_text = exec_result.stdout
stderr_text = exec_result.stderr
except Exception as e:
error = str(e)
return {"input_data": input_data, "backend_result": backend_result, "stdout": stdout_text, "stderr": stderr_text, "error": error}
except json.JSONDecodeError:
return {"error": "Failed to parse generated data as JSON", "input_data": None, "backend_result": None}
except Exception as e:
logger.exception("Auto-run failed")
return {"error": str(e), "input_data": None, "backend_result": None}
AUTO_RUN_AGENT_SYSTEM_PROMPT = """\
You are a data-gathering agent. Your job is to use the available tools to collect \
real data, then render it into a structured View.
You have access to MCP tools (e.g. Gmail, calendar, etc.) that let you fetch live data. \
Use them as needed to fulfil the user's request.
When you have gathered enough data, call the **RenderOutput** tool with:
- `output_id`: `{output_id}`
- `input_data`: a JSON object conforming to this schema:
```json
{schema}
```
Do NOT fabricate data. Use the tools to get real information, then structure it to match \
the schema above. If a tool call fails, report the error clearly.\
"""
async def auto_run_agent(body: AutoRunAgentRequest):
from backend.apps.agents.agent_manager import agent_manager
from backend.apps.agents.mcp_builder import FULL_TOOLS
from backend.apps.agents.models import AgentConfig
from backend.apps.outputs.outputs import _load
output = _load(body.output_id)
schema_str = json.dumps(body.input_schema or output.input_schema, indent=2)
system_prompt = AUTO_RUN_AGENT_SYSTEM_PROMPT.format(
output_id=body.output_id, schema=schema_str,
)
allowed_tools = list(FULL_TOOLS)
for tool_name in body.forced_tools:
if tool_name not in allowed_tools:
allowed_tools.append(tool_name)
config = AgentConfig(
name=f"AutoRun: {output.name}", model=body.model,
mode="agent", system_prompt=system_prompt,
allowed_tools=allowed_tools, max_turns=20,
)
session = await agent_manager.launch_agent(config)
await agent_manager.send_message(
session.id, body.prompt,
context_paths=body.context_paths if body.context_paths else None,
forced_tools=body.forced_tools if body.forced_tools else None,
)
return {"session_id": session.id}
async def cleanup_auto_run_agent(session_id: str):
from backend.apps.agents.agent_manager import agent_manager
try:
await agent_manager.delete_session(session_id)
except Exception as e:
logger.warning(f"Auto-run agent cleanup failed for {session_id}: {e}")
return {"ok": True}
backend/apps/outputs/helpers.py
+72
View File
@@ -0,0 +1,72 @@
"""Pure helpers for data injection, validation, and directory walking."""
from __future__ import annotations
import base64
import json
import os
from jsonschema import validate as schema_validate, ValidationError as SchemaValidationError
def _validate_against_schema(data: dict, schema: dict) -> str | None:
"""Validate *data* against *schema*. Return an error string or None."""
try:
schema_validate(instance=data, schema=schema)
return None
except SchemaValidationError as exc:
path = " -> ".join(str(p) for p in exc.absolute_path) if exc.absolute_path else "(root)"
return f"Schema validation failed at {path}: {exc.message}"
def _build_data_injection(input_json: str, result_json: str) -> str:
return (
"<script>\n"
"(function() {\n"
" window.OUTPUT_INPUT = " + input_json + ";\n"
" window.OUTPUT_BACKEND_RESULT = " + result_json + ";\n"
" window.addEventListener('message', function(e) {\n"
" if (e.data && e.data.type === 'OUTPUT_DATA') {\n"
" window.OUTPUT_INPUT = e.data.input || {};\n"
" window.OUTPUT_BACKEND_RESULT = e.data.backendResult || null;\n"
" window.dispatchEvent(new CustomEvent('output-data-ready'));\n"
" }\n"
" });\n"
"})();\n"
"</script>"
)
def _inject_data_into_html(html: str, input_json: str = "{}", result_json: str = "null") -> str:
injection = _build_data_injection(input_json, result_json)
if "</head>" in html:
return html.replace("</head>", f"{injection}\n</head>", 1)
if "<body" in html:
return html.replace("<body", f"{injection}\n<body", 1)
return f"{injection}\n{html}"
def _decode_data_param(d: str) -> tuple[str, str]:
try:
decoded = json.loads(base64.b64decode(d))
input_json = json.dumps(decoded.get("i", {}))
result_json = json.dumps(decoded.get("r", None))
return input_json, result_json
except Exception:
return "{}", "null"
def _walk_directory(folder: str) -> dict[str, str]:
files: dict[str, str] = {}
if not os.path.isdir(folder):
return files
for root, _dirs, filenames in os.walk(folder):
for fname in filenames:
full_path = os.path.join(root, fname)
rel_path = os.path.relpath(full_path, folder)
try:
with open(full_path) as f:
files[rel_path] = f.read()
except Exception:
pass
return files
backend/apps/outputs/outputs.py
+35 -359
View File
@@ -1,87 +1,34 @@
"""Outputs SubApp — CRUD, workspace management, and file serving.
AI-generation endpoints live in ``ai_generation.py``; pure helpers in ``helpers.py``.
"""
from __future__ import annotations
import json
import os
import logging
import mimetypes
import base64
import os
from datetime import datetime
from contextlib import asynccontextmanager
from fastapi import HTTPException, Query
from fastapi import HTTPException
from fastapi.responses import Response
from jsonschema import validate as schema_validate, ValidationError as SchemaValidationError
from backend.config.Apps import SubApp
from backend.apps.common.json_store import JsonStore
from backend.apps.outputs.models import (
Output, OutputCreate, OutputUpdate, OutputExecute, OutputExecuteResult,
VibeCodeRequest, AutoRunRequest, AutoRunConfig, AutoRunAgentRequest,
WorkspaceSeedRequest,
AutoRunConfig, WorkspaceSeedRequest,
)
from backend.apps.outputs.executor import execute_backend_code
from backend.apps.outputs.view_builder_templates import VIEW_BUILDER_SKILL, VIEW_TEMPLATE_FILES
from backend.apps.common.model_registry import resolve_model_id as _resolve_model
from backend.apps.settings.settings import load_settings
logger = logging.getLogger(__name__)
def _get_anthropic_client():
"""Create an AsyncAnthropic client using the API key from app settings."""
from backend.apps.settings.credentials import get_anthropic_client
settings = load_settings()
return get_anthropic_client(settings)
def _validate_against_schema(data: dict, schema: dict) -> str | None:
"""Validate *data* against *schema*. Return an error string or None."""
try:
schema_validate(instance=data, schema=schema)
return None
except SchemaValidationError as exc:
path = " -> ".join(str(p) for p in exc.absolute_path) if exc.absolute_path else "(root)"
return f"Schema validation failed at {path}: {exc.message}"
from backend.apps.outputs.helpers import (
_validate_against_schema, _inject_data_into_html, _decode_data_param, _walk_directory,
)
from backend.apps.outputs import ai_generation
from backend.config.paths import OUTPUTS_DIR as DATA_DIR, OUTPUTS_WORKSPACE_DIR as WORKSPACE_DIR
def _build_data_injection(input_json: str, result_json: str) -> str:
"""Build a <script> tag that sets OUTPUT_INPUT / OUTPUT_BACKEND_RESULT
and listens for postMessage updates."""
return (
"<script>\n"
"(function() {\n"
" window.OUTPUT_INPUT = " + input_json + ";\n"
" window.OUTPUT_BACKEND_RESULT = " + result_json + ";\n"
" window.addEventListener('message', function(e) {\n"
" if (e.data && e.data.type === 'OUTPUT_DATA') {\n"
" window.OUTPUT_INPUT = e.data.input || {};\n"
" window.OUTPUT_BACKEND_RESULT = e.data.backendResult || null;\n"
" window.dispatchEvent(new CustomEvent('output-data-ready'));\n"
" }\n"
" });\n"
"})();\n"
"</script>"
)
def _inject_data_into_html(html: str, input_json: str = "{}", result_json: str = "null") -> str:
injection = _build_data_injection(input_json, result_json)
if "</head>" in html:
return html.replace("</head>", f"{injection}\n</head>", 1)
if "<body" in html:
return html.replace("<body", f"{injection}\n<body", 1)
return f"{injection}\n{html}"
def _decode_data_param(d: str) -> tuple[str, str]:
"""Decode the base64-encoded _d query param into (input_json, result_json)."""
try:
decoded = json.loads(base64.b64decode(d))
input_json = json.dumps(decoded.get("i", {}))
result_json = json.dumps(decoded.get("r", None))
return input_json, result_json
except Exception:
return "{}", "null"
@asynccontextmanager
async def outputs_lifespan():
os.makedirs(DATA_DIR, exist_ok=True)
@@ -91,76 +38,46 @@ async def outputs_lifespan():
outputs = SubApp("outputs", outputs_lifespan)
_store = JsonStore(Output, DATA_DIR, not_found_detail="Output not found")
_load_all = _store.load_all
_save = _store.save
_load = _store.load
load_output = _store.load_or_none
def _walk_directory(folder: str) -> dict[str, str]:
"""Walk a directory tree and return {relative_path: content} for all text files."""
files: dict[str, str] = {}
if not os.path.isdir(folder):
return files
for root, _dirs, filenames in os.walk(folder):
for fname in filenames:
full_path = os.path.join(root, fname)
rel_path = os.path.relpath(full_path, folder)
try:
with open(full_path) as f:
files[rel_path] = f.read()
except Exception:
pass
return files
# ---------------------------------------------------------------------------
# File-serving endpoints (for iframe preview with multi-file support)
# ---------------------------------------------------------------------------
# -- File serving --
@outputs.router.get("/workspace/{workspace_id}/serve/{filepath:path}")
async def serve_workspace_file(workspace_id: str, filepath: str, _d: str = ""):
"""Serve a file from a workspace folder. For index.html, inject OUTPUT data."""
folder = os.path.join(WORKSPACE_DIR, workspace_id)
full_path = os.path.normpath(os.path.join(folder, filepath))
if not full_path.startswith(os.path.normpath(folder)):
raise HTTPException(status_code=403, detail="Path traversal not allowed")
if not os.path.isfile(full_path):
raise HTTPException(status_code=404, detail="File not found")
with open(full_path) as f:
content = f.read()
if filepath == "index.html":
input_json, result_json = _decode_data_param(_d) if _d else ("{}", "null")
content = _inject_data_into_html(content, input_json, result_json)
mime, _ = mimetypes.guess_type(filepath)
return Response(content=content, media_type=mime or "text/plain")
@outputs.router.get("/{output_id}/serve/{filepath:path}")
async def serve_output_file(output_id: str, filepath: str, _d: str = ""):
"""Serve a file from a saved output's files dict. For index.html, inject OUTPUT data."""
output = _load(output_id)
content = output.files.get(filepath)
if content is None:
raise HTTPException(status_code=404, detail="File not found in output")
if filepath == "index.html":
input_json, result_json = _decode_data_param(_d) if _d else ("{}", "null")
content = _inject_data_into_html(content, input_json, result_json)
mime, _ = mimetypes.guess_type(filepath)
return Response(content=content, media_type=mime or "text/plain")
# ---------------------------------------------------------------------------
# CRUD + workspace endpoints
# ---------------------------------------------------------------------------
# -- CRUD --
@outputs.router.get("/list")
async def list_outputs():
@@ -169,29 +86,23 @@ async def list_outputs():
@outputs.router.get("/workspace/{workspace_id}")
async def read_workspace(workspace_id: str):
"""Read all files from an output workspace folder."""
folder = os.path.join(WORKSPACE_DIR, workspace_id)
if not os.path.isdir(folder):
raise HTTPException(status_code=404, detail="Workspace not found")
files = _walk_directory(folder)
meta = None
if "meta.json" in files:
try:
meta = json.loads(files["meta.json"])
except (json.JSONDecodeError, ValueError):
pass
return {"files": files, "meta": meta}
@outputs.router.post("/workspace/seed")
async def seed_workspace(body: WorkspaceSeedRequest):
"""Create a workspace folder and optionally pre-seed it with files."""
folder = os.path.join(WORKSPACE_DIR, body.workspace_id)
os.makedirs(folder, exist_ok=True)
if body.files:
for rel_path, content in body.files.items():
full_path = os.path.normpath(os.path.join(folder, rel_path))
@@ -205,20 +116,16 @@ async def seed_workspace(body: WorkspaceSeedRequest):
full_path = os.path.join(folder, rel_path)
with open(full_path, "w") as f:
f.write(content)
with open(os.path.join(folder, "SKILL.md"), "w") as f:
f.write(VIEW_BUILDER_SKILL)
if body.meta:
with open(os.path.join(folder, "meta.json"), "w") as f:
json.dump(body.meta, f, indent=2)
return {"path": os.path.abspath(folder)}
@outputs.router.put("/workspace/{workspace_id}/file/{filepath:path}")
async def write_workspace_file(workspace_id: str, filepath: str, body: dict):
"""Write (create/overwrite) a single file in a workspace."""
folder = os.path.join(WORKSPACE_DIR, workspace_id)
if not os.path.isdir(folder):
raise HTTPException(status_code=404, detail="Workspace not found")
@@ -233,7 +140,6 @@ async def write_workspace_file(workspace_id: str, filepath: str, body: dict):
@outputs.router.delete("/workspace/{workspace_id}/file/{filepath:path}")
async def delete_workspace_file(workspace_id: str, filepath: str):
"""Delete a single file from a workspace."""
folder = os.path.join(WORKSPACE_DIR, workspace_id)
if not os.path.isdir(folder):
raise HTTPException(status_code=404, detail="Workspace not found")
@@ -261,15 +167,10 @@ async def get_output(output_id: str):
async def create_output(body: OutputCreate):
now = datetime.now().isoformat()
output = Output(
name=body.name,
description=body.description,
icon=body.icon,
input_schema=body.input_schema,
files=body.files,
auto_run_config=body.auto_run_config,
thumbnail=body.thumbnail,
created_at=now,
updated_at=now,
name=body.name, description=body.description, icon=body.icon,
input_schema=body.input_schema, files=body.files,
auto_run_config=body.auto_run_config, thumbnail=body.thumbnail,
created_at=now, updated_at=now,
)
_save(output)
from backend.apps.analytics.collector import record as _analytics
@@ -298,263 +199,38 @@ async def delete_output(output_id: str):
return {"ok": True}
VIBE_CODE_SYSTEM_PROMPT = """\
You are an expert at building self-contained HTML/JS/CSS applications that run in an iframe.
The user will describe what they want, and you will generate:
1. **frontend_code**: A complete HTML document. React 18 is available via esm.sh CDN.
- Use: <script type="importmap">{"imports":{"react":"https://esm.sh/react@18","react-dom/client":"https://esm.sh/react-dom@18/client"}}</script>
- Input data is at window.OUTPUT_INPUT (object), backend result at window.OUTPUT_BACKEND_RESULT.
2. **input_schema**: A JSON Schema object defining the structured input.
3. **backend_code** (optional): Python code where input_data is a global dict and result is a global dict to assign to.
4. **name**: A short name for the view.
5. **description**: A one-sentence description.
6. **message**: A brief explanation of what you did/changed.
Return ONLY valid JSON with these keys. No markdown fences, no extra text.\
"""
@outputs.router.post("/vibe-code")
async def vibe_code(body: VibeCodeRequest):
"""Use an LLM to generate or iterate on Output code from a natural language prompt."""
from backend.apps.analytics.collector import record as _analytics
_analytics("feature.used", {"feature": "vibe_code.used"})
try:
import anthropic
except ImportError:
return {
"message": "anthropic SDK not installed. Install with: pip install anthropic",
"frontend_code": body.current_frontend_code,
"backend_code": body.current_backend_code,
"input_schema": body.current_schema,
}
context_parts = []
if body.current_frontend_code:
context_parts.append(f"Current frontend code:\n```html\n{body.current_frontend_code}\n```")
if body.current_backend_code:
context_parts.append(f"Current backend code:\n```python\n{body.current_backend_code}\n```")
if body.current_schema:
context_parts.append(f"Current input schema:\n```json\n{body.current_schema}\n```")
if body.name:
context_parts.append(f"Current name: {body.name}")
if body.description:
context_parts.append(f"Current description: {body.description}")
user_message = body.prompt
if context_parts:
user_message = "\n\n".join(context_parts) + "\n\nUser request: " + body.prompt
client = _get_anthropic_client()
try:
resp = await client.messages.create(
model="claude-sonnet-4-20250514",
max_tokens=8000,
system=VIBE_CODE_SYSTEM_PROMPT,
messages=[{"role": "user", "content": user_message}],
)
raw = resp.content[0].text.strip()
if raw.startswith("```"):
raw = raw.split("\n", 1)[1] if "\n" in raw else raw[3:]
if raw.endswith("```"):
raw = raw[:-3]
result = json.loads(raw)
return {
"message": result.get("message", "View updated."),
"frontend_code": result.get("frontend_code", body.current_frontend_code),
"backend_code": result.get("backend_code", body.current_backend_code),
"input_schema": result.get("input_schema", body.current_schema),
"name": result.get("name", body.name),
"description": result.get("description", body.description),
}
except json.JSONDecodeError:
return {
"message": "I generated code but couldn't parse the response. Please try again.",
"frontend_code": body.current_frontend_code,
"backend_code": body.current_backend_code,
"input_schema": body.current_schema,
}
except Exception as e:
logger.exception("Vibe code generation failed")
return {
"message": f"Error: {str(e)}",
"frontend_code": body.current_frontend_code,
"backend_code": body.current_backend_code,
"input_schema": body.current_schema,
}
AUTO_RUN_SYSTEM_PROMPT = """\
You generate structured JSON data matching a given schema.
The user provides a prompt describing what data to generate and a JSON Schema.
Return ONLY valid JSON that conforms to the schema. No markdown fences, no extra text, no explanation.
Every required field must be present. Use realistic, meaningful data.\
"""
@outputs.router.post("/auto-run")
async def auto_run_output(body: AutoRunRequest):
"""Use an LLM to generate input data matching the schema, then optionally execute backend code."""
try:
import anthropic
except ImportError:
return {"error": "anthropic SDK not installed", "input_data": None, "backend_result": None}
schema_str = json.dumps(body.input_schema, indent=2)
user_message = f"Schema:\n```json\n{schema_str}\n```\n\nGenerate data for: {body.prompt}"
api_model = _resolve_model(body.model)
client = _get_anthropic_client()
try:
resp = await client.messages.create(
model=api_model,
max_tokens=4000,
system=AUTO_RUN_SYSTEM_PROMPT,
messages=[{"role": "user", "content": user_message}],
)
raw = resp.content[0].text.strip()
if raw.startswith("```"):
raw = raw.split("\n", 1)[1] if "\n" in raw else raw[3:]
if raw.endswith("```"):
raw = raw[:-3]
input_data = json.loads(raw)
validation_err = _validate_against_schema(input_data, body.input_schema)
if validation_err:
return {"input_data": input_data, "backend_result": None, "error": validation_err}
backend_result = None
stdout_text = None
stderr_text = None
error = None
if body.backend_code:
try:
exec_result = await execute_backend_code(body.backend_code, input_data)
backend_result = exec_result.result
stdout_text = exec_result.stdout
stderr_text = exec_result.stderr
except Exception as e:
error = str(e)
return {"input_data": input_data, "backend_result": backend_result, "stdout": stdout_text, "stderr": stderr_text, "error": error}
except json.JSONDecodeError:
return {"error": "Failed to parse generated data as JSON", "input_data": None, "backend_result": None}
except Exception as e:
logger.exception("Auto-run failed")
return {"error": str(e), "input_data": None, "backend_result": None}
@outputs.router.post("/execute")
async def execute_output(body: OutputExecute):
output = _load(body.output_id)
validation_err = _validate_against_schema(body.input_data, output.input_schema)
if validation_err:
return OutputExecuteResult(
output_id=output.id,
output_name=output.name,
frontend_code=output.frontend_code,
input_data=body.input_data,
backend_result=None,
error=validation_err,
output_id=output.id, output_name=output.name,
frontend_code=output.frontend_code, input_data=body.input_data,
backend_result=None, error=validation_err,
).model_dump()
backend_result = None
stdout_text = None
stderr_text = None
error = None
if output.backend_code:
try:
exec_result = await execute_backend_code(
output.backend_code, body.input_data
)
exec_result = await execute_backend_code(output.backend_code, body.input_data)
backend_result = exec_result.result
stdout_text = exec_result.stdout
stderr_text = exec_result.stderr
except Exception as e:
error = str(e)
return OutputExecuteResult(
output_id=output.id,
output_name=output.name,
frontend_code=output.frontend_code,
input_data=body.input_data,
backend_result=backend_result,
stdout=stdout_text,
stderr=stderr_text,
error=error,
output_id=output.id, output_name=output.name,
frontend_code=output.frontend_code, input_data=body.input_data,
backend_result=backend_result, stdout=stdout_text,
stderr=stderr_text, error=error,
).model_dump()
AUTO_RUN_AGENT_SYSTEM_PROMPT = """\
You are a data-gathering agent. Your job is to use the available tools to collect \
real data, then render it into a structured View.
You have access to MCP tools (e.g. Gmail, calendar, etc.) that let you fetch live data. \
Use them as needed to fulfil the user's request.
When you have gathered enough data, call the **RenderOutput** tool with:
- `output_id`: `{output_id}`
- `input_data`: a JSON object conforming to this schema:
```json
{schema}
```
Do NOT fabricate data. Use the tools to get real information, then structure it to match \
the schema above. If a tool call fails, report the error clearly.\
"""
@outputs.router.post("/auto-run-agent")
async def auto_run_agent(body: AutoRunAgentRequest):
"""Launch a temporary agent session that uses MCP tools to gather data for a view."""
from backend.apps.agents.agent_manager import agent_manager, FULL_TOOLS
from backend.apps.agents.models import AgentConfig
output = _load(body.output_id)
schema_str = json.dumps(body.input_schema or output.input_schema, indent=2)
system_prompt = AUTO_RUN_AGENT_SYSTEM_PROMPT.format(
output_id=body.output_id,
schema=schema_str,
)
allowed_tools = list(FULL_TOOLS)
for tool_name in body.forced_tools:
if tool_name not in allowed_tools:
allowed_tools.append(tool_name)
config = AgentConfig(
name=f"AutoRun: {output.name}",
model=body.model,
mode="agent",
system_prompt=system_prompt,
allowed_tools=allowed_tools,
max_turns=20,
)
session = await agent_manager.launch_agent(config)
await agent_manager.send_message(
session.id,
body.prompt,
context_paths=body.context_paths if body.context_paths else None,
forced_tools=body.forced_tools if body.forced_tools else None,
)
return {"session_id": session.id}
@outputs.router.delete("/auto-run-agent/{session_id}")
async def cleanup_auto_run_agent(session_id: str):
"""Delete a temporary auto-run agent session."""
from backend.apps.agents.agent_manager import agent_manager
try:
await agent_manager.delete_session(session_id)
except Exception as e:
logger.warning(f"Auto-run agent cleanup failed for {session_id}: {e}")
return {"ok": True}
# -- AI generation routes --
outputs.router.add_api_route("/vibe-code", ai_generation.vibe_code, methods=["POST"])
outputs.router.add_api_route("/auto-run", ai_generation.auto_run_output, methods=["POST"])
outputs.router.add_api_route("/auto-run-agent", ai_generation.auto_run_agent, methods=["POST"])
outputs.router.add_api_route("/auto-run-agent/{session_id}", ai_generation.cleanup_auto_run_agent, methods=["DELETE"])
backend/apps/tools_lib/__init__.py
+38
View File
@@ -0,0 +1,38 @@
"""tools_lib package — SubApp instance, lifespan, and route wiring."""
from __future__ import annotations
import os
from contextlib import asynccontextmanager
from backend.config.Apps import SubApp
from backend.config.paths import TOOLS_DIR as DATA_DIR
from backend.apps.tools_lib import routes, oauth, mcp_discovery
@asynccontextmanager
async def tools_lib_lifespan():
os.makedirs(DATA_DIR, exist_ok=True)
yield
tools_lib = SubApp("tools", tools_lib_lifespan)
# CRUD + permissions
tools_lib.router.add_api_route("/builtin", routes.list_builtin_tools, methods=["GET"])
tools_lib.router.add_api_route("/builtin/permissions", routes.get_builtin_permissions, methods=["GET"])
tools_lib.router.add_api_route("/builtin/permissions", routes.update_builtin_permissions, methods=["PUT"])
tools_lib.router.add_api_route("/list", routes.list_tools, methods=["GET"])
tools_lib.router.add_api_route("/create", routes.create_tool, methods=["POST"])
# OAuth
tools_lib.router.add_api_route("/oauth/callback", oauth.oauth_callback, methods=["GET"])
# Per-tool routes (order matters: specific paths before {tool_id})
tools_lib.router.add_api_route("/{tool_id}/discover", mcp_discovery.discover_tools, methods=["POST"])
tools_lib.router.add_api_route("/{tool_id}/oauth/disconnect", oauth.oauth_disconnect, methods=["POST"])
tools_lib.router.add_api_route("/{tool_id}/oauth/start", oauth.oauth_start, methods=["POST"])
tools_lib.router.add_api_route("/{tool_id}", routes.get_tool, methods=["GET"])
tools_lib.router.add_api_route("/{tool_id}", routes.update_tool, methods=["PUT"])
tools_lib.router.add_api_route("/{tool_id}", routes.delete_tool, methods=["DELETE"])
backend/apps/tools_lib/classification.py
+102
View File
@@ -0,0 +1,102 @@
"""Tool categorization — pure data + logic, no routes."""
from __future__ import annotations
_READ_PREFIXES = ("get", "list", "read", "search", "fetch", "find", "query", "count", "check", "describe", "show", "download", "browse", "analy", "explain")
_WRITE_PREFIXES = ("create", "write", "delete", "update", "send", "remove", "modify", "add", "set", "put", "post", "patch", "insert", "move", "copy", "rename", "archive", "trash", "publish", "approve", "reject")
_SERVICE_RULES: list[tuple[list[str], str, str]] = [
(["gmail"], "Gmail", "Google"),
(["drive"], "Drive", "Google"),
(["calendar", "event", "freebusy"], "Calendar", "Google"),
(["spreadsheet", "sheet"], "Sheets", "Google"),
(["doc", "paragraph", "table"], "Docs", "Google"),
(["chat", "space", "reaction", "message"], "Chat", "Google"),
(["form", "publish_settings"], "Forms", "Google"),
(["presentation", "slide", "page"], "Slides", "Google"),
(["task_list", "task"], "Tasks", "Google"),
(["contact"], "Contacts", "Google"),
(["script", "deployment", "version", "trigger"], "Apps Script", "Google"),
(["search_custom", "search_engine"], "Search", "Google"),
(["subreddit"], "Subreddits", "Reddit"),
(["search_reddit"], "Search", "Reddit"),
(["post_detail"], "Posts", "Reddit"),
(["user_analysis"], "Users", "Reddit"),
(["reddit_explain"], "Reference", "Reddit"),
(["sequentialthinking", "thinking"], "Thinking", "Sequential Thinking"),
(["create_entities", "create_relations", "add_observations", "delete_entities",
"delete_observations", "delete_relations", "read_graph", "search_nodes",
"open_nodes"], "Knowledge Graph", "Memory"),
(["read_file", "read_multiple_files", "write_file", "edit_file",
"create_directory", "list_directory", "directory_tree", "move_file",
"search_files", "get_file_info", "list_allowed_directories"], "Files", "Filesystem"),
(["browser_navigate", "browser_screenshot", "browser_click", "browser_fill",
"browser_select", "browser_hover", "browser_evaluate", "browser_console",
"browser_tab", "browser_close", "browser_resize", "browser_snapshot",
"browser_wait", "browser_pdf", "browser_drag"], "Browser", "Playwright"),
(["git_status", "git_diff", "git_diff_unstaged", "git_diff_staged",
"git_commit", "git_log", "git_add", "git_reset", "git_show",
"git_create_branch", "git_checkout", "git_list_branches", "git_init",
"git_clone"], "Repository", "Git"),
(["get_transcript"], "Transcripts", "YouTube"),
(["execute_command", "read_output", "force_terminate", "list_sessions",
"list_processes", "kill_process", "block_command", "unblock_command",
"read_file", "write_file", "search_code", "list_directory",
"get_file_info", "edit_block"], "System", "Desktop Commander"),
(["repository", "issue", "pull_request", "commit", "branch", "fork", "star",
"create_issue", "list_issues", "get_issue", "create_pull_request",
"list_commits", "search_repositories", "create_repository",
"get_file_contents", "push_files", "create_branch",
"search_code", "search_issues"], "Repository", "GitHub"),
(["channel", "slack_message", "thread", "reply", "workspace",
"list_channels", "post_message", "reply_to_thread", "search_messages",
"get_channel_history", "get_thread_replies", "get_users",
"get_user_profile"], "Messaging", "Slack"),
(["notion_page", "database", "block", "create_page", "update_page",
"search_pages", "get_page", "get_database", "query_database",
"create_database", "append_block_children"], "Pages", "Notion"),
(["play", "pause", "skip", "playlist", "track", "album", "artist",
"search_tracks", "get_playlist", "get_currently_playing",
"add_to_playlist", "create_playlist", "get_recommendations",
"get_top_items"], "Music", "Spotify"),
(["figma", "design", "component", "style", "node",
"get_file", "get_file_nodes", "get_image", "get_comments",
"get_team_projects", "get_project_files"], "Design", "Figma"),
(["airtable", "base", "record", "field", "view",
"list_records", "get_record", "create_record", "update_record",
"delete_record", "list_bases", "get_base_schema"], "Data", "Airtable"),
(["hubspot", "contact", "deal", "company", "ticket", "pipeline",
"crm", "engagement", "association"], "CRM", "HubSpot"),
(["discord", "guild", "server", "channel_message", "send_message",
"get_messages", "get_guilds", "get_channels", "add_reaction"], "Messaging", "Discord"),
(["tweetsave", "get_tweet", "get_thread", "to_blog", "batch",
"extract_media"], "Tweets", "Twitter"),
(["shopify", "introspect", "graphql", "search_dev_docs", "liquid",
"polaris", "admin_api", "storefront_api"], "Developer", "Shopify"),
(["zoom", "meeting", "recording", "participant", "webinar",
"create_meeting", "list_meetings", "get_meeting", "delete_meeting",
"update_meeting"], "Meetings", "Zoom"),
(["outlook", "onedrive", "ms365", "microsoft", "mail_folder",
"email", "calendar_event", "contact", "drive_item"], "Mail & Files", "Microsoft 365"),
]
def _categorize_tool(name: str) -> str:
lower = name.lower().replace("_", " ").replace("-", " ").strip()
for word in lower.split():
for prefix in _READ_PREFIXES:
if word.startswith(prefix):
return "read"
for prefix in _WRITE_PREFIXES:
if word.startswith(prefix):
return "write"
return "write"
def _extract_service(name: str) -> tuple[str, str]:
lower = name.lower()
for keywords, display, group in _SERVICE_RULES:
for kw in keywords:
if kw in lower:
return display, group
return "Other", ""
backend/apps/tools_lib/mcp_config.py
+143
View File
@@ -0,0 +1,143 @@
"""MCP server config derivation and path resolution helpers."""
from __future__ import annotations
import json
import logging
import os
import shutil
from typing import Optional
from backend.apps.tools_lib.oauth_providers import OAUTH_PROVIDERS
logger = logging.getLogger(__name__)
def _extra_bin_dirs() -> list[str]:
"""Well-known user-local bin directories that may not be on PATH in packaged apps."""
home = os.path.expanduser("~")
_backend = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
dirs = [
os.path.join(_backend, "uv-bin"),
os.path.join(home, ".bun", "bin"),
os.path.join(home, ".cargo", "bin"),
os.path.join(home, ".local", "bin"),
os.path.join(home, ".volta", "bin"),
"/opt/homebrew/bin",
"/usr/local/bin",
]
nvm_node = os.path.join(home, ".nvm", "versions", "node")
try:
if os.path.isdir(nvm_node):
versions = sorted(os.listdir(nvm_node), reverse=True)
if versions:
dirs.insert(0, os.path.join(nvm_node, versions[0], "bin"))
except OSError:
pass
fnm_bin = os.path.join(home, "Library", "Application Support", "fnm", "aliases", "default", "bin")
if os.path.isdir(fnm_bin):
dirs.insert(0, fnm_bin)
return dirs
def _resolve_command(command: str) -> str | None:
found = shutil.which(command)
if found:
return found
for d in _extra_bin_dirs():
candidate = os.path.join(d, command)
if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
return candidate
_backend = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
candidate = os.path.join(_backend, "uv-bin", command)
if os.path.isfile(candidate) and os.access(candidate, os.X_OK):
return candidate
return None
def _augmented_path() -> str:
extra = [d for d in _extra_bin_dirs() if os.path.isdir(d)]
current = os.environ.get("PATH", "")
seen: set[str] = set()
parts: list[str] = []
for p in extra + current.split(os.pathsep):
if p and p not in seen:
seen.add(p)
parts.append(p)
return os.pathsep.join(parts)
def derive_mcp_config(tool) -> Optional[dict]:
"""Build the claude_agent_sdk mcp_servers config entry for a tool."""
if not tool.mcp_config:
return None
config: dict = dict(tool.mcp_config)
if tool.credentials:
if config.get("type") in ("http", "sse"):
headers = config.setdefault("headers", {})
for key, val in tool.credentials.items():
if key.lower() in ("authorization", "api_key", "api-key"):
headers.setdefault("Authorization", f"Bearer {val}")
else:
env = config.setdefault("env", {})
env.update(tool.credentials)
if tool.auth_type == "oauth2" and tool.oauth_tokens.get("access_token"):
if config.get("type") in ("http", "sse"):
headers = config.setdefault("headers", {})
headers["Authorization"] = f"Bearer {tool.oauth_tokens['access_token']}"
else:
env = config.setdefault("env", {})
provider_key = tool.oauth_provider or "google"
provider = OAUTH_PROVIDERS.get(provider_key)
if provider:
for token_field, env_var in provider.token_env_mapping.items():
if token_field.startswith("_client_id"):
val = os.environ.get(provider.client_id_env, "")
elif token_field.startswith("_client_secret"):
val = os.environ.get(provider.client_secret_env, "")
else:
val = tool.oauth_tokens.get(token_field, "")
if val:
if provider.env_value_transform == "notion_headers" and token_field == "access_token":
val = json.dumps({
"Authorization": f"Bearer {val}",
"Notion-Version": "2022-06-28",
})
env[env_var] = val
for _, env_var in provider.extra_token_fields.items():
val = tool.oauth_tokens.get(env_var, "")
if val:
env[env_var] = val
if provider_key == "figma" and tool.oauth_tokens.get("access_token"):
args = config.get("args", [])
if "--figma-api-key" not in args:
config["args"] = args + ["--figma-api-key", tool.oauth_tokens["access_token"]]
else:
env["OAUTH_ACCESS_TOKEN"] = tool.oauth_tokens["access_token"]
if config.get("type") == "stdio":
if config.get("command"):
resolved = _resolve_command(config["command"])
if resolved:
config["command"] = resolved
else:
logger.warning(f"Command '{config['command']}' not found on PATH or bundled directories")
env = config.setdefault("env", {})
env.setdefault("PATH", _augmented_path())
env.setdefault("PYTHONPATH", "")
_is_packaged = os.environ.get("OPENSWARM_PACKAGED") == "1"
if _is_packaged:
_resources = os.path.dirname(os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
_bundled_python = os.path.join(_resources, "python-env", "bin", "python3")
if os.path.exists(_bundled_python):
env.setdefault("UV_PYTHON", _bundled_python)
else:
_backend = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
_venv_python = os.path.join(_backend, ".venv", "bin", "python3")
if os.path.exists(_venv_python):
env.setdefault("UV_PYTHON", _venv_python)
return config
backend/apps/tools_lib/mcp_discovery.py
+238
View File
@@ -0,0 +1,238 @@
"""MCP tool discovery — HTTP, SSE, and stdio transports."""
from __future__ import annotations
import asyncio
import json
import logging
import os
import time
from typing import Any
import httpx
from fastapi import HTTPException
from backend.apps.tools_lib.mcp_config import (
_resolve_command, _augmented_path, derive_mcp_config,
)
from backend.apps.tools_lib.classification import _categorize_tool, _extract_service
from backend.apps.common.mcp_utils import parse_sse_json as _parse_sse_json
logger = logging.getLogger(__name__)
async def _discover_mcp_tools_http(url: str, headers: dict | None = None) -> list[dict]:
h = {
"Content-Type": "application/json",
"Accept": "application/json, text/event-stream",
**(headers or {}),
}
async with httpx.AsyncClient(timeout=30.0) as client:
init_resp = await client.post(url, headers=h, json={
"jsonrpc": "2.0", "id": 1, "method": "initialize",
"params": {"protocolVersion": "2025-03-26", "capabilities": {},
"clientInfo": {"name": "self-swarm", "version": "0.1.0"}},
})
if init_resp.status_code not in (200, 201):
raise HTTPException(status_code=502, detail=f"MCP initialize failed: {init_resp.status_code}")
session_id = init_resp.headers.get("mcp-session-id", "")
if session_id:
h["mcp-session-id"] = session_id
await client.post(url, headers=h, json={
"jsonrpc": "2.0", "method": "notifications/initialized",
})
list_resp = await client.post(url, headers=h, json={
"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {},
})
if list_resp.status_code not in (200, 201):
raise HTTPException(status_code=502, detail=f"MCP tools/list failed: {list_resp.status_code}")
ct = list_resp.headers.get("content-type", "")
if "text/event-stream" in ct:
data = _parse_sse_json(list_resp.text)
else:
data = list_resp.json()
if not data:
raise HTTPException(status_code=502, detail="Empty response from MCP server")
tools_list = data.get("result", {}).get("tools", [])
return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list]
async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list[dict]:
from mcp.client.sse import sse_client
from mcp import ClientSession
from mcp.types import Implementation
try:
async with sse_client(url=url, headers=headers, timeout=30, sse_read_timeout=30) as (read_stream, write_stream):
async with ClientSession(
read_stream, write_stream,
client_info=Implementation(name="self-swarm", version="0.1.0"),
) as session:
await session.initialize()
result = await session.list_tools()
return [{"name": t.name, "description": t.description or "", "inputSchema": t.inputSchema if t.inputSchema else None} for t in result.tools]
except BaseExceptionGroup as eg:
first = eg.exceptions[0] if eg.exceptions else eg
raise HTTPException(status_code=502, detail=f"SSE discovery failed: {first}") from first
async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None, env: dict | None = None) -> list[dict]:
cmd_path = _resolve_command(command)
if not cmd_path:
raise HTTPException(status_code=400, detail=f"Command '{command}' not found on PATH or common install locations")
proc_env = {**os.environ, **(env or {}), "PATH": _augmented_path()}
proc_env.pop("PYTHONPATH", None)
proc = await asyncio.create_subprocess_exec(
cmd_path, *(args or []),
stdin=asyncio.subprocess.PIPE, stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE, env=proc_env, limit=1024 * 1024,
)
async def _send(msg: dict) -> None:
line = json.dumps(msg) + "\n"
proc.stdin.write(line.encode())
await proc.stdin.drain()
async def _recv() -> dict:
while True:
line = await asyncio.wait_for(proc.stdout.readline(), timeout=30.0)
if not line:
stderr_out = ""
try:
stderr_out = (await asyncio.wait_for(proc.stderr.read(4096), timeout=2.0)).decode(errors="replace")
except (asyncio.TimeoutError, Exception):
pass
raise HTTPException(
status_code=502,
detail=f"MCP stdio process exited unexpectedly{': ' + stderr_out if stderr_out else ''}",
)
stripped = line.decode(errors="replace").strip()
if not stripped:
continue
try:
data = json.loads(stripped)
except json.JSONDecodeError:
continue
if "id" in data:
return data
try:
await _send({
"jsonrpc": "2.0", "id": 1, "method": "initialize",
"params": {"protocolVersion": "2025-03-26", "capabilities": {},
"clientInfo": {"name": "self-swarm", "version": "0.1.0"}},
})
await _recv()
await _send({"jsonrpc": "2.0", "method": "notifications/initialized"})
await _send({"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {}})
data = await _recv()
tools_list = data.get("result", {}).get("tools", [])
return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list]
except HTTPException:
raise
except asyncio.TimeoutError:
raise HTTPException(status_code=504, detail="MCP stdio server timed out during discovery")
finally:
try:
proc.stdin.close()
except Exception:
pass
try:
proc.terminate()
await asyncio.wait_for(proc.wait(), timeout=5.0)
except Exception:
try:
proc.kill()
except Exception:
pass
async def discover_tools(tool_id: str):
from backend.apps.tools_lib.oauth import refresh_oauth_token
from backend.apps.tools_lib.routes import _store
tool = _store.load(tool_id)
if tool.auth_type == "oauth2" and tool.auth_status == "connected":
refreshed = await refresh_oauth_token(tool)
if not refreshed and tool.oauth_tokens.get("access_token"):
expiry = tool.oauth_tokens.get("token_expiry", 0)
if time.time() >= expiry - 60:
client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "")
if not client_id:
raise HTTPException(
status_code=400,
detail="OAuth token expired and GOOGLE_OAUTH_CLIENT_ID is not set.",
)
raise HTTPException(status_code=502, detail="OAuth token expired and refresh failed. Try reconnecting Google.")
config = derive_mcp_config(tool)
if not config:
raise HTTPException(status_code=400, detail="Cannot derive MCP config for tool")
transport = config.get("type", "")
try:
if transport == "stdio":
command = config.get("command", "")
if not command:
raise HTTPException(status_code=400, detail="stdio transport requires a 'command' in MCP config")
raw_tools = await _discover_mcp_tools_stdio(command=command, args=config.get("args"), env=config.get("env"))
elif transport in ("http", "sse") or config.get("url"):
url = config.get("url", "")
if not url:
raise HTTPException(status_code=400, detail="HTTP/SSE transport requires a 'url' in MCP config")
if transport == "sse":
raw_tools = await _discover_mcp_tools_sse(url, config.get("headers"))
else:
try:
raw_tools = await _discover_mcp_tools_http(url, config.get("headers"))
except HTTPException:
logger.info(f"Streamable HTTP failed for {tool.name}, retrying with SSE transport")
raw_tools = await _discover_mcp_tools_sse(url, config.get("headers"))
else:
raise HTTPException(status_code=400, detail=f"Unsupported MCP transport type: '{transport}'.")
except HTTPException:
raise
except Exception as e:
msg = str(e).strip() or type(e).__name__
logger.warning(f"MCP tool discovery failed for {tool.name}: {msg}", exc_info=True)
raise HTTPException(status_code=502, detail=f"Discovery failed: {msg}")
services: dict[str, dict[str, list[str]]] = {}
service_groups: dict[str, list[str]] = {}
permissions: dict[str, Any] = {}
for t in raw_tools:
name = t["name"]
cat = _categorize_tool(name)
svc, group = _extract_service(name)
if svc not in services:
services[svc] = {"read": [], "write": []}
services[svc][cat].append(name)
permissions[name] = tool.tool_permissions.get(name, "ask")
if group:
service_groups.setdefault(group, [])
if svc not in service_groups[group]:
service_groups[group].append(svc)
all_read = [n for s in services.values() for n in s["read"]]
all_write = [n for s in services.values() for n in s["write"]]
permissions["_categories"] = {"read": all_read, "write": all_write}
permissions["_services"] = services
permissions["_service_groups"] = service_groups
permissions["_tool_descriptions"] = {t["name"]: t["description"] for t in raw_tools}
permissions["_tool_schemas"] = {t["name"]: t.get("inputSchema") for t in raw_tools if t.get("inputSchema")}
tool.tool_permissions = permissions
_store.save(tool)
return {"ok": True, "tool": tool.model_dump()}
backend/apps/tools_lib/oauth.py
+245
View File
@@ -0,0 +1,245 @@
"""OAuth flow logic — callback, start, disconnect, refresh."""
from __future__ import annotations
import base64
import hashlib
import json
import logging
import os
import secrets
import time
from typing import Any, Optional
from urllib.parse import urlencode
import httpx
from fastapi import HTTPException, Query
from fastapi.responses import HTMLResponse
from backend.apps.tools_lib.oauth_providers import (
OAuthProvider, OAUTH_PROVIDERS, _resolve_oauth_provider,
)
logger = logging.getLogger(__name__)
_pending_oauth: dict[str, str] = {}
_pending_pkce: dict[str, str] = {}
def _get_store():
from backend.apps.tools_lib.routes import _store
return _store
async def oauth_callback(code: str = Query(...), state: str = Query("")):
tool_id = _pending_oauth.pop(state, None)
if not tool_id:
tool_id = _pending_oauth.pop(state.split(":")[-1] if ":" in state else state, None)
if not tool_id:
return HTMLResponse("<html><body><h2>Invalid OAuth state</h2></body></html>", status_code=400)
tool = _get_store().load(tool_id)
provider = _resolve_oauth_provider(tool)
client_id = os.environ.get(provider.client_id_env, "")
client_secret = os.environ.get(provider.client_secret_env, "")
_port = os.environ.get("OPENSWARM_PORT", "8324")
redirect_uri = f"http://localhost:{_port}/api/tools/oauth/callback"
token_data: dict[str, str] = {
"code": code, "redirect_uri": redirect_uri, "grant_type": "authorization_code",
}
headers: dict[str, str] = {}
if provider.token_auth_method == "basic":
creds = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode()
headers["Authorization"] = f"Basic {creds}"
elif provider.token_auth_method == "basic_json":
creds = base64.b64encode(f"{client_id}:{client_secret}".encode()).decode()
headers["Authorization"] = f"Basic {creds}"
headers["Content-Type"] = "application/json"
else:
token_data["client_id"] = client_id
token_data["client_secret"] = client_secret
if (tool.oauth_provider or "google") == "github":
headers["Accept"] = "application/json"
code_verifier = _pending_pkce.pop(state, None)
if code_verifier:
token_data["code_verifier"] = code_verifier
async with httpx.AsyncClient(timeout=15.0) as client:
if provider.token_auth_method == "basic_json":
resp = await client.post(provider.token_url, json=token_data, headers=headers)
else:
resp = await client.post(provider.token_url, data=token_data, headers=headers)
if resp.status_code != 200:
logger.warning(f"OAuth token exchange failed: {resp.text}")
return HTMLResponse(f"<html><body><h2>Token exchange failed</h2><pre>{resp.text}</pre></body></html>", status_code=400)
tokens = resp.json()
access_token = tokens.get("access_token", "")
if provider.token_response_path and not access_token:
obj = tokens
for part in provider.token_response_path.split("."):
obj = obj.get(part, {}) if isinstance(obj, dict) else ""
if isinstance(obj, str) and obj:
access_token = obj
tool.oauth_tokens = {
"access_token": access_token,
"refresh_token": tokens.get("refresh_token", ""),
"token_expiry": time.time() + tokens.get("expires_in", 3600),
}
for response_path, env_var in provider.extra_token_fields.items():
obj_val: Any = tokens
for part in response_path.split("."):
obj_val = obj_val.get(part, "") if isinstance(obj_val, dict) else ""
if obj_val:
tool.oauth_tokens[env_var] = str(obj_val)
tool.auth_status = "connected"
if access_token and provider.userinfo_url:
try:
async with httpx.AsyncClient(timeout=10.0) as info_client:
info_resp = await info_client.get(
provider.userinfo_url,
headers={"Authorization": f"Bearer {access_token}"},
)
if info_resp.status_code == 200:
tool.connected_account_email = info_resp.json().get(provider.userinfo_field)
except Exception as e:
logger.warning(f"Failed to fetch userinfo for {tool.oauth_provider or 'google'}: {e}")
if (tool.oauth_provider or "google") == "notion" and not tool.connected_account_email:
workspace_name = tokens.get("workspace_name")
if workspace_name:
tool.connected_account_email = workspace_name
_get_store().save(tool)
return HTMLResponse("""
<html><body>
<h2 style="font-family:sans-serif;color:#22c55e">Connected successfully!</h2>
<p style="font-family:sans-serif;color:#666">You can close this window.</p>
<script>
if (window.opener) window.opener.postMessage({type:'oauth_complete', tool_id:'""" + tool_id + """'}, '*');
setTimeout(() => window.close(), 1500);
</script>
</body></html>
""")
async def oauth_disconnect(tool_id: str):
tool = _get_store().load(tool_id)
access_token = tool.oauth_tokens.get("access_token")
if access_token:
provider = _resolve_oauth_provider(tool)
revoke_url = provider.revoke_url or "https://oauth2.googleapis.com/revoke"
try:
async with httpx.AsyncClient(timeout=10.0) as client:
await client.post(
revoke_url, params={"token": access_token},
headers={"Content-Type": "application/x-www-form-urlencoded"},
)
except Exception as e:
logger.warning(f"Failed to revoke token for tool {tool.id}: {e}")
tool.oauth_tokens = {}
tool.auth_status = "configured"
tool.connected_account_email = None
_get_store().save(tool)
return {"ok": True, "tool": tool.model_dump()}
async def oauth_start(tool_id: str):
tool = _get_store().load(tool_id)
provider = _resolve_oauth_provider(tool)
client_id = os.environ.get(provider.client_id_env, "")
if not client_id:
raise HTTPException(status_code=400, detail=f"{provider.client_id_env} not set in backend .env")
_port = os.environ.get("OPENSWARM_PORT", "8324")
redirect_uri = f"http://localhost:{_port}/api/tools/oauth/callback"
provider_key = tool.oauth_provider or "google"
state = f"{provider_key}:{tool_id}"
_pending_oauth[state] = tool_id
params = {
"client_id": client_id, "redirect_uri": redirect_uri,
"response_type": "code", "state": state,
**provider.extra_auth_params,
}
if provider.scopes:
params["scope"] = " ".join(provider.scopes)
if provider.pkce_required:
code_verifier = secrets.token_urlsafe(64)
code_challenge = base64.urlsafe_b64encode(
hashlib.sha256(code_verifier.encode()).digest()
).rstrip(b"=").decode()
params["code_challenge"] = code_challenge
params["code_challenge_method"] = "S256"
_pending_pkce[state] = code_verifier
auth_url = f"{provider.auth_url}?{urlencode(params)}"
return {"auth_url": auth_url}
async def refresh_oauth_token(tool) -> Optional[str]:
"""Refresh an expired OAuth token. Returns the fresh access_token or None."""
if tool.auth_type != "oauth2":
return None
refresh_token = tool.oauth_tokens.get("refresh_token")
if not refresh_token:
return None
expiry = tool.oauth_tokens.get("token_expiry", 0)
if time.time() < expiry - 60:
return tool.oauth_tokens.get("access_token")
provider = _resolve_oauth_provider(tool)
client_id = os.environ.get(provider.client_id_env, "")
client_secret = os.environ.get(provider.client_secret_env, "")
if not client_id or not client_secret:
return None
try:
async with httpx.AsyncClient(timeout=15.0) as client:
resp = await client.post(provider.token_url, data={
"client_id": client_id, "client_secret": client_secret,
"refresh_token": refresh_token, "grant_type": "refresh_token",
})
if resp.status_code == 200:
data = resp.json()
new_token = data["access_token"]
tool.oauth_tokens["access_token"] = new_token
tool.oauth_tokens["token_expiry"] = time.time() + data.get("expires_in", 3600)
if not tool.connected_account_email and provider.userinfo_url:
try:
async with httpx.AsyncClient(timeout=10.0) as info_client:
info_resp = await info_client.get(
provider.userinfo_url,
headers={"Authorization": f"Bearer {new_token}"},
)
if info_resp.status_code == 200:
tool.connected_account_email = info_resp.json().get(provider.userinfo_field)
except Exception:
pass
_get_store().save(tool)
return new_token
except Exception as e:
logger.warning(f"OAuth token refresh failed for tool {tool.id}: {e}")
return None
refresh_google_token = refresh_oauth_token
backend/apps/tools_lib/oauth_providers.py
+166
View File
@@ -0,0 +1,166 @@
"""OAuth provider definitions — pure data, no route handlers."""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from fastapi import HTTPException
# Default OAuth credentials (public client IDs safe to embed per vendor docs).
_DEFAULT_GOOGLE_CLIENT_ID = "6741219524-8vpt07arcc5rvkdb4j1b6v9g53469ugq.apps.googleusercontent.com"
_DEFAULT_GOOGLE_CLIENT_SECRET = "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"
os.environ.setdefault("GOOGLE_OAUTH_CLIENT_ID", _DEFAULT_GOOGLE_CLIENT_ID)
os.environ.setdefault("GOOGLE_OAUTH_CLIENT_SECRET", _DEFAULT_GOOGLE_CLIENT_SECRET)
os.environ.setdefault("GITHUB_OAUTH_CLIENT_ID", "Ov23liDcwNJaKMjXY2jI")
os.environ.setdefault("GITHUB_OAUTH_CLIENT_SECRET", "b25fe39409896aad3fd5155f032e9868440002f8")
os.environ.setdefault("SLACK_CLIENT_ID", "10795695056323.10799999254534")
os.environ.setdefault("SLACK_CLIENT_SECRET", "d3a85a286bb0205157d7e4963502a91d")
os.environ.setdefault("FIGMA_CLIENT_ID", "q6WduT7UuPaO6lM88v6ddN")
os.environ.setdefault("FIGMA_CLIENT_SECRET", "dhNZdbEuyEWC15cKLwWpqTclyOSplD")
os.environ.setdefault("AIRTABLE_CLIENT_ID", "0699038b-a3a4-46b2-8fa6-690eb76fadfa")
os.environ.setdefault("AIRTABLE_CLIENT_SECRET", "187fa83c8bab8ebcd11b8f226d75e7a1f14a8174ac0494463c1a53e66a3036d0")
os.environ.setdefault("HUBSPOT_CLIENT_ID", "6f4a1d4c-6a2f-4336-9b65-2cd84e218ff6")
os.environ.setdefault("HUBSPOT_CLIENT_SECRET", "5747b5de-0800-4c35-a2da-e0655ee7ea37")
GOOGLE_AUTH_URL = "https://accounts.google.com/o/oauth2/v2/auth"
GOOGLE_TOKEN_URL = "https://oauth2.googleapis.com/token"
GOOGLE_USERINFO_URL = "https://www.googleapis.com/oauth2/v2/userinfo"
GOOGLE_SCOPES = [
"openid",
"https://www.googleapis.com/auth/userinfo.email",
"https://www.googleapis.com/auth/gmail.modify",
"https://www.googleapis.com/auth/calendar",
"https://www.googleapis.com/auth/drive",
"https://www.googleapis.com/auth/contacts.readonly",
]
@dataclass
class OAuthProvider:
auth_url: str
token_url: str
scopes: list[str]
userinfo_url: str | None
userinfo_field: str
client_id_env: str
client_secret_env: str
token_env_mapping: dict[str, str]
extra_auth_params: dict[str, str] = field(default_factory=dict)
revoke_url: str | None = None
token_response_path: str | None = None
token_auth_method: str = "form"
pkce_required: bool = False
env_value_transform: str | None = None
extra_token_fields: dict[str, str] = field(default_factory=dict)
OAUTH_PROVIDERS: dict[str, OAuthProvider] = {
"google": OAuthProvider(
auth_url=GOOGLE_AUTH_URL, token_url=GOOGLE_TOKEN_URL,
scopes=GOOGLE_SCOPES, userinfo_url=GOOGLE_USERINFO_URL,
userinfo_field="email", client_id_env="GOOGLE_OAUTH_CLIENT_ID",
client_secret_env="GOOGLE_OAUTH_CLIENT_SECRET",
token_env_mapping={
"access_token": "OAUTH_ACCESS_TOKEN",
"refresh_token": "GOOGLE_WORKSPACE_REFRESH_TOKEN",
"_client_id": "GOOGLE_WORKSPACE_CLIENT_ID",
"_client_secret": "GOOGLE_WORKSPACE_CLIENT_SECRET",
},
extra_auth_params={"access_type": "offline", "prompt": "consent"},
),
"github": OAuthProvider(
auth_url="https://github.com/login/oauth/authorize",
token_url="https://github.com/login/oauth/access_token",
scopes=["repo", "read:user", "user:email"],
userinfo_url="https://api.github.com/user", userinfo_field="login",
client_id_env="GITHUB_OAUTH_CLIENT_ID", client_secret_env="GITHUB_OAUTH_CLIENT_SECRET",
token_env_mapping={"access_token": "GITHUB_PERSONAL_ACCESS_TOKEN"},
),
"slack": OAuthProvider(
auth_url="https://slack.com/oauth/v2/authorize",
token_url="https://slack.com/api/oauth.v2.access",
scopes=[
"channels:read", "channels:history", "chat:write",
"groups:read", "groups:history", "im:read", "im:history",
"mpim:read", "mpim:history", "users:read", "users:read.email",
"team:read", "reactions:read", "reactions:write",
"files:read", "files:write",
],
userinfo_url="https://slack.com/api/auth.test", userinfo_field="user",
client_id_env="SLACK_CLIENT_ID", client_secret_env="SLACK_CLIENT_SECRET",
token_env_mapping={"access_token": "SLACK_BOT_TOKEN"},
extra_token_fields={"team.id": "SLACK_TEAM_ID"},
),
"notion": OAuthProvider(
auth_url="https://api.notion.com/v1/oauth/authorize",
token_url="https://api.notion.com/v1/oauth/token",
scopes=[], userinfo_url=None, userinfo_field="owner",
client_id_env="NOTION_OAUTH_CLIENT_ID", client_secret_env="NOTION_OAUTH_CLIENT_SECRET",
token_env_mapping={"access_token": "OPENAPI_MCP_HEADERS"},
extra_auth_params={"owner": "user"},
token_auth_method="basic_json", env_value_transform="notion_headers",
),
"spotify": OAuthProvider(
auth_url="https://accounts.spotify.com/authorize",
token_url="https://accounts.spotify.com/api/token",
scopes=[
"user-read-playback-state", "user-modify-playback-state",
"user-read-currently-playing", "playlist-read-private",
"playlist-modify-public", "playlist-modify-private",
"user-library-read", "user-library-modify",
"user-read-recently-played", "user-top-read",
],
userinfo_url="https://api.spotify.com/v1/me", userinfo_field="display_name",
client_id_env="SPOTIFY_CLIENT_ID", client_secret_env="SPOTIFY_CLIENT_SECRET",
token_env_mapping={
"access_token": "SPOTIFY_ACCESS_TOKEN",
"refresh_token": "SPOTIFY_REFRESH_TOKEN",
"_client_id": "SPOTIFY_CLIENT_ID",
"_client_secret": "SPOTIFY_CLIENT_SECRET",
},
token_auth_method="basic",
),
"figma": OAuthProvider(
auth_url="https://www.figma.com/oauth",
token_url="https://api.figma.com/v1/oauth/token",
scopes=["current_user:read", "file_content:read", "file_metadata:read", "file_comments:read", "file_comments:write", "file_versions:read", "file_variables:read"],
userinfo_url="https://api.figma.com/v1/me", userinfo_field="email",
client_id_env="FIGMA_CLIENT_ID", client_secret_env="FIGMA_CLIENT_SECRET",
token_env_mapping={"access_token": "FIGMA_API_KEY"},
),
"airtable": OAuthProvider(
auth_url="https://airtable.com/oauth2/v1/authorize",
token_url="https://airtable.com/oauth2/v1/token",
scopes=[
"data.records:read", "data.records:write",
"data.recordComments:read", "data.recordComments:write",
"schema.bases:read", "schema.bases:write",
"user.email:read", "webhook:manage",
],
userinfo_url="https://api.airtable.com/v0/meta/whoami", userinfo_field="email",
client_id_env="AIRTABLE_CLIENT_ID", client_secret_env="AIRTABLE_CLIENT_SECRET",
token_env_mapping={"access_token": "AIRTABLE_API_KEY"},
pkce_required=True, token_auth_method="basic",
),
"hubspot": OAuthProvider(
auth_url="https://mcp-na2.hubspot.com/oauth/authorize/user",
token_url="https://api.hubapi.com/oauth/v1/token",
scopes=[], userinfo_url=None, userinfo_field="user",
client_id_env="HUBSPOT_CLIENT_ID", client_secret_env="HUBSPOT_CLIENT_SECRET",
token_env_mapping={
"access_token": "PRIVATE_APP_ACCESS_TOKEN",
"refresh_token": "HUBSPOT_REFRESH_TOKEN",
},
pkce_required=True,
),
}
def _resolve_oauth_provider(tool) -> OAuthProvider:
"""Resolve the OAuth provider for a tool, defaulting to Google."""
key = tool.oauth_provider or "google"
provider = OAUTH_PROVIDERS.get(key)
if not provider:
raise HTTPException(status_code=400, detail=f"Unknown OAuth provider: {key}")
return provider
backend/apps/tools_lib/routes.py
+84
View File
@@ -0,0 +1,84 @@
"""Tool CRUD endpoints and builtin permission management."""
from __future__ import annotations
import json
import os
from fastapi import HTTPException
from backend.apps.common.json_store import JsonStore
from backend.apps.tools_lib.models import ToolDefinition, ToolCreate, ToolUpdate, BUILTIN_TOOLS
from backend.config.paths import TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH
_store = JsonStore(ToolDefinition, DATA_DIR, not_found_detail="Tool not found")
_load_all = _store.load_all
_save = _store.save
_load = _store.load
def load_builtin_permissions() -> dict[str, str]:
if not os.path.exists(BUILTIN_PERMS_PATH):
return {}
with open(BUILTIN_PERMS_PATH) as f:
return json.load(f)
def save_builtin_permissions(perms: dict[str, str]):
os.makedirs(os.path.dirname(BUILTIN_PERMS_PATH), exist_ok=True)
with open(BUILTIN_PERMS_PATH, "w") as f:
json.dump(perms, f, indent=2)
async def list_builtin_tools():
return {"tools": [t.model_dump() for t in BUILTIN_TOOLS]}
async def get_builtin_permissions():
return {"permissions": load_builtin_permissions()}
async def update_builtin_permissions(body: dict):
valid_tools = {t.name for t in BUILTIN_TOOLS}
valid_policies = {"always_allow", "ask", "deny"}
perms = load_builtin_permissions()
for name, policy in body.get("permissions", {}).items():
if name in valid_tools and policy in valid_policies:
perms[name] = policy
save_builtin_permissions(perms)
return {"permissions": perms}
async def list_tools():
return {"tools": [t.model_dump() for t in _load_all()]}
async def get_tool(tool_id: str):
return _load(tool_id).model_dump()
async def create_tool(body: ToolCreate):
tool = ToolDefinition(
name=body.name, description=body.description,
command=body.command, mcp_config=body.mcp_config,
credentials=body.credentials, auth_type=body.auth_type,
auth_status=body.auth_status, oauth_provider=body.oauth_provider,
)
_save(tool)
return {"ok": True, "tool": tool.model_dump()}
async def update_tool(tool_id: str, body: ToolUpdate):
tool = _load(tool_id)
for k, v in body.model_dump(exclude_none=True).items():
setattr(tool, k, v)
_save(tool)
return {"ok": True, "tool": tool.model_dump()}
async def delete_tool(tool_id: str):
path = os.path.join(DATA_DIR, f"{tool_id}.json")
if os.path.exists(path):
os.remove(path)
return {"ok": True}
backend/apps/tools_lib/tools_lib.py
+16 -1112
View File
File diff suppressed because it is too large Load Diff
assets/icon.png → readme_assets/icon.png
View File
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 320 KiB

After

Width:  |  Height:  |  Size: 320 KiB

assets/screenshot.png → readme_assets/screenshot.png
View File
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 64 KiB

After

Width:  |  Height:  |  Size: 64 KiB