Merge pull request #7 from openswarm-ai/eric/cute

v1.0.12 — Multi-provider agents, 9Router subscriptions, onboarding
This commit is contained in:
ciregenz
2026-03-23 18:56:23 -07:00
committed by GitHub
130 changed files with 18689 additions and 1500 deletions
+2
View File
@@ -8,7 +8,9 @@ backend/data/**
# Electron build artifacts
electron/dist/
electron/python-env/
electron/build-staging/
electron/node_modules/
electron/package-lock.json
# Frontend build output
frontend/dist/
-8
View File
@@ -229,14 +229,6 @@ Please open an issue first for larger changes so we can discuss the approach.
<br>
## Community
- [Twitter / X](https://twitter.com/openswarm_ai)
- [Discord](https://discord.gg/openswarm)
- [Website](https://openswarm.ai)
<br>
## License
MIT — see [LICENSE](LICENSE) for details.
+22
View File
@@ -17,3 +17,25 @@ APPLE_TEAM_ID=ABCDE12345
# GitHub Releases (required for --publish)
# =============================================================================
GH_TOKEN=ghp_your-github-personal-access-token
# =============================================================================
# Channels: SMS, WhatsApp, Voice Calling
# =============================================================================
TWILIO_ACCOUNT_SID=your-twilio-account-sid
TWILIO_AUTH_TOKEN=your-twilio-auth-token
TWILIO_PHONE_NUMBER=+1234567890
TELNYX_API_KEY=your-telnyx-api-key
TELNYX_PUBLIC_KEY=your-telnyx-public-key
# =============================================================================
# TTS / STT Providers
# =============================================================================
ELEVENLABS_API_KEY=your-elevenlabs-api-key
DEEPGRAM_API_KEY=your-deepgram-api-key
OPENAI_API_KEY=your-openai-api-key
# =============================================================================
# Webhook URL (required for inbound SMS/calls — use ngrok or Tailscale)
# =============================================================================
WEBHOOK_BASE_URL=https://your-public-url.ngrok.io
+331
View File
@@ -0,0 +1,331 @@
"""Owned agent loop — replaces claude_agent_sdk's query() function.
Generalizes the pattern from browser_agent.py (lines 243-334) into a
provider-agnostic, streaming, HITL-aware tool-use loop.
"""
from __future__ import annotations
import json
import logging
import time
from typing import Any, Callable, Awaitable
from uuid import uuid4
from backend.apps.agents.providers.base import (
BaseProvider, ContentBlock, ModelResponse, ProviderMessage,
StreamEvent, ToolCall, ToolSchema,
)
logger = logging.getLogger(__name__)
# Type aliases for callbacks
ToolExecutor = Callable[[str, dict], Awaitable[list[dict]]]
# hitl_handler(tool_name, tool_input) -> (approved, updated_input_or_None)
HITLHandler = Callable[[str, dict], Awaitable[tuple[bool, dict | None]]]
# ws_emitter(event_type, data) -> None
WSEmitter = Callable[[str, dict], Awaitable[None]]
class AgentLoop:
"""Provider-agnostic agent loop with streaming and HITL support.
The loop:
1. Sends user message to the model
2. Streams the response (emitting WebSocket events)
3. If the model requests tool use:
a. For each tool call: check HITL permission → execute → collect result
b. Append tool results → go to step 2
4. If the model stops (end_turn/max_tokens): done
"""
def __init__(
self,
session_id: str,
provider: BaseProvider,
model: str,
system_prompt: str | None,
tools: list[ToolSchema],
tool_executor: ToolExecutor,
hitl_handler: HITLHandler,
ws_emitter: WSEmitter,
max_turns: int | None = None,
cwd: str | None = None,
):
self.session_id = session_id
self.provider = provider
self.model = model
self.system_prompt = system_prompt
self.tools = tools
self.tool_executor = tool_executor
self.hitl_handler = hitl_handler
self.ws_emitter = ws_emitter
self.max_turns = max_turns
self.cwd = cwd
# Conversation history in provider-agnostic format
self.messages: list[ProviderMessage] = []
# Token tracking
self.total_input_tokens = 0
self.total_output_tokens = 0
async def run(self, user_content: Any) -> None:
"""Run the agent loop for a single user turn."""
# Append user message
user_msg = self.provider.format_user_message(user_content)
self.messages.append(user_msg)
turn = 0
while True:
if self.max_turns and turn >= self.max_turns:
logger.info(f"Agent {self.session_id}: max turns ({self.max_turns}) reached")
break
turn += 1
# Stream the model response and collect it
response = await self._stream_and_collect()
# Track usage
self.total_input_tokens += response.usage.get("input_tokens", 0)
self.total_output_tokens += response.usage.get("output_tokens", 0)
# Append assistant message to conversation history
assistant_msg = self.provider.format_assistant_message(response)
self.messages.append(assistant_msg)
# If no tool use, we're done
if response.stop_reason != "tool_use":
break
# Execute tools
tool_results = await self._execute_tools(response)
if not tool_results:
break
# Append tool results
self.messages.append(ProviderMessage(role="tool_result", content=tool_results))
async def _stream_and_collect(self) -> ModelResponse:
"""Stream model output, emit WebSocket events, collect full response."""
collected_content: list[ContentBlock] = []
collected_usage: dict[str, int] = {}
stop_reason = "end_turn"
# Track streaming state for WS emissions
stream_text_msg_id: str | None = None
stream_tool_msg_ids: dict[int, str] = {} # block index -> msg_id
block_index_map: dict[int, str] = {} # block index -> msg_id
# Buffers for collecting content
text_buffers: dict[int, str] = {}
json_buffers: dict[int, str] = {}
tool_names: dict[int, str] = {}
tool_ids: dict[int, str] = {}
block_types: dict[int, str] = {}
async for event in self.provider.stream_message(
model=self.model,
system=self.system_prompt,
messages=self.messages,
tools=self.tools,
):
if event.type == "content_block_start":
if event.block_type == "text":
if stream_text_msg_id is None:
stream_text_msg_id = uuid4().hex
await self.ws_emitter("agent:stream_start", {
"message_id": stream_text_msg_id,
"role": "assistant",
})
block_index_map[event.index] = stream_text_msg_id
block_types[event.index] = "text"
text_buffers[event.index] = ""
elif event.block_type == "tool_use":
tool_msg_id = uuid4().hex
stream_tool_msg_ids[event.index] = tool_msg_id
block_index_map[event.index] = tool_msg_id
block_types[event.index] = "tool_use"
tool_names[event.index] = event.tool_name
tool_ids[event.index] = event.tool_id
json_buffers[event.index] = ""
await self.ws_emitter("agent:stream_start", {
"message_id": tool_msg_id,
"role": "tool_call",
"tool_name": event.tool_name,
})
elif event.type == "content_block_delta":
msg_id = block_index_map.get(event.index)
if not msg_id:
continue
if event.delta_type == "text_delta":
text_buffers.setdefault(event.index, "")
text_buffers[event.index] += event.text
await self.ws_emitter("agent:stream_delta", {
"message_id": msg_id,
"delta": event.text,
})
elif event.delta_type == "input_json_delta":
json_buffers.setdefault(event.index, "")
json_buffers[event.index] += event.text
await self.ws_emitter("agent:stream_delta", {
"message_id": msg_id,
"delta": event.text,
})
elif event.type == "content_block_stop":
msg_id = block_index_map.get(event.index)
bt = block_types.get(event.index, "")
if bt == "text":
collected_content.append(
ContentBlock(type="text", text=text_buffers.get(event.index, ""))
)
elif bt == "tool_use":
try:
tool_input = json.loads(json_buffers.get(event.index, "{}"))
except json.JSONDecodeError:
tool_input = {}
collected_content.append(ContentBlock(
type="tool_use",
tool_call=ToolCall(
id=tool_ids.get(event.index, uuid4().hex),
name=tool_names.get(event.index, ""),
input=tool_input,
),
))
# Send stream_end for tool blocks (text block ends at message_stop)
if msg_id and bt == "tool_use":
await self.ws_emitter("agent:stream_end", {
"message_id": msg_id,
})
elif event.type == "usage":
# Accumulate token usage from provider stream
for k, v in event.usage.items():
collected_usage[k] = collected_usage.get(k, 0) + v
elif event.type == "message_stop":
# Check if any tool calls means stop_reason is tool_use
has_tool_use = any(b.type == "tool_use" for b in collected_content)
if has_tool_use:
stop_reason = "tool_use"
# End text stream
if stream_text_msg_id:
await self.ws_emitter("agent:stream_end", {
"message_id": stream_text_msg_id,
})
# Build and emit the collected messages
await self._emit_collected_messages(
collected_content, stream_text_msg_id, stream_tool_msg_ids,
)
return ModelResponse(
content=collected_content,
stop_reason=stop_reason,
usage=collected_usage,
)
async def _emit_collected_messages(
self,
content: list[ContentBlock],
text_msg_id: str | None,
tool_msg_ids: dict[int, str],
) -> None:
"""Emit finalized agent:message events for the collected response."""
from backend.apps.agents.models import Message
# Emit text message
text_parts = [b.text for b in content if b.type == "text" and b.text]
if text_parts:
msg = Message(
id=text_msg_id or uuid4().hex,
role="assistant",
content="\n".join(text_parts),
)
await self.ws_emitter("agent:message", {
"message": msg.model_dump(mode="json"),
})
# Emit tool call messages
tool_blocks = [b for b in content if b.type == "tool_use" and b.tool_call]
tool_id_list = sorted(tool_msg_ids.items(), key=lambda x: x[0])
for i, block in enumerate(tool_blocks):
tc = block.tool_call
msg_id = tool_id_list[i][1] if i < len(tool_id_list) else uuid4().hex
msg = Message(
id=msg_id,
role="tool_call",
content={
"id": tc.id,
"tool": tc.name,
"input": tc.input,
},
)
await self.ws_emitter("agent:message", {
"message": msg.model_dump(mode="json"),
})
async def _execute_tools(self, response: ModelResponse) -> list[dict]:
"""Execute all tool calls from a response, respecting HITL permissions.
Returns a list of tool result dicts formatted for the provider.
"""
from backend.apps.agents.models import Message
results = []
for block in response.content:
if block.type != "tool_use" or not block.tool_call:
continue
tc = block.tool_call
start_time = time.time()
# HITL permission check
approved, updated_input = await self.hitl_handler(tc.name, tc.input)
if not approved:
result_content = [{"type": "text", "text": "Tool use was denied by the user."}]
else:
tool_input = updated_input if updated_input else tc.input
try:
result_content = await self.tool_executor(tc.name, tool_input)
except Exception as e:
logger.warning(f"Tool execution error: {tc.name}: {e}")
result_content = [{"type": "text", "text": f"Error executing {tc.name}: {e}"}]
elapsed_ms = int((time.time() - start_time) * 1000)
# Emit tool result to frontend
result_text = ""
for block_item in result_content:
if isinstance(block_item, dict) and block_item.get("type") == "text":
result_text = block_item.get("text", "")
break
result_msg = Message(
role="tool_result",
content={
"text": result_text[:15000] if result_text else "Done.",
"tool_name": tc.name,
"elapsed_ms": elapsed_ms,
},
)
await self.ws_emitter("agent:message", {
"message": result_msg.model_dump(mode="json"),
})
# Format for provider
results.append(
self.provider.format_tool_result(tc.id, result_content)
)
return results
File diff suppressed because it is too large Load Diff
+216
View File
@@ -52,10 +52,13 @@ async def send_message(session_id: str, body: dict):
prompt,
mode=body.get("mode"),
model=body.get("model"),
provider=body.get("provider"),
images=body.get("images"),
context_paths=body.get("context_paths"),
forced_tools=body.get("forced_tools"),
attached_skills=body.get("attached_skills"),
hidden=body.get("hidden", False),
selected_browser_ids=body.get("selected_browser_ids"),
)
return {"ok": True}
@@ -131,6 +134,18 @@ async def get_branches(session_id: str):
"active_branch_id": session.active_branch_id,
}
@agents.router.post("/sessions/{session_id}/duplicate")
async def duplicate_session(session_id: str, body: dict = {}):
try:
session = await agent_manager.duplicate_session(
session_id,
dashboard_id=body.get("dashboard_id"),
up_to_message_id=body.get("up_to_message_id"),
)
except ValueError as e:
raise HTTPException(status_code=404, detail=str(e))
return {"session": session.model_dump(mode="json")}
@agents.router.post("/sessions/{session_id}/close")
async def close_session(session_id: str):
try:
@@ -151,6 +166,11 @@ async def get_history(q: str = "", limit: int = 20, offset: int = 0, dashboard_i
dashboard_id=dashboard_id or None,
)
@agents.router.get("/sessions/{session_id}/browser-agents")
async def get_browser_agent_children(session_id: str):
children = agent_manager.get_browser_agent_children(session_id)
return {"sessions": children}
@agents.router.post("/sessions/{session_id}/resume")
async def resume_session(session_id: str):
try:
@@ -159,3 +179,199 @@ async def resume_session(session_id: str):
raise HTTPException(status_code=404, detail=str(e))
return {"session": session.model_dump(mode="json")}
@agents.router.get("/models")
async def list_models():
"""Return available models grouped by provider, filtered by configured credentials."""
from backend.apps.agents.providers.registry import get_available_models
from backend.apps.settings.settings import load_settings
settings = load_settings()
return {"models": get_available_models(settings)}
# ── GitHub Copilot Auth ──
@agents.router.post("/copilot/start-auth")
async def copilot_start_auth():
"""Start GitHub device flow for Copilot auth."""
from backend.apps.agents.copilot_auth import start_device_flow
try:
result = await start_device_flow()
return result
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@agents.router.post("/copilot/poll-auth")
async def copilot_poll_auth(body: dict):
"""Poll for GitHub auth completion. Returns token on success."""
from backend.apps.agents.copilot_auth import poll_for_token, exchange_for_copilot_token, get_github_username, list_copilot_models
from backend.apps.settings.settings import load_settings, _save_settings
device_code = body.get("device_code", "")
if not device_code:
raise HTTPException(status_code=400, detail="device_code required")
try:
github_token = await poll_for_token(device_code)
if github_token is None:
return {"status": "pending"}
# Got GitHub token — exchange for Copilot token
copilot_result = await exchange_for_copilot_token(github_token)
username = await get_github_username(github_token)
# Fetch available models
models = await list_copilot_models(copilot_result["token"])
# Save to settings
settings = load_settings()
settings.copilot_github_token = github_token
settings.copilot_token = copilot_result["token"]
settings.copilot_token_expires = copilot_result["expires_at"]
_save_settings(settings)
return {
"status": "connected",
"username": username,
"models": models,
}
except ValueError as e:
raise HTTPException(status_code=400, detail=str(e))
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@agents.router.get("/copilot/models")
async def copilot_models():
"""List models available through Copilot."""
from backend.apps.agents.copilot_auth import list_copilot_models, get_copilot_token
from backend.apps.settings.settings import load_settings, _save_settings
settings = load_settings()
github_token = getattr(settings, "copilot_github_token", None)
if not github_token:
return {"models": []}
try:
result = await get_copilot_token(
github_token,
getattr(settings, "copilot_token", None),
getattr(settings, "copilot_token_expires", None),
)
settings.copilot_token = result["token"]
settings.copilot_token_expires = result["expires_at"]
_save_settings(settings)
models = await list_copilot_models(result["token"])
return {"models": models}
except Exception as e:
return {"models": [], "error": str(e)}
@agents.router.post("/copilot/disconnect")
async def copilot_disconnect():
"""Clear Copilot tokens."""
from backend.apps.settings.settings import load_settings, _save_settings
settings = load_settings()
settings.copilot_github_token = None
settings.copilot_token = None
settings.copilot_token_expires = None
_save_settings(settings)
return {"ok": True}
# ── Subscription Management (via 9Router) ──
@agents.router.get("/subscriptions/status")
async def subscriptions_status():
"""Check if 9Router is running and list connected providers."""
from backend.apps.nine_router import is_running, get_providers, get_models
if not is_running():
return {"running": False, "providers": [], "models": []}
providers = await get_providers()
models = await get_models()
return {"running": True, "providers": providers, "models": models}
@agents.router.post("/subscriptions/connect")
async def subscriptions_connect(body: dict):
"""Start OAuth flow for a subscription provider."""
from backend.apps.nine_router import is_running, ensure_running, start_oauth
provider = body.get("provider", "")
if not provider:
raise HTTPException(status_code=400, detail="provider required")
if not is_running():
import asyncio
await ensure_running()
if not is_running():
raise HTTPException(status_code=503, detail="9Router not available. Please install Node.js.")
try:
result = await start_oauth(provider)
# For auth_code flows, store pending state so the callback can exchange
if result.get("flow") == "authorization_code" and result.get("state"):
from backend.main import _pending_oauth
_pending_oauth[result["state"]] = {
"provider": provider,
"code_verifier": result.get("code_verifier", ""),
"redirect_uri": result.get("redirect_uri", ""),
}
return result
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@agents.router.post("/subscriptions/poll")
async def subscriptions_poll(body: dict):
"""Poll for OAuth completion."""
from backend.apps.nine_router import poll_oauth
provider = body.get("provider", "")
device_code = body.get("device_code", "")
if not provider or not device_code:
raise HTTPException(status_code=400, detail="provider and device_code required")
try:
result = await poll_oauth(
provider, device_code,
code_verifier=body.get("code_verifier"),
extra_data=body.get("extra_data"),
)
return result
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@agents.router.post("/subscriptions/exchange")
async def subscriptions_exchange(body: dict):
"""Exchange OAuth code for tokens via 9Router."""
from backend.apps.nine_router import exchange_oauth
provider = body.get("provider", "")
code = body.get("code", "")
redirect_uri = body.get("redirect_uri", "")
code_verifier = body.get("code_verifier", "")
state = body.get("state", "")
if not provider or not code:
raise HTTPException(status_code=400, detail="provider and code required")
try:
result = await exchange_oauth(provider, code, redirect_uri, code_verifier, state)
return result
except Exception as e:
raise HTTPException(status_code=500, detail=str(e))
@agents.router.get("/subscriptions/models")
async def subscriptions_models():
"""List all models available through connected subscriptions."""
from backend.apps.nine_router import is_running, get_models
if not is_running():
return {"models": []}
models = await get_models()
return {"models": models}
+274 -34
View File
@@ -15,15 +15,16 @@ from uuid import uuid4
import anthropic
from backend.apps.agents.models import AgentSession, Message
from backend.apps.agents.models import AgentSession, ApprovalRequest, Message
from backend.apps.agents.ws_manager import ws_manager
from backend.apps.tools_lib.tools_lib import load_builtin_permissions
logger = logging.getLogger(__name__)
MODEL_MAP = {
"sonnet": "claude-sonnet-4-20250514",
"opus": "claude-opus-4-20250514",
"haiku": "claude-haiku-4-20250414",
"haiku": "claude-haiku-4-5-20251001",
}
BROWSER_TOOLS_SCHEMA = [
@@ -112,6 +113,48 @@ BROWSER_TOOLS_SCHEMA = [
"required": [],
},
},
{
"name": "BrowserScroll",
"description": (
"Scroll the page up or down. Automatically finds the correct scrollable "
"container (works on SPAs like Notion, Gmail, etc. that use nested scroll "
"containers instead of window-level scrolling). Returns scroll position info "
"including whether top/bottom has been reached."
),
"input_schema": {
"type": "object",
"properties": {
"direction": {
"type": "string",
"enum": ["up", "down"],
"description": "Scroll direction. Defaults to 'down'.",
},
"amount": {
"type": "number",
"description": "Pixels to scroll. Defaults to 500.",
},
},
"required": [],
},
},
{
"name": "BrowserWait",
"description": (
"Wait for a specified duration. Useful after navigation or actions that "
"trigger page loads, animations, or async content rendering. "
"Min 100ms, max 10000ms."
),
"input_schema": {
"type": "object",
"properties": {
"milliseconds": {
"type": "number",
"description": "Duration to wait in milliseconds. Defaults to 1000.",
},
},
"required": [],
},
},
]
ACTION_MAP = {
@@ -122,17 +165,29 @@ ACTION_MAP = {
"BrowserType": "type",
"BrowserEvaluate": "evaluate",
"BrowserGetElements": "get_elements",
"BrowserScroll": "scroll",
"BrowserWait": "wait",
}
SYSTEM_PROMPT = (
"You are a browser automation agent. You control a single browser tab and "
"execute the task you are given.\n\n"
"Strategy:\n"
"1. Start by taking a screenshot or calling BrowserGetElements to understand the page.\n"
"2. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n"
"3. After performing actions, take a screenshot to verify the result.\n"
"4. If an action fails, try alternative selectors or approaches.\n"
"5. When the task is complete, provide a clear summary of what you accomplished.\n\n"
"1. Start by taking a screenshot to understand the page.\n"
"2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n"
"3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with "
"window.scrollBy() as many sites use nested scroll containers that BrowserScroll "
"handles automatically.\n"
"4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n"
"5. After performing actions, take a screenshot to verify the result.\n"
"6. If an action fails, try alternative selectors or approaches.\n"
"7. When the task is complete, provide a clear summary of what you accomplished.\n\n"
"Important notes:\n"
"- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n"
"- BrowserScroll returns position info including atTop/atBottom — use this to know when "
"you've reached the end of the page.\n"
"- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n"
"- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n"
"You have access ONLY to browser tools. Do not ask the user questions — "
"complete the task autonomously to the best of your ability."
)
@@ -143,16 +198,29 @@ MAX_TURNS = 25
async def execute_browser_tool(
tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "",
) -> dict:
"""Execute a browser tool via ws_manager directly (no MCP/HTTP round-trip)."""
"""Execute a browser tool. Tries Electron webview first, falls back to headless Playwright."""
action = ACTION_MAP.get(tool_name)
if not action:
return {"error": f"Unknown browser tool: {tool_name}"}
params = {k: v for k, v in tool_input.items()}
request_id = uuid4().hex
# Try Electron webview first
result = await ws_manager.send_browser_command(
request_id, action, browser_id, params, tab_id=tab_id,
)
# If Electron webview not available, fall back to headless Playwright
if isinstance(result, dict) and result.get("error") and "not found" in result.get("error", "").lower():
try:
from backend.apps.agents.headless_browser import execute as headless_execute
result = await headless_execute(browser_id, action, params)
except ImportError:
return {"error": "Browser not available. Install playwright: pip install playwright && playwright install chromium"}
except Exception as e:
return {"error": f"Headless browser error: {e}"}
return result
@@ -179,6 +247,40 @@ def _format_tool_result(result: dict, tool_name: str) -> list[dict]:
return [{"type": "text", "text": str(text)}]
async def _request_browser_approval(
session: AgentSession, tool_name: str, tool_input: dict,
) -> dict:
"""Send an approval request for a browser sub-agent tool and wait for the decision."""
request_id = uuid4().hex
approval_req = ApprovalRequest(
id=request_id,
session_id=session.id,
tool_name=tool_name,
tool_input=tool_input,
)
session.pending_approvals.append(approval_req)
session.status = "waiting_approval"
await ws_manager.send_to_session(session.id, "agent:status", {
"session_id": session.id,
"status": "waiting_approval",
})
decision = await ws_manager.send_approval_request(
session.id, request_id, tool_name, tool_input,
)
session.pending_approvals = [
a for a in session.pending_approvals if a.id != request_id
]
session.status = "running"
await ws_manager.send_to_session(session.id, "agent:status", {
"session_id": session.id,
"status": "running",
})
return decision
async def run_browser_agent(
task: str,
browser_id: str,
@@ -188,6 +290,9 @@ async def run_browser_agent(
tab_id: str = "",
pre_selected: bool = False,
initial_url: str | None = None,
parent_session_id: str | None = None,
auth_token: str | None = None,
base_url: str | None = None,
) -> dict:
"""Run a browser sub-agent loop for a single browser card.
@@ -196,7 +301,10 @@ async def run_browser_agent(
"""
from backend.apps.agents.agent_manager import agent_manager
_browser_perms = load_builtin_permissions()
session_id = uuid4().hex
cancel_event = asyncio.Event()
session = AgentSession(
id=session_id,
name=f"Browser Agent",
@@ -206,7 +314,9 @@ async def run_browser_agent(
dashboard_id=dashboard_id,
browser_id=browser_id,
system_prompt=SYSTEM_PROMPT,
parent_session_id=parent_session_id,
)
session._cancel_event = cancel_event
agent_manager.sessions[session_id] = session
await ws_manager.send_to_session(session_id, "agent:status", {
@@ -222,7 +332,17 @@ async def run_browser_agent(
logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}")
api_model = MODEL_MAP.get(model, model)
client = anthropic.AsyncAnthropic(api_key=api_key)
# Use OpenAI client for 9Router, Anthropic client for direct API
_use_openai_client = base_url is not None
if _use_openai_client:
from openai import AsyncOpenAI
# Map to 9Router model IDs
_9r_map = {"sonnet": "cc/claude-sonnet-4-6", "opus": "cc/claude-opus-4-6", "haiku": "cc/claude-haiku-4-5-20251001"}
api_model = _9r_map.get(model, f"cc/{api_model}" if not api_model.startswith("cc/") else api_model)
client = AsyncOpenAI(api_key=api_key, base_url=base_url)
else:
client = anthropic.AsyncAnthropic(api_key=api_key)
messages: list[dict] = [{"role": "user", "content": task}]
action_log: list[dict] = []
@@ -237,30 +357,69 @@ async def run_browser_agent(
try:
for turn in range(MAX_TURNS):
response = await client.messages.create(
model=api_model,
max_tokens=4096,
system=SYSTEM_PROMPT,
tools=BROWSER_TOOLS_SCHEMA,
messages=messages,
)
if cancel_event.is_set():
break
assistant_content = []
text_parts = []
tool_uses = []
if _use_openai_client:
# OpenAI-compatible format (9Router)
import json as _json
oai_tools = [{"type": "function", "function": {"name": t["name"], "description": t["description"], "parameters": t["input_schema"]}} for t in BROWSER_TOOLS_SCHEMA]
oai_messages = [{"role": "system", "content": SYSTEM_PROMPT}] + messages
resp = await client.chat.completions.create(model=api_model, max_tokens=4096, tools=oai_tools, messages=oai_messages)
choice = resp.choices[0]
for block in response.content:
if block.type == "text":
text_parts.append(block.text)
assistant_content.append({"type": "text", "text": block.text})
elif block.type == "tool_use":
tool_uses.append(block)
assistant_content.append({
"type": "tool_use",
"id": block.id,
"name": block.name,
"input": block.input,
})
assistant_content = []
text_parts = []
tool_uses = []
if choice.message.content:
text_parts.append(choice.message.content)
assistant_content.append({"type": "text", "text": choice.message.content})
if choice.message.tool_calls:
for tc in choice.message.tool_calls:
try:
inp = _json.loads(tc.function.arguments)
except Exception:
inp = {}
# Create a simple object with .id, .name, .input
class _TC:
pass
tool_obj = _TC()
tool_obj.id = tc.id
tool_obj.name = tc.function.name
tool_obj.input = inp
tool_uses.append(tool_obj)
assistant_content.append({"type": "tool_use", "id": tc.id, "name": tc.function.name, "input": inp})
stop_reason = "tool_use" if choice.message.tool_calls else "end_turn"
else:
# Anthropic format (direct API)
response = await client.messages.create(
model=api_model,
max_tokens=4096,
system=SYSTEM_PROMPT,
tools=BROWSER_TOOLS_SCHEMA,
messages=messages,
)
assistant_content = []
text_parts = []
tool_uses = []
stop_reason = response.stop_reason
for block in response.content:
if block.type == "text":
text_parts.append(block.text)
assistant_content.append({"type": "text", "text": block.text})
elif block.type == "tool_use":
tool_uses.append(block)
assistant_content.append({
"type": "tool_use",
"id": block.id,
"name": block.name,
"input": block.input,
})
if text_parts:
asst_msg = Message(
@@ -284,13 +443,67 @@ async def run_browser_agent(
"message": tool_msg.model_dump(mode="json"),
})
messages.append({"role": "assistant", "content": assistant_content})
if _use_openai_client:
# OpenAI format: assistant message with tool_calls
asst_api_msg: dict = {"role": "assistant", "content": choice.message.content}
if choice.message.tool_calls:
asst_api_msg["tool_calls"] = [{"id": tc.id, "type": "function", "function": {"name": tc.function.name, "arguments": tc.function.arguments}} for tc in (choice.message.tool_calls or [])]
messages.append(asst_api_msg)
else:
messages.append({"role": "assistant", "content": assistant_content})
if response.stop_reason != "tool_use":
if stop_reason != "tool_use":
break
tool_results = []
cancelled = False
for tu in tool_uses:
if cancel_event.is_set():
cancelled = True
break
policy = _browser_perms.get(tu.name, "always_allow")
if policy == "deny":
denied_text = f"Tool {tu.name} is denied by permission policy."
tool_results.append({
"type": "tool_result",
"tool_use_id": tu.id,
"content": [{"type": "text", "text": denied_text}],
})
result_msg = Message(
role="tool_result",
content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0},
)
session.messages.append(result_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": result_msg.model_dump(mode="json"),
})
continue
if policy == "ask":
decision = await _request_browser_approval(
session, tu.name, tu.input,
)
if decision.get("behavior") == "deny":
denied_text = decision.get("message") or f"Tool {tu.name} denied by user."
tool_results.append({
"type": "tool_result",
"tool_use_id": tu.id,
"content": [{"type": "text", "text": denied_text}],
})
result_msg = Message(
role="tool_result",
content={"text": denied_text, "tool_name": tu.name, "elapsed_ms": 0},
)
session.messages.append(result_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": result_msg.model_dump(mode="json"),
})
continue
start = time.time()
result = await execute_browser_tool(
tu.name, tu.input, browser_id, tab_id,
@@ -325,7 +538,26 @@ async def run_browser_agent(
"message": result_msg.model_dump(mode="json"),
})
messages.append({"role": "user", "content": tool_results})
if _use_openai_client:
# OpenAI format: each tool result is a separate message
for tr in tool_results:
text_content = ""
for block in (tr.get("content") or []):
if isinstance(block, dict) and block.get("type") == "text":
text_content += block.get("text", "")
messages.append({"role": "tool", "tool_call_id": tr["tool_use_id"], "content": text_content or "Done."})
else:
messages.append({"role": "user", "content": tool_results})
if cancelled:
break
if cancel_event.is_set():
session.status = "stopped"
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id,
"status": "stopped",
})
summary_parts = text_parts if text_parts else ["Task completed."]
summary = "\n".join(summary_parts)
@@ -407,6 +639,8 @@ async def _create_browser_card(dashboard_id: str, url: str) -> str:
activeTabId=tab_id,
x=40,
y=100,
width=1280,
height=800,
)
dashboard.layout.browser_cards[browser_id] = card
dashboard.updated_at = datetime.now()
@@ -425,6 +659,9 @@ async def run_browser_agents(
api_key: str,
dashboard_id: str | None = None,
pre_selected_browser_ids: list[str] | None = None,
parent_session_id: str | None = None,
auth_token: str | None = None,
base_url: str | None = None,
) -> list[dict]:
"""Run multiple browser sub-agents in parallel.
@@ -451,6 +688,9 @@ async def run_browser_agents(
dashboard_id=dashboard_id,
pre_selected=is_pre_selected,
initial_url=url if url and browser_id not in pre_selected else None,
parent_session_id=parent_session_id,
auth_token=auth_token,
base_url=base_url,
)
results = await asyncio.gather(*[_run_one(t) for t in tasks], return_exceptions=True)
+61 -23
View File
@@ -25,26 +25,20 @@ BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/browser-agent/run"
MODEL = os.environ.get("OPENSWARM_AGENT_MODEL", "sonnet")
DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "")
PRE_SELECTED_BROWSER_IDS = os.environ.get("OPENSWARM_PRE_SELECTED_BROWSER_IDS", "")
PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "")
TOOLS = [
{
"name": "BrowserAgent",
"name": "CreateBrowserAgent",
"description": (
"Delegate a browser task to a dedicated browser agent. The browser agent "
"Create a new browser card and run a task on it. A dedicated browser agent "
"will autonomously perform the task (navigating, clicking, typing, etc.) "
"and return a summary of actions taken plus a final screenshot. "
"Use this for any task that requires interacting with a web page."
"Use this when you need a fresh browser for a new task."
),
"inputSchema": {
"type": "object",
"properties": {
"browser_id": {
"type": "string",
"description": (
"The ID of the browser card to use. If omitted, a new browser "
"card will be automatically created."
),
},
"task": {
"type": "string",
"description": (
@@ -55,20 +49,47 @@ TOOLS = [
"url": {
"type": "string",
"description": (
"Optional starting URL. If provided and no browser_id is given, "
"the new browser will navigate here first."
"Optional starting URL. The new browser will navigate here "
"before beginning the task."
),
},
},
"required": ["task"],
},
},
{
"name": "BrowserAgent",
"description": (
"Delegate a browser task to a dedicated browser agent on an existing "
"browser card. The browser agent will autonomously perform the task "
"(navigating, clicking, typing, etc.) and return a summary of actions "
"taken plus a final screenshot."
),
"inputSchema": {
"type": "object",
"properties": {
"browser_id": {
"type": "string",
"description": "The ID of the existing browser card to use.",
},
"task": {
"type": "string",
"description": (
"The task for the browser agent to perform. Be specific and "
"detailed about what you want accomplished."
),
},
},
"required": ["browser_id", "task"],
},
},
{
"name": "BrowserAgents",
"description": (
"Delegate multiple browser tasks to run in parallel, each on a different "
"browser. All tasks execute concurrently and results are returned together. "
"Use this when you need to perform tasks on multiple web pages simultaneously."
"Delegate multiple browser tasks to run in parallel, each on an existing "
"browser card. All tasks execute concurrently and results are returned "
"together. Use this when you need to perform tasks on multiple web pages "
"simultaneously."
),
"inputSchema": {
"type": "object",
@@ -81,18 +102,14 @@ TOOLS = [
"properties": {
"browser_id": {
"type": "string",
"description": "Optional browser card ID. If omitted, a new browser will be created.",
"description": "The ID of the existing browser card to use.",
},
"task": {
"type": "string",
"description": "The task for this browser agent.",
},
"url": {
"type": "string",
"description": "Optional starting URL.",
},
},
"required": ["task"],
"required": ["browser_id", "task"],
},
},
},
@@ -119,6 +136,7 @@ def call_backend(tasks: list[dict]) -> dict:
"model": MODEL,
"dashboard_id": DASHBOARD_ID,
"pre_selected_browser_ids": pre_selected,
"parent_session_id": PARENT_SESSION_ID,
}).encode()
req = urllib.request.Request(
BACKEND_URL,
@@ -219,10 +237,10 @@ def format_batch_results(results: list[dict]) -> dict:
def handle_tool_call(tool_name: str, arguments: dict) -> dict:
if tool_name == "BrowserAgent":
if tool_name == "CreateBrowserAgent":
task_def = {
"task": arguments.get("task", ""),
"browser_id": arguments.get("browser_id", ""),
"browser_id": "",
"url": arguments.get("url", ""),
}
result = call_backend([task_def])
@@ -233,10 +251,30 @@ def handle_tool_call(tool_name: str, arguments: dict) -> dict:
return format_result(results[0])
return {"content": [{"type": "text", "text": "No result returned."}], "isError": True}
elif tool_name == "BrowserAgent":
browser_id = arguments.get("browser_id", "")
if not browser_id:
return {"content": [{"type": "text", "text": "Error: browser_id is required"}], "isError": True}
task_def = {
"task": arguments.get("task", ""),
"browser_id": browser_id,
"url": "",
}
result = call_backend([task_def])
if "error" in result:
return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True}
results = result.get("results", [result])
if results:
return format_result(results[0])
return {"content": [{"type": "text", "text": "No result returned."}], "isError": True}
elif tool_name == "BrowserAgents":
tasks = arguments.get("tasks", [])
if not tasks:
return {"content": [{"type": "text", "text": "Error: tasks array is empty"}], "isError": True}
for t in tasks:
if not t.get("browser_id"):
return {"content": [{"type": "text", "text": "Error: browser_id is required for each task"}], "isError": True}
result = call_backend(tasks)
if "error" in result:
return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True}
+54
View File
@@ -181,6 +181,58 @@ TOOLS = [
"required": ["browser_id"],
},
},
{
"name": "BrowserScroll",
"description": (
"Scroll the page up or down. Automatically finds the correct scrollable "
"container (works on SPAs like Notion, Gmail, etc. that use nested scroll "
"containers instead of window-level scrolling). Returns scroll position info "
"including whether top/bottom has been reached."
),
"inputSchema": {
"type": "object",
"properties": {
"browser_id": {
"type": "string",
"description": "The browser card ID.",
},
"tab_id": TAB_ID_PROP,
"direction": {
"type": "string",
"enum": ["up", "down"],
"description": "Scroll direction. Defaults to 'down'.",
},
"amount": {
"type": "number",
"description": "Pixels to scroll. Defaults to 500.",
},
},
"required": ["browser_id"],
},
},
{
"name": "BrowserWait",
"description": (
"Wait for a specified duration. Useful after navigation or actions that "
"trigger page loads, animations, or async content rendering. "
"Min 100ms, max 10000ms."
),
"inputSchema": {
"type": "object",
"properties": {
"browser_id": {
"type": "string",
"description": "The browser card ID.",
},
"tab_id": TAB_ID_PROP,
"milliseconds": {
"type": "number",
"description": "Duration to wait in milliseconds. Defaults to 1000.",
},
},
"required": ["browser_id"],
},
},
]
@@ -260,6 +312,8 @@ def handle_tool_call(tool_name: str, arguments: dict) -> dict:
"BrowserType": "type",
"BrowserEvaluate": "evaluate",
"BrowserGetElements": "get_elements",
"BrowserScroll": "scroll",
"BrowserWait": "wait",
}
action = action_map.get(tool_name)
if not action:
+175
View File
@@ -0,0 +1,175 @@
"""GitHub Copilot device flow OAuth + token management.
Handles the full flow:
1. Start device flow → get user_code for user to enter at github.com/login/device
2. Poll until user authorizes → get GitHub access token
3. Exchange GitHub token → Copilot JWT (expires every ~30min)
4. Auto-refresh Copilot token before expiry
"""
import logging
import time
import httpx
logger = logging.getLogger(__name__)
CLIENT_ID = "Iv1.b507a08c87ecfe98" # Copilot's public OAuth app ID
DEVICE_CODE_URL = "https://github.com/login/device/code"
TOKEN_URL = "https://github.com/login/oauth/access_token"
COPILOT_TOKEN_URL = "https://api.github.com/copilot_internal/v2/token"
COPILOT_API_BASE = "https://api.githubcopilot.com"
HEADERS = {
"accept": "application/json",
"content-type": "application/json",
"editor-version": "vscode/1.100.0",
"editor-plugin-version": "copilot-chat/0.30.0",
"user-agent": "GithubCopilot/1.200.0",
}
async def start_device_flow() -> dict:
"""Start GitHub device flow.
Returns: {user_code, verification_uri, device_code, expires_in, interval}
"""
async with httpx.AsyncClient(timeout=15.0) as client:
resp = await client.post(
DEVICE_CODE_URL,
headers=HEADERS,
json={"client_id": CLIENT_ID, "scope": "read:user"},
)
resp.raise_for_status()
data = resp.json()
return {
"user_code": data["user_code"],
"verification_uri": data["verification_uri"],
"device_code": data["device_code"],
"expires_in": data.get("expires_in", 900),
"interval": data.get("interval", 5),
}
async def poll_for_token(device_code: str) -> str | None:
"""Poll GitHub for token after user authorizes.
Returns the GitHub access token (gho_xxx), or None if still pending.
Raises on error (expired, denied, etc.)
"""
async with httpx.AsyncClient(timeout=15.0) as client:
resp = await client.post(
TOKEN_URL,
headers=HEADERS,
json={
"client_id": CLIENT_ID,
"device_code": device_code,
"grant_type": "urn:ietf:params:oauth:grant-type:device_code",
},
)
data = resp.json()
if "access_token" in data:
return data["access_token"]
error = data.get("error", "")
if error == "authorization_pending":
return None # Still waiting
if error == "slow_down":
return None # Need to slow down polling
if error in ("expired_token", "access_denied"):
raise ValueError(f"GitHub auth failed: {error}")
return None
async def exchange_for_copilot_token(github_token: str) -> dict:
"""Exchange GitHub OAuth token for Copilot JWT.
Returns: {token, expires_at}
"""
async with httpx.AsyncClient(timeout=15.0) as client:
resp = await client.get(
COPILOT_TOKEN_URL,
headers={
**HEADERS,
"authorization": f"token {github_token}",
},
)
if resp.status_code == 401:
raise ValueError("GitHub token invalid or expired. Please re-authenticate.")
if resp.status_code == 403:
raise ValueError("No Copilot subscription found for this GitHub account.")
resp.raise_for_status()
data = resp.json()
token = data.get("token", "")
# Extract expiry from token (format: tid=xxx;exp=1234567890;...)
expires_at = time.time() + 25 * 60 # Default 25 min
if "exp=" in token:
try:
for pair in token.split(";"):
if pair.strip().startswith("exp="):
expires_at = int(pair.strip().split("=")[1])
break
except (ValueError, IndexError):
pass
return {"token": token, "expires_at": expires_at}
async def get_copilot_token(github_token: str, current_token: str | None = None, expires_at: float | None = None) -> dict:
"""Get a valid Copilot token, refreshing if needed.
Returns: {token, expires_at}
"""
# If current token is still valid (with 2 min buffer), return it
if current_token and expires_at and time.time() < expires_at - 120:
return {"token": current_token, "expires_at": expires_at}
# Otherwise refresh
return await exchange_for_copilot_token(github_token)
async def list_copilot_models(copilot_token: str) -> list[dict]:
"""Fetch available models from Copilot API."""
async with httpx.AsyncClient(timeout=15.0) as client:
resp = await client.get(
f"{COPILOT_API_BASE}/models",
headers={
"authorization": f"Bearer {copilot_token}",
"copilot-integration-id": "vscode-chat",
**HEADERS,
},
)
if resp.status_code != 200:
logger.warning(f"Failed to list Copilot models: {resp.status_code}")
return []
data = resp.json()
models = data.get("data", data.get("models", []))
return [
{
"value": m.get("id", m.get("name", "")),
"label": m.get("name", m.get("id", "")),
"context_window": m.get("context_window", 128_000),
}
for m in models
if isinstance(m, dict)
]
async def get_github_username(github_token: str) -> str | None:
"""Get the GitHub username for display."""
try:
async with httpx.AsyncClient(timeout=10.0) as client:
resp = await client.get(
"https://api.github.com/user",
headers={"authorization": f"token {github_token}", "accept": "application/json"},
)
if resp.status_code == 200:
return resp.json().get("login")
except Exception:
pass
return None
+149
View File
@@ -0,0 +1,149 @@
"""Headless browser fallback using Playwright.
Used when Electron's <webview> is not available (running in regular browser).
Provides the same browser actions as the Electron webview bridge:
navigate, click, type, screenshot, get_text, get_elements, evaluate.
"""
import asyncio
import base64
import logging
from typing import Any
logger = logging.getLogger(__name__)
_playwright = None
_browser = None
_pages: dict[str, Any] = {} # browser_id -> Page
async def _ensure_browser():
"""Start Playwright browser if not running."""
global _playwright, _browser
if _browser and _browser.is_connected():
return
from playwright.async_api import async_playwright
_playwright = await async_playwright().start()
_browser = await _playwright.chromium.launch(
headless=True,
args=["--no-sandbox", "--disable-gpu"],
)
logger.info("Headless browser started (Playwright/Chromium)")
async def _get_page(browser_id: str) -> Any:
"""Get or create a page for a browser_id."""
await _ensure_browser()
if browser_id not in _pages or _pages[browser_id].is_closed():
page = await _browser.new_page()
await page.set_viewport_size({"width": 1280, "height": 800})
_pages[browser_id] = page
return _pages[browser_id]
async def execute(browser_id: str, action: str, params: dict) -> dict:
"""Execute a browser action. Returns same format as Electron webview bridge."""
try:
page = await _get_page(browser_id)
if action == "navigate":
url = params.get("url", "")
if url:
await page.goto(url, wait_until="domcontentloaded", timeout=30000)
await page.wait_for_timeout(1000) # Let JS settle
return {"text": f"Navigated to {page.url}", "url": page.url}
elif action == "screenshot":
screenshot_bytes = await page.screenshot(type="png")
b64 = base64.b64encode(screenshot_bytes).decode()
return {"image": b64, "url": page.url}
elif action == "get_text":
text = await page.evaluate("document.body?.innerText || ''")
return {"text": text[:15000]}
elif action == "click":
selector = params.get("selector", "")
if selector:
await page.click(selector, timeout=5000)
await page.wait_for_timeout(500)
return {"text": f"Clicked {selector}"}
elif action == "type":
selector = params.get("selector", "")
text = params.get("text", "")
if selector and text:
await page.fill(selector, text)
return {"text": f"Typed into {selector}"}
elif action == "evaluate":
expression = params.get("expression", "")
result = await page.evaluate(expression)
return {"text": str(result) if result is not None else "undefined"}
elif action == "get_elements":
selector = params.get("selector", "body")
elements = await page.evaluate(f"""
(() => {{
const els = document.querySelectorAll('{selector} a, {selector} button, {selector} input, {selector} select, {selector} textarea, {selector} [role="button"], {selector} [onclick]');
return Array.from(els).slice(0, 50).map(el => ({{
tag: el.tagName.toLowerCase(),
text: (el.textContent || '').trim().slice(0, 100),
selector: el.id ? '#' + el.id : (el.className ? '.' + el.className.split(' ')[0] : el.tagName.toLowerCase()),
href: el.href || null,
type: el.type || null,
placeholder: el.placeholder || null,
}}));
}})()
""")
lines = []
for el in (elements or []):
desc = f"{el['tag']}"
if el.get("text"):
desc += f" \"{el['text'][:50]}\""
if el.get("href"):
desc += f"{el['href'][:80]}"
desc += f" selector: {el.get('selector', '?')}"
lines.append(desc)
return {"text": "\n".join(lines) if lines else "No interactive elements found"}
else:
return {"error": f"Unknown action: {action}"}
except Exception as e:
logger.warning(f"Headless browser error ({action}): {e}")
return {"error": str(e)}
async def close_page(browser_id: str):
"""Close a specific page."""
page = _pages.pop(browser_id, None)
if page and not page.is_closed():
await page.close()
async def shutdown():
"""Close all pages and the browser."""
global _browser, _playwright
for page in _pages.values():
try:
if not page.is_closed():
await page.close()
except Exception:
pass
_pages.clear()
if _browser:
try:
await _browser.close()
except Exception:
pass
_browser = None
if _playwright:
try:
await _playwright.stop()
except Exception:
pass
_playwright = None
@@ -0,0 +1,160 @@
#!/usr/bin/env python3
"""
Stdio MCP server that exposes the InvokeAgent tool.
Launched as a subprocess by the Claude Agent SDK. Proxies invocation
requests to the OpenSwarm backend via HTTP, which forks the target
agent session and runs it with the new message.
"""
import json
import sys
import os
import urllib.request
import urllib.error
BACKEND_PORT = os.environ.get("OPENSWARM_PORT", "8324")
BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/invoke-agent/run"
PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "")
DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "")
TOOLS = [
{
"name": "InvokeAgent",
"description": (
"Invoke a copy of an existing agent session with a new message. "
"The invoked agent will have full context of its prior conversation "
"and will process the new message independently. Use this when you "
"need to query another agent about its prior work or ask it to "
"perform a follow-up task."
),
"inputSchema": {
"type": "object",
"properties": {
"session_id": {
"type": "string",
"description": (
"The session ID of the agent to invoke. This is the ID "
"from a selected Agent Card in the context."
),
},
"message": {
"type": "string",
"description": (
"The message to send to the invoked agent. Be specific "
"about what you need from it."
),
},
},
"required": ["session_id", "message"],
},
},
]
def send_response(id_, result=None, error=None):
msg = {"jsonrpc": "2.0", "id": id_}
if error is not None:
msg["error"] = error
else:
msg["result"] = result
sys.stdout.write(json.dumps(msg) + "\n")
sys.stdout.flush()
def call_backend(session_id: str, message: str) -> dict:
payload = json.dumps({
"session_id": session_id,
"message": message,
"parent_session_id": PARENT_SESSION_ID,
"dashboard_id": DASHBOARD_ID,
}).encode()
req = urllib.request.Request(
BACKEND_URL,
data=payload,
headers={"Content-Type": "application/json"},
method="POST",
)
try:
with urllib.request.urlopen(req, timeout=600) as resp:
return json.loads(resp.read().decode())
except urllib.error.HTTPError as e:
body = e.read().decode() if e.fp else str(e)
return {"error": f"HTTP {e.code}: {body}"}
except Exception as e:
return {"error": str(e)}
def handle_tool_call(tool_name: str, arguments: dict) -> dict:
if tool_name != "InvokeAgent":
return {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True}
session_id = arguments.get("session_id", "")
message = arguments.get("message", "")
if not session_id:
return {"content": [{"type": "text", "text": "Error: session_id is required"}], "isError": True}
if not message:
return {"content": [{"type": "text", "text": "Error: message is required"}], "isError": True}
result = call_backend(session_id, message)
if "error" in result:
return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True}
forked_id = result.get("forked_session_id", "")
response = result.get("response", "No response from invoked agent.")
cost = result.get("cost_usd", 0)
source_name = result.get("source_name", "")
lines = [f"**Invoked Agent Result** (forked session: {forked_id})"]
if source_name:
lines[0] = f"**Invoked Agent Result** — {source_name} (forked session: {forked_id})"
if cost > 0:
lines.append(f"*Cost: ${cost:.4f}*")
lines.append("")
lines.append(response)
return {"content": [{"type": "text", "text": "\n".join(lines)}]}
def main():
for line in sys.stdin:
line = line.strip()
if not line:
continue
try:
msg = json.loads(line)
except json.JSONDecodeError:
continue
method = msg.get("method")
id_ = msg.get("id")
params = msg.get("params", {})
if method == "initialize":
send_response(id_, {
"protocolVersion": "2024-11-05",
"capabilities": {"tools": {}},
"serverInfo": {
"name": "openswarm-invoke-agent",
"version": "1.0.0",
},
})
elif method == "notifications/initialized":
pass
elif method == "tools/list":
send_response(id_, {"tools": TOOLS})
elif method == "tools/call":
tool_name = params.get("name", "")
arguments = params.get("arguments", {})
result = handle_tool_call(tool_name, arguments)
send_response(id_, result)
elif method == "ping":
send_response(id_, {})
elif id_ is not None:
send_response(id_, error={"code": -32601, "message": f"Method not found: {method}"})
if __name__ == "__main__":
main()
+360
View File
@@ -0,0 +1,360 @@
"""Standalone MCP client manager for agent sessions.
Replaces claude_agent_sdk's internal MCP server management.
One MCPClientManager instance per agent session — manages connections
to stdio/http/sse MCP servers, discovers tools, and routes tool calls.
"""
from __future__ import annotations
import asyncio
import json
import logging
from contextlib import AsyncExitStack
from dataclasses import dataclass, field
from typing import Any
from backend.apps.agents.providers.base import ToolSchema
logger = logging.getLogger(__name__)
@dataclass
class MCPConnection:
"""A live connection to an MCP server."""
server_name: str
session: Any # mcp.ClientSession
tools: list[ToolSchema] = field(default_factory=list)
class MCPClientManager:
"""Manages connections to MCP servers for a single agent session."""
def __init__(self):
self._connections: dict[str, MCPConnection] = {}
self._exit_stack = AsyncExitStack()
self._started = False
async def __aenter__(self):
await self._exit_stack.__aenter__()
self._started = True
return self
async def __aexit__(self, *exc):
await self.disconnect_all()
try:
await self._exit_stack.__aexit__(*exc)
except (BaseExceptionGroup, ExceptionGroup, Exception) as e:
# MCP subprocess cleanup errors are non-fatal
logger.warning(f"MCP cleanup error (non-fatal): {e}")
self._started = False
async def connect(self, server_name: str, config: dict, timeout: float = 30.0) -> list[ToolSchema]:
"""Connect to an MCP server and return its available tools.
The tools are returned with names prefixed as mcp__<server_name>__<tool_name>.
"""
transport = config.get("type", "stdio")
try:
if transport == "stdio":
coro = self._connect_stdio(server_name, config)
elif transport == "sse":
coro = self._connect_sse(server_name, config)
elif transport == "http":
coro = self._connect_http(server_name, config)
else:
logger.warning(f"Unsupported MCP transport: {transport} for {server_name}")
return []
conn = await asyncio.wait_for(coro, timeout=timeout)
self._connections[server_name] = conn
logger.info(f"MCP connected: {server_name} ({len(conn.tools)} tools)")
return conn.tools
except asyncio.TimeoutError:
logger.warning(f"MCP server {server_name} connection timed out after {timeout}s")
return []
except Exception as e:
logger.warning(f"Failed to connect MCP server {server_name}: {e}")
return []
async def _connect_stdio(self, server_name: str, config: dict) -> MCPConnection:
"""Connect to a stdio MCP server (spawns a subprocess)."""
from mcp import ClientSession
from mcp.client.stdio import stdio_client, StdioServerParameters
command = config.get("command", "")
args = config.get("args", [])
env = config.get("env")
params = StdioServerParameters(
command=command,
args=args,
env=env,
)
transport = await self._exit_stack.enter_async_context(
stdio_client(params)
)
read_stream, write_stream = transport
session = await self._exit_stack.enter_async_context(
ClientSession(read_stream, write_stream)
)
await session.initialize()
result = await session.list_tools()
tools = [
ToolSchema(
name=f"mcp__{server_name}__{t.name}",
description=t.description or "",
input_schema=t.inputSchema if hasattr(t, "inputSchema") else (t.input_schema if hasattr(t, "input_schema") else {}),
)
for t in result.tools
]
return MCPConnection(server_name=server_name, session=session, tools=tools)
async def _connect_sse(self, server_name: str, config: dict) -> MCPConnection:
"""Connect to an SSE MCP server."""
from mcp import ClientSession
from mcp.client.sse import sse_client
url = config.get("url", "")
headers = config.get("headers")
transport = await self._exit_stack.enter_async_context(
sse_client(url=url, headers=headers, timeout=30, sse_read_timeout=300)
)
read_stream, write_stream = transport
session = await self._exit_stack.enter_async_context(
ClientSession(read_stream, write_stream)
)
await session.initialize()
result = await session.list_tools()
tools = [
ToolSchema(
name=f"mcp__{server_name}__{t.name}",
description=t.description or "",
input_schema=t.inputSchema if hasattr(t, "inputSchema") else (t.input_schema if hasattr(t, "input_schema") else {}),
)
for t in result.tools
]
return MCPConnection(server_name=server_name, session=session, tools=tools)
async def _connect_http(self, server_name: str, config: dict) -> MCPConnection:
"""Connect to a Streamable HTTP MCP server.
Falls back to SSE if streamable HTTP fails.
"""
url = config.get("url", "")
headers = config.get("headers")
# Try streamable HTTP first, fall back to SSE
try:
return await self._connect_http_streamable(server_name, url, headers)
except Exception as e:
logger.info(f"Streamable HTTP failed for {server_name}, trying SSE: {e}")
return await self._connect_sse(server_name, config)
async def _connect_http_streamable(
self, server_name: str, url: str, headers: dict | None,
) -> MCPConnection:
"""Connect via Streamable HTTP (JSON-RPC POST)."""
import httpx
from mcp import ClientSession
# Use httpx for streamable HTTP — keep client alive in the exit stack
client = await self._exit_stack.enter_async_context(
httpx.AsyncClient(timeout=30.0)
)
h = {
"Content-Type": "application/json",
"Accept": "application/json, text/event-stream",
**(headers or {}),
}
# Initialize
init_resp = await client.post(url, headers=h, json={
"jsonrpc": "2.0", "id": 1, "method": "initialize",
"params": {
"protocolVersion": "2025-03-26",
"capabilities": {},
"clientInfo": {"name": "self-swarm", "version": "0.1.0"},
},
})
if init_resp.status_code not in (200, 201):
raise ConnectionError(f"MCP initialize failed: {init_resp.status_code}")
session_id = init_resp.headers.get("mcp-session-id", "")
if session_id:
h["mcp-session-id"] = session_id
# Notify initialized
await client.post(url, headers=h, json={
"jsonrpc": "2.0", "method": "notifications/initialized",
})
# List tools
list_resp = await client.post(url, headers=h, json={
"jsonrpc": "2.0", "id": 2, "method": "tools/list", "params": {},
})
if list_resp.status_code not in (200, 201):
raise ConnectionError(f"MCP tools/list failed: {list_resp.status_code}")
ct = list_resp.headers.get("content-type", "")
if "text/event-stream" in ct:
data = self._parse_sse_json(list_resp.text)
else:
data = list_resp.json()
if not data:
raise ConnectionError("Empty response from MCP server")
tools_list = data.get("result", {}).get("tools", [])
tools = [
ToolSchema(
name=f"mcp__{server_name}__{t.get('name', '')}",
description=t.get("description", ""),
input_schema=t.get("inputSchema", t.get("input_schema", {})),
)
for t in tools_list
]
# Store the HTTP client info for call_tool
conn = MCPConnection(server_name=server_name, session=None, tools=tools)
conn._http_client = client # type: ignore[attr-defined]
conn._http_url = url # type: ignore[attr-defined]
conn._http_headers = h # type: ignore[attr-defined]
conn._next_id = 3 # type: ignore[attr-defined]
return conn
@staticmethod
def _parse_sse_json(text: str) -> dict | None:
"""Extract JSON from an SSE response body."""
for line in text.splitlines():
stripped = line.strip()
if stripped.startswith("data:"):
payload = stripped[len("data:"):].strip()
if payload:
try:
return json.loads(payload)
except json.JSONDecodeError:
continue
try:
return json.loads(text)
except json.JSONDecodeError:
return None
async def call_tool(
self, server_name: str, tool_name: str, arguments: dict,
) -> list[dict]:
"""Call a tool on a specific MCP server.
Args:
server_name: The MCP server name (e.g. "google-workspace")
tool_name: The bare tool name (without mcp__prefix)
arguments: Tool input arguments
Returns:
List of content blocks: [{"type": "text", "text": "..."}]
"""
conn = self._connections.get(server_name)
if not conn:
return [{"type": "text", "text": f"MCP server {server_name} not connected"}]
try:
if conn.session is not None:
# stdio or SSE — use MCP ClientSession
result = await conn.session.call_tool(tool_name, arguments)
return self._format_mcp_result(result)
elif hasattr(conn, "_http_client"):
# Streamable HTTP — use JSON-RPC
return await self._call_tool_http(conn, tool_name, arguments)
else:
return [{"type": "text", "text": f"No session for MCP server {server_name}"}]
except Exception as e:
logger.warning(f"MCP tool call failed: {server_name}/{tool_name}: {e}")
return [{"type": "text", "text": f"Error calling {tool_name}: {e}"}]
async def _call_tool_http(
self, conn: MCPConnection, tool_name: str, arguments: dict,
) -> list[dict]:
"""Call a tool via Streamable HTTP."""
client = conn._http_client # type: ignore[attr-defined]
url = conn._http_url # type: ignore[attr-defined]
headers = conn._http_headers # type: ignore[attr-defined]
req_id = conn._next_id # type: ignore[attr-defined]
conn._next_id = req_id + 1 # type: ignore[attr-defined]
resp = await client.post(url, headers=headers, json={
"jsonrpc": "2.0",
"id": req_id,
"method": "tools/call",
"params": {"name": tool_name, "arguments": arguments},
}, timeout=300.0)
ct = resp.headers.get("content-type", "")
if "text/event-stream" in ct:
data = self._parse_sse_json(resp.text)
else:
data = resp.json()
if not data:
return [{"type": "text", "text": "Empty response from MCP server"}]
if "error" in data:
return [{"type": "text", "text": f"MCP error: {data['error']}"}]
result = data.get("result", {})
content = result.get("content", [])
return content if content else [{"type": "text", "text": json.dumps(result)}]
@staticmethod
def _format_mcp_result(result: Any) -> list[dict]:
"""Convert an MCP CallToolResult to content blocks."""
if hasattr(result, "content"):
blocks = []
for item in result.content:
if hasattr(item, "text"):
blocks.append({"type": "text", "text": item.text})
elif hasattr(item, "data"):
blocks.append({
"type": "image",
"source": {
"type": "base64",
"media_type": getattr(item, "mimeType", "image/png"),
"data": item.data,
},
})
else:
blocks.append({"type": "text", "text": str(item)})
return blocks if blocks else [{"type": "text", "text": "Done."}]
return [{"type": "text", "text": str(result)}]
def get_all_tool_schemas(self) -> list[ToolSchema]:
"""Return tool schemas from all connected MCP servers."""
schemas = []
for conn in self._connections.values():
schemas.extend(conn.tools)
return schemas
def parse_mcp_tool_name(self, full_name: str) -> tuple[str, str] | None:
"""Parse mcp__<server>__<tool> into (server_name, tool_name).
Returns None if the name doesn't match the MCP naming convention.
"""
import re
m = re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", full_name)
if m:
return m.group(1), m.group(2)
return None
async def disconnect_all(self):
"""Disconnect all MCP servers. Called on session end."""
self._connections.clear()
# The AsyncExitStack handles actual cleanup of transports/sessions
+4
View File
@@ -5,6 +5,7 @@ from uuid import uuid4
class AgentConfig(BaseModel):
name: str = Field(default_factory=lambda: f"Agent-{uuid4().hex[:6]}")
provider: str = "anthropic"
model: str = "sonnet"
mode: str = "agent"
system_prompt: Optional[str] = None
@@ -37,6 +38,7 @@ class Message(BaseModel):
attached_skills: Optional[list[dict]] = None
forced_tools: Optional[list[str]] = None
images: Optional[list[dict]] = None
hidden: bool = False
class MessageBranch(BaseModel):
id: str = Field(default_factory=lambda: uuid4().hex)
@@ -54,6 +56,7 @@ class AgentSession(BaseModel):
id: str = Field(default_factory=lambda: uuid4().hex)
name: str
status: Literal["running", "waiting_approval", "completed", "error", "stopped"] = "running"
provider: str = "anthropic"
model: str = "sonnet"
mode: str = "agent"
sdk_session_id: Optional[str] = None
@@ -72,3 +75,4 @@ class AgentSession(BaseModel):
tool_group_meta: dict[str, "ToolGroupMeta"] = Field(default_factory=dict)
dashboard_id: Optional[str] = None
browser_id: Optional[str] = None
parent_session_id: Optional[str] = None
+260
View File
@@ -0,0 +1,260 @@
"""Anthropic provider adapter using the native Anthropic SDK."""
from __future__ import annotations
import json
import logging
from typing import Any, AsyncIterator
import anthropic
from backend.apps.agents.providers.base import (
BaseProvider, ContentBlock, ModelResponse, ProviderMessage,
StreamEvent, ToolCall, ToolSchema,
)
logger = logging.getLogger(__name__)
MODEL_MAP = {
"sonnet": "claude-sonnet-4-6",
"opus": "claude-opus-4-6",
"haiku": "claude-haiku-4-5",
}
class AnthropicProvider(BaseProvider):
"""Provider adapter for Anthropic's Messages API."""
def __init__(
self,
api_key: str | None = None,
auth_token: str | None = None,
base_url: str | None = None,
):
kwargs: dict[str, Any] = {}
if auth_token:
kwargs["auth_token"] = auth_token
elif api_key:
kwargs["api_key"] = api_key
if base_url:
kwargs["base_url"] = base_url
self.client = anthropic.AsyncAnthropic(**kwargs)
def get_model_id(self, short_name: str) -> str:
return MODEL_MAP.get(short_name, short_name)
def clean_tool_schema(self, schema: ToolSchema) -> dict:
return {
"name": schema.name,
"description": schema.description,
"input_schema": schema.input_schema,
}
def format_tool_result(self, tool_use_id: str, content: list[dict]) -> dict:
return {
"type": "tool_result",
"tool_use_id": tool_use_id,
"content": content,
}
def format_user_message(self, content: Any) -> ProviderMessage:
return ProviderMessage(role="user", content=content)
def format_assistant_message(self, response: ModelResponse) -> ProviderMessage:
blocks = []
for block in response.content:
if block.type == "text":
blocks.append({"type": "text", "text": block.text})
elif block.type == "tool_use" and block.tool_call:
blocks.append({
"type": "tool_use",
"id": block.tool_call.id,
"name": block.tool_call.name,
"input": block.tool_call.input,
})
return ProviderMessage(role="assistant", content=blocks)
def _build_messages(self, messages: list[ProviderMessage]) -> list[dict]:
"""Convert ProviderMessages to Anthropic API format."""
result = []
for msg in messages:
if msg.role == "tool_result":
# Tool results: content is a list of tool_result dicts
if isinstance(msg.content, list):
result.append({"role": "user", "content": msg.content})
else:
result.append({"role": "user", "content": [msg.content]})
elif msg.role == "assistant":
result.append({"role": "assistant", "content": msg.content})
elif msg.role == "user":
result.append({"role": "user", "content": msg.content})
return result
async def create_message(
self,
model: str,
system: str | None,
messages: list[ProviderMessage],
tools: list[ToolSchema],
max_tokens: int = 8192,
) -> ModelResponse:
kwargs: dict[str, Any] = {
"model": self.get_model_id(model),
"max_tokens": max_tokens,
"messages": self._build_messages(messages),
}
if system:
kwargs["system"] = system
if tools:
kwargs["tools"] = [self.clean_tool_schema(t) for t in tools]
resp = await self.client.messages.create(**kwargs)
content = []
for block in resp.content:
if block.type == "text":
content.append(ContentBlock(type="text", text=block.text))
elif block.type == "tool_use":
content.append(ContentBlock(
type="tool_use",
tool_call=ToolCall(
id=block.id,
name=block.name,
input=block.input,
),
))
return ModelResponse(
content=content,
stop_reason="tool_use" if resp.stop_reason == "tool_use" else "end_turn",
usage={
"input_tokens": resp.usage.input_tokens,
"output_tokens": resp.usage.output_tokens,
},
)
async def stream_message(
self,
model: str,
system: str | None,
messages: list[ProviderMessage],
tools: list[ToolSchema],
max_tokens: int = 8192,
) -> AsyncIterator[StreamEvent]:
kwargs: dict[str, Any] = {
"model": self.get_model_id(model),
"max_tokens": max_tokens,
"messages": self._build_messages(messages),
}
if system:
kwargs["system"] = system
if tools:
kwargs["tools"] = [self.clean_tool_schema(t) for t in tools]
# Use create() with stream=True for raw SSE events
kwargs["stream"] = True
raw_stream = await self.client.messages.create(**kwargs)
current_block_type: dict[int, str] = {}
current_tool_name: dict[int, str] = {}
current_tool_id: dict[int, str] = {}
current_text: dict[int, str] = {}
current_json: dict[int, str] = {}
async for event in raw_stream:
event_type = getattr(event, "type", "")
if event_type == "content_block_start":
index = event.index
block = event.content_block
block_type = block.type
current_block_type[index] = block_type
if block_type == "text":
current_text[index] = ""
yield StreamEvent(
type="content_block_start",
index=index,
block_type="text",
)
elif block_type == "tool_use":
current_tool_name[index] = block.name
current_tool_id[index] = block.id
current_json[index] = ""
yield StreamEvent(
type="content_block_start",
index=index,
block_type="tool_use",
tool_name=block.name,
tool_id=block.id,
)
elif event_type == "content_block_delta":
index = event.index
delta = event.delta
delta_type = delta.type
if delta_type == "text_delta":
current_text.setdefault(index, "")
current_text[index] += delta.text
yield StreamEvent(
type="content_block_delta",
index=index,
delta_type="text_delta",
text=delta.text,
)
elif delta_type == "input_json_delta":
current_json.setdefault(index, "")
current_json[index] += delta.partial_json
yield StreamEvent(
type="content_block_delta",
index=index,
delta_type="input_json_delta",
text=delta.partial_json,
)
elif event_type == "content_block_stop":
yield StreamEvent(type="content_block_stop", index=event.index)
elif event_type == "message_delta":
# Extract output token usage from the final delta
usage_data = {}
delta_usage = getattr(event, "usage", None)
if delta_usage:
output_tokens = getattr(delta_usage, "output_tokens", 0)
if output_tokens:
usage_data["output_tokens"] = output_tokens
if usage_data:
yield StreamEvent(type="usage", usage=usage_data)
elif event_type == "message_start":
# Extract input token usage from the message start
msg = getattr(event, "message", None)
if msg:
msg_usage = getattr(msg, "usage", None)
if msg_usage:
usage_data = {}
input_tokens = getattr(msg_usage, "input_tokens", 0)
output_tokens = getattr(msg_usage, "output_tokens", 0)
if input_tokens:
usage_data["input_tokens"] = input_tokens
if output_tokens:
usage_data["output_tokens"] = output_tokens
if usage_data:
yield StreamEvent(type="usage", usage=usage_data)
yield StreamEvent(type="message_stop")
async def stream_and_collect(
self,
model: str,
system: str | None,
messages: list[ProviderMessage],
tools: list[ToolSchema],
max_tokens: int = 8192,
) -> tuple[AsyncIterator[StreamEvent], ModelResponse]:
"""Helper: stream events and also return the full collected response.
Not used directly — the AgentLoop handles collection.
"""
raise NotImplementedError("Use stream_message() directly; AgentLoop collects.")
+135
View File
@@ -0,0 +1,135 @@
"""Provider-agnostic base classes for multi-model support.
All provider adapters (Anthropic, OpenAI, Gemini, OpenAI-compatible)
implement BaseProvider, translating their native APIs into these
common data structures.
"""
from __future__ import annotations
from abc import ABC, abstractmethod
from dataclasses import dataclass, field
from typing import Any, AsyncIterator
@dataclass
class ToolSchema:
"""Provider-agnostic tool definition."""
name: str
description: str
input_schema: dict[str, Any]
@dataclass
class ToolCall:
"""A tool invocation requested by the model."""
id: str
name: str
input: dict[str, Any]
@dataclass
class ContentBlock:
"""A block of content from the model response."""
type: str # "text" | "tool_use"
text: str = ""
tool_call: ToolCall | None = None
@dataclass
class ModelResponse:
"""Complete (non-streaming) response from a provider."""
content: list[ContentBlock]
stop_reason: str # "end_turn" | "tool_use" | "max_tokens"
usage: dict[str, int] = field(default_factory=dict)
@dataclass
class StreamEvent:
"""A single streaming event, normalized across providers.
The event types match what the frontend already expects via WebSocket:
content_block_start, content_block_delta, content_block_stop, message_stop.
"""
type: str
index: int = 0
block_type: str = "" # "text" | "tool_use"
delta_type: str = "" # "text_delta" | "input_json_delta"
text: str = ""
tool_name: str = ""
tool_id: str = ""
usage: dict[str, int] = field(default_factory=dict)
@dataclass
class ProviderMessage:
"""Provider-agnostic message for conversation history.
Each provider adapter converts these to/from its native format.
"""
role: str # "user" | "assistant" | "tool_result"
content: Any # str, list[dict], or provider-specific content
class BaseProvider(ABC):
"""Abstract base for LLM provider adapters."""
@abstractmethod
async def stream_message(
self,
model: str,
system: str | None,
messages: list[ProviderMessage],
tools: list[ToolSchema],
max_tokens: int = 8192,
) -> AsyncIterator[StreamEvent]:
"""Stream a model response, yielding normalized StreamEvents."""
...
@abstractmethod
async def create_message(
self,
model: str,
system: str | None,
messages: list[ProviderMessage],
tools: list[ToolSchema],
max_tokens: int = 8192,
) -> ModelResponse:
"""Non-streaming message creation."""
...
@abstractmethod
def format_tool_result(
self,
tool_use_id: str,
content: list[dict],
) -> dict:
"""Format a tool result in this provider's expected message format."""
...
@abstractmethod
def format_user_message(self, content: Any) -> ProviderMessage:
"""Wrap user content (str or multimodal blocks) into a ProviderMessage."""
...
@abstractmethod
def format_assistant_message(self, response: ModelResponse) -> ProviderMessage:
"""Convert a ModelResponse into a ProviderMessage for conversation history."""
...
@abstractmethod
def get_model_id(self, short_name: str) -> str:
"""Resolve a short model name to the full API model ID."""
...
def clean_tool_schema(self, schema: ToolSchema) -> dict:
"""Convert a ToolSchema to the provider's native tool format.
Default: Anthropic-style format. Override for providers that need
different formats or schema cleaning (e.g. Gemini).
"""
return {
"name": schema.name,
"description": schema.description,
"input_schema": schema.input_schema,
}
+121
View File
@@ -0,0 +1,121 @@
"""GitHub Copilot provider — routes through Copilot's OpenAI-compatible API.
Uses the user's GitHub Copilot subscription to access Claude, GPT, and other models.
Extends OpenAICompatProvider since Copilot's API speaks the OpenAI format.
"""
from __future__ import annotations
import logging
import time
from typing import Any, AsyncIterator
from openai import AsyncOpenAI
from backend.apps.agents.providers.base import (
BaseProvider, ProviderMessage, StreamEvent, ToolSchema, ModelResponse,
)
from backend.apps.agents.providers.openai_compat import OpenAICompatProvider
logger = logging.getLogger(__name__)
COPILOT_API_BASE = "https://api.githubcopilot.com"
class CopilotProvider(OpenAICompatProvider):
"""Provider that routes through GitHub Copilot's API."""
def __init__(self, copilot_token: str):
# Initialize OpenAI client pointing at Copilot's API
self.client = AsyncOpenAI(
api_key=copilot_token,
base_url=COPILOT_API_BASE,
)
# Store token for header injection
self._copilot_token = copilot_token
def get_model_id(self, short_name: str) -> str:
# Copilot uses same model IDs — pass through
return short_name
async def stream_message(
self,
model: str,
system: str | None,
messages: list[ProviderMessage],
tools: list[ToolSchema],
max_tokens: int = 8192,
) -> AsyncIterator[StreamEvent]:
"""Stream with Copilot-specific headers."""
kwargs: dict[str, Any] = {
"model": self.get_model_id(model),
"max_tokens": max_tokens,
"messages": self._build_messages(system, messages),
"stream": True,
"extra_headers": {
"copilot-integration-id": "vscode-chat",
},
}
if tools:
kwargs["tools"] = [self.clean_tool_schema(t) for t in tools]
stream = await self.client.chat.completions.create(**kwargs)
# Reuse parent's stream parsing logic
text_started = False
text_index = 0
tool_indices: dict[int, dict] = {}
next_block_index = 0
from uuid import uuid4
async for chunk in stream:
if not chunk.choices:
continue
delta = chunk.choices[0].delta
finish_reason = chunk.choices[0].finish_reason
if delta.content is not None:
if not text_started:
text_started = True
text_index = next_block_index
next_block_index += 1
yield StreamEvent(type="content_block_start", index=text_index, block_type="text")
yield StreamEvent(type="content_block_delta", index=text_index, delta_type="text_delta", text=delta.content)
if delta.tool_calls:
for tc_delta in delta.tool_calls:
tc_idx = tc_delta.index
if tc_idx not in tool_indices:
if text_started:
yield StreamEvent(type="content_block_stop", index=text_index)
text_started = False
block_idx = next_block_index
next_block_index += 1
tool_indices[tc_idx] = {
"block_index": block_idx,
"id": tc_delta.id or uuid4().hex,
"name": tc_delta.function.name if tc_delta.function else "",
"json_buf": "",
}
yield StreamEvent(
type="content_block_start", index=block_idx, block_type="tool_use",
tool_name=tool_indices[tc_idx]["name"], tool_id=tool_indices[tc_idx]["id"],
)
info = tool_indices[tc_idx]
if tc_delta.function and tc_delta.function.name:
info["name"] = tc_delta.function.name
if tc_delta.function and tc_delta.function.arguments:
info["json_buf"] += tc_delta.function.arguments
yield StreamEvent(
type="content_block_delta", index=info["block_index"],
delta_type="input_json_delta", text=tc_delta.function.arguments,
)
if finish_reason is not None:
if text_started:
yield StreamEvent(type="content_block_stop", index=text_index)
for info in tool_indices.values():
yield StreamEvent(type="content_block_stop", index=info["block_index"])
yield StreamEvent(type="message_stop")
+600
View File
@@ -0,0 +1,600 @@
"""Gemini provider adapter using the new google-genai SDK."""
from __future__ import annotations
import base64
import json
import logging
from copy import deepcopy
from typing import Any, AsyncIterator
from uuid import uuid4
from google import genai
from google.genai import types
from backend.apps.agents.providers.base import (
BaseProvider,
ContentBlock,
ModelResponse,
ProviderMessage,
StreamEvent,
ToolCall,
ToolSchema,
)
logger = logging.getLogger(__name__)
MODEL_MAP = {
"flash": "gemini-2.5-flash",
"pro": "gemini-2.5-pro",
}
# JSON Schema keywords that Gemini does not support
_UNSUPPORTED_VALIDATION_KEYS = frozenset({
"minLength",
"maxLength",
"minimum",
"maximum",
"exclusiveMinimum",
"exclusiveMaximum",
"pattern",
"minItems",
"maxItems",
"uniqueItems",
"minProperties",
"maxProperties",
"multipleOf",
"format",
"const",
})
_UNSUPPORTED_STRUCTURAL_KEYS = frozenset({
"$ref",
"$defs",
"patternProperties",
})
# ---------------------------------------------------------------------------
# Schema cleaning — port of OpenClaw's clean-for-gemini.ts logic
# ---------------------------------------------------------------------------
def _resolve_ref(ref: str, root_defs: dict[str, Any]) -> dict[str, Any]:
"""Attempt to resolve a $ref pointer like '#/$defs/Foo'."""
if ref.startswith("#/$defs/"):
name = ref[len("#/$defs/"):]
if name in root_defs:
return deepcopy(root_defs[name])
# Cannot resolve — return empty object
return {"type": "object"}
def _clean_schema_node(node: dict[str, Any], root_defs: dict[str, Any]) -> dict[str, Any]:
"""Recursively clean a single JSON Schema node for Gemini compatibility."""
if not isinstance(node, dict):
return node
# If this node is just a $ref, resolve it first then clean the result
if "$ref" in node and len(node) <= 2: # $ref possibly with description
resolved = _resolve_ref(node["$ref"], root_defs)
# Carry over description if the ref node had one
if "description" in node:
resolved["description"] = node["description"]
return _clean_schema_node(resolved, root_defs)
result: dict[str, Any] = {}
for key, value in node.items():
# Drop unsupported keys
if key in _UNSUPPORTED_VALIDATION_KEYS:
continue
if key in _UNSUPPORTED_STRUCTURAL_KEYS:
continue
# Handle additionalProperties: drop if boolean, recurse if schema
if key == "additionalProperties":
if isinstance(value, bool):
continue
# It's a schema dict — clean and keep it
result[key] = _clean_schema_node(value, root_defs)
continue
# Handle anyOf / oneOf: flatten into something Gemini can use
if key in ("anyOf", "oneOf"):
if isinstance(value, list):
flattened = _flatten_union(value, root_defs)
if flattened is not None:
result.update(flattened)
continue
# Handle allOf: merge all members
if key == "allOf":
if isinstance(value, list):
merged = _merge_all_of(value, root_defs)
result.update(merged)
continue
# Recurse into properties
if key == "properties" and isinstance(value, dict):
result[key] = {
prop_name: _clean_schema_node(prop_schema, root_defs)
for prop_name, prop_schema in value.items()
}
continue
# Recurse into items
if key == "items":
if isinstance(value, dict):
result[key] = _clean_schema_node(value, root_defs)
elif isinstance(value, list):
result[key] = [_clean_schema_node(item, root_defs) for item in value]
else:
result[key] = value
continue
# Keep everything else (type, description, title, default, enum, required, etc.)
result[key] = value
return result
def _flatten_union(
variants: list[dict[str, Any]],
root_defs: dict[str, Any],
) -> dict[str, Any] | None:
"""Collapse anyOf/oneOf into a Gemini-compatible schema.
Strategies (applied in order):
1. If all variants are literal types (with const or single-value enums),
collapse into a single enum.
2. If there's a null variant mixed with non-null variants, strip the null
variant and return the remaining schema (nullable).
3. If only one non-null variant remains after stripping, unwrap it.
4. Otherwise return the first variant (best-effort).
"""
if not variants:
return None
# Clean each variant first (resolve refs, etc.)
cleaned = [_clean_schema_node(v, root_defs) for v in variants]
# Strategy 1: all literal types → collapse to enum
enum_values: list[Any] = []
all_literals = True
for v in cleaned:
if "const" in v:
enum_values.append(v["const"])
elif "enum" in v and isinstance(v["enum"], list) and len(v["enum"]) == 1:
enum_values.append(v["enum"][0])
else:
all_literals = False
break
if all_literals and enum_values:
return {"type": "string", "enum": enum_values}
# Strategy 2 & 3: strip null variants
non_null = [v for v in cleaned if v.get("type") != "null"]
if len(non_null) == 0:
# All variants are null
return {"type": "string"}
if len(non_null) == 1:
# Single non-null variant — unwrap it, mark as nullable
result = non_null[0].copy()
result["nullable"] = True
return result
# Strategy 4: multiple non-null variants, just use first (best-effort)
return non_null[0]
def _merge_all_of(
members: list[dict[str, Any]],
root_defs: dict[str, Any],
) -> dict[str, Any]:
"""Merge allOf members into a single cleaned schema."""
merged: dict[str, Any] = {}
for member in members:
cleaned = _clean_schema_node(member, root_defs)
for key, value in cleaned.items():
if key == "properties" and key in merged:
merged[key].update(value)
elif key == "required" and key in merged:
existing = set(merged[key])
existing.update(value)
merged[key] = sorted(existing)
else:
merged[key] = value
return merged
def clean_schema_for_gemini(schema: dict[str, Any]) -> dict[str, Any]:
"""Top-level entry point: clean a full JSON Schema for Gemini compatibility."""
root_defs = schema.get("$defs", schema.get("definitions", {}))
return _clean_schema_node(schema, root_defs)
# ---------------------------------------------------------------------------
# Provider implementation
# ---------------------------------------------------------------------------
class GeminiProvider(BaseProvider):
"""Provider adapter for Google Gemini via the google-genai SDK."""
def __init__(self, api_key: str):
self.client = genai.Client(api_key=api_key)
def get_model_id(self, short_name: str) -> str:
return MODEL_MAP.get(short_name, short_name)
def clean_tool_schema(self, schema: ToolSchema) -> dict:
"""Convert a ToolSchema to Gemini function declaration format.
Cleans the JSON Schema of unsupported features before sending.
"""
cleaned_params = clean_schema_for_gemini(schema.input_schema)
return {
"name": schema.name,
"description": schema.description,
"parameters": cleaned_params,
}
def format_tool_result(self, tool_use_id: str, content: list[dict]) -> dict:
"""Format a tool result for Gemini conversation history.
Gemini uses FunctionResponse parts; we store enough info to reconstruct them.
"""
# Extract text content for the function response
text_parts = []
for block in content:
if block.get("type") == "text":
text_parts.append(block.get("text", ""))
elif block.get("type") == "image":
text_parts.append("[image]")
else:
text_parts.append(json.dumps(block))
return {
"tool_use_id": tool_use_id,
"output": "\n".join(text_parts) if text_parts else "Done.",
}
def format_user_message(self, content: Any) -> ProviderMessage:
if isinstance(content, str):
return ProviderMessage(role="user", content=content)
return ProviderMessage(role="user", content=content)
def format_assistant_message(self, response: ModelResponse) -> ProviderMessage:
"""Convert a ModelResponse into a ProviderMessage for conversation history."""
blocks = []
for block in response.content:
if block.type == "text":
blocks.append({"type": "text", "text": block.text})
elif block.type == "tool_use" and block.tool_call:
blocks.append({
"type": "tool_use",
"id": block.tool_call.id,
"name": block.tool_call.name,
"input": block.tool_call.input,
})
return ProviderMessage(role="assistant", content=blocks)
# ------------------------------------------------------------------
# Internal helpers for building Gemini API messages
# ------------------------------------------------------------------
def _build_tools(self, tools: list[ToolSchema]) -> list[types.Tool] | None:
"""Convert ToolSchemas to Gemini Tool objects."""
if not tools:
return None
declarations = []
for t in tools:
cleaned = self.clean_tool_schema(t)
declarations.append(types.FunctionDeclaration(
name=cleaned["name"],
description=cleaned["description"],
parameters=cleaned["parameters"],
))
return [types.Tool(function_declarations=declarations)]
def _build_contents(
self, messages: list[ProviderMessage],
) -> list[types.Content]:
"""Convert ProviderMessages into a list of Gemini Content objects.
Gemini requires:
- Conversation starts with a user message
- Strict alternating user/model turns
We merge consecutive same-role messages to satisfy this.
"""
raw_contents: list[types.Content] = []
for msg in messages:
if msg.role == "user":
parts = self._user_content_to_parts(msg.content)
raw_contents.append(types.Content(role="user", parts=parts))
elif msg.role == "assistant":
parts = self._assistant_content_to_parts(msg.content)
raw_contents.append(types.Content(role="model", parts=parts))
elif msg.role == "tool_result":
# Tool results become user turns with FunctionResponse parts
parts = self._tool_result_to_parts(msg.content)
raw_contents.append(types.Content(role="user", parts=parts))
# Ensure conversation starts with user
if raw_contents and raw_contents[0].role != "user":
raw_contents.insert(
0,
types.Content(
role="user",
parts=[types.Part.from_text("Hello.")],
),
)
# Merge consecutive same-role turns
merged: list[types.Content] = []
for content in raw_contents:
if merged and merged[-1].role == content.role:
merged[-1].parts.extend(content.parts)
else:
merged.append(content)
return merged
def _user_content_to_parts(self, content: Any) -> list[types.Part]:
"""Convert user message content to Gemini Part objects."""
if isinstance(content, str):
return [types.Part.from_text(content)]
if isinstance(content, list):
parts = []
for block in content:
if isinstance(block, str):
parts.append(types.Part.from_text(block))
elif isinstance(block, dict):
if block.get("type") == "text":
parts.append(types.Part.from_text(block.get("text", "")))
elif block.get("type") == "image":
source = block.get("source", {})
media_type = source.get("media_type", "image/png")
data = source.get("data", "")
parts.append(types.Part.from_bytes(
data=base64.b64decode(data),
mime_type=media_type,
))
else:
parts.append(types.Part.from_text(json.dumps(block)))
return parts if parts else [types.Part.from_text("")]
return [types.Part.from_text(str(content))]
def _assistant_content_to_parts(self, content: Any) -> list[types.Part]:
"""Convert assistant message content to Gemini Part objects."""
if isinstance(content, str):
return [types.Part.from_text(content)]
if isinstance(content, list):
parts = []
for block in content:
if isinstance(block, dict):
if block.get("type") == "text":
text = block.get("text", "")
if text:
parts.append(types.Part.from_text(text))
elif block.get("type") == "tool_use":
parts.append(types.Part(
function_call=types.FunctionCall(
name=block.get("name", ""),
args=block.get("input", {}),
)
))
return parts if parts else [types.Part.from_text("")]
return [types.Part.from_text(str(content))]
def _tool_result_to_parts(self, content: Any) -> list[types.Part]:
"""Convert tool result content to Gemini FunctionResponse parts."""
parts = []
if isinstance(content, list):
for item in content:
if isinstance(item, dict) and "tool_use_id" in item:
# The tool_use_id is the function name in our format_tool_result
func_name = item.get("tool_use_id", "unknown")
output = item.get("output", "Done.")
parts.append(types.Part(
function_response=types.FunctionResponse(
name=func_name,
response={"result": output},
)
))
elif isinstance(content, dict) and "tool_use_id" in content:
func_name = content.get("tool_use_id", "unknown")
output = content.get("output", "Done.")
parts.append(types.Part(
function_response=types.FunctionResponse(
name=func_name,
response={"result": output},
)
))
return parts if parts else [types.Part.from_text("Tool result unavailable.")]
# ------------------------------------------------------------------
# Non-streaming message creation
# ------------------------------------------------------------------
async def create_message(
self,
model: str,
system: str | None,
messages: list[ProviderMessage],
tools: list[ToolSchema],
max_tokens: int = 8192,
) -> ModelResponse:
contents = self._build_contents(messages)
gemini_tools = self._build_tools(tools)
config = types.GenerateContentConfig(
max_output_tokens=max_tokens,
)
if system:
config.system_instruction = system
if gemini_tools:
config.tools = gemini_tools
resp = await self.client.aio.models.generate_content(
model=self.get_model_id(model),
contents=contents,
config=config,
)
return self._parse_response(resp)
def _parse_response(self, resp: Any) -> ModelResponse:
"""Parse a Gemini GenerateContentResponse into a ModelResponse."""
content: list[ContentBlock] = []
has_tool_calls = False
if resp.candidates:
candidate = resp.candidates[0]
if candidate.content and candidate.content.parts:
for part in candidate.content.parts:
if part.text is not None:
content.append(ContentBlock(type="text", text=part.text))
elif part.function_call is not None:
has_tool_calls = True
fc = part.function_call
content.append(ContentBlock(
type="tool_use",
tool_call=ToolCall(
id=uuid4().hex,
name=fc.name,
input=dict(fc.args) if fc.args else {},
),
))
stop_reason = "tool_use" if has_tool_calls else "end_turn"
usage = {}
if resp.usage_metadata:
usage = {
"input_tokens": getattr(resp.usage_metadata, "prompt_token_count", 0) or 0,
"output_tokens": getattr(resp.usage_metadata, "candidates_token_count", 0) or 0,
}
return ModelResponse(content=content, stop_reason=stop_reason, usage=usage)
# ------------------------------------------------------------------
# Streaming message creation
# ------------------------------------------------------------------
async def stream_message(
self,
model: str,
system: str | None,
messages: list[ProviderMessage],
tools: list[ToolSchema],
max_tokens: int = 8192,
) -> AsyncIterator[StreamEvent]:
contents = self._build_contents(messages)
gemini_tools = self._build_tools(tools)
config = types.GenerateContentConfig(
max_output_tokens=max_tokens,
)
if system:
config.system_instruction = system
if gemini_tools:
config.tools = gemini_tools
stream = self.client.aio.models.generate_content_stream(
model=self.get_model_id(model),
contents=contents,
config=config,
)
# Track streaming state
block_index = 0
text_block_open = False
tool_blocks: dict[str, int] = {} # tool_name -> block_index (for dedup)
async for chunk in stream:
if not chunk.candidates:
continue
candidate = chunk.candidates[0]
if not candidate.content or not candidate.content.parts:
continue
for part in candidate.content.parts:
if part.text is not None:
text = part.text
if not text_block_open:
text_block_open = True
yield StreamEvent(
type="content_block_start",
index=block_index,
block_type="text",
)
yield StreamEvent(
type="content_block_delta",
index=block_index,
delta_type="text_delta",
text=text,
)
elif part.function_call is not None:
fc = part.function_call
# Close text block if open
if text_block_open:
yield StreamEvent(
type="content_block_stop",
index=block_index,
)
block_index += 1
text_block_open = False
tool_id = uuid4().hex
tool_block_idx = block_index
block_index += 1
args = dict(fc.args) if fc.args else {}
args_json = json.dumps(args)
yield StreamEvent(
type="content_block_start",
index=tool_block_idx,
block_type="tool_use",
tool_name=fc.name,
tool_id=tool_id,
)
yield StreamEvent(
type="content_block_delta",
index=tool_block_idx,
delta_type="input_json_delta",
text=args_json,
)
yield StreamEvent(
type="content_block_stop",
index=tool_block_idx,
)
# Close any remaining open text block
if text_block_open:
yield StreamEvent(
type="content_block_stop",
index=block_index,
)
# Emit usage from the last chunk if available
if chunk and hasattr(chunk, 'usage_metadata') and chunk.usage_metadata:
um = chunk.usage_metadata
usage_data = {
"input_tokens": getattr(um, "prompt_token_count", 0) or 0,
"output_tokens": getattr(um, "candidates_token_count", 0) or 0,
}
if any(v > 0 for v in usage_data.values()):
yield StreamEvent(type="usage", usage=usage_data)
yield StreamEvent(type="message_stop")
@@ -0,0 +1,330 @@
"""OpenAI-compatible provider adapter.
Works with ANY endpoint that speaks the OpenAI Chat Completions API:
OpenAI, OpenRouter, Together, Groq, Fireworks, Mistral, Ollama, vLLM, etc.
"""
from __future__ import annotations
import json
import logging
from typing import Any, AsyncIterator
from uuid import uuid4
from openai import AsyncOpenAI
from backend.apps.agents.providers.base import (
BaseProvider, ContentBlock, ModelResponse, ProviderMessage,
StreamEvent, ToolCall, ToolSchema,
)
logger = logging.getLogger(__name__)
class OpenAICompatProvider(BaseProvider):
"""Provider adapter for any OpenAI-compatible API endpoint."""
def __init__(
self,
api_key: str = "",
base_url: str | None = None,
):
kwargs: dict[str, Any] = {}
# Always set api_key — use "none" as placeholder if empty (some endpoints don't need real keys)
kwargs["api_key"] = api_key if api_key else "none"
if base_url:
kwargs["base_url"] = base_url
self.client = AsyncOpenAI(**kwargs)
def get_model_id(self, short_name: str) -> str:
# Pass through — user selects exact model ID
return short_name
def clean_tool_schema(self, schema: ToolSchema) -> dict:
"""Convert to OpenAI function calling format."""
return {
"type": "function",
"function": {
"name": schema.name,
"description": schema.description,
"parameters": schema.input_schema,
},
}
def format_tool_result(self, tool_use_id: str, content: list[dict]) -> dict:
"""Format tool result as OpenAI expects."""
# OpenAI wants a single string for tool results
text_parts = []
for block in content:
if block.get("type") == "text":
text_parts.append(block.get("text", ""))
elif block.get("type") == "image":
text_parts.append("[image]")
else:
text_parts.append(json.dumps(block))
return {
"role": "tool",
"tool_call_id": tool_use_id,
"content": "\n".join(text_parts) if text_parts else "Done.",
}
def format_user_message(self, content: Any) -> ProviderMessage:
"""Convert user content to OpenAI format."""
if isinstance(content, str):
return ProviderMessage(role="user", content=content)
# Multimodal content (text + images)
if isinstance(content, list):
parts = []
for block in content:
if isinstance(block, dict):
if block.get("type") == "text":
parts.append({"type": "text", "text": block["text"]})
elif block.get("type") == "image":
source = block.get("source", {})
media_type = source.get("media_type", "image/png")
data = source.get("data", "")
parts.append({
"type": "image_url",
"image_url": {"url": f"data:{media_type};base64,{data}"},
})
elif isinstance(block, str):
parts.append({"type": "text", "text": block})
return ProviderMessage(role="user", content=parts)
return ProviderMessage(role="user", content=str(content))
def format_assistant_message(self, response: ModelResponse) -> ProviderMessage:
"""Convert ModelResponse to OpenAI assistant message format."""
text_parts = []
tool_calls = []
for block in response.content:
if block.type == "text":
text_parts.append(block.text)
elif block.type == "tool_use" and block.tool_call:
tool_calls.append({
"id": block.tool_call.id,
"type": "function",
"function": {
"name": block.tool_call.name,
"arguments": json.dumps(block.tool_call.input),
},
})
msg: dict[str, Any] = {"role": "assistant"}
if text_parts:
msg["content"] = "\n".join(text_parts)
else:
msg["content"] = None
if tool_calls:
msg["tool_calls"] = tool_calls
return ProviderMessage(role="assistant", content=msg)
def _build_messages(
self,
system: str | None,
messages: list[ProviderMessage],
) -> list[dict]:
"""Convert ProviderMessages to OpenAI API format."""
result = []
if system:
result.append({"role": "system", "content": system})
for msg in messages:
if msg.role == "assistant":
# Assistant messages are already in OpenAI format from format_assistant_message
if isinstance(msg.content, dict) and "role" in msg.content:
result.append(msg.content)
else:
# Raw content blocks from provider-agnostic format
text_parts = []
tool_calls = []
if isinstance(msg.content, list):
for block in msg.content:
if isinstance(block, dict):
if block.get("type") == "text":
text_parts.append(block["text"])
elif block.get("type") == "tool_use":
tool_calls.append({
"id": block.get("id", uuid4().hex),
"type": "function",
"function": {
"name": block.get("name", ""),
"arguments": json.dumps(block.get("input", {})),
},
})
api_msg: dict[str, Any] = {
"role": "assistant",
"content": "\n".join(text_parts) if text_parts else None,
}
if tool_calls:
api_msg["tool_calls"] = tool_calls
result.append(api_msg)
elif msg.role == "tool_result":
# Tool results: content is a list of tool result dicts
if isinstance(msg.content, list):
for tr in msg.content:
if isinstance(tr, dict) and "tool_call_id" in tr:
result.append(tr)
elif isinstance(msg.content, dict) and "tool_call_id" in msg.content:
result.append(msg.content)
elif msg.role == "user":
result.append({"role": "user", "content": msg.content})
return result
async def create_message(
self,
model: str,
system: str | None,
messages: list[ProviderMessage],
tools: list[ToolSchema],
max_tokens: int = 8192,
) -> ModelResponse:
kwargs: dict[str, Any] = {
"model": self.get_model_id(model),
"max_tokens": max_tokens,
"messages": self._build_messages(system, messages),
}
if tools:
kwargs["tools"] = [self.clean_tool_schema(t) for t in tools]
resp = await self.client.chat.completions.create(**kwargs)
choice = resp.choices[0]
message = choice.message
content: list[ContentBlock] = []
if message.content:
content.append(ContentBlock(type="text", text=message.content))
if message.tool_calls:
for tc in message.tool_calls:
try:
args = json.loads(tc.function.arguments)
except json.JSONDecodeError:
args = {}
content.append(ContentBlock(
type="tool_use",
tool_call=ToolCall(
id=tc.id,
name=tc.function.name,
input=args,
),
))
stop = "end_turn"
if choice.finish_reason == "tool_calls":
stop = "tool_use"
elif message.tool_calls:
stop = "tool_use"
usage_dict = {}
if resp.usage:
usage_dict = {
"input_tokens": resp.usage.prompt_tokens,
"output_tokens": resp.usage.completion_tokens,
}
return ModelResponse(content=content, stop_reason=stop, usage=usage_dict)
async def stream_message(
self,
model: str,
system: str | None,
messages: list[ProviderMessage],
tools: list[ToolSchema],
max_tokens: int = 8192,
) -> AsyncIterator[StreamEvent]:
kwargs: dict[str, Any] = {
"model": self.get_model_id(model),
"max_tokens": max_tokens,
"messages": self._build_messages(system, messages),
"stream": True,
"stream_options": {"include_usage": True},
}
if tools:
kwargs["tools"] = [self.clean_tool_schema(t) for t in tools]
stream = await self.client.chat.completions.create(**kwargs)
# Track streaming state to emit normalized events
text_started = False
text_index = 0
tool_indices: dict[int, dict] = {} # openai tool_call index -> {name, id, json_buf}
next_block_index = 0
async for chunk in stream:
if not chunk.choices:
# Usage-only chunk at the end
if chunk.usage:
yield StreamEvent(type="usage", usage={
"input_tokens": chunk.usage.prompt_tokens or 0,
"output_tokens": chunk.usage.completion_tokens or 0,
})
continue
delta = chunk.choices[0].delta
finish_reason = chunk.choices[0].finish_reason
# Text content
if delta.content is not None:
if not text_started:
text_started = True
text_index = next_block_index
next_block_index += 1
yield StreamEvent(
type="content_block_start",
index=text_index,
block_type="text",
)
yield StreamEvent(
type="content_block_delta",
index=text_index,
delta_type="text_delta",
text=delta.content,
)
# Tool calls
if delta.tool_calls:
for tc_delta in delta.tool_calls:
tc_idx = tc_delta.index
if tc_idx not in tool_indices:
# New tool call starting
if text_started:
yield StreamEvent(type="content_block_stop", index=text_index)
text_started = False
block_idx = next_block_index
next_block_index += 1
tool_indices[tc_idx] = {
"block_index": block_idx,
"id": tc_delta.id or uuid4().hex,
"name": tc_delta.function.name if tc_delta.function else "",
"json_buf": "",
}
yield StreamEvent(
type="content_block_start",
index=block_idx,
block_type="tool_use",
tool_name=tool_indices[tc_idx]["name"],
tool_id=tool_indices[tc_idx]["id"],
)
info = tool_indices[tc_idx]
if tc_delta.function and tc_delta.function.name:
info["name"] = tc_delta.function.name
if tc_delta.function and tc_delta.function.arguments:
info["json_buf"] += tc_delta.function.arguments
yield StreamEvent(
type="content_block_delta",
index=info["block_index"],
delta_type="input_json_delta",
text=tc_delta.function.arguments,
)
# Finish
if finish_reason is not None:
if text_started:
yield StreamEvent(type="content_block_stop", index=text_index)
for info in tool_indices.values():
yield StreamEvent(type="content_block_stop", index=info["block_index"])
yield StreamEvent(type="message_stop")
+362
View File
@@ -0,0 +1,362 @@
"""Provider factory and model registry.
Two-tier system:
1. Built-in providers (Anthropic, OpenAI, Gemini) with curated model lists
2. User-configured custom providers (any OpenAI-compatible endpoint)
- Includes built-in OpenRouter integration for 300+ models
"""
from __future__ import annotations
import logging
from typing import Any, TYPE_CHECKING
from backend.apps.agents.providers.base import BaseProvider
if TYPE_CHECKING:
from backend.apps.settings.models import AppSettings
logger = logging.getLogger(__name__)
# ---------------------------------------------------------------------------
# Tier 1: Built-in models (curated, we know their quirks)
# ---------------------------------------------------------------------------
BUILTIN_MODELS: dict[str, list[dict[str, Any]]] = {
# ── Native API providers (use direct SDK) ──
"Anthropic": [
{"value": "sonnet", "label": "Claude Sonnet 4.6", "context_window": 1_000_000, "model_id": "claude-sonnet-4-6", "api": "anthropic"},
{"value": "opus", "label": "Claude Opus 4.6", "context_window": 1_000_000, "model_id": "claude-opus-4-6", "api": "anthropic"},
{"value": "haiku", "label": "Claude Haiku 4.5", "context_window": 200_000, "model_id": "claude-haiku-4-5", "api": "anthropic"},
],
"OpenAI": [
{"value": "gpt-5.4", "label": "GPT-5.4", "context_window": 1_000_000, "api": "openai"},
{"value": "gpt-5.4-mini", "label": "GPT-5.4 Mini", "context_window": 400_000, "api": "openai"},
{"value": "o3", "label": "o3", "context_window": 200_000, "api": "openai"},
{"value": "o4-mini", "label": "o4-mini", "context_window": 200_000, "api": "openai"},
],
"Google": [
{"value": "gemini-2.5-pro", "label": "Gemini 2.5 Pro", "context_window": 1_048_576, "api": "gemini"},
{"value": "gemini-2.5-flash", "label": "Gemini 2.5 Flash", "context_window": 1_048_576, "api": "gemini"},
],
# ── Via OpenRouter (need OpenRouter API key) ──
"xAI": [
{"value": "x-ai/grok-4-0214", "label": "Grok 4", "context_window": 2_000_000, "api": "openrouter"},
],
"Meta": [
{"value": "meta-llama/llama-4-maverick", "label": "Llama 4 Maverick", "context_window": 1_000_000, "api": "openrouter"},
{"value": "meta-llama/llama-4-scout", "label": "Llama 4 Scout", "context_window": 10_000_000, "api": "openrouter"},
],
"DeepSeek": [
{"value": "deepseek/deepseek-chat-v3-0324", "label": "DeepSeek V3", "context_window": 163_840, "api": "openrouter"},
{"value": "deepseek/deepseek-r1", "label": "DeepSeek R1", "context_window": 163_840, "api": "openrouter"},
],
"Mistral": [
{"value": "mistralai/mistral-large-2501", "label": "Mistral Large", "context_window": 256_000, "api": "openrouter"},
{"value": "mistralai/mistral-small-3.1-24b-instruct", "label": "Mistral Small 3.1", "context_window": 128_000, "api": "openrouter"},
],
"Qwen": [
{"value": "qwen/qwen3-coder", "label": "Qwen3 Coder 480B", "context_window": 262_144, "api": "openrouter"},
{"value": "qwen/qwen3-235b-a22b", "label": "Qwen3 235B", "context_window": 131_072, "api": "openrouter"},
],
"Cohere": [
{"value": "cohere/command-a-03-2025", "label": "Command A", "context_window": 256_000, "api": "openrouter"},
],
}
# ---------------------------------------------------------------------------
# OpenRouter: built-in integration for 300+ models
# ---------------------------------------------------------------------------
OPENROUTER_BASE_URL = "https://openrouter.ai/api/v1"
_9router_cache: dict = {"available": None, "checked_at": 0}
def _is_9router_available() -> bool:
"""Check if 9Router is running on localhost:20128. Caches for 30 seconds."""
import time as _time
now = _time.time()
if _9router_cache["available"] is not None and now - _9router_cache["checked_at"] < 30:
return _9router_cache["available"]
try:
import httpx
r = httpx.get("http://localhost:20128/v1/models", timeout=2.0)
available = r.status_code == 200
except Exception:
available = False
_9router_cache["available"] = available
_9router_cache["checked_at"] = now
return available
# ---------------------------------------------------------------------------
# Provider factory
# ---------------------------------------------------------------------------
def create_provider(
provider_name: str,
settings: AppSettings,
provider_config: dict | None = None,
) -> BaseProvider:
"""Create a provider adapter.
Routes based on the 'api' field in BUILTIN_MODELS:
- "anthropic" native Anthropic SDK
- "openai" native OpenAI SDK (direct API)
- "gemini" native Google GenAI SDK
- "openrouter" OpenAI-compat via openrouter.ai (Meta, Mistral, DeepSeek, Qwen, xAI, etc.)
Custom providers use OpenAI-compat with user's base_url.
"""
api_type = _get_api_type(provider_name)
# Check for 9Router first
if provider_name in ("9Router", "9router"):
from backend.apps.agents.providers.openai_compat import OpenAICompatProvider
return OpenAICompatProvider(api_key="9router", base_url="http://localhost:20128/v1")
# Check for GitHub Copilot
if provider_name in ("GitHub Copilot", "copilot"):
from backend.apps.agents.providers.copilot import CopilotProvider
copilot_token = getattr(settings, "copilot_token", None)
if not copilot_token:
raise ValueError("GitHub Copilot not connected. Sign in via Settings → Models.")
# Auto-refresh if expired
import time as _time
expires = getattr(settings, "copilot_token_expires", None)
if expires and _time.time() > expires - 120:
github_token = getattr(settings, "copilot_github_token", None)
if github_token:
import asyncio
from backend.apps.agents.copilot_auth import exchange_for_copilot_token
try:
loop = asyncio.get_event_loop()
result = loop.run_until_complete(exchange_for_copilot_token(github_token))
copilot_token = result["token"]
settings.copilot_token = copilot_token
settings.copilot_token_expires = result["expires_at"]
from backend.apps.settings.settings import _save_settings
_save_settings(settings)
except Exception as e:
logger.warning(f"Copilot token refresh failed: {e}")
return CopilotProvider(copilot_token=copilot_token)
if api_type == "anthropic":
from backend.apps.agents.providers.anthropic import AnthropicProvider
if getattr(settings, "connection_mode", "own_key") == "managed":
return AnthropicProvider(
auth_token=getattr(settings, "openswarm_auth_token", None),
base_url=getattr(settings, "openswarm_proxy_url", None) or "https://api.openswarm.ai",
)
if settings.anthropic_api_key:
return AnthropicProvider(api_key=settings.anthropic_api_key)
# No API key — try 9Router as fallback
if _is_9router_available():
from backend.apps.agents.providers.openai_compat import OpenAICompatProvider
provider = OpenAICompatProvider(api_key="9router", base_url="http://localhost:20128/v1")
# Override get_model_id to map our short names to 9Router's cc/ prefixed IDs
_original_get_model = provider.get_model_id
_9r_model_map = {
"sonnet": "cc/claude-sonnet-4-6",
"opus": "cc/claude-opus-4-6",
"haiku": "cc/claude-haiku-4-5-20251001",
}
provider.get_model_id = lambda name: _9r_model_map.get(name, f"cc/{name}" if not name.startswith("cc/") else name)
return provider
raise ValueError("Anthropic API key not configured. Set it in Settings, or connect 9Router.")
if api_type == "openai":
from backend.apps.agents.providers.openai_compat import OpenAICompatProvider
if settings.openai_api_key:
return OpenAICompatProvider(api_key=settings.openai_api_key, base_url="https://api.openai.com/v1")
# No API key — try 9Router as fallback
if _is_9router_available():
return OpenAICompatProvider(api_key="9router", base_url="http://localhost:20128/v1")
raise ValueError("OpenAI API key not configured. Set it in Settings, or connect 9Router.")
if api_type == "gemini":
from backend.apps.agents.providers.gemini import GeminiProvider
if settings.google_api_key:
return GeminiProvider(api_key=settings.google_api_key)
# No API key — try 9Router as fallback
if _is_9router_available():
from backend.apps.agents.providers.openai_compat import OpenAICompatProvider
return OpenAICompatProvider(api_key="9router", base_url="http://localhost:20128/v1")
raise ValueError("Google API key not configured. Set it in Settings, or connect 9Router.")
if api_type == "openrouter":
from backend.apps.agents.providers.openai_compat import OpenAICompatProvider
openrouter_key = getattr(settings, "openrouter_api_key", None)
if openrouter_key:
return OpenAICompatProvider(api_key=openrouter_key, base_url=OPENROUTER_BASE_URL)
# No OpenRouter key — try 9Router as fallback
if _is_9router_available():
return OpenAICompatProvider(api_key="9router", base_url="http://localhost:20128/v1")
raise ValueError(f"OpenRouter API key not configured for {provider_name}. Set it in Settings, or connect a subscription.")
# Custom provider — look up in settings.custom_providers
if provider_config:
from backend.apps.agents.providers.openai_compat import OpenAICompatProvider
return OpenAICompatProvider(
api_key=provider_config.get("api_key", ""),
base_url=provider_config.get("base_url", ""),
)
for cp in getattr(settings, "custom_providers", []):
if cp.name == provider_name:
from backend.apps.agents.providers.openai_compat import OpenAICompatProvider
return OpenAICompatProvider(
api_key=cp.api_key,
base_url=cp.base_url,
)
raise ValueError(f"Unknown provider: {provider_name}")
def _get_api_type(provider_name: str) -> str:
"""Get the API type for a provider from BUILTIN_MODELS.
Accepts both display names ('Anthropic') and lowercase API names ('anthropic').
"""
# Direct lookup first (display name like 'Anthropic', 'OpenAI', etc.)
models = BUILTIN_MODELS.get(provider_name, [])
if models:
return models[0].get("api", "openrouter")
# Lowercase API name mapping
_API_NAME_MAP = {
"anthropic": "anthropic",
"openai": "openai",
"gemini": "gemini",
"google": "gemini",
"openrouter": "openrouter",
}
if provider_name.lower() in _API_NAME_MAP:
return _API_NAME_MAP[provider_name.lower()]
# Case-insensitive lookup into BUILTIN_MODELS
lower = provider_name.lower()
for key, models in BUILTIN_MODELS.items():
if key.lower() == lower:
return models[0].get("api", "openrouter")
return "openrouter"
def _has_credentials(provider_name: str, settings: AppSettings) -> bool:
"""Check if a provider has credentials configured."""
api_type = _get_api_type(provider_name)
if api_type == "anthropic":
if getattr(settings, "connection_mode", "own_key") == "managed":
return bool(getattr(settings, "openswarm_auth_token", None))
return bool(settings.anthropic_api_key)
if api_type == "openai":
return bool(settings.openai_api_key)
if api_type == "gemini":
return bool(getattr(settings, "google_api_key", None))
if api_type == "openrouter":
return bool(getattr(settings, "openrouter_api_key", None))
return False
def get_available_models(settings: AppSettings) -> dict[str, list[dict]]:
"""Return all models — always show everything, mark which have keys configured.
Like Cursor: show all models upfront, prompt for key when user tries to use one.
Returns: {"provider_name": [{"value": ..., "label": ..., "context_window": ..., "configured": bool}, ...]}
"""
result: dict[str, list[dict]] = {}
# Built-in providers — always show all
for provider_name, models in BUILTIN_MODELS.items():
configured = _has_credentials(provider_name, settings)
result[provider_name] = [
{**m, "configured": configured}
for m in models
]
# Custom providers
for cp in getattr(settings, "custom_providers", []):
if cp.models:
result[cp.name] = [
{
"value": m.get("value", m.get("id", "")),
"label": m.get("label", m.get("value", m.get("id", ""))),
"context_window": m.get("context_window", 128_000),
"configured": True,
}
for m in cp.models
]
return result
def get_context_window(provider: str, model: str, settings: AppSettings | None = None) -> int:
"""Look up context window for any model."""
# Check built-in models first
for models in BUILTIN_MODELS.values():
for m in models:
if m["value"] == model:
return m.get("context_window", 128_000)
# Check custom providers
if settings:
for cp in getattr(settings, "custom_providers", []):
for m in cp.models:
if m.get("value") == model or m.get("id") == model:
return m.get("context_window", 128_000)
return 128_000 # safe default
# ---------------------------------------------------------------------------
# Cost tracking
# ---------------------------------------------------------------------------
COST_PER_1M_TOKENS: dict[tuple[str, str], tuple[float, float]] = {
# (provider, model): (input_cost_per_1M, output_cost_per_1M)
# Anthropic
("Anthropic", "sonnet"): (3.0, 15.0),
("Anthropic", "opus"): (5.0, 25.0),
("Anthropic", "haiku"): (1.0, 5.0),
# OpenAI
("OpenAI", "gpt-5.4"): (2.50, 15.0),
("OpenAI", "gpt-5.4-mini"): (0.75, 3.0),
("OpenAI", "o3"): (2.0, 8.0),
("OpenAI", "o4-mini"): (1.10, 4.40),
# Google
("Google", "gemini-2.5-flash"): (0.15, 0.60),
("Google", "gemini-2.5-pro"): (1.25, 10.0),
# OpenRouter-backed (approximate)
("xAI", "x-ai/grok-4-0214"): (3.0, 15.0),
("Meta", "meta-llama/llama-4-maverick"): (0.50, 0.70),
("Meta", "meta-llama/llama-4-scout"): (0.15, 0.40),
("DeepSeek", "deepseek/deepseek-chat-v3-0324"): (0.30, 0.90),
("DeepSeek", "deepseek/deepseek-r1"): (0.80, 2.40),
("Mistral", "mistralai/mistral-large-2501"): (2.0, 6.0),
("Mistral", "mistralai/mistral-small-3.1-24b-instruct"): (0.10, 0.30),
("Qwen", "qwen/qwen3-coder"): (0.0, 0.0),
("Qwen", "qwen/qwen3-235b-a22b"): (0.20, 0.70),
("Cohere", "cohere/command-a-03-2025"): (2.50, 10.0),
}
def calculate_cost(
provider: str, model: str,
input_tokens: int, output_tokens: int,
) -> float:
"""Calculate cost in USD from token counts."""
# Direct lookup first
rates = COST_PER_1M_TOKENS.get((provider, model))
if not rates:
# Case-insensitive provider lookup
lower = provider.lower()
for (p, m), r in COST_PER_1M_TOKENS.items():
if p.lower() == lower and m == model:
rates = r
break
if not rates:
return 0.0
input_rate, output_rate = rates
return (input_tokens * input_rate + output_tokens * output_rate) / 1_000_000
+47
View File
@@ -0,0 +1,47 @@
"""Base classes for builtin tool implementations."""
from abc import ABC, abstractmethod
from dataclasses import dataclass
from typing import Any
@dataclass
class ToolContext:
"""Runtime context passed to every tool execution."""
cwd: str
session_id: str
class BaseTool(ABC):
"""Abstract base for all builtin tools.
Subclasses must set ``name`` and ``description`` as class attributes and
implement ``get_schema`` (JSON Schema for tool input) and ``execute``.
"""
name: str
description: str
@abstractmethod
def get_schema(self) -> dict:
"""Return JSON Schema for this tool's input parameters."""
...
@abstractmethod
async def execute(self, input_data: dict, context: ToolContext) -> list[dict]:
"""Execute the tool.
Returns a list of content blocks, e.g.
``[{"type": "text", "text": "..."}]``.
"""
...
def to_tool_schema(self):
"""Convert to the provider-agnostic ``ToolSchema`` used everywhere."""
from backend.apps.agents.providers.base import ToolSchema
return ToolSchema(
name=self.name,
description=self.description,
input_schema=self.get_schema(),
)
+476
View File
@@ -0,0 +1,476 @@
"""Filesystem tools: Read, Write, Edit, Glob, Grep."""
from __future__ import annotations
import asyncio
import base64
import mimetypes
import os
import re
from pathlib import Path
from typing import Any
from backend.apps.agents.tools.base import BaseTool, ToolContext
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
_IMAGE_EXTENSIONS = {".png", ".jpg", ".jpeg", ".gif", ".webp", ".svg"}
_MAX_OUTPUT_BYTES = 50 * 1024 # ~50 KB cap for grep output
def _resolve(file_path: str, cwd: str) -> Path:
"""Resolve *file_path* against *cwd* when it is relative."""
p = Path(file_path)
if not p.is_absolute():
p = Path(cwd) / p
return p.resolve()
def _text_block(text: str) -> list[dict]:
return [{"type": "text", "text": text}]
# ───────────────────────────────────────────────────────────────────────────
# ReadTool
# ───────────────────────────────────────────────────────────────────────────
class ReadTool(BaseTool):
name = "Read"
description = (
"Read a file from the filesystem. Returns lines with line numbers "
"(cat -n style). For image files returns base64 content. Supports "
"offset and limit parameters for reading portions of large files."
)
def get_schema(self) -> dict:
return {
"type": "object",
"properties": {
"file_path": {
"type": "string",
"description": "Absolute or relative path to the file to read.",
},
"offset": {
"type": "integer",
"description": "1-based line number to start reading from.",
},
"limit": {
"type": "integer",
"description": "Maximum number of lines to return (default 2000).",
},
},
"required": ["file_path"],
"additionalProperties": False,
}
async def execute(self, input_data: dict, context: ToolContext) -> list[dict]:
file_path = _resolve(input_data["file_path"], context.cwd)
if not file_path.exists():
return _text_block(f"Error: file not found: {file_path}")
if not file_path.is_file():
return _text_block(f"Error: not a regular file: {file_path}")
# Binary / image files → base64
ext = file_path.suffix.lower()
if ext in _IMAGE_EXTENSIONS:
try:
raw = file_path.read_bytes()
b64 = base64.b64encode(raw).decode("ascii")
media = mimetypes.guess_type(str(file_path))[0] or "application/octet-stream"
return [
{
"type": "image",
"source": {
"type": "base64",
"media_type": media,
"data": b64,
},
}
]
except Exception as exc:
return _text_block(f"Error reading image {file_path}: {exc}")
# Text files
offset = max(input_data.get("offset", 1), 1)
limit = input_data.get("limit", 2000)
if limit <= 0:
limit = 2000
try:
with open(file_path, "r", errors="replace") as fh:
lines: list[str] = []
for lineno, line in enumerate(fh, start=1):
if lineno < offset:
continue
if len(lines) >= limit:
break
# cat -n style: right-justified line number + tab + content
lines.append(f"{lineno:>6}\t{line.rstrip()}")
if not lines:
return _text_block(f"(file is empty or offset beyond end of file: {file_path})")
return _text_block("\n".join(lines))
except Exception as exc:
return _text_block(f"Error reading {file_path}: {exc}")
# ───────────────────────────────────────────────────────────────────────────
# WriteTool
# ───────────────────────────────────────────────────────────────────────────
class WriteTool(BaseTool):
name = "Write"
description = (
"Write content to a file. Creates parent directories if they do not "
"exist. Overwrites the file if it already exists."
)
def get_schema(self) -> dict:
return {
"type": "object",
"properties": {
"file_path": {
"type": "string",
"description": "Absolute or relative path to the file to write.",
},
"content": {
"type": "string",
"description": "The full content to write to the file.",
},
},
"required": ["file_path", "content"],
"additionalProperties": False,
}
async def execute(self, input_data: dict, context: ToolContext) -> list[dict]:
file_path = _resolve(input_data["file_path"], context.cwd)
content: str = input_data["content"]
try:
file_path.parent.mkdir(parents=True, exist_ok=True)
file_path.write_text(content, encoding="utf-8")
return _text_block(f"Successfully wrote {len(content)} bytes to {file_path}")
except Exception as exc:
return _text_block(f"Error writing {file_path}: {exc}")
# ───────────────────────────────────────────────────────────────────────────
# EditTool
# ───────────────────────────────────────────────────────────────────────────
class EditTool(BaseTool):
name = "Edit"
description = (
"Perform exact string replacements in a file. By default the "
"old_string must appear exactly once (not unique → error). Pass "
"replace_all=true to replace every occurrence."
)
def get_schema(self) -> dict:
return {
"type": "object",
"properties": {
"file_path": {
"type": "string",
"description": "Absolute or relative path to the file to edit.",
},
"old_string": {
"type": "string",
"description": "The exact text to find in the file.",
},
"new_string": {
"type": "string",
"description": "The text to replace old_string with.",
},
"replace_all": {
"type": "boolean",
"description": "If true, replace all occurrences. Default false.",
"default": False,
},
},
"required": ["file_path", "old_string", "new_string"],
"additionalProperties": False,
}
async def execute(self, input_data: dict, context: ToolContext) -> list[dict]:
file_path = _resolve(input_data["file_path"], context.cwd)
old_string: str = input_data["old_string"]
new_string: str = input_data["new_string"]
replace_all: bool = input_data.get("replace_all", False)
if not file_path.exists():
return _text_block(f"Error: file not found: {file_path}")
if not file_path.is_file():
return _text_block(f"Error: not a regular file: {file_path}")
try:
content = file_path.read_text(encoding="utf-8")
except Exception as exc:
return _text_block(f"Error reading {file_path}: {exc}")
count = content.count(old_string)
if count == 0:
return _text_block(
f"Error: old_string not found in {file_path}. "
"Make sure the string matches exactly, including whitespace and indentation."
)
if not replace_all and count > 1:
return _text_block(
f"Error: old_string appears {count} times in {file_path}. "
"Provide more surrounding context to make the match unique, "
"or set replace_all=true to replace every occurrence."
)
if replace_all:
new_content = content.replace(old_string, new_string)
else:
# Replace only the first (and only) occurrence
new_content = content.replace(old_string, new_string, 1)
try:
file_path.write_text(new_content, encoding="utf-8")
except Exception as exc:
return _text_block(f"Error writing {file_path}: {exc}")
replacements = count if replace_all else 1
return _text_block(
f"Successfully edited {file_path} ({replacements} replacement{'s' if replacements != 1 else ''})."
)
# ───────────────────────────────────────────────────────────────────────────
# GlobTool
# ───────────────────────────────────────────────────────────────────────────
class GlobTool(BaseTool):
name = "Glob"
description = (
"Fast file pattern matching. Supports glob patterns like '**/*.py'. "
"Returns matching file paths sorted by modification time (newest first)."
)
def get_schema(self) -> dict:
return {
"type": "object",
"properties": {
"pattern": {
"type": "string",
"description": "Glob pattern to match files (e.g. '**/*.py', 'src/**/*.ts').",
},
"path": {
"type": "string",
"description": "Directory to search in. Defaults to the working directory.",
},
},
"required": ["pattern"],
"additionalProperties": False,
}
async def execute(self, input_data: dict, context: ToolContext) -> list[dict]:
pattern: str = input_data["pattern"]
base = Path(input_data.get("path") or context.cwd)
if not base.is_dir():
return _text_block(f"Error: directory not found: {base}")
try:
matches: list[Path] = []
for p in base.glob(pattern):
if p.is_file():
matches.append(p)
if len(matches) >= 500:
break
# Sort by modification time, newest first
matches.sort(key=lambda p: p.stat().st_mtime, reverse=True)
if not matches:
return _text_block(f"No files matched pattern '{pattern}' in {base}")
result = "\n".join(str(p) for p in matches)
return _text_block(result)
except Exception as exc:
return _text_block(f"Error during glob '{pattern}' in {base}: {exc}")
# ───────────────────────────────────────────────────────────────────────────
# GrepTool
# ───────────────────────────────────────────────────────────────────────────
class GrepTool(BaseTool):
name = "Grep"
description = (
"Search file contents using regular expressions. Uses ripgrep (rg) "
"when available, otherwise falls back to Python's re module. "
"Supports output modes: files_with_matches, content, count."
)
def get_schema(self) -> dict:
return {
"type": "object",
"properties": {
"pattern": {
"type": "string",
"description": "Regular expression pattern to search for.",
},
"path": {
"type": "string",
"description": "File or directory to search in. Defaults to the working directory.",
},
"glob": {
"type": "string",
"description": "Glob pattern to filter files (e.g. '*.py', '*.{ts,tsx}').",
},
"output_mode": {
"type": "string",
"enum": ["files_with_matches", "content", "count"],
"description": "Output mode. Default: files_with_matches.",
"default": "files_with_matches",
},
},
"required": ["pattern"],
"additionalProperties": False,
}
async def execute(self, input_data: dict, context: ToolContext) -> list[dict]:
pattern: str = input_data["pattern"]
search_path: str = input_data.get("path") or context.cwd
file_glob: str | None = input_data.get("glob")
output_mode: str = input_data.get("output_mode", "files_with_matches")
# Try ripgrep first
try:
result = await self._run_rg(pattern, search_path, file_glob, output_mode)
if result is not None:
return result
except FileNotFoundError:
pass # rg not installed, fall through to Python fallback
# Python fallback
return await self._python_grep(pattern, search_path, file_glob, output_mode)
async def _run_rg(
self,
pattern: str,
search_path: str,
file_glob: str | None,
output_mode: str,
) -> list[dict] | None:
"""Run ripgrep and return results, or None if rg is not available."""
cmd = ["rg", "--no-heading", "--color=never"]
if output_mode == "files_with_matches":
cmd.append("--files-with-matches")
elif output_mode == "count":
cmd.append("--count")
else:
cmd.extend(["--line-number"])
if file_glob:
cmd.extend(["--glob", file_glob])
cmd.append(pattern)
cmd.append(search_path)
try:
proc = await asyncio.create_subprocess_exec(
*cmd,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
)
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=30)
except FileNotFoundError:
raise # re-raise so caller knows rg is missing
except asyncio.TimeoutError:
return _text_block("Error: grep timed out after 30 seconds.")
except Exception as exc:
return _text_block(f"Error running ripgrep: {exc}")
output = stdout.decode("utf-8", errors="replace")
if proc.returncode not in (0, 1):
err = stderr.decode("utf-8", errors="replace").strip()
if err:
return _text_block(f"Grep error: {err}")
if not output.strip():
return _text_block(f"No matches found for pattern '{pattern}'.")
# Truncate if too large
if len(output) > _MAX_OUTPUT_BYTES:
output = output[:_MAX_OUTPUT_BYTES] + "\n... (output truncated)"
return _text_block(output.rstrip())
async def _python_grep(
self,
pattern: str,
search_path: str,
file_glob: str | None,
output_mode: str,
) -> list[dict]:
"""Pure-Python grep fallback using the re module."""
try:
regex = re.compile(pattern)
except re.error as exc:
return _text_block(f"Invalid regex pattern: {exc}")
base = Path(search_path)
if base.is_file():
files = [base]
elif base.is_dir():
glob_pat = file_glob or "**/*"
files = [p for p in base.glob(glob_pat) if p.is_file()]
else:
return _text_block(f"Error: path not found: {search_path}")
lines_out: list[str] = []
total_bytes = 0
truncated = False
for fp in sorted(files):
try:
text = fp.read_text(encoding="utf-8", errors="replace")
except Exception:
continue
file_matches: list[tuple[int, str]] = []
for lineno, line in enumerate(text.splitlines(), start=1):
if regex.search(line):
file_matches.append((lineno, line))
if not file_matches:
continue
if output_mode == "files_with_matches":
entry = str(fp)
elif output_mode == "count":
entry = f"{fp}:{len(file_matches)}"
else:
parts = [f"{fp}:{ln}:{txt}" for ln, txt in file_matches]
entry = "\n".join(parts)
total_bytes += len(entry)
if total_bytes > _MAX_OUTPUT_BYTES:
truncated = True
break
lines_out.append(entry)
if not lines_out:
return _text_block(f"No matches found for pattern '{pattern}'.")
result = "\n".join(lines_out)
if truncated:
result += "\n... (output truncated)"
return _text_block(result)
+61
View File
@@ -0,0 +1,61 @@
"""Central tool registry.
Importing this module automatically registers all builtin tools.
"""
from __future__ import annotations
from backend.apps.agents.tools.base import BaseTool
from backend.apps.agents.providers.base import ToolSchema
_TOOLS: dict[str, BaseTool] = {}
def register_tool(tool: BaseTool) -> None:
"""Register a tool instance by its name."""
_TOOLS[tool.name] = tool
def get_tool(name: str) -> BaseTool | None:
"""Look up a registered tool by name. Returns None if not found."""
return _TOOLS.get(name)
def get_all_tools() -> list[BaseTool]:
"""Return all registered tool instances."""
return list(_TOOLS.values())
def get_all_tool_schemas() -> list[ToolSchema]:
"""Return provider-agnostic ToolSchema for every registered tool."""
return [t.to_tool_schema() for t in _TOOLS.values()]
def init_tools() -> None:
"""Import and register all builtin tools."""
from backend.apps.agents.tools.filesystem import (
ReadTool,
WriteTool,
EditTool,
GlobTool,
GrepTool,
)
from backend.apps.agents.tools.system import BashTool, AskUserQuestionTool
from backend.apps.agents.tools.web import WebSearchTool, WebFetchTool
for tool_cls in [
ReadTool,
WriteTool,
EditTool,
GlobTool,
GrepTool,
BashTool,
AskUserQuestionTool,
WebSearchTool,
WebFetchTool,
]:
register_tool(tool_cls())
# Auto-register on import
init_tools()
+125
View File
@@ -0,0 +1,125 @@
"""System tools: Bash and AskUserQuestion."""
from __future__ import annotations
import asyncio
from backend.apps.agents.tools.base import BaseTool, ToolContext
_MAX_OUTPUT_BYTES = 100 * 1024 # ~100 KB cap
class BashTool(BaseTool):
name = "Bash"
description = (
"Execute a shell command and return its output. The command runs in "
"the session's working directory. Supports an optional timeout "
"(default 120 000 ms). Stdout and stderr are captured and returned."
)
def get_schema(self) -> dict:
return {
"type": "object",
"properties": {
"command": {
"type": "string",
"description": "The shell command to execute.",
},
"timeout": {
"type": "integer",
"description": "Timeout in milliseconds (default 120000, max 600000).",
"default": 120000,
},
"description": {
"type": "string",
"description": "Optional human-readable description of what this command does.",
},
},
"required": ["command"],
"additionalProperties": False,
}
async def execute(self, input_data: dict, context: ToolContext) -> list[dict]:
command: str = input_data["command"]
timeout_ms: int = min(input_data.get("timeout", 120000), 600000)
timeout_s: float = timeout_ms / 1000.0
try:
proc = await asyncio.create_subprocess_shell(
command,
stdout=asyncio.subprocess.PIPE,
stderr=asyncio.subprocess.PIPE,
cwd=context.cwd,
)
except Exception as exc:
return [{"type": "text", "text": f"Error starting command: {exc}"}]
try:
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=timeout_s)
except asyncio.TimeoutError:
# Attempt to kill the process
try:
proc.kill()
stdout, stderr = await asyncio.wait_for(proc.communicate(), timeout=5)
except Exception:
stdout, stderr = b"", b""
partial = self._decode(stdout, stderr)
msg = (
f"Command timed out after {timeout_ms}ms.\n"
f"Partial output:\n{partial}"
)
return [{"type": "text", "text": self._truncate(msg)}]
except Exception as exc:
return [{"type": "text", "text": f"Error executing command: {exc}"}]
output = self._decode(stdout, stderr)
if proc.returncode != 0:
output = f"Exit code: {proc.returncode}\n{output}"
if not output.strip():
output = f"(command completed with exit code {proc.returncode})"
return [{"type": "text", "text": self._truncate(output)}]
@staticmethod
def _decode(stdout: bytes, stderr: bytes) -> str:
parts: list[str] = []
if stdout:
parts.append(stdout.decode("utf-8", errors="replace"))
if stderr:
parts.append(stderr.decode("utf-8", errors="replace"))
return "\n".join(parts)
@staticmethod
def _truncate(text: str) -> str:
if len(text) > _MAX_OUTPUT_BYTES:
return text[:_MAX_OUTPUT_BYTES] + "\n... (output truncated)"
return text
class AskUserQuestionTool(BaseTool):
name = "AskUserQuestion"
description = (
"Ask the user a clarifying question. The actual blocking/HITL "
"interaction is handled by the agent loop's hitl_handler; this tool "
"simply surfaces the question text."
)
def get_schema(self) -> dict:
return {
"type": "object",
"properties": {
"question": {
"type": "string",
"description": "The question to ask the user.",
},
},
"required": ["question"],
"additionalProperties": False,
}
async def execute(self, input_data: dict, context: ToolContext) -> list[dict]:
question: str = input_data.get("question", "")
return [{"type": "text", "text": question}]
+214
View File
@@ -0,0 +1,214 @@
"""Web tools: WebSearch and WebFetch."""
from __future__ import annotations
import html
import re
from typing import Any
import httpx
from backend.apps.agents.tools.base import BaseTool, ToolContext
_MAX_OUTPUT_BYTES = 100 * 1024 # ~100 KB
_HTTP_TIMEOUT = 30 # seconds
_USER_AGENT = (
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) "
"AppleWebKit/537.36 (KHTML, like Gecko) Chrome/120.0.0.0 Safari/537.36"
)
def _truncate(text: str, limit: int = _MAX_OUTPUT_BYTES) -> str:
if len(text) > limit:
return text[:limit] + "\n... (output truncated)"
return text
def _strip_html(raw_html: str) -> str:
"""Naive but effective HTML → plain-text conversion."""
# Remove script/style blocks
text = re.sub(r"<(script|style)[^>]*>.*?</\1>", "", raw_html, flags=re.DOTALL | re.IGNORECASE)
# Remove HTML tags
text = re.sub(r"<[^>]+>", " ", text)
# Decode HTML entities
text = html.unescape(text)
# Collapse whitespace
text = re.sub(r"[ \t]+", " ", text)
text = re.sub(r"\n{3,}", "\n\n", text)
return text.strip()
# ───────────────────────────────────────────────────────────────────────────
# WebSearchTool
# ───────────────────────────────────────────────────────────────────────────
class WebSearchTool(BaseTool):
name = "WebSearch"
description = (
"Search the web using DuckDuckGo and return titles, URLs, and "
"snippets for the top results."
)
def get_schema(self) -> dict:
return {
"type": "object",
"properties": {
"query": {
"type": "string",
"description": "The search query.",
},
"num_results": {
"type": "integer",
"description": "Maximum number of results to return (default 5).",
"default": 5,
},
},
"required": ["query"],
"additionalProperties": False,
}
async def execute(self, input_data: dict, context: ToolContext) -> list[dict]:
query: str = input_data["query"]
num_results: int = input_data.get("num_results", 5)
try:
results = await self._search_ddg(query, num_results)
if not results:
return [{"type": "text", "text": f"No search results found for: {query}"}]
return [{"type": "text", "text": results}]
except Exception as exc:
return [{"type": "text", "text": f"Web search error: {exc}"}]
@staticmethod
async def _search_ddg(query: str, num_results: int) -> str:
"""Query DuckDuckGo HTML endpoint and parse results."""
async with httpx.AsyncClient(
timeout=_HTTP_TIMEOUT,
follow_redirects=True,
headers={"User-Agent": _USER_AGENT},
) as client:
resp = await client.post(
"https://html.duckduckgo.com/html/",
data={"q": query},
)
resp.raise_for_status()
body = resp.text
# Parse result blocks DuckDuckGo wraps each result in
# <div class="result ..."> ... </div>
result_blocks = re.findall(
r'<div[^>]*class="[^"]*result[^"]*"[^>]*>(.*?)</div>\s*(?=<div[^>]*class="[^"]*result|$)',
body,
flags=re.DOTALL,
)
entries: list[str] = []
for block in result_blocks:
if len(entries) >= num_results:
break
# Title + URL — handle both class-before-href and href-before-class
link_match = re.search(
r'<a[^>]*class="[^"]*result__a[^"]*"[^>]*href="([^"]*)"[^>]*>(.*?)</a>',
block,
flags=re.DOTALL,
)
if not link_match:
# Try reversed attribute order
link_match = re.search(
r'<a[^>]*href="([^"]*)"[^>]*class="[^"]*result__a[^"]*"[^>]*>(.*?)</a>',
block,
flags=re.DOTALL,
)
if not link_match:
continue
raw_url = html.unescape(link_match.group(1))
title = _strip_html(link_match.group(2)).strip()
# Snippet
snippet_match = re.search(
r'<a[^>]*class="[^"]*result__snippet[^"]*"[^>]*>(.*?)</a>',
block,
flags=re.DOTALL,
)
snippet = _strip_html(snippet_match.group(1)).strip() if snippet_match else ""
# DuckDuckGo wraps URLs through a redirect; try to extract the real URL
real_url_match = re.search(r"uddg=([^&]+)", raw_url)
if real_url_match:
from urllib.parse import unquote
url = unquote(real_url_match.group(1))
else:
url = raw_url
entry = f"[{len(entries) + 1}] {title}\n {url}"
if snippet:
entry += f"\n {snippet}"
entries.append(entry)
return "\n\n".join(entries)
# ───────────────────────────────────────────────────────────────────────────
# WebFetchTool
# ───────────────────────────────────────────────────────────────────────────
class WebFetchTool(BaseTool):
name = "WebFetch"
description = (
"Fetch the contents of a URL and return the extracted text. "
"HTML is stripped to plain text. Output is truncated to ~100 KB."
)
def get_schema(self) -> dict:
return {
"type": "object",
"properties": {
"url": {
"type": "string",
"description": "The URL to fetch.",
},
"prompt": {
"type": "string",
"description": "Optional prompt/context describing what information to look for.",
},
},
"required": ["url"],
"additionalProperties": False,
}
async def execute(self, input_data: dict, context: ToolContext) -> list[dict]:
url: str = input_data["url"]
prompt: str | None = input_data.get("prompt")
try:
async with httpx.AsyncClient(
timeout=_HTTP_TIMEOUT,
follow_redirects=True,
headers={"User-Agent": _USER_AGENT},
) as client:
resp = await client.get(url)
resp.raise_for_status()
except httpx.HTTPStatusError as exc:
return [{"type": "text", "text": f"HTTP error {exc.response.status_code} fetching {url}"}]
except Exception as exc:
return [{"type": "text", "text": f"Error fetching {url}: {exc}"}]
content_type = resp.headers.get("content-type", "")
if "html" in content_type or resp.text.strip().startswith("<!"):
text = _strip_html(resp.text)
else:
text = resp.text
text = _truncate(text)
header = f"Contents of {url}:"
if prompt:
header += f"\n(Looking for: {prompt})"
return [{"type": "text", "text": f"{header}\n\n{text}"}]
+8 -3
View File
@@ -61,10 +61,12 @@ class ConnectionManager:
pass
async def send_approval_request(
self, session_id: str, request_id: str, tool_name: str, tool_input: dict
self, session_id: str, request_id: str, tool_name: str, tool_input: dict,
timeout: float = 600.0,
) -> dict:
"""Send an approval request and wait for the user's response.
Returns the approval decision dict."""
Returns the approval decision dict. Times out after *timeout* seconds
(default 10 minutes) to prevent permanently stuck agents."""
future = asyncio.get_event_loop().create_future()
self.pending_futures[request_id] = future
@@ -75,8 +77,11 @@ class ConnectionManager:
})
try:
result = await future
result = await asyncio.wait_for(future, timeout=timeout)
return result
except asyncio.TimeoutError:
logger.warning("Approval %s for session %s timed out after %ss", request_id, session_id, timeout)
return {"behavior": "deny", "message": "Approval timed out"}
finally:
self.pending_futures.pop(request_id, None)
View File
+168
View File
@@ -0,0 +1,168 @@
"""Analytics SubApp: PostHog for product analytics + local usage summary from session data."""
import json
import logging
import os
import platform
from collections import Counter
from contextlib import asynccontextmanager
from backend.config.Apps import SubApp
from backend.config.paths import SESSIONS_DIR
from backend.apps.analytics.collector import init as init_collector, shutdown as shutdown_collector, record, identify
logger = logging.getLogger(__name__)
@asynccontextmanager
async def analytics_lifespan():
init_collector()
logger.info("PostHog analytics initialised")
try:
from backend.apps.settings.settings import load_settings
settings = load_settings()
providers = []
if getattr(settings, "anthropic_api_key", None):
providers.append("anthropic")
if getattr(settings, "openai_api_key", None):
providers.append("openai")
if getattr(settings, "google_api_key", None):
providers.append("gemini")
if getattr(settings, "openrouter_api_key", None):
providers.append("openrouter")
for cp in getattr(settings, "custom_providers", []):
providers.append(cp.name)
record("app.opened", {
"os": platform.system(),
"platform": platform.platform(),
"provider_count": len(providers),
"providers": providers,
"connection_mode": getattr(settings, "connection_mode", "own_key"),
})
identify({
"providers_configured": providers,
"provider_count": len(providers),
"connection_mode": getattr(settings, "connection_mode", "own_key"),
})
except Exception as e:
logger.debug(f"Analytics startup event failed (non-critical): {e}")
# Auto-start 9Router for subscription access
try:
from backend.apps.nine_router import ensure_running as ensure_9router, stop as stop_9router
await ensure_9router()
except Exception as e:
logger.debug(f"9Router auto-start skipped: {e}")
yield
# Stop 9Router
try:
from backend.apps.nine_router import stop as stop_9router
stop_9router()
except Exception:
pass
shutdown_collector()
logger.info("PostHog analytics shut down")
analytics = SubApp("analytics", analytics_lifespan)
def _load_all_sessions() -> list[dict]:
"""Load all persisted session JSON files."""
results = []
if not os.path.exists(SESSIONS_DIR):
return results
for fname in os.listdir(SESSIONS_DIR):
if fname.endswith(".json"):
try:
with open(os.path.join(SESSIONS_DIR, fname)) as f:
results.append(json.load(f))
except Exception:
pass
return results
@analytics.router.get("/usage-summary")
async def usage_summary():
"""Compute usage stats from persisted sessions for the Settings page."""
from backend.apps.agents.agent_manager import agent_manager
# Combine persisted + active sessions
sessions = _load_all_sessions()
for s in agent_manager.get_all_sessions():
sessions.append(s.model_dump(mode="json"))
total_sessions = len(sessions)
total_cost = sum(s.get("cost_usd", 0) for s in sessions)
total_messages = 0
total_tool_calls = 0
total_duration = 0.0
model_counts: Counter = Counter()
provider_counts: Counter = Counter()
tool_counts: Counter = Counter()
status_counts: Counter = Counter()
for s in sessions:
messages = s.get("messages", [])
user_msgs = [m for m in messages if m.get("role") in ("user", "assistant")]
tool_msgs = [m for m in messages if m.get("role") == "tool_call"]
total_messages += len(user_msgs)
total_tool_calls += len(tool_msgs)
model_counts[s.get("model", "unknown")] += 1
provider_counts[s.get("provider", "anthropic")] += 1
status_counts[s.get("status", "unknown")] += 1
# Duration
created = s.get("created_at")
closed = s.get("closed_at")
if created and closed:
try:
from datetime import datetime
fmt = "%Y-%m-%dT%H:%M:%S"
c_str = created[:19]
cl_str = closed[:19]
dur = (datetime.fromisoformat(cl_str) - datetime.fromisoformat(c_str)).total_seconds()
if dur > 0:
total_duration += dur
except Exception:
pass
# Count individual tools
for m in tool_msgs:
content = m.get("content", {})
if isinstance(content, dict):
tool_name = content.get("tool", "")
if tool_name:
tool_counts[tool_name] += 1
avg_duration = total_duration / total_sessions if total_sessions > 0 else 0
avg_cost = total_cost / total_sessions if total_sessions > 0 else 0
completed = status_counts.get("completed", 0)
completion_rate = completed / total_sessions if total_sessions > 0 else 0
return {
"total_sessions": total_sessions,
"total_cost_usd": round(total_cost, 4),
"total_messages": total_messages,
"total_tool_calls": total_tool_calls,
"avg_duration_seconds": round(avg_duration, 1),
"avg_cost_per_session": round(avg_cost, 4),
"completion_rate": round(completion_rate, 3),
"models_used": dict(model_counts.most_common(10)),
"providers_used": dict(provider_counts.most_common(10)),
"top_tools": dict(tool_counts.most_common(15)),
"status_breakdown": dict(status_counts),
}
@analytics.router.get("/status")
async def analytics_status():
return {"status": "posthog", "enabled": True}
+123
View File
@@ -0,0 +1,123 @@
"""PostHog-only analytics collector.
All events go directly to PostHog. No local SQLite storage.
Usage from any module:
from backend.apps.analytics.collector import record
record("session.started", {"model": "opus"}, session_id="abc123")
"""
import logging
import platform
from uuid import uuid4
from posthog import Posthog
logger = logging.getLogger(__name__)
POSTHOG_API_KEY = "phc_KdVLvAdjCuHeacFoDm1CM1Gb23XikewRqlX67Mj6TNB"
POSTHOG_HOST = "https://us.i.posthog.com"
_posthog: Posthog | None = None
_installation_id: str | None = None
def init():
"""Initialise PostHog. Called once at app startup."""
global _posthog
if _posthog is None:
_posthog = Posthog(
project_api_key=POSTHOG_API_KEY,
host=POSTHOG_HOST,
)
return _posthog
def shutdown():
"""Flush and close. Called at app shutdown."""
global _posthog
if _posthog:
try:
_posthog.shutdown()
except Exception:
pass
_posthog = None
def _get_installation_id() -> str:
"""Get or create a stable anonymous installation ID."""
global _installation_id
if _installation_id:
return _installation_id
try:
from backend.apps.settings.settings import load_settings, _save_settings
settings = load_settings()
iid = getattr(settings, "installation_id", None)
if not iid:
iid = uuid4().hex
settings.installation_id = iid
_save_settings(settings)
_installation_id = iid
except Exception:
_installation_id = uuid4().hex
return _installation_id
def _is_opted_in() -> bool:
"""Check if user has opted in to analytics."""
try:
from backend.apps.settings.settings import load_settings
return getattr(load_settings(), "analytics_opt_in", True)
except Exception:
return True
def record(
event_type: str,
properties: dict | None = None,
session_id: str | None = None,
dashboard_id: str | None = None,
):
"""Record an analytics event to PostHog."""
if not _posthog or not _is_opted_in():
return
props = {**(properties or {})}
if session_id:
props["session_id"] = session_id
if dashboard_id:
props["dashboard_id"] = dashboard_id
props["os"] = platform.system()
props["platform"] = platform.platform()
try:
_posthog.capture(
event_type,
distinct_id=_get_installation_id(),
properties=props,
)
except Exception as e:
logger.debug(f"PostHog capture failed (non-critical): {e}")
def identify(extra_properties: dict | None = None):
"""Identify the current installation with properties."""
if not _posthog or not _is_opted_in():
return
try:
_posthog.identify(
_get_installation_id(),
properties={
"os": platform.system(),
"platform": platform.platform(),
**(extra_properties or {}),
},
)
except Exception as e:
logger.debug(f"PostHog identify failed (non-critical): {e}")
def get_collector():
"""Backward compat — returns None since we no longer have a local collector."""
return None
+37
View File
@@ -0,0 +1,37 @@
from pydantic import BaseModel
from typing import Optional
class AnalyticsEvent(BaseModel):
id: Optional[int] = None
timestamp: str
event_type: str
properties: dict
session_id: Optional[str] = None
dashboard_id: Optional[str] = None
class UsageSummary(BaseModel):
total_sessions: int = 0
total_cost_usd: float = 0.0
total_messages: int = 0
total_tool_calls: int = 0
avg_session_duration_seconds: float = 0.0
session_completion_rate: float = 0.0
approval_rate: float = 0.0
models_used: dict[str, int] = {}
modes_used: dict[str, int] = {}
top_tools: list[list] = []
class TimeSeriesPoint(BaseModel):
date: str
value: float
class ExportPayload(BaseModel):
export_version: str = "1.0"
exported_at: str = ""
app_version: str = "unknown"
period: dict = {}
summary: dict = {}
View File
+168
View File
@@ -0,0 +1,168 @@
"""Auth SubApp — handles managed-mode authentication with Open Swarm service.
All endpoints are currently stubbed with mock responses so the full UI flow
works end-to-end without a real proxy server.
"""
from contextlib import asynccontextmanager
from uuid import uuid4
from pydantic import BaseModel
from typing import Optional
from backend.config.Apps import SubApp
from backend.apps.settings.settings import load_settings, update_settings
# ── Models ──────────────────────────────────────────────────────────────
class LoginRequest(BaseModel):
email: str
password: str
class GoogleCallbackRequest(BaseModel):
code: str
redirect_uri: str = ""
class LoginResponse(BaseModel):
ok: bool
token: Optional[str] = None
email: Optional[str] = None
proxy_url: Optional[str] = None
error: Optional[str] = None
class ValidateResponse(BaseModel):
valid: bool
email: Optional[str] = None
class UsageResponse(BaseModel):
used_usd: float
quota_usd: float
reset_date: str
# ── SubApp setup ────────────────────────────────────────────────────────
@asynccontextmanager
async def _lifespan():
yield
auth = SubApp("auth", _lifespan)
router = auth.router
# ── Endpoints ───────────────────────────────────────────────────────────
@router.post("/login")
async def login(req: LoginRequest) -> LoginResponse:
"""Authenticate with email + password.
TODO: replace with real API call to Open Swarm auth server.
"""
if not req.email or not req.password:
return LoginResponse(ok=False, error="Email and password are required")
# Stub: generate a mock token for any valid-looking input
mock_token = f"osw_{uuid4().hex}"
proxy_url = "https://api.openswarm.ai"
# Persist credentials to settings
settings = load_settings()
settings.connection_mode = "managed"
settings.openswarm_auth_token = mock_token
settings.openswarm_proxy_url = proxy_url
settings.openswarm_user_email = req.email
await _save_settings(settings)
return LoginResponse(ok=True, token=mock_token, email=req.email, proxy_url=proxy_url)
@router.post("/google-url")
async def google_auth_url() -> dict:
"""Return the Google OAuth authorize URL.
TODO: replace with real Google OAuth URL construction.
"""
# Stub: return a placeholder URL
return {
"url": "https://accounts.google.com/o/oauth2/v2/auth?client_id=PLACEHOLDER&response_type=code&scope=email+profile&redirect_uri=http://localhost:8324/api/auth/google-callback"
}
@router.post("/google-callback")
async def google_callback(req: GoogleCallbackRequest) -> LoginResponse:
"""Exchange Google OAuth code for a session token.
TODO: replace with real OAuth code exchange + Open Swarm auth server call.
"""
if not req.code:
return LoginResponse(ok=False, error="Authorization code is required")
# Stub: generate a mock token
mock_token = f"osw_{uuid4().hex}"
proxy_url = "https://api.openswarm.ai"
mock_email = "user@gmail.com"
settings = load_settings()
settings.connection_mode = "managed"
settings.openswarm_auth_token = mock_token
settings.openswarm_proxy_url = proxy_url
settings.openswarm_user_email = mock_email
await _save_settings(settings)
return LoginResponse(ok=True, token=mock_token, email=mock_email, proxy_url=proxy_url)
@router.post("/validate")
async def validate_token() -> ValidateResponse:
"""Check if the stored auth token is still valid.
TODO: replace with real validation call to Open Swarm auth server.
"""
settings = load_settings()
if not settings.openswarm_auth_token:
return ValidateResponse(valid=False)
# Stub: always return valid
return ValidateResponse(valid=True, email=settings.openswarm_user_email)
@router.post("/logout")
async def logout() -> dict:
"""Clear managed-mode credentials from settings."""
settings = load_settings()
settings.connection_mode = "own_key"
settings.openswarm_auth_token = None
settings.openswarm_proxy_url = None
settings.openswarm_user_email = None
await _save_settings(settings)
return {"ok": True}
@router.get("/usage")
async def get_usage() -> UsageResponse:
"""Fetch usage and quota information for the current managed-mode user.
TODO: replace with real API call to Open Swarm proxy server.
"""
settings = load_settings()
if not settings.openswarm_auth_token:
return UsageResponse(used_usd=0, quota_usd=0, reset_date="")
# Stub: return mock usage data
return UsageResponse(used_usd=0, quota_usd=50, reset_date="2026-04-01")
# ── Helpers ─────────────────────────────────────────────────────────────
async def _save_settings(settings):
"""Persist settings to disk (reuses the settings module's update logic)."""
import json
from backend.apps.settings.settings import SETTINGS_FILE
with open(SETTINGS_FILE, "w") as f:
json.dump(settings.model_dump(), f, indent=2)
View File
@@ -0,0 +1,146 @@
"""Telnyx implementation of BaseChannelAdapter.
Uses Telnyx Call Control v2 for voice and Messaging API for SMS.
"""
import asyncio
import hashlib
import hmac
import logging
from typing import Optional
from functools import partial
from backend.apps.channels.base_adapter import BaseChannelAdapter
logger = logging.getLogger(__name__)
class TelnyxAdapter(BaseChannelAdapter):
def __init__(self, api_key: str, public_key: str = ""):
self._api_key = api_key
self._public_key = public_key
self._telnyx = None
def _get_telnyx(self):
if self._telnyx is None:
import telnyx
telnyx.api_key = self._api_key
self._telnyx = telnyx
return self._telnyx
async def send_sms(self, to: str, from_: str, body: str) -> dict:
telnyx = self._get_telnyx()
loop = asyncio.get_event_loop()
msg = await loop.run_in_executor(
None,
partial(
telnyx.Message.create,
to=to,
from_=from_,
text=body,
),
)
return {"id": msg.id, "status": getattr(msg, "status", "queued")}
async def send_whatsapp(self, to: str, from_: str, body: str) -> dict:
# Telnyx WhatsApp uses the same messaging API with messaging_profile_id
return await self.send_sms(to, from_, body)
async def initiate_call(
self, to: str, from_: str, webhook_url: str, greeting: str = ""
) -> dict:
telnyx = self._get_telnyx()
loop = asyncio.get_event_loop()
call = await loop.run_in_executor(
None,
partial(
telnyx.Call.create,
to=to,
from_=from_,
connection_id=self._api_key, # connection_id should be set separately
webhook_url=webhook_url,
),
)
return {"call_control_id": call.call_control_id, "status": "initiated"}
def verify_webhook_signature(
self, request_url: str, params: dict, signature: str, auth_token: str
) -> bool:
if not self._public_key:
# Fail closed: no public key means reject
logger.error("Telnyx public key not configured — rejecting webhook")
return False
try:
# Telnyx uses Ed25519 signature verification
# The signature and timestamp are in webhook headers
import base64
from cryptography.hazmat.primitives.asymmetric.ed25519 import Ed25519PublicKey
from cryptography.hazmat.primitives.serialization import load_pem_public_key
public_key = load_pem_public_key(self._public_key.encode())
sig_bytes = base64.b64decode(signature)
payload = params.get("_raw_body", "")
timestamp = params.get("_timestamp", "")
signed_payload = f"{timestamp}|{payload}"
public_key.verify(sig_bytes, signed_payload.encode())
return True
except Exception:
logger.exception("Telnyx signature verification failed")
return False
def generate_twiml_gather(
self,
prompt: str,
action_url: str,
voice: str = "Polly.Joanna",
language: str = "en-US",
timeout: int = 10,
) -> str:
# Telnyx uses TeXML (Twilio-compatible XML)
return (
'<?xml version="1.0" encoding="UTF-8"?>'
"<Response>"
f'<Gather input="speech" action="{action_url}" '
f'language="{language}" speechTimeout="{timeout}">'
f'<Say voice="{voice}">{_escape_xml(prompt)}</Say>'
"</Gather>"
f'<Say voice="{voice}">I didn\'t hear anything. Goodbye.</Say>'
"</Response>"
)
def generate_twiml_say(
self, text: str, voice: str = "Polly.Joanna", language: str = "en-US"
) -> str:
return (
'<?xml version="1.0" encoding="UTF-8"?>'
"<Response>"
f'<Say voice="{voice}">{_escape_xml(text)}</Say>'
"</Response>"
)
def generate_twiml_hangup(self) -> str:
return (
'<?xml version="1.0" encoding="UTF-8"?>'
"<Response><Hangup/></Response>"
)
async def get_recording_audio(self, recording_url: str, auth_token: str) -> bytes:
import httpx
async with httpx.AsyncClient() as client:
resp = await client.get(
recording_url,
headers={"Authorization": f"Bearer {self._api_key}"},
follow_redirects=True,
)
resp.raise_for_status()
return resp.content
def _escape_xml(text: str) -> str:
return (
text.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replace('"', "&quot;")
.replace("'", "&apos;")
)
@@ -0,0 +1,139 @@
"""Twilio implementation of BaseChannelAdapter.
Handles SMS, WhatsApp, and Voice via the Twilio Python SDK.
"""
import asyncio
import logging
from typing import Optional
from functools import partial
from backend.apps.channels.base_adapter import BaseChannelAdapter
logger = logging.getLogger(__name__)
class TwilioAdapter(BaseChannelAdapter):
def __init__(self, account_sid: str, auth_token: str):
self._account_sid = account_sid
self._auth_token = auth_token
self._client = None
def _get_client(self):
if self._client is None:
from twilio.rest import Client
self._client = Client(self._account_sid, self._auth_token)
return self._client
async def send_sms(self, to: str, from_: str, body: str) -> dict:
client = self._get_client()
loop = asyncio.get_event_loop()
msg = await loop.run_in_executor(
None,
partial(
client.messages.create,
to=to,
from_=from_,
body=body,
),
)
return {"sid": msg.sid, "status": msg.status}
async def send_whatsapp(self, to: str, from_: str, body: str) -> dict:
wa_to = to if to.startswith("whatsapp:") else f"whatsapp:{to}"
wa_from = from_ if from_.startswith("whatsapp:") else f"whatsapp:{from_}"
client = self._get_client()
loop = asyncio.get_event_loop()
msg = await loop.run_in_executor(
None,
partial(
client.messages.create,
to=wa_to,
from_=wa_from,
body=body,
),
)
return {"sid": msg.sid, "status": msg.status}
async def initiate_call(
self, to: str, from_: str, webhook_url: str, greeting: str = ""
) -> dict:
client = self._get_client()
loop = asyncio.get_event_loop()
call = await loop.run_in_executor(
None,
partial(
client.calls.create,
to=to,
from_=from_,
url=webhook_url,
),
)
return {"sid": call.sid, "status": call.status}
def verify_webhook_signature(
self, request_url: str, params: dict, signature: str, auth_token: str
) -> bool:
try:
from twilio.request_validator import RequestValidator
validator = RequestValidator(auth_token)
return validator.validate(request_url, params, signature)
except Exception:
logger.exception("Twilio signature verification failed")
return False
def generate_twiml_gather(
self,
prompt: str,
action_url: str,
voice: str = "Polly.Joanna",
language: str = "en-US",
timeout: int = 10,
) -> str:
return (
'<?xml version="1.0" encoding="UTF-8"?>'
"<Response>"
f'<Gather input="speech" action="{action_url}" '
f'language="{language}" speechTimeout="{timeout}">'
f'<Say voice="{voice}">{_escape_xml(prompt)}</Say>'
"</Gather>"
f'<Say voice="{voice}">I didn\'t hear anything. Goodbye.</Say>'
"</Response>"
)
def generate_twiml_say(
self, text: str, voice: str = "Polly.Joanna", language: str = "en-US"
) -> str:
return (
'<?xml version="1.0" encoding="UTF-8"?>'
"<Response>"
f'<Say voice="{voice}">{_escape_xml(text)}</Say>'
"</Response>"
)
def generate_twiml_hangup(self) -> str:
return (
'<?xml version="1.0" encoding="UTF-8"?>'
"<Response><Hangup/></Response>"
)
async def get_recording_audio(self, recording_url: str, auth_token: str) -> bytes:
import httpx
async with httpx.AsyncClient() as client:
resp = await client.get(
recording_url,
auth=(self._account_sid, auth_token),
follow_redirects=True,
)
resp.raise_for_status()
return resp.content
def _escape_xml(text: str) -> str:
return (
text.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replace('"', "&quot;")
.replace("'", "&apos;")
)
+88
View File
@@ -0,0 +1,88 @@
from abc import ABC, abstractmethod
from typing import Optional
class BaseChannelAdapter(ABC):
"""Provider-agnostic interface for telephony operations."""
@abstractmethod
async def send_sms(self, to: str, from_: str, body: str) -> dict:
"""Send an SMS message. Returns provider response dict."""
...
@abstractmethod
async def send_whatsapp(self, to: str, from_: str, body: str) -> dict:
"""Send a WhatsApp message. Returns provider response dict."""
...
@abstractmethod
async def initiate_call(
self, to: str, from_: str, webhook_url: str, greeting: str = ""
) -> dict:
"""Initiate an outbound voice call. Returns provider response dict."""
...
@abstractmethod
def verify_webhook_signature(
self, request_url: str, params: dict, signature: str, auth_token: str
) -> bool:
"""Verify that an inbound webhook is authentic."""
...
@abstractmethod
def generate_twiml_gather(
self,
prompt: str,
action_url: str,
voice: str = "Polly.Joanna",
language: str = "en-US",
timeout: int = 10,
) -> str:
"""Generate TwiML (or equivalent) to play a prompt and gather speech."""
...
@abstractmethod
def generate_twiml_say(
self, text: str, voice: str = "Polly.Joanna", language: str = "en-US"
) -> str:
"""Generate TwiML (or equivalent) to speak text."""
...
@abstractmethod
def generate_twiml_hangup(self) -> str:
"""Generate TwiML (or equivalent) to end a call."""
...
@abstractmethod
async def get_recording_audio(self, recording_url: str, auth_token: str) -> bytes:
"""Download audio from a recording URL."""
...
def chunk_message(self, text: str, max_length: int = 1600) -> list[str]:
"""Split a long message into chunks respecting sentence boundaries."""
if len(text) <= max_length:
return [text]
chunks: list[str] = []
remaining = text
while remaining:
if len(remaining) <= max_length:
chunks.append(remaining)
break
# Try to split at sentence boundary
split_at = -1
for sep in [". ", "! ", "? ", "\n\n", "\n", " "]:
idx = remaining.rfind(sep, 0, max_length)
if idx > 0:
split_at = idx + len(sep)
break
if split_at <= 0:
split_at = max_length
chunks.append(remaining[:split_at].rstrip())
remaining = remaining[split_at:].lstrip()
return chunks
+129
View File
@@ -0,0 +1,129 @@
import logging
from datetime import datetime
from typing import Optional, Literal
logger = logging.getLogger(__name__)
CallStatus = Literal[
"ringing", "connected", "gathering", "processing", "responding", "completed", "failed"
]
VALID_TRANSITIONS: dict[CallStatus, set[CallStatus]] = {
"ringing": {"connected", "completed", "failed"},
"connected": {"gathering", "completed", "failed"},
"gathering": {"processing", "completed", "failed"},
"processing": {"responding", "completed", "failed"},
"responding": {"gathering", "completed", "failed"},
"completed": set(),
"failed": set(),
}
class CallState:
"""Tracks the lifecycle of a single voice call."""
def __init__(
self,
call_sid: str,
channel_id: str,
from_number: str,
to_number: str,
):
self.call_sid = call_sid
self.channel_id = channel_id
self.from_number = from_number
self.to_number = to_number
self.agent_session_id: Optional[str] = None
self.status: CallStatus = "ringing"
self.turns: list[dict] = []
self.created_at = datetime.now()
self.last_activity = datetime.now()
self.error: Optional[str] = None
def transition(self, new_status: CallStatus) -> bool:
"""Attempt a state transition. Returns True if valid."""
if new_status in VALID_TRANSITIONS.get(self.status, set()):
logger.info(
"Call %s: %s -> %s", self.call_sid, self.status, new_status
)
self.status = new_status
self.last_activity = datetime.now()
return True
logger.warning(
"Call %s: invalid transition %s -> %s",
self.call_sid, self.status, new_status,
)
return False
def add_turn(self, role: str, content: str):
self.turns.append({
"role": role,
"content": content,
"timestamp": datetime.now().isoformat(),
})
self.last_activity = datetime.now()
@property
def is_active(self) -> bool:
return self.status not in ("completed", "failed")
@property
def duration_seconds(self) -> float:
return (datetime.now() - self.created_at).total_seconds()
def to_dict(self) -> dict:
return {
"call_sid": self.call_sid,
"channel_id": self.channel_id,
"from_number": self.from_number,
"to_number": self.to_number,
"agent_session_id": self.agent_session_id,
"status": self.status,
"turns": self.turns,
"created_at": self.created_at.isoformat(),
"last_activity": self.last_activity.isoformat(),
"duration_seconds": self.duration_seconds,
"error": self.error,
}
class CallManager:
"""Manages all active voice calls."""
def __init__(self):
self.calls: dict[str, CallState] = {}
def create_call(
self,
call_sid: str,
channel_id: str,
from_number: str,
to_number: str,
) -> CallState:
call = CallState(call_sid, channel_id, from_number, to_number)
self.calls[call_sid] = call
return call
def get_call(self, call_sid: str) -> Optional[CallState]:
return self.calls.get(call_sid)
def end_call(self, call_sid: str, status: CallStatus = "completed"):
call = self.calls.get(call_sid)
if call:
call.transition(status)
def cleanup_stale(self, max_duration_seconds: int = 3600):
"""Remove calls that have exceeded max duration."""
stale = [
sid
for sid, call in self.calls.items()
if not call.is_active or call.duration_seconds > max_duration_seconds
]
for sid in stale:
if self.calls[sid].is_active:
self.calls[sid].transition("failed")
self.calls[sid].error = "Exceeded max call duration"
del self.calls[sid]
def get_active_calls(self) -> list[dict]:
return [c.to_dict() for c in self.calls.values() if c.is_active]
+387
View File
@@ -0,0 +1,387 @@
"""Channels SubApp — REST endpoints and Twilio/Telnyx webhooks."""
import logging
import os
from contextlib import asynccontextmanager
from datetime import datetime
from fastapi import HTTPException, Request
from fastapi.responses import JSONResponse, Response
from backend.config.Apps import SubApp
from backend.apps.channels.models import (
ChannelConfig, ChannelCreate, ChannelUpdate, VoiceConfig, TTSConfig, STTConfig,
)
from backend.apps.channels.orchestrator import channel_orchestrator
from backend.apps.channels import ws_events
logger = logging.getLogger(__name__)
@asynccontextmanager
async def channels_lifespan():
logger.info("Channels sub-app starting")
await channel_orchestrator.restore_all()
yield
logger.info("Channels sub-app shutting down")
await channel_orchestrator.persist_all()
channels = SubApp("channels", channels_lifespan)
# ─── CRUD Endpoints ──────────────────────────────────────────────
@channels.router.get("/list")
async def list_channels():
configs = list(channel_orchestrator.configs.values())
return {
"channels": [c.model_dump(mode="json") for c in configs],
}
@channels.router.get("/{channel_id}")
async def get_channel(channel_id: str):
config = channel_orchestrator.configs.get(channel_id)
if not config:
raise HTTPException(404, "Channel not found")
return config.model_dump(mode="json")
@channels.router.post("/create")
async def create_channel(body: ChannelCreate):
config = ChannelConfig(
name=body.name,
channel_type=body.channel_type,
provider=body.provider,
phone_number=body.phone_number,
credentials=body.credentials,
)
if body.agent_config:
config.agent_config = body.agent_config
if body.security:
config.security = body.security
if body.voice_config:
config.voice_config = body.voice_config
if body.tts_config:
config.tts_config = body.tts_config
if body.stt_config:
config.stt_config = body.stt_config
channel_orchestrator.save_config(config)
return {"channel": config.model_dump(mode="json")}
@channels.router.put("/{channel_id}")
async def update_channel(channel_id: str, body: ChannelUpdate):
config = channel_orchestrator.configs.get(channel_id)
if not config:
raise HTTPException(404, "Channel not found")
updates = body.model_dump(exclude_none=True)
for key, val in updates.items():
setattr(config, key, val)
# Re-create adapter if credentials changed
if "credentials" in updates or "provider" in updates:
channel_orchestrator.adapters.pop(channel_id, None)
channel_orchestrator.save_config(config)
return {"channel": config.model_dump(mode="json")}
@channels.router.delete("/{channel_id}")
async def delete_channel(channel_id: str):
if channel_id not in channel_orchestrator.configs:
raise HTTPException(404, "Channel not found")
channel_orchestrator.delete_config(channel_id)
return {"ok": True}
# ─── Enable / Disable / Test ─────────────────────────────────────
@channels.router.post("/{channel_id}/enable")
async def enable_channel(channel_id: str):
config = channel_orchestrator.configs.get(channel_id)
if not config:
raise HTTPException(404, "Channel not found")
try:
channel_orchestrator.get_adapter(config)
config.enabled = True
config.status = "active"
config.status_message = None
channel_orchestrator.save_config(config)
await ws_events.emit_channel_status(channel_id, "active")
return {"ok": True, "status": "active"}
except Exception as e:
config.status = "error"
config.status_message = str(e)
channel_orchestrator.save_config(config)
raise HTTPException(400, f"Failed to enable channel: {e}")
@channels.router.post("/{channel_id}/disable")
async def disable_channel(channel_id: str):
config = channel_orchestrator.configs.get(channel_id)
if not config:
raise HTTPException(404, "Channel not found")
config.enabled = False
config.status = "inactive"
channel_orchestrator.adapters.pop(channel_id, None)
channel_orchestrator.save_config(config)
await ws_events.emit_channel_status(channel_id, "inactive")
return {"ok": True}
@channels.router.post("/{channel_id}/test")
async def test_channel(channel_id: str, body: dict | None = None):
config = channel_orchestrator.configs.get(channel_id)
if not config:
raise HTTPException(404, "Channel not found")
to_number = (body or {}).get("to_number", "")
if not to_number:
raise HTTPException(400, "to_number is required for test")
try:
adapter = channel_orchestrator.get_adapter(config)
if config.channel_type == "whatsapp":
result = await adapter.send_whatsapp(to_number, config.phone_number, "Test message from Open Swarm")
elif config.channel_type == "voice":
result = {"message": "Voice test: configure webhook and call the number"}
else:
result = await adapter.send_sms(to_number, config.phone_number, "Test message from Open Swarm")
return {"ok": True, "result": result}
except Exception as e:
raise HTTPException(400, f"Test failed: {e}")
# ─── Conversations ────────────────────────────────────────────────
@channels.router.get("/{channel_id}/conversations")
async def list_conversations(channel_id: str):
convs = [
c.model_dump(mode="json")
for c in channel_orchestrator.conversations.values()
if c.channel_id == channel_id
]
return {"conversations": convs}
@channels.router.get("/{channel_id}/conversations/{conversation_id}")
async def get_conversation(channel_id: str, conversation_id: str):
for conv in channel_orchestrator.conversations.values():
if conv.id == conversation_id and conv.channel_id == channel_id:
return conv.model_dump(mode="json")
raise HTTPException(404, "Conversation not found")
# ─── Outbound ─────────────────────────────────────────────────────
@channels.router.post("/{channel_id}/send")
async def send_outbound(channel_id: str, body: dict):
to_number = body.get("to_number", "")
message = body.get("message", "")
if not to_number or not message:
raise HTTPException(400, "to_number and message are required")
try:
result = await channel_orchestrator.send_outbound(channel_id, to_number, message)
return result
except ValueError as e:
raise HTTPException(404, str(e))
@channels.router.post("/{channel_id}/call")
async def initiate_call(channel_id: str, body: dict):
to_number = body.get("to_number", "")
if not to_number:
raise HTTPException(400, "to_number is required")
try:
result = await channel_orchestrator.initiate_outbound_call(channel_id, to_number)
return result
except ValueError as e:
raise HTTPException(404, str(e))
# ─── Twilio Webhooks ─────────────────────────────────────────────
@channels.router.post("/webhooks/twilio/sms")
async def twilio_sms_webhook(request: Request):
"""Inbound SMS webhook from Twilio."""
form = await request.form()
channel_id = request.query_params.get("channel_id", "")
# Find channel by phone number if channel_id not provided
if not channel_id:
to_number = form.get("To", "")
for cfg in channel_orchestrator.configs.values():
if cfg.phone_number == to_number and cfg.channel_type == "sms":
channel_id = cfg.id
break
config = channel_orchestrator.configs.get(channel_id)
if not config:
return Response(status_code=404)
# Verify signature
if config.security.verify_signatures:
adapter = channel_orchestrator.get_adapter(config)
sig = request.headers.get("X-Twilio-Signature", "")
url = str(request.url)
if not adapter.verify_webhook_signature(url, dict(form), sig, config.credentials.get("auth_token", "")):
logger.warning("Invalid Twilio signature for channel %s", channel_id)
return Response(status_code=403)
from_number = form.get("From", "")
body = form.get("Body", "")
num_media = int(form.get("NumMedia", "0"))
media_urls = [form.get(f"MediaUrl{i}", "") for i in range(num_media)]
media_urls = [u for u in media_urls if u]
await channel_orchestrator.handle_inbound_sms(channel_id, from_number, body, media_urls)
# Return empty TwiML (Twilio expects XML response)
return Response(
content='<?xml version="1.0"?><Response></Response>',
media_type="application/xml",
)
@channels.router.post("/webhooks/twilio/whatsapp")
async def twilio_whatsapp_webhook(request: Request):
"""Inbound WhatsApp webhook from Twilio."""
form = await request.form()
channel_id = request.query_params.get("channel_id", "")
if not channel_id:
to_number = form.get("To", "").replace("whatsapp:", "")
for cfg in channel_orchestrator.configs.values():
if cfg.phone_number == to_number and cfg.channel_type == "whatsapp":
channel_id = cfg.id
break
config = channel_orchestrator.configs.get(channel_id)
if not config:
return Response(status_code=404)
if config.security.verify_signatures:
adapter = channel_orchestrator.get_adapter(config)
sig = request.headers.get("X-Twilio-Signature", "")
if not adapter.verify_webhook_signature(str(request.url), dict(form), sig, config.credentials.get("auth_token", "")):
return Response(status_code=403)
from_number = form.get("From", "").replace("whatsapp:", "")
body = form.get("Body", "")
num_media = int(form.get("NumMedia", "0"))
media_urls = [form.get(f"MediaUrl{i}", "") for i in range(num_media)]
await channel_orchestrator.handle_inbound_sms(channel_id, from_number, body, media_urls or None)
return Response(
content='<?xml version="1.0"?><Response></Response>',
media_type="application/xml",
)
@channels.router.post("/webhooks/twilio/voice")
async def twilio_voice_webhook(request: Request):
"""Inbound voice call webhook from Twilio."""
form = await request.form()
channel_id = request.query_params.get("channel_id", "")
if not channel_id:
to_number = form.get("To", "")
for cfg in channel_orchestrator.configs.values():
if cfg.phone_number == to_number and cfg.channel_type == "voice":
channel_id = cfg.id
break
call_sid = form.get("CallSid", "")
from_number = form.get("From", "")
to_number = form.get("To", "")
twiml = await channel_orchestrator.handle_inbound_call(
channel_id, call_sid, from_number, to_number
)
return Response(content=twiml, media_type="application/xml")
@channels.router.post("/webhooks/twilio/voice/gather")
async def twilio_voice_gather_webhook(request: Request):
"""Speech gathered from a voice call."""
form = await request.form()
channel_id = request.query_params.get("channel_id", "")
call_sid = request.query_params.get("call_sid", "") or form.get("CallSid", "")
speech_result = form.get("SpeechResult", "")
if not speech_result:
# No speech detected, ask again or hang up
config = channel_orchestrator.configs.get(channel_id)
if config:
adapter = channel_orchestrator.get_adapter(config)
voice_cfg = config.voice_config or VoiceConfig()
twiml = adapter.generate_twiml_say(
"I didn't catch that. Goodbye.", voice=voice_cfg.voice
)
else:
twiml = '<?xml version="1.0"?><Response><Say>Goodbye.</Say><Hangup/></Response>'
return Response(content=twiml, media_type="application/xml")
twiml = await channel_orchestrator.handle_voice_gather(
channel_id, call_sid, speech_result
)
return Response(content=twiml, media_type="application/xml")
@channels.router.post("/webhooks/twilio/voice/status")
async def twilio_voice_status_webhook(request: Request):
"""Call status update from Twilio."""
form = await request.form()
call_sid = form.get("CallSid", "")
status = form.get("CallStatus", "")
channel_orchestrator.handle_call_status(call_sid, status)
return Response(status_code=204)
# ─── Telnyx Webhook ───────────────────────────────────────────────
@channels.router.post("/webhooks/telnyx")
async def telnyx_webhook(request: Request):
"""Unified Telnyx webhook for SMS and Voice events."""
body = await request.json()
event_type = body.get("data", {}).get("event_type", "")
payload = body.get("data", {}).get("payload", {})
channel_id = request.query_params.get("channel_id", "")
if event_type == "message.received":
from_number = payload.get("from", {}).get("phone_number", "")
text = payload.get("text", "")
await channel_orchestrator.handle_inbound_sms(channel_id, from_number, text)
elif event_type in ("call.initiated", "call.answered"):
call_sid = payload.get("call_control_id", "")
from_number = payload.get("from", "")
to_number = payload.get("to", "")
# Telnyx voice uses Call Control commands rather than TwiML
logger.info("Telnyx call event: %s for %s", event_type, call_sid)
return JSONResponse({"ok": True})
# ─── Active Calls ─────────────────────────────────────────────────
@channels.router.get("/calls/active")
async def list_active_calls():
return {"calls": channel_orchestrator.call_manager.get_active_calls()}
+60
View File
@@ -0,0 +1,60 @@
"""Audio attachment processing for WhatsApp voice notes and media messages."""
import logging
from typing import Optional
import httpx
from backend.apps.channels.models import STTConfig
from backend.apps.channels import stt_service
logger = logging.getLogger(__name__)
SUPPORTED_FORMATS = {
"audio/ogg", "audio/mpeg", "audio/wav", "audio/mp4",
"audio/flac", "audio/webm", "audio/x-wav",
}
MAX_MEDIA_BYTES = 20 * 1024 * 1024
async def process_audio_attachment(
url: str,
content_type: str,
stt_config: STTConfig,
auth: tuple[str, str] | None = None,
) -> Optional[str]:
"""Download an audio attachment and return its transcript.
Args:
url: URL to download the audio from.
content_type: MIME type of the audio.
stt_config: STT configuration for transcription.
auth: Optional (username, password) tuple for basic auth (e.g. Twilio).
Returns:
Transcript string, or None on failure.
"""
if content_type not in SUPPORTED_FORMATS:
logger.warning("Unsupported audio format: %s", content_type)
return None
try:
async with httpx.AsyncClient(timeout=60) as client:
kwargs = {"follow_redirects": True}
if auth:
kwargs["auth"] = auth
resp = await client.get(url, **kwargs)
resp.raise_for_status()
audio_bytes = resp.content
except Exception:
logger.exception("Failed to download audio from %s", url)
return None
if len(audio_bytes) > MAX_MEDIA_BYTES:
logger.warning("Audio attachment exceeds %d bytes", MAX_MEDIA_BYTES)
return None
if len(audio_bytes) < 1024:
logger.debug("Audio attachment too small, skipping")
return None
return await stt_service.transcribe(audio_bytes, stt_config, content_type)
+116
View File
@@ -0,0 +1,116 @@
from pydantic import BaseModel, Field
from typing import Optional, Literal, Any
from datetime import datetime
from uuid import uuid4
class ChannelAgentConfig(BaseModel):
mode: str = "agent"
model: str = "sonnet"
system_prompt: Optional[str] = None
max_turns: int = 10
allowed_tools: Optional[list[str]] = None
class ChannelSecurityConfig(BaseModel):
verify_signatures: bool = True
allowlist: list[str] = Field(default_factory=list)
blocklist: list[str] = Field(default_factory=list)
rate_limit_per_minute: int = 10
rate_limit_per_hour: int = 60
class VoiceConfig(BaseModel):
mode: Literal["conversation", "notify"] = "conversation"
greeting_message: str = "Hello, how can I help you?"
silence_timeout_ms: int = 700
max_call_duration_seconds: int = 600
gather_timeout_seconds: int = 10
voice: str = "Polly.Joanna"
language: str = "en-US"
class TTSConfig(BaseModel):
provider: Literal["twilio_say", "elevenlabs", "openai_tts", "edge_tts"] = "twilio_say"
auto_tts_mode: Literal["off", "always", "inbound", "tagged"] = "off"
elevenlabs_voice_id: Optional[str] = None
elevenlabs_model_id: str = "eleven_v3"
openai_voice: str = "alloy"
skip_short_text: bool = True
summarize_long_replies: bool = True
max_tts_chars: int = 4000
class STTConfig(BaseModel):
provider: Literal["twilio_builtin", "deepgram", "openai_whisper"] = "twilio_builtin"
deepgram_model: str = "nova-3"
language: str = "en-US"
fallback_chain: list[str] = Field(default_factory=lambda: ["twilio_builtin"])
class ChannelConfig(BaseModel):
id: str = Field(default_factory=lambda: uuid4().hex)
name: str = ""
channel_type: Literal["sms", "whatsapp", "voice"] = "sms"
provider: Literal["twilio", "telnyx"] = "twilio"
enabled: bool = False
phone_number: str = ""
credentials: dict[str, str] = Field(default_factory=dict)
agent_config: ChannelAgentConfig = Field(default_factory=ChannelAgentConfig)
security: ChannelSecurityConfig = Field(default_factory=ChannelSecurityConfig)
voice_config: Optional[VoiceConfig] = None
tts_config: Optional[TTSConfig] = None
stt_config: Optional[STTConfig] = None
status: Literal["inactive", "active", "error"] = "inactive"
status_message: Optional[str] = None
created_at: str = Field(default_factory=lambda: datetime.now().isoformat())
updated_at: str = Field(default_factory=lambda: datetime.now().isoformat())
last_message_at: Optional[str] = None
message_count: int = 0
class ChannelMessage(BaseModel):
id: str = Field(default_factory=lambda: uuid4().hex)
direction: Literal["inbound", "outbound"] = "inbound"
content: str = ""
media_urls: list[str] = Field(default_factory=list)
timestamp: str = Field(default_factory=lambda: datetime.now().isoformat())
channel_type: str = ""
provider_message_id: Optional[str] = None
class ChannelConversation(BaseModel):
id: str = Field(default_factory=lambda: uuid4().hex)
channel_id: str = ""
phone_number: str = ""
agent_session_id: Optional[str] = None
messages: list[ChannelMessage] = Field(default_factory=list)
created_at: str = Field(default_factory=lambda: datetime.now().isoformat())
updated_at: str = Field(default_factory=lambda: datetime.now().isoformat())
status: Literal["active", "closed"] = "active"
class ChannelCreate(BaseModel):
name: str
channel_type: Literal["sms", "whatsapp", "voice"] = "sms"
provider: Literal["twilio", "telnyx"] = "twilio"
phone_number: str = ""
credentials: dict[str, str] = Field(default_factory=dict)
agent_config: Optional[ChannelAgentConfig] = None
security: Optional[ChannelSecurityConfig] = None
voice_config: Optional[VoiceConfig] = None
tts_config: Optional[TTSConfig] = None
stt_config: Optional[STTConfig] = None
class ChannelUpdate(BaseModel):
name: Optional[str] = None
channel_type: Optional[Literal["sms", "whatsapp", "voice"]] = None
provider: Optional[Literal["twilio", "telnyx"]] = None
phone_number: Optional[str] = None
credentials: Optional[dict[str, str]] = None
agent_config: Optional[ChannelAgentConfig] = None
security: Optional[ChannelSecurityConfig] = None
voice_config: Optional[VoiceConfig] = None
tts_config: Optional[TTSConfig] = None
stt_config: Optional[STTConfig] = None
+559
View File
@@ -0,0 +1,559 @@
"""Channel orchestrator — routes inbound messages/calls to agent sessions.
This is the central routing layer that bridges telephony events to the
existing AgentManager. Each phone number gets its own ChannelConversation
which maps to an AgentSession.
"""
import asyncio
import json
import logging
import os
import time
from datetime import datetime
from typing import Optional
from backend.apps.channels.models import (
ChannelConfig, ChannelConversation, ChannelMessage,
)
from backend.apps.channels.call_state import CallManager, CallState
from backend.apps.channels.base_adapter import BaseChannelAdapter
from backend.apps.channels import ws_events
from backend.apps.agents.models import AgentConfig
from backend.config.paths import DATA_ROOT
logger = logging.getLogger(__name__)
CHANNELS_DIR = os.path.join(DATA_ROOT, "channels")
CHANNELS_SESSIONS_DIR = os.path.join(DATA_ROOT, "channels", "sessions")
PLATFORM_MAX_LENGTH = {
"sms": 1600,
"whatsapp": 4096,
"voice": 100000,
}
class RateLimiter:
"""Simple token-bucket rate limiter per phone number."""
def __init__(self):
self._buckets: dict[str, list[float]] = {}
def check(self, key: str, per_minute: int, per_hour: int) -> bool:
now = time.time()
if key not in self._buckets:
self._buckets[key] = []
# Prune old entries
self._buckets[key] = [t for t in self._buckets[key] if now - t < 3600]
recent_minute = sum(1 for t in self._buckets[key] if now - t < 60)
recent_hour = len(self._buckets[key])
if recent_minute >= per_minute or recent_hour >= per_hour:
return False
self._buckets[key].append(now)
return True
class ChannelOrchestrator:
"""Manages channel configs, conversations, and message routing."""
def __init__(self):
self.configs: dict[str, ChannelConfig] = {}
self.conversations: dict[str, ChannelConversation] = {} # key: "{channel_id}:{phone}"
self.adapters: dict[str, BaseChannelAdapter] = {}
self.call_manager = CallManager()
self.rate_limiter = RateLimiter()
self._agent_listeners: dict[str, asyncio.Task] = {}
# ─── Config persistence ───────────────────────────────────────
def _ensure_dirs(self):
os.makedirs(CHANNELS_DIR, exist_ok=True)
os.makedirs(CHANNELS_SESSIONS_DIR, exist_ok=True)
def _config_path(self, channel_id: str) -> str:
return os.path.join(CHANNELS_DIR, f"{channel_id}.json")
def _conv_path(self, channel_id: str) -> str:
return os.path.join(CHANNELS_SESSIONS_DIR, f"{channel_id}.json")
def save_config(self, config: ChannelConfig):
self._ensure_dirs()
config.updated_at = datetime.now().isoformat()
self.configs[config.id] = config
with open(self._config_path(config.id), "w") as f:
json.dump(config.model_dump(mode="json"), f, indent=2)
def delete_config(self, channel_id: str):
self.configs.pop(channel_id, None)
self.adapters.pop(channel_id, None)
path = self._config_path(channel_id)
if os.path.exists(path):
os.remove(path)
def load_all_configs(self):
self._ensure_dirs()
self.configs.clear()
for fname in os.listdir(CHANNELS_DIR):
if fname.endswith(".json"):
try:
with open(os.path.join(CHANNELS_DIR, fname)) as f:
data = json.load(f)
config = ChannelConfig(**data)
self.configs[config.id] = config
except Exception:
logger.exception("Failed to load channel config: %s", fname)
# ─── Conversation persistence ─────────────────────────────────
def save_conversations(self, channel_id: str):
self._ensure_dirs()
convs = [
c.model_dump(mode="json")
for c in self.conversations.values()
if c.channel_id == channel_id
]
with open(self._conv_path(channel_id), "w") as f:
json.dump(convs, f, indent=2)
def load_all_conversations(self):
self._ensure_dirs()
self.conversations.clear()
for fname in os.listdir(CHANNELS_SESSIONS_DIR):
if fname.endswith(".json"):
try:
with open(os.path.join(CHANNELS_SESSIONS_DIR, fname)) as f:
convs = json.load(f)
for data in convs:
conv = ChannelConversation(**data)
key = f"{conv.channel_id}:{conv.phone_number}"
self.conversations[key] = conv
except Exception:
logger.exception("Failed to load conversations: %s", fname)
# ─── Adapter management ───────────────────────────────────────
def get_adapter(self, config: ChannelConfig) -> BaseChannelAdapter:
if config.id not in self.adapters:
self.adapters[config.id] = self._create_adapter(config)
return self.adapters[config.id]
def _create_adapter(self, config: ChannelConfig) -> BaseChannelAdapter:
if config.provider == "twilio":
from backend.apps.channels.adapters.twilio_adapter import TwilioAdapter
return TwilioAdapter(
account_sid=config.credentials.get("account_sid", ""),
auth_token=config.credentials.get("auth_token", ""),
)
elif config.provider == "telnyx":
from backend.apps.channels.adapters.telnyx_adapter import TelnyxAdapter
return TelnyxAdapter(
api_key=config.credentials.get("api_key", ""),
public_key=config.credentials.get("public_key", ""),
)
raise ValueError(f"Unknown provider: {config.provider}")
# ─── Security checks ─────────────────────────────────────────
def _check_allowlist(self, config: ChannelConfig, phone: str) -> bool:
sec = config.security
if phone in sec.blocklist:
return False
if sec.allowlist and phone not in sec.allowlist:
return False
return True
def _check_rate_limit(self, config: ChannelConfig, phone: str) -> bool:
sec = config.security
return self.rate_limiter.check(
phone, sec.rate_limit_per_minute, sec.rate_limit_per_hour
)
# ─── Inbound SMS / WhatsApp ───────────────────────────────────
async def handle_inbound_sms(
self,
channel_id: str,
from_number: str,
body: str,
media_urls: list[str] | None = None,
) -> Optional[str]:
"""Handle an inbound SMS or WhatsApp message. Returns agent response or None."""
config = self.configs.get(channel_id)
if not config or not config.enabled:
logger.warning("Channel %s not found or disabled", channel_id)
return None
if not self._check_allowlist(config, from_number):
logger.info("Blocked message from %s (not in allowlist)", from_number)
return None
if not self._check_rate_limit(config, from_number):
logger.info("Rate limited: %s", from_number)
return None
# Process media attachments (voice notes)
if media_urls and config.stt_config:
from backend.apps.channels.media_handler import process_audio_attachment
for url in media_urls:
transcript = await process_audio_attachment(
url, "audio/ogg", config.stt_config,
auth=(
config.credentials.get("account_sid", ""),
config.credentials.get("auth_token", ""),
) if config.provider == "twilio" else None,
)
if transcript:
body = f"{body}\n\n[Voice Note Transcript]: {transcript}" if body else transcript
# Get or create conversation
conv_key = f"{channel_id}:{from_number}"
conv = self.conversations.get(conv_key)
if not conv:
conv = ChannelConversation(
channel_id=channel_id,
phone_number=from_number,
)
self.conversations[conv_key] = conv
# Record inbound message
inbound_msg = ChannelMessage(
direction="inbound",
content=body,
media_urls=media_urls or [],
channel_type=config.channel_type,
)
conv.messages.append(inbound_msg)
conv.updated_at = datetime.now().isoformat()
await ws_events.emit_channel_message(
channel_id, conv.id, inbound_msg.model_dump(mode="json")
)
# Launch or reuse agent session
agent_response = await self._route_to_agent(config, conv, body)
if agent_response:
# Send response back via SMS/WhatsApp
adapter = self.get_adapter(config)
max_len = PLATFORM_MAX_LENGTH.get(config.channel_type, 1600)
chunks = adapter.chunk_message(agent_response, max_len)
for chunk in chunks:
if config.channel_type == "whatsapp":
await adapter.send_whatsapp(from_number, config.phone_number, chunk)
else:
await adapter.send_sms(from_number, config.phone_number, chunk)
outbound_msg = ChannelMessage(
direction="outbound",
content=agent_response,
channel_type=config.channel_type,
)
conv.messages.append(outbound_msg)
conv.updated_at = datetime.now().isoformat()
config.message_count += 1
config.last_message_at = datetime.now().isoformat()
await ws_events.emit_channel_message(
channel_id, conv.id, outbound_msg.model_dump(mode="json")
)
self.save_conversations(channel_id)
self.save_config(config)
return agent_response
# ─── Inbound Voice ────────────────────────────────────────────
async def handle_inbound_call(
self, channel_id: str, call_sid: str, from_number: str, to_number: str
) -> str:
"""Handle an inbound voice call. Returns initial TwiML."""
config = self.configs.get(channel_id)
if not config or not config.enabled:
adapter = self._fallback_adapter(config)
return adapter.generate_twiml_hangup()
if not self._check_allowlist(config, from_number):
adapter = self.get_adapter(config)
return adapter.generate_twiml_say("Sorry, you are not authorized to call this number.")
voice_cfg = config.voice_config or VoiceConfig()
adapter = self.get_adapter(config)
# Create call state
call = self.call_manager.create_call(call_sid, channel_id, from_number, to_number)
call.transition("connected")
call.transition("gathering")
await ws_events.emit_call_event(channel_id, call_sid, "call_started", {
"from": from_number, "to": to_number,
})
# Return TwiML to greet and gather speech
from backend.apps.settings.settings import load_settings
settings = load_settings()
webhook_base = getattr(settings, "webhook_base_url", "") or ""
gather_url = f"{webhook_base}/api/channels/webhooks/twilio/voice/gather?channel_id={channel_id}&call_sid={call_sid}"
return adapter.generate_twiml_gather(
prompt=voice_cfg.greeting_message,
action_url=gather_url,
voice=voice_cfg.voice,
language=voice_cfg.language,
timeout=voice_cfg.gather_timeout_seconds,
)
async def handle_voice_gather(
self, channel_id: str, call_sid: str, speech_result: str
) -> str:
"""Handle gathered speech from a voice call. Returns response TwiML."""
config = self.configs.get(channel_id)
if not config:
return '<?xml version="1.0"?><Response><Hangup/></Response>'
call = self.call_manager.get_call(call_sid)
if not call or not call.is_active:
adapter = self.get_adapter(config)
return adapter.generate_twiml_hangup()
call.transition("processing")
call.add_turn("user", speech_result)
voice_cfg = config.voice_config or VoiceConfig()
adapter = self.get_adapter(config)
# Route speech to agent
conv_key = f"{channel_id}:{call.from_number}"
conv = self.conversations.get(conv_key)
if not conv:
conv = ChannelConversation(
channel_id=channel_id,
phone_number=call.from_number,
)
self.conversations[conv_key] = conv
agent_response = await self._route_to_agent(config, conv, speech_result)
if not agent_response:
agent_response = "I'm sorry, I couldn't process that. Could you try again?"
call.transition("responding")
call.add_turn("assistant", agent_response)
await ws_events.emit_call_event(channel_id, call_sid, "turn_complete", {
"user": speech_result, "assistant": agent_response,
})
# Check if we should continue or end
if voice_cfg.mode == "notify":
call.transition("completed")
return adapter.generate_twiml_say(agent_response, voice=voice_cfg.voice)
# Conversation mode: say response then gather again
from backend.apps.settings.settings import load_settings
settings = load_settings()
webhook_base = getattr(settings, "webhook_base_url", "") or ""
gather_url = f"{webhook_base}/api/channels/webhooks/twilio/voice/gather?channel_id={channel_id}&call_sid={call_sid}"
call.transition("gathering")
return (
'<?xml version="1.0" encoding="UTF-8"?>'
"<Response>"
f'<Say voice="{voice_cfg.voice}">{_escape_xml(agent_response)}</Say>'
f'<Gather input="speech" action="{gather_url}" '
f'language="{voice_cfg.language}" speechTimeout="{voice_cfg.gather_timeout_seconds}">'
"</Gather>"
f'<Say voice="{voice_cfg.voice}">Are you still there? Goodbye.</Say>'
"</Response>"
)
def handle_call_status(self, call_sid: str, status: str):
"""Handle Twilio call status callback."""
call = self.call_manager.get_call(call_sid)
if not call:
return
if status in ("completed", "busy", "no-answer", "canceled", "failed"):
final = "failed" if status == "failed" else "completed"
call.transition(final)
asyncio.create_task(
ws_events.emit_call_event(call.channel_id, call_sid, "call_ended", {
"status": status,
})
)
# ─── Agent routing ────────────────────────────────────────────
async def _route_to_agent(
self, config: ChannelConfig, conv: ChannelConversation, text: str
) -> Optional[str]:
"""Send a message to an agent session and wait for the response."""
from backend.apps.agents.agent_manager import agent_manager
from backend.apps.agents.ws_manager import ws_manager
# Launch agent if no session exists
if not conv.agent_session_id or not agent_manager.get_session(conv.agent_session_id):
ac = config.agent_config
agent_cfg = AgentConfig(
name=f"{config.channel_type}: {conv.phone_number}",
model=ac.model,
mode=ac.mode,
system_prompt=ac.system_prompt,
max_turns=ac.max_turns,
)
if ac.allowed_tools:
agent_cfg.allowed_tools = ac.allowed_tools
session = await agent_manager.launch_agent(agent_cfg)
conv.agent_session_id = session.id
session_id = conv.agent_session_id
# Set up a future to capture the agent's response
response_future: asyncio.Future[str] = asyncio.get_event_loop().create_future()
async def _on_agent_event(event: str, data: dict):
if response_future.done():
return
if event == "agent:message":
msg = data.get("message", {})
if msg.get("role") == "assistant":
content = msg.get("content", "")
if isinstance(content, list):
# Extract text from content blocks
parts = [
b.get("text", "")
for b in content
if isinstance(b, dict) and b.get("type") == "text"
]
content = "\n".join(parts)
if content and not response_future.done():
response_future.set_result(content)
elif event == "agent:status":
status = data.get("status", "")
if status in ("completed", "error", "stopped") and not response_future.done():
response_future.set_result("")
# Register listener for this session's events
# We tap into ws_manager's send_to_session by monkey-patching temporarily
original_send = ws_manager.send_to_session
async def _hooked_send(sid: str, event: str, data: dict):
await original_send(sid, event, data)
if sid == session_id:
await _on_agent_event(event, data)
ws_manager.send_to_session = _hooked_send
try:
await agent_manager.send_message(session_id, text)
response = await asyncio.wait_for(response_future, timeout=120)
return response if response else None
except asyncio.TimeoutError:
logger.warning("Agent response timed out for session %s", session_id)
return None
except Exception:
logger.exception("Error routing to agent")
return None
finally:
ws_manager.send_to_session = original_send
def _fallback_adapter(self, config: Optional[ChannelConfig] = None) -> BaseChannelAdapter:
"""Return a minimal adapter for generating hangup TwiML."""
from backend.apps.channels.adapters.twilio_adapter import TwilioAdapter
return TwilioAdapter("", "")
# ─── Outbound ─────────────────────────────────────────────────
async def send_outbound(
self, channel_id: str, to_number: str, message: str
) -> dict:
config = self.configs.get(channel_id)
if not config:
raise ValueError(f"Channel {channel_id} not found")
adapter = self.get_adapter(config)
max_len = PLATFORM_MAX_LENGTH.get(config.channel_type, 1600)
chunks = adapter.chunk_message(message, max_len)
results = []
for chunk in chunks:
if config.channel_type == "whatsapp":
r = await adapter.send_whatsapp(to_number, config.phone_number, chunk)
else:
r = await adapter.send_sms(to_number, config.phone_number, chunk)
results.append(r)
# Record outbound
conv_key = f"{channel_id}:{to_number}"
conv = self.conversations.get(conv_key)
if not conv:
conv = ChannelConversation(channel_id=channel_id, phone_number=to_number)
self.conversations[conv_key] = conv
conv.messages.append(ChannelMessage(
direction="outbound", content=message, channel_type=config.channel_type,
))
conv.updated_at = datetime.now().isoformat()
self.save_conversations(channel_id)
return {"sent": len(chunks), "results": results}
async def initiate_outbound_call(
self, channel_id: str, to_number: str
) -> dict:
config = self.configs.get(channel_id)
if not config:
raise ValueError(f"Channel {channel_id} not found")
from backend.apps.settings.settings import load_settings
settings = load_settings()
webhook_base = getattr(settings, "webhook_base_url", "") or ""
voice_webhook = f"{webhook_base}/api/channels/webhooks/twilio/voice?channel_id={channel_id}"
adapter = self.get_adapter(config)
result = await adapter.initiate_call(
to=to_number,
from_=config.phone_number,
webhook_url=voice_webhook,
)
return result
# ─── Lifecycle ────────────────────────────────────────────────
async def restore_all(self):
self.load_all_configs()
self.load_all_conversations()
for config in self.configs.values():
if config.enabled:
try:
self.get_adapter(config)
config.status = "active"
except Exception:
config.status = "error"
config.status_message = "Failed to initialize adapter"
async def persist_all(self):
for config in self.configs.values():
self.save_config(config)
self.save_conversations(config.id)
def _escape_xml(text: str) -> str:
return (
text.replace("&", "&amp;")
.replace("<", "&lt;")
.replace(">", "&gt;")
.replace('"', "&quot;")
.replace("'", "&apos;")
)
# Singleton
from backend.apps.channels.models import VoiceConfig # noqa: E402
channel_orchestrator = ChannelOrchestrator()
+142
View File
@@ -0,0 +1,142 @@
"""Provider-abstracted Speech-to-Text service.
Supports: Twilio built-in (via Gather), Deepgram Nova-3, OpenAI Whisper.
"""
import logging
from typing import Optional
import httpx
from backend.apps.channels.models import STTConfig
from backend.apps.settings.settings import load_settings
logger = logging.getLogger(__name__)
SUPPORTED_AUDIO_FORMATS = {
"audio/ogg", "audio/mpeg", "audio/wav", "audio/mp4",
"audio/flac", "audio/webm", "audio/x-wav",
}
MAX_MEDIA_BYTES = 20 * 1024 * 1024 # 20 MB
async def transcribe(
audio_bytes: bytes,
config: STTConfig,
content_type: str = "audio/wav",
) -> Optional[str]:
"""Transcribe audio bytes to text using the configured provider chain."""
if len(audio_bytes) > MAX_MEDIA_BYTES:
logger.warning("Audio exceeds %d bytes limit", MAX_MEDIA_BYTES)
return None
if len(audio_bytes) < 1024:
logger.debug("Audio too short, skipping")
return None
providers = config.fallback_chain or [config.provider]
for provider in providers:
try:
if provider == "twilio_builtin":
# Twilio STT is handled inline by <Gather> — no bytes to process
continue
elif provider == "deepgram":
result = await _deepgram_transcribe(audio_bytes, config, content_type)
elif provider == "openai_whisper":
result = await _openai_transcribe(audio_bytes, config, content_type)
else:
logger.warning("Unknown STT provider: %s", provider)
continue
if result:
return result
except Exception:
logger.exception("STT provider %s failed, trying next", provider)
return None
async def transcribe_from_url(
url: str, config: STTConfig, content_type: str = "audio/ogg"
) -> Optional[str]:
"""Download audio from URL and transcribe."""
try:
async with httpx.AsyncClient(timeout=60) as client:
resp = await client.get(url, follow_redirects=True)
resp.raise_for_status()
return await transcribe(resp.content, config, content_type)
except Exception:
logger.exception("Failed to download audio from %s", url)
return None
async def _deepgram_transcribe(
audio_bytes: bytes, config: STTConfig, content_type: str
) -> Optional[str]:
settings = load_settings()
api_key = settings.deepgram_api_key if hasattr(settings, "deepgram_api_key") else None
if not api_key:
logger.warning("Deepgram API key not configured")
return None
try:
async with httpx.AsyncClient(timeout=60) as client:
resp = await client.post(
"https://api.deepgram.com/v1/listen",
headers={
"Authorization": f"Token {api_key}",
"Content-Type": content_type,
},
params={
"model": config.deepgram_model,
"language": config.language,
"smart_format": "true",
"punctuate": "true",
},
content=audio_bytes,
)
resp.raise_for_status()
data = resp.json()
return (
data.get("results", {})
.get("channels", [{}])[0]
.get("alternatives", [{}])[0]
.get("transcript", "")
)
except Exception:
logger.exception("Deepgram transcription failed")
return None
async def _openai_transcribe(
audio_bytes: bytes, config: STTConfig, content_type: str
) -> Optional[str]:
settings = load_settings()
api_key = settings.openai_api_key if hasattr(settings, "openai_api_key") else None
if not api_key:
logger.warning("OpenAI API key not configured")
return None
ext_map = {
"audio/ogg": "ogg",
"audio/mpeg": "mp3",
"audio/wav": "wav",
"audio/x-wav": "wav",
"audio/mp4": "m4a",
"audio/flac": "flac",
"audio/webm": "webm",
}
ext = ext_map.get(content_type, "wav")
try:
async with httpx.AsyncClient(timeout=60) as client:
resp = await client.post(
"https://api.openai.com/v1/audio/transcriptions",
headers={"Authorization": f"Bearer {api_key}"},
files={"file": (f"audio.{ext}", audio_bytes, content_type)},
data={"model": "whisper-1", "language": config.language[:2]},
)
resp.raise_for_status()
return resp.json().get("text", "")
except Exception:
logger.exception("OpenAI Whisper transcription failed")
return None
+163
View File
@@ -0,0 +1,163 @@
"""Browser-based Talk Mode — continuous voice conversation via WebSocket.
Pipeline: Mic WebSocket STT Agent TTS WebSocket Speaker
This runs as a separate WebSocket endpoint /ws/talk/{session_id} that
streams audio bidirectionally between the browser and the STT/TTS services.
"""
import asyncio
import json
import logging
from typing import Optional
from fastapi import WebSocket, WebSocketDisconnect
from backend.apps.channels import stt_service, tts_service
from backend.apps.channels.models import STTConfig, TTSConfig
from backend.apps.agents.agent_manager import agent_manager
from backend.apps.agents.ws_manager import ws_manager
from backend.apps.settings.settings import load_settings
logger = logging.getLogger(__name__)
# Default configs for talk mode
DEFAULT_STT = STTConfig(
provider="openai_whisper",
fallback_chain=["openai_whisper", "deepgram"],
)
DEFAULT_TTS = TTSConfig(
provider="elevenlabs",
skip_short_text=False,
)
async def handle_talk_session(websocket: WebSocket, session_id: str):
"""Handle a talk-mode WebSocket connection.
Protocol:
- Client sends: {"type": "audio", "data": "<base64 audio>", "format": "webm"}
- Client sends: {"type": "config", "stt": {...}, "tts": {...}}
- Client sends: {"type": "end_utterance"} when silence detected
- Server sends: {"type": "transcript", "text": "..."}
- Server sends: {"type": "audio", "data": "<base64 audio>", "format": "mp3"}
- Server sends: {"type": "agent_response", "text": "..."}
- Server sends: {"type": "status", "status": "listening|processing|speaking"}
"""
await websocket.accept()
stt_config = DEFAULT_STT
tts_config = DEFAULT_TTS
audio_buffer = bytearray()
try:
while True:
data = await websocket.receive_text()
msg = json.loads(data)
msg_type = msg.get("type", "")
if msg_type == "config":
if msg.get("stt"):
stt_config = STTConfig(**msg["stt"])
if msg.get("tts"):
tts_config = TTSConfig(**msg["tts"])
await websocket.send_text(json.dumps({"type": "status", "status": "listening"}))
elif msg_type == "audio":
import base64
chunk = base64.b64decode(msg.get("data", ""))
audio_buffer.extend(chunk)
elif msg_type == "end_utterance":
if not audio_buffer:
continue
await websocket.send_text(json.dumps({"type": "status", "status": "processing"}))
audio_bytes = bytes(audio_buffer)
audio_buffer.clear()
audio_format = msg.get("format", "webm")
content_type = f"audio/{audio_format}"
# STT
transcript = await stt_service.transcribe(
audio_bytes, stt_config, content_type
)
if not transcript:
await websocket.send_text(json.dumps({"type": "status", "status": "listening"}))
continue
await websocket.send_text(json.dumps({
"type": "transcript", "text": transcript,
}))
# Route to agent
agent_response = await _get_agent_response(session_id, transcript)
if agent_response:
await websocket.send_text(json.dumps({
"type": "agent_response", "text": agent_response,
}))
# TTS
await websocket.send_text(json.dumps({"type": "status", "status": "speaking"}))
audio = await tts_service.synthesize(agent_response, tts_config)
if audio:
import base64 as b64
await websocket.send_text(json.dumps({
"type": "audio",
"data": b64.b64encode(audio).decode(),
"format": "mp3",
}))
await websocket.send_text(json.dumps({"type": "status", "status": "listening"}))
elif msg_type == "stop":
break
except WebSocketDisconnect:
logger.info("Talk mode disconnected for session %s", session_id)
except Exception:
logger.exception("Talk mode error for session %s", session_id)
async def _get_agent_response(session_id: str, text: str) -> Optional[str]:
"""Send text to agent and wait for response."""
session = agent_manager.get_session(session_id)
if not session:
return None
response_future: asyncio.Future[str] = asyncio.get_event_loop().create_future()
original_send = ws_manager.send_to_session
async def _hooked_send(sid: str, event: str, data: dict):
await original_send(sid, event, data)
if sid == session_id and not response_future.done():
if event == "agent:message":
msg = data.get("message", {})
if msg.get("role") == "assistant":
content = msg.get("content", "")
if isinstance(content, list):
parts = [
b.get("text", "")
for b in content
if isinstance(b, dict) and b.get("type") == "text"
]
content = "\n".join(parts)
if content:
response_future.set_result(content)
elif event == "agent:status":
if data.get("status") in ("completed", "error", "stopped"):
response_future.set_result("")
ws_manager.send_to_session = _hooked_send
try:
await agent_manager.send_message(session_id, text)
return await asyncio.wait_for(response_future, timeout=120) or None
except asyncio.TimeoutError:
return None
finally:
ws_manager.send_to_session = original_send
+127
View File
@@ -0,0 +1,127 @@
"""Provider-abstracted Text-to-Speech service.
Supports: Twilio built-in Say, ElevenLabs, OpenAI TTS, Microsoft Edge TTS.
"""
import logging
from typing import Optional
import httpx
from backend.apps.channels.models import TTSConfig
from backend.apps.settings.settings import load_settings
logger = logging.getLogger(__name__)
async def synthesize(text: str, config: TTSConfig) -> Optional[bytes]:
"""Convert text to audio bytes. Returns None for twilio_say (handled in TwiML)."""
if should_skip(text, config):
return None
if len(text) > config.max_tts_chars and config.summarize_long_replies:
text = text[: config.max_tts_chars]
provider = config.provider
if provider == "twilio_say":
# Twilio renders speech inline via <Say> — no audio bytes needed
return None
elif provider == "elevenlabs":
return await _elevenlabs_synthesize(text, config)
elif provider == "openai_tts":
return await _openai_synthesize(text, config)
elif provider == "edge_tts":
return await _edge_synthesize(text, config)
logger.warning("Unknown TTS provider: %s", provider)
return None
def should_skip(text: str, config: TTSConfig) -> bool:
if config.skip_short_text and len(text.strip()) < 20:
return True
return False
async def _elevenlabs_synthesize(text: str, config: TTSConfig) -> Optional[bytes]:
settings = load_settings()
api_key = settings.elevenlabs_api_key if hasattr(settings, "elevenlabs_api_key") else None
if not api_key:
logger.warning("ElevenLabs API key not configured, falling back to edge_tts")
return await _edge_synthesize(text, config)
voice_id = config.elevenlabs_voice_id or "21m00Tcm4TlvDq8ikWAM" # Rachel default
model_id = config.elevenlabs_model_id
try:
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.post(
f"https://api.elevenlabs.io/v1/text-to-speech/{voice_id}",
headers={
"xi-api-key": api_key,
"Content-Type": "application/json",
"Accept": "audio/mpeg",
},
json={
"text": text,
"model_id": model_id,
"voice_settings": {
"stability": 0.5,
"similarity_boost": 0.75,
},
},
)
resp.raise_for_status()
return resp.content
except Exception:
logger.exception("ElevenLabs TTS failed, falling back to edge_tts")
return await _edge_synthesize(text, config)
async def _openai_synthesize(text: str, config: TTSConfig) -> Optional[bytes]:
settings = load_settings()
api_key = settings.openai_api_key if hasattr(settings, "openai_api_key") else None
if not api_key:
logger.warning("OpenAI API key not configured, falling back to edge_tts")
return await _edge_synthesize(text, config)
try:
async with httpx.AsyncClient(timeout=30) as client:
resp = await client.post(
"https://api.openai.com/v1/audio/speech",
headers={
"Authorization": f"Bearer {api_key}",
"Content-Type": "application/json",
},
json={
"model": "tts-1",
"input": text,
"voice": config.openai_voice,
"response_format": "mp3",
},
)
resp.raise_for_status()
return resp.content
except Exception:
logger.exception("OpenAI TTS failed, falling back to edge_tts")
return await _edge_synthesize(text, config)
async def _edge_synthesize(text: str, config: TTSConfig) -> Optional[bytes]:
"""Free fallback TTS via Microsoft Edge neural voices. No API key needed."""
try:
import edge_tts
import tempfile
import os
communicate = edge_tts.Communicate(text, "en-US-JennyNeural")
with tempfile.NamedTemporaryFile(suffix=".mp3", delete=False) as f:
tmp_path = f.name
await communicate.save(tmp_path)
with open(tmp_path, "rb") as f:
audio = f.read()
os.unlink(tmp_path)
return audio
except Exception:
logger.exception("Edge TTS failed")
return None
+68
View File
@@ -0,0 +1,68 @@
"""Voice Wake Word Detection — scaffolded interface.
Matches OpenClaw's current state: the interface is defined but full
implementation is deferred. Supports future integration with Vosk
(offline) or Porcupine wake word engines.
Usage:
This module defines the configuration and interface. Actual wake word
detection runs on the client device (macOS/iOS/Android) and sends
a "wake" event to the gateway when triggered.
"""
import logging
from typing import Optional
from pydantic import BaseModel, Field
logger = logging.getLogger(__name__)
class WakeWordConfig(BaseModel):
"""Configuration for wake word detection."""
enabled: bool = False
wake_words: list[str] = Field(default_factory=lambda: ["hey swarm", "open swarm"])
sensitivity: float = 0.5 # 0.0 - 1.0
engine: str = "vosk" # "vosk" | "porcupine"
class WakeWordManager:
"""Manages wake word detection state.
In the current scaffolded implementation, this stores configuration
and handles wake events from client devices. The actual audio
processing runs on the client side.
"""
def __init__(self):
self.config = WakeWordConfig()
self._active_devices: dict[str, bool] = {}
def update_config(self, **kwargs):
for k, v in kwargs.items():
if hasattr(self.config, k):
setattr(self.config, k, v)
def register_device(self, device_id: str):
self._active_devices[device_id] = True
logger.info("Wake word device registered: %s", device_id)
def unregister_device(self, device_id: str):
self._active_devices.pop(device_id, None)
def handle_wake_event(self, device_id: str, wake_word: str) -> bool:
"""Called when a client device detects a wake word.
Returns True if the wake event should trigger a talk session.
"""
if not self.config.enabled:
return False
if device_id not in self._active_devices:
return False
if wake_word.lower() not in [w.lower() for w in self.config.wake_words]:
return False
logger.info("Wake word detected: '%s' from device %s", wake_word, device_id)
return True
# Singleton
wake_word_manager = WakeWordManager()
+40
View File
@@ -0,0 +1,40 @@
"""WebSocket event emitters for channel events.
Uses the existing ws_manager.broadcast_global() no new WebSocket
infrastructure needed.
"""
import logging
from backend.apps.agents.ws_manager import ws_manager
logger = logging.getLogger(__name__)
async def emit_channel_message(
channel_id: str, conversation_id: str, message: dict
):
await ws_manager.broadcast_global("channel:message", {
"channel_id": channel_id,
"conversation_id": conversation_id,
"message": message,
})
async def emit_channel_status(
channel_id: str, status: str, detail: str = ""
):
await ws_manager.broadcast_global("channel:status", {
"channel_id": channel_id,
"status": status,
"detail": detail,
})
async def emit_call_event(
channel_id: str, call_sid: str, event: str, data: dict | None = None
):
await ws_manager.broadcast_global("channel:call_event", {
"channel_id": channel_id,
"call_sid": call_sid,
"event": event,
**(data or {}),
})
+5 -6
View File
@@ -123,8 +123,10 @@ async def list_dashboards():
@dashboards.router.post("/create")
async def create_dashboard(body: DashboardCreate):
from backend.apps.analytics.collector import record as _analytics
dashboard = Dashboard(name=body.name)
_save(dashboard)
_analytics("dashboard.created", {}, dashboard_id=dashboard.id)
return dashboard.model_dump(mode="json")
@@ -151,13 +153,10 @@ async def generate_name(dashboard_id: str):
fallback = prompts[0][:40]
try:
import anthropic
from backend.apps.settings.settings import load_settings
from backend.apps.settings.credentials import get_anthropic_client
global_settings = load_settings()
if not global_settings.anthropic_api_key:
raise ValueError("API key not configured")
client = anthropic.AsyncAnthropic(api_key=global_settings.anthropic_api_key)
client = get_anthropic_client(global_settings)
if len(prompts) == 1:
system = (
@@ -173,7 +172,7 @@ async def generate_name(dashboard_id: str):
user_content = "\n".join(f"- {p}" for p in prompts)
resp = await client.messages.create(
model="claude-haiku-4-20250414",
model="claude-haiku-4-5-20251001",
max_tokens=30,
system=system,
messages=[{"role": "user", "content": user_content}],
+2 -2
View File
@@ -34,8 +34,8 @@ class BrowserCardPosition(BaseModel):
activeTabId: str = ""
x: float = 0
y: float = 0
width: float = 640
height: float = 480
width: float = 1280
height: float = 800
class DashboardLayout(BaseModel):
+21 -36
View File
@@ -76,43 +76,28 @@ BUILTIN_MODES: list[Mode] = [
),
Mode(
id="view-builder",
name="View Builder",
description="Create and iterate on reusable View artifacts.",
name="App Builder",
description="Create and iterate on reusable App artifacts.",
system_prompt=(
"You are helping the user build a reusable View — a self-contained "
"web app rendered in an iframe.\n\n"
"Your working directory is a dedicated workspace folder for this view. "
"You can create any file structure you need using the Write tool.\n\n"
"## Required files\n\n"
"1. **index.html** — The entry point. A complete HTML document. "
"React 18 is available via esm.sh CDN imports:\n"
' <script type="importmap">{"imports":{"react":"https://esm.sh/react@18",'
'"react-dom/client":"https://esm.sh/react-dom@18/client"}}</script>\n'
" The structured input data is available at `window.OUTPUT_INPUT` (object) "
"and any server-side result at `window.OUTPUT_BACKEND_RESULT`.\n\n"
"2. **schema.json** — A JSON Schema object defining the structured input "
"the view accepts. Example:\n"
' {"type":"object","properties":{"name":{"type":"string"}},"required":["name"]}\n\n'
"3. **meta.json** — Metadata for this view. Always write this file with "
"a short name and one-sentence description. Example:\n"
' {"name":"Sales Dashboard","description":"Interactive dashboard showing sales metrics"}\n\n'
"## Optional files\n\n"
"- **backend.py** — Python code that receives `input_data` as "
"a global dict and must assign its result to a global `result` dict.\n"
"- **Any additional files** — You can create subdirectories and split code "
"across multiple files. For example:\n"
" - `components/Chart.js` — Reusable components\n"
" - `utils/helpers.js` — Utility functions\n"
" - `styles/main.css` — Stylesheets\n\n"
"Files are served from the workspace, so relative imports work naturally:\n"
' `<script type="module" src="./components/Chart.js"></script>`\n'
' `<link rel="stylesheet" href="./styles/main.css">`\n'
" `import { helper } from './utils/helpers.js'` (in ES modules)\n\n"
"## Guidelines\n\n"
"Write files immediately when you have code ready. The user can see "
"a live preview that auto-refreshes from these files. Always write the "
"complete file content (do not use Edit for partial patches on first creation). "
"For complex views, split code into separate files to keep things organized."
"You are an App Builder — an AI assistant that creates self-contained "
"web apps rendered in an iframe preview.\n\n"
"Your working directory is a dedicated workspace folder pre-seeded with "
"template files. Read the existing files before making changes.\n\n"
"## Critical rules\n\n"
"- The entry point MUST be named `index.html`. Never rename it or create "
"a different HTML file as the main entry point.\n"
"- Write files immediately when you have code ready — the user sees a "
"live preview that auto-refreshes from these files.\n"
"- Always write the complete file content on first creation (do not use "
"Edit for partial patches on new files).\n"
"- For complex apps, split code into separate files (JS, CSS, etc.) "
"and reference them from index.html with relative paths.\n"
"- Always update meta.json with a short name and one-sentence description.\n"
"- Build beautiful, polished UIs with modern design — dark themes, smooth "
"transitions, proper spacing, and responsive layouts.\n\n"
"Read the SKILL.md reference in your workspace for the full technical "
"specification of the App platform (available globals, file conventions, "
"schema format, backend.py usage, and examples)."
),
tools=None,
default_next_mode=None,
+12 -1
View File
@@ -59,7 +59,8 @@ def load_mode(mode_id: str) -> Mode | None:
@modes.router.get("/list")
async def list_modes():
return {"modes": [m.model_dump() for m in _load_all()]}
builtin_defaults = {m.id: m.model_dump() for m in BUILTIN_MODES}
return {"modes": [m.model_dump() for m in _load_all()], "builtin_defaults": builtin_defaults}
@modes.router.get("/{mode_id}")
@@ -93,6 +94,16 @@ async def update_mode(mode_id: str, body: ModeUpdate):
return {"ok": True, "mode": mode.model_dump()}
@modes.router.post("/{mode_id}/reset")
async def reset_mode(mode_id: str):
"""Reset a built-in mode to its hardcoded defaults."""
builtin = next((m for m in BUILTIN_MODES if m.id == mode_id), None)
if not builtin:
raise HTTPException(status_code=400, detail="Only built-in modes can be reset")
_save(builtin)
return {"ok": True, "mode": builtin.model_dump()}
@modes.router.delete("/{mode_id}")
async def delete_mode(mode_id: str):
mode = _load(mode_id)
+199
View File
@@ -0,0 +1,199 @@
"""Auto-start and manage 9Router subprocess.
9Router is a free AI subscription proxy that lets users connect their
Claude/ChatGPT/Gemini subscriptions to OpenSwarm without API keys.
It runs silently in the background on port 20128 and exposes an
OpenAI-compatible API at localhost:20128/v1.
"""
import asyncio
import logging
import os
import shutil
import subprocess
import httpx
logger = logging.getLogger(__name__)
NINE_ROUTER_PORT = 20128
NINE_ROUTER_URL = f"http://localhost:{NINE_ROUTER_PORT}"
NINE_ROUTER_API = f"{NINE_ROUTER_URL}/api"
NINE_ROUTER_V1 = f"{NINE_ROUTER_URL}/v1"
_process: subprocess.Popen | None = None
def is_running() -> bool:
"""Check if 9Router is running."""
try:
r = httpx.get(f"{NINE_ROUTER_V1}/models", timeout=2.0)
return r.status_code == 200
except Exception:
return False
async def ensure_running():
"""Start 9Router if not already running."""
global _process
if is_running():
logger.info("9Router already running on port %d", NINE_ROUTER_PORT)
return
npx = shutil.which("npx")
if not npx:
logger.warning("npx not found — cannot auto-start 9Router. Install Node.js or run 9Router manually.")
return
logger.info("Starting 9Router on port %d...", NINE_ROUTER_PORT)
try:
env = {**os.environ, "PORT": str(NINE_ROUTER_PORT)}
_process = subprocess.Popen(
[npx, "9router"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
env=env,
)
# Wait up to 15 seconds for it to start
for _ in range(30):
await asyncio.sleep(0.5)
if is_running():
logger.info("9Router started successfully")
return
logger.warning("9Router did not start within 15 seconds")
except Exception as e:
logger.warning(f"Failed to start 9Router: {e}")
def stop():
"""Stop the 9Router subprocess."""
global _process
if _process:
try:
_process.terminate()
_process.wait(timeout=5)
except Exception:
try:
_process.kill()
except Exception:
pass
_process = None
logger.info("9Router stopped")
# ---------------------------------------------------------------------------
# API proxy helpers — call 9Router's API from OpenSwarm
# ---------------------------------------------------------------------------
async def get_providers() -> list[dict]:
"""Get all providers and their connection status from 9Router."""
try:
async with httpx.AsyncClient(timeout=5.0) as client:
r = await client.get(f"{NINE_ROUTER_API}/providers")
if r.status_code == 200:
return r.json()
except Exception as e:
logger.debug(f"9Router providers fetch failed: {e}")
return []
async def start_oauth(provider: str) -> dict:
"""Start OAuth flow for a provider.
For device_code providers (github, qwen, kiro): returns {user_code, verification_uri, device_code}
For authorization_code providers (claude, codex, gemini-cli): returns {authUrl, codeVerifier, state}
"""
async with httpx.AsyncClient(timeout=15.0) as client:
# Try device-code flow first
try:
r = await client.get(f"{NINE_ROUTER_API}/oauth/{provider}/device-code")
if r.status_code == 200:
data = r.json()
return {
"flow": "device_code",
"user_code": data.get("user_code", ""),
"verification_uri": data.get("verification_uri", data.get("verification_uri_complete", "")),
"device_code": data.get("device_code", ""),
"code_verifier": data.get("codeVerifier", ""),
"extra_data": {k: v for k, v in data.items() if k.startswith("_")},
}
except Exception:
pass
# Authorization code flow — redirect to 9Router's own callback page
# (Anthropic only accepts redirect URIs registered with 9Router's client ID)
callback_url = f"http://localhost:{NINE_ROUTER_PORT}/callback"
r = await client.get(
f"{NINE_ROUTER_API}/oauth/{provider}/authorize",
params={"redirect_uri": callback_url},
)
r.raise_for_status()
data = r.json()
return {
"flow": "authorization_code",
"auth_url": data.get("authUrl", ""),
"code_verifier": data.get("codeVerifier", ""),
"state": data.get("state", ""),
"redirect_uri": callback_url,
}
async def poll_oauth(provider: str, device_code: str, code_verifier: str | None = None, extra_data: dict | None = None) -> dict:
"""Poll for OAuth completion.
Returns: {success: true, connection: {...}} or {success: false, pending: true}
"""
body: dict = {"deviceCode": device_code}
if code_verifier:
body["codeVerifier"] = code_verifier
if extra_data:
body["extraData"] = extra_data
async with httpx.AsyncClient(timeout=15.0) as client:
r = await client.post(
f"{NINE_ROUTER_API}/oauth/{provider}/poll",
json=body,
)
r.raise_for_status()
return r.json()
async def exchange_oauth(provider: str, code: str, redirect_uri: str, code_verifier: str, state: str = "") -> dict:
"""Exchange OAuth code for tokens via 9Router."""
async with httpx.AsyncClient(timeout=15.0) as client:
r = await client.post(
f"{NINE_ROUTER_API}/oauth/{provider}/exchange",
json={
"code": code,
"redirectUri": redirect_uri,
"codeVerifier": code_verifier,
"state": state,
},
)
r.raise_for_status()
return r.json()
async def get_models() -> list[dict]:
"""Get all available models from 9Router."""
try:
async with httpx.AsyncClient(timeout=5.0) as client:
r = await client.get(f"{NINE_ROUTER_V1}/models")
if r.status_code == 200:
data = r.json()
models = data.get("data", [])
return [
{
"value": m.get("id", ""),
"label": m.get("id", "").split("/")[-1] if "/" in m.get("id", "") else m.get("id", ""),
"context_window": 200_000,
"provider": m.get("owned_by", "subscription"),
}
for m in models
]
except Exception as e:
logger.debug(f"9Router models fetch failed: {e}")
return []
+13 -6
View File
@@ -15,6 +15,7 @@ from backend.apps.outputs.models import (
WorkspaceSeedRequest,
)
from backend.apps.outputs.executor import execute_backend_code
from backend.apps.outputs.view_builder_templates import VIEW_BUILDER_SKILL, VIEW_TEMPLATE_FILES
from backend.apps.settings.settings import load_settings
logger = logging.getLogger(__name__)
@@ -22,7 +23,7 @@ logger = logging.getLogger(__name__)
MODEL_MAP = {
"sonnet": "claude-sonnet-4-20250514",
"opus": "claude-opus-4-20250514",
"haiku": "claude-haiku-4-20250414",
"haiku": "claude-haiku-4-5-20251001",
}
@@ -31,13 +32,11 @@ def _resolve_model(short_name: str) -> str:
def _get_anthropic_client():
"""Create an AsyncAnthropic client using the API key from app settings."""
import anthropic
"""Create an AsyncAnthropic client using credentials from app settings."""
from backend.apps.settings.credentials import get_anthropic_client
settings = load_settings()
if not settings.anthropic_api_key:
raise ValueError("Anthropic API key not configured. Set it in Settings.")
return anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key)
return get_anthropic_client(settings)
def _validate_against_schema(data: dict, schema: dict) -> str | None:
@@ -235,6 +234,14 @@ async def seed_workspace(body: WorkspaceSeedRequest):
os.makedirs(os.path.dirname(full_path), exist_ok=True)
with open(full_path, "w") as f:
f.write(content)
else:
for rel_path, content in VIEW_TEMPLATE_FILES.items():
full_path = os.path.join(folder, rel_path)
with open(full_path, "w") as f:
f.write(content)
with open(os.path.join(folder, "SKILL.md"), "w") as f:
f.write(VIEW_BUILDER_SKILL)
if body.meta:
with open(os.path.join(folder, "meta.json"), "w") as f:
+223
View File
@@ -0,0 +1,223 @@
# App Builder — Platform Reference
You are building an **App**: a self-contained web app served in an iframe.
The workspace you're working in is the source of truth — every file you write
here is served directly to the live preview.
---
## File conventions
| File | Required | Purpose |
|------|----------|---------|
| `index.html` | **Yes** | Entry point. Must be a complete HTML document. This is the ONLY file the preview iframe loads — never rename it. |
| `meta.json` | **Yes** | `{"name":"…","description":"…"}` — displayed in the UI header. Always write this. |
| `schema.json` | Recommended | JSON Schema defining the input form (the "Test Input" tab). |
| `backend.py` | Optional | Server-side Python executed before rendering. |
| Everything else | Optional | JS, CSS, images, subdirectories — referenced from `index.html` via relative paths. |
### ⚠️ Do NOT
- Name the main HTML file anything other than `index.html` — the platform
will not find it and the preview will be blank.
- Use `document.write()` — it breaks the injected data globals.
- Assume any external server or API is available unless the user provides one.
---
## Injected globals
Before `index.html` loads, the platform injects two globals:
```javascript
window.OUTPUT_INPUT // Object — structured input from the schema form
window.OUTPUT_BACKEND_RESULT // Object | null — result from backend.py execution
```
These are available immediately in any `<script>` tag. You can also listen for
live updates when the user changes input:
```javascript
window.addEventListener('output-data-ready', () => {
const input = window.OUTPUT_INPUT;
const result = window.OUTPUT_BACKEND_RESULT;
// re-render with new data
});
```
---
## schema.json format
Standard JSON Schema. The platform renders a form from this automatically.
```json
{
"type": "object",
"properties": {
"title": { "type": "string", "default": "My Dashboard" },
"count": { "type": "number", "default": 10 },
"enabled": { "type": "boolean", "default": true },
"items": {
"type": "array",
"items": { "type": "string" },
"default": ["alpha", "beta"]
}
},
"required": ["title"]
}
```
Supported types: `string`, `number`, `integer`, `boolean`, `array`, `object`.
Use `"default"` values so the preview works without manual input.
---
## backend.py
Optional server-side Python that runs before the frontend renders.
It receives a global `input_data` dict (the schema form values) and must
assign its result to a global `result` dict.
```python
# input_data is pre-populated from the schema form
import json
result = {
"processed_items": [item.upper() for item in input_data.get("items", [])],
"timestamp": "2024-01-01T00:00:00Z",
}
```
The `result` dict becomes `window.OUTPUT_BACKEND_RESULT` in the frontend.
---
## Multi-file projects
Split code across files for organization. All files are served from the
workspace root, so relative imports work naturally:
```
workspace/
├── index.html
├── meta.json
├── schema.json
├── styles/
│ └── main.css
├── components/
│ └── Chart.js
└── utils/
└── helpers.js
```
Reference from `index.html`:
```html
<link rel="stylesheet" href="./styles/main.css">
<script type="module" src="./components/Chart.js"></script>
```
ES module imports between JS files:
```javascript
// components/Chart.js
import { formatNumber } from '../utils/helpers.js';
```
---
## Using React
React 18 is available via esm.sh CDN — no build step needed:
```html
<script type="importmap">
{
"imports": {
"react": "https://esm.sh/react@18",
"react-dom/client": "https://esm.sh/react-dom@18/client"
}
}
</script>
<div id="root"></div>
<script type="module">
import React from 'react';
import { createRoot } from 'react-dom/client';
function App() {
const input = window.OUTPUT_INPUT || {};
return React.createElement('div', null,
React.createElement('h1', null, input.title || 'Hello')
);
}
createRoot(document.getElementById('root')).render(
React.createElement(App)
);
</script>
```
Other CDN libraries work too — use `https://esm.sh/` or `https://cdn.jsdelivr.net/npm/` for any npm package.
---
## Design guidelines
- **Dark theme by default** — use dark backgrounds (#0f1117, #1a1d27) with
light text (#e2e8f0) unless the user requests otherwise.
- **Modern aesthetics** — rounded corners (8-12px), subtle borders, box shadows,
smooth transitions (0.15-0.3s ease).
- **Responsive** — use flexbox/grid, test at different sizes.
- **Typography** — system font stack for UI, monospace for code/data.
- **Color accents** — use a single accent color with variations for hover/active states.
- **Spacing** — consistent padding (12-20px), adequate whitespace between sections.
- **Interactivity** — hover effects, focus states, loading indicators where appropriate.
---
## Complete minimal example
```html
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>My App</title>
<style>
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif;
background: #0f1117;
color: #e2e8f0;
min-height: 100vh;
display: flex;
align-items: center;
justify-content: center;
}
.card {
background: #1a1d27;
border: 1px solid #2e3248;
border-radius: 12px;
padding: 32px;
max-width: 480px;
width: 100%;
}
h1 { font-size: 1.5rem; margin-bottom: 8px; }
p { color: #8892a4; line-height: 1.6; }
</style>
</head>
<body>
<div class="card">
<h1 id="title">Loading…</h1>
<p id="desc"></p>
</div>
<script>
const input = window.OUTPUT_INPUT || {};
document.getElementById('title').textContent = input.title || 'Untitled';
document.getElementById('desc').textContent = input.description || 'No description provided.';
</script>
</body>
</html>
```
@@ -0,0 +1,74 @@
"""Default template files seeded into new App Builder workspaces."""
import os
_SKILL_PATH = os.path.join(os.path.dirname(__file__), "view_builder_skill.md")
with open(_SKILL_PATH) as _f:
VIEW_BUILDER_SKILL = _f.read()
VIEW_TEMPLATE_INDEX = """\
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>App</title>
<style>
* { box-sizing: border-box; margin: 0; padding: 0; }
body {
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', Roboto, sans-serif;
background: #0f1117;
color: #e2e8f0;
min-height: 100vh;
display: flex;
align-items: center;
justify-content: center;
padding: 24px;
}
.container {
background: #1a1d27;
border: 1px solid #2e3248;
border-radius: 12px;
padding: 32px;
max-width: 600px;
width: 100%;
text-align: center;
}
h1 { font-size: 1.5rem; font-weight: 600; margin-bottom: 8px; }
p { color: #8892a4; font-size: 0.95rem; line-height: 1.6; }
</style>
</head>
<body>
<div class="container">
<h1 id="title">Ready</h1>
<p id="desc">Describe what you want to build and the agent will update this app.</p>
</div>
<script>
const input = window.OUTPUT_INPUT || {};
const result = window.OUTPUT_BACKEND_RESULT || null;
</script>
</body>
</html>
"""
VIEW_TEMPLATE_SCHEMA = """\
{
"type": "object",
"properties": {},
"required": []
}
"""
VIEW_TEMPLATE_META = """\
{
"name": "",
"description": ""
}
"""
VIEW_TEMPLATE_FILES = {
"index.html": VIEW_TEMPLATE_INDEX,
"schema.json": VIEW_TEMPLATE_SCHEMA,
"meta.json": VIEW_TEMPLATE_META,
}
+141
View File
@@ -0,0 +1,141 @@
"""Centralized credential resolution for LLM API calls.
Supports multiple providers: Anthropic (native), OpenAI, Gemini,
OpenRouter, and user-configured custom providers.
"""
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
import anthropic
from backend.apps.settings.models import AppSettings
OPENSWARM_DEFAULT_PROXY_URL = "https://api.openswarm.ai"
def _check_9router() -> bool:
"""Check if 9Router is running locally."""
try:
import httpx
r = httpx.get("http://localhost:20128/v1/models", timeout=2.0)
return r.status_code == 200
except Exception:
return False
def validate_credentials(settings: AppSettings, provider: str = "anthropic") -> None:
"""Raise ValueError if credentials are missing for the given provider.
Allows through if 9Router is running as a fallback.
Handles both display names ('Anthropic') and lowercase ('anthropic').
"""
p = provider.lower().strip()
# 9Router or GitHub Copilot providers don't need traditional credentials
if p in ("9router", "github copilot", "copilot"):
return
# If 9Router is running, all providers are accessible
if _check_9router():
return
if p == "anthropic":
if getattr(settings, "connection_mode", "own_key") == "managed":
if not getattr(settings, "openswarm_auth_token", None):
raise ValueError("Open Swarm account not connected. Sign in via Settings → API.")
return
if settings.anthropic_api_key:
return
raise ValueError("Anthropic API key not configured. Set it in Settings, or connect a subscription.")
elif p == "openai":
if settings.openai_api_key:
return
raise ValueError("OpenAI API key not configured. Set it in Settings, or connect a subscription.")
elif p in ("gemini", "google"):
if getattr(settings, "google_api_key", None):
return
raise ValueError("Google API key not configured. Set it in Settings, or connect a subscription.")
elif p == "openrouter":
if getattr(settings, "openrouter_api_key", None):
return
raise ValueError("OpenRouter API key not configured. Set it in Settings.")
elif p in ("xai", "meta", "deepseek", "mistral", "qwen", "cohere"):
# These route through OpenRouter — need either OpenRouter key or 9Router
if getattr(settings, "openrouter_api_key", None):
return
raise ValueError(f"{provider} requires an OpenRouter API key, or connect a subscription via 9Router.")
else:
# Custom provider — check if it exists in custom_providers
for cp in getattr(settings, "custom_providers", []):
if cp.name.lower() == p:
return
# Unknown provider — allow through (create_provider will handle the error)
return
def get_provider_credentials(settings: AppSettings, provider: str) -> dict[str, str]:
"""Return credential dict for a specific provider."""
validate_credentials(settings, provider)
if provider == "anthropic":
if getattr(settings, "connection_mode", "own_key") == "managed":
return {
"auth_token": getattr(settings, "openswarm_auth_token", "") or "",
"base_url": getattr(settings, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL,
}
return {"api_key": settings.anthropic_api_key or ""}
if provider == "openai":
return {"api_key": settings.openai_api_key or ""}
if provider == "gemini":
return {"api_key": getattr(settings, "google_api_key", "") or ""}
if provider == "openrouter":
return {"api_key": getattr(settings, "openrouter_api_key", "") or ""}
# Custom provider
for cp in getattr(settings, "custom_providers", []):
if cp.name == provider:
return {"api_key": cp.api_key, "base_url": cp.base_url}
raise ValueError(f"No credentials for provider: {provider}")
# ---------------------------------------------------------------------------
# Legacy helpers (kept for backward compat during migration)
# ---------------------------------------------------------------------------
def get_agent_sdk_env(settings: AppSettings) -> dict[str, str]:
"""Return the env dict for ClaudeAgentOptions based on connection mode.
DEPRECATED: Use create_provider() from providers.registry instead.
"""
validate_credentials(settings, "anthropic")
if getattr(settings, "connection_mode", "own_key") == "managed":
proxy_url = getattr(settings, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL
return {
"ANTHROPIC_AUTH_TOKEN": getattr(settings, "openswarm_auth_token", ""),
"ANTHROPIC_BASE_URL": proxy_url,
}
return {"ANTHROPIC_API_KEY": settings.anthropic_api_key}
def get_anthropic_client(settings: AppSettings) -> anthropic.AsyncAnthropic:
"""Return a configured AsyncAnthropic client based on connection mode."""
import anthropic
validate_credentials(settings, "anthropic")
if getattr(settings, "connection_mode", "own_key") == "managed":
proxy_url = getattr(settings, "openswarm_proxy_url", None) or OPENSWARM_DEFAULT_PROXY_URL
return anthropic.AsyncAnthropic(
auth_token=getattr(settings, "openswarm_auth_token", None),
base_url=proxy_url,
)
return anthropic.AsyncAnthropic(api_key=settings.anthropic_api_key)
+51 -7
View File
@@ -1,12 +1,22 @@
from pydantic import BaseModel
from typing import Optional
from pydantic import BaseModel, Field
from typing import Optional, Any
DEFAULT_SYSTEM_PROMPT = (
'"Ask the user as many follow ups as needed in order to eliminate any possible ambiguity. '
"When asking the user questions, use the AskUserQuestion tool.\n\n"
"You are an unstopable Agent that does whatever is needed to achieve the task. "
"You are particularly gifted at coding, so when needed, transpose ordinary tasks into coding tasks.\n\n"
'If a Browser is selected, prioritize this over other tools when it makes sense (so the user also has observability)."'
"You are a personal AI assistant running inside OpenSwarm.\n\n"
"## Tool Priority\n"
"When a dedicated MCP tool exists for a task, use it directly — do not use the browser for things MCP tools can handle.\n"
"Priority order:\n"
"1. MCP tools first (Reddit, Google Workspace, Twitter, etc.) — fastest and most reliable\n"
"2. WebSearch / WebFetch — for general web lookups without a dedicated MCP\n"
"3. BrowserAgent — only when you need to visually interact with a website, fill forms, or do something no other tool can handle\n\n"
"## Tool Call Style\n"
"Default: do not narrate routine tool calls — just call the tool.\n"
"Narrate only when it helps: multi-step work, complex problems, or when the user explicitly asks.\n"
"Keep narration brief. Use plain language.\n\n"
"## Interaction Style\n"
"Be direct and action-oriented. Do not ask clarifying questions unless genuinely ambiguous — "
"make reasonable assumptions and act. If you need to ask, use the AskUserQuestion tool.\n"
"Do not over-explain what you are about to do. Just do it and show the results.\n"
)
@@ -21,3 +31,37 @@ class AppSettings(BaseModel):
new_agent_shortcut: str = "Meta+l"
anthropic_api_key: Optional[str] = None
browser_homepage: str = "https://www.google.com"
# Telephony / Channel credentials
twilio_account_sid: Optional[str] = None
twilio_auth_token: Optional[str] = None
telnyx_api_key: Optional[str] = None
elevenlabs_api_key: Optional[str] = None
deepgram_api_key: Optional[str] = None
openai_api_key: Optional[str] = None
google_api_key: Optional[str] = None
openrouter_api_key: Optional[str] = None
custom_providers: list["CustomProvider"] = Field(default_factory=list)
webhook_base_url: Optional[str] = None
# Dashboard / UI preferences
auto_select_mode_on_new_agent: bool = False
expand_new_chats_in_dashboard: bool = False
auto_reveal_sub_agents: bool = True
dev_mode: bool = False
# Subscription tokens (from CLI tools — alternative to API keys)
claude_subscription_token: Optional[str] = None
openai_subscription_token: Optional[str] = None
gemini_subscription_token: Optional[str] = None
# GitHub Copilot
copilot_github_token: Optional[str] = None
copilot_token: Optional[str] = None
copilot_token_expires: Optional[float] = None
# Analytics: opted in by default, user can toggle off
analytics_opt_in: bool = True
installation_id: Optional[str] = None
class CustomProvider(BaseModel):
name: str
base_url: str
api_key: str = ""
models: list[dict[str, Any]] = Field(default_factory=list)
+7
View File
@@ -38,6 +38,13 @@ def load_settings() -> AppSettings:
return AppSettings()
def _save_settings(settings: AppSettings):
"""Persist settings to JSON file."""
os.makedirs(DATA_DIR, exist_ok=True)
with open(SETTINGS_FILE, "w") as f:
json.dump(settings.model_dump(), f, indent=2)
@settings.router.get("")
async def get_settings():
return load_settings().model_dump()
+18
View File
@@ -5,6 +5,7 @@ from uuid import uuid4
class BuiltinTool(BaseModel):
name: str
display_name: Optional[str] = None
description: str
category: str = "filesystem"
deferred: bool = False
@@ -33,6 +34,23 @@ BUILTIN_TOOLS: list[BuiltinTool] = [
BuiltinTool(name="CronList", description="List all scheduled tasks", category="scheduling", deferred=True),
BuiltinTool(name="CronDelete", description="Delete a scheduled task", category="scheduling", deferred=True),
BuiltinTool(name="RenderOutput", description="Render a reusable View artifact with structured input data", category="views", deferred=True),
# Agent tools
BuiltinTool(name="Agent", display_name="CreateAgent", description="Spawn a sub-agent to handle a complex subtask", category="agents"),
BuiltinTool(name="InvokeAgent", description="Invoke a copy of an existing agent with a new message, preserving full conversation context", category="agents"),
# Browser delegation tools (Layer 1 — what the main agent calls)
BuiltinTool(name="CreateBrowserAgent", description="Create a new browser and run a task on it", category="browser_delegation"),
BuiltinTool(name="BrowserAgent", description="Delegate a browser task to an existing browser agent", category="browser_delegation"),
BuiltinTool(name="BrowserAgents", description="Run multiple browser tasks in parallel on existing browsers", category="browser_delegation"),
# Browser action tools (Layer 2 — what the sub-agent executes)
BuiltinTool(name="BrowserScreenshot", description="Capture a screenshot of the browser page", category="browser_action"),
BuiltinTool(name="BrowserNavigate", description="Navigate the browser to a URL", category="browser_action"),
BuiltinTool(name="BrowserClick", description="Click an element by CSS selector", category="browser_action"),
BuiltinTool(name="BrowserType", description="Type text into an input element", category="browser_action"),
BuiltinTool(name="BrowserEvaluate", description="Execute JavaScript in the browser", category="browser_action"),
BuiltinTool(name="BrowserGetText", description="Get visible text content of the page", category="browser_action"),
BuiltinTool(name="BrowserGetElements", description="List interactive elements with CSS selectors", category="browser_action"),
BuiltinTool(name="BrowserScroll", description="Scroll the page up or down", category="browser_action"),
BuiltinTool(name="BrowserWait", description="Wait for page loads or animations", category="browser_action"),
]
+37 -7
View File
@@ -18,9 +18,11 @@ from backend.apps.tools_lib.models import ToolDefinition, ToolCreate, ToolUpdate
logger = logging.getLogger(__name__)
from backend.config.paths import BACKEND_DIR, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH
from backend.config.paths import BACKEND_DIR, DATA_ROOT, TOOLS_DIR as DATA_DIR, BUILTIN_PERMISSIONS_PATH as BUILTIN_PERMS_PATH
load_dotenv(os.path.join(BACKEND_DIR, ".env"))
if os.environ.get("OPENSWARM_PACKAGED") == "1":
load_dotenv(os.path.join(os.path.dirname(DATA_ROOT), ".env"), override=True)
@asynccontextmanager
@@ -380,6 +382,7 @@ def derive_mcp_config(tool: ToolDefinition) -> Optional[dict]:
config["command"] = resolved
env = config.setdefault("env", {})
env.setdefault("PATH", _augmented_path())
env.setdefault("PYTHONPATH", "")
return config
@@ -510,7 +513,7 @@ async def _discover_mcp_tools_http(url: str, headers: dict | None = None) -> lis
raise HTTPException(status_code=502, detail="Empty response from MCP server")
tools_list = data.get("result", {}).get("tools", [])
return [{"name": t.get("name", ""), "description": t.get("description", "")} for t in tools_list]
return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list]
async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list[dict]:
@@ -533,7 +536,7 @@ async def _discover_mcp_tools_sse(url: str, headers: dict | None = None) -> list
) as session:
await session.initialize()
result = await session.list_tools()
return [{"name": t.name, "description": t.description or ""} for t in result.tools]
return [{"name": t.name, "description": t.description or "", "inputSchema": t.inputSchema if t.inputSchema else None} for t in result.tools]
except BaseExceptionGroup as eg:
first = eg.exceptions[0] if eg.exceptions else eg
raise HTTPException(status_code=502, detail=f"SSE discovery failed: {first}") from first
@@ -546,6 +549,7 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None,
raise HTTPException(status_code=400, detail=f"Command '{command}' not found on PATH or common install locations")
proc_env = {**os.environ, **(env or {}), "PATH": _augmented_path()}
proc_env.pop("PYTHONPATH", None)
proc = await asyncio.create_subprocess_exec(
cmd_path, *(args or []),
@@ -601,7 +605,7 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None,
data = await _recv()
tools_list = data.get("result", {}).get("tools", [])
return [{"name": t.get("name", ""), "description": t.get("description", "")} for t in tools_list]
return [{"name": t.get("name", ""), "description": t.get("description", ""), "inputSchema": t.get("inputSchema")} for t in tools_list]
except HTTPException:
raise
@@ -616,12 +620,34 @@ async def _discover_mcp_tools_stdio(command: str, args: list[str] | None = None,
proc.terminate()
await asyncio.wait_for(proc.wait(), timeout=5.0)
except Exception:
proc.kill()
try:
proc.kill()
except Exception:
pass
@tools_lib.router.post("/{tool_id}/discover")
async def discover_tools(tool_id: str):
tool = _load(tool_id)
if tool.auth_type == "oauth2" and tool.auth_status == "connected":
refreshed = await refresh_google_token(tool)
if not refreshed and tool.oauth_tokens.get("access_token"):
expiry = tool.oauth_tokens.get("token_expiry", 0)
if time.time() >= expiry - 60:
client_id = os.environ.get("GOOGLE_OAUTH_CLIENT_ID", "")
if not client_id:
raise HTTPException(
status_code=400,
detail="OAuth token expired and GOOGLE_OAUTH_CLIENT_ID is not set. "
"In the packaged app, create ~/.openswarm.env or "
"~/Library/Application Support/OpenSwarm/.env with your Google OAuth credentials.",
)
raise HTTPException(
status_code=502,
detail="OAuth token expired and refresh failed. Try reconnecting Google.",
)
config = derive_mcp_config(tool)
if not config:
raise HTTPException(status_code=400, detail="Cannot derive MCP config for tool")
@@ -655,8 +681,11 @@ async def discover_tools(tool_id: str):
except HTTPException:
raise
except Exception as e:
logger.warning(f"MCP tool discovery failed for {tool.name}: {e}")
raise HTTPException(status_code=502, detail=f"Discovery failed: {e}")
msg = str(e).strip()
if not msg:
msg = type(e).__name__
logger.warning(f"MCP tool discovery failed for {tool.name}: {msg}", exc_info=True)
raise HTTPException(status_code=502, detail=f"Discovery failed: {msg}")
services: dict[str, dict[str, list[str]]] = {}
service_groups: dict[str, list[str]] = {}
@@ -681,6 +710,7 @@ async def discover_tools(tool_id: str):
permissions["_services"] = services
permissions["_service_groups"] = service_groups
permissions["_tool_descriptions"] = {t["name"]: t["description"] for t in raw_tools}
permissions["_tool_schemas"] = {t["name"]: t.get("inputSchema") for t in raw_tools if t.get("inputSchema")}
tool.tool_permissions = permissions
_save(tool)
+3
View File
@@ -35,5 +35,8 @@ OUTPUTS_WORKSPACE_DIR = os.path.join(DATA_ROOT, "outputs_workspace")
SKILLS_WORKSPACE_DIR = os.path.join(DATA_ROOT, "skills_workspace")
DASHBOARD_LAYOUT_DIR = os.path.join(DATA_ROOT, "dashboard_layout")
BUILTIN_PERMISSIONS_PATH = os.path.join(DATA_ROOT, "builtin_permissions.json")
CHANNELS_DIR = os.path.join(DATA_ROOT, "channels")
CHANNELS_SESSIONS_DIR = os.path.join(DATA_ROOT, "channels", "sessions")
ANALYTICS_DIR = os.path.join(DATA_ROOT, "analytics")
BACKEND_DIR = _BACKEND_DIR
+116 -5
View File
@@ -1,8 +1,14 @@
import logging
import os
from uuid import uuid4
from fastapi.responses import JSONResponse
logger = logging.getLogger(__name__)
from fastapi.responses import JSONResponse, HTMLResponse
from fastapi import Request
# In-memory store for pending OAuth flows (state → {provider, code_verifier, redirect_uri})
_pending_oauth: dict[str, dict] = {}
from backend.config.Apps import MainApp
from backend.apps.health.health import health
from backend.apps.agents.agents import agents
@@ -16,11 +22,14 @@ from backend.apps.mcp_registry.mcp_registry import mcp_registry
from backend.apps.skill_registry.skill_registry import skill_registry
from backend.apps.outputs.outputs import outputs
from backend.apps.dashboards.dashboards import dashboards
from backend.apps.channels.channels import channels
from backend.apps.analytics.analytics import analytics
from backend.apps.auth.auth import auth
from fastapi.middleware.cors import CORSMiddleware
from fastapi import WebSocket, WebSocketDisconnect
import json
main_app = MainApp([health, agents, templates, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, dashboards])
main_app = MainApp([health, agents, templates, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, dashboards, channels, analytics, auth])
app = main_app.app
app.add_middleware(
@@ -48,6 +57,7 @@ async def websocket_session(websocket: WebSocket, session_id: str):
payload.get("prompt", ""),
mode=payload.get("mode"),
model=payload.get("model"),
provider=payload.get("provider"),
images=payload.get("images"),
)
elif event == "agent:approval_response":
@@ -96,6 +106,12 @@ async def websocket_dashboard(websocket: WebSocket):
ws_manager.disconnect_global(websocket)
@app.websocket("/ws/talk/{session_id}")
async def websocket_talk_mode(websocket: WebSocket, session_id: str):
from backend.apps.channels.talk_mode import handle_talk_session
await handle_talk_session(websocket, session_id)
@app.post("/api/browser/command")
async def browser_command(request: Request):
"""HTTP endpoint called by the browser MCP server subprocess.
@@ -114,6 +130,53 @@ async def browser_command(request: Request):
return JSONResponse(result)
@app.get("/api/subscriptions/pending/{state}")
async def subscriptions_pending(state: str):
"""Return pending OAuth data for a state param. Called by 9Router's callback page."""
pending = _pending_oauth.get(state)
if not pending:
return JSONResponse({"error": "not found"}, status_code=404,
headers={"Access-Control-Allow-Origin": "*"})
return JSONResponse({
"provider": pending["provider"],
"code_verifier": pending["code_verifier"],
"redirect_uri": pending["redirect_uri"],
}, headers={"Access-Control-Allow-Origin": "*"})
@app.get("/api/subscriptions/callback")
async def subscriptions_callback(request: Request):
"""Catch OAuth redirect from provider, exchange code via 9Router, close window."""
code = request.query_params.get("code", "")
state = request.query_params.get("state", "")
error = request.query_params.get("error", "")
if error:
desc = request.query_params.get("error_description", error)
return HTMLResponse(f'<html><body style="background:#1a1a1a;color:#fff;display:flex;align-items:center;justify-content:center;height:100vh;font-family:sans-serif"><div style="text-align:center"><h2>Authorization failed</h2><p style="color:#888">{desc}</p></div></body></html>')
pending = _pending_oauth.pop(state, None)
if not pending:
return HTMLResponse('<html><body style="background:#1a1a1a;color:#fff;display:flex;align-items:center;justify-content:center;height:100vh;font-family:sans-serif"><div style="text-align:center"><h2>Session expired</h2><p style="color:#888">Please try connecting again.</p></div></body></html>')
from backend.apps.nine_router import exchange_oauth
try:
await exchange_oauth(pending["provider"], code, pending["redirect_uri"], pending["code_verifier"], state)
except Exception as e:
return HTMLResponse(f'<html><body style="background:#1a1a1a;color:#fff;display:flex;align-items:center;justify-content:center;height:100vh;font-family:sans-serif"><div style="text-align:center"><h2>Connection failed</h2><p style="color:#888">{e}</p></div></body></html>')
return HTMLResponse(
'<html><body style="background:#1a1a1a;color:#fff;display:flex;align-items:center;justify-content:center;height:100vh;font-family:sans-serif">'
'<div style="text-align:center">'
'<div style="width:64px;height:64px;border-radius:50%;background:#22c55e20;display:flex;align-items:center;justify-content:center;margin:0 auto 16px;font-size:32px">✓</div>'
'<h2 style="margin:0 0 8px">Connected!</h2>'
'<p style="color:#888;margin:0">You can close this window</p>'
'</div>'
'<script>setTimeout(()=>window.close(),1500)</script>'
'</body></html>'
)
@app.post("/api/browser-agent/run")
async def browser_agent_run(request: Request):
"""Run one or more browser sub-agents in parallel.
@@ -126,24 +189,72 @@ async def browser_agent_run(request: Request):
model = body.get("model", "sonnet")
dashboard_id = body.get("dashboard_id", "")
pre_selected_browser_ids = body.get("pre_selected_browser_ids", [])
parent_session_id = body.get("parent_session_id", "")
if not tasks:
return JSONResponse({"error": "tasks array is required"}, status_code=400)
settings = load_settings()
if not settings.anthropic_api_key:
return JSONResponse({"error": "Anthropic API key not configured"}, status_code=400)
# Determine API credentials — check API key, then 9Router
api_key = settings.anthropic_api_key
auth_token = None
base_url = None
if not api_key:
# Try 9Router
from backend.apps.nine_router import is_running as _9r_running
if _9r_running():
api_key = "9router"
base_url = "http://localhost:20128/v1"
auth_token = None
else:
return JSONResponse({"error": "No AI provider configured. Set an API key or connect a subscription."}, status_code=400)
results = await run_browser_agents(
tasks=tasks,
model=model,
api_key=settings.anthropic_api_key,
api_key=api_key,
dashboard_id=dashboard_id or None,
pre_selected_browser_ids=pre_selected_browser_ids,
parent_session_id=parent_session_id or None,
auth_token=auth_token,
base_url=base_url,
)
return JSONResponse({"results": results})
@app.post("/api/invoke-agent/run")
async def invoke_agent_run(request: Request):
"""Fork an existing agent session and send it a new message.
Called by the invoke_agent_mcp_server stdio subprocess."""
body = await request.json()
session_id = body.get("session_id", "")
message = body.get("message", "")
parent_session_id = body.get("parent_session_id", "")
dashboard_id = body.get("dashboard_id", "")
if not session_id:
return JSONResponse({"error": "session_id is required"}, status_code=400)
if not message:
return JSONResponse({"error": "message is required"}, status_code=400)
try:
from backend.apps.agents.agent_manager import agent_manager
result = await agent_manager.invoke_agent(
source_session_id=session_id,
message=message,
parent_session_id=parent_session_id or None,
dashboard_id=dashboard_id or None,
)
return JSONResponse(result)
except ValueError as e:
return JSONResponse({"error": str(e)}, status_code=404)
except Exception as e:
logger.exception("invoke_agent_run failed")
return JSONResponse({"error": str(e)}, status_code=500)
if __name__ == "__main__":
import argparse
import uvicorn
+12 -2
View File
@@ -1,5 +1,8 @@
anthropic
claude-agent-sdk
openai
google-genai
mcp
posthog
jsonschema
fastapi[standard]
pydantic==2.10.5
@@ -9,4 +12,11 @@ pytest==8.3.4
pytest-asyncio==0.25.2
typeguard==4.4.2
python-dotenv==1.1.1
Pillow
Pillow
# Channels: SMS, WhatsApp, Voice
twilio>=9.0.0
telnyx>=2.0.0
edge-tts>=6.1.0
httpx>=0.27.0
playwright
cryptography>=42.0.0
+213 -12
View File
@@ -1,5 +1,6 @@
const { app, BrowserWindow, ipcMain, shell } = require('electron');
const { autoUpdater } = require('electron-updater');
const { app, components, BrowserWindow, ipcMain, shell, session } = require('electron');
let autoUpdater;
try { autoUpdater = require('electron-updater').autoUpdater; } catch (_) {}
const path = require('path');
const { spawn, execFileSync } = require('child_process');
const os = require('os');
@@ -7,9 +8,16 @@ const fs = require('fs');
const getPort = require('get-port');
const http = require('http');
app.commandLine.appendSwitch('disable-features', 'HardwareMediaKeyHandling');
app.commandLine.appendSwitch('ignore-gpu-blocklist');
app.commandLine.appendSwitch('enable-gpu-rasterization');
app.commandLine.appendSwitch('enable-zero-copy');
app.commandLine.appendSwitch('autoplay-policy', 'no-user-gesture-required');
let mainWindow = null;
let backendProcess = null;
let backendPort = null;
let cachedUpdateStatus = { status: 'idle', info: null, error: null };
const isPackaged = app.isPackaged;
const isDev = process.env.ELECTRON_DEV === '1';
@@ -24,9 +32,10 @@ const iconPath = path.join(__dirname, 'build', 'icon.png');
function getShellPath() {
if (process.platform !== 'darwin' || isDev) return process.env.PATH || '';
// Strategy 1: ask the user's login shell for its PATH
try {
const shell = process.env.SHELL || '/bin/zsh';
const result = execFileSync(shell, ['-ilc', 'echo $PATH'], {
const userShell = process.env.SHELL || '/bin/zsh';
const result = execFileSync(userShell, ['-ilc', 'echo $PATH'], {
encoding: 'utf8',
timeout: 5000,
env: { ...process.env, HOME: os.homedir() },
@@ -35,19 +44,40 @@ function getShellPath() {
if (resolved) return resolved;
} catch (_) { /* fall through */ }
// Strategy 2: read macOS system PATH config (/etc/paths + /etc/paths.d/*)
const systemPaths = [];
try {
const base = fs.readFileSync('/etc/paths', 'utf8');
for (const line of base.split('\n')) {
const p = line.trim();
if (p) systemPaths.push(p);
}
} catch (_) { /* ignore */ }
try {
const pathsD = '/etc/paths.d';
if (fs.existsSync(pathsD)) {
for (const file of fs.readdirSync(pathsD).sort()) {
const content = fs.readFileSync(path.join(pathsD, file), 'utf8');
for (const line of content.split('\n')) {
const p = line.trim();
if (p) systemPaths.push(p);
}
}
}
} catch (_) { /* ignore */ }
// Strategy 3: well-known user-local bin directories
const home = os.homedir();
const fallbackDirs = [
path.join(home, '.nvm/versions/node'),
path.join(home, '.local/bin'),
path.join(home, '.volta/bin'),
path.join(home, '.fnm/aliases/default/bin'),
path.join(home, '.bun/bin'),
path.join(home, '.cargo/bin'),
path.join(home, '.local/bin'),
'/opt/homebrew/bin',
'/usr/local/bin',
];
// For nvm, resolve the current default version dynamically
const nvmDir = path.join(home, '.nvm/versions/node');
try {
if (fs.existsSync(nvmDir)) {
@@ -58,10 +88,14 @@ function getShellPath() {
}
} catch (_) { /* ignore */ }
const existing = fallbackDirs.filter((d) => {
try { return fs.statSync(d).isDirectory(); } catch { return false; }
});
return [...existing, process.env.PATH || ''].join(':');
const seen = new Set();
const dirs = [];
for (const d of [...fallbackDirs, ...systemPaths, ...(process.env.PATH || '').split(':')]) {
if (!d || seen.has(d)) continue;
seen.add(d);
try { if (fs.statSync(d).isDirectory()) dirs.push(d); } catch { /* skip */ }
}
return dirs.join(':');
}
function getResourcePath(...segments) {
@@ -190,6 +224,18 @@ function createWindow() {
mainWindow.loadFile(frontendPath);
}
mainWindow.webContents.on('will-attach-webview', (_event, webPreferences, _params) => {
webPreferences.plugins = true;
webPreferences.enableBlinkFeatures = 'EncryptedMedia';
});
mainWindow.webContents.on('will-navigate', (event, url) => {
if (isDev && url.startsWith('http://localhost:3000')) return;
if (url.startsWith('file://')) return;
event.preventDefault();
mainWindow.webContents.send('webview-new-window', url, mainWindow.webContents.id);
});
mainWindow.on('closed', () => {
mainWindow = null;
});
@@ -202,30 +248,36 @@ function sendToRenderer(channel, ...args) {
}
function setupAutoUpdater() {
if (!autoUpdater) return;
autoUpdater.autoDownload = false;
autoUpdater.autoInstallOnAppQuit = false;
autoUpdater.on('update-available', (info) => {
console.log(`Update available: ${info.version}`);
cachedUpdateStatus = { status: 'available', info, error: null };
sendToRenderer('update-available', info);
});
autoUpdater.on('update-not-available', (info) => {
console.log('App is up to date');
cachedUpdateStatus = { status: 'not-available', info, error: null };
sendToRenderer('update-not-available', info);
});
autoUpdater.on('download-progress', (progress) => {
cachedUpdateStatus = { status: 'downloading', info: progress, error: null };
sendToRenderer('download-progress', progress);
});
autoUpdater.on('update-downloaded', (info) => {
console.log(`Update downloaded: ${info.version}`);
cachedUpdateStatus = { status: 'downloaded', info, error: null };
sendToRenderer('update-downloaded', info);
});
autoUpdater.on('error', (err) => {
console.error('Auto-update error:', err);
cachedUpdateStatus = { status: 'error', info: null, error: err?.message || String(err) };
sendToRenderer('update-error', err?.message || String(err));
});
@@ -252,6 +304,68 @@ app.whenReady().then(async () => {
try { app.dock.setIcon(iconPath); } catch (_) {}
}
session.defaultSession.setPermissionRequestHandler((_wc, permission, callback) => {
const allowed = [
'media', 'mediaKeySystem', 'protected-media-identifier',
'geolocation', 'notifications', 'midi', 'midiSysex',
'clipboard-read', 'clipboard-sanitized-write',
'pointerLock', 'fullscreen', 'idle-detection',
];
console.log('Permission request:', permission, '->', allowed.includes(permission) ? 'granted' : 'denied');
callback(allowed.includes(permission));
});
session.defaultSession.setPermissionCheckHandler((_wc, permission) => {
const allowed = [
'media', 'mediaKeySystem', 'protected-media-identifier',
'clipboard-read', 'clipboard-sanitized-write',
'pointerLock', 'fullscreen', 'idle-detection',
];
return allowed.includes(permission);
});
// Read-only logging for DRM license requests — no modifying interceptors
// so the network stack can set Content-Type and other headers normally.
session.defaultSession.webRequest.onSendHeaders(
{ urls: ['*://*/*widevine*license*'] },
(details) => {
console.log(`[drm-req] ${details.method} ${details.url}`);
for (const [k, v] of Object.entries(details.requestHeaders || {})) {
if (/content-type|origin|referer|auth|accept/i.test(k)) {
console.log(`[drm-req] ${k}: ${v}`);
}
}
},
);
session.defaultSession.webRequest.onCompleted(
{ urls: ['*://*/*widevine*', '*://*/*license*'] },
(details) => {
console.log(`[drm-net] ${details.method} ${details.url}${details.statusCode}`);
},
);
session.defaultSession.webRequest.onErrorOccurred(
{ urls: ['*://*/*widevine*', '*://*/*license*'] },
(details) => {
console.log(`[drm-net] FAILED ${details.method} ${details.url}${details.error}`);
},
);
// Wait for the Widevine CDM to be downloaded/ready (CastLabs Component
// Updater Service). On first launch this downloads the CDM; subsequent
// launches use the cached version.
if (components && typeof components.whenReady === 'function') {
try {
await components.whenReady();
console.log('Widevine CDM ready');
if (typeof components.status === 'function') {
console.log('CDM component status:', JSON.stringify(components.status()));
}
} catch (err) {
console.warn('Widevine CDM not available:', err.message);
}
} else {
console.log('CastLabs components API not available — using standard Electron (no DRM)');
}
try {
if (isDev) {
backendPort = parseInt(process.env.OPENSWARM_PORT || '8324', 10);
@@ -262,6 +376,13 @@ app.whenReady().then(async () => {
createWindow();
if (!isDev) {
setupAutoUpdater();
mainWindow.webContents.on('did-finish-load', () => {
if (cachedUpdateStatus.status === 'available') {
sendToRenderer('update-available', cachedUpdateStatus.info);
} else if (cachedUpdateStatus.status === 'downloaded') {
sendToRenderer('update-downloaded', cachedUpdateStatus.info);
}
});
}
} catch (err) {
console.error('Failed to start:', err);
@@ -269,6 +390,79 @@ app.whenReady().then(async () => {
}
});
app.on('web-contents-created', (_event, contents) => {
contents.setWindowOpenHandler(({ url, disposition }) => {
if (disposition === 'foreground-tab' || disposition === 'background-tab') {
if (mainWindow && !mainWindow.isDestroyed()) {
mainWindow.webContents.send('webview-new-window', url, contents.id);
}
return { action: 'deny' };
}
return {
action: 'allow',
overrideBrowserWindowOptions: {
parent: mainWindow || undefined,
},
};
});
contents.on('did-create-window', (childWindow) => {
if (mainWindow && !mainWindow.isDestroyed() && !childWindow.isDestroyed()) {
childWindow.setParentWindow(mainWindow);
}
});
if (contents.getType() === 'webview') {
contents.on('console-message', (_e, level, message, line, sourceId) => {
if (message.includes('widevine') || message.includes('drm') ||
message.includes('license') || message.includes('MediaKeySession') ||
message.includes('EME') || message.includes('[drm-diag]') || level >= 2) {
const tag = ['LOG', 'INFO', 'WARN', 'ERROR'][level] || 'LOG';
const src = sourceId ? sourceId.split('/').pop() : '';
console.log(`[webview:${tag}] ${message}${src ? ` (${src}:${line})` : ''}`);
}
});
contents.on('dom-ready', () => {
const url = contents.getURL();
if (url.includes('spotify')) {
contents.executeJavaScript(`
(function() {
const origFetch = window.fetch;
window.fetch = async function(...args) {
const resp = await origFetch.apply(this, args);
const url = typeof args[0] === 'string' ? args[0] : args[0]?.url || '';
if (url.includes('widevine-license') && !resp.ok) {
const clone = resp.clone();
try {
const text = await clone.text();
console.log('[drm-diag] License response ' + resp.status + ': ' + text.substring(0, 500));
} catch(e) {}
}
return resp;
};
// Check EME availability
if (navigator.requestMediaKeySystemAccess) {
navigator.requestMediaKeySystemAccess('com.widevine.alpha', [{
initDataTypes: ['cenc'],
audioCapabilities: [{contentType: 'audio/mp4; codecs="mp4a.40.2"'}],
}]).then(function(access) {
console.log('[drm-diag] Widevine EME access: ' + access.keySystem);
}).catch(function(err) {
console.log('[drm-diag] Widevine EME FAILED: ' + err.message);
});
} else {
console.log('[drm-diag] EME API not available');
}
})();
`).catch(() => {});
}
});
}
});
app.on('window-all-closed', () => {
if (!isDev) killBackend();
app.quit();
@@ -286,9 +480,14 @@ app.on('activate', () => {
ipcMain.handle('get-backend-port', () => backendPort);
ipcMain.handle('get-app-version', () => app.getVersion());
ipcMain.handle('get-webview-preload-path', () => {
return `file://${path.join(__dirname, 'webview-preload.js')}`;
});
ipcMain.handle('get-update-status', () => cachedUpdateStatus);
ipcMain.handle('check-for-updates', async () => {
if (!isPackaged) {
if (!autoUpdater || !isPackaged) {
sendToRenderer('update-error', 'Update check is only available in the packaged app.');
return { success: false, error: 'Not packaged' };
}
@@ -305,6 +504,7 @@ ipcMain.handle('check-for-updates', async () => {
});
ipcMain.handle('download-update', async () => {
if (!autoUpdater) return { success: false, error: 'Updater not available' };
try {
await autoUpdater.downloadUpdate();
return { success: true };
@@ -314,6 +514,7 @@ ipcMain.handle('download-update', async () => {
});
ipcMain.handle('install-update', () => {
if (!autoUpdater) return;
autoUpdater.quitAndInstall(false, true);
});
+12 -12
View File
@@ -1,19 +1,20 @@
{
"name": "openswarm",
"version": "1.0.3",
"version": "1.0.11",
"lockfileVersion": 3,
"requires": true,
"packages": {
"": {
"name": "openswarm",
"version": "1.0.3",
"version": "1.0.11",
"hasInstallScript": true,
"dependencies": {
"electron-updater": "^6.3.0",
"get-port": "^5.1.1"
},
"devDependencies": {
"@electron/notarize": "^3.1.1",
"electron": "^33.0.0",
"electron": "castlabs/electron-releases#v33.4.11+wvcus",
"electron-builder": "^25.1.0"
}
},
@@ -2206,9 +2207,8 @@
}
},
"node_modules/electron": {
"version": "33.4.11",
"resolved": "https://registry.npmjs.org/electron/-/electron-33.4.11.tgz",
"integrity": "sha512-xmdAs5QWRkInC7TpXGNvzo/7exojubk+72jn1oJL7keNeIlw7xNglf8TGtJtkR4rWC5FJq0oXiIXPS9BcK2Irg==",
"version": "33.4.11+wvcus",
"resolved": "git+ssh://git@github.com/castlabs/electron-releases.git#d1cf58c11ec0a8a04f307ed362d7efde2816778d",
"dev": true,
"hasInstallScript": true,
"license": "MIT",
@@ -3953,9 +3953,9 @@
}
},
"node_modules/node-abi": {
"version": "3.88.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.88.0.tgz",
"integrity": "sha512-At6b4UqIEVudaqPsXjmUO1r/N5BUr4yhDGs5PkBE8/oG5+TfLPhFechiskFsnT6Ql0VfUXbalUUCbfXxtj7K+w==",
"version": "3.89.0",
"resolved": "https://registry.npmjs.org/node-abi/-/node-abi-3.89.0.tgz",
"integrity": "sha512-6u9UwL0HlAl21+agMN3YAMXcKByMqwGx+pq+P76vii5f7hTPtKDp08/H9py6DY+cfDw7kQNTGEj/rly3IgbNQA==",
"dev": true,
"license": "MIT",
"dependencies": {
@@ -4605,9 +4605,9 @@
}
},
"node_modules/sax": {
"version": "1.5.0",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.5.0.tgz",
"integrity": "sha512-21IYA3Q5cQf089Z6tgaUTr7lDAyzoTPx5HRtbhsME8Udispad8dC/+sziTNugOEx54ilvatQ9YCzl4KQLPcRHA==",
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.6.0.tgz",
"integrity": "sha512-6R3J5M4AcbtLUdZmRv2SygeVaM7IhrLXu9BmnOGmmACak8fiUtOsYNWUS4uK7upbmHIBbLBeFeI//477BKLBzA==",
"license": "BlueOak-1.0.0",
"engines": {
"node": ">=11.0.0"
+13 -17
View File
@@ -1,11 +1,13 @@
{
"name": "openswarm",
"version": "1.0.3",
"version": "1.0.12",
"description": "OpenSwarm — AI Agent Orchestrator",
"main": "main.js",
"scripts": {
"start": "electron .",
"dev": "ELECTRON_DEV=1 electron .",
"postinstall": "bash scripts/sign-vmp.sh",
"sign-vmp": "bash scripts/sign-vmp.sh",
"dist": "electron-builder --mac --publish never",
"dist:publish": "electron-builder --mac --publish always",
"dist:all": "electron-builder --mac --win --linux"
@@ -16,12 +18,15 @@
},
"devDependencies": {
"@electron/notarize": "^3.1.1",
"electron": "^33.0.0",
"electron": "castlabs/electron-releases#v33.4.11+wvcus",
"electron-builder": "^25.1.0"
},
"build": {
"appId": "com.clusterlabs.openswarm",
"productName": "OpenSwarm",
"electronDownload": {
"mirror": "https://github.com/castlabs/electron-releases/releases/download/v"
},
"directories": {
"output": "dist"
},
@@ -38,6 +43,7 @@
"entitlementsInherit": "build/entitlements.mac.plist"
},
"dmg": {
"artifactName": "OpenSwarm-${arch}.${ext}",
"title": "OpenSwarm",
"contents": [
{
@@ -54,34 +60,24 @@
},
"extraResources": [
{
"from": "../frontend/dist",
"from": "build-staging/frontend",
"to": "frontend",
"filter": [
"**/*"
]
},
{
"from": "../backend",
"from": "build-staging/backend",
"to": "backend",
"filter": [
"**/*",
"!__pycache__/**",
"!**/__pycache__/**",
"!.venv/**",
"!*.pyc"
"**/*"
]
},
{
"from": "../debugger",
"from": "build-staging/debugger",
"to": "debugger",
"filter": [
"**/*",
"!__pycache__/**",
"!**/__pycache__/**",
"!*.pyc",
"!.venv/**",
"!**/.venv/**",
"!**/node_modules/**"
"**/*"
]
},
{
+9
View File
@@ -2,15 +2,18 @@ const { contextBridge, ipcRenderer } = require('electron');
(async () => {
const port = await ipcRenderer.invoke('get-backend-port');
const webviewPreloadPath = await ipcRenderer.invoke('get-webview-preload-path');
contextBridge.exposeInMainWorld('__OPENSWARM_PORT__', port);
contextBridge.exposeInMainWorld('openswarm', {
getBackendPort: () => port,
getWebviewPreloadPath: () => webviewPreloadPath,
getAppVersion: () => ipcRenderer.invoke('get-app-version'),
openExternal: (url) => ipcRenderer.invoke('open-external', url),
capturePage: (rect) => ipcRenderer.invoke('capture-page', rect),
getUpdateStatus: () => ipcRenderer.invoke('get-update-status'),
checkForUpdates: () => ipcRenderer.invoke('check-for-updates'),
downloadUpdate: () => ipcRenderer.invoke('download-update'),
installUpdate: () => ipcRenderer.invoke('install-update'),
@@ -40,5 +43,11 @@ const { contextBridge, ipcRenderer } = require('electron');
ipcRenderer.on('update-error', listener);
return () => ipcRenderer.removeListener('update-error', listener);
},
onWebviewNewWindow: (cb) => {
const listener = (_event, url, webContentsId) => cb(url, webContentsId);
ipcRenderer.on('webview-new-window', listener);
return () => ipcRenderer.removeListener('webview-new-window', listener);
},
});
})();
+61
View File
@@ -0,0 +1,61 @@
#!/bin/bash
# Signs the CastLabs Electron binary with a production VMP certificate via EVS,
# then repairs macOS framework symlinks that npm/signing may strip.
#
# First-time setup (one-time):
# pip3 install --user castlabs-evs
# python3 -m castlabs_evs.account signup
#
# After signup, this script runs automatically.
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ELECTRON_DIR="$SCRIPT_DIR/../node_modules/electron/dist"
FW_BASE="$ELECTRON_DIR/Electron.app/Contents/Frameworks"
fix_framework_symlinks() {
[ -d "$FW_BASE" ] || return 0
for fw in "$FW_BASE"/*.framework; do
[ -d "$fw/Versions/A" ] || continue
local name
name=$(basename "$fw" .framework)
cd "$fw"
(cd Versions && ln -sf A Current 2>/dev/null)
ln -sf "Versions/Current/$name" "$name" 2>/dev/null
[ -d "Versions/A/Resources" ] && ln -sf Versions/Current/Resources Resources 2>/dev/null
[ -d "Versions/A/Libraries" ] && ln -sf Versions/Current/Libraries Libraries 2>/dev/null
[ -d "Versions/A/Helpers" ] && ln -sf Versions/Current/Helpers Helpers 2>/dev/null
done
}
if [ ! -d "$ELECTRON_DIR" ]; then
echo "[vmp] Electron dist not found at $ELECTRON_DIR — skipping VMP signing"
fix_framework_symlinks
exit 0
fi
# Always fix symlinks first (npm git installs strip them)
fix_framework_symlinks
if ! python3 -c "import castlabs_evs" 2>/dev/null; then
echo "[vmp] castlabs-evs not installed. Install with: pip3 install --user castlabs-evs"
echo "[vmp] Skipping VMP signing — DRM playback will be limited"
exit 0
fi
VERIFY_OUTPUT=$(python3 -m castlabs_evs.vmp verify-pkg "$ELECTRON_DIR" 2>&1)
if echo "$VERIFY_OUTPUT" | grep -q "Signature is valid" && ! echo "$VERIFY_OUTPUT" | grep -q "development only"; then
echo "[vmp] Electron already has a valid production VMP signature"
exit 0
fi
echo "[vmp] Signing Electron with production VMP certificate..."
if python3 -m castlabs_evs.vmp sign-pkg "$ELECTRON_DIR" 2>&1; then
echo "[vmp] VMP signing successful — full DRM playback enabled"
# Re-fix symlinks in case signing modified the bundle
fix_framework_symlinks
else
echo "[vmp] VMP signing failed — you may need to run: python3 -m castlabs_evs.account signup"
echo "[vmp] DRM playback will be limited to previews until signed"
fi
exit 0
+87
View File
@@ -0,0 +1,87 @@
/**
* Webview preload script patches browser fingerprinting so sites like
* Spotify/Netflix don't detect an Electron shell and disable features.
* Loaded via the webview's `preload` attribute before any page script runs.
*/
'use strict';
// Hide webdriver flag
Object.defineProperty(navigator, 'webdriver', {
get: () => false,
configurable: true,
});
// Spoof navigator.plugins (Chrome has a few built-in ones)
const fakePlugins = {
0: { name: 'Chrome PDF Plugin', filename: 'internal-pdf-viewer', description: 'Portable Document Format' },
1: { name: 'Chrome PDF Viewer', filename: 'mhjfbmdgcfjbbpaeojofohoefgiehjai', description: '' },
2: { name: 'Native Client', filename: 'internal-nacl-plugin', description: '' },
length: 3,
item: (i) => fakePlugins[i] || null,
namedItem: (name) => {
for (let i = 0; i < fakePlugins.length; i++) {
if (fakePlugins[i].name === name) return fakePlugins[i];
}
return null;
},
refresh: () => {},
[Symbol.iterator]: function* () {
for (let i = 0; i < this.length; i++) yield this[i];
},
};
try {
Object.defineProperty(navigator, 'plugins', {
get: () => fakePlugins,
configurable: true,
});
} catch (_) {}
// Ensure window.chrome exists (sites test for it)
if (!window.chrome) {
window.chrome = {};
}
if (!window.chrome.runtime) {
window.chrome.runtime = {
connect: () => {},
sendMessage: () => {},
onMessage: { addListener: () => {}, removeListener: () => {} },
};
}
// Ensure navigator.languages has sensible values
try {
Object.defineProperty(navigator, 'languages', {
get: () => ['en-US', 'en'],
configurable: true,
});
} catch (_) {}
// Patch permissions.query to report 'granted' for common permissions
const originalQuery = navigator.permissions?.query?.bind(navigator.permissions);
if (originalQuery) {
navigator.permissions.query = (params) => {
if (params.name === 'notifications') {
return Promise.resolve({ state: 'granted', onchange: null });
}
return originalQuery(params).catch(() =>
Promise.resolve({ state: 'prompt', onchange: null })
);
};
}
// Prevent iframe detection heuristics
try {
Object.defineProperty(document, 'hidden', {
get: () => false,
configurable: true,
});
Object.defineProperty(document, 'visibilityState', {
get: () => 'visible',
configurable: true,
});
} catch (_) {}
// Fix console.debug detection (some sites use it as a breakpoint detector)
const noop = () => {};
if (!window.console.debug) window.console.debug = noop;
+334
View File
@@ -0,0 +1,334 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1.0">
<title>Pixel Face</title>
<style>
* { margin: 0; padding: 0; }
body { background: #000; overflow: hidden; cursor: none; }
canvas {
display: block;
image-rendering: pixelated;
image-rendering: crisp-edges;
}
</style>
</head>
<body>
<canvas id="c"></canvas>
<script>
const C = document.getElementById('c');
const X = C.getContext('2d');
// Grid
const PX = 20;
let COLS, ROWS;
function resize() {
COLS = Math.ceil(window.innerWidth / PX);
ROWS = Math.ceil(window.innerHeight / PX);
C.width = COLS * PX;
C.height = ROWS * PX;
}
resize();
window.addEventListener('resize', resize);
const BG = '#E8927A';
const EYE = '#1E1E1E';
const MOUTH = '#1E1E1E';
// State
let state = 'idle';
let breath = 0;
let talk = 0;
let think = 0;
let sleepZ = 0;
let heartP = 0;
// Lerped eye/mouth sizes
let eyeH = 3, eyeHTarget = 3;
let mouthW = 2, mouthWTarget = 2;
let mouthH = 2, mouthHTarget = 2;
let eyeOffX = 0, eyeOffXTarget = 0;
let eyeOffY = 0, eyeOffYTarget = 0;
// Idle behavior system
let idleTimer = 0;
let idleAction = 'none';
let idleActionTimer = 0;
let idleGlanceX = 0;
let idleGlanceY = 0;
let blinkOpen = true;
let blinkCD = 120 + Math.random() * 200;
let doubleBlink = false;
let idleSinceInput = 0;
let sleepTransitioned = false;
function lerp(a, b, t) { return a + (b - a) * t; }
function px(col, row, color) {
X.fillStyle = color;
X.fillRect(col * PX, row * PX, PX, PX);
}
function pxRect(x, y, w, h, color) {
for (let r = 0; r < Math.round(h); r++)
for (let c = 0; c < Math.round(w); c++)
px(Math.round(x) + c, Math.round(y) + r, color);
}
function set(s) {
state = s;
idleSinceInput = 0;
sleepTransitioned = false;
}
// --- Idle behavior ---
function updateIdle() {
idleSinceInput++;
// Auto-sleep after ~45s no input
if (idleSinceInput > 2700 && state === 'idle' && !sleepTransitioned) {
state = 'sleeping';
sleepTransitioned = true;
return;
}
// Wake from auto-sleep on input (handled in set())
if (state !== 'idle') return;
idleTimer++;
idleActionTimer--;
// Blink system
blinkCD--;
if (blinkCD <= 0 && blinkOpen) {
blinkOpen = false;
blinkCD = 6;
doubleBlink = Math.random() < 0.3;
} else if (!blinkOpen && blinkCD <= 0) {
blinkOpen = true;
if (doubleBlink) {
doubleBlink = false;
blinkCD = 8;
} else {
blinkCD = 100 + Math.random() * 280;
}
}
// Pick new idle action
if (idleActionTimer <= 0) {
const roll = Math.random();
if (roll < 0.30) {
idleAction = 'glance';
idleGlanceX = Math.floor(Math.random() * 5) - 2;
idleGlanceY = (Math.random() - 0.5) * 1.2;
idleActionTimer = 50 + Math.random() * 120;
} else if (roll < 0.45) {
idleAction = 'scan';
idleActionTimer = 180;
} else if (roll < 0.55) {
idleAction = 'squint';
idleActionTimer = 35 + Math.random() * 40;
} else if (roll < 0.65) {
idleAction = 'lookup';
idleActionTimer = 50 + Math.random() * 70;
} else {
idleAction = 'none';
idleGlanceX = 0;
idleGlanceY = 0;
idleActionTimer = 60 + Math.random() * 200;
}
}
}
function draw() {
breath += 0.025;
talk += 0.3;
think += 0.025;
sleepZ += 0.012;
heartP += 0.05;
updateIdle();
// Blink for non-idle blinking states
if (state === 'talking' || state === 'thinking') {
blinkCD--;
if (blinkCD <= 0 && blinkOpen) { blinkOpen = false; blinkCD = 6; }
else if (!blinkOpen && blinkCD <= 0) { blinkOpen = true; blinkCD = 120 + Math.random() * 250; }
}
if (state !== 'idle' && state !== 'talking' && state !== 'thinking') {
blinkOpen = true;
}
const b = Math.sin(breath) * 0.3;
switch (state) {
case 'idle': {
eyeHTarget = blinkOpen ? 3 : 0;
mouthWTarget = 2; mouthHTarget = 2;
let gx = 0, gy = b;
if (idleAction === 'glance') {
gx = idleGlanceX; gy = idleGlanceY + b;
} else if (idleAction === 'scan') {
const t = 1 - (idleActionTimer / 180);
gx = Math.sin(t * Math.PI * 2) * 2.5; gy = b;
} else if (idleAction === 'squint') {
eyeHTarget = blinkOpen ? 2 : 0;
gy = b + 0.3;
} else if (idleAction === 'lookup') {
gy = -1.2 + b;
}
eyeOffXTarget = gx; eyeOffYTarget = gy;
break;
}
case 'happy':
eyeHTarget = 1; mouthWTarget = 6; mouthHTarget = 1;
eyeOffXTarget = 0; eyeOffYTarget = b;
break;
case 'thinking':
eyeHTarget = blinkOpen ? 3 : 0;
mouthWTarget = 2; mouthHTarget = 2;
eyeOffXTarget = 2; eyeOffYTarget = b;
break;
case 'talking': {
eyeHTarget = blinkOpen ? 3 : 0;
const open = Math.round(Math.abs(Math.sin(talk)) * 2 + 1);
mouthWTarget = 4; mouthHTarget = open;
eyeOffXTarget = 0; eyeOffYTarget = b;
break;
}
case 'surprised':
eyeHTarget = 4; mouthWTarget = 3; mouthHTarget = 3;
eyeOffXTarget = 0; eyeOffYTarget = b;
break;
case 'sleeping':
eyeHTarget = 1; mouthWTarget = 2; mouthHTarget = 1;
eyeOffXTarget = 0; eyeOffYTarget = b * 2;
break;
case 'angry':
eyeHTarget = 2; mouthWTarget = 6; mouthHTarget = 1;
eyeOffXTarget = 0; eyeOffYTarget = b * 0.3;
break;
case 'love':
eyeHTarget = 3; mouthWTarget = 2; mouthHTarget = 2;
eyeOffXTarget = 0; eyeOffYTarget = b;
break;
}
// Lerp
eyeH = lerp(eyeH, eyeHTarget, 0.18);
mouthW = lerp(mouthW, mouthWTarget, 0.15);
mouthH = lerp(mouthH, mouthHTarget, 0.2);
eyeOffX = lerp(eyeOffX, eyeOffXTarget, 0.1);
eyeOffY = lerp(eyeOffY, eyeOffYTarget, 0.15);
// --- Draw ---
// Full screen pink
X.fillStyle = BG;
X.fillRect(0, 0, C.width, C.height);
// Center of screen in grid coords
const cx = Math.floor(COLS / 2);
const cy = Math.floor(ROWS / 2);
// Eye positions: spread apart, centered
const eyeSpread = 5;
const eyeW = 3;
const eh = Math.max(1, Math.round(eyeH));
const eOffX = Math.round(eyeOffX);
const eOffY = Math.round(eyeOffY);
// Vertical centering for blink (close from middle)
const eyeBaseY = cy - 2;
const blinkOff = Math.round((3 - eh) / 2);
if (state === 'love') {
// Hearts instead of eyes
const pulse = Math.sin(heartP) > 0 ? '#CC2244' : '#BB1E3E';
function heart(hx, hy) {
px(hx-1, hy, pulse); px(hx+1, hy, pulse);
px(hx-2, hy+1, pulse); px(hx-1, hy+1, pulse); px(hx, hy+1, pulse); px(hx+1, hy+1, pulse); px(hx+2, hy+1, pulse);
px(hx-1, hy+2, pulse); px(hx, hy+2, pulse); px(hx+1, hy+2, pulse);
px(hx, hy+3, pulse);
}
heart(cx - eyeSpread + eOffX, eyeBaseY + eOffY);
heart(cx + eyeSpread + eOffX, eyeBaseY + eOffY);
} else {
// Left eye
pxRect(cx - eyeSpread - 1 + eOffX, eyeBaseY + blinkOff + eOffY, eyeW, eh, EYE);
// Right eye
pxRect(cx + eyeSpread - 1 + eOffX, eyeBaseY + blinkOff + eOffY, eyeW, eh, EYE);
}
// Angry eyebrows
if (state === 'angry') {
const lx = cx - eyeSpread - 1 + eOffX;
const ly = eyeBaseY + blinkOff + eOffY - 2;
px(lx, ly + 1, EYE); px(lx + 1, ly, EYE); px(lx + 2, ly, EYE);
const rx = cx + eyeSpread - 1 + eOffX;
px(rx + 2, ly + 1, EYE); px(rx + 1, ly, EYE); px(rx, ly, EYE);
}
// Mouth
const mw = Math.max(1, Math.round(mouthW));
const mh = Math.max(1, Math.round(mouthH));
const mouthY = cy + 4 + Math.round(eyeOffY);
if (state === 'happy') {
// Smile: line with corners up
pxRect(cx - Math.floor(mw/2), mouthY, mw, 1, MOUTH);
px(cx - Math.floor(mw/2), mouthY - 1, MOUTH);
px(cx - Math.floor(mw/2) + mw - 1, mouthY - 1, MOUTH);
} else if (state === 'angry') {
// Frown: line with corners down
pxRect(cx - Math.floor(mw/2), mouthY, mw, 1, MOUTH);
px(cx - Math.floor(mw/2), mouthY + 1, MOUTH);
px(cx - Math.floor(mw/2) + mw - 1, mouthY + 1, MOUTH);
} else {
// Simple rect
pxRect(cx - Math.floor(mw/2), mouthY, mw, mh, MOUTH);
}
// Sleeping Z's
if (state === 'sleeping') {
const zFrame = Math.floor(sleepZ * 60) % 90;
const zy = Math.round(cy - 5 - (zFrame / 90) * 4);
const zx = cx + eyeSpread + 3;
if (zy >= 1 && zFrame < 70) {
px(zx, zy, '#5577CC'); px(zx+1, zy, '#5577CC');
px(zx+1, zy+1, '#5577CC');
px(zx, zy+2, '#5577CC'); px(zx+1, zy+2, '#5577CC');
}
}
// Thinking dots
if (state === 'thinking') {
const phase = Math.floor(think * 10) % 4;
const dx = cx + eyeSpread + 3;
const dy = cy - 5;
if (phase >= 1) px(dx, dy, '#7799DD');
if (phase >= 2) px(dx + 1, dy - 1, '#7799DD');
if (phase >= 3) px(dx + 2, dy - 2, '#7799DD');
}
requestAnimationFrame(draw);
}
// Keyboard
document.addEventListener('keydown', e => {
const m = {'1':'idle','2':'happy','3':'thinking','4':'talking','5':'surprised','6':'sleeping','7':'angry','8':'love'};
if (m[e.key]) set(m[e.key]);
});
// Click cycles through states
let si = 0;
const STATES = ['idle','happy','thinking','talking','surprised','sleeping','angry','love'];
document.addEventListener('click', () => {
si = (si + 1) % STATES.length;
set(STATES[si]);
});
draw();
</script>
</body>
</html>
+32 -1
View File
@@ -3,8 +3,9 @@ import { Provider } from 'react-redux';
import { HashRouter, Routes, Route } from 'react-router-dom';
import { ThemeProvider as MuiThemeProvider, createTheme, CssBaseline } from '@mui/material';
import { store } from '../shared/state/store';
import { useAppDispatch } from '@/shared/hooks';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import { fetchSettings } from '@/shared/state/settingsSlice';
import { fetchModels } from '@/shared/state/modelsSlice';
import {
setAppVersion,
setUpdateAvailable,
@@ -22,9 +23,13 @@ import Tools from './pages/Tools/Tools';
import Modes from './pages/Modes/Modes';
import Views from './pages/Views/Views';
import Customization from './pages/Customization/Customization';
import Channels from './pages/Channels/Channels';
import Analytics from './pages/Analytics/Analytics';
import AnalyticsOptIn from './components/AnalyticsOptIn';
import { useKeyboardShortcuts } from '@/shared/hooks/useKeyboardShortcuts';
import KeyboardShortcutsHelp from './components/KeyboardShortcutsHelp';
import { ThemeProvider, useThemeMode, useClaudeTokens } from '@/shared/styles/ThemeContext';
import OnboardingModal from './components/OnboardingModal';
import { ClaudeTokens } from '@/shared/styles/claudeTokens';
function buildMuiTheme(c: ClaudeTokens, mode: 'light' | 'dark') {
@@ -155,9 +160,16 @@ const ShortcutsProvider: React.FC<{ children: React.ReactNode }> = ({ children }
const SettingsLoader: React.FC<{ children: React.ReactNode }> = ({ children }) => {
const dispatch = useAppDispatch();
const { setMode: setThemeMode } = useThemeMode();
const theme = useAppSelector((s) => s.settings.data.theme);
const loaded = useAppSelector((s) => s.settings.loaded);
useEffect(() => {
dispatch(fetchSettings());
dispatch(fetchModels());
}, [dispatch]);
useEffect(() => {
if (loaded) setThemeMode(theme as 'light' | 'dark');
}, [loaded, theme, setThemeMode]);
return <>{children}</>;
};
@@ -170,6 +182,21 @@ const UpdateListener: React.FC<{ children: React.ReactNode }> = ({ children }) =
api.getAppVersion().then((v: string) => dispatch(setAppVersion(v)));
api.getUpdateStatus?.().then((cached) => {
if (!cached) return;
if (cached.status === 'available' && cached.info?.version) {
dispatch(setUpdateAvailable(cached.info.version));
} else if (cached.status === 'not-available') {
dispatch(setUpdateNotAvailable());
} else if (cached.status === 'downloading' && cached.info?.percent != null) {
dispatch(setDownloading(cached.info.percent));
} else if (cached.status === 'downloaded') {
dispatch(setUpdateDownloaded());
} else if (cached.status === 'error' && cached.error) {
dispatch(setUpdateError(cached.error));
}
});
const cleanups = [
api.onUpdateAvailable?.((info: OpenSwarmUpdateInfo) => dispatch(setUpdateAvailable(info.version))),
api.onUpdateNotAvailable?.(() => dispatch(setUpdateNotAvailable())),
@@ -207,8 +234,12 @@ const ThemedApp: React.FC = () => {
<Route path="/modes" element={<Modes />} />
<Route path="/apps" element={<Views />} />
<Route path="/apps/:id" element={<Views />} />
<Route path="/channels" element={<Channels />} />
<Route path="/analytics" element={<Analytics />} />
</Route>
</Routes>
<AnalyticsOptIn />
<OnboardingModal />
</UpdateListener>
</SettingsLoader>
</ShortcutsProvider>
@@ -0,0 +1,84 @@
import React from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import Button from '@mui/material/Button';
import Paper from '@mui/material/Paper';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import { updateSettings } from '@/shared/state/settingsSlice';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
const AnalyticsOptIn: React.FC = () => {
const c = useClaudeTokens();
const dispatch = useAppDispatch();
const settings = useAppSelector((s) => s.settings.data);
const loaded = useAppSelector((s) => s.settings.loaded);
if (!loaded || settings.analytics_opt_in !== null) return null;
const handleChoice = (optIn: boolean) => {
dispatch(updateSettings({ ...settings, analytics_opt_in: optIn }));
};
return (
<Box
sx={{
position: 'fixed',
bottom: 24,
left: '50%',
transform: 'translateX(-50%)',
zIndex: 1400,
maxWidth: 480,
width: '90%',
}}
>
<Paper
elevation={0}
sx={{
p: 2.5,
bgcolor: c.bg.surface,
border: `1px solid ${c.border.medium}`,
borderRadius: 3,
boxShadow: c.shadow.lg,
}}
>
<Typography sx={{ color: c.text.primary, fontSize: '0.9rem', fontWeight: 600, mb: 0.5 }}>
Help improve OpenSwarm
</Typography>
<Typography sx={{ color: c.text.muted, fontSize: '0.8rem', lineHeight: 1.5, mb: 2 }}>
Share anonymous usage statistics like session counts, feature usage, and model preferences.
No conversations, file paths, or personal information ever.
You can change this anytime in Settings.
</Typography>
<Box sx={{ display: 'flex', gap: 1, justifyContent: 'flex-end' }}>
<Button
onClick={() => handleChoice(false)}
sx={{
color: c.text.muted,
textTransform: 'none',
fontSize: '0.82rem',
'&:hover': { bgcolor: `${c.text.tertiary}0A` },
}}
>
No thanks
</Button>
<Button
variant="contained"
onClick={() => handleChoice(true)}
sx={{
bgcolor: c.accent.primary,
'&:hover': { bgcolor: c.accent.pressed },
textTransform: 'none',
fontSize: '0.82rem',
borderRadius: 1.5,
px: 2,
}}
>
Share anonymous data
</Button>
</Box>
</Paper>
</Box>
);
};
export default AnalyticsOptIn;
@@ -0,0 +1,993 @@
import React, { useMemo, useCallback, useState, useEffect, useRef } from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import IconButton from '@mui/material/IconButton';
import Tooltip from '@mui/material/Tooltip';
import Collapse from '@mui/material/Collapse';
import SearchIcon from '@mui/icons-material/Search';
import StopCircleOutlinedIcon from '@mui/icons-material/StopCircleOutlined';
import CloseIcon from '@mui/icons-material/Close';
import CheckIcon from '@mui/icons-material/Check';
import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
import ExpandLessIcon from '@mui/icons-material/ExpandLess';
import { motion, AnimatePresence } from 'framer-motion';
import { useNavigate } from 'react-router-dom';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import {
handleApproval,
stopAgent,
dismissAgentNotification,
dismissAllFinishedNotifications,
ApprovalRequest,
AgentSession,
HistorySession,
} from '@/shared/state/agentsSlice';
import { setPendingFocusAgentId } from '@/shared/state/tempStateSlice';
import ApprovalBar, { BatchApprovalBar, parseMcpToolName, useMcpToolMeta, getToolIcon } from '@/app/pages/AgentChat/ApprovalBar';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
// ---------------------------------------------------------------------------
// Types
// ---------------------------------------------------------------------------
type IslandState = 'idle' | 'compact' | 'compact-actionable' | 'expanded';
interface SessionApprovalGroup {
sessionId: string;
sessionName: string;
approvals: ApprovalRequest[];
}
type TrackedAgent = {
id: string;
name: string;
status: AgentSession['status'] | string;
dashboardId?: string;
};
const STATUS_CONFIG: Record<string, { label: string; tokenKey?: string }> = {
running: { label: 'Running', tokenKey: 'success' },
waiting_approval: { label: 'Waiting', tokenKey: 'warning' },
completed: { label: 'Done', tokenKey: 'success' },
error: { label: 'Error', tokenKey: 'error' },
stopped: { label: 'Stopped', tokenKey: 'info' },
};
// ---------------------------------------------------------------------------
// Spring configs
// ---------------------------------------------------------------------------
const SPRING_LAYOUT = { type: 'spring' as const, stiffness: 400, damping: 30 };
const SPRING_BOUNCE = { type: 'spring' as const, stiffness: 500, damping: 25 };
// ---------------------------------------------------------------------------
// Sub-components
// ---------------------------------------------------------------------------
const StatusDot: React.FC<{ status: string; c: ReturnType<typeof useClaudeTokens> }> = ({ status, c }) => {
const cfg = STATUS_CONFIG[status];
const color = cfg?.tokenKey ? (c.status as any)[cfg.tokenKey] : c.text.ghost;
const isActive = status === 'running';
return (
<Box
sx={{
width: 6,
height: 6,
borderRadius: '50%',
bgcolor: color,
flexShrink: 0,
opacity: 0.8,
...(isActive && {
animation: 'islandPulse 2s ease-in-out infinite',
'@keyframes islandPulse': {
'0%, 100%': { opacity: 0.8, transform: 'scale(1)' },
'50%': { opacity: 0.4, transform: 'scale(1.3)' },
},
}),
}}
/>
);
};
const AgentStatusRow: React.FC<{
agent: TrackedAgent;
c: ReturnType<typeof useClaudeTokens>;
onStop: (id: string) => void;
onDismiss: (id: string) => void;
onNavigate: (dashboardId: string, agentId: string) => void;
}> = ({ agent, c, onStop, onDismiss, onNavigate }) => {
const isActive = agent.status === 'running' || agent.status === 'waiting_approval';
const cfg = STATUS_CONFIG[agent.status] ?? { label: agent.status };
return (
<Box
onClick={() => agent.dashboardId && onNavigate(agent.dashboardId, agent.id)}
sx={{
display: 'flex',
alignItems: 'center',
gap: 1,
px: 2,
py: 0.75,
cursor: agent.dashboardId ? 'pointer' : 'default',
'&:hover': { bgcolor: c.border.subtle },
transition: 'background-color 0.15s',
minHeight: 34,
}}
>
<StatusDot status={agent.status} c={c} />
<Typography
sx={{
fontSize: '0.78rem',
fontWeight: 500,
color: c.text.secondary,
flex: 1,
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
}}
>
{agent.name}
</Typography>
<Typography
sx={{
fontSize: '0.6rem',
color: c.text.ghost,
textTransform: 'uppercase',
letterSpacing: '0.04em',
flexShrink: 0,
}}
>
{cfg.label}
</Typography>
{isActive ? (
<Tooltip title="Stop agent" arrow>
<IconButton
size="small"
onClick={(e) => { e.stopPropagation(); onStop(agent.id); }}
sx={{ p: 0.25, color: c.text.ghost, '&:hover': { color: c.status.error, bgcolor: c.border.subtle } }}
>
<StopCircleOutlinedIcon sx={{ fontSize: 15 }} />
</IconButton>
</Tooltip>
) : (
<Tooltip title="Dismiss" arrow>
<IconButton
size="small"
onClick={(e) => { e.stopPropagation(); onDismiss(agent.id); }}
sx={{ p: 0.25, color: c.text.ghost, '&:hover': { bgcolor: c.border.subtle } }}
>
<CloseIcon sx={{ fontSize: 13 }} />
</IconButton>
</Tooltip>
)}
</Box>
);
};
// ---------------------------------------------------------------------------
// Compact activity indicator — subtle breathing dot
// ---------------------------------------------------------------------------
const ActivityIndicator: React.FC<{ c: ReturnType<typeof useClaudeTokens> }> = ({ c }) => (
<Box
sx={{
width: 6,
height: 6,
borderRadius: '50%',
bgcolor: c.text.tertiary,
flexShrink: 0,
animation: 'subtlePulse 2.2s ease-in-out infinite',
'@keyframes subtlePulse': {
'0%, 100%': { opacity: 0.6, transform: 'scale(1)' },
'50%': { opacity: 1, transform: 'scale(1.15)' },
},
}}
/>
);
// ---------------------------------------------------------------------------
// Main component
// ---------------------------------------------------------------------------
const DynamicIsland: React.FC = () => {
const c = useClaudeTokens();
const dispatch = useAppDispatch();
const navigate = useNavigate();
const islandRef = useRef<HTMLDivElement>(null);
const sessions = useAppSelector((state) => state.agents.sessions);
const history = useAppSelector((state) => state.agents.history);
const trackedIds = useAppSelector((state) => state.agents.trackedNotificationIds);
const [userExpanded, setUserExpanded] = useState(false);
// ---- Derived data ----
const groups: SessionApprovalGroup[] = useMemo(() => {
const result: SessionApprovalGroup[] = [];
for (const [sessionId, session] of Object.entries(sessions)) {
if (session.pending_approvals?.length > 0) {
result.push({
sessionId,
sessionName: session.name || 'Agent',
approvals: session.pending_approvals,
});
}
}
return result;
}, [sessions]);
const totalApprovals = useMemo(
() => groups.reduce((sum, g) => sum + g.approvals.length, 0),
[groups],
);
const trackedAgents: TrackedAgent[] = useMemo(() => {
const agents = trackedIds
.map((id): TrackedAgent | null => {
const session = sessions[id];
if (session && session.status !== 'draft') {
return { id, name: session.name, status: session.status, dashboardId: session.dashboard_id };
}
const hist: HistorySession | undefined = history[id];
if (hist) {
return { id, name: hist.name, status: hist.status, dashboardId: hist.dashboard_id };
}
return null;
})
.filter((a): a is TrackedAgent => a !== null);
const trackedIdSet = new Set(trackedIds);
for (const g of groups) {
if (!trackedIdSet.has(g.sessionId)) {
const session = sessions[g.sessionId];
if (session && session.status !== 'draft') {
agents.push({ id: g.sessionId, name: session.name, status: session.status, dashboardId: session.dashboard_id });
}
}
}
return agents;
}, [trackedIds, sessions, history, groups]);
const activeAgents = useMemo(
() => trackedAgents.filter((a) => a.status === 'running' || a.status === 'waiting_approval'),
[trackedAgents],
);
const finishedAgents = useMemo(
() => trackedAgents.filter((a) => a.status !== 'running' && a.status !== 'waiting_approval'),
[trackedAgents],
);
const hasApprovals = totalApprovals > 0;
const hasAgents = trackedAgents.length > 0;
const hasOnlyQuestionApprovals = useMemo(() => {
if (!hasApprovals) return false;
const allApprovals = groups.flatMap((g) => g.approvals);
return allApprovals.every((a) => a.tool_name === 'AskUserQuestion');
}, [hasApprovals, groups]);
const nonQuestionApprovalCount = useMemo(
() => groups.reduce((sum, g) => sum + g.approvals.filter((a) => a.tool_name !== 'AskUserQuestion').length, 0),
[groups],
);
const oldestNonQuestionApproval = useMemo(() => {
const all = groups
.flatMap((g) => g.approvals)
.filter((a) => a.tool_name !== 'AskUserQuestion');
if (all.length === 0) return null;
return all.reduce((oldest, a) =>
a.created_at < oldest.created_at ? a : oldest,
);
}, [groups]);
// ---- Island state machine ----
const islandState: IslandState = useMemo(() => {
if (userExpanded && (hasAgents || hasApprovals)) return 'expanded';
if (hasApprovals && hasOnlyQuestionApprovals) return 'expanded';
if (hasApprovals) return 'compact-actionable';
if (hasAgents) return 'compact';
return 'idle';
}, [hasApprovals, hasOnlyQuestionApprovals, userExpanded, hasAgents]);
useEffect(() => {
if (!hasAgents && !hasApprovals) {
setUserExpanded(false);
}
}, [hasAgents, hasApprovals]);
// ---- Click outside to collapse ----
useEffect(() => {
if (islandState !== 'expanded') return;
const handler = (e: MouseEvent) => {
if (islandRef.current && !islandRef.current.contains(e.target as Node)) {
setUserExpanded(false);
}
};
document.addEventListener('mousedown', handler);
return () => document.removeEventListener('mousedown', handler);
}, [islandState]);
// ---- Callbacks ----
const onApprove = useCallback(
(requestId: string, updatedInput?: Record<string, any>) => {
dispatch(handleApproval({ requestId, behavior: 'allow', updatedInput }));
},
[dispatch],
);
const onDeny = useCallback(
(requestId: string, message?: string) => {
dispatch(handleApproval({ requestId, behavior: 'deny', message }));
},
[dispatch],
);
const onStopAgent = useCallback(
(sessionId: string) => dispatch(stopAgent({ sessionId })),
[dispatch],
);
const onDismissAgent = useCallback(
(sessionId: string) => dispatch(dismissAgentNotification(sessionId)),
[dispatch],
);
const onNavigateToDashboard = useCallback(
(dashboardId: string, agentId: string) => {
dispatch(setPendingFocusAgentId(agentId));
navigate(`/dashboard/${dashboardId}`);
},
[navigate, dispatch],
);
const onApproveAllNonQuestion = useCallback(() => {
for (const g of groups) {
for (const req of g.approvals) {
if (req.tool_name !== 'AskUserQuestion') {
dispatch(handleApproval({ requestId: req.id, behavior: 'allow' }));
}
}
}
}, [dispatch, groups]);
const onDenyAllNonQuestion = useCallback(() => {
for (const g of groups) {
for (const req of g.approvals) {
if (req.tool_name !== 'AskUserQuestion') {
dispatch(handleApproval({ requestId: req.id, behavior: 'deny' }));
}
}
}
}, [dispatch, groups]);
const onClearAllFinished = useCallback(() => {
dispatch(dismissAllFinishedNotifications());
}, [dispatch]);
const handleIslandClick = useCallback(() => {
if (islandState === 'compact' || islandState === 'compact-actionable') {
setUserExpanded(true);
} else if (islandState === 'expanded') {
setUserExpanded(false);
}
}, [islandState]);
// ---- Styling — uses the same neutral palette as the rest of the UI ----
const islandWidth = islandState === 'idle'
? 200
: islandState === 'compact'
? 210
: islandState === 'compact-actionable'
? 310
: 400;
const islandBorderRadius = islandState === 'expanded' ? 14 : 50;
const shadow = islandState === 'idle'
? 'none'
: islandState === 'compact'
? c.shadow.sm
: c.shadow.md;
// ---- Compact summary text ----
const compactText = useMemo(() => {
const parts: string[] = [];
if (activeAgents.length > 0) {
parts.push(`${activeAgents.length} running`);
}
if (finishedAgents.length > 0) {
parts.push(`${finishedAgents.length} done`);
}
return parts.join(' · ') || 'Agents';
}, [activeAgents.length, finishedAgents.length]);
const glowKeyframes = useMemo(() => `
@keyframes approvalGlow {
0%, 100% { box-shadow: 0 0 6px 1px ${c.status.warning}30; }
50% { box-shadow: 0 0 12px 3px ${c.status.warning}60; }
}
`, [c.status.warning]);
// ---- Render ----
return (
<>
{islandState === 'compact-actionable' && <style>{glowKeyframes}</style>}
<motion.div
ref={islandRef}
layout
transition={islandState === 'expanded' ? SPRING_LAYOUT : SPRING_BOUNCE}
style={{
position: 'absolute',
left: '50%',
top: 6,
x: '-50%',
zIndex: 9999,
width: islandWidth,
borderRadius: islandBorderRadius,
cursor: islandState === 'expanded' ? 'default' : 'pointer',
// @ts-expect-error -- vendor prefix
WebkitAppRegion: 'no-drag',
}}
onClick={islandState !== 'expanded' && islandState !== 'compact-actionable' ? handleIslandClick : undefined}
>
<motion.div
layout
transition={SPRING_LAYOUT}
style={{
background: c.bg.secondary,
border: islandState === 'compact-actionable'
? `1px solid ${c.status.warning}`
: `0.5px solid ${c.border.medium}`,
borderRadius: islandBorderRadius,
boxShadow: islandState === 'compact-actionable'
? `0 0 8px 1px ${c.status.warning}40`
: shadow,
overflow: 'hidden',
animation: islandState === 'compact-actionable'
? 'approvalGlow 2.5s ease-in-out infinite'
: 'none',
}}
>
<AnimatePresence mode="wait">
{islandState === 'idle' && (
<IdlePill key="idle" c={c} />
)}
{islandState === 'compact' && (
<CompactPill
key="compact"
c={c}
text={compactText}
activeCount={activeAgents.length}
hasApprovals={hasApprovals}
/>
)}
{islandState === 'compact-actionable' && oldestNonQuestionApproval && (
<CompactActionablePill
key="compact-actionable"
c={c}
request={oldestNonQuestionApproval}
remainingCount={nonQuestionApprovalCount}
onApprove={onApprove}
onDeny={onDeny}
onExpand={() => setUserExpanded(true)}
/>
)}
{islandState === 'expanded' && (
<ExpandedCard
key="expanded"
c={c}
groups={groups}
totalApprovals={totalApprovals}
activeAgents={activeAgents}
finishedAgents={finishedAgents}
hasApprovals={hasApprovals}
hasAgents={hasAgents}
onApprove={onApprove}
onDeny={onDeny}
onStopAgent={onStopAgent}
onDismissAgent={onDismissAgent}
onNavigateToDashboard={onNavigateToDashboard}
onClearAllFinished={onClearAllFinished}
onCollapse={() => setUserExpanded(false)}
/>
)}
</AnimatePresence>
</motion.div>
</motion.div>
</>
);
};
// ---------------------------------------------------------------------------
// Idle pill — disabled search bar
// ---------------------------------------------------------------------------
const IdlePill: React.FC<{ c: ReturnType<typeof useClaudeTokens> }> = ({ c }) => (
<motion.div
initial={{ opacity: 0, scale: 0.92 }}
animate={{ opacity: 1, scale: 1 }}
exit={{ opacity: 0, scale: 0.92 }}
transition={{ duration: 0.2 }}
>
<Tooltip title="Coming soon" arrow placement="bottom">
<Box
sx={{
display: 'flex',
alignItems: 'center',
gap: 0.75,
px: 1.25,
height: 24,
userSelect: 'none',
cursor: 'default',
}}
>
<SearchIcon sx={{ fontSize: 13, color: c.text.ghost, flexShrink: 0 }} />
<Typography
sx={{
color: c.text.ghost,
fontSize: '0.66rem',
fontWeight: 400,
lineHeight: 1,
whiteSpace: 'nowrap',
}}
>
Search...
</Typography>
</Box>
</Tooltip>
</motion.div>
);
// ---------------------------------------------------------------------------
// Compact pill
// ---------------------------------------------------------------------------
const CompactPill: React.FC<{
c: ReturnType<typeof useClaudeTokens>;
text: string;
activeCount: number;
hasApprovals: boolean;
}> = ({ c, text, activeCount, hasApprovals }) => (
<motion.div
initial={{ opacity: 0, scale: 0.92 }}
animate={{ opacity: 1, scale: 1 }}
exit={{ opacity: 0, scale: 0.92 }}
transition={SPRING_BOUNCE}
>
<Box
sx={{
display: 'flex',
alignItems: 'center',
gap: 0.75,
px: 1.5,
height: 24,
userSelect: 'none',
}}
>
<ActivityIndicator c={c} />
<Typography
sx={{
fontSize: '0.68rem',
fontWeight: 500,
color: c.text.tertiary,
flex: 1,
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
}}
>
{text}
</Typography>
{hasApprovals && (
<Box
sx={{
width: 4,
height: 4,
borderRadius: '50%',
bgcolor: c.accent.primary,
flexShrink: 0,
opacity: 0.8,
}}
/>
)}
</Box>
</motion.div>
);
// ---------------------------------------------------------------------------
// Compact-actionable pill — single approval with icon + name + approve/deny
// ---------------------------------------------------------------------------
const CompactActionablePill: React.FC<{
c: ReturnType<typeof useClaudeTokens>;
request: ApprovalRequest;
remainingCount: number;
onApprove: (requestId: string) => void;
onDeny: (requestId: string) => void;
onExpand: () => void;
}> = ({ c, request, remainingCount, onApprove, onDeny, onExpand }) => {
const parsed = useMemo(() => parseMcpToolName(request.tool_name), [request.tool_name]);
const meta = useMcpToolMeta(parsed);
const icon = parsed.isMcp
? (meta.integration?.icon || null)
: getToolIcon(request.tool_name);
return (
<motion.div
initial={{ opacity: 0, scale: 0.92 }}
animate={{ opacity: 1, scale: 1 }}
exit={{ opacity: 0, scale: 0.92 }}
transition={SPRING_BOUNCE}
>
<Box
sx={{
display: 'flex',
alignItems: 'center',
gap: 0.75,
px: 0.5,
height: 24,
userSelect: 'none',
}}
>
<Box
sx={{
width: 16,
height: 16,
borderRadius: 1,
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
flexShrink: 0,
color: c.text.tertiary,
'& svg': { width: 12, height: 12 },
}}
>
{icon}
</Box>
<Typography
sx={{
fontSize: '0.68rem',
fontWeight: 600,
color: c.text.secondary,
flex: 1,
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
minWidth: 0,
}}
>
{parsed.displayName}
</Typography>
{remainingCount > 1 && (
<Typography
sx={{
fontSize: '0.6rem',
fontWeight: 600,
color: c.text.ghost,
flexShrink: 0,
}}
>
+{remainingCount - 1}
</Typography>
)}
<Tooltip title="Approve" arrow>
<IconButton
size="small"
onClick={(e) => { e.stopPropagation(); onApprove(request.id); }}
sx={{
p: 0,
width: 18,
height: 18,
color: '#fff',
bgcolor: c.status.success,
'&:hover': { bgcolor: c.status.success, filter: 'brightness(0.85)' },
}}
>
<CheckIcon sx={{ fontSize: 11 }} />
</IconButton>
</Tooltip>
<Tooltip title="Deny" arrow>
<IconButton
size="small"
onClick={(e) => { e.stopPropagation(); onDeny(request.id); }}
sx={{
p: 0,
width: 18,
height: 18,
color: c.status.error,
border: `1px solid ${c.status.error}`,
'&:hover': { bgcolor: `${c.status.error}0a` },
}}
>
<CloseIcon sx={{ fontSize: 11 }} />
</IconButton>
</Tooltip>
<Tooltip title="Show details" arrow>
<IconButton
size="small"
onClick={(e) => { e.stopPropagation(); onExpand(); }}
sx={{ p: 0.25, color: c.text.ghost, '&:hover': { color: c.text.tertiary } }}
>
<ExpandMoreIcon sx={{ fontSize: 15 }} />
</IconButton>
</Tooltip>
</Box>
</motion.div>
);
};
// ---------------------------------------------------------------------------
// Expanded card
// ---------------------------------------------------------------------------
const ExpandedCard: React.FC<{
c: ReturnType<typeof useClaudeTokens>;
groups: SessionApprovalGroup[];
totalApprovals: number;
activeAgents: TrackedAgent[];
finishedAgents: TrackedAgent[];
hasApprovals: boolean;
hasAgents: boolean;
onApprove: (requestId: string, updatedInput?: Record<string, any>) => void;
onDeny: (requestId: string, message?: string) => void;
onStopAgent: (id: string) => void;
onDismissAgent: (id: string) => void;
onNavigateToDashboard: (dashboardId: string, agentId: string) => void;
onClearAllFinished: () => void;
onCollapse: () => void;
}> = ({
c, groups, totalApprovals,
activeAgents, finishedAgents, hasApprovals, hasAgents,
onApprove, onDeny, onStopAgent, onDismissAgent, onNavigateToDashboard, onClearAllFinished, onCollapse,
}) => {
const [completedExpanded, setCompletedExpanded] = useState(false);
const headerTitle = hasApprovals && !hasAgents
? 'Approval Required'
: hasAgents && !hasApprovals
? 'Agents'
: 'Notifications';
const badgeCount = totalApprovals + activeAgents.length;
return (
<motion.div
initial={{ opacity: 0, scale: 0.96 }}
animate={{ opacity: 1, scale: 1 }}
exit={{ opacity: 0, scale: 0.96 }}
transition={{ duration: 0.18 }}
>
{/* Header */}
<Box
onClick={!hasApprovals ? onCollapse : undefined}
sx={{
display: 'flex',
alignItems: 'center',
gap: 1,
px: 2,
py: 1,
cursor: hasApprovals ? 'default' : 'pointer',
userSelect: 'none',
borderBottom: `0.5px solid ${c.border.subtle}`,
'&:hover': !hasApprovals ? { bgcolor: c.border.subtle } : {},
transition: 'background-color 0.15s',
}}
>
<Typography
sx={{
fontSize: '0.76rem',
fontWeight: 600,
color: c.text.muted,
flex: 1,
}}
>
{headerTitle}
</Typography>
{badgeCount > 0 && (
<Typography
sx={{
fontSize: '0.65rem',
fontWeight: 600,
color: c.text.ghost,
flexShrink: 0,
}}
>
{badgeCount}
</Typography>
)}
{!hasApprovals && (
<IconButton
size="small"
onClick={(e) => { e.stopPropagation(); onCollapse(); }}
sx={{ p: 0.25, color: c.text.ghost, '&:hover': { color: c.text.tertiary } }}
>
<CloseIcon sx={{ fontSize: 13 }} />
</IconButton>
)}
</Box>
{/* Content */}
<Box
sx={{
overflow: 'auto',
maxHeight: 'min(420px, calc(100vh - 100px))',
'&::-webkit-scrollbar': { width: 4 },
'&::-webkit-scrollbar-track': { background: 'transparent' },
'&::-webkit-scrollbar-thumb': {
background: c.border.medium,
borderRadius: 3,
'&:hover': { background: c.border.strong },
},
scrollbarWidth: 'thin',
scrollbarColor: `${c.border.medium} transparent`,
}}
>
{hasApprovals && (
<Box sx={{ py: 1 }}>
{hasAgents && (
<Typography
sx={{
fontSize: '0.58rem',
fontWeight: 600,
color: c.text.ghost,
textTransform: 'uppercase',
letterSpacing: '0.06em',
px: 2,
pb: 0.5,
}}
>
Approvals
</Typography>
)}
{groups.map((group) => (
<Box key={group.sessionId} sx={{ mb: 1, '&:last-child': { mb: 0 } }}>
{groups.length > 1 && (
<Typography
sx={{
fontSize: '0.65rem',
fontWeight: 600,
color: c.text.ghost,
textTransform: 'uppercase',
letterSpacing: '0.04em',
px: 2,
py: 0.5,
}}
>
{group.sessionName}
</Typography>
)}
{group.approvals.length > 1 ? (
<BatchApprovalBar
requests={group.approvals}
onApprove={onApprove}
onDeny={onDeny}
/>
) : (
group.approvals.map((req) => (
<ApprovalBar
key={req.id}
request={req}
onApprove={onApprove}
onDeny={onDeny}
/>
))
)}
</Box>
))}
</Box>
)}
{hasApprovals && hasAgents && (
<Box sx={{ mx: 2, borderTop: `0.5px solid ${c.border.subtle}` }} />
)}
{hasAgents && (
<Box sx={{ py: 0.75 }}>
{hasApprovals && (
<Typography
sx={{
fontSize: '0.58rem',
fontWeight: 600,
color: c.text.ghost,
textTransform: 'uppercase',
letterSpacing: '0.06em',
px: 2,
pb: 0.5,
pt: 0.25,
}}
>
Agents
</Typography>
)}
{activeAgents.map((agent) => (
<AgentStatusRow
key={agent.id}
agent={agent}
c={c}
onStop={onStopAgent}
onDismiss={onDismissAgent}
onNavigate={onNavigateToDashboard}
/>
))}
{finishedAgents.length > 0 && (
<>
{activeAgents.length > 0 && (
<Box sx={{ mx: 2, my: 0.5, borderTop: `0.5px solid ${c.border.subtle}` }} />
)}
<Box
onClick={() => setCompletedExpanded((v) => !v)}
sx={{
display: 'flex',
alignItems: 'center',
gap: 1,
px: 2,
py: 0.5,
cursor: 'pointer',
userSelect: 'none',
'&:hover': { bgcolor: c.border.subtle },
transition: 'background-color 0.15s',
}}
>
<Typography
sx={{
fontSize: '0.58rem',
fontWeight: 600,
color: c.text.ghost,
textTransform: 'uppercase',
letterSpacing: '0.06em',
flex: 1,
}}
>
Completed ({finishedAgents.length})
</Typography>
<Typography
component="span"
onClick={(e: React.MouseEvent) => { e.stopPropagation(); onClearAllFinished(); }}
sx={{
fontSize: '0.58rem',
fontWeight: 600,
color: c.text.ghost,
cursor: 'pointer',
'&:hover': { color: c.text.secondary },
transition: 'color 0.15s',
}}
>
Clear all
</Typography>
<IconButton size="small" sx={{ p: 0, color: c.text.ghost }}>
{completedExpanded
? <ExpandLessIcon sx={{ fontSize: 14 }} />
: <ExpandMoreIcon sx={{ fontSize: 14 }} />}
</IconButton>
</Box>
<Collapse in={completedExpanded}>
{finishedAgents.map((agent) => (
<AgentStatusRow
key={agent.id}
agent={agent}
c={c}
onStop={onStopAgent}
onDismiss={onDismissAgent}
onNavigate={onNavigateToDashboard}
/>
))}
</Collapse>
</>
)}
</Box>
)}
</Box>
</motion.div>
);
};
export default DynamicIsland;
@@ -1,4 +1,4 @@
import React, { createContext, useContext, useState, useRef, useCallback, RefObject } from 'react';
import React, { createContext, useContext, useState, useRef, useCallback, useMemo, RefObject } from 'react';
export interface SelectedElement {
id: string;
@@ -20,11 +20,17 @@ interface ElementSelectionContextValue {
setSelectMode: (active: boolean) => void;
excludeSelectId: string | null;
setExcludeSelectId: (id: string | null) => void;
activeOwnerId: string | null;
setActiveOwnerId: (id: string | null) => void;
selectedElements: SelectedElement[];
addSelectedElement: (el: SelectedElement) => void;
updateSelectedElement: (id: string, patch: Partial<SelectedElement>) => void;
removeSelectedElement: (id: string) => void;
clearSelectedElements: () => void;
elementsByOwner: Record<string, SelectedElement[]>;
addElementForOwner: (ownerId: string, el: SelectedElement) => void;
removeOwnerElement: (ownerId: string, elementId: string) => void;
clearOwnerElements: (ownerId: string) => void;
iframeRef: RefObject<HTMLIFrameElement | null>;
}
@@ -37,9 +43,18 @@ export function useElementSelection() {
export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
const [selectMode, setSelectMode] = useState(false);
const [excludeSelectId, setExcludeSelectId] = useState<string | null>(null);
const [selectedElements, setSelectedElements] = useState<SelectedElement[]>([]);
const [activeOwnerId, setActiveOwnerId] = useState<string | null>(null);
const [elementsByOwner, setElementsByOwner] = useState<Record<string, SelectedElement[]>>({});
const iframeRef = useRef<HTMLIFrameElement | null>(null);
const activeOwnerIdRef = useRef(activeOwnerId);
activeOwnerIdRef.current = activeOwnerId;
const selectedElements = useMemo(
() => (activeOwnerId ? elementsByOwner[activeOwnerId] ?? [] : []),
[activeOwnerId, elementsByOwner],
);
const toggleSelectMode = useCallback(() => {
setSelectMode((prev) => {
if (prev) setExcludeSelectId(null);
@@ -48,22 +63,65 @@ export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> =
}, []);
const addSelectedElement = useCallback((el: SelectedElement) => {
setSelectedElements((prev) => {
if (prev.some((e) => e.id === el.id)) return prev;
return [...prev, el];
const ownerId = activeOwnerIdRef.current;
if (!ownerId) return;
setElementsByOwner((prev) => {
const existing = prev[ownerId] ?? [];
if (existing.some((e) => e.id === el.id)) return prev;
return { ...prev, [ownerId]: [...existing, el] };
});
}, []);
const updateSelectedElement = useCallback((id: string, patch: Partial<SelectedElement>) => {
setSelectedElements((prev) => prev.map((e) => e.id === id ? { ...e, ...patch } : e));
const ownerId = activeOwnerIdRef.current;
if (!ownerId) return;
setElementsByOwner((prev) => {
const existing = prev[ownerId];
if (!existing) return prev;
return { ...prev, [ownerId]: existing.map((e) => (e.id === id ? { ...e, ...patch } : e)) };
});
}, []);
const removeSelectedElement = useCallback((id: string) => {
setSelectedElements((prev) => prev.filter((e) => e.id !== id));
const ownerId = activeOwnerIdRef.current;
if (!ownerId) return;
setElementsByOwner((prev) => {
const existing = prev[ownerId];
if (!existing) return prev;
return { ...prev, [ownerId]: existing.filter((e) => e.id !== id) };
});
}, []);
const clearSelectedElements = useCallback(() => {
setSelectedElements([]);
const ownerId = activeOwnerIdRef.current;
if (!ownerId) return;
setElementsByOwner((prev) => {
if (!prev[ownerId]?.length) return prev;
return { ...prev, [ownerId]: [] };
});
}, []);
const addElementForOwner = useCallback((ownerId: string, el: SelectedElement) => {
setElementsByOwner((prev) => {
const existing = prev[ownerId] ?? [];
if (existing.some((e) => e.semanticData?.selectId === el.semanticData?.selectId)) return prev;
return { ...prev, [ownerId]: [...existing, el] };
});
}, []);
const removeOwnerElement = useCallback((ownerId: string, elementId: string) => {
setElementsByOwner((prev) => {
const existing = prev[ownerId];
if (!existing) return prev;
return { ...prev, [ownerId]: existing.filter((e) => e.id !== elementId) };
});
}, []);
const clearOwnerElements = useCallback((ownerId: string) => {
setElementsByOwner((prev) => {
if (!prev[ownerId]?.length) return prev;
return { ...prev, [ownerId]: [] };
});
}, []);
return (
@@ -74,11 +132,17 @@ export const ElementSelectionProvider: React.FC<{ children: React.ReactNode }> =
setSelectMode,
excludeSelectId,
setExcludeSelectId,
activeOwnerId,
setActiveOwnerId,
selectedElements,
addSelectedElement,
updateSelectedElement,
removeSelectedElement,
clearSelectedElements,
elementsByOwner,
addElementForOwner,
removeOwnerElement,
clearOwnerElements,
iframeRef,
}}
>
@@ -1,203 +0,0 @@
import React, { useMemo, useCallback, useState, useEffect } from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import Chip from '@mui/material/Chip';
import IconButton from '@mui/material/IconButton';
import ExpandLessIcon from '@mui/icons-material/ExpandLess';
import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
import NotificationsActiveIcon from '@mui/icons-material/NotificationsActive';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import { handleApproval, ApprovalRequest } from '@/shared/state/agentsSlice';
import ApprovalBar, { BatchApprovalBar } from '@/app/pages/AgentChat/ApprovalBar';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
interface SessionApprovalGroup {
sessionId: string;
sessionName: string;
approvals: ApprovalRequest[];
}
const GlobalApprovalOverlay: React.FC = () => {
const c = useClaudeTokens();
const dispatch = useAppDispatch();
const sessions = useAppSelector((state) => state.agents.sessions);
const [collapsed, setCollapsed] = useState(false);
const groups: SessionApprovalGroup[] = useMemo(() => {
const result: SessionApprovalGroup[] = [];
for (const [sessionId, session] of Object.entries(sessions)) {
if (session.pending_approvals.length > 0) {
result.push({
sessionId,
sessionName: session.name || 'Agent',
approvals: session.pending_approvals,
});
}
}
return result;
}, [sessions]);
const totalApprovals = useMemo(
() => groups.reduce((sum, g) => sum + g.approvals.length, 0),
[groups],
);
useEffect(() => {
if (totalApprovals > 0) {
setCollapsed(false);
}
}, [totalApprovals]);
const onApprove = useCallback(
(requestId: string, updatedInput?: Record<string, any>) => {
dispatch(handleApproval({ requestId, behavior: 'allow', updatedInput }));
},
[dispatch],
);
const onDeny = useCallback(
(requestId: string, message?: string) => {
dispatch(handleApproval({ requestId, behavior: 'deny', message }));
},
[dispatch],
);
if (totalApprovals === 0) return null;
return (
<Box
sx={{
position: 'fixed',
top: 16,
right: 16,
zIndex: 9999,
width: collapsed ? 'auto' : 420,
maxWidth: 'calc(100vw - 280px)',
maxHeight: 'calc(100vh - 32px)',
display: 'flex',
flexDirection: 'column',
borderRadius: `${c.radius.xl}px`,
bgcolor: c.bg.surface,
border: `1px solid ${c.status.warning}40`,
boxShadow: `0 8px 32px rgba(0,0,0,0.25), 0 0 0 1px ${c.status.warning}20`,
overflow: 'hidden',
animation: 'approvalSlideIn 0.25s ease-out',
'@keyframes approvalSlideIn': {
from: { opacity: 0, transform: 'translateY(-12px) scale(0.97)' },
to: { opacity: 1, transform: 'translateY(0) scale(1)' },
},
}}
>
{/* Header */}
<Box
onClick={() => setCollapsed((v) => !v)}
sx={{
display: 'flex',
alignItems: 'center',
gap: 1,
px: 2,
py: 1.25,
bgcolor: c.status.warningBg,
borderBottom: collapsed ? 'none' : `1px solid ${c.status.warning}20`,
cursor: 'pointer',
userSelect: 'none',
'&:hover': { bgcolor: `${c.status.warning}18` },
transition: 'background-color 0.15s',
}}
>
<NotificationsActiveIcon
sx={{
fontSize: 18,
color: c.status.warning,
animation: 'approvalBell 0.6s ease-in-out',
'@keyframes approvalBell': {
'0%': { transform: 'rotate(0)' },
'20%': { transform: 'rotate(12deg)' },
'40%': { transform: 'rotate(-10deg)' },
'60%': { transform: 'rotate(6deg)' },
'80%': { transform: 'rotate(-3deg)' },
'100%': { transform: 'rotate(0)' },
},
}}
/>
<Typography sx={{ fontSize: '0.85rem', fontWeight: 700, color: c.status.warning, flex: 1 }}>
Approval Required
</Typography>
<Chip
label={totalApprovals}
size="small"
sx={{
height: 22,
minWidth: 28,
fontSize: '0.75rem',
fontWeight: 700,
bgcolor: `${c.status.warning}20`,
color: c.status.warning,
border: 'none',
}}
/>
<IconButton size="small" sx={{ color: c.text.ghost, p: 0.25 }}>
{collapsed ? <ExpandMoreIcon sx={{ fontSize: 18 }} /> : <ExpandLessIcon sx={{ fontSize: 18 }} />}
</IconButton>
</Box>
{/* Content */}
{!collapsed && (
<Box
sx={{
overflow: 'auto',
py: 1,
maxHeight: 'calc(100vh - 120px)',
'&::-webkit-scrollbar': { width: 5 },
'&::-webkit-scrollbar-track': { background: 'transparent' },
'&::-webkit-scrollbar-thumb': {
background: c.border.medium,
borderRadius: 3,
'&:hover': { background: c.border.strong },
},
scrollbarWidth: 'thin',
scrollbarColor: `${c.border.medium} transparent`,
}}
>
{groups.map((group) => (
<Box key={group.sessionId} sx={{ mb: 1, '&:last-child': { mb: 0 } }}>
{groups.length > 1 && (
<Typography
sx={{
fontSize: '0.7rem',
fontWeight: 600,
color: c.text.muted,
textTransform: 'uppercase',
letterSpacing: '0.04em',
px: 2,
py: 0.5,
}}
>
{group.sessionName}
</Typography>
)}
{group.approvals.length > 1 ? (
<BatchApprovalBar
requests={group.approvals}
onApprove={onApprove}
onDeny={onDeny}
/>
) : (
group.approvals.map((req) => (
<ApprovalBar
key={req.id}
request={req}
onApprove={onApprove}
onDeny={onDeny}
/>
))
)}
</Box>
))}
</Box>
)}
</Box>
);
};
export default GlobalApprovalOverlay;
@@ -11,6 +11,7 @@ const shortcuts = [
{ key: 't', description: 'Go to Templates' },
{ key: '1-9', description: 'Open agent by position' },
{ key: '⌘M', description: 'Add App' },
{ key: '⌘N', description: 'New Browser' },
{ key: '⌘O', description: 'History' },
{ key: 'Shift+A', description: 'Approve all pending' },
{ key: 'Shift+D', description: 'Deny all pending' },
+247 -27
View File
@@ -27,17 +27,24 @@ import ViewSidebarOutlinedIcon from '@mui/icons-material/ViewSidebarOutlined';
import ArrowBackOutlinedIcon from '@mui/icons-material/ArrowBackOutlined';
import ArrowForwardOutlinedIcon from '@mui/icons-material/ArrowForwardOutlined';
import RestartAltIcon from '@mui/icons-material/RestartAlt';
import SystemUpdateAltIcon from '@mui/icons-material/SystemUpdateAlt';
import CloseIcon from '@mui/icons-material/Close';
import LinearProgress from '@mui/material/LinearProgress';
import Settings from '@/app/pages/Settings/Settings';
import GlobalApprovalOverlay from '@/app/components/GlobalApprovalOverlay';
import DynamicIsland from '@/app/components/DynamicIsland';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import { fetchDashboards, createDashboard, renameDashboard } from '@/shared/state/dashboardsSlice';
import { addBrowserCard, addBrowserTab } from '@/shared/state/dashboardLayoutSlice';
import { setPendingBrowserUrl } from '@/shared/state/tempStateSlice';
import { fetchOutputs } from '@/shared/state/outputsSlice';
import { findBrowserByWebContentsId } from '@/shared/browserRegistry';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
const SIDEBAR_MIN = 160;
const SIDEBAR_MAX = 400;
const SIDEBAR_DEFAULT = 220;
const SIDEBAR_WIDTH_KEY = 'openswarm-sidebar-width';
const UPDATE_DISMISS_KEY = 'openswarm-update-dismissed';
const CUSTOMIZATION_ITEMS = [
{ label: 'Prompts', path: '/templates', icon: <DescriptionIcon /> },
@@ -75,10 +82,34 @@ const AppShell: React.FC = () => {
const updateStatus = useAppSelector((state) => state.update.status);
const availableVersion = useAppSelector((state) => state.update.availableVersion);
const [updateBannerDismissed, setUpdateBannerDismissed] = useState(false);
const downloadPercent = useAppSelector((state) => state.update.downloadPercent);
const showUpdateDot = updateStatus === 'available' || updateStatus === 'downloaded';
const showUpdateBanner = updateStatus === 'downloaded' && !updateBannerDismissed;
const [dismissedVersion, setDismissedVersion] = useState<string | null>(() => {
try { return localStorage.getItem(UPDATE_DISMISS_KEY); } catch { return null; }
});
const [snackbarDismissed, setSnackbarDismissed] = useState(false);
const bannerDismissedForVersion = availableVersion != null && dismissedVersion === availableVersion;
const isUpdateActionable = updateStatus === 'available' || updateStatus === 'downloaded' || updateStatus === 'downloading';
const showUpdateDot = (updateStatus === 'available' || updateStatus === 'downloaded') && !bannerDismissedForVersion;
const showUpdateBanner = isUpdateActionable && !bannerDismissedForVersion;
const showUpdateSnackbar = (updateStatus === 'available' || updateStatus === 'downloaded') && !bannerDismissedForVersion && !snackbarDismissed;
const handleDismissBanner = useCallback(() => {
if (availableVersion) {
try { localStorage.setItem(UPDATE_DISMISS_KEY, availableVersion); } catch {}
setDismissedVersion(availableVersion);
}
}, [availableVersion]);
const handleDownloadUpdate = useCallback(async () => {
try { await (window as any).openswarm?.downloadUpdate(); } catch {}
}, []);
const handleInstallUpdate = useCallback(() => {
(window as any).openswarm?.installUpdate();
}, []);
const dashboardItems = useAppSelector((state) => state.dashboards.items);
const dashboardList = Object.values(dashboardItems).sort(
@@ -95,6 +126,75 @@ const AppShell: React.FC = () => {
dispatch(fetchOutputs());
}, [dispatch]);
const openUrlInBrowser = useCallback((url: string, webContentsId?: number) => {
const dashMatch = location.pathname.match(/^\/dashboard\/(.+)/);
if (dashMatch) {
if (webContentsId != null) {
const browserId = findBrowserByWebContentsId(webContentsId);
if (browserId) {
dispatch(addBrowserTab({ browserId, url, makeActive: true }));
return;
}
}
dispatch(addBrowserCard({ url }));
} else {
dispatch(setPendingBrowserUrl(url));
const lastId = (window as any).__openswarm_last_dashboard_id as string | undefined;
const firstDashboard = dashboardList[0];
const targetId = lastId || firstDashboard?.id;
if (targetId) {
navigate(`/dashboard/${targetId}`);
} else {
dispatch(createDashboard('Untitled Dashboard')).then((result: any) => {
if (createDashboard.fulfilled.match(result)) {
navigate(`/dashboard/${result.payload.id}`);
}
});
}
}
}, [location.pathname, dashboardList, dispatch, navigate]);
useEffect(() => {
let lastUrl = '';
let lastTime = 0;
const handleClick = (e: MouseEvent) => {
const anchor = (e.target as HTMLElement)?.closest?.('a');
if (!anchor) return;
const href = anchor.getAttribute('href');
if (!href) return;
if (!/^https?:\/\//i.test(href)) return;
if (href.startsWith('http://localhost:')) return;
e.preventDefault();
e.stopPropagation();
const now = Date.now();
if (href === lastUrl && now - lastTime < 1000) return;
lastUrl = href;
lastTime = now;
openUrlInBrowser(href);
};
document.addEventListener('click', handleClick, true);
return () => document.removeEventListener('click', handleClick, true);
}, [openUrlInBrowser]);
useEffect(() => {
const w = window as any;
if (!w.openswarm?.onWebviewNewWindow) return;
let lastUrl = '';
let lastTime = 0;
return w.openswarm.onWebviewNewWindow((url: string, webContentsId: number) => {
const now = Date.now();
if (url === lastUrl && now - lastTime < 1000) return;
lastUrl = url;
lastTime = now;
openUrlInBrowser(url, webContentsId);
});
}, [openUrlInBrowser]);
useEffect(() => {
try { localStorage.setItem(SIDEBAR_WIDTH_KEY, String(sidebarWidth)); } catch {}
}, [sidebarWidth]);
@@ -196,6 +296,8 @@ const AppShell: React.FC = () => {
borderBottom: `0.5px solid ${c.border.medium}`,
display: 'flex',
alignItems: 'center',
position: 'relative',
overflow: 'visible',
WebkitAppRegion: 'drag',
userSelect: 'none',
pl: '78px',
@@ -248,6 +350,8 @@ const AppShell: React.FC = () => {
</IconButton>
</Tooltip>
<DynamicIsland />
<Box sx={{ flex: 1 }} />
<Box
@@ -263,12 +367,12 @@ const AppShell: React.FC = () => {
component="img"
src="./logo.png"
alt="OpenSwarm"
sx={{ width: 18, height: 18, borderRadius: 0.5, opacity: 0.7 }}
sx={{ width: 16, height: 16, borderRadius: 0.5, opacity: 0.6 }}
/>
<Typography
sx={{
color: c.text.tertiary,
fontSize: '0.75rem',
fontSize: '0.72rem',
fontWeight: 500,
letterSpacing: 0.3,
lineHeight: 1,
@@ -279,6 +383,98 @@ const AppShell: React.FC = () => {
</Box>
</Box>
{showUpdateBanner && (
<Box
sx={{
display: 'flex',
alignItems: 'center',
gap: 1.5,
px: 2,
py: 0.5,
bgcolor: `${c.accent.primary}14`,
borderBottom: `1px solid ${c.accent.primary}30`,
flexShrink: 0,
}}
>
<SystemUpdateAltIcon sx={{ fontSize: 16, color: c.accent.primary, flexShrink: 0 }} />
<Typography sx={{ fontSize: '0.8rem', color: c.text.secondary, flex: 1, whiteSpace: 'nowrap', overflow: 'hidden', textOverflow: 'ellipsis' }}>
{updateStatus === 'available' && `OpenSwarm ${availableVersion} is available`}
{updateStatus === 'downloading' && `Downloading OpenSwarm ${availableVersion}`}
{updateStatus === 'downloaded' && `OpenSwarm ${availableVersion} is ready to install`}
</Typography>
{updateStatus === 'downloading' && (
<LinearProgress
variant="determinate"
value={downloadPercent}
sx={{
width: 120,
height: 3,
flexShrink: 0,
borderRadius: 2,
bgcolor: `${c.accent.primary}20`,
'& .MuiLinearProgress-bar': { bgcolor: c.accent.primary, borderRadius: 2 },
}}
/>
)}
{updateStatus === 'downloading' && (
<Typography sx={{ fontSize: '0.72rem', color: c.text.tertiary, flexShrink: 0 }}>
{Math.round(downloadPercent)}%
</Typography>
)}
{updateStatus === 'available' && (
<Button
size="small"
variant="contained"
onClick={handleDownloadUpdate}
sx={{
bgcolor: c.accent.primary,
'&:hover': { bgcolor: c.accent.pressed },
textTransform: 'none',
fontSize: '0.75rem',
fontWeight: 600,
borderRadius: 1.5,
minWidth: 'auto',
py: 0.25,
px: 1.5,
lineHeight: 1.5,
flexShrink: 0,
}}
>
Download
</Button>
)}
{updateStatus === 'downloaded' && (
<Button
size="small"
variant="contained"
onClick={handleInstallUpdate}
sx={{
bgcolor: c.accent.primary,
'&:hover': { bgcolor: c.accent.pressed },
textTransform: 'none',
fontSize: '0.75rem',
fontWeight: 600,
borderRadius: 1.5,
minWidth: 'auto',
py: 0.25,
px: 1.5,
lineHeight: 1.5,
flexShrink: 0,
}}
>
Restart & Update
</Button>
)}
<IconButton
size="small"
onClick={handleDismissBanner}
sx={{ color: c.text.tertiary, p: 0.25, flexShrink: 0, '&:hover': { color: c.text.secondary } }}
>
<CloseIcon sx={{ fontSize: 14 }} />
</IconButton>
</Box>
)}
<Box sx={{ display: 'flex', flex: 1, minHeight: 0 }}>
{!sidebarCollapsed && (
<>
@@ -732,39 +928,62 @@ const AppShell: React.FC = () => {
</Box>
<Settings />
<GlobalApprovalOverlay />
<Snackbar
open={showUpdateBanner}
open={showUpdateSnackbar}
autoHideDuration={10000}
onClose={() => setSnackbarDismissed(true)}
anchorOrigin={{ vertical: 'bottom', horizontal: 'center' }}
>
<Alert
severity="info"
icon={<RestartAltIcon sx={{ fontSize: 18 }} />}
icon={updateStatus === 'downloaded'
? <RestartAltIcon sx={{ fontSize: 18 }} />
: <SystemUpdateAltIcon sx={{ fontSize: 18 }} />
}
action={
<Box sx={{ display: 'flex', gap: 1, alignItems: 'center' }}>
<Button
size="small"
onClick={() => setUpdateBannerDismissed(true)}
onClick={() => setSnackbarDismissed(true)}
sx={{ color: c.text.muted, textTransform: 'none', fontSize: '0.8rem', minWidth: 'auto' }}
>
Later
</Button>
<Button
size="small"
variant="contained"
onClick={() => (window as any).openswarm?.installUpdate()}
sx={{
bgcolor: c.accent.primary,
'&:hover': { bgcolor: c.accent.pressed },
textTransform: 'none',
fontSize: '0.8rem',
borderRadius: 1.5,
minWidth: 'auto',
}}
>
Restart
Dismiss
</Button>
{updateStatus === 'available' && (
<Button
size="small"
variant="contained"
onClick={handleDownloadUpdate}
sx={{
bgcolor: c.accent.primary,
'&:hover': { bgcolor: c.accent.pressed },
textTransform: 'none',
fontSize: '0.8rem',
borderRadius: 1.5,
minWidth: 'auto',
}}
>
Download
</Button>
)}
{updateStatus === 'downloaded' && (
<Button
size="small"
variant="contained"
onClick={handleInstallUpdate}
sx={{
bgcolor: c.accent.primary,
'&:hover': { bgcolor: c.accent.pressed },
textTransform: 'none',
fontSize: '0.8rem',
borderRadius: 1.5,
minWidth: 'auto',
}}
>
Restart & Update
</Button>
)}
</Box>
}
sx={{
@@ -775,7 +994,8 @@ const AppShell: React.FC = () => {
'& .MuiAlert-icon': { color: c.accent.primary },
}}
>
OpenSwarm {availableVersion} downloaded restart to update
{updateStatus === 'available' && `OpenSwarm ${availableVersion} is available`}
{updateStatus === 'downloaded' && `OpenSwarm ${availableVersion} downloaded — restart to update`}
</Alert>
</Snackbar>
</Box>
@@ -0,0 +1,222 @@
import React, { useState, useEffect } from 'react';
import { Box, Typography, Modal, Button } from '@mui/material';
import { useAppSelector } from '@/shared/hooks';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
import { API_BASE } from '@/shared/config';
const SUBSCRIPTION_PROVIDERS = [
{ id: 'claude', name: 'Claude', desc: 'Sonnet, Opus, Haiku', color: '#E8927A', preview: false },
{ id: 'gemini-cli', name: 'Gemini', desc: 'Gemini 2.5 Pro & Flash', color: '#4285F4', preview: true },
{ id: 'codex', name: 'ChatGPT', desc: 'GPT-5.4, o3, o4-mini', color: '#74AA9C', preview: true },
{ id: 'github', name: 'GitHub Copilot', desc: 'Claude + GPT models', color: '#8B949E', preview: true },
];
const OnboardingModal: React.FC = () => {
const c = useClaudeTokens();
const settings = useAppSelector((s) => s.settings);
const [open, setOpen] = useState(false);
const [dismissed, setDismissed] = useState(false);
const [connecting, setConnecting] = useState<string | null>(null);
const [nineRouterStatus, setNineRouterStatus] = useState<any>(null);
// Check if user has any credentials configured
const hasAnyKey = !!(
settings.anthropic_api_key ||
settings.openai_api_key ||
settings.google_api_key ||
settings.openrouter_api_key
);
// Check 9Router subscription status
useEffect(() => {
fetch(`${API_BASE}/agents/subscriptions/status`)
.then((r) => r.json())
.then(setNineRouterStatus)
.catch(() => setNineRouterStatus(null));
}, []);
const hasSubscription = (() => {
if (!nineRouterStatus?.running) return false;
const connections = nineRouterStatus?.providers?.connections || [];
return connections.some((p: any) => p.isActive);
})();
// Show modal if no keys AND no subscriptions AND not dismissed
useEffect(() => {
if (!hasAnyKey && !hasSubscription && !dismissed && nineRouterStatus !== null) {
setOpen(true);
} else {
setOpen(false);
}
}, [hasAnyKey, hasSubscription, dismissed, nineRouterStatus]);
const handleConnect = async (providerId: string) => {
setConnecting(providerId);
try {
const r = await fetch(`${API_BASE}/agents/subscriptions/connect`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ provider: providerId }),
});
const data = await r.json();
if (data.flow === 'device_code') {
const verifyUrl = data.verification_uri;
if (verifyUrl) window.open(verifyUrl, '_blank');
// Poll for completion
const timer = setInterval(async () => {
try {
const pr = await fetch(`${API_BASE}/agents/subscriptions/poll`, {
method: 'POST',
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ provider: providerId, device_code: data.device_code, code_verifier: data.code_verifier, extra_data: data.extra_data }),
});
const pd = await pr.json();
if (pd.success) {
clearInterval(timer);
setConnecting(null);
setOpen(false);
}
} catch {}
}, 5000);
setTimeout(() => { clearInterval(timer); setConnecting(null); }, 300000);
} else if (data.flow === 'authorization_code') {
// Open auth URL as popup — window.opener lets callback page postMessage back
const popup = window.open(data.auth_url, 'oauth_connect', 'width=600,height=700');
// Listen for postMessage from 9Router's callback page
// 9Router sends: { type: "oauth_callback", data: { code, state, ... } }
const msgHandler = async (event: MessageEvent) => {
const d = event.data;
const callbackData = d?.type === 'oauth_callback' ? d.data : d;
if (callbackData?.code) {
window.removeEventListener('message', msgHandler);
clearInterval(statusPoller);
if (popup && !popup.closed) popup.close();
try {
await fetch(`${API_BASE}/agents/subscriptions/exchange`, {
method: 'POST', headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({
provider: providerId, code: callbackData.code,
redirect_uri: data.redirect_uri, code_verifier: data.code_verifier,
state: callbackData.state || data.state,
}),
});
} catch {}
setConnecting(null);
setOpen(false);
}
};
window.addEventListener('message', msgHandler);
// Also poll status as fallback (in case postMessage doesn't work in Electron)
const statusPoller = setInterval(async () => {
try {
const sr = await fetch(`${API_BASE}/agents/subscriptions/status`);
const sd = await sr.json();
const conns = sd.providers?.connections || [];
if (conns.some((p: any) => p.provider === providerId && p.isActive)) {
clearInterval(statusPoller);
window.removeEventListener('message', msgHandler);
setConnecting(null);
setOpen(false);
}
} catch {}
}, 2000);
setTimeout(() => { clearInterval(statusPoller); window.removeEventListener('message', msgHandler); setConnecting(null); }, 300000);
}
} catch {
setConnecting(null);
}
};
const handleApiKey = () => {
setDismissed(true);
setOpen(false);
// User will manually go to Settings → Models to add API keys
};
const handleSkip = () => {
setDismissed(true);
setOpen(false);
};
if (!open) return null;
return (
<Modal open={open} onClose={handleSkip} sx={{ display: 'flex', alignItems: 'center', justifyContent: 'center' }}>
<Box sx={{
width: 480, maxWidth: '90vw', bgcolor: c.bg.surface, borderRadius: `${c.radius.xl}px`,
border: `1px solid ${c.border.subtle}`, p: 3.5, outline: 'none',
boxShadow: '0 20px 60px rgba(0,0,0,0.4)',
}}>
<Typography sx={{ fontSize: '1.3rem', fontWeight: 700, color: c.text.primary, mb: 0.5, textAlign: 'center' }}>
Welcome to OpenSwarm
</Typography>
<Typography sx={{ fontSize: '0.78rem', color: c.text.muted, mb: 3, textAlign: 'center' }}>
Connect an AI model to get started
</Typography>
{/* Subscription options */}
<Typography sx={{ fontSize: '0.65rem', fontWeight: 600, color: c.text.tertiary, textTransform: 'uppercase', letterSpacing: '0.08em', mb: 1 }}>
Use your existing subscription
</Typography>
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 0.75, mb: 2.5 }}>
{SUBSCRIPTION_PROVIDERS.map((p) => (
<Box
key={p.id}
onClick={() => !p.preview && !connecting && handleConnect(p.id)}
sx={{
display: 'flex', alignItems: 'center', justifyContent: 'space-between',
p: 1.5, borderRadius: `${c.radius.md}px`, border: `1px solid ${c.border.subtle}`,
cursor: p.preview ? 'default' : connecting ? 'wait' : 'pointer',
opacity: p.preview ? 0.5 : 1,
transition: 'border-color 0.15s, background 0.15s',
...(!p.preview && { '&:hover': { borderColor: c.border.medium, bgcolor: `${c.accent.primary}05` } }),
}}
>
<Box>
<Typography sx={{ fontSize: '0.82rem', fontWeight: 600, color: c.text.primary }}>{p.name}</Typography>
<Typography sx={{ fontSize: '0.65rem', color: c.text.muted }}>{p.desc}</Typography>
</Box>
<Typography sx={{ fontSize: '0.68rem', color: p.preview ? c.text.ghost : connecting === p.id ? c.accent.primary : c.text.tertiary, fontStyle: p.preview ? 'italic' : 'normal' }}>
{p.preview ? 'Coming soon' : connecting === p.id ? 'Connecting...' : 'Connect →'}
</Typography>
</Box>
))}
</Box>
{/* API key option */}
<Typography sx={{ fontSize: '0.65rem', fontWeight: 600, color: c.text.tertiary, textTransform: 'uppercase', letterSpacing: '0.08em', mb: 1 }}>
Or use an API key
</Typography>
<Box
onClick={handleApiKey}
sx={{
p: 1.5, borderRadius: `${c.radius.md}px`, border: `1px solid ${c.border.subtle}`,
cursor: 'pointer', mb: 2.5,
'&:hover': { borderColor: c.border.medium, bgcolor: `${c.accent.primary}05` },
}}
>
<Typography sx={{ fontSize: '0.78rem', color: c.text.primary }}>
I have an API key
</Typography>
<Typography sx={{ fontSize: '0.65rem', color: c.text.muted }}>
Go to Settings Models to enter your key
</Typography>
</Box>
{/* Skip */}
<Button
onClick={handleSkip}
fullWidth
sx={{ textTransform: 'none', fontSize: '0.72rem', color: c.text.ghost, '&:hover': { bgcolor: 'transparent', color: c.text.muted } }}
>
Skip for now
</Button>
</Box>
</Modal>
);
};
export default OnboardingModal;
@@ -0,0 +1,526 @@
import React, { useEffect, useRef, useState, useCallback } from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import IconButton from '@mui/material/IconButton';
import MicIcon from '@mui/icons-material/Mic';
import MicOffIcon from '@mui/icons-material/MicOff';
import VolumeUpIcon from '@mui/icons-material/VolumeUp';
import CloseIcon from '@mui/icons-material/Close';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
import { WS_BASE } from '@/shared/config';
type FaceState = 'idle' | 'happy' | 'thinking' | 'talking' | 'surprised' | 'sleeping' | 'angry' | 'love';
type TalkStatus = 'idle' | 'listening' | 'processing' | 'speaking';
interface Props {
open: boolean;
onClose: () => void;
sessionId?: string;
}
// ─── Pixel Face Canvas Renderer ──────────────────────────────────
// Ported from face.html — all the draw logic in one hook.
function usePixelFace(
canvasRef: React.RefObject<HTMLCanvasElement | null>,
faceState: FaceState,
size: number,
) {
const stateRef = useRef<FaceState>('idle');
const animRef = useRef<number>(0);
useEffect(() => {
stateRef.current = faceState;
}, [faceState]);
useEffect(() => {
const canvas = canvasRef.current;
if (!canvas) return;
const ctx = canvas.getContext('2d');
if (!ctx) return;
const PX = Math.max(4, Math.floor(size / 30));
const COLS = Math.ceil(size / PX);
const ROWS = Math.ceil(size / PX);
canvas.width = COLS * PX;
canvas.height = ROWS * PX;
const BG = '#E8927A';
const EYE = '#1E1E1E';
const MOUTH = '#1E1E1E';
let breath = 0, talk = 0, think = 0, sleepZ = 0, heartP = 0;
let eyeH = 3, eyeHTarget = 3;
let mouthW = 2, mouthWTarget = 2;
let mouthH = 2, mouthHTarget = 2;
let eyeOffX = 0, eyeOffXTarget = 0;
let eyeOffY = 0, eyeOffYTarget = 0;
let blinkOpen = true, blinkCD = 120 + Math.random() * 200;
let doubleBlink = false;
let idleSinceInput = 0, sleepTransitioned = false;
let idleAction = 'none', idleActionTimer = 0;
let idleGlanceX = 0, idleGlanceY = 0;
const lerp = (a: number, b: number, t: number) => a + (b - a) * t;
const px = (col: number, row: number, color: string) => {
ctx.fillStyle = color;
ctx.fillRect(col * PX, row * PX, PX, PX);
};
const pxRect = (x: number, y: number, w: number, h: number, color: string) => {
for (let r = 0; r < Math.round(h); r++)
for (let c = 0; c < Math.round(w); c++)
px(Math.round(x) + c, Math.round(y) + r, color);
};
function draw() {
const state = stateRef.current;
breath += 0.025;
talk += 0.3;
think += 0.025;
sleepZ += 0.012;
heartP += 0.05;
idleSinceInput++;
if (idleSinceInput > 2700 && state === 'idle' && !sleepTransitioned) {
stateRef.current = 'sleeping';
sleepTransitioned = true;
}
if (state === 'idle') {
idleActionTimer--;
blinkCD--;
if (blinkCD <= 0 && blinkOpen) { blinkOpen = false; blinkCD = 6; doubleBlink = Math.random() < 0.3; }
else if (!blinkOpen && blinkCD <= 0) { blinkOpen = true; blinkCD = doubleBlink ? 8 : 100 + Math.random() * 280; doubleBlink = false; }
if (idleActionTimer <= 0) {
const roll = Math.random();
if (roll < 0.3) { idleAction = 'glance'; idleGlanceX = Math.floor(Math.random() * 5) - 2; idleGlanceY = (Math.random() - 0.5) * 1.2; idleActionTimer = 50 + Math.random() * 120; }
else if (roll < 0.45) { idleAction = 'scan'; idleActionTimer = 180; }
else if (roll < 0.55) { idleAction = 'squint'; idleActionTimer = 35 + Math.random() * 40; }
else if (roll < 0.65) { idleAction = 'lookup'; idleActionTimer = 50 + Math.random() * 70; }
else { idleAction = 'none'; idleGlanceX = 0; idleGlanceY = 0; idleActionTimer = 60 + Math.random() * 200; }
}
}
if (state === 'talking' || state === 'thinking') {
blinkCD--;
if (blinkCD <= 0 && blinkOpen) { blinkOpen = false; blinkCD = 6; }
else if (!blinkOpen && blinkCD <= 0) { blinkOpen = true; blinkCD = 120 + Math.random() * 250; }
}
if (state !== 'idle' && state !== 'talking' && state !== 'thinking') blinkOpen = true;
const b = Math.sin(breath) * 0.3;
switch (state) {
case 'idle': {
eyeHTarget = blinkOpen ? 3 : 0; mouthWTarget = 2; mouthHTarget = 2;
let gx = 0, gy = b;
if (idleAction === 'glance') { gx = idleGlanceX; gy = idleGlanceY + b; }
else if (idleAction === 'scan') { const t = 1 - (idleActionTimer / 180); gx = Math.sin(t * Math.PI * 2) * 2.5; }
else if (idleAction === 'squint') { eyeHTarget = blinkOpen ? 2 : 0; gy = b + 0.3; }
else if (idleAction === 'lookup') { gy = -1.2 + b; }
eyeOffXTarget = gx; eyeOffYTarget = gy; break;
}
case 'happy': eyeHTarget = 1; mouthWTarget = 6; mouthHTarget = 1; eyeOffXTarget = 0; eyeOffYTarget = b; break;
case 'thinking': eyeHTarget = blinkOpen ? 3 : 0; mouthWTarget = 2; mouthHTarget = 2; eyeOffXTarget = 2; eyeOffYTarget = b; break;
case 'talking': { eyeHTarget = blinkOpen ? 3 : 0; const open = Math.round(Math.abs(Math.sin(talk)) * 2 + 1); mouthWTarget = 4; mouthHTarget = open; eyeOffXTarget = 0; eyeOffYTarget = b; break; }
case 'surprised': eyeHTarget = 4; mouthWTarget = 3; mouthHTarget = 3; eyeOffXTarget = 0; eyeOffYTarget = b; break;
case 'sleeping': eyeHTarget = 1; mouthWTarget = 2; mouthHTarget = 1; eyeOffXTarget = 0; eyeOffYTarget = b * 2; break;
case 'angry': eyeHTarget = 2; mouthWTarget = 6; mouthHTarget = 1; eyeOffXTarget = 0; eyeOffYTarget = b * 0.3; break;
case 'love': eyeHTarget = 3; mouthWTarget = 2; mouthHTarget = 2; eyeOffXTarget = 0; eyeOffYTarget = b; break;
}
eyeH = lerp(eyeH, eyeHTarget, 0.18);
mouthW = lerp(mouthW, mouthWTarget, 0.15);
mouthH = lerp(mouthH, mouthHTarget, 0.2);
eyeOffX = lerp(eyeOffX, eyeOffXTarget, 0.1);
eyeOffY = lerp(eyeOffY, eyeOffYTarget, 0.15);
// Draw
ctx.fillStyle = BG;
ctx.fillRect(0, 0, canvas.width, canvas.height);
const cx = Math.floor(COLS / 2);
const cy = Math.floor(ROWS / 2);
const eyeSpread = 5, eyeW = 3;
const eh = Math.max(1, Math.round(eyeH));
const eOffX = Math.round(eyeOffX), eOffY = Math.round(eyeOffY);
const eyeBaseY = cy - 2;
const blinkOff = Math.round((3 - eh) / 2);
if (state === 'love') {
const pulse = Math.sin(heartP) > 0 ? '#CC2244' : '#BB1E3E';
const heart = (hx: number, hy: number) => {
px(hx - 1, hy, pulse); px(hx + 1, hy, pulse);
px(hx - 2, hy + 1, pulse); px(hx - 1, hy + 1, pulse); px(hx, hy + 1, pulse); px(hx + 1, hy + 1, pulse); px(hx + 2, hy + 1, pulse);
px(hx - 1, hy + 2, pulse); px(hx, hy + 2, pulse); px(hx + 1, hy + 2, pulse);
px(hx, hy + 3, pulse);
};
heart(cx - eyeSpread + eOffX, eyeBaseY + eOffY);
heart(cx + eyeSpread + eOffX, eyeBaseY + eOffY);
} else {
pxRect(cx - eyeSpread - 1 + eOffX, eyeBaseY + blinkOff + eOffY, eyeW, eh, EYE);
pxRect(cx + eyeSpread - 1 + eOffX, eyeBaseY + blinkOff + eOffY, eyeW, eh, EYE);
}
if (state === 'angry') {
const lx = cx - eyeSpread - 1 + eOffX, ly = eyeBaseY + blinkOff + eOffY - 2;
px(lx, ly + 1, EYE); px(lx + 1, ly, EYE); px(lx + 2, ly, EYE);
const rx = cx + eyeSpread - 1 + eOffX;
px(rx + 2, ly + 1, EYE); px(rx + 1, ly, EYE); px(rx, ly, EYE);
}
const mw = Math.max(1, Math.round(mouthW)), mh = Math.max(1, Math.round(mouthH));
const mouthY = cy + 4 + Math.round(eyeOffY);
if (state === 'happy') {
pxRect(cx - Math.floor(mw / 2), mouthY, mw, 1, MOUTH);
px(cx - Math.floor(mw / 2), mouthY - 1, MOUTH);
px(cx - Math.floor(mw / 2) + mw - 1, mouthY - 1, MOUTH);
} else if (state === 'angry') {
pxRect(cx - Math.floor(mw / 2), mouthY, mw, 1, MOUTH);
px(cx - Math.floor(mw / 2), mouthY + 1, MOUTH);
px(cx - Math.floor(mw / 2) + mw - 1, mouthY + 1, MOUTH);
} else {
pxRect(cx - Math.floor(mw / 2), mouthY, mw, mh, MOUTH);
}
if (state === 'sleeping') {
const zFrame = Math.floor(sleepZ * 60) % 90;
const zy = Math.round(cy - 5 - (zFrame / 90) * 4);
const zx = cx + eyeSpread + 3;
if (zy >= 1 && zFrame < 70) {
px(zx, zy, '#5577CC'); px(zx + 1, zy, '#5577CC');
px(zx + 1, zy + 1, '#5577CC');
px(zx, zy + 2, '#5577CC'); px(zx + 1, zy + 2, '#5577CC');
}
}
if (state === 'thinking') {
const phase = Math.floor(think * 10) % 4;
const dx = cx + eyeSpread + 3, dy = cy - 5;
if (phase >= 1) px(dx, dy, '#7799DD');
if (phase >= 2) px(dx + 1, dy - 1, '#7799DD');
if (phase >= 3) px(dx + 2, dy - 2, '#7799DD');
}
animRef.current = requestAnimationFrame(draw);
}
animRef.current = requestAnimationFrame(draw);
return () => cancelAnimationFrame(animRef.current);
}, [canvasRef, size]);
}
// ─── Status Labels ───────────────────────────────────────────────
const STATUS_LABELS: Record<TalkStatus, string> = {
idle: 'Tap to speak',
listening: 'Listening...',
processing: 'Thinking...',
speaking: 'Speaking...',
};
// ─── Main Component ─────────────────────────────────────────────
const TalkModeOverlay: React.FC<Props> = ({ open, onClose, sessionId }) => {
const c = useClaudeTokens();
const canvasRef = useRef<HTMLCanvasElement>(null);
const [faceState, setFaceState] = useState<FaceState>('idle');
const [talkStatus, setTalkStatus] = useState<TalkStatus>('idle');
const [transcript, setTranscript] = useState('');
const [agentResponse, setAgentResponse] = useState('');
const wsRef = useRef<WebSocket | null>(null);
const mediaRecorderRef = useRef<MediaRecorder | null>(null);
const silenceTimerRef = useRef<number>(0);
const audioContextRef = useRef<AudioContext | null>(null);
usePixelFace(canvasRef, faceState, 240);
// Map talk status to face state
useEffect(() => {
switch (talkStatus) {
case 'idle': setFaceState('idle'); break;
case 'listening': setFaceState('idle'); break;
case 'processing': setFaceState('thinking'); break;
case 'speaking': setFaceState('talking'); break;
}
}, [talkStatus]);
// WebSocket connection for talk mode
useEffect(() => {
if (!open || !sessionId) return;
const ws = new WebSocket(`${WS_BASE}/ws/talk/${sessionId}`);
wsRef.current = ws;
ws.onopen = () => {
ws.send(JSON.stringify({ type: 'config', stt: {}, tts: {} }));
};
ws.onmessage = (event) => {
const msg = JSON.parse(event.data);
switch (msg.type) {
case 'status':
if (msg.status === 'listening') setTalkStatus('idle');
else if (msg.status === 'processing') setTalkStatus('processing');
else if (msg.status === 'speaking') setTalkStatus('speaking');
break;
case 'transcript':
setTranscript(msg.text);
break;
case 'agent_response':
setAgentResponse(msg.text);
setFaceState('happy');
setTimeout(() => setFaceState('idle'), 2000);
break;
case 'audio': {
const audioData = atob(msg.data);
const audioArray = new Uint8Array(audioData.length);
for (let i = 0; i < audioData.length; i++) audioArray[i] = audioData.charCodeAt(i);
if (!audioContextRef.current) audioContextRef.current = new AudioContext();
const audioCtx = audioContextRef.current;
audioCtx.decodeAudioData(audioArray.buffer.slice(0), (buffer) => {
const source = audioCtx.createBufferSource();
source.buffer = buffer;
source.connect(audioCtx.destination);
source.onended = () => setTalkStatus('idle');
source.start(0);
setTalkStatus('speaking');
});
break;
}
}
};
ws.onerror = () => setFaceState('angry');
ws.onclose = () => {};
return () => {
ws.close();
wsRef.current = null;
};
}, [open, sessionId]);
// Keyboard: Esc to close
useEffect(() => {
if (!open) return;
const handler = (e: KeyboardEvent) => {
if (e.key === 'Escape') onClose();
};
window.addEventListener('keydown', handler);
return () => window.removeEventListener('keydown', handler);
}, [open, onClose]);
const startRecording = useCallback(async () => {
try {
const stream = await navigator.mediaDevices.getUserMedia({ audio: true });
const recorder = new MediaRecorder(stream, { mimeType: 'audio/webm' });
mediaRecorderRef.current = recorder;
const chunks: Blob[] = [];
recorder.ondataavailable = (e) => {
if (e.data.size > 0) chunks.push(e.data);
};
recorder.onstop = async () => {
stream.getTracks().forEach((t) => t.stop());
const blob = new Blob(chunks, { type: 'audio/webm' });
const reader = new FileReader();
reader.onload = () => {
const base64 = (reader.result as string).split(',')[1];
if (wsRef.current?.readyState === WebSocket.OPEN) {
wsRef.current.send(JSON.stringify({ type: 'audio', data: base64, format: 'webm' }));
wsRef.current.send(JSON.stringify({ type: 'end_utterance', format: 'webm' }));
}
};
reader.readAsDataURL(blob);
setTalkStatus('processing');
};
recorder.start();
setTalkStatus('listening');
setTranscript('');
setAgentResponse('');
// Auto-stop after silence (simple timeout approach)
silenceTimerRef.current = window.setTimeout(() => {
if (mediaRecorderRef.current?.state === 'recording') {
mediaRecorderRef.current.stop();
}
}, 5000);
} catch {
setFaceState('angry');
}
}, []);
const stopRecording = useCallback(() => {
window.clearTimeout(silenceTimerRef.current);
if (mediaRecorderRef.current?.state === 'recording') {
mediaRecorderRef.current.stop();
}
}, []);
const handleMicClick = useCallback(() => {
if (talkStatus === 'listening') {
stopRecording();
} else if (talkStatus === 'idle') {
startRecording();
}
}, [talkStatus, startRecording, stopRecording]);
if (!open) return null;
const micBg =
talkStatus === 'listening' ? c.accent.primary :
talkStatus === 'speaking' ? '#4caf50' :
c.bg.elevated;
return (
<Box
sx={{
position: 'fixed',
inset: 0,
zIndex: 9999,
bgcolor: 'rgba(0, 0, 0, 0.75)',
display: 'flex',
flexDirection: 'column',
alignItems: 'center',
justifyContent: 'center',
backdropFilter: 'blur(8px)',
animation: 'fadeIn 300ms cubic-bezier(0.165, 0.85, 0.45, 1)',
'@keyframes fadeIn': { from: { opacity: 0 }, to: { opacity: 1 } },
'@keyframes pulse': {
'0%': { boxShadow: `0 0 0 0 ${c.accent.primary}60` },
'70%': { boxShadow: `0 0 0 16px ${c.accent.primary}00` },
'100%': { boxShadow: `0 0 0 0 ${c.accent.primary}00` },
},
}}
onClick={(e) => { if (e.target === e.currentTarget) onClose(); }}
>
{/* Close button */}
<IconButton
onClick={onClose}
sx={{
position: 'absolute',
top: 24,
right: 24,
color: 'rgba(255,255,255,0.5)',
'&:hover': { color: 'rgba(255,255,255,0.9)' },
}}
>
<CloseIcon />
</IconButton>
{/* Face canvas */}
<Box
sx={{
borderRadius: 4,
overflow: 'hidden',
boxShadow: '0 8px 40px rgba(0,0,0,0.4)',
mb: 3,
}}
>
<canvas
ref={canvasRef}
style={{
display: 'block',
imageRendering: 'pixelated',
width: 240,
height: 240,
borderRadius: 16,
}}
/>
</Box>
{/* Status label */}
<Typography
sx={{
color: 'rgba(255,255,255,0.5)',
fontSize: '0.85rem',
fontWeight: 500,
mb: 3,
fontFamily: c.font.sans,
letterSpacing: 0.3,
}}
>
{STATUS_LABELS[talkStatus]}
</Typography>
{/* Transcript area */}
<Box sx={{ maxWidth: 440, width: '100%', px: 3, mb: 2, minHeight: 80 }}>
{transcript && (
<Typography
sx={{
color: 'rgba(255,255,255,0.4)',
fontSize: '0.9rem',
fontStyle: 'italic',
textAlign: 'center',
mb: 1.5,
fontFamily: c.font.sans,
animation: 'fadeIn 200ms ease',
}}
>
"{transcript}"
</Typography>
)}
{agentResponse && (
<Typography
sx={{
color: 'rgba(255,255,255,0.85)',
fontSize: '0.95rem',
textAlign: 'center',
fontFamily: c.font.sans,
lineHeight: 1.5,
animation: 'fadeIn 300ms ease',
}}
>
{agentResponse}
</Typography>
)}
</Box>
{/* Mic button */}
<IconButton
onClick={handleMicClick}
sx={{
width: 56,
height: 56,
bgcolor: micBg,
color: talkStatus === 'listening' ? '#fff' : c.text.primary,
'&:hover': { bgcolor: micBg, opacity: 0.9 },
transition: 'all 200ms ease',
animation: talkStatus === 'listening' ? 'pulse 1.5s infinite' : 'none',
mt: 2,
}}
>
{talkStatus === 'listening' ? <MicIcon sx={{ fontSize: 28 }} /> :
talkStatus === 'speaking' ? <VolumeUpIcon sx={{ fontSize: 28 }} /> :
<MicIcon sx={{ fontSize: 28 }} />}
</IconButton>
{/* Hint */}
<Typography
sx={{
color: 'rgba(255,255,255,0.2)',
fontSize: '0.7rem',
mt: 3,
fontFamily: c.font.sans,
}}
>
esc to close
</Typography>
</Box>
);
};
export default TalkModeOverlay;
@@ -120,6 +120,7 @@ export function useDomElementSelector(): DomSelectorState {
const dragOriginRef = useRef<{ x: number; y: number } | null>(null);
const isDraggingRef = useRef(false);
const dragBoundsRef = useRef<{ left: number; top: number; right: number; bottom: number } | null>(null);
const preDragFocusRef = useRef<HTMLElement | null>(null);
const excludeIdRef = useRef<string | null>(null);
useEffect(() => {
@@ -245,8 +246,11 @@ export function useDomElementSelector(): DomSelectorState {
if (e.button !== 0) return;
if (e.metaKey || e.ctrlKey) return;
const target = e.target as Element;
// Only start drag on "empty" canvas areas (not on selectable elements)
if (target && findSelectableAncestor(target, excludeIdRef.current)) return;
if (target && findSelectableAncestor(target, excludeIdRef.current)) {
e.preventDefault();
return;
}
preDragFocusRef.current = document.activeElement instanceof HTMLElement ? document.activeElement : null;
dragOriginRef.current = { x: e.clientX, y: e.clientY };
isDraggingRef.current = false;
}, []);
@@ -291,12 +295,17 @@ export function useDomElementSelector(): DomSelectorState {
});
}
const wasDragging = isDraggingRef.current;
dragOriginRef.current = null;
isDraggingRef.current = false;
dragBoundsRef.current = null;
setDragRect(EMPTY_DRAG);
setDragPreview([]);
if (dragPreviewRafRef.current) cancelAnimationFrame(dragPreviewRafRef.current);
if (wasDragging && preDragFocusRef.current) {
preDragFocusRef.current.focus();
}
preDragFocusRef.current = null;
}, [ctx]);
const handleClick = useCallback((e: MouseEvent) => {
@@ -327,6 +336,7 @@ export function useDomElementSelector(): DomSelectorState {
dragOriginRef.current = null;
dragBoundsRef.current = null;
isDraggingRef.current = false;
preDragFocusRef.current = null;
return;
}
@@ -353,6 +363,7 @@ export function useDomElementSelector(): DomSelectorState {
dragOriginRef.current = null;
dragBoundsRef.current = null;
isDraggingRef.current = false;
preDragFocusRef.current = null;
};
}, [ctx?.selectMode, handleMouseMove, handleMouseDown, handleMouseUp, handleClick]);
+581 -72
View File
@@ -5,8 +5,16 @@ import Typography from '@mui/material/Typography';
import Chip from '@mui/material/Chip';
import IconButton from '@mui/material/IconButton';
import Tooltip from '@mui/material/Tooltip';
import TextField from '@mui/material/TextField';
import ClickAwayListener from '@mui/material/ClickAwayListener';
import CloseIcon from '@mui/icons-material/Close';
import KeyboardArrowDownIcon from '@mui/icons-material/KeyboardArrowDown';
import KeyboardArrowUpIcon from '@mui/icons-material/KeyboardArrowUp';
import PlayArrowIcon from '@mui/icons-material/PlayArrow';
import EditOutlinedIcon from '@mui/icons-material/EditOutlined';
import DeleteOutlineIcon from '@mui/icons-material/DeleteOutline';
import CheckIcon from '@mui/icons-material/Check';
import DragIndicatorIcon from '@mui/icons-material/DragIndicator';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import {
sendMessage as sendMessageThunk,
@@ -17,6 +25,9 @@ import {
handleApproval,
editMessage,
switchBranch,
duplicateSession,
setActiveSession,
updateSessionProvider,
updateSessionModel,
updateSessionMode,
fetchSession,
@@ -25,19 +36,19 @@ import {
import { fetchModes } from '@/shared/state/modesSlice';
import { createSessionWs } from '@/shared/ws/WebSocketManager';
import MessageBubble from './MessageBubble';
import MessageActionBar from './MessageActionBar';
import ToolCallBubble, { ToolPair } from './ToolCallBubble';
import ToolGroupBubble, { RenderItem, ToolGroup, isToolGroup, isToolPair } from './ToolGroupBubble';
import ApprovalBar, { BatchApprovalBar } from './ApprovalBar';
import ChatInput, { ChatInputHandle } from './ChatInput';
import { ContextPath } from '@/app/components/DirectoryBrowser';
import BranchNavigator from './BranchNavigator';
import DiffViewer from './DiffViewer';
import { setGlowingBrowserCards, clearGlowingBrowserCards } from '@/shared/state/dashboardLayoutSlice';
import { setGlowingBrowserCards, fadeGlowingBrowserCards, clearGlowingBrowserCards } from '@/shared/state/dashboardLayoutSlice';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
const CONTEXT_WINDOWS: Record<string, number> = {
sonnet: 200_000,
opus: 200_000,
const CONTEXT_WINDOWS_DEFAULT: Record<string, number> = {
sonnet: 1_000_000,
opus: 1_000_000,
haiku: 200_000,
};
@@ -91,14 +102,27 @@ const ThinkingBubble: React.FC = () => {
);
};
interface QueuedMessage {
prompt: string;
images?: Array<{ data: string; media_type: string }>;
contextPaths?: Array<{ path: string; type: 'file' | 'directory' }>;
forcedTools?: string[];
attachedSkills?: Array<{ id: string; name: string; content: string }>;
selectedBrowserIds?: string[];
}
interface AgentChatProps {
sessionId?: string;
onClose?: () => void;
embedded?: boolean;
autoFocus?: boolean;
isGlowing?: boolean;
onDismissGlow?: () => void;
initialContextPaths?: ContextPath[];
onBranch?: (newSessionId: string) => void;
}
const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose, embedded, initialContextPaths }) => {
const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose, embedded, autoFocus, isGlowing, onDismissGlow, initialContextPaths, onBranch }) => {
const c = useClaudeTokens();
const STATUS_STYLES: Record<string, { color: string; bg: string }> = {
running: { color: c.status.success, bg: c.status.successBg },
@@ -112,15 +136,26 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
const dispatch = useAppDispatch();
const session = useAppSelector((state) => (id ? state.agents.sessions[id] : undefined));
const modesMap = useAppSelector((state) => state.modes.items);
const modelsByProvider = useAppSelector((state) => state.models.byProvider);
const scrollContainerRef = useRef<HTMLDivElement>(null);
const chatInputRef = useRef<ChatInputHandle>(null);
const isAtBottomRef = useRef(true);
const [showScrollButton, setShowScrollButton] = useState(false);
const [showResumeBubble, setShowResumeBubble] = useState(false);
const [awaitingResponse, setAwaitingResponse] = useState(false);
const [mode, setMode] = useState('agent');
const [model, setModel] = useState('sonnet');
const [provider, setProvider] = useState('anthropic');
const wsRef = useRef<ReturnType<typeof createSessionWs> | null>(null);
const initialContextApplied = useRef(false);
const messageQueueRef = useRef<QueuedMessage[]>([]);
const [queueLength, setQueueLength] = useState(0);
const [queueExpanded, setQueueExpanded] = useState(false);
const [editingQueueIdx, setEditingQueueIdx] = useState<number | null>(null);
const [editingQueueText, setEditingQueueText] = useState('');
const [dragIdx, setDragIdx] = useState<number | null>(null);
const [dropTargetIdx, setDropTargetIdx] = useState<number | null>(null);
const isDraft = session?.status === 'draft';
@@ -153,17 +188,75 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
if (session) setModel(session.model);
}, [session?.model]);
useEffect(() => {
if (session?.provider) setProvider(session.provider);
}, [session?.provider]);
useEffect(() => {
if (Object.keys(modesMap).length === 0) dispatch(fetchModes());
}, [dispatch, modesMap]);
const dispatchMessage = useCallback((msg: QueuedMessage) => {
if (!id) return;
setShowResumeBubble(false);
setAwaitingResponse(true);
if (isDraft) {
const config: Record<string, any> = { provider, model, mode };
if (session?.system_prompt) config.system_prompt = session.system_prompt;
if (session?.target_directory) config.target_directory = session.target_directory;
dispatch(
launchAndSendFirstMessage({ draftId: id, config, prompt: msg.prompt, mode, model, provider, images: msg.images, contextPaths: msg.contextPaths, forcedTools: msg.forcedTools, attachedSkills: msg.attachedSkills, selectedBrowserIds: msg.selectedBrowserIds })
).then((action) => {
if (launchAndSendFirstMessage.fulfilled.match(action)) {
const realId = action.payload.session.id;
dispatch(generateTitle({ sessionId: realId, prompt: msg.prompt }));
if (msg.selectedBrowserIds?.length) {
dispatch(setGlowingBrowserCards({ browserIds: msg.selectedBrowserIds, sessionId: realId, label: 'Use Browser' }));
}
}
});
} else {
if (msg.selectedBrowserIds?.length) {
dispatch(setGlowingBrowserCards({ browserIds: msg.selectedBrowserIds, sessionId: id, label: 'Use Browser' }));
}
dispatch(sendMessageThunk({ sessionId: id, prompt: msg.prompt, mode, model, provider, images: msg.images, contextPaths: msg.contextPaths, forcedTools: msg.forcedTools, attachedSkills: msg.attachedSkills, selectedBrowserIds: msg.selectedBrowserIds }))
.then((action) => {
if (sendMessageThunk.rejected.match(action)) {
setAwaitingResponse(false);
}
});
}
}, [id, isDraft, mode, model, provider, session?.system_prompt, session?.target_directory, dispatch]);
const agentBusy = awaitingResponse || (!isDraft && (session?.status === 'running' || session?.status === 'waiting_approval'));
const prevStatusRef = useRef(session?.status);
useEffect(() => {
const prev = prevStatusRef.current;
const curr = session?.status;
prevStatusRef.current = curr;
if (prev === 'running' && (curr === 'completed' || curr === 'stopped' || curr === 'error')) {
if (id) dispatch(clearGlowingBrowserCards(id));
let didDispatchQueued = false;
const wasActive = prev === 'running' || prev === 'waiting_approval';
const isTerminal = curr === 'completed' || curr === 'stopped' || curr === 'error';
if (wasActive && isTerminal) {
if (id) {
dispatch(fadeGlowingBrowserCards(id));
setTimeout(() => dispatch(clearGlowingBrowserCards(id)), 2800);
}
const nextQueued = messageQueueRef.current.shift();
if (nextQueued) {
setQueueLength(messageQueueRef.current.length);
dispatchMessage(nextQueued);
didDispatchQueued = true;
} else {
if (curr === 'stopped') {
setShowResumeBubble(true);
}
}
const currentMode = modesMap[mode];
if (currentMode?.default_next_mode && modesMap[currentMode.default_next_mode]) {
setMode(currentMode.default_next_mode);
@@ -172,7 +265,13 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
}
}
}
}, [session?.status, mode, modesMap, id, isDraft, dispatch]);
if (curr === 'running') {
setShowResumeBubble(false);
}
if (curr !== 'draft' && !didDispatchQueued) {
setAwaitingResponse(false);
}
}, [session?.status, mode, modesMap, id, isDraft, dispatch, dispatchMessage]);
const SCROLL_THRESHOLD = 50;
@@ -201,27 +300,13 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
const handleSend = (prompt: string, images?: Array<{ data: string; media_type: string }>, contextPaths?: Array<{ path: string; type: 'file' | 'directory' }>, forcedTools?: string[], attachedSkills?: Array<{ id: string; name: string; content: string }>, selectedBrowserIds?: string[]) => {
if (!id) return;
if (isDraft) {
const config: Record<string, any> = { model, mode };
if (session?.system_prompt) config.system_prompt = session.system_prompt;
if (session?.target_directory) config.target_directory = session.target_directory;
dispatch(
launchAndSendFirstMessage({ draftId: id, config, prompt, mode, model, images, contextPaths, forcedTools, attachedSkills })
).then((action) => {
if (launchAndSendFirstMessage.fulfilled.match(action)) {
const realId = action.payload.session.id;
dispatch(generateTitle({ sessionId: realId, prompt }));
if (selectedBrowserIds?.length) {
dispatch(setGlowingBrowserCards({ browserIds: selectedBrowserIds, sessionId: realId }));
}
}
});
} else {
if (selectedBrowserIds?.length) {
dispatch(setGlowingBrowserCards({ browserIds: selectedBrowserIds, sessionId: id }));
}
dispatch(sendMessageThunk({ sessionId: id, prompt, mode, model, images, contextPaths, forcedTools, attachedSkills }));
const msg: QueuedMessage = { prompt, images, contextPaths, forcedTools, attachedSkills, selectedBrowserIds };
if (agentBusy) {
messageQueueRef.current.push(msg);
setQueueLength(messageQueueRef.current.length);
return;
}
dispatchMessage(msg);
};
const handleModeChange = useCallback((newMode: string) => {
@@ -229,6 +314,11 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
if (id && !isDraft) dispatch(updateSessionMode({ sessionId: id, mode: newMode }));
}, [id, isDraft, dispatch]);
const handleProviderChange = useCallback((newProvider: string) => {
setProvider(newProvider);
if (id && !isDraft) dispatch(updateSessionProvider({ sessionId: id, provider: newProvider }));
}, [id, isDraft, dispatch]);
const handleModelChange = useCallback((newModel: string) => {
setModel(newModel);
if (id && !isDraft) dispatch(updateSessionModel({ sessionId: id, model: newModel }));
@@ -247,14 +337,34 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
dispatch(stopAgent({ sessionId: id }));
};
const handleEdit = useCallback(
const handleResume = useCallback(() => {
if (!id) return;
setShowResumeBubble(false);
dispatch(sendMessageThunk({
sessionId: id,
prompt: "Continue where you left off. Start you're response EXACTLY with 'Sorry, let me pick up where I left off",
mode,
model,
provider,
hidden: true,
}));
}, [id, mode, model, provider, dispatch]);
const [editingMessageId, setEditingMessageId] = useState<string | null>(null);
const handleSaveEdit = useCallback(
(messageId: string, newContent: string) => {
if (!id) return;
dispatch(editMessage({ sessionId: id, messageId, content: newContent }));
setEditingMessageId(null);
},
[id, dispatch]
);
const handleCancelEdit = useCallback(() => {
setEditingMessageId(null);
}, []);
const activeBranchMessages = useMemo(() => {
if (!session) return [];
const branchId = session.active_branch_id || 'main';
@@ -264,16 +374,81 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
return session.messages.filter((m) => m.branch_id === 'main' || m.branch_id === branchId);
}
const forkIdx = session.messages.findIndex((m) => m.id === branch.fork_point_message_id);
const preMessages = session.messages
.slice(0, forkIdx)
.filter((m) => m.branch_id === (branch.parent_branch_id || 'main'));
const branchMessages = session.messages.filter((m) => m.branch_id === branchId);
return [...preMessages, ...branchMessages];
const segments: Array<{ branchId: string; upToMessageId?: string }> = [];
let cur = branch;
let curId = branchId;
while (cur && cur.fork_point_message_id) {
segments.unshift({ branchId: curId, upToMessageId: cur.fork_point_message_id });
curId = cur.parent_branch_id || 'main';
cur = session.branches?.[curId];
}
segments.unshift({ branchId: curId });
const result: typeof session.messages = [];
for (let i = 0; i < segments.length; i++) {
const seg = segments[i];
const nextForkMsgId = seg.upToMessageId;
if (nextForkMsgId) {
const forkIdx = session.messages.findIndex((m) => m.id === nextForkMsgId);
const pre = session.messages
.slice(0, forkIdx)
.filter((m) => m.branch_id === seg.branchId);
result.push(...pre);
} else if (i < segments.length - 1) {
const nextFork = segments[i + 1].upToMessageId;
const forkIdx = nextFork
? session.messages.findIndex((m) => m.id === nextFork)
: session.messages.length;
result.push(
...session.messages.slice(0, forkIdx).filter((m) => m.branch_id === seg.branchId)
);
} else {
result.push(...session.messages.filter((m) => m.branch_id === seg.branchId));
}
}
const leafMsgs = session.messages.filter((m) => m.branch_id === branchId);
if (!result.some((m) => m.branch_id === branchId)) {
result.push(...leafMsgs);
}
return result;
}, [session?.messages, session?.active_branch_id, session?.branches]);
const handleRegenerate = useCallback(
(assistantMsg: AgentMessage) => {
if (!id) return;
const idx = activeBranchMessages.findIndex((m) => m.id === assistantMsg.id);
for (let i = idx - 1; i >= 0; i--) {
if (activeBranchMessages[i].role === 'user') {
const userMsg = activeBranchMessages[i];
const content = typeof userMsg.content === 'string' ? userMsg.content : JSON.stringify(userMsg.content);
dispatch(editMessage({ sessionId: id, messageId: userMsg.id, content }));
break;
}
}
},
[id, activeBranchMessages, dispatch]
);
const handleBranchChat = useCallback(async (upToMessageId: string) => {
if (!id) return;
const dashId = session?.dashboard_id;
const action = await dispatch(duplicateSession({ sessionId: id, dashboardId: dashId, upToMessageId }));
if (duplicateSession.fulfilled.match(action)) {
if (onBranch) {
onBranch(action.payload.id);
} else {
dispatch(setActiveSession(action.payload.id));
}
}
}, [id, dispatch, onBranch, session?.dashboard_id]);
const contextEstimate = useMemo(() => {
const limit = CONTEXT_WINDOWS[model] || 200_000;
// Look up context window from dynamic models first, then fall back to defaults
let limit = CONTEXT_WINDOWS_DEFAULT[model] || 200_000;
for (const models of Object.values(modelsByProvider)) {
const found = models.find((m: any) => m.value === model);
if (found?.context_window) { limit = found.context_window; break; }
}
let totalChars = 0;
if (session?.system_prompt) totalChars += session.system_prompt.length;
for (const msg of activeBranchMessages) {
@@ -284,7 +459,7 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
}
const used = Math.round(totalChars / 4);
return { used, limit };
}, [activeBranchMessages, session?.system_prompt, session?.streamingMessage?.content, model]);
}, [activeBranchMessages, session?.system_prompt, session?.streamingMessage?.content, model, modelsByProvider]);
const sessionRunning = session?.status === 'running' || session?.status === 'waiting_approval';
@@ -373,13 +548,33 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
items.push(...outputItems);
} else {
items.push(msg);
if (!msg.hidden) {
items.push(msg);
}
i++;
}
}
return items;
}, [activeBranchMessages]);
const lastAssistantIdsInTurn = useMemo(() => {
const ids = new Set<string>();
let lastAssistantId: string | null = null;
for (const item of renderItems) {
if (!isToolGroup(item) && !isToolPair(item)) {
const msg = item as AgentMessage;
if (msg.role === 'assistant') {
lastAssistantId = msg.id;
} else if (msg.role === 'user') {
if (lastAssistantId) ids.add(lastAssistantId);
lastAssistantId = null;
}
}
}
if (lastAssistantId) ids.add(lastAssistantId);
return ids;
}, [renderItems]);
const groupMetaRequestedRef = useRef<Set<string>>(new Set());
const groupMetaRefinedRef = useRef<Set<string>>(new Set());
@@ -427,11 +622,33 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
const getSiblingBranches = useCallback(
(messageId: string): string[] => {
if (!session?.branches) return [];
return Object.values(session.branches)
const directForks = Object.values(session.branches)
.filter((b) => b.fork_point_message_id === messageId)
.map((b) => b.id);
if (directForks.length > 0) {
const originalMsg = session.messages.find((m) => m.id === messageId);
const parentBranchId = originalMsg?.branch_id || 'main';
return [parentBranchId, ...directForks];
}
const msg = session.messages.find((m) => m.id === messageId);
if (!msg || msg.role !== 'user') return [];
const msgBranch = session.branches[msg.branch_id];
if (!msgBranch?.fork_point_message_id) return [];
const branchUserMsgs = session.messages.filter(
(m) => m.branch_id === msg.branch_id && m.role === 'user'
);
if (branchUserMsgs.length === 0 || branchUserMsgs[0].id !== messageId) return [];
const forkPointId = msgBranch.fork_point_message_id;
const siblingBranches = Object.values(session.branches)
.filter((b) => b.fork_point_message_id === forkPointId)
.map((b) => b.id);
const parentBranchId = msgBranch.parent_branch_id || 'main';
return [parentBranchId, ...siblingBranches];
},
[session?.branches]
[session?.branches, session?.messages]
);
if (!session) {
@@ -527,37 +744,55 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
{renderItems.map((item) => {
if (isToolGroup(item)) {
const groupMeta = session.tool_group_meta?.[item.id];
return <ToolGroupBubble key={item.id} group={item} isSessionRunning={sessionRunning} meta={groupMeta} />;
return <ToolGroupBubble key={item.id} group={item} isSessionRunning={sessionRunning} meta={groupMeta} sessionId={session.id} />;
}
if (isToolPair(item)) {
const isPending = item.result === null && sessionRunning;
return <ToolCallBubble key={item.id} call={item.call} result={item.result} isPending={isPending} />;
return <ToolCallBubble key={item.id} call={item.call} result={item.result} isPending={isPending} sessionId={session.id} />;
}
const msg = item;
const isEditing = editingMessageId === msg.id;
const siblings = getSiblingBranches(msg.id);
const hasBranches = siblings.length > 0;
const currentBranchIdx = hasBranches
? siblings.indexOf(session.active_branch_id || 'main')
: 0;
const rawText = typeof msg.content === 'string' ? msg.content : JSON.stringify(msg.content);
return (
<React.Fragment key={msg.id}>
<MessageBubble message={msg} onEdit={msg.role === 'user' ? handleEdit : undefined} />
{hasBranches && (
<BranchNavigator
currentIndex={Math.max(0, currentBranchIdx)}
totalBranches={siblings.length}
onPrevious={() => {
const prevBranch = siblings[Math.max(0, currentBranchIdx - 1)];
if (prevBranch && id) dispatch(switchBranch({ sessionId: id, branchId: prevBranch }));
}}
onNext={() => {
const nextBranch = siblings[Math.min(siblings.length - 1, currentBranchIdx + 1)];
if (nextBranch && id) dispatch(switchBranch({ sessionId: id, branchId: nextBranch }));
}}
<Box key={msg.id} sx={{ '&:hover .msg-actions': { opacity: 1 } }}>
<MessageBubble
message={msg}
editing={isEditing}
onSaveEdit={handleSaveEdit}
onCancelEdit={handleCancelEdit}
/>
{!isEditing && (msg.role === 'user' || (msg.role === 'assistant' && lastAssistantIdsInTurn.has(msg.id))) && (
<MessageActionBar
role={msg.role as 'user' | 'assistant'}
onCopy={() => navigator.clipboard.writeText(rawText)}
onEdit={msg.role === 'user' ? () => setEditingMessageId(msg.id) : undefined}
onRegenerate={msg.role === 'assistant' ? () => handleRegenerate(msg) : undefined}
onBranch={msg.role === 'assistant' ? () => handleBranchChat(msg.id) : undefined}
branchNav={
hasBranches
? {
currentIndex: Math.max(0, currentBranchIdx),
totalBranches: siblings.length,
onPrevious: () => {
const prevBranch = siblings[Math.max(0, currentBranchIdx - 1)];
if (prevBranch && id) dispatch(switchBranch({ sessionId: id, branchId: prevBranch }));
},
onNext: () => {
const nextBranch = siblings[Math.min(siblings.length - 1, currentBranchIdx + 1)];
if (nextBranch && id) dispatch(switchBranch({ sessionId: id, branchId: nextBranch }));
},
}
: undefined
}
/>
)}
</React.Fragment>
</Box>
);
})}
{session.streamingMessage && (
@@ -566,6 +801,7 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
key={`streaming-${session.streamingMessage.id}`}
isStreaming
isPending
sessionId={session.id}
call={{
id: session.streamingMessage.id,
role: 'tool_call',
@@ -590,9 +826,37 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
/>
)
)}
{session.status === 'running' && !session.streamingMessage && (
{(awaitingResponse || (session.status === 'running' && !session.streamingMessage)) && (
<ThinkingBubble />
)}
{showResumeBubble && session.status === 'stopped' && (
<Box sx={{ display: 'flex', justifyContent: 'flex-start', my: 0.75 }}>
<Box
onClick={handleResume}
sx={{
display: 'inline-flex',
alignItems: 'center',
gap: 0.5,
px: 1.5,
py: 0.75,
borderRadius: '12px',
cursor: 'pointer',
bgcolor: `${c.accent.primary}10`,
border: `1px solid ${c.accent.primary}30`,
transition: 'all 0.15s',
'&:hover': {
bgcolor: `${c.accent.primary}1a`,
border: `1px solid ${c.accent.primary}50`,
},
}}
>
<PlayArrowIcon sx={{ fontSize: 14, color: c.accent.primary }} />
<Typography sx={{ fontSize: '0.78rem', fontWeight: 500, color: c.accent.primary }}>
Resume Agent Response
</Typography>
</Box>
</Box>
)}
</Box>
{showScrollButton && (
<Tooltip title="Scroll to bottom">
@@ -627,19 +891,264 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
))
)}
<ChatInput
ref={chatInputRef}
onSend={handleSend}
disabled={false}
mode={mode}
onModeChange={handleModeChange}
model={model}
onModelChange={handleModelChange}
isRunning={!isDraft && (session.status === 'running' || session.status === 'waiting_approval')}
onStop={handleStop}
contextEstimate={contextEstimate}
sessionId={id}
/>
{isGlowing ? (
<Box
onClick={(e) => { e.stopPropagation(); onDismissGlow?.(); }}
sx={{
mx: 1.5,
mb: 1.5,
py: 1.25,
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
borderRadius: 2.5,
cursor: 'pointer',
fontWeight: 600,
fontSize: '0.85rem',
color: c.accent.primary,
border: `1.5px solid ${c.accent.primary}`,
background: `${c.accent.primary}08`,
boxShadow: `0 0 12px ${c.accent.primary}25, inset 0 0 12px ${c.accent.primary}08`,
animation: 'continue-chat-glow 2s ease-in-out infinite',
transition: 'background 0.15s, box-shadow 0.15s',
'@keyframes continue-chat-glow': {
'0%, 100%': {
boxShadow: `0 0 12px ${c.accent.primary}25, inset 0 0 12px ${c.accent.primary}08`,
},
'50%': {
boxShadow: `0 0 20px ${c.accent.primary}40, inset 0 0 20px ${c.accent.primary}15`,
},
},
'&:hover': {
background: `${c.accent.primary}14`,
boxShadow: `0 0 24px ${c.accent.primary}50, inset 0 0 20px ${c.accent.primary}18`,
},
}}
>
Continue chat
</Box>
) : (
<ClickAwayListener onClickAway={() => { if (queueExpanded) { setQueueExpanded(false); setEditingQueueIdx(null); } }}>
<Box>
{queueLength > 0 && (
<Box sx={{ ml: 3, mr: 1.5 }}>
<Box
onClick={() => { setQueueExpanded((v) => !v); setEditingQueueIdx(null); }}
sx={{
display: 'inline-flex',
alignItems: 'center',
gap: 0.5,
px: 1.25,
py: 0.25,
borderRadius: '8px 8px 0 0',
bgcolor: c.bg.surface,
border: `1px solid ${c.border.subtle}`,
borderBottom: 'none',
cursor: 'pointer',
userSelect: 'none',
'&:hover': { bgcolor: c.bg.secondary },
transition: 'background 0.12s',
}}
>
{queueExpanded
? <KeyboardArrowDownIcon sx={{ fontSize: 12, color: c.text.tertiary }} />
: <KeyboardArrowUpIcon sx={{ fontSize: 12, color: c.text.tertiary }} />
}
<Typography sx={{ fontSize: '0.68rem', fontWeight: 600, color: c.text.muted, letterSpacing: 0.2 }}>
{queueLength} queued
</Typography>
<Tooltip title="Clear all">
<IconButton
size="small"
onClick={(e) => { e.stopPropagation(); messageQueueRef.current = []; setQueueLength(0); setQueueExpanded(false); setEditingQueueIdx(null); }}
sx={{ p: 0.15, color: c.text.tertiary, '&:hover': { color: c.status.error } }}
>
<CloseIcon sx={{ fontSize: 10 }} />
</IconButton>
</Tooltip>
</Box>
{queueExpanded && (
<Box
sx={{
bgcolor: c.bg.surface,
border: `1px solid ${c.border.subtle}`,
borderBottom: 'none',
borderRadius: '0 8px 0 0',
maxHeight: 240,
overflowY: 'auto',
'&::-webkit-scrollbar': { width: 4 },
'&::-webkit-scrollbar-thumb': { background: c.border.medium, borderRadius: 2 },
}}
>
{messageQueueRef.current.map((msg, idx) => (
<Box
key={idx}
draggable={editingQueueIdx !== idx}
onDragStart={(e) => {
setDragIdx(idx);
e.dataTransfer.effectAllowed = 'move';
}}
onDragOver={(e) => {
e.preventDefault();
e.dataTransfer.dropEffect = 'move';
if (dragIdx !== null && dragIdx !== idx) setDropTargetIdx(idx);
}}
onDragLeave={() => { if (dropTargetIdx === idx) setDropTargetIdx(null); }}
onDrop={(e) => {
e.preventDefault();
if (dragIdx !== null && dragIdx !== idx) {
const q = messageQueueRef.current;
const [item] = q.splice(dragIdx, 1);
q.splice(idx, 0, item);
setQueueLength(q.length);
}
setDragIdx(null);
setDropTargetIdx(null);
}}
onDragEnd={() => { setDragIdx(null); setDropTargetIdx(null); }}
sx={{
display: 'flex',
alignItems: 'flex-start',
gap: 0.75,
px: 1.5,
py: 1,
borderBottom: idx < queueLength - 1 ? `1px solid ${c.border.subtle}` : 'none',
'&:hover': { bgcolor: c.bg.secondary },
transition: 'background 0.1s, opacity 0.15s',
...(dragIdx === idx ? { opacity: 0.35 } : {}),
...(dropTargetIdx === idx && dragIdx !== null && dragIdx !== idx
? { borderTop: `2px solid ${c.accent.primary}` }
: {}),
}}
>
<Box
sx={{
cursor: editingQueueIdx === idx ? 'default' : 'grab',
display: 'flex',
alignItems: 'center',
mt: 0.3,
color: c.text.ghost,
'&:hover': { color: c.text.tertiary },
'&:active': { cursor: 'grabbing' },
}}
>
<DragIndicatorIcon sx={{ fontSize: 14 }} />
</Box>
{editingQueueIdx === idx ? (
<Box sx={{ flex: 1, display: 'flex', gap: 0.5, alignItems: 'flex-start' }}>
<TextField
multiline
fullWidth
size="small"
value={editingQueueText}
onChange={(e) => setEditingQueueText(e.target.value)}
autoFocus
onKeyDown={(e) => {
if (e.key === 'Enter' && !e.shiftKey) {
e.preventDefault();
const trimmed = editingQueueText.trim();
if (trimmed) {
messageQueueRef.current[idx] = { ...messageQueueRef.current[idx], prompt: trimmed };
setQueueLength(messageQueueRef.current.length);
}
setEditingQueueIdx(null);
}
if (e.key === 'Escape') setEditingQueueIdx(null);
}}
sx={{
'& .MuiOutlinedInput-root': {
fontSize: '0.78rem',
color: c.text.primary,
'& fieldset': { borderColor: c.border.medium },
'&.Mui-focused fieldset': { borderColor: c.accent.primary },
},
}}
/>
<IconButton
size="small"
onClick={() => {
const trimmed = editingQueueText.trim();
if (trimmed) {
messageQueueRef.current[idx] = { ...messageQueueRef.current[idx], prompt: trimmed };
setQueueLength(messageQueueRef.current.length);
}
setEditingQueueIdx(null);
}}
sx={{ p: 0.25, color: c.accent.primary, mt: 0.25 }}
>
<CheckIcon sx={{ fontSize: 14 }} />
</IconButton>
</Box>
) : (
<Typography
sx={{
flex: 1,
fontSize: '0.78rem',
color: c.text.secondary,
lineHeight: 1.5,
overflow: 'hidden',
display: '-webkit-box',
WebkitLineClamp: 3,
WebkitBoxOrient: 'vertical',
wordBreak: 'break-word',
}}
>
{msg.prompt}
</Typography>
)}
{editingQueueIdx !== idx && (
<Box sx={{ display: 'flex', gap: 0.25, flexShrink: 0, mt: 0.15 }}>
<Tooltip title="Edit">
<IconButton
size="small"
onClick={() => { setEditingQueueIdx(idx); setEditingQueueText(msg.prompt); }}
sx={{ p: 0.25, color: c.text.tertiary, '&:hover': { color: c.text.primary } }}
>
<EditOutlinedIcon sx={{ fontSize: 13 }} />
</IconButton>
</Tooltip>
<Tooltip title="Remove">
<IconButton
size="small"
onClick={() => {
messageQueueRef.current.splice(idx, 1);
setQueueLength(messageQueueRef.current.length);
if (messageQueueRef.current.length === 0) setQueueExpanded(false);
}}
sx={{ p: 0.25, color: c.text.tertiary, '&:hover': { color: c.status.error } }}
>
<DeleteOutlineIcon sx={{ fontSize: 13 }} />
</IconButton>
</Tooltip>
</Box>
)}
</Box>
))}
</Box>
)}
</Box>
)}
<ChatInput
ref={chatInputRef}
onSend={handleSend}
disabled={false}
mode={mode}
onModeChange={handleModeChange}
model={model}
onModelChange={handleModelChange}
provider={provider}
onProviderChange={handleProviderChange}
isRunning={agentBusy}
onStop={handleStop}
queueLength={queueLength}
contextEstimate={contextEstimate}
sessionId={id}
autoFocus={autoFocus}
/>
</Box>
</ClickAwayListener>
)}
</Box>
</Box>
);
@@ -59,14 +59,14 @@ const INTEGRATION_META: Record<string, IntegrationMeta> = {
// MCP tool name parser
// ---------------------------------------------------------------------------
interface ParsedTool {
export interface ParsedTool {
isMcp: boolean;
serverSlug: string;
actionName: string;
displayName: string;
}
function parseMcpToolName(rawName: string): ParsedTool {
export function parseMcpToolName(rawName: string): ParsedTool {
const m = rawName.match(/^mcp__([^_]+(?:-[^_]+)*)__(.+)$/);
if (!m) {
return { isMcp: false, serverSlug: '', actionName: rawName, displayName: rawName };
@@ -93,7 +93,7 @@ interface McpToolMeta {
serverLabel: string;
}
function useMcpToolMeta(parsed: ParsedTool): McpToolMeta {
export function useMcpToolMeta(parsed: ParsedTool): McpToolMeta {
const toolItems = useAppSelector((s) => s.tools.items);
return useMemo(() => {
@@ -185,7 +185,7 @@ interface Props {
onDeny: (requestId: string, message?: string) => void;
}
function getToolIcon(toolName: string) {
export function getToolIcon(toolName: string) {
switch (toolName) {
case 'Bash': return <TerminalIcon sx={{ fontSize: '1rem' }} />;
case 'Read': return <DescriptionIcon sx={{ fontSize: '1rem' }} />;
@@ -21,31 +21,38 @@ const BranchNavigator: React.FC<Props> = ({ currentIndex, totalBranches, onPrevi
<Box
sx={{
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
gap: 0.5,
my: 0.25,
justifyContent: 'flex-end',
mt: -0.25,
mb: 0.5,
}}
>
<IconButton
size="small"
onClick={onPrevious}
disabled={currentIndex === 0}
sx={{ color: c.text.tertiary, p: 0.25, '&.Mui-disabled': { color: c.border.medium } }}
<Box
sx={{
display: 'flex',
alignItems: 'center',
gap: 0.25,
}}
>
<ChevronLeftIcon sx={{ fontSize: 18 }} />
</IconButton>
<Typography sx={{ color: c.text.tertiary, fontSize: '0.7rem', minWidth: 32, textAlign: 'center' }}>
{currentIndex + 1}/{totalBranches}
</Typography>
<IconButton
size="small"
onClick={onNext}
disabled={currentIndex === totalBranches - 1}
sx={{ color: c.text.tertiary, p: 0.25, '&.Mui-disabled': { color: c.border.medium } }}
>
<ChevronRightIcon sx={{ fontSize: 18 }} />
</IconButton>
<IconButton
size="small"
onClick={onPrevious}
disabled={currentIndex === 0}
sx={{ color: c.text.tertiary, p: 0.25, '&.Mui-disabled': { color: c.border.medium } }}
>
<ChevronLeftIcon sx={{ fontSize: 16 }} />
</IconButton>
<Typography sx={{ color: c.text.tertiary, fontSize: '0.7rem', minWidth: 28, textAlign: 'center', userSelect: 'none' }}>
{currentIndex + 1} / {totalBranches}
</Typography>
<IconButton
size="small"
onClick={onNext}
disabled={currentIndex === totalBranches - 1}
sx={{ color: c.text.tertiary, p: 0.25, '&.Mui-disabled': { color: c.border.medium } }}
>
<ChevronRightIcon sx={{ fontSize: 16 }} />
</IconButton>
</Box>
</Box>
);
};
@@ -0,0 +1,403 @@
import React, { useEffect, useRef, useMemo } from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import SmartToyOutlinedIcon from '@mui/icons-material/SmartToyOutlined';
import CheckCircleOutlineIcon from '@mui/icons-material/CheckCircleOutline';
import ErrorOutlineIcon from '@mui/icons-material/ErrorOutline';
import LanguageIcon from '@mui/icons-material/Language';
import OpenInNewIcon from '@mui/icons-material/OpenInNew';
import TouchAppOutlinedIcon from '@mui/icons-material/TouchAppOutlined';
import KeyboardOutlinedIcon from '@mui/icons-material/KeyboardOutlined';
import CameraAltOutlinedIcon from '@mui/icons-material/CameraAltOutlined';
import ArticleOutlinedIcon from '@mui/icons-material/ArticleOutlined';
import AccountTreeOutlinedIcon from '@mui/icons-material/AccountTreeOutlined';
import CodeOutlinedIcon from '@mui/icons-material/CodeOutlined';
import BuildOutlinedIcon from '@mui/icons-material/BuildOutlined';
import { createSelector } from '@reduxjs/toolkit';
import { useAppSelector, useAppDispatch } from '@/shared/hooks';
import { AgentMessage, AgentSession, fetchBrowserAgentChildren } from '@/shared/state/agentsSlice';
import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext';
import type { RootState } from '@/shared/state/store';
interface Props {
parentSessionId: string;
browserId?: string;
}
interface FeedEntry {
type: 'thought' | 'action' | 'result' | 'system';
text: string;
actionTool?: string;
sessionLabel?: string;
}
function formatMessage(msg: AgentMessage): FeedEntry | null {
if (msg.role === 'user') return null;
if (msg.role === 'assistant' && typeof msg.content === 'string') {
const trimmed = msg.content.trim();
if (!trimmed) return null;
return { type: 'thought', text: trimmed };
}
if (msg.role === 'tool_call') {
const content =
typeof msg.content === 'string'
? (() => { try { return JSON.parse(msg.content); } catch { return {}; } })()
: msg.content;
const tool = content?.tool || content?.name || '?';
const input = content?.input || {};
let brief = '';
switch (tool) {
case 'BrowserNavigate':
brief = `Navigate → ${input.url || '...'}`;
break;
case 'BrowserClick':
brief = `Click ${input.selector || '...'}`;
break;
case 'BrowserType': {
const txt = (input.text || '').slice(0, 40);
const ellipsis = (input.text || '').length > 40 ? '…' : '';
brief = `Type "${txt}${ellipsis}" into ${input.selector || '...'}`;
break;
}
case 'BrowserScreenshot':
brief = 'Screenshot';
break;
case 'BrowserGetText':
brief = 'Read page text';
break;
case 'BrowserGetElements':
brief = `Inspect elements${input.selector ? ` (${input.selector})` : ''}`;
break;
case 'BrowserEvaluate':
brief = `Evaluate JS`;
break;
default:
brief = `${tool}(${JSON.stringify(input).slice(0, 60)})`;
}
return { type: 'action', text: brief, actionTool: tool };
}
if (msg.role === 'tool_result') {
const content =
typeof msg.content === 'string'
? (() => { try { return JSON.parse(msg.content); } catch { return { text: msg.content }; } })()
: msg.content;
const toolName = content?.tool_name || '';
const elapsed = content?.elapsed_ms;
const text = content?.text || '';
if (toolName === 'BrowserScreenshot') {
return { type: 'result', text: `Screenshot captured${elapsed ? ` (${elapsed}ms)` : ''}` };
}
const preview = text.length > 120 ? text.slice(0, 120) + '…' : text;
return { type: 'result', text: `${preview}${elapsed ? ` (${elapsed}ms)` : ''}` };
}
if (msg.role === 'system') {
return { type: 'system', text: typeof msg.content === 'string' ? msg.content : '' };
}
return null;
}
type SvgIconComponent = typeof OpenInNewIcon;
function getActionIcon(tool?: string): SvgIconComponent {
switch (tool) {
case 'BrowserNavigate': return OpenInNewIcon;
case 'BrowserClick': return TouchAppOutlinedIcon;
case 'BrowserType': return KeyboardOutlinedIcon;
case 'BrowserScreenshot': return CameraAltOutlinedIcon;
case 'BrowserGetText': return ArticleOutlinedIcon;
case 'BrowserGetElements': return AccountTreeOutlinedIcon;
case 'BrowserEvaluate': return CodeOutlinedIcon;
default: return BuildOutlinedIcon;
}
}
interface FeedColors {
thought: string;
thoughtIcon: string;
result: string;
error: string;
errorIcon: string;
scrollThumb: string;
}
const darkFeedColors: FeedColors = {
thought: '#a0aab8',
thoughtIcon: '#555b6e',
result: '#555b6e',
error: '#ff8787',
errorIcon: '#ff8787',
scrollThumb: '#2a2d3e',
};
const lightFeedColors: FeedColors = {
thought: '#555550',
thoughtIcon: '#9e9c95',
result: '#9e9c95',
error: '#c03030',
errorIcon: '#c03030',
scrollThumb: '#ccc9c0',
};
const selectBrowserSessions = createSelector(
[(state: RootState) => state.agents.sessions,
(_: RootState, parentSessionId: string) => parentSessionId,
(_: RootState, __: string, browserId?: string) => browserId],
(sessions, parentSessionId, browserId) =>
Object.values(sessions).filter(
(s): s is AgentSession =>
s.mode === 'browser-agent' &&
s.parent_session_id === parentSessionId &&
(!browserId || s.browser_id === browserId),
),
);
const BrowserAgentInlineFeed: React.FC<Props> = ({ parentSessionId, browserId }) => {
const c = useClaudeTokens();
const dispatch = useAppDispatch();
const { mode } = useThemeMode();
const fc = mode === 'dark' ? darkFeedColors : lightFeedColors;
const scrollRef = useRef<HTMLDivElement>(null);
const fetchedForSession = useRef<string | null>(null);
const browserSessions = useAppSelector((state) =>
selectBrowserSessions(state, parentSessionId, browserId),
);
useEffect(() => {
if (browserSessions.length === 0 && fetchedForSession.current !== parentSessionId) {
fetchedForSession.current = parentSessionId;
dispatch(fetchBrowserAgentChildren(parentSessionId))
.unwrap()
.catch(() => { fetchedForSession.current = null; });
}
}, [browserSessions.length, parentSessionId, dispatch]);
const sessionsWithEntries = useMemo(() => {
return browserSessions.map((session) => {
const entries: FeedEntry[] = [];
for (const msg of session.messages) {
const entry = formatMessage(msg);
if (entry) entries.push(entry);
}
if (session.streamingMessage?.role === 'assistant' && session.streamingMessage.content) {
entries.push({ type: 'thought', text: session.streamingMessage.content });
}
return { session, entries };
});
}, [browserSessions]);
const totalMessages = browserSessions.reduce(
(n, s) => n + s.messages.length + (s.streamingMessage ? 1 : 0),
0,
);
useEffect(() => {
if (scrollRef.current) {
scrollRef.current.scrollTop = scrollRef.current.scrollHeight;
}
}, [totalMessages]);
if (browserSessions.length === 0) return null;
const showLabels = sessionsWithEntries.length > 1;
const accentColor = c.accent.primary;
return (
<Box
ref={scrollRef}
sx={{
maxHeight: 300,
overflowY: 'auto',
px: 1.5,
py: 1,
display: 'flex',
flexDirection: 'column',
gap: 0.25,
scrollbarWidth: 'thin',
scrollbarColor: `${fc.scrollThumb} transparent`,
'&::-webkit-scrollbar': { width: 4 },
'&::-webkit-scrollbar-thumb': {
background: fc.scrollThumb,
borderRadius: 2,
},
}}
>
{sessionsWithEntries.map(({ session, entries }, si) => (
<Box key={session.id} sx={{ display: 'flex', flexDirection: 'column', gap: 0.25 }}>
{showLabels && (
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5, mt: si > 0 ? 1 : 0, mb: 0.25 }}>
<LanguageIcon sx={{ fontSize: 12, color: accentColor, opacity: 0.7 }} />
<Typography
sx={{
fontSize: '0.65rem',
fontWeight: 600,
color: accentColor,
opacity: 0.8,
textTransform: 'uppercase',
letterSpacing: '0.04em',
}}
>
{session.browser_id || `Browser ${si + 1}`}
</Typography>
<SessionStatusChip status={session.status} />
</Box>
)}
{!showLabels && entries.length === 0 && session.status === 'running' && (
<Typography
sx={{
fontSize: '0.7rem',
color: c.text.tertiary,
fontStyle: 'italic',
fontFamily: c.font.mono,
}}
>
Starting browser agent...
</Typography>
)}
{entries.map((entry, i) => (
<EntryRow key={i} entry={entry} accentColor={accentColor} fc={fc} />
))}
{!showLabels && session.status === 'running' && entries.length > 0 && (
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5, mt: 0.25 }}>
<Box
sx={{
width: 6,
height: 6,
borderRadius: '50%',
bgcolor: accentColor,
animation: 'ba-feed-pulse 1.4s ease-in-out infinite',
'@keyframes ba-feed-pulse': {
'0%, 100%': { opacity: 0.3, transform: 'scale(0.8)' },
'50%': { opacity: 1, transform: 'scale(1.2)' },
},
}}
/>
</Box>
)}
</Box>
))}
</Box>
);
};
const EntryRow: React.FC<{ entry: FeedEntry; accentColor: string; fc: FeedColors }> = ({ entry, accentColor, fc }) => {
const c = useClaudeTokens();
if (entry.type === 'thought') {
return (
<Box sx={{ display: 'flex', gap: 0.5, alignItems: 'flex-start', minWidth: 0 }}>
<SmartToyOutlinedIcon
sx={{ fontSize: 10, color: fc.thoughtIcon, mt: '3px', flexShrink: 0 }}
/>
<Typography
sx={{
fontSize: '0.7rem',
color: fc.thought,
lineHeight: 1.45,
wordBreak: 'break-word',
fontFamily: c.font.mono,
}}
>
{entry.text}
</Typography>
</Box>
);
}
if (entry.type === 'action') {
const ActionIcon = getActionIcon(entry.actionTool);
return (
<Box sx={{ display: 'flex', gap: 0.5, alignItems: 'flex-start', minWidth: 0 }}>
<ActionIcon sx={{ fontSize: 11, color: accentColor, mt: '2px', flexShrink: 0 }} />
<Typography
sx={{
fontSize: '0.7rem',
fontFamily: c.font.mono,
color: accentColor,
lineHeight: 1.45,
wordBreak: 'break-word',
}}
>
{entry.text}
</Typography>
</Box>
);
}
if (entry.type === 'result') {
return (
<Box sx={{ display: 'flex', gap: 0.5, alignItems: 'flex-start', minWidth: 0, pl: 1.25 }}>
<Typography
sx={{
fontSize: '0.65rem',
fontFamily: c.font.mono,
color: fc.result,
lineHeight: 1.45,
wordBreak: 'break-word',
}}
>
{entry.text}
</Typography>
</Box>
);
}
if (entry.type === 'system') {
return (
<Box sx={{ display: 'flex', gap: 0.5, alignItems: 'center', minWidth: 0 }}>
<ErrorOutlineIcon sx={{ fontSize: 10, color: fc.errorIcon, flexShrink: 0 }} />
<Typography
sx={{
fontSize: '0.68rem',
fontFamily: c.font.mono,
color: fc.error,
lineHeight: 1.45,
}}
>
{entry.text}
</Typography>
</Box>
);
}
return null;
};
const SessionStatusChip: React.FC<{ status: string }> = ({ status }) => {
const c = useClaudeTokens();
if (status === 'running') {
return (
<Box
sx={{
width: 6,
height: 6,
borderRadius: '50%',
bgcolor: c.status.success,
animation: 'ba-feed-pulse 1.4s ease-in-out infinite',
'@keyframes ba-feed-pulse': {
'0%, 100%': { opacity: 0.3, transform: 'scale(0.8)' },
'50%': { opacity: 1, transform: 'scale(1.2)' },
},
}}
/>
);
}
if (status === 'completed') {
return <CheckCircleOutlineIcon sx={{ fontSize: 10, color: c.status.success }} />;
}
if (status === 'error') {
return <ErrorOutlineIcon sx={{ fontSize: 10, color: c.status.error }} />;
}
return null;
};
export default React.memo(BrowserAgentInlineFeed);
+225 -118
View File
@@ -26,6 +26,7 @@ import AttachFileIcon from '@mui/icons-material/AttachFile';
import AdsClickIcon from '@mui/icons-material/AdsClick';
import CommandPicker, { CommandPickerItem, getToolGroupIcon } from '@/app/components/CommandPicker';
import { useElementSelection, SelectedElement } from '@/app/components/ElementSelectionContext';
import { getClipboardCards, clearClipboard } from '@/shared/dashboardClipboard';
import { getWebview } from '@/shared/browserRegistry';
import { API_BASE } from '@/shared/config';
import { ContextPath } from '@/app/components/DirectoryBrowser';
@@ -66,6 +67,8 @@ interface Props {
onModeChange: (mode: string) => void;
model: string;
onModelChange: (model: string) => void;
provider?: string;
onProviderChange?: (provider: string) => void;
isRunning?: boolean;
onStop?: () => void;
autoRunMode?: boolean;
@@ -73,6 +76,7 @@ interface Props {
embedded?: boolean;
autoFocus?: boolean;
sessionId?: string;
queueLength?: number;
}
export interface ChatInputHandle {
@@ -90,10 +94,10 @@ const ICON_MAP: Record<string, React.ReactNode> = {
const FALLBACK_MODE_BASE = { label: 'Agent', icon: ICON_MAP.smart_toy };
const MODEL_OPTIONS = [
{ value: 'sonnet', label: 'Sonnet', version: '4.6' },
{ value: 'opus', label: 'Opus', version: '4.6' },
{ value: 'haiku', label: 'Haiku', version: '3.5' },
const FALLBACK_MODELS = [
{ value: 'sonnet', label: 'Claude Sonnet 4.6', context_window: 1_000_000 },
{ value: 'opus', label: 'Claude Opus 4.6', context_window: 1_000_000 },
{ value: 'haiku', label: 'Claude Haiku 4.5', context_window: 200_000 },
];
function formatTokenCount(n: number): string {
@@ -130,7 +134,7 @@ const ContextRing: React.FC<{ used: number; limit: number; accentColor: string;
);
};
const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode, onModeChange, model, onModelChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus, sessionId }, ref) => {
const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode, onModeChange, model, onModelChange, provider, onProviderChange, isRunning, onStop, autoRunMode, contextEstimate, embedded, autoFocus, sessionId, queueLength = 0 }, ref) => {
const c = useClaudeTokens();
const editorRef = useRef<HTMLDivElement>(null);
const containerRef = useRef<HTMLDivElement>(null);
@@ -138,6 +142,9 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
const dispatch = useAppDispatch();
const elementSelection = useElementSelection();
const fallbackOwnerIdRef = useRef(`input-${Date.now().toString(36)}-${Math.random().toString(36).slice(2, 6)}`);
const ownerId = sessionId || fallbackOwnerIdRef.current;
useEffect(() => {
if (autoFocus) editorRef.current?.focus();
}, [autoFocus]);
@@ -153,6 +160,24 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
const skills = useAppSelector((state) => state.skills.items);
const modesMap = useAppSelector((state) => state.modes.items);
const modesArr = useMemo(() => Object.values(modesMap), [modesMap]);
const modelsByProvider = useAppSelector((state) => state.models.byProvider);
const modelsLoaded = useAppSelector((state) => state.models.loaded);
// Build flat model list with provider grouping
const allModelOptions = useMemo(() => {
if (!modelsLoaded || Object.keys(modelsByProvider).length === 0) {
return { flat: FALLBACK_MODELS.map(m => ({ ...m, provider: 'Anthropic' })), grouped: { Anthropic: FALLBACK_MODELS } };
}
const flat: Array<{ value: string; label: string; context_window: number; provider: string }> = [];
const grouped: Record<string, Array<{ value: string; label: string; context_window: number }>> = {};
for (const [prov, models] of Object.entries(modelsByProvider)) {
grouped[prov] = models.map(m => ({ value: m.value, label: m.label, context_window: m.context_window ?? 200_000 }));
for (const m of models) {
flat.push({ value: m.value, label: m.label, context_window: m.context_window ?? 200_000, provider: prov });
}
}
return { flat, grouped };
}, [modelsByProvider, modelsLoaded]);
useEffect(() => {
if (modesArr.length === 0) dispatch(fetchModes());
@@ -281,7 +306,7 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
let trimmed = serialized.trim();
if (!trimmed) return;
const selectedEls = elementSelection?.selectedElements ?? [];
const selectedEls = elementSelection?.elementsByOwner?.[ownerId] ?? [];
let allImages = images.length > 0
? images.map(({ data, media_type }) => ({ data, media_type }))
: [];
@@ -298,7 +323,7 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
lines.push(`${i + 1}. [Browser Card] ${title}`);
lines.push(` browser_id: ${el.semanticData.selectId}`);
if (url) lines.push(` URL: ${url}`);
lines.push(` (Use BrowserAgent with this browser_id to interact with it)`);
lines.push(` (Use BrowserAgent with this browser_id to interact with it, or CreateBrowserAgent for a new browser)`);
} else if (el.semanticType && el.semanticData) {
const typeLabel = {
'agent-card': 'Agent Card',
@@ -317,6 +342,9 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
.map(([k, v]) => `${k}: ${typeof v === 'string' ? v : JSON.stringify(v)}`)
.join(', ');
if (metaStr) lines.push(` ${metaStr}`);
if (el.semanticType === 'agent-card' && selectId) {
lines.push(` (Use InvokeAgent with session_id "${selectId}" to query this agent with full conversation context)`);
}
} else {
const styleStr = Object.entries(el.computedStyles)
.map(([k, v]) => `${k}: ${v}`)
@@ -359,8 +387,8 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
setForcedTools([]);
setAttachedSkills({});
setHasContent(false);
elementSelection?.clearSelectedElements();
}, [disabled, images, contextPaths, forcedTools, onSend, elementSelection]);
elementSelection?.clearOwnerElements(ownerId);
}, [disabled, images, contextPaths, forcedTools, onSend, elementSelection, ownerId]);
const detectTrigger = useCallback(() => {
const result = detectEditorTrigger();
@@ -469,6 +497,41 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
};
const handlePaste = useCallback((e: React.ClipboardEvent) => {
const copied = getClipboardCards();
if (copied.length > 0 && elementSelection) {
e.preventDefault();
for (const card of copied) {
const semanticTypeMap: Record<string, SelectedElement['semanticType']> = {
agent: 'agent-card',
view: 'view-card',
browser: 'browser-card',
};
const semanticType = semanticTypeMap[card.type];
if (!semanticType) continue;
const labelMap: Record<string, string> = {
'agent-card': 'Agent',
'view-card': 'View',
'browser-card': 'Browser',
};
const semanticLabel = (labelMap[semanticType] || semanticType) + ': ' + card.name;
const el: SelectedElement = {
id: `sel-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
selectorPath: `[data-select-type="${semanticType}"][data-select-id="${card.id}"]`,
tagName: 'DIV',
className: '',
outerHTML: '',
computedStyles: {},
boundingRect: { x: 0, y: 0, width: 0, height: 0 },
semanticType,
semanticLabel,
semanticData: { ...card.meta, selectId: card.id },
};
elementSelection.addElementForOwner(ownerId, el);
}
clearClipboard();
return;
}
const items = e.clipboardData?.items;
if (!items) return;
const imageFiles: File[] = [];
@@ -486,7 +549,7 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
e.preventDefault();
const plain = e.clipboardData.getData('text/plain');
if (plain) document.execCommand('insertText', false, plain);
}, [addImageFiles]);
}, [addImageFiles, elementSelection, ownerId]);
const handleDragOver = useCallback((e: React.DragEvent) => {
e.preventDefault();
@@ -523,7 +586,8 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
bgcolor: c.bg.surface,
border: `1px solid ${c.border.subtle}`,
borderRadius: '10px',
minWidth: 140,
minWidth: 180,
maxHeight: 400,
boxShadow: c.shadow.lg,
'& .MuiMenuItem-root': {
fontSize: '0.8rem',
@@ -535,7 +599,7 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
},
};
const selectedElements = elementSelection?.selectedElements ?? [];
const selectedElements = elementSelection?.elementsByOwner?.[ownerId] ?? [];
const hasAttachments = images.length > 0 || contextPaths.length > 0 || forcedTools.length > 0 || selectedElements.length > 0;
return (
@@ -783,7 +847,7 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
icon={<AdsClickIcon sx={{ fontSize: 14 }} />}
label={chipLabel}
size="small"
onDelete={() => elementSelection?.removeSelectedElement(el.id)}
onDelete={() => elementSelection?.removeOwnerElement(ownerId, el.id)}
sx={{
bgcolor: 'rgba(59, 130, 246, 0.1)',
color: '#3b82f6',
@@ -849,7 +913,7 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
userSelect: 'none',
}}
>
{disabled ? 'Agent is working...' : autoRunMode ? 'Describe what data to generate…' : `${modeConf.label}, @ for context, / for commands`}
{disabled ? 'Agent is working...' : autoRunMode ? 'Describe what data to generate…' : isRunning ? (queueLength > 0 ? `${queueLength} queued — type another or wait…` : 'Agent is working — messages will queue…') : `${modeConf.label}, @ for context, / for commands`}
</div>
)}
</Box>
@@ -936,7 +1000,7 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
}}
>
<Typography sx={{ fontSize: '0.75rem', fontWeight: 500, color: 'inherit', lineHeight: 1 }}>
{(() => { const m = MODEL_OPTIONS.find((m) => m.value === model); return m ? `${m.label} ${m.version}` : model; })()}
{(() => { const m = allModelOptions.flat.find((m) => m.value === model); return m ? m.label : model; })()}
</Typography>
<KeyboardArrowDownIcon sx={{ fontSize: 14, color: 'inherit', opacity: 0.7 }} />
</Box>
@@ -949,21 +1013,45 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
transformOrigin={{ vertical: 'bottom', horizontal: 'left' }}
slotProps={{ paper: menuPaperProps }}
>
{MODEL_OPTIONS.map((opt) => (
<MenuItem
key={opt.value}
selected={model === opt.value}
onClick={() => {
onModelChange(opt.value);
setModelAnchor(null);
}}
>
<ListItemText
primary={`${opt.label} ${opt.version}`}
slotProps={{ primary: { sx: { fontSize: '0.8rem', color: model === opt.value ? c.text.primary : c.text.muted } } }}
/>
</MenuItem>
))}
{Object.entries(allModelOptions.grouped).map(([prov, models]) => [
<MenuItem key={`header-${prov}`} disabled sx={{ opacity: '0.7 !important', py: 0.5, px: 1.5, minHeight: 'auto' }}>
<Typography sx={{ fontSize: '0.65rem', fontWeight: 700, letterSpacing: '0.06em', textTransform: 'uppercase', color: c.text.tertiary }}>
{prov}
</Typography>
</MenuItem>,
...models.map((opt) => (
<MenuItem
key={opt.value}
selected={model === opt.value}
onClick={() => {
onModelChange(opt.value);
if (onProviderChange) {
// Derive API-level provider key from the display group name
const provLower = prov.toLowerCase();
const providerMap: Record<string, string> = {
anthropic: 'anthropic',
openai: 'openai',
google: 'gemini',
// OpenRouter-backed providers
xai: 'openrouter',
meta: 'openrouter',
deepseek: 'openrouter',
mistral: 'openrouter',
qwen: 'openrouter',
cohere: 'openrouter',
};
onProviderChange(providerMap[provLower] || provLower);
}
setModelAnchor(null);
}}
>
<ListItemText
primary={opt.label}
slotProps={{ primary: { sx: { fontSize: '0.8rem', color: model === opt.value ? c.text.primary : c.text.muted } } }}
/>
</MenuItem>
)),
]).flat()}
</Menu>
<Box sx={{ flex: 1 }} />
@@ -977,40 +1065,54 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
/>
)}
{elementSelection && !autoRunMode && (
<Tooltip title={elementSelection.selectMode ? 'Exit select mode' : 'Select UI element'}>
<IconButton
size="small"
onClick={() => {
if (!elementSelection.selectMode && sessionId) {
elementSelection.setExcludeSelectId(sessionId);
}
elementSelection.toggleSelectMode();
}}
sx={{
p: 0.5,
...(elementSelection.selectMode
? {
bgcolor: '#3b82f6',
color: '#fff',
'&:hover': { bgcolor: '#2563eb' },
animation: 'selectBtnPulse 2s ease-in-out infinite',
'@keyframes selectBtnPulse': {
'0%, 100%': { boxShadow: '0 0 0 0 rgba(59,130,246,0.4)' },
'50%': { boxShadow: '0 0 0 4px rgba(59,130,246,0.1)' },
},
{elementSelection && !autoRunMode && (() => {
const isMySelectMode = elementSelection.selectMode && elementSelection.activeOwnerId === ownerId;
return (
<Tooltip title={isMySelectMode ? 'Exit select mode' : 'Select UI element'}>
<IconButton
size="small"
onMouseDown={(e) => e.preventDefault()}
onClick={() => {
if (isMySelectMode) {
elementSelection.setSelectMode(false);
} else {
if (elementSelection.activeOwnerId !== ownerId) {
elementSelection.clearOwnerElements(ownerId);
}
: {
color: c.text.tertiary,
'&:hover': { color: c.text.secondary, bgcolor: 'rgba(0,0,0,0.04)' },
}),
transition: 'background-color 0.15s, color 0.15s',
}}
>
<AdsClickIcon sx={{ fontSize: 18 }} />
</IconButton>
</Tooltip>
)}
elementSelection.setActiveOwnerId(ownerId);
if (sessionId) {
elementSelection.setExcludeSelectId(sessionId);
} else {
elementSelection.setExcludeSelectId(null);
}
elementSelection.setSelectMode(true);
}
}}
sx={{
p: 0.5,
...(isMySelectMode
? {
bgcolor: '#3b82f6',
color: '#fff',
'&:hover': { bgcolor: '#2563eb' },
animation: 'selectBtnPulse 2s ease-in-out infinite',
'@keyframes selectBtnPulse': {
'0%, 100%': { boxShadow: '0 0 0 0 rgba(59,130,246,0.4)' },
'50%': { boxShadow: '0 0 0 4px rgba(59,130,246,0.1)' },
},
}
: {
color: c.text.tertiary,
'&:hover': { color: c.text.secondary, bgcolor: 'rgba(0,0,0,0.04)' },
}),
transition: 'background-color 0.15s, color 0.15s',
}}
>
<AdsClickIcon sx={{ fontSize: 18 }} />
</IconButton>
</Tooltip>
);
})()}
<input
ref={generalFileInputRef}
@@ -1040,61 +1142,66 @@ const ChatInput = forwardRef<ChatInputHandle, Props>(({ onSend, disabled, mode,
<AttachFileIcon sx={{ fontSize: 18 }} />
</IconButton>
</Tooltip>
{!autoRunMode && (isRunning ? (
<Tooltip title="Stop agent">
<IconButton
size="small"
onClick={onStop}
sx={{
bgcolor: c.status.error,
color: c.text.inverse,
p: 0.5,
width: 26,
height: 26,
'&:hover': { bgcolor: c.status.error, opacity: 0.85 },
transition: c.transition,
}}
>
<StopIcon sx={{ fontSize: 16 }} />
</IconButton>
</Tooltip>
) : hasContent ? (
<Tooltip title="Send message">
<IconButton
size="small"
onClick={handleSend}
disabled={disabled}
sx={{
bgcolor: c.accent.primary,
color: c.text.inverse,
p: 0.5,
width: 26,
height: 26,
'&:hover': { bgcolor: c.accent.hover },
'&.Mui-disabled': { bgcolor: c.bg.secondary, color: c.text.ghost },
transition: c.transition,
}}
>
<ArrowUpwardIcon sx={{ fontSize: 16 }} />
</IconButton>
</Tooltip>
) : (
<Tooltip title="Voice input (coming soon)">
<span>
<IconButton
size="small"
disabled
sx={{
color: c.text.tertiary,
p: 0.5,
'&.Mui-disabled': { color: c.text.ghost },
}}
>
<MicNoneOutlinedIcon sx={{ fontSize: 18 }} />
</IconButton>
</span>
</Tooltip>
))}
{!autoRunMode && (
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
{hasContent && (
<Tooltip title={isRunning ? 'Queue message' : 'Send message'}>
<IconButton
size="small"
onClick={handleSend}
disabled={disabled}
sx={{
bgcolor: c.accent.primary,
color: c.text.inverse,
p: 0.5,
width: 26,
height: 26,
'&:hover': { bgcolor: c.accent.hover },
'&.Mui-disabled': { bgcolor: c.bg.secondary, color: c.text.ghost },
transition: c.transition,
}}
>
<ArrowUpwardIcon sx={{ fontSize: 16 }} />
</IconButton>
</Tooltip>
)}
{isRunning ? (
<Tooltip title="Stop agent">
<IconButton
size="small"
onClick={onStop}
sx={{
bgcolor: c.status.error,
color: c.text.inverse,
p: 0.5,
width: 26,
height: 26,
'&:hover': { bgcolor: c.status.error, opacity: 0.85 },
transition: c.transition,
}}
>
<StopIcon sx={{ fontSize: 16 }} />
</IconButton>
</Tooltip>
) : !hasContent ? (
<Tooltip title="Voice input (coming soon)">
<span>
<IconButton
size="small"
disabled
sx={{
color: c.text.tertiary,
p: 0.5,
'&.Mui-disabled': { color: c.text.ghost },
}}
>
<MicNoneOutlinedIcon sx={{ fontSize: 18 }} />
</IconButton>
</span>
</Tooltip>
) : null}
</Box>
)}
</Box>
{selectedTemplate && (
@@ -0,0 +1,153 @@
import React, { useState } from 'react';
import Box from '@mui/material/Box';
import IconButton from '@mui/material/IconButton';
import Typography from '@mui/material/Typography';
import Tooltip from '@mui/material/Tooltip';
import ContentCopyIcon from '@mui/icons-material/ContentCopy';
import CheckIcon from '@mui/icons-material/Check';
import EditIcon from '@mui/icons-material/Edit';
import BookmarkBorderIcon from '@mui/icons-material/BookmarkBorder';
import ReplayIcon from '@mui/icons-material/Replay';
import CallSplitIcon from '@mui/icons-material/CallSplit';
import ChevronLeftIcon from '@mui/icons-material/ChevronLeft';
import ChevronRightIcon from '@mui/icons-material/ChevronRight';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
interface BranchNavProps {
currentIndex: number;
totalBranches: number;
onPrevious: () => void;
onNext: () => void;
}
interface Props {
role: 'user' | 'assistant';
onCopy: () => void;
onEdit?: () => void;
onRegenerate?: () => void;
onBranch?: () => void;
branchNav?: BranchNavProps;
}
const btnSx = (c: ReturnType<typeof useClaudeTokens>) => ({
color: c.text.tertiary,
p: 0.4,
'&:hover': { color: c.text.secondary, bgcolor: 'transparent' },
'&.Mui-disabled': { color: c.border.medium },
});
const MessageActionBar: React.FC<Props> = ({
role,
onCopy,
onEdit,
onRegenerate,
onBranch,
branchNav,
}) => {
const c = useClaudeTokens();
const [copied, setCopied] = useState(false);
const handleCopy = () => {
onCopy();
setCopied(true);
setTimeout(() => setCopied(false), 1500);
};
const isUser = role === 'user';
return (
<Box
className="msg-actions"
sx={{
display: 'flex',
alignItems: 'center',
justifyContent: isUser ? 'flex-end' : 'flex-start',
gap: 0,
opacity: 0,
transition: 'opacity 0.15s',
mt: -0.25,
mb: 0.25,
minHeight: 28,
}}
>
{isUser ? (
<>
<Tooltip title="Coming soon" arrow>
<span>
<IconButton size="small" disabled sx={btnSx(c)}>
<BookmarkBorderIcon sx={{ fontSize: 16 }} />
</IconButton>
</span>
</Tooltip>
<Tooltip title={copied ? 'Copied!' : 'Copy'} arrow>
<IconButton size="small" onClick={handleCopy} sx={btnSx(c)}>
{copied ? <CheckIcon sx={{ fontSize: 16 }} /> : <ContentCopyIcon sx={{ fontSize: 16 }} />}
</IconButton>
</Tooltip>
{onEdit && (
<Tooltip title="Edit" arrow>
<IconButton size="small" onClick={onEdit} sx={btnSx(c)}>
<EditIcon sx={{ fontSize: 16 }} />
</IconButton>
</Tooltip>
)}
{branchNav && branchNav.totalBranches > 1 && (
<Box sx={{ display: 'inline-flex', alignItems: 'center', ml: 0.25 }}>
<IconButton
size="small"
onClick={branchNav.onPrevious}
disabled={branchNav.currentIndex === 0}
sx={btnSx(c)}
>
<ChevronLeftIcon sx={{ fontSize: 16 }} />
</IconButton>
<Typography
sx={{
color: c.text.tertiary,
fontSize: '0.7rem',
minWidth: 28,
textAlign: 'center',
userSelect: 'none',
}}
>
{branchNav.currentIndex + 1} / {branchNav.totalBranches}
</Typography>
<IconButton
size="small"
onClick={branchNav.onNext}
disabled={branchNav.currentIndex === branchNav.totalBranches - 1}
sx={btnSx(c)}
>
<ChevronRightIcon sx={{ fontSize: 16 }} />
</IconButton>
</Box>
)}
</>
) : (
<>
<Tooltip title={copied ? 'Copied!' : 'Copy'} arrow>
<IconButton size="small" onClick={handleCopy} sx={btnSx(c)}>
{copied ? <CheckIcon sx={{ fontSize: 16 }} /> : <ContentCopyIcon sx={{ fontSize: 16 }} />}
</IconButton>
</Tooltip>
{onRegenerate && (
<Tooltip title="Regenerate" arrow>
<IconButton size="small" onClick={onRegenerate} sx={btnSx(c)}>
<ReplayIcon sx={{ fontSize: 16 }} />
</IconButton>
</Tooltip>
)}
{onBranch && (
<Tooltip title="Branch chat" arrow>
<IconButton size="small" onClick={onBranch} sx={btnSx(c)}>
<CallSplitIcon sx={{ fontSize: 16 }} />
</IconButton>
</Tooltip>
)}
</>
)}
</Box>
);
};
export default MessageActionBar;
@@ -8,7 +8,6 @@ import Chip from '@mui/material/Chip';
import Tooltip from '@mui/material/Tooltip';
import Collapse from '@mui/material/Collapse';
import Modal from '@mui/material/Modal';
import EditIcon from '@mui/icons-material/Edit';
import CloseIcon from '@mui/icons-material/Close';
import AdsClickIcon from '@mui/icons-material/AdsClick';
import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
@@ -391,13 +390,14 @@ const MessageImageThumbnails: React.FC<{
interface Props {
message: AgentMessage;
onEdit?: (messageId: string, newContent: string) => void;
editing?: boolean;
onSaveEdit?: (messageId: string, newContent: string) => void;
onCancelEdit?: () => void;
isStreaming?: boolean;
}
const MessageBubble: React.FC<Props> = React.memo(({ message, onEdit, isStreaming }) => {
const MessageBubble: React.FC<Props> = React.memo(({ message, editing = false, onSaveEdit, onCancelEdit, isStreaming }) => {
const c = useClaudeTokens();
const [editing, setEditing] = useState(false);
const [editText, setEditText] = useState('');
const { role, content } = message;
@@ -440,23 +440,22 @@ const MessageBubble: React.FC<Props> = React.memo(({ message, onEdit, isStreamin
? parseElementContext(rawText)
: { userMessage: rawText, elements: [] };
const handleStartEdit = () => {
setEditText(rawText);
setEditing(true);
};
React.useEffect(() => {
if (editing) setEditText(rawText);
}, [editing, rawText]);
const handleCancelEdit = () => {
setEditing(false);
setEditText('');
onCancelEdit?.();
};
const handleSaveEdit = () => {
const trimmed = editText.trim();
if (trimmed && trimmed !== rawText && onEdit) {
onEdit(message.id, trimmed);
if (trimmed && trimmed !== rawText && onSaveEdit) {
onSaveEdit(message.id, trimmed);
}
setEditing(false);
setEditText('');
onCancelEdit?.();
};
const truncatedContent = typeof content === 'string'
@@ -472,26 +471,8 @@ const MessageBubble: React.FC<Props> = React.memo(({ message, onEdit, isStreamin
display: 'flex',
justifyContent: isUser ? 'flex-end' : 'flex-start',
my: 0.75,
'&:hover .edit-btn': { opacity: 1 },
}}
>
{isUser && onEdit && !editing && (
<IconButton
className="edit-btn"
size="small"
onClick={handleStartEdit}
sx={{
opacity: 0,
transition: 'opacity 0.15s',
color: c.text.tertiary,
alignSelf: 'center',
mr: 0.5,
p: 0.5,
}}
>
<EditIcon sx={{ fontSize: 16 }} />
</IconButton>
)}
<Box
sx={{
maxWidth: '85%',
@@ -641,7 +622,14 @@ const MessageBubble: React.FC<Props> = React.memo(({ message, onEdit, isStreamin
'& a': { color: c.accent.primary },
}}
>
<ReactMarkdown remarkPlugins={[remarkGfm]}>{rawText}</ReactMarkdown>
<ReactMarkdown
remarkPlugins={[remarkGfm]}
components={{
a: ({ children, ...props }) => (
<a {...props} style={{ cursor: 'pointer' }}>{children}</a>
),
}}
>{rawText}</ReactMarkdown>
{isStreaming && <StreamingCursor />}
</Box>
)}
@@ -1,8 +1,9 @@
import React, { useState, useEffect, useCallback, useMemo } from 'react';
import React, { useState, useEffect, useCallback, useMemo, useRef } from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import Collapse from '@mui/material/Collapse';
import IconButton from '@mui/material/IconButton';
import Tooltip from '@mui/material/Tooltip';
import ExpandMoreIcon from '@mui/icons-material/ExpandMore';
import ExpandLessIcon from '@mui/icons-material/ExpandLess';
import TerminalIcon from '@mui/icons-material/Terminal';
@@ -15,10 +16,14 @@ import FolderIcon from '@mui/icons-material/Folder';
import AttachFileIcon from '@mui/icons-material/AttachFile';
import SearchIcon from '@mui/icons-material/Search';
import SendIcon from '@mui/icons-material/Send';
import CallSplitIcon from '@mui/icons-material/CallSplit';
import ReactMarkdown from 'react-markdown';
import remarkGfm from 'remark-gfm';
import { AgentMessage } from '@/shared/state/agentsSlice';
import { AgentMessage, expandSession, collapseSession, fetchSession } from '@/shared/state/agentsSlice';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import { placeCard, removeCard, setGlowingAgentCard, clearGlowingAgentCard, DEFAULT_CARD_W, DEFAULT_CARD_H, EXPANDED_CARD_MIN_H, GRID_GAP } from '@/shared/state/dashboardLayoutSlice';
import { useClaudeTokens, useThemeMode } from '@/shared/styles/ThemeContext';
import BrowserAgentInlineFeed from './BrowserAgentInlineFeed';
const GoogleServiceIcon: React.FC<{ service: string; size?: number }> = ({ service, size = 14 }) => {
if (service === 'gmail') {
@@ -68,7 +73,13 @@ export interface ToolPair {
result: AgentMessage | null;
}
const pulsingKeyframes = `
let toolCallKeyframesInjected = false;
function ensureToolCallKeyframes() {
if (toolCallKeyframesInjected) return;
toolCallKeyframesInjected = true;
const style = document.createElement('style');
style.setAttribute('data-tool-call-keyframes', '');
style.textContent = `
@keyframes tool-pulse {
0%, 100% { opacity: 1; }
50% { opacity: 0.4; }
@@ -77,14 +88,13 @@ const pulsingKeyframes = `
0%, 100% { box-shadow: 0 0 0 0 rgba(var(--glow-rgb), 0); }
50% { box-shadow: 0 0 10px 2px rgba(var(--glow-rgb), 0.12); }
}
`;
const streamingCursorKeyframes = `
@keyframes blink-cursor {
0%, 100% { opacity: 1; }
50% { opacity: 0; }
}
`;
document.head.appendChild(style);
}
const ElapsedTimer: React.FC<{ startTime: string }> = ({ startTime }) => {
const c = useClaudeTokens();
@@ -466,6 +476,7 @@ interface ToolCallBubbleProps {
isPending?: boolean;
isStreaming?: boolean;
mcpCompact?: boolean;
sessionId?: string;
}
interface TermColors {
@@ -929,7 +940,7 @@ const GmailCard: React.FC<{ data: Record<string, any>; action: string; hideSubje
}}>
<ReactMarkdown
remarkPlugins={[remarkGfm]}
components={{ a: ({ children, ...props }) => <a {...props} target="_blank" rel="noopener noreferrer">{children}</a> }}
components={{ a: ({ children, ...props }) => <a {...props}>{children}</a> }}
>
{email.bodyPreview || email.snippet}
</ReactMarkdown>
@@ -1169,18 +1180,92 @@ const McpResultCard: React.FC<{ parsed: ParsedMcpResult; compact?: boolean }> =
return <GenericMcpCard data={data} />;
};
function isBrowserAgentTool(name: string): boolean {
if (name === 'CreateBrowserAgent' || name === 'BrowserAgent' || name === 'BrowserAgents') return true;
const mcp = parseMcpToolName(name);
return mcp.isMcp && mcp.serverSlug === 'openswarm-browser-agent';
}
function isInvokeAgentTool(name: string): boolean {
if (name === 'InvokeAgent') return true;
const mcp = parseMcpToolName(name);
return mcp.isMcp && mcp.serverSlug === 'openswarm-invoke-agent';
}
function isCreateAgentTool(name: string): boolean {
return name === 'Agent';
}
function parseInvokedSessionId(rawText: string): string | null {
const match = rawText.match(/\(forked session:\s*([a-f0-9]+)\)/);
return match ? match[1] : null;
}
interface InvokeAgentParsed {
agentName: string;
sessionId: string | null;
cost: string | null;
response: string;
}
function parseCreateAgentResult(rawText: string): string {
if (!rawText) return '';
try {
const parsed = JSON.parse(rawText);
if (typeof parsed === 'string') return parsed;
if (typeof parsed === 'object' && parsed !== null) {
if (parsed.text) return parsed.text;
if (parsed.content) return typeof parsed.content === 'string' ? parsed.content : JSON.stringify(parsed.content);
if (parsed.result) return typeof parsed.result === 'string' ? parsed.result : JSON.stringify(parsed.result);
}
} catch {}
return rawText;
}
function parseInvokeAgentResult(rawText: string): InvokeAgentParsed | null {
const headerMatch = rawText.match(
/\*\*Invoked Agent Result\*\*(?:\s*—\s*(.+?))?\s*\(forked session:\s*([a-f0-9]+)\)/,
);
if (!headerMatch) return null;
const agentName = headerMatch[1]?.trim() || 'Agent';
const sessionId = headerMatch[2];
const costMatch = rawText.match(/\*Cost:\s*\$([0-9.]+)\*/);
const cost = costMatch ? costMatch[1] : null;
const bodyStart = rawText.indexOf('\n\n');
let response = bodyStart >= 0 ? rawText.slice(bodyStart + 2).trim() : '';
if (response.startsWith('*Cost:')) {
const afterCost = response.indexOf('\n');
response = afterCost >= 0 ? response.slice(afterCost + 1).trim() : '';
}
return { agentName, sessionId, cost, response };
}
const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false }) => {
({ call, result = null, isPending = false, isStreaming = false, mcpCompact = false, sessionId }) => {
ensureToolCallKeyframes();
const c = useClaudeTokens();
const tc = useTermColors();
const dispatch = useAppDispatch();
const cards = useAppSelector((s) => s.dashboardLayout.cards);
const [expanded, setExpanded] = useState(false);
const bubbleRef = useRef<HTMLDivElement>(null);
const { toolName, input, isDenied } = getToolData(call);
const mcpInfo = useMemo(() => parseMcpToolName(toolName), [toolName]);
const inputSummary = getInputSummary(toolName, input);
const formattedInput = useMemo(() => formatInputDisplay(toolName, input), [toolName, input]);
const showTimer = isPending && !isDenied && !isStreaming;
const showBody = expanded || isStreaming;
const isBrowserAgent = isBrowserAgentTool(toolName);
const isInvokeAgent = isInvokeAgentTool(toolName);
const isCreateAgent = isCreateAgentTool(toolName);
const browserAgentAutoExpand = isBrowserAgent && isPending && !isStreaming;
const showBody = expanded || isStreaming || browserAgentAutoExpand;
const resultContent = result?.content;
const hasStructuredResult =
@@ -1206,6 +1291,95 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
(parsedResult?.type === 'bash' && parsedResult.exitCode !== null && parsedResult.exitCode !== 0) ||
(parsedResult?.type === 'text' && parsedResult.isError);
const invokedSessionId = useMemo(
() => (isInvokeAgent && result ? parseInvokedSessionId(resultRawText) : null),
[isInvokeAgent, result, resultRawText],
);
const invokeAgentParsed = useMemo(
() => (isInvokeAgent && result ? parseInvokeAgentResult(resultRawText) : null),
[isInvokeAgent, result, resultRawText],
);
const createAgentResponse = useMemo(
() => (isCreateAgent && result ? parseCreateAgentResult(resultRawText) : ''),
[isCreateAgent, result, resultRawText],
);
const createAgentSessionId: string | null = useMemo(
() => (isCreateAgent && hasStructuredResult && resultContent?.sub_session_id) ? resultContent.sub_session_id : null,
[isCreateAgent, hasStructuredResult, resultContent],
);
const revealTargetSessionId = invokedSessionId || createAgentSessionId;
const sessions = useAppSelector((s) => s.agents.sessions);
const handleRevealAgent = useCallback(
(e: React.MouseEvent) => {
e.stopPropagation();
if (!revealTargetSessionId || !sessionId) return;
if (cards[revealTargetSessionId]) {
dispatch(collapseSession(revealTargetSessionId));
dispatch(removeCard(revealTargetSessionId));
setTimeout(() => {
dispatch(clearGlowingAgentCard(revealTargetSessionId));
}, 500);
return;
}
let sourceYRatio: number | undefined;
if (bubbleRef.current) {
const bubbleEl = bubbleRef.current;
const cardEl = bubbleEl.closest('[data-select-type="agent-card"]') as HTMLElement | null;
if (cardEl) {
const cardRect = cardEl.getBoundingClientRect();
const bubbleRect = bubbleEl.getBoundingClientRect();
const bubbleCenterY = bubbleRect.top + bubbleRect.height / 2;
const ratio = (bubbleCenterY - cardRect.top) / cardRect.height;
sourceYRatio = Math.max(0, Math.min(1, ratio));
}
}
const doPlace = () => {
const parentCard = cards[sessionId];
const targetX = parentCard
? parentCard.x + parentCard.width + GRID_GAP * 12
: 40;
let targetY = parentCard ? parentCard.y : 100;
if (parentCard) {
const columnCards = Object.values(cards).filter(
(c) => Math.abs(c.x - targetX) < 50 && c.session_id !== revealTargetSessionId,
);
if (columnCards.length > 0) {
const lowestBottom = Math.max(
...columnCards.map((c) => c.y + Math.max(EXPANDED_CARD_MIN_H, c.height)),
);
targetY = lowestBottom + GRID_GAP;
}
}
dispatch(placeCard({
sessionId: revealTargetSessionId,
x: targetX,
y: targetY,
width: DEFAULT_CARD_W,
height: DEFAULT_CARD_H,
}));
dispatch(expandSession(revealTargetSessionId));
const label = isCreateAgent ? 'Create Agent' : isInvokeAgent ? 'Invoke Agent' : 'Agent';
dispatch(setGlowingAgentCard({ sessionId: revealTargetSessionId, sourceId: sessionId, sourceYRatio, label }));
};
if (!sessions[revealTargetSessionId]) {
dispatch(fetchSession(revealTargetSessionId)).then(doPlace);
} else {
doPlace();
}
},
[revealTargetSessionId, sessionId, cards, sessions, dispatch],
);
const toggle = useCallback(() => {
if (!isStreaming) setExpanded((v) => !v);
}, [isStreaming]);
@@ -1233,10 +1407,415 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
'data-select-meta': JSON.stringify({ tool: toolName, inputSummary }),
};
if (isInvokeAgent) {
const agentName = invokeAgentParsed?.agentName || input?.session_id || 'Agent';
const responsePreview = invokeAgentParsed?.response || '';
const costLabel = invokeAgentParsed?.cost ? `$${invokeAgentParsed.cost}` : null;
const hasResponse = !!invokeAgentParsed;
return (
<Box ref={bubbleRef} {...selectAttrs} sx={{ maxWidth: '85%', my: 0.5 }}>
<Box
sx={{
'--glow-rgb': accentRgb,
bgcolor: c.bg.elevated,
border: `1px solid ${
isPending ? c.accent.primary : isDenied ? c.status.error + '60' : c.border.subtle
}`,
borderRadius: 2,
overflow: 'hidden',
animation: isPending ? 'border-glow 2s ease-in-out infinite' : 'none',
transition: 'border-color 0.3s, box-shadow 0.3s',
} as any}
>
{/* Header */}
<Box
onClick={toggle}
sx={{
display: 'flex',
alignItems: 'center',
gap: 0.75,
px: 1.5,
py: 0.75,
cursor: hasResponse ? 'pointer' : 'default',
'&:hover': hasResponse ? { bgcolor: 'rgba(0,0,0,0.02)' } : {},
}}
>
<CallSplitIcon sx={{ fontSize: 15, color: c.accent.primary, flexShrink: 0 }} />
<Typography
sx={{
color: c.accent.primary,
fontSize: '0.8rem',
fontWeight: 600,
flexShrink: 0,
}}
>
InvokeAgent
</Typography>
<Box
sx={{
display: 'inline-flex',
alignItems: 'center',
bgcolor: `${c.accent.primary}14`,
borderRadius: 1,
px: 0.75,
py: 0.15,
maxWidth: 180,
overflow: 'hidden',
}}
>
<Typography
noWrap
sx={{
fontSize: '0.72rem',
fontWeight: 500,
color: c.text.secondary,
fontFamily: c.font.sans,
}}
>
{agentName}
</Typography>
</Box>
{!hasResponse && !showTimer && <Box sx={{ flex: 1 }} />}
{hasResponse && responsePreview && !expanded && (
<Typography
noWrap
sx={{
flex: 1,
minWidth: 0,
fontSize: '0.73rem',
color: c.text.tertiary,
fontFamily: c.font.sans,
}}
>
{responsePreview.slice(0, 100)}{responsePreview.length > 100 ? '…' : ''}
</Typography>
)}
{expanded && <Box sx={{ flex: 1 }} />}
{isDenied && (
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.3 }}>
<BlockIcon sx={{ fontSize: 13, color: c.status.error }} />
<Typography sx={{ color: c.status.error, fontSize: '0.7rem', fontWeight: 500 }}>denied</Typography>
</Box>
)}
{hasResponse && !isDenied && (
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
{isError ? (
<ErrorOutlineIcon sx={{ fontSize: 13, color: c.status.error }} />
) : (
<CheckCircleOutlineIcon sx={{ fontSize: 13, color: c.status.success }} />
)}
{resultElapsedMs != null && (
<Typography sx={{ fontSize: '0.65rem', fontFamily: c.font.mono, color: c.text.tertiary }}>
{formatElapsed(resultElapsedMs)}
</Typography>
)}
{costLabel && (
<Typography sx={{ fontSize: '0.63rem', fontFamily: c.font.mono, color: c.text.tertiary }}>
{costLabel}
</Typography>
)}
</Box>
)}
{showTimer && <ElapsedTimer startTime={call.timestamp} />}
{invokedSessionId && (
<Tooltip title="Reveal on dashboard" arrow>
<IconButton
size="small"
onClick={handleRevealAgent}
sx={{
color: c.accent.primary,
p: 0.25,
flexShrink: 0,
'&:hover': { bgcolor: `${c.accent.primary}18` },
}}
>
<CallSplitIcon sx={{ fontSize: 15, transform: 'rotate(180deg)' }} />
</IconButton>
</Tooltip>
)}
{hasResponse && (
<IconButton size="small" sx={{ color: c.text.tertiary, p: 0.25, flexShrink: 0 }}>
{expanded ? <ExpandLessIcon sx={{ fontSize: 18 }} /> : <ExpandMoreIcon sx={{ fontSize: 18 }} />}
</IconButton>
)}
</Box>
{/* Expanded body — markdown rendered, not terminal */}
<Collapse in={expanded && hasResponse}>
<Box
sx={{
borderTop: `1px solid ${c.border.subtle}`,
px: 1.5,
py: 1.25,
maxHeight: 400,
overflowY: 'auto',
overflowX: 'hidden',
color: c.text.secondary,
fontFamily: c.font.sans,
fontSize: '0.78rem',
lineHeight: 1.65,
overflowWrap: 'anywhere',
wordBreak: 'break-word',
'& p': { m: 0, mb: 0.75, '&:last-child': { mb: 0 } },
'& h1, & h2, & h3, & h4': {
color: c.text.primary, fontFamily: c.font.sans,
mt: 1, mb: 0.5, '&:first-of-type': { mt: 0 },
},
'& h1': { fontSize: '0.88rem' }, '& h2': { fontSize: '0.84rem' },
'& h3': { fontSize: '0.8rem' }, '& h4': { fontSize: '0.78rem' },
'& strong': { color: c.text.primary, fontWeight: 600 },
'& a': { color: c.accent.primary, textDecoration: 'none', '&:hover': { textDecoration: 'underline' } },
'& ul, & ol': { pl: 2, mb: 0.75, mt: 0 },
'& li': { mb: 0.2 },
'& blockquote': {
m: 0, mb: 0.75, pl: 1, ml: 0,
borderLeft: `2px solid ${c.border.subtle}`,
color: c.text.tertiary, fontStyle: 'italic',
},
'& code': {
bgcolor: c.bg.secondary, px: 0.4, py: 0.15,
borderRadius: 0.5, fontSize: '0.72rem', fontFamily: c.font.mono,
},
'& pre': {
bgcolor: c.bg.secondary, borderRadius: 1, p: 1,
overflow: 'auto', fontSize: '0.72rem', fontFamily: c.font.mono,
m: 0, mb: 0.75,
},
'& pre code': { bgcolor: 'transparent', p: 0 },
'& hr': { border: 'none', borderTop: `1px solid ${c.border.subtle}`, my: 0.75 },
'&::-webkit-scrollbar': { width: 5 },
'&::-webkit-scrollbar-track': { background: 'transparent' },
'&::-webkit-scrollbar-thumb': { background: c.border.medium, borderRadius: 3 },
}}
>
<ReactMarkdown
remarkPlugins={[remarkGfm]}
components={{
a: ({ children, ...props }) => (
<a {...props}>{children}</a>
),
}}
>
{responsePreview}
</ReactMarkdown>
</Box>
</Collapse>
</Box>
</Box>
);
}
if (isCreateAgent) {
const taskPrompt = input?.prompt || input?.task || input?.message || '';
const taskLabel = taskPrompt
? taskPrompt.length > 40 ? taskPrompt.slice(0, 40) + '…' : taskPrompt
: 'Sub-agent';
const hasResponse = !!createAgentResponse;
return (
<Box ref={bubbleRef} {...selectAttrs} sx={{ maxWidth: '85%', my: 0.5 }}>
<Box
sx={{
'--glow-rgb': accentRgb,
bgcolor: c.bg.elevated,
border: `1px solid ${
isPending ? c.accent.primary : isDenied ? c.status.error + '60' : c.border.subtle
}`,
borderRadius: 2,
overflow: 'hidden',
animation: isPending ? 'border-glow 2s ease-in-out infinite' : 'none',
transition: 'border-color 0.3s, box-shadow 0.3s',
} as any}
>
<Box
onClick={toggle}
sx={{
display: 'flex',
alignItems: 'center',
gap: 0.75,
px: 1.5,
py: 0.75,
cursor: hasResponse ? 'pointer' : 'default',
'&:hover': hasResponse ? { bgcolor: 'rgba(0,0,0,0.02)' } : {},
}}
>
<CallSplitIcon sx={{ fontSize: 15, color: c.accent.primary, flexShrink: 0 }} />
<Typography
sx={{
color: c.accent.primary,
fontSize: '0.8rem',
fontWeight: 600,
flexShrink: 0,
}}
>
CreateAgent
</Typography>
<Box
sx={{
display: 'inline-flex',
alignItems: 'center',
bgcolor: `${c.accent.primary}14`,
borderRadius: 1,
px: 0.75,
py: 0.15,
maxWidth: 180,
overflow: 'hidden',
}}
>
<Typography
noWrap
sx={{
fontSize: '0.72rem',
fontWeight: 500,
color: c.text.secondary,
fontFamily: c.font.sans,
}}
>
{taskLabel}
</Typography>
</Box>
{!hasResponse && !showTimer && <Box sx={{ flex: 1 }} />}
{hasResponse && createAgentResponse && !expanded && (
<Typography
noWrap
sx={{
flex: 1,
minWidth: 0,
fontSize: '0.73rem',
color: c.text.tertiary,
fontFamily: c.font.sans,
}}
>
{createAgentResponse.slice(0, 100)}{createAgentResponse.length > 100 ? '…' : ''}
</Typography>
)}
{expanded && <Box sx={{ flex: 1 }} />}
{isDenied && (
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.3 }}>
<BlockIcon sx={{ fontSize: 13, color: c.status.error }} />
<Typography sx={{ color: c.status.error, fontSize: '0.7rem', fontWeight: 500 }}>denied</Typography>
</Box>
)}
{hasResponse && !isDenied && (
<Box sx={{ display: 'flex', alignItems: 'center', gap: 0.5 }}>
{isError ? (
<ErrorOutlineIcon sx={{ fontSize: 13, color: c.status.error }} />
) : (
<CheckCircleOutlineIcon sx={{ fontSize: 13, color: c.status.success }} />
)}
{resultElapsedMs != null && (
<Typography sx={{ fontSize: '0.65rem', fontFamily: c.font.mono, color: c.text.tertiary }}>
{formatElapsed(resultElapsedMs)}
</Typography>
)}
</Box>
)}
{showTimer && <ElapsedTimer startTime={call.timestamp} />}
{createAgentSessionId && (
<Tooltip title="Reveal on dashboard" arrow>
<IconButton
size="small"
onClick={handleRevealAgent}
sx={{
color: c.accent.primary,
p: 0.25,
flexShrink: 0,
'&:hover': { bgcolor: `${c.accent.primary}18` },
}}
>
<CallSplitIcon sx={{ fontSize: 15, transform: 'rotate(180deg)' }} />
</IconButton>
</Tooltip>
)}
{hasResponse && (
<IconButton size="small" sx={{ color: c.text.tertiary, p: 0.25, flexShrink: 0 }}>
{expanded ? <ExpandLessIcon sx={{ fontSize: 18 }} /> : <ExpandMoreIcon sx={{ fontSize: 18 }} />}
</IconButton>
)}
</Box>
<Collapse in={expanded && hasResponse}>
<Box
sx={{
borderTop: `1px solid ${c.border.subtle}`,
px: 1.5,
py: 1.25,
maxHeight: 400,
overflowY: 'auto',
overflowX: 'hidden',
color: c.text.secondary,
fontFamily: c.font.sans,
fontSize: '0.78rem',
lineHeight: 1.65,
overflowWrap: 'anywhere',
wordBreak: 'break-word',
'& p': { m: 0, mb: 0.75, '&:last-child': { mb: 0 } },
'& h1, & h2, & h3, & h4': {
color: c.text.primary, fontFamily: c.font.sans,
mt: 1, mb: 0.5, '&:first-of-type': { mt: 0 },
},
'& h1': { fontSize: '0.88rem' }, '& h2': { fontSize: '0.84rem' },
'& h3': { fontSize: '0.8rem' }, '& h4': { fontSize: '0.78rem' },
'& strong': { color: c.text.primary, fontWeight: 600 },
'& a': { color: c.accent.primary, textDecoration: 'none', '&:hover': { textDecoration: 'underline' } },
'& ul, & ol': { pl: 2, mb: 0.75, mt: 0 },
'& li': { mb: 0.2 },
'& blockquote': {
m: 0, mb: 0.75, pl: 1, ml: 0,
borderLeft: `2px solid ${c.border.subtle}`,
color: c.text.tertiary, fontStyle: 'italic',
},
'& code': {
bgcolor: c.bg.secondary, px: 0.4, py: 0.15,
borderRadius: 0.5, fontSize: '0.72rem', fontFamily: c.font.mono,
},
'& pre': {
bgcolor: c.bg.secondary, borderRadius: 1, p: 1,
overflow: 'auto', fontSize: '0.72rem', fontFamily: c.font.mono,
m: 0, mb: 0.75,
},
'& pre code': { bgcolor: 'transparent', p: 0 },
'& hr': { border: 'none', borderTop: `1px solid ${c.border.subtle}`, my: 0.75 },
'&::-webkit-scrollbar': { width: 5 },
'&::-webkit-scrollbar-track': { background: 'transparent' },
'&::-webkit-scrollbar-thumb': { background: c.border.medium, borderRadius: 3 },
}}
>
<ReactMarkdown
remarkPlugins={[remarkGfm]}
components={{
a: ({ children, ...props }) => (
<a {...props}>{children}</a>
),
}}
>
{createAgentResponse}
</ReactMarkdown>
</Box>
</Collapse>
</Box>
</Box>
);
}
if (mcpCompact && mcpInfo.isMcp) {
return (
<Box {...selectAttrs} sx={{ my: 0 }}>
<style>{pulsingKeyframes}</style>
<Box
onClick={toggle}
sx={{
@@ -1320,6 +1899,12 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
'&::-webkit-scrollbar-thumb': { background: tc.SCROLLBAR_THUMB, borderRadius: 3 },
}}
>
{isBrowserAgent && sessionId && (
<BrowserAgentInlineFeed
parentSessionId={sessionId}
browserId={input?.browser_id}
/>
)}
{parsedResult && parsedResult.type === 'mcp' ? (
<McpResultCard parsed={parsedResult} compact />
) : parsedResult ? (
@@ -1330,7 +1915,7 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
{parsedResult.type === 'text' ? parsedResult.content : ''}
</pre>
) : null}
{!parsedResult && isPending && !isStreaming && (
{!parsedResult && isPending && !isStreaming && !isBrowserAgent && (
<Box sx={{ px: 1.5, py: 1 }}>
<Box sx={{ width: 8, height: 2, bgcolor: tc.PROMPT_COLOR, animation: 'tool-pulse 1s ease-in-out infinite', borderRadius: 1 }} />
</Box>
@@ -1343,8 +1928,6 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
return (
<Box {...selectAttrs} sx={{ maxWidth: mcpCompact ? '100%' : '85%', my: mcpCompact ? 0 : 0.5 }}>
<style>{pulsingKeyframes}</style>
{isStreaming && <style>{streamingCursorKeyframes}</style>}
<Box
sx={{
'--glow-rgb': accentRgb,
@@ -1525,6 +2108,14 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
)}
</pre>
{/* Browser agent inline feed */}
{isBrowserAgent && sessionId && (
<BrowserAgentInlineFeed
parentSessionId={sessionId}
browserId={input?.browser_id}
/>
)}
{/* Output */}
{parsedResult && parsedResult.type === 'mcp' ? (
<McpResultCard parsed={parsedResult} />
@@ -1566,8 +2157,8 @@ const ToolCallBubble: React.FC<ToolCallBubbleProps> = React.memo(
</pre>
) : null}
{/* Pending indicator when waiting for result */}
{!parsedResult && isPending && !isStreaming && (
{/* Pending indicator when waiting for result (skip for browser agent — feed replaces it) */}
{!parsedResult && isPending && !isStreaming && !isBrowserAgent && (
<Box sx={{ px: 1.5, pb: 1, pt: 0.5 }}>
<Box
sx={{
@@ -68,9 +68,10 @@ interface Props {
group: ToolGroup;
isSessionRunning?: boolean;
meta?: ToolGroupMeta;
sessionId?: string;
}
const ToolGroupBubble: React.FC<Props> = React.memo(({ group, isSessionRunning = false, meta }) => {
const ToolGroupBubble: React.FC<Props> = React.memo(({ group, isSessionRunning = false, meta, sessionId }) => {
const c = useClaudeTokens();
const isMcp = !!group.mcpServer;
const [expanded, setExpanded] = useState(isMcp);
@@ -186,6 +187,7 @@ const ToolGroupBubble: React.FC<Props> = React.memo(({ group, isSessionRunning =
result={pair.result}
isPending={pair.result === null && isSessionRunning}
mcpCompact
sessionId={sessionId}
/>
))}
</Box>
@@ -0,0 +1,63 @@
import React from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import Paper from '@mui/material/Paper';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
const Analytics: React.FC = () => {
const c = useClaudeTokens();
return (
<Box sx={{ height: '100%', overflow: 'auto', p: 3 }}>
<Box sx={{ maxWidth: 800, mx: 'auto' }}>
<Typography variant="h5" sx={{ color: c.text.primary, fontWeight: 600, mb: 3 }}>
Analytics
</Typography>
<Paper sx={{
p: 4,
bgcolor: c.bg.surface,
border: `1px solid ${c.border.subtle}`,
textAlign: 'center',
}}>
<Box sx={{ mb: 2 }}>
<svg width="48" height="48" viewBox="0 0 24 24" fill="none" stroke={c.accent.primary} strokeWidth="1.5">
<path d="M3 3v18h18" />
<path d="M7 16l4-4 4 4 5-5" />
<circle cx="20" cy="7" r="1.5" fill={c.accent.primary} />
</svg>
</Box>
<Typography sx={{ color: c.text.primary, fontSize: '1.1rem', fontWeight: 600, mb: 1 }}>
Analytics powered by PostHog
</Typography>
<Typography sx={{ color: c.text.muted, fontSize: '0.85rem', lineHeight: 1.6, mb: 3, maxWidth: 500, mx: 'auto' }}>
Usage data is automatically collected sessions, costs, tool usage, model distribution, and task categories.
All data is anonymous and can be disabled in Settings.
</Typography>
<Box sx={{ display: 'grid', gridTemplateColumns: 'repeat(3, 1fr)', gap: 2, mt: 3, textAlign: 'left' }}>
{[
{ label: 'Sessions & Usage', desc: 'How often agents are launched, session duration, completion rates' },
{ label: 'Cost Tracking', desc: 'Spend by model, provider, and time period' },
{ label: 'Task Categories', desc: 'What users do — coding, email, research, social, browsing' },
{ label: 'Model Distribution', desc: 'Which models and providers are most popular' },
{ label: 'Tool Usage', desc: 'Most used MCP tools, execution times, approval rates' },
{ label: 'Retention & Funnels', desc: 'User engagement, feature adoption, onboarding flow' },
].map((item) => (
<Box key={item.label} sx={{ p: 2, borderRadius: `${c.radius.md}px`, bgcolor: c.bg.elevated }}>
<Typography sx={{ color: c.text.primary, fontSize: '0.82rem', fontWeight: 600, mb: 0.5 }}>
{item.label}
</Typography>
<Typography sx={{ color: c.text.muted, fontSize: '0.72rem', lineHeight: 1.4 }}>
{item.desc}
</Typography>
</Box>
))}
</Box>
</Paper>
</Box>
</Box>
);
};
export default Analytics;
@@ -0,0 +1,406 @@
import React, { useRef, useEffect, useCallback } from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
const PALETTES = {
salmon: ['#C46B57', '#D4795F', '#E8927A', '#F0A088', '#F5B49E'],
blue: ['#445588', '#5577AA', '#6688BB', '#7799CC', '#88AADD'],
coral: ['#993344', '#AA3D4E', '#BB4455', '#CC5566', '#DD6677'],
green: ['#447755', '#558866', '#669977', '#77AA88', '#88BB99'],
purple: ['#665588', '#7766AA', '#8877BB', '#9988CC', '#AA99DD'],
} as const;
type PaletteKey = keyof typeof PALETTES;
interface PixelChartProps {
data: { label: string; value: number }[];
palette?: PaletteKey;
height?: number;
pixelSize?: number;
formatValue?: (v: number) => string;
glow?: boolean;
showXLabels?: boolean;
showYScale?: boolean;
mode?: 'bar' | 'area'; // 'area' draws a filled line chart instead of bars
}
const PixelChart: React.FC<PixelChartProps> = ({
data,
palette = 'salmon',
height = 140,
pixelSize = 6,
formatValue,
glow = true,
showXLabels = true,
showYScale = true,
mode = 'bar',
}) => {
const canvasRef = useRef<HTMLCanvasElement>(null);
const containerRef = useRef<HTMLDivElement>(null);
const animRef = useRef(0);
const progressRef = useRef(0);
const hoverIdxRef = useRef(-1);
const tooltipRef = useRef<HTMLDivElement>(null);
const c = useClaudeTokens();
const colors = PALETTES[palette];
const maxVal = Math.max(...data.map((d) => d.value), 0.001);
// Compute nice Y-axis ticks
const yTicks = (() => {
if (maxVal <= 0) return [0];
const rawStep = maxVal / 3;
const magnitude = Math.pow(10, Math.floor(Math.log10(rawStep)));
const normalised = rawStep / magnitude;
let niceStep: number;
if (normalised <= 1) niceStep = magnitude;
else if (normalised <= 2) niceStep = 2 * magnitude;
else if (normalised <= 5) niceStep = 5 * magnitude;
else niceStep = 10 * magnitude;
const ticks: number[] = [];
for (let v = 0; v <= maxVal * 1.1; v += niceStep) {
ticks.push(v);
}
if (ticks.length < 2) ticks.push(niceStep);
return ticks;
})();
// X-axis labels: show first, last, and up to 3 evenly spaced
const xLabels = (() => {
if (data.length <= 1) return data.map((d, i) => ({ idx: i, label: d.label }));
if (data.length <= 5) return data.map((d, i) => ({ idx: i, label: d.label }));
const result: { idx: number; label: string }[] = [];
result.push({ idx: 0, label: data[0].label });
const step = Math.floor(data.length / 4);
for (let i = 1; i <= 3; i++) {
const idx = Math.min(i * step, data.length - 2);
if (idx > 0 && idx < data.length - 1) {
result.push({ idx, label: data[idx].label });
}
}
result.push({ idx: data.length - 1, label: data[data.length - 1].label });
return result;
})();
const Y_LABEL_WIDTH = showYScale ? 80 : 0;
const draw = useCallback(() => {
const canvas = canvasRef.current;
const container = containerRef.current;
if (!canvas || !container || data.length === 0) return;
const dpr = window.devicePixelRatio || 1;
const totalW = container.clientWidth;
const chartW = totalW - Y_LABEL_WIDTH;
const h = height;
canvas.width = totalW * dpr;
canvas.height = h * dpr;
canvas.style.width = `${totalW}px`;
canvas.style.height = `${h}px`;
const ctx = canvas.getContext('2d');
if (!ctx) return;
ctx.scale(dpr, dpr);
const px = pixelSize;
const gridCols = Math.floor(chartW / px);
const gridRows = Math.floor(h / px);
const effectiveMax = yTicks[yTicks.length - 1] || maxVal;
ctx.clearRect(0, 0, totalW, h);
// Y-axis labels and horizontal grid lines
if (showYScale) {
ctx.font = '10px monospace';
ctx.textAlign = 'right';
ctx.textBaseline = 'middle';
for (const tick of yTicks) {
const yNorm = effectiveMax > 0 ? tick / effectiveMax : 0;
const yPx = h - yNorm * (h - px);
// Grid line
ctx.strokeStyle = c.border.subtle;
ctx.lineWidth = 0.5;
ctx.setLineDash([2, 4]);
ctx.beginPath();
ctx.moveTo(Y_LABEL_WIDTH, yPx);
ctx.lineTo(totalW, yPx);
ctx.stroke();
ctx.setLineDash([]);
// Label
const label = formatValue ? formatValue(tick) : (tick % 1 === 0 ? String(tick) : tick.toFixed(1));
ctx.fillStyle = c.text.ghost;
ctx.fillText(label, Y_LABEL_WIDTH - 8, yPx);
}
}
// Subtle grid dots in chart area
ctx.fillStyle = c.border.subtle;
for (let gy = 0; gy < gridRows; gy += 5) {
for (let gx = 0; gx < gridCols; gx += 5) {
ctx.fillRect(Y_LABEL_WIDTH + gx * px, gy * px, 1, 1);
}
}
const progress = Math.min(progressRef.current, 1);
const hoverIdx = hoverIdxRef.current;
if (mode === 'area') {
// ── Area / line chart mode ──
// Draw a smooth filled area under a line
const usableH = h - px * 2;
const points: { x: number; y: number }[] = [];
for (let i = 0; i < data.length; i++) {
const val = data[i].value;
const norm = effectiveMax > 0 ? val / effectiveMax : 0;
const x = Y_LABEL_WIDTH + (i / Math.max(data.length - 1, 1)) * chartW;
const y = h - px - norm * usableH * progress;
points.push({ x, y });
}
if (points.length > 0) {
// Filled area with gradient
const gradient = ctx.createLinearGradient(0, 0, 0, h);
gradient.addColorStop(0, colors[colors.length - 1] + '60');
gradient.addColorStop(0.5, colors[Math.floor(colors.length / 2)] + '30');
gradient.addColorStop(1, colors[0] + '08');
ctx.beginPath();
ctx.moveTo(points[0].x, h);
// Smooth curve through points
for (let i = 0; i < points.length; i++) {
if (i === 0) {
ctx.lineTo(points[i].x, points[i].y);
} else {
// Cubic bezier for smoothing
const prev = points[i - 1];
const curr = points[i];
const cpx = (prev.x + curr.x) / 2;
ctx.bezierCurveTo(cpx, prev.y, cpx, curr.y, curr.x, curr.y);
}
}
ctx.lineTo(points[points.length - 1].x, h);
ctx.closePath();
ctx.fillStyle = gradient;
ctx.fill();
// Line on top
ctx.beginPath();
for (let i = 0; i < points.length; i++) {
if (i === 0) {
ctx.moveTo(points[i].x, points[i].y);
} else {
const prev = points[i - 1];
const curr = points[i];
const cpx = (prev.x + curr.x) / 2;
ctx.bezierCurveTo(cpx, prev.y, cpx, curr.y, curr.x, curr.y);
}
}
ctx.strokeStyle = colors[colors.length - 1];
ctx.lineWidth = 2;
ctx.stroke();
// Glow on line
if (glow) {
ctx.shadowColor = colors[colors.length - 1];
ctx.shadowBlur = 8;
ctx.stroke();
ctx.shadowBlur = 0;
}
// Data point dots
for (let i = 0; i < points.length; i++) {
if (data[i].value > 0) {
const isHov = i === hoverIdx;
ctx.beginPath();
ctx.arc(points[i].x, points[i].y, isHov ? 4 : 2.5, 0, Math.PI * 2);
ctx.fillStyle = isHov ? colors[colors.length - 1] : colors[Math.floor(colors.length / 2)];
ctx.fill();
if (isHov) {
ctx.strokeStyle = colors[colors.length - 1];
ctx.lineWidth = 1.5;
ctx.stroke();
}
}
}
// Pixel scatter in the filled area for the pixel art feel
for (let i = 0; i < points.length - 1; i++) {
const p1 = points[i];
const p2 = points[i + 1];
const steps = Math.ceil((p2.x - p1.x) / px);
for (let s = 0; s < steps; s++) {
const t = s / steps;
const x = p1.x + t * (p2.x - p1.x);
const lineY = p1.y + t * (p2.y - p1.y);
// Scatter pixels below the line
for (let py = lineY + px * 2; py < h - px; py += px * 2) {
if (Math.random() > 0.65) {
const depth = (py - lineY) / (h - lineY);
const ci = Math.max(0, Math.floor((1 - depth) * (colors.length - 1)));
ctx.globalAlpha = 0.15 + (1 - depth) * 0.2;
ctx.fillStyle = colors[ci];
ctx.fillRect(Math.floor(x / px) * px, Math.floor(py / px) * px, px - 1, px - 1);
}
}
}
}
ctx.globalAlpha = 1;
}
} else {
// ── Bar chart mode (original) ──
const barSlots = data.length;
const totalBarPx = Math.max(1, Math.floor(gridCols / barSlots));
const barW = Math.max(1, totalBarPx - 1);
for (let i = 0; i < data.length; i++) {
const val = data[i].value;
const normalised = effectiveMax > 0 ? val / effectiveMax : 0;
const usableRows = gridRows - 2;
const targetH = Math.max(normalised > 0 ? 1 : 0, Math.round(normalised * usableRows));
const barH = Math.round(targetH * progress);
const barX = i * totalBarPx;
const isHovered = i === hoverIdx;
for (let row = 0; row < barH; row++) {
const y = gridRows - 1 - row;
const colorIdx = Math.min(colors.length - 1, Math.floor((row / Math.max(barH - 1, 1)) * (colors.length - 1)));
const baseColor = isHovered ? colors[Math.min(colorIdx + 1, colors.length - 1)] : colors[colorIdx];
for (let col = 0; col < barW; col++) {
ctx.fillStyle = baseColor;
ctx.fillRect(Y_LABEL_WIDTH + (barX + col) * px, y * px, px - 1, px - 1);
}
}
if (glow && barH > 0) {
const topY = (gridRows - 1 - barH + 1) * px;
ctx.shadowColor = colors[colors.length - 1];
ctx.shadowBlur = 6;
ctx.fillStyle = colors[colors.length - 1];
for (let col = 0; col < barW; col++) {
ctx.fillRect(Y_LABEL_WIDTH + (barX + col) * px, topY, px - 1, px - 1);
}
ctx.shadowBlur = 0;
}
}
}
}, [data, height, pixelSize, c, colors, glow, maxVal, yTicks, showYScale, Y_LABEL_WIDTH, formatValue, mode]);
useEffect(() => {
progressRef.current = 0;
let start: number | null = null;
const animate = (ts: number) => {
if (!start) start = ts;
progressRef.current = Math.min(1, (ts - start) / 600);
draw();
if (progressRef.current < 1) animRef.current = requestAnimationFrame(animate);
};
animRef.current = requestAnimationFrame(animate);
return () => cancelAnimationFrame(animRef.current);
}, [data, draw]);
useEffect(() => {
const handleResize = () => draw();
window.addEventListener('resize', handleResize);
return () => window.removeEventListener('resize', handleResize);
}, [draw]);
const handleMouseMove = useCallback(
(e: React.MouseEvent) => {
const canvas = canvasRef.current;
const tooltip = tooltipRef.current;
if (!canvas || !tooltip || data.length === 0) return;
const rect = canvas.getBoundingClientRect();
const mx = e.clientX - rect.left - Y_LABEL_WIDTH;
if (mx < 0) { hoverIdxRef.current = -1; tooltip.style.opacity = '0'; draw(); return; }
const chartW = rect.width - Y_LABEL_WIDTH;
const gridCols = Math.floor(chartW / pixelSize);
const totalBarPx = Math.max(1, Math.floor(gridCols / data.length));
const idx = Math.floor(mx / (totalBarPx * pixelSize));
if (idx >= 0 && idx < data.length) {
hoverIdxRef.current = idx;
const d = data[idx];
const valStr = formatValue ? formatValue(d.value) : d.value.toFixed(2);
tooltip.textContent = `${d.label}: ${valStr}`;
tooltip.style.opacity = '1';
tooltip.style.left = `${e.clientX - rect.left}px`;
tooltip.style.top = `${e.clientY - rect.top - 28}px`;
} else {
hoverIdxRef.current = -1;
tooltip.style.opacity = '0';
}
draw();
},
[data, pixelSize, draw, formatValue, Y_LABEL_WIDTH],
);
const handleMouseLeave = useCallback(() => {
hoverIdxRef.current = -1;
if (tooltipRef.current) tooltipRef.current.style.opacity = '0';
draw();
}, [draw]);
return (
<Box ref={containerRef} sx={{ position: 'relative', width: '100%' }}>
<canvas
ref={canvasRef}
onMouseMove={handleMouseMove}
onMouseLeave={handleMouseLeave}
style={{ display: 'block', width: '100%', imageRendering: 'pixelated', cursor: 'crosshair' }}
/>
{/* X-axis labels */}
{showXLabels && data.length > 0 && (
<Box sx={{ display: 'flex', justifyContent: 'space-between', mt: 0.5, pl: `${Y_LABEL_WIDTH}px` }}>
{xLabels.map((xl) => (
<Typography
key={xl.idx}
sx={{
color: c.text.ghost,
fontSize: '0.58rem',
fontFamily: c.font.mono,
whiteSpace: 'nowrap',
overflow: 'hidden',
textOverflow: 'ellipsis',
maxWidth: 60,
}}
>
{xl.label}
</Typography>
))}
</Box>
)}
{/* Tooltip */}
<Box
ref={tooltipRef}
sx={{
position: 'absolute',
pointerEvents: 'none',
opacity: 0,
transition: 'opacity 0.12s',
bgcolor: c.bg.inverse,
color: c.text.inverse,
fontSize: '0.7rem',
fontFamily: c.font.mono,
fontWeight: 500,
px: 1,
py: 0.35,
borderRadius: 0.75,
whiteSpace: 'nowrap',
transform: 'translateX(-50%)',
zIndex: 10,
boxShadow: '0 2px 8px rgba(0,0,0,0.3)',
}}
/>
</Box>
);
};
export default PixelChart;
@@ -0,0 +1,681 @@
import React, { useEffect, useState } from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import Button from '@mui/material/Button';
import IconButton from '@mui/material/IconButton';
import TextField from '@mui/material/TextField';
import Select from '@mui/material/Select';
import MenuItem from '@mui/material/MenuItem';
import Chip from '@mui/material/Chip';
import Dialog from '@mui/material/Dialog';
import DialogTitle from '@mui/material/DialogTitle';
import DialogContent from '@mui/material/DialogContent';
import DialogActions from '@mui/material/DialogActions';
import FormControl from '@mui/material/FormControl';
import InputLabel from '@mui/material/InputLabel';
import Switch from '@mui/material/Switch';
import FormControlLabel from '@mui/material/FormControlLabel';
import Tabs from '@mui/material/Tabs';
import Tab from '@mui/material/Tab';
import Tooltip from '@mui/material/Tooltip';
import AddIcon from '@mui/icons-material/Add';
import DeleteIcon from '@mui/icons-material/Delete';
import PhoneIcon from '@mui/icons-material/Phone';
import SmsIcon from '@mui/icons-material/Sms';
import WhatsAppIcon from '@mui/icons-material/WhatsApp';
import PlayArrowIcon from '@mui/icons-material/PlayArrow';
import StopIcon from '@mui/icons-material/Stop';
import SendIcon from '@mui/icons-material/Send';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import {
fetchChannels,
createChannel,
updateChannel,
deleteChannel,
enableChannel,
disableChannel,
testChannel,
fetchConversations,
ChannelConfig,
} from '@/shared/state/channelsSlice';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
const CHANNEL_TYPE_ICONS: Record<string, React.ReactNode> = {
sms: <SmsIcon />,
whatsapp: <WhatsAppIcon />,
voice: <PhoneIcon />,
};
const STATUS_COLORS: Record<string, string> = {
active: '#4caf50',
inactive: '#9e9e9e',
error: '#f44336',
};
const Channels: React.FC = () => {
const c = useClaudeTokens();
const dispatch = useAppDispatch();
const channels = useAppSelector((s) => s.channels.items);
const conversations = useAppSelector((s) => s.channels.conversations);
const channelList = Object.values(channels).sort(
(a, b) => new Date(b.updated_at).getTime() - new Date(a.updated_at).getTime(),
);
const [selectedId, setSelectedId] = useState<string | null>(null);
const [createOpen, setCreateOpen] = useState(false);
const [tab, setTab] = useState(0);
// Create form
const [newName, setNewName] = useState('');
const [newType, setNewType] = useState<'sms' | 'whatsapp' | 'voice'>('sms');
const [newProvider, setNewProvider] = useState<'twilio' | 'telnyx'>('twilio');
const [newPhone, setNewPhone] = useState('');
const [newAccountSid, setNewAccountSid] = useState('');
const [newAuthToken, setNewAuthToken] = useState('');
// Test
const [testNumber, setTestNumber] = useState('');
const [testResult, setTestResult] = useState('');
useEffect(() => {
dispatch(fetchChannels());
}, [dispatch]);
const selected = selectedId ? channels[selectedId] : null;
useEffect(() => {
if (selectedId) dispatch(fetchConversations(selectedId));
}, [selectedId, dispatch]);
const handleCreate = async () => {
const result = await dispatch(
createChannel({
name: newName,
channel_type: newType,
provider: newProvider,
phone_number: newPhone,
credentials: {
account_sid: newAccountSid,
auth_token: newAuthToken,
},
}),
);
if (createChannel.fulfilled.match(result)) {
setSelectedId(result.payload.id);
setCreateOpen(false);
setNewName('');
setNewPhone('');
setNewAccountSid('');
setNewAuthToken('');
}
};
const handleDelete = async (id: string) => {
await dispatch(deleteChannel(id));
if (selectedId === id) setSelectedId(null);
};
const handleTest = async () => {
if (!selectedId || !testNumber) return;
try {
const result = await dispatch(testChannel({ id: selectedId, to_number: testNumber }));
if (testChannel.fulfilled.match(result)) {
setTestResult('Test sent successfully!');
} else {
setTestResult('Test failed');
}
} catch {
setTestResult('Test failed');
}
};
const convList = Object.values(conversations).filter(
(cv) => cv.channel_id === selectedId,
);
return (
<Box sx={{ display: 'flex', height: '100%', overflow: 'hidden' }}>
{/* Left: Channel list */}
<Box
sx={{
width: 320,
flexShrink: 0,
borderRight: `1px solid ${c.border.subtle}`,
display: 'flex',
flexDirection: 'column',
bgcolor: c.bg.secondary,
}}
>
<Box sx={{ p: 2, display: 'flex', alignItems: 'center', justifyContent: 'space-between' }}>
<Typography sx={{ fontSize: '1rem', fontWeight: 600, color: c.text.primary }}>
Channels
</Typography>
<Tooltip title="New channel">
<IconButton
size="small"
onClick={() => setCreateOpen(true)}
sx={{ color: c.accent.primary }}
>
<AddIcon />
</IconButton>
</Tooltip>
</Box>
<Box sx={{ px: 1.5, pb: 1 }}>
<Typography sx={{ fontSize: '0.75rem', color: c.text.muted, lineHeight: 1.4 }}>
Connect SMS, WhatsApp, or voice calls to your agents.
</Typography>
</Box>
<Box sx={{ flex: 1, overflow: 'auto', px: 1 }}>
{channelList.length === 0 && (
<Box sx={{ p: 3, textAlign: 'center' }}>
<Typography sx={{ color: c.text.muted, fontSize: '0.85rem' }}>
No channels configured yet
</Typography>
</Box>
)}
{channelList.map((ch) => (
<Box
key={ch.id}
onClick={() => setSelectedId(ch.id)}
sx={{
p: 1.5,
mb: 0.5,
borderRadius: 2,
cursor: 'pointer',
bgcolor: selectedId === ch.id ? `${c.accent.primary}14` : 'transparent',
border: selectedId === ch.id ? `1px solid ${c.accent.primary}40` : '1px solid transparent',
'&:hover': { bgcolor: `${c.text.tertiary}0A` },
transition: 'all 0.15s',
}}
>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
<Box sx={{ color: c.text.muted }}>{CHANNEL_TYPE_ICONS[ch.channel_type]}</Box>
<Box sx={{ flex: 1, minWidth: 0 }}>
<Typography
sx={{
fontSize: '0.85rem',
fontWeight: 500,
color: c.text.primary,
overflow: 'hidden',
textOverflow: 'ellipsis',
whiteSpace: 'nowrap',
}}
>
{ch.name}
</Typography>
<Typography sx={{ fontSize: '0.72rem', color: c.text.muted }}>
{ch.phone_number} · {ch.provider}
</Typography>
</Box>
<Box
sx={{
width: 8,
height: 8,
borderRadius: '50%',
bgcolor: STATUS_COLORS[ch.status] || '#9e9e9e',
}}
/>
</Box>
</Box>
))}
</Box>
</Box>
{/* Right: Detail panel */}
<Box sx={{ flex: 1, overflow: 'auto', p: 3 }}>
{!selected ? (
<Box sx={{ display: 'flex', alignItems: 'center', justifyContent: 'center', height: '100%' }}>
<Typography sx={{ color: c.text.muted }}>Select a channel or create a new one</Typography>
</Box>
) : (
<>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 2, mb: 3 }}>
<Box sx={{ color: c.text.muted, fontSize: 28 }}>{CHANNEL_TYPE_ICONS[selected.channel_type]}</Box>
<Box sx={{ flex: 1 }}>
<Typography sx={{ fontSize: '1.2rem', fontWeight: 600, color: c.text.primary }}>
{selected.name}
</Typography>
<Box sx={{ display: 'flex', gap: 1, mt: 0.5 }}>
<Chip
label={selected.channel_type.toUpperCase()}
size="small"
sx={{ fontSize: '0.7rem', height: 22 }}
/>
<Chip
label={selected.status}
size="small"
sx={{
fontSize: '0.7rem',
height: 22,
bgcolor: `${STATUS_COLORS[selected.status]}20`,
color: STATUS_COLORS[selected.status],
}}
/>
<Chip label={`${selected.message_count} messages`} size="small" sx={{ fontSize: '0.7rem', height: 22 }} />
</Box>
</Box>
<Box sx={{ display: 'flex', gap: 1 }}>
{selected.enabled ? (
<Button
size="small"
variant="outlined"
startIcon={<StopIcon />}
onClick={() => dispatch(disableChannel(selected.id))}
color="error"
>
Disable
</Button>
) : (
<Button
size="small"
variant="contained"
startIcon={<PlayArrowIcon />}
onClick={() => dispatch(enableChannel(selected.id))}
sx={{ bgcolor: c.accent.primary }}
>
Enable
</Button>
)}
<IconButton size="small" onClick={() => handleDelete(selected.id)} sx={{ color: c.status.error }}>
<DeleteIcon />
</IconButton>
</Box>
</Box>
<Tabs value={tab} onChange={(_, v) => setTab(v)} sx={{ mb: 2, borderBottom: `1px solid ${c.border.subtle}` }}>
<Tab label="Connection" sx={{ textTransform: 'none' }} />
<Tab label="Agent" sx={{ textTransform: 'none' }} />
<Tab label="Security" sx={{ textTransform: 'none' }} />
{selected.channel_type === 'voice' && <Tab label="Voice" sx={{ textTransform: 'none' }} />}
<Tab label="Conversations" sx={{ textTransform: 'none' }} />
<Tab label="Test" sx={{ textTransform: 'none' }} />
</Tabs>
{/* Connection Tab */}
{tab === 0 && (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxWidth: 500 }}>
<TextField
label="Phone Number"
value={selected.phone_number}
size="small"
onChange={(e) =>
dispatch(updateChannel({ id: selected.id, phone_number: e.target.value }))
}
/>
<FormControl size="small">
<InputLabel>Provider</InputLabel>
<Select
value={selected.provider}
label="Provider"
onChange={(e) =>
dispatch(updateChannel({ id: selected.id, provider: e.target.value as any }))
}
>
<MenuItem value="twilio">Twilio</MenuItem>
<MenuItem value="telnyx">Telnyx</MenuItem>
</Select>
</FormControl>
<TextField
label="Account SID / API Key"
value={selected.credentials.account_sid || selected.credentials.api_key || ''}
size="small"
type="password"
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
credentials: {
...selected.credentials,
[selected.provider === 'twilio' ? 'account_sid' : 'api_key']: e.target.value,
},
}),
)
}
/>
<TextField
label="Auth Token / Public Key"
value={selected.credentials.auth_token || selected.credentials.public_key || ''}
size="small"
type="password"
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
credentials: {
...selected.credentials,
[selected.provider === 'twilio' ? 'auth_token' : 'public_key']: e.target.value,
},
}),
)
}
/>
<Typography sx={{ fontSize: '0.75rem', color: c.text.muted, mt: 1 }}>
Webhook URL for Twilio: <code>{window.location.origin.replace(/:\d+$/, ':8324')}/api/channels/webhooks/twilio/{selected.channel_type}?channel_id={selected.id}</code>
</Typography>
</Box>
)}
{/* Agent Tab */}
{tab === 1 && (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxWidth: 500 }}>
<FormControl size="small">
<InputLabel>Mode</InputLabel>
<Select
value={selected.agent_config.mode}
label="Mode"
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
agent_config: { ...selected.agent_config, mode: e.target.value },
}),
)
}
>
<MenuItem value="agent">Agent</MenuItem>
<MenuItem value="ask">Ask</MenuItem>
<MenuItem value="plan">Plan</MenuItem>
</Select>
</FormControl>
<FormControl size="small">
<InputLabel>Model</InputLabel>
<Select
value={selected.agent_config.model}
label="Model"
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
agent_config: { ...selected.agent_config, model: e.target.value },
}),
)
}
>
<MenuItem value="haiku">Haiku</MenuItem>
<MenuItem value="sonnet">Sonnet</MenuItem>
<MenuItem value="opus">Opus</MenuItem>
</Select>
</FormControl>
<TextField
label="System Prompt (optional)"
value={selected.agent_config.system_prompt || ''}
size="small"
multiline
rows={4}
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
agent_config: { ...selected.agent_config, system_prompt: e.target.value || undefined },
}),
)
}
/>
</Box>
)}
{/* Security Tab */}
{tab === 2 && (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxWidth: 500 }}>
<FormControlLabel
control={
<Switch
checked={selected.security.verify_signatures}
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
security: { ...selected.security, verify_signatures: e.target.checked },
}),
)
}
/>
}
label="Verify webhook signatures"
/>
<TextField
label="Allowlist (one phone per line)"
value={selected.security.allowlist.join('\n')}
size="small"
multiline
rows={4}
placeholder="+1234567890"
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
security: {
...selected.security,
allowlist: e.target.value.split('\n').filter(Boolean),
},
}),
)
}
/>
<TextField
label="Rate limit (per minute)"
value={selected.security.rate_limit_per_minute}
size="small"
type="number"
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
security: {
...selected.security,
rate_limit_per_minute: parseInt(e.target.value) || 10,
},
}),
)
}
/>
</Box>
)}
{/* Voice Tab */}
{tab === 3 && selected.channel_type === 'voice' && (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxWidth: 500 }}>
<FormControl size="small">
<InputLabel>Call Mode</InputLabel>
<Select
value={selected.voice_config?.mode || 'conversation'}
label="Call Mode"
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
voice_config: {
...(selected.voice_config || {}),
mode: e.target.value,
} as any,
}),
)
}
>
<MenuItem value="conversation">Conversation (multi-turn)</MenuItem>
<MenuItem value="notify">Notify (one-shot)</MenuItem>
</Select>
</FormControl>
<TextField
label="Greeting Message"
value={selected.voice_config?.greeting_message || 'Hello, how can I help you?'}
size="small"
multiline
rows={2}
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
voice_config: {
...(selected.voice_config || {}),
greeting_message: e.target.value,
} as any,
}),
)
}
/>
<TextField
label="Voice (e.g. Polly.Joanna)"
value={selected.voice_config?.voice || 'Polly.Joanna'}
size="small"
onChange={(e) =>
dispatch(
updateChannel({
id: selected.id,
voice_config: {
...(selected.voice_config || {}),
voice: e.target.value,
} as any,
}),
)
}
/>
</Box>
)}
{/* Conversations Tab */}
{tab === (selected.channel_type === 'voice' ? 4 : 3) && (
<Box>
{convList.length === 0 ? (
<Typography sx={{ color: c.text.muted }}>No conversations yet</Typography>
) : (
convList.map((conv) => (
<Box
key={conv.id}
sx={{
p: 2,
mb: 1,
borderRadius: 2,
border: `1px solid ${c.border.subtle}`,
bgcolor: c.bg.surface,
}}
>
<Typography sx={{ fontWeight: 500, fontSize: '0.85rem', color: c.text.primary }}>
{conv.phone_number}
</Typography>
<Typography sx={{ fontSize: '0.72rem', color: c.text.muted }}>
{conv.messages.length} messages · {conv.status}
</Typography>
<Box sx={{ mt: 1, maxHeight: 200, overflow: 'auto' }}>
{conv.messages.slice(-5).map((msg) => (
<Box
key={msg.id}
sx={{
p: 0.75,
mb: 0.5,
borderRadius: 1,
bgcolor: msg.direction === 'inbound' ? `${c.accent.primary}0A` : c.bg.secondary,
fontSize: '0.78rem',
color: c.text.primary,
}}
>
<Typography sx={{ fontSize: '0.65rem', color: c.text.muted, mb: 0.25 }}>
{msg.direction === 'inbound' ? 'Received' : 'Sent'} ·{' '}
{new Date(msg.timestamp).toLocaleTimeString()}
</Typography>
{msg.content}
</Box>
))}
</Box>
</Box>
))
)}
</Box>
)}
{/* Test Tab */}
{tab === (selected.channel_type === 'voice' ? 5 : 4) && (
<Box sx={{ display: 'flex', flexDirection: 'column', gap: 2, maxWidth: 500 }}>
<TextField
label="Send test to phone number"
value={testNumber}
size="small"
placeholder="+1234567890"
onChange={(e) => setTestNumber(e.target.value)}
/>
<Button
variant="contained"
startIcon={<SendIcon />}
onClick={handleTest}
sx={{ bgcolor: c.accent.primary, alignSelf: 'flex-start' }}
>
Send Test
</Button>
{testResult && (
<Typography sx={{ fontSize: '0.85rem', color: testResult.includes('success') ? c.status.success : c.status.error }}>
{testResult}
</Typography>
)}
</Box>
)}
</>
)}
</Box>
{/* Create Dialog */}
<Dialog open={createOpen} onClose={() => setCreateOpen(false)} maxWidth="sm" fullWidth>
<DialogTitle>New Channel</DialogTitle>
<DialogContent sx={{ display: 'flex', flexDirection: 'column', gap: 2, pt: '16px !important' }}>
<TextField
label="Name"
value={newName}
onChange={(e) => setNewName(e.target.value)}
size="small"
placeholder="My SMS Channel"
/>
<FormControl size="small">
<InputLabel>Type</InputLabel>
<Select value={newType} label="Type" onChange={(e) => setNewType(e.target.value as any)}>
<MenuItem value="sms">SMS</MenuItem>
<MenuItem value="whatsapp">WhatsApp</MenuItem>
<MenuItem value="voice">Voice</MenuItem>
</Select>
</FormControl>
<FormControl size="small">
<InputLabel>Provider</InputLabel>
<Select value={newProvider} label="Provider" onChange={(e) => setNewProvider(e.target.value as any)}>
<MenuItem value="twilio">Twilio</MenuItem>
<MenuItem value="telnyx">Telnyx</MenuItem>
</Select>
</FormControl>
<TextField
label="Phone Number"
value={newPhone}
onChange={(e) => setNewPhone(e.target.value)}
size="small"
placeholder="+1234567890"
/>
<TextField
label="Account SID / API Key"
value={newAccountSid}
onChange={(e) => setNewAccountSid(e.target.value)}
size="small"
type="password"
/>
<TextField
label="Auth Token"
value={newAuthToken}
onChange={(e) => setNewAuthToken(e.target.value)}
size="small"
type="password"
/>
</DialogContent>
<DialogActions>
<Button onClick={() => setCreateOpen(false)}>Cancel</Button>
<Button
variant="contained"
onClick={handleCreate}
disabled={!newName || !newPhone}
sx={{ bgcolor: c.accent.primary }}
>
Create
</Button>
</DialogActions>
</Dialog>
</Box>
);
};
export default Channels;
+240 -77
View File
@@ -22,12 +22,16 @@ import {
import {
setCardPosition,
setCardSize,
fadeGlowingAgentCard,
clearGlowingAgentCard,
removeCard,
} from '@/shared/state/dashboardLayoutSlice';
import { useAppDispatch } from '@/shared/hooks';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import { QuestionForm } from '@/app/pages/AgentChat/ApprovalBar';
import AgentChat from '@/app/pages/AgentChat/AgentChat';
import { parseMcpToolName, getMcpShortAction } from '@/app/pages/AgentChat/ToolCallBubble';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
import { useOverlayScrollPassthrough } from './useOverlayScrollPassthrough';
// ---------------------------------------------------------------------------
// Helper components & functions (unchanged)
@@ -168,7 +172,8 @@ interface Props {
cardWidth: number;
cardHeight: number;
zoom?: number;
spawnFrom?: { x: number; y: number };
spawnFrom?: { x: number; y: number; type?: 'branch' };
exitTarget?: { x: number; y: number };
isSelected?: boolean;
isHighlighted?: boolean;
multiDragDelta?: { dx: number; dy: number } | null;
@@ -176,6 +181,12 @@ interface Props {
onDragStart?: (id: string, type: 'agent' | 'view') => void;
onDragMove?: (dx: number, dy: number) => void;
onDragEnd?: (dx: number, dy: number, didDrag: boolean) => void;
onBranch?: (sourceSessionId: string, newSessionId: string) => void;
onMeasuredHeight?: (sessionId: string, height: number) => void;
snapColumn?: { x: number; width: number };
autoFocusInput?: boolean;
cardZOrder?: number;
onBringToFront?: (id: string, type: 'agent' | 'view' | 'browser') => void;
}
const MIN_W = 480;
@@ -183,13 +194,54 @@ const MIN_H = 120;
const EXPANDED_OVERLAY_H = 620;
const SPAWN_SPRING = { type: 'spring' as const, stiffness: 400, damping: 28, mass: 0.6 };
const BRANCH_SPRING = { type: 'spring' as const, stiffness: 300, damping: 26, mass: 0.8 };
const EXIT_SPRING = { type: 'spring' as const, stiffness: 350, damping: 30, mass: 0.7 };
const GLOW_FADE_MS = 2500;
const SNAP_THRESHOLD = 60;
const AgentCard: React.FC<Props> = ({
session, expanded, cardX, cardY, cardWidth, cardHeight, zoom = 1, spawnFrom,
session, expanded, cardX, cardY, cardWidth, cardHeight, zoom = 1, spawnFrom, exitTarget,
isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd,
onBranch, onMeasuredHeight, snapColumn, autoFocusInput, cardZOrder = 0, onBringToFront,
}) => {
const c = useClaudeTokens();
const dispatch = useAppDispatch();
const scrollOverlayRef = useOverlayScrollPassthrough(isSelected);
const cardBoxRef = useRef<HTMLDivElement>(null);
useEffect(() => {
const el = cardBoxRef.current;
if (!el || !onMeasuredHeight) return;
const ro = new ResizeObserver((entries) => {
for (const entry of entries) {
onMeasuredHeight(session.id, entry.contentRect.height);
}
});
ro.observe(el);
return () => ro.disconnect();
}, [session.id, onMeasuredHeight]);
// ---- Glow state (for branched cards) ----
const glowEntry = useAppSelector((s) => s.dashboardLayout.glowingAgentCards[session.id]);
const isGlowingRedux = !!glowEntry;
const glowFading = glowEntry?.fading ?? false;
const glowFadeTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const dismissGlow = useCallback(() => {
if (!isGlowingRedux || glowFading) return;
dispatch(fadeGlowingAgentCard(session.id));
glowFadeTimer.current = setTimeout(() => {
dispatch(clearGlowingAgentCard(session.id));
}, GLOW_FADE_MS + 300);
}, [isGlowingRedux, glowFading, dispatch, session.id]);
useEffect(() => () => {
if (glowFadeTimer.current) clearTimeout(glowFadeTimer.current);
}, []);
const accentColor = c.accent.primary;
const accentHover = c.accent.hover;
const STATUS_COLORS: Record<string, { color: string; bg: string }> = {
running: { color: c.status.success, bg: c.status.successBg },
@@ -212,6 +264,7 @@ const AgentCard: React.FC<Props> = ({
const justDraggedRef = useRef(false);
const handleDragPointerDown = useCallback((e: React.PointerEvent) => {
if (e.button !== 0) return;
e.preventDefault();
e.stopPropagation();
dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY };
@@ -241,11 +294,15 @@ const AgentCard: React.FC<Props> = ({
const dx = (e.clientX - dragState.current.startX) / zoom;
const dy = (e.clientY - dragState.current.startY) / zoom;
if (didDrag.current) {
dispatch(setCardPosition({
sessionId: session.id,
x: dragState.current.origX + dx,
y: dragState.current.origY + dy,
}));
let finalX = dragState.current.origX + dx;
const finalY = dragState.current.origY + dy;
if (snapColumn && Math.abs(finalX - snapColumn.x) < SNAP_THRESHOLD) {
finalX = snapColumn.x;
dispatch(setCardSize({ sessionId: session.id, width: snapColumn.width, height: cardHeight }));
}
dispatch(setCardPosition({ sessionId: session.id, x: finalX, y: finalY }));
justDraggedRef.current = true;
requestAnimationFrame(() => { justDraggedRef.current = false; });
}
@@ -255,7 +312,7 @@ const AgentCard: React.FC<Props> = ({
setLocalDragPos(null);
setIsDragging(false);
(e.currentTarget as HTMLElement).releasePointerCapture(e.pointerId);
}, [zoom, dispatch, session.id, onDragEnd]);
}, [zoom, dispatch, session.id, onDragEnd, snapColumn, cardHeight]);
// ---- Unified edge / corner resize ----
const resizeRef = useRef<{
@@ -272,6 +329,7 @@ const AgentCard: React.FC<Props> = ({
const handleResizeDown = useCallback(
(dir: ResizeDir) => (e: React.PointerEvent) => {
if (e.button !== 0) return;
e.preventDefault();
e.stopPropagation();
const effectiveW = Math.max(cardWidth, MIN_W);
@@ -337,14 +395,17 @@ const AgentCard: React.FC<Props> = ({
const handleRemove = (e: React.MouseEvent) => {
e.stopPropagation();
e.preventDefault();
dispatch(closeSession({ sessionId: session.id }));
dispatch(collapseSession(session.id));
dispatch(removeCard(session.id));
if (glowEntry) {
setTimeout(() => {
dispatch(clearGlowingAgentCard(session.id));
}, 500);
} else {
dispatch(closeSession({ sessionId: session.id }));
}
};
const handleCollapse = (e: React.MouseEvent) => {
e.stopPropagation();
e.preventDefault();
dispatch(collapseSession(session.id));
};
useEffect(() => {
if (session.status === 'running' || session.status === 'waiting_approval') {
@@ -376,33 +437,54 @@ const AgentCard: React.FC<Props> = ({
const activeW = localResize?.w ?? cardWidth;
const activeH = localResize?.h ?? cardHeight;
const isBranchSpawn = spawnFrom?.type === 'branch';
const spawnInitial = spawnFrom
? isBranchSpawn
? { opacity: 0.5, scale: 0.92, left: spawnFrom.x, top: spawnFrom.y }
: { opacity: 0, scale: 0.3, left: spawnFrom.x, top: spawnFrom.y }
: false;
const spawnTransition = noTransition || !spawnFrom
? { duration: 0 }
: isBranchSpawn
? { left: BRANCH_SPRING, top: BRANCH_SPRING, scale: BRANCH_SPRING, opacity: { duration: 0.25 } }
: { left: SPAWN_SPRING, top: SPAWN_SPRING, scale: SPAWN_SPRING, opacity: { duration: 0.12 } };
const exitAnimation = exitTarget
? {
opacity: 0,
scale: 0.3,
left: exitTarget.x,
top: exitTarget.y,
transition: { left: EXIT_SPRING, top: EXIT_SPRING, scale: EXIT_SPRING, opacity: { duration: 0.2 } },
}
: { opacity: 0, scale: 0.85, transition: { duration: 0.2 } };
return (
<motion.div
initial={spawnFrom
? { opacity: 0, scale: 0.3, left: spawnFrom.x, top: spawnFrom.y }
: { opacity: 0, scale: 0.92, left: activeX, top: activeY }
}
layout={false}
initial={spawnInitial}
animate={{ opacity: 1, scale: 1, left: activeX, top: activeY }}
transition={noTransition
? { duration: 0 }
: spawnFrom
? { left: SPAWN_SPRING, top: SPAWN_SPRING, scale: SPAWN_SPRING, opacity: { duration: 0.12 } }
: { duration: 0.25, ease: [0.25, 0.1, 0.25, 1] }
}
exit={exitAnimation}
transition={spawnTransition}
onPointerDownCapture={() => onBringToFront?.(session.id, 'agent')}
style={{
position: 'absolute',
zIndex: isDragging || isResizing ? 999 : expanded ? 100 : 'auto',
zIndex: isDragging || isResizing ? 999999 : cardZOrder,
}}
>
<Box
ref={cardBoxRef}
data-select-type="agent-card"
data-select-id={session.id}
data-select-meta={JSON.stringify({ name: session.name || session.id, status: session.status, model: session.model, mode: session.mode })}
onClick={(e: React.MouseEvent) => {
if (justDraggedRef.current) return;
if (!isSelected && !e.shiftKey) {
dispatch(toggleExpandSession(session.id));
}
onCardSelect?.(session.id, 'agent', e.shiftKey);
}}
onDoubleClick={() => dispatch(toggleExpandSession(session.id))}
sx={{
position: 'relative',
width: localResize ? activeW : Math.max(cardWidth, MIN_W),
@@ -410,26 +492,34 @@ const AgentCard: React.FC<Props> = ({
bgcolor: c.bg.surface,
border: isHighlighted
? `2px solid ${c.accent.primary}`
: isSelected
? '2px solid #3b82f6'
: hasPending && !expanded
? `1px solid ${c.status.warning}`
: expanded
? `1px solid ${c.border.strong}`
: `1px solid ${c.border.subtle}`,
: (isGlowingRedux && !glowFading)
? `2px solid ${accentColor}`
: isSelected
? '2px solid #3b82f6'
: hasPending && !expanded
? `1px solid ${c.status.warning}`
: expanded
? `1px solid ${c.border.strong}`
: `1px solid ${c.border.subtle}`,
borderRadius: 3,
p: 2,
cursor: expanded ? 'default' : 'pointer',
transition: noTransition ? 'none' : c.transition,
transition: noTransition
? 'none'
: glowFading
? `border ${GLOW_FADE_MS}ms ease-out, box-shadow ${GLOW_FADE_MS}ms ease-out`
: c.transition,
boxShadow: isHighlighted
? `0 0 0 3px ${c.accent.primary}50, 0 0 20px ${c.accent.primary}35, 0 0 40px ${c.accent.primary}15`
: isDragging
? c.shadow.lg
: isSelected
? `0 0 0 1px #3b82f6, ${c.shadow.md}`
: expanded
? c.shadow.md
: c.shadow.sm,
: (isGlowingRedux && !glowFading)
? `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15`
: isDragging
? c.shadow.lg
: isSelected
? `0 0 0 1px #3b82f6, ${c.shadow.md}`
: expanded
? c.shadow.md
: c.shadow.sm,
display: 'flex',
flexDirection: 'column',
overflow: 'hidden',
@@ -452,9 +542,19 @@ const AgentCard: React.FC<Props> = ({
boxShadow: c.shadow.sm,
},
},
zIndex: 50,
}),
...(!isHighlighted && !expanded && !isDragging && !isSelected && {
...(!isHighlighted && isGlowingRedux && !glowFading && {
animation: 'agent-card-glow-pulse 2s ease-in-out infinite',
'@keyframes agent-card-glow-pulse': {
'0%, 100%': {
boxShadow: `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15`,
},
'50%': {
boxShadow: `0 0 0 3px ${accentColor}60, 0 0 28px ${accentColor}45, 0 0 56px ${accentColor}25`,
},
},
}),
...(!isHighlighted && !(isGlowingRedux && !glowFading) && !expanded && !isDragging && !isSelected && {
'&:hover': {
boxShadow: c.shadow.md,
borderColor: hasPending ? c.status.warning : c.border.strong,
@@ -462,6 +562,82 @@ const AgentCard: React.FC<Props> = ({
}),
}}
>
{/* Glow overlays for branched cards */}
{isGlowingRedux && (
<Box
className="agent-card-glow-overlays"
sx={{
position: 'absolute',
inset: 0,
pointerEvents: 'none',
borderRadius: 'inherit',
zIndex: 20,
opacity: glowFading ? 0 : 1,
transition: `opacity ${GLOW_FADE_MS}ms ease-out`,
}}
>
{/* Rotating conic gradient border */}
<Box
sx={{
position: 'absolute',
inset: 0,
borderRadius: 'inherit',
overflow: 'hidden',
padding: '3px',
mask: 'linear-gradient(#fff 0 0) content-box, linear-gradient(#fff 0 0)',
WebkitMask: 'linear-gradient(#fff 0 0) content-box, linear-gradient(#fff 0 0)',
maskComposite: 'exclude',
WebkitMaskComposite: 'xor',
'&::before': {
content: '""',
position: 'absolute',
inset: '-50%',
background: `conic-gradient(from 0deg, transparent 0%, ${accentColor} 25%, transparent 50%, ${accentColor} 75%, transparent 100%)`,
animation: 'agent-card-rotate-glow 3s linear infinite',
},
'@keyframes agent-card-rotate-glow': {
'100%': { transform: 'rotate(360deg)' },
},
}}
/>
{/* Top edge shimmer */}
<Box
sx={{
position: 'absolute',
top: 0,
left: 0,
right: 0,
height: '2px',
background: `linear-gradient(90deg, transparent, ${accentColor}, ${accentHover}, ${accentColor}, transparent)`,
backgroundSize: '200% 100%',
animation: 'agent-card-border-shimmer 2s linear infinite',
'@keyframes agent-card-border-shimmer': {
'0%': { backgroundPosition: '200% 0' },
'100%': { backgroundPosition: '-200% 0' },
},
}}
/>
{/* Inner shadow overlay */}
<Box
sx={{
position: 'absolute',
inset: 0,
borderRadius: 'inherit',
boxShadow: `inset 0 0 40px ${accentColor}30, inset 0 0 80px ${accentColor}12`,
animation: 'agent-card-inner-pulse 2s ease-in-out infinite',
'@keyframes agent-card-inner-pulse': {
'0%, 100%': {
boxShadow: `inset 0 0 40px ${accentColor}30, inset 0 0 80px ${accentColor}12`,
},
'50%': {
boxShadow: `inset 0 0 50px ${accentColor}40, inset 0 0 100px ${accentColor}18`,
},
},
}}
/>
</Box>
)}
{/* Resize handles: 4 edges + 4 corners */}
{HANDLE_DEFS.map(({ dir, sx }) => (
<Box
@@ -481,9 +657,10 @@ const AgentCard: React.FC<Props> = ({
/>
))}
{/* Selection overlay blocks content interaction while selected, enabling drag from anywhere */}
{/* Selection overlay blocks click interaction while selected, enabling drag from anywhere */}
{isSelected && (
<Box
ref={scrollOverlayRef}
onPointerDown={handleDragPointerDown}
onPointerMove={handleDragPointerMove}
onPointerUp={handleDragPointerUp}
@@ -491,7 +668,6 @@ const AgentCard: React.FC<Props> = ({
if (justDraggedRef.current) return;
onCardSelect?.(session.id, 'agent', e.shiftKey);
}}
onDoubleClick={() => dispatch(toggleExpandSession(session.id))}
sx={{
position: 'absolute',
inset: 0,
@@ -571,37 +747,20 @@ const AgentCard: React.FC<Props> = ({
onPointerDown={(e) => e.stopPropagation()}
sx={{ display: 'flex', alignItems: 'center', gap: 0.5, flexShrink: 0, ml: 0.5 }}
>
{expanded ? (
<Tooltip title="Collapse">
<IconButton
size="small"
onClick={handleCollapse}
onMouseDown={(e) => e.stopPropagation()}
sx={{
color: c.text.ghost,
p: 0.5,
'&:hover': { color: c.text.secondary, bgcolor: c.bg.secondary },
}}
>
<CloseIcon sx={{ fontSize: 16 }} />
</IconButton>
</Tooltip>
) : (
<Tooltip title={isDraft ? 'Remove' : 'Close chat'}>
<IconButton
size="small"
onClick={handleRemove}
onMouseDown={(e) => e.stopPropagation()}
sx={{
color: c.text.ghost,
p: 0.5,
'&:hover': { color: c.status.error, bgcolor: `${c.status.errorBg}` },
}}
>
<CloseIcon sx={{ fontSize: 16 }} />
</IconButton>
</Tooltip>
)}
<Tooltip title={isDraft ? 'Remove' : 'Close chat'}>
<IconButton
size="small"
onClick={handleRemove}
onMouseDown={(e) => e.stopPropagation()}
sx={{
color: c.text.ghost,
p: 0.5,
'&:hover': { color: c.status.error, bgcolor: `${c.status.errorBg}` },
}}
>
<CloseIcon sx={{ fontSize: 16 }} />
</IconButton>
</Tooltip>
</Box>
</Box>
@@ -649,6 +808,10 @@ const AgentCard: React.FC<Props> = ({
sessionId={session.id}
onClose={() => dispatch(collapseSession(session.id))}
embedded
autoFocus={autoFocusInput}
isGlowing={isGlowingRedux && !glowFading}
onDismissGlow={dismissGlow}
onBranch={onBranch ? (newId: string) => onBranch(session.id, newId) : undefined}
/>
</Box>
)}
@@ -58,12 +58,24 @@ const BrowserAgentOverlay: React.FC<Props> = ({ session, browserWidth, browserHe
const [expanded, setExpanded] = useState(false);
const [confirmStop, setConfirmStop] = useState(false);
const [fadeOut, setFadeOut] = useState(false);
const [hidden, setHidden] = useState(false);
const confirmTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const fadeTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const hideTimer = useRef<ReturnType<typeof setTimeout> | null>(null);
const isRunning = session.status === 'running';
const isDone = session.status === 'completed' || session.status === 'error' || session.status === 'stopped';
const prevSessionId = useRef(session.id);
useEffect(() => {
if (session.id !== prevSessionId.current) {
prevSessionId.current = session.id;
setFadeOut(false);
setHidden(false);
setConfirmStop(false);
}
}, [session.id]);
useEffect(() => {
if (isDone) {
fadeTimer.current = setTimeout(() => setFadeOut(true), 2000);
@@ -71,6 +83,13 @@ const BrowserAgentOverlay: React.FC<Props> = ({ session, browserWidth, browserHe
return () => { if (fadeTimer.current) clearTimeout(fadeTimer.current); };
}, [isDone]);
useEffect(() => {
if (fadeOut) {
hideTimer.current = setTimeout(() => setHidden(true), 400);
}
return () => { if (hideTimer.current) clearTimeout(hideTimer.current); };
}, [fadeOut]);
useEffect(() => {
if (scrollRef.current) {
scrollRef.current.scrollTop = scrollRef.current.scrollHeight;
@@ -111,7 +130,7 @@ const BrowserAgentOverlay: React.FC<Props> = ({ session, browserWidth, browserHe
const panelW = expanded ? expandedW : collapsedW;
const panelH = expanded ? expandedH : collapsedH;
if (fadeOut) return null;
if (hidden) return null;
return (
<Box
@@ -133,7 +152,7 @@ const BrowserAgentOverlay: React.FC<Props> = ({ session, browserWidth, browserHe
flexDirection: 'column',
overflow: 'hidden',
transition: 'width 0.25s ease, height 0.25s ease, opacity 0.4s ease',
opacity: isDone && !fadeOut ? 0.7 : 1,
opacity: fadeOut ? 0 : isDone ? 0.7 : 1,
animation: 'overlay-enter 0.3s ease-out',
'@keyframes overlay-enter': {
'0%': { opacity: 0, transform: 'translateY(8px) scale(0.95)' },
@@ -40,6 +40,7 @@ import { useBrowserActivity } from '@/shared/useBrowserActivity';
import { getActionLabel } from '@/shared/browserCommandHandler';
import { resolveInput, isGoogleSearch } from '@/shared/resolveUrl';
import BrowserAgentOverlay from './BrowserAgentOverlay';
import { useOverlayScrollPassthrough } from './useOverlayScrollPassthrough';
import { useElementSelection } from '@/app/components/ElementSelectionContext';
type ResizeDir = 'n' | 's' | 'e' | 'w' | 'ne' | 'nw' | 'se' | 'sw';
@@ -67,6 +68,14 @@ const HANDLE_DEFS: { dir: ResizeDir; sx: Record<string, any> }[] = [
const isElectron = navigator.userAgent.includes('Electron');
const chromeUserAgent = navigator.userAgent
.replace(/\s*Electron\/\S+/, '')
.replace(/\s*OpenSwarm\/\S+/, '');
const webviewPreloadPath: string | undefined = isElectron
? (window as any).openswarm?.getWebviewPreloadPath?.()
: undefined;
type WebviewElement = BrowserWebview;
interface TabLocalState {
@@ -84,6 +93,7 @@ interface Props {
cardWidth: number;
cardHeight: number;
zoom?: number;
cmdHeld?: boolean;
isSelected?: boolean;
isHighlighted?: boolean;
multiDragDelta?: { dx: number; dy: number } | null;
@@ -91,29 +101,35 @@ interface Props {
onDragStart?: (id: string, type: 'agent' | 'view' | 'browser') => void;
onDragMove?: (dx: number, dy: number) => void;
onDragEnd?: (dx: number, dy: number, didDrag: boolean) => void;
cardZOrder?: number;
onBringToFront?: (id: string, type: 'agent' | 'view' | 'browser') => void;
}
const BrowserCard: React.FC<Props> = ({
browserId, tabs, activeTabId, cardX, cardY, cardWidth, cardHeight, zoom = 1,
browserId, tabs, activeTabId, cardX, cardY, cardWidth, cardHeight, zoom = 1, cmdHeld = false,
isSelected = false, isHighlighted = false, multiDragDelta, onCardSelect, onDragStart, onDragMove, onDragEnd,
cardZOrder = 0, onBringToFront,
}) => {
const c = useClaudeTokens();
const dispatch = useAppDispatch();
const scrollOverlayRef = useOverlayScrollPassthrough(isSelected);
const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage);
const elementSelectionCtx = useElementSelection();
const isElementSelectMode = elementSelectionCtx?.selectMode ?? false;
const browserAgentSession = useAppSelector((state) => {
const sessions = state.agents.sessions;
return Object.values(sessions).find(
const matches = Object.values(sessions).filter(
(s) => s.browser_id === browserId && s.mode === 'browser-agent'
&& (s.status === 'running' || s.status === 'completed' || s.status === 'error'),
) ?? null;
&& (s.status === 'running' || s.status === 'completed' || s.status === 'error' || s.status === 'stopped'),
);
return matches.find((s) => s.status === 'running') ?? matches[matches.length - 1] ?? null;
});
const activity = useBrowserActivity(browserId);
const agentActive = activity.active;
const agentRunning = browserAgentSession?.status === 'running';
const agentActive = activity.active || agentRunning;
const agentAction = activity.action;
const lastAction = activity.lastAction;
@@ -192,10 +208,6 @@ const BrowserCard: React.FC<Props> = ({
onTitleUpdate();
};
const onNewWindow = (e: any) => {
if (e.url) dispatch(addBrowserTab({ browserId, url: e.url, makeActive: true }));
};
const onFaviconUpdate = (e: any) => {
const favicons = e.favicons || (e.detail && e.detail.favicons);
if (favicons?.[0]) {
@@ -208,7 +220,6 @@ const BrowserCard: React.FC<Props> = ({
wv.addEventListener('page-title-updated', onTitleUpdate);
wv.addEventListener('did-start-loading', onLoadStart);
wv.addEventListener('did-stop-loading', onLoadStop);
wv.addEventListener('new-window', onNewWindow);
wv.addEventListener('page-favicon-updated', onFaviconUpdate);
cleanups.push(() => {
@@ -218,7 +229,6 @@ const BrowserCard: React.FC<Props> = ({
wv.removeEventListener('page-title-updated', onTitleUpdate);
wv.removeEventListener('did-start-loading', onLoadStart);
wv.removeEventListener('did-stop-loading', onLoadStop);
wv.removeEventListener('new-window', onNewWindow);
wv.removeEventListener('page-favicon-updated', onFaviconUpdate);
});
}
@@ -371,6 +381,7 @@ const BrowserCard: React.FC<Props> = ({
const justDraggedRef = useRef(false);
const handleDragPointerDown = useCallback((e: React.PointerEvent) => {
if (e.button !== 0) return;
e.preventDefault();
e.stopPropagation();
dragState.current = { startX: e.clientX, startY: e.clientY, origX: cardX, origY: cardY };
@@ -426,6 +437,7 @@ const BrowserCard: React.FC<Props> = ({
const handleResizeDown = useCallback(
(dir: ResizeDir) => (e: React.PointerEvent) => {
if (e.button !== 0) return;
e.preventDefault();
e.stopPropagation();
resizeRef.current = {
@@ -491,20 +503,13 @@ const BrowserCard: React.FC<Props> = ({
const accentColor = c.accent.primary;
const accentHover = c.accent.hover;
const accentRgb = accentColor.replace('#', '').match(/.{2}/g)?.map(h => parseInt(h, 16)).join(',') || '189,100,57';
// ---- Glow state ----
const glowingBrowserCards = useAppSelector((s) => s.dashboardLayout.glowingBrowserCards);
const isGlowingFromRedux = !!glowingBrowserCards[browserId];
const [hasBeenTouched, setHasBeenTouched] = useState(false);
useEffect(() => {
if (isGlowingFromRedux && agentActive) setHasBeenTouched(true);
}, [isGlowingFromRedux, agentActive]);
useEffect(() => {
if (!isGlowingFromRedux) setHasBeenTouched(false);
}, [isGlowingFromRedux]);
const showGlow = isGlowingFromRedux && hasBeenTouched;
const showGlow = isGlowingFromRedux;
const agentBorder = isHighlighted
? `2px solid ${c.accent.primary}`
@@ -535,6 +540,7 @@ const BrowserCard: React.FC<Props> = ({
data-select-type="browser-card"
data-select-id={browserId}
data-select-meta={JSON.stringify({ name: activeTitle || 'Browser', url: activeUrl })}
onPointerDownCapture={() => onBringToFront?.(browserId, 'browser')}
onClick={(e: React.MouseEvent) => {
if (justDraggedRef.current) return;
onCardSelect?.(browserId, 'browser', e.shiftKey);
@@ -552,7 +558,7 @@ const BrowserCard: React.FC<Props> = ({
overflow: 'hidden',
display: 'flex',
flexDirection: 'column',
zIndex: isHighlighted ? 50 : (isDragging || isResizing) ? 100 : (agentActive || showGlow) ? 50 : 1,
zIndex: (isDragging || isResizing) ? 999999 : cardZOrder,
transition: noTransition ? 'none' : 'box-shadow 0.4s ease, border 0.3s ease',
'&:hover .resize-handle': { opacity: 1 },
...(isHighlighted && {
@@ -576,8 +582,8 @@ const BrowserCard: React.FC<Props> = ({
},
}),
...(!isHighlighted && (agentActive || showGlow) && {
animation: 'agent-glow-pulse 2s ease-in-out infinite',
'@keyframes agent-glow-pulse': {
animation: `agent-glow-${browserId} 2s ease-in-out infinite`,
[`@keyframes agent-glow-${browserId}`]: {
'0%, 100%': {
boxShadow: `0 0 0 2px ${accentColor}40, 0 0 18px ${accentColor}30, 0 0 40px ${accentColor}15${innerGlow}`,
},
@@ -588,9 +594,10 @@ const BrowserCard: React.FC<Props> = ({
}),
}}
>
{/* Selection overlay blocks content interaction while selected, enabling drag from anywhere */}
{/* Selection overlay blocks click interaction while selected, enabling drag from anywhere */}
{isSelected && (
<Box
ref={scrollOverlayRef}
onPointerDown={handleDragPointerDown}
onPointerMove={handleDragPointerMove}
onPointerUp={handleDragPointerUp}
@@ -977,6 +984,9 @@ const BrowserCard: React.FC<Props> = ({
{isElementSelectMode && (
<Box sx={{ position: 'absolute', inset: 0, zIndex: 10 }} />
)}
{cmdHeld && !isSelected && (
<Box sx={{ position: 'absolute', inset: 0, zIndex: 12 }} />
)}
{isElectron ? (
tabs.map((tab) => (
<webview
@@ -988,6 +998,9 @@ const BrowserCard: React.FC<Props> = ({
data-tab-id={tab.id}
src="about:blank"
allowpopups="true"
useragent={chromeUserAgent}
{...(webviewPreloadPath ? { preload: webviewPreloadPath } : {})}
webpreferences="plugins=yes, autoplayPolicy=no-user-gesture-required"
style={{
position: 'absolute',
top: 0,
@@ -1035,6 +1048,7 @@ const BrowserCard: React.FC<Props> = ({
{/* Camera flash — screenshot */}
{(agentAction === 'screenshot' || lastAction === 'screenshot') && (
<Box
key={`flash-${activity.actionSeq}`}
sx={{
position: 'absolute',
inset: 0,
@@ -1062,10 +1076,10 @@ const BrowserCard: React.FC<Props> = ({
pointerEvents: 'none',
background: `linear-gradient(180deg, transparent, ${accentColor}90, transparent)`,
boxShadow: `0 0 12px ${accentColor}60`,
animation: 'scan-sweep 1.5s ease-in-out infinite',
animation: 'scan-sweep 1.5s ease-in-out infinite alternate',
'@keyframes scan-sweep': {
'0%': { top: '0%' },
'100%': { top: '100%' },
'100%': { top: 'calc(100% - 3px)' },
},
}}
/>
@@ -1074,10 +1088,11 @@ const BrowserCard: React.FC<Props> = ({
{/* Click ripple */}
{(agentAction === 'click' || lastAction === 'click') && (
<Box
key={`ripple-${activity.actionSeq}`}
sx={{
position: 'absolute',
top: '50%',
left: '50%',
top: `${(activity.coords?.yPercent ?? 0.5) * 100}%`,
left: `${(activity.coords?.xPercent ?? 0.5) * 100}%`,
width: 40,
height: 40,
borderRadius: '50%',
@@ -1133,7 +1148,7 @@ const BrowserCard: React.FC<Props> = ({
</Box>
)}
{/* Orange inner shadow overlay for selection / streaming glow */}
{/* Accent inner shadow overlay for selection / streaming glow */}
{showGlow && !agentActive && (
<Box
sx={{
@@ -1142,14 +1157,14 @@ const BrowserCard: React.FC<Props> = ({
zIndex: 14,
pointerEvents: 'none',
borderRadius: 'inherit',
boxShadow: 'inset 0 0 40px rgba(255,140,0,0.35), inset 0 0 80px rgba(255,100,0,0.15)',
animation: 'orange-glow-pulse 2s ease-in-out infinite',
'@keyframes orange-glow-pulse': {
boxShadow: `inset 0 0 40px rgba(${accentRgb},0.35), inset 0 0 80px rgba(${accentRgb},0.15)`,
animation: `accent-glow-${browserId} 2s ease-in-out infinite`,
[`@keyframes accent-glow-${browserId}`]: {
'0%, 100%': {
boxShadow: 'inset 0 0 40px rgba(255,140,0,0.35), inset 0 0 80px rgba(255,100,0,0.15)',
boxShadow: `inset 0 0 40px rgba(${accentRgb},0.35), inset 0 0 80px rgba(${accentRgb},0.15)`,
},
'50%': {
boxShadow: 'inset 0 0 50px rgba(255,140,0,0.45), inset 0 0 100px rgba(255,100,0,0.22)',
boxShadow: `inset 0 0 50px rgba(${accentRgb},0.45), inset 0 0 100px rgba(${accentRgb},0.22)`,
},
},
}}
File diff suppressed because it is too large Load Diff
@@ -92,7 +92,7 @@ const DashboardHeader: React.FC<DashboardHeaderProps> = ({
const handleFocus = useCallback(
(cardId: string, card: { x: number; y: number; width: number; height: number }) => {
canvasActions.fitToCards([card], 1.0);
canvasActions.fitToCards([card], 1.0, true);
onHighlightCard?.(cardId);
setExpanded(false);
},
@@ -42,6 +42,7 @@ interface Props {
dashboardId?: string;
}
const TOOLBAR_OWNER_ID = '__toolbar__';
const BTN = 40;
const WarmTooltip = styled(
@@ -90,8 +91,19 @@ const DashboardToolbar = React.forwardRef<HTMLDivElement, Props>(
const searchInputRef = useRef<HTMLInputElement>(null);
const historyInputRef = useRef<HTMLInputElement>(null);
const historyListRef = useRef<HTMLDivElement>(null);
const [mode, setMode] = useState('agent');
const [model, setModel] = useState('sonnet');
const defaultMode = useAppSelector((s) => s.settings.data.default_mode);
const defaultModel = useAppSelector((s) => s.settings.data.default_model);
const [mode, setMode] = useState(defaultMode || 'agent');
const [model, setModel] = useState(defaultModel || 'sonnet');
const [provider, setProvider] = useState('anthropic');
const settingsApplied = useRef(false);
useEffect(() => {
if (!settingsApplied.current) {
setMode(defaultMode || 'agent');
setModel(defaultModel || 'sonnet');
settingsApplied.current = true;
}
}, [defaultMode, defaultModel]);
const [viewPickerOpen, setViewPickerOpen] = useState(false);
const [viewSearch, setViewSearch] = useState('');
const [historyOpen, setHistoryOpen] = useState(false);
@@ -204,13 +216,23 @@ const DashboardToolbar = React.forwardRef<HTMLDivElement, Props>(
const isExpanded = inputOpen || viewPickerOpen || historyOpen;
const autoSelectOnNew = useAppSelector((s) => s.settings.data.auto_select_mode_on_new_agent);
const prevInputOpenRef = useRef(inputOpen);
useEffect(() => {
if (prevInputOpenRef.current && !inputOpen && elementSelection?.selectMode) {
elementSelection.setSelectMode(false);
if (prevInputOpenRef.current && !inputOpen && elementSelection) {
elementSelection.clearOwnerElements(TOOLBAR_OWNER_ID);
if (elementSelection.selectMode && elementSelection.activeOwnerId === TOOLBAR_OWNER_ID) {
elementSelection.setSelectMode(false);
}
}
if (!prevInputOpenRef.current && inputOpen && autoSelectOnNew && elementSelection) {
elementSelection.clearOwnerElements(TOOLBAR_OWNER_ID);
elementSelection.setActiveOwnerId(TOOLBAR_OWNER_ID);
elementSelection.setExcludeSelectId(null);
elementSelection.setSelectMode(true);
}
prevInputOpenRef.current = inputOpen;
}, [inputOpen, elementSelection]);
}, [inputOpen, elementSelection, autoSelectOnNew]);
useEffect(() => {
if (!isExpanded) return;
@@ -226,21 +248,44 @@ const DashboardToolbar = React.forwardRef<HTMLDivElement, Props>(
useEffect(() => {
if (!isExpanded) return;
const handleClick = (e: MouseEvent) => {
if (elementSelection?.selectMode) return;
let downPos: { x: number; y: number; target: Node } | null = null;
const DRAG_THRESHOLD = 5;
const handleDown = (e: MouseEvent) => {
const target = e.target as Node;
if (containerRef.current && !containerRef.current.contains(target)) {
const el = target instanceof Element ? target : target.parentElement;
if (el?.closest('[role="dialog"], [role="presentation"], .MuiModal-root, .MuiPopover-root')) {
return;
}
handleDismiss();
downPos = { x: e.clientX, y: e.clientY, target };
} else {
downPos = null;
}
};
const t = setTimeout(() => document.addEventListener('mousedown', handleClick), 50);
const handleUp = (e: MouseEvent) => {
if (!downPos) return;
const dx = e.clientX - downPos.x;
const dy = e.clientY - downPos.y;
const target = downPos.target;
downPos = null;
if (Math.abs(dx) > DRAG_THRESHOLD || Math.abs(dy) > DRAG_THRESHOLD) return;
const el = target instanceof Element ? target : (target as Node).parentElement;
if (el?.closest('[role="dialog"], [role="presentation"], .MuiModal-root, .MuiPopover-root')) {
return;
}
if (elementSelection?.selectMode && el?.closest('[data-select-type]')) {
return;
}
handleDismiss();
};
const t = setTimeout(() => {
document.addEventListener('mousedown', handleDown, true);
document.addEventListener('mouseup', handleUp, true);
}, 50);
return () => {
clearTimeout(t);
document.removeEventListener('mousedown', handleClick);
document.removeEventListener('mousedown', handleDown, true);
document.removeEventListener('mouseup', handleUp, true);
};
}, [isExpanded, handleDismiss, elementSelection?.selectMode]);
@@ -266,10 +311,14 @@ const DashboardToolbar = React.forwardRef<HTMLDivElement, Props>(
e.preventDefault();
handleOpenHistory();
}
if (e.metaKey && e.key.toLowerCase() === 'n' && !e.ctrlKey && !e.shiftKey && !e.altKey) {
e.preventDefault();
onAddBrowser();
}
};
window.addEventListener('keydown', handleKey);
return () => window.removeEventListener('keydown', handleKey);
}, [handleOpenViewPicker, handleOpenHistory]);
}, [handleOpenViewPicker, handleOpenHistory, onAddBrowser]);
useEffect(() => {
if (!historyOpen) return;
@@ -317,8 +366,11 @@ const DashboardToolbar = React.forwardRef<HTMLDivElement, Props>(
onModeChange={setMode}
model={model}
onModelChange={setModel}
provider={provider}
onProviderChange={setProvider}
embedded
autoFocus
sessionId={TOOLBAR_OWNER_ID}
/>
</div>
) : historyOpen ? (
@@ -596,6 +648,39 @@ const DashboardToolbar = React.forwardRef<HTMLDivElement, Props>(
</Box>
</WarmTooltip>
<WarmTooltip
tokens={c}
placement="top"
arrow
enterDelay={200}
title={
<Box sx={{ textAlign: 'center' }}>
<Box sx={{ fontWeight: 600 }}>Browser N</Box>
</Box>
}
>
<Box
role="button"
aria-label="Browser"
tabIndex={0}
onClick={onAddBrowser}
sx={{
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
width: BTN,
height: BTN,
borderRadius: `${c.radius.md}px`,
color: c.text.tertiary,
cursor: 'pointer',
transition: 'opacity 0.15s, background-color 0.15s',
'&:hover': { opacity: 1, bgcolor: c.bg.secondary, color: c.accent.primary },
}}
>
<LanguageIcon sx={{ fontSize: 22 }} />
</Box>
</WarmTooltip>
<WarmTooltip
tokens={c}
placement="top"
@@ -629,39 +714,6 @@ const DashboardToolbar = React.forwardRef<HTMLDivElement, Props>(
</Box>
</WarmTooltip>
<WarmTooltip
tokens={c}
placement="top"
arrow
enterDelay={200}
title={
<Box sx={{ textAlign: 'center' }}>
<Box sx={{ fontWeight: 600 }}>Browser</Box>
</Box>
}
>
<Box
role="button"
aria-label="Browser"
tabIndex={0}
onClick={onAddBrowser}
sx={{
display: 'flex',
alignItems: 'center',
justifyContent: 'center',
width: BTN,
height: BTN,
borderRadius: `${c.radius.md}px`,
color: c.text.tertiary,
cursor: 'pointer',
transition: 'opacity 0.15s, background-color 0.15s',
'&:hover': { opacity: 1, bgcolor: c.bg.secondary, color: c.accent.primary },
}}
>
<LanguageIcon sx={{ fontSize: 22 }} />
</Box>
</WarmTooltip>
{placeholderItems.map(({ icon: PlaceholderIcon, label, sub }) => (
<WarmTooltip
key={label}

Some files were not shown because too many files have changed in this diff Show More