diff --git a/backend/apps/agents/manager/permissions/build_effective_tool_lists.py b/backend/apps/agents/manager/permissions/build_effective_tool_lists.py index 7d308f6e..3dc7f3bf 100644 --- a/backend/apps/agents/manager/permissions/build_effective_tool_lists.py +++ b/backend/apps/agents/manager/permissions/build_effective_tool_lists.py @@ -72,6 +72,14 @@ def build_effective_tool_lists( effective_disallowed.append("mcp__openswarm-skill__Skill") continue + if name == "openswarm-ui": + policy = builtin_perms.get("ShowUI", "always_allow") + if policy == "always_allow": + effective_allowed.append("mcp__openswarm-ui__ShowUI") + else: + effective_disallowed.append("mcp__openswarm-ui__ShowUI") + continue + if name == "openswarm-web": # Expose our DDG-backed web tools under an MCP prefix. Honor existing WebSearch/WebFetch permission policy, if the user disabled them in Settings, don't offer the MCP variants either. for wt in ("WebSearch", "WebFetch"): diff --git a/backend/apps/agents/manager/register_builtin_mcp_servers.py b/backend/apps/agents/manager/register_builtin_mcp_servers.py index a954ca13..75bd4d5a 100644 --- a/backend/apps/agents/manager/register_builtin_mcp_servers.py +++ b/backend/apps/agents/manager/register_builtin_mcp_servers.py @@ -142,6 +142,19 @@ def register_builtin_mcp_servers( "type": "stdio", } + # Display-only ShowUI server: renders rich inline components (weather, plan, stats, links) + # in the transcript. Pure display, no state mutation; the frontend renders from the + # tool_call input, the server only validates. Gated on the ShowUI builtin perm. + show_ui_denied = builtin_perms.get("ShowUI", "always_allow") == "deny" + if not show_ui_denied: + show_ui_server_path = os.path.join(agents_dir, "show_ui_mcp_server.py") + mcp_servers["openswarm-ui"] = { + "command": sys.executable, + "args": [show_ui_server_path], + "env": {}, + "type": "stdio", + } + # Always-on schedule server: ScheduleWorkflow + CRUD + AddWorkflowStep/EditWorkflowStep so the agent (and the workflow Edit Agent) can build and schedule recurring work via the native scheduler instead of cron/launchctl. The 4 scheduling tools are force-gated in path_gate; Cron* is denied in build_effective_tool_lists. schedule_server_path = os.path.join( agents_dir, "schedule_mcp_server.py" diff --git a/backend/apps/agents/show_ui_mcp_server.py b/backend/apps/agents/show_ui_mcp_server.py new file mode 100644 index 00000000..dfc52719 --- /dev/null +++ b/backend/apps/agents/show_ui_mcp_server.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +"""Stdio MCP server exposing ShowUI: render a rich inline component in the chat transcript. + +Display-only. The frontend renders the component straight from the tool_call input it already +has in the transcript, so this server just validates the payload and acknowledges; there is no +backend round-trip and nothing here can mutate state. +""" + +import json +import sys + +MAX_PROPS_BYTES = 20_000 + +COMPONENT_SPECS = { + "weather": "props: {location: str, temp: number, unit?: 'F'|'C', high?: number, low?: number, condition?: str, forecast?: [{day: str, condition?: str, high: number, low?: number}] (max 7)}", + "plan": "props: {title?: str, steps: [{label: str, status: 'pending'|'in_progress'|'completed'}] (max 20)}", + "stats": "props: {title?: str, stats: [{label: str, value: str, delta?: str, direction?: 'up'|'down'}] (max 8)}", + "links": "props: {links: [{title: str, url: str, description?: str}] (max 10)}", +} + +TOOLS = [ + { + "name": "ShowUI", + "description": ( + "Render a rich inline UI component in the chat instead of describing data as text. " + "Use it whenever a result fits one of the shapes. Supported components:\n" + + "\n".join(f"- '{name}': {spec}" for name, spec in COMPONENT_SPECS.items()) + + "\nCall it with the component name and a props object matching that shape. " + "The component renders in place of raw text; still give a one-line text summary after." + ), + "inputSchema": { + "type": "object", + "properties": { + "component": { + "type": "string", + "enum": list(COMPONENT_SPECS.keys()), + "description": "Which component to render.", + }, + "props": { + "type": "object", + "description": "Data for the component, matching its documented shape.", + }, + }, + "required": ["component", "props"], + }, + }, +] + + +def send_response(id_, result=None, error=None): + msg = {"jsonrpc": "2.0", "id": id_} + if error is not None: + msg["error"] = error + else: + msg["result"] = result + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def validate(component: str, props: dict) -> str: + if component not in COMPONENT_SPECS: + return f"Unknown component {component!r}. Supported: {', '.join(COMPONENT_SPECS)}." + try: + size = len(json.dumps(props)) + except (TypeError, ValueError): + return "props must be JSON-serializable." + if size > MAX_PROPS_BYTES: + return f"props too large ({size} bytes; max {MAX_PROPS_BYTES})." + if component == "weather" and not (isinstance(props.get("location"), str) and isinstance(props.get("temp"), (int, float))): + return f"weather needs at least location + temp. {COMPONENT_SPECS['weather']}" + if component == "plan" and not (isinstance(props.get("steps"), list) and props["steps"]): + return f"plan needs a non-empty steps list. {COMPONENT_SPECS['plan']}" + if component == "stats" and not (isinstance(props.get("stats"), list) and props["stats"]): + return f"stats needs a non-empty stats list. {COMPONENT_SPECS['stats']}" + if component == "links" and not (isinstance(props.get("links"), list) and props["links"]): + return f"links needs a non-empty links list. {COMPONENT_SPECS['links']}" + return "" + + +def handle_tool_call(tool_name: str, arguments: dict) -> dict: + if tool_name != "ShowUI": + return {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True} + component = str(arguments.get("component", "")).strip() + props = arguments.get("props") + if not isinstance(props, dict): + return {"content": [{"type": "text", "text": "props must be an object."}], "isError": True} + problem = validate(component, props) + if problem: + return {"content": [{"type": "text", "text": f"Not rendered: {problem}"}], "isError": True} + return {"content": [{"type": "text", "text": f"Rendered a '{component}' component inline."}]} + + +def main(): + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + + method = msg.get("method") + id_ = msg.get("id") + params = msg.get("params", {}) or {} + + if method == "initialize": + send_response(id_, { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": { + "name": "openswarm-ui", + "version": "1.0.0", + }, + }) + elif method == "notifications/initialized": + pass + elif method == "tools/list": + send_response(id_, {"tools": TOOLS}) + elif method == "tools/call": + tool_name = params.get("name", "") + arguments = params.get("arguments", {}) or {} + result = handle_tool_call(tool_name, arguments) + send_response(id_, result) + elif method == "ping": + send_response(id_, {}) + elif id_ is not None: + send_response(id_, error={"code": -32601, "message": f"Method not found: {method}"}) + + +if __name__ == "__main__": + main() diff --git a/frontend/src/app/pages/AgentChat/AgentChat.tsx b/frontend/src/app/pages/AgentChat/AgentChat.tsx index 2e573e09..ffa6a056 100644 --- a/frontend/src/app/pages/AgentChat/AgentChat.tsx +++ b/frontend/src/app/pages/AgentChat/AgentChat.tsx @@ -58,6 +58,8 @@ import CompactionMarker from './bubbles/CompactionMarker'; import MessageActionBar from './shell/MessageActionBar'; import ToolCallBubble, { ToolPair } from './tool-bubbles/ToolCallBubble'; import ToolGroupBubble, { RenderItem, ToolGroup, isToolGroup, isToolPair } from './tool-bubbles/ToolGroupBubble'; +import ToolUiBubble from './tool-ui/ToolUiBubble'; +import { isShowUiPair } from './tool-ui/showUiPayload'; import ApprovalBar, { BatchApprovalBar } from './shell/ApprovalBar'; import ForceStopAgentBar from './ForceStopAgentBar'; import { RateLimitPill } from './shell/RateLimitPill'; @@ -1063,15 +1065,21 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose i++; } - const calls = group.filter((m) => m.role === 'tool_call'); + const allCalls = group.filter((m) => m.role === 'tool_call'); const results = group.filter((m) => m.role === 'tool_result'); - const pairs: ToolPair[] = calls.map((call, idx) => ({ + const allPairs: ToolPair[] = allCalls.map((call, idx) => ({ type: 'tool_pair' as const, id: `pair-${call.id}`, call, result: results[idx] || null, })); + // ShowUI calls render as inline components, never buried inside a collapsed group. + // They typically cap a run of work, so the quiet group row stays above the widget. + const showUiPairs = allPairs.filter(isShowUiPair); + const pairs = allPairs.filter((p) => !isShowUiPair(p)); + const calls = pairs.map((p) => p.call); + const mcpServers = new Set( calls.map((m) => { const tool = typeof m.content === 'object' ? m.content.tool || '' : ''; @@ -1113,6 +1121,7 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose callCount: calls.length, } satisfies ToolGroup); } + items.push(...showUiPairs); } else { if (!msg.hidden) { items.push(msg); @@ -1593,6 +1602,14 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose } if (isToolPair(item)) { const isPending = item.result === null && sessionRunning; + if (isShowUiPair(item)) { + return ( + + + {compactionChip} + + ); + } return ( diff --git a/frontend/src/app/pages/AgentChat/tool-ui/LinksWidget.tsx b/frontend/src/app/pages/AgentChat/tool-ui/LinksWidget.tsx new file mode 100644 index 00000000..8d980491 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/tool-ui/LinksWidget.tsx @@ -0,0 +1,56 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import type { LinksProps } from './showUiPayload'; + +function hostOf(url: string): string { + try { + return new URL(url).hostname.replace(/^www\./, ''); + } catch { + return url; + } +} + +/** Tool-UI-style link previews: domain, title, description; opens like any transcript link. */ +function LinksWidget({ props }: { props: LinksProps }): React.ReactElement { + const c = useClaudeTokens(); + return ( + + {props.links.map((l, i) => ( + + + {hostOf(l.url)} + + + {l.title} + + {l.description && ( + + {l.description} + + )} + + ))} + + ); +} + +export default LinksWidget; diff --git a/frontend/src/app/pages/AgentChat/tool-ui/PlanWidget.tsx b/frontend/src/app/pages/AgentChat/tool-ui/PlanWidget.tsx new file mode 100644 index 00000000..92b6fd8a --- /dev/null +++ b/frontend/src/app/pages/AgentChat/tool-ui/PlanWidget.tsx @@ -0,0 +1,72 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import CheckCircleIcon from '@mui/icons-material/CheckCircle'; +import RadioButtonUncheckedIcon from '@mui/icons-material/RadioButtonUnchecked'; +import CircularProgress from '@mui/material/CircularProgress'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import type { PlanProps } from './showUiPayload'; + +const MAX_VISIBLE = 6; + +/** Tool-UI-style plan card: progress summary bar + step checklist. */ +function PlanWidget({ props }: { props: PlanProps }): React.ReactElement { + const c = useClaudeTokens(); + const done = props.steps.filter((s) => s.status === 'completed').length; + const visible = props.steps.slice(0, MAX_VISIBLE); + const hidden = props.steps.length - visible.length; + + return ( + + {props.title && ( + + {props.title} + + )} + + {done} of {props.steps.length} complete + + + + + {visible.map((step, i) => ( + + {step.status === 'completed' ? ( + + ) : step.status === 'in_progress' ? ( + + ) : ( + + )} + + {step.label} + + + ))} + {hidden > 0 && ( + + ... {hidden} more + + )} + + ); +} + +export default PlanWidget; diff --git a/frontend/src/app/pages/AgentChat/tool-ui/StatsWidget.tsx b/frontend/src/app/pages/AgentChat/tool-ui/StatsWidget.tsx new file mode 100644 index 00000000..22ddaf15 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/tool-ui/StatsWidget.tsx @@ -0,0 +1,58 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import ArrowUpwardIcon from '@mui/icons-material/ArrowUpward'; +import ArrowDownwardIcon from '@mui/icons-material/ArrowDownward'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import type { StatsProps } from './showUiPayload'; + +/** Tool-UI-style stat tiles: label, value, optional signed delta. */ +function StatsWidget({ props }: { props: StatsProps }): React.ReactElement { + const c = useClaudeTokens(); + return ( + + {props.title && ( + + {props.title} + + )} + + {props.stats.map((s, i) => ( + + + {s.label} + + + {s.value} + + {s.delta && ( + + {s.direction === 'down' ? ( + + ) : ( + + )} + + {s.delta} + + + )} + + ))} + + + ); +} + +export default StatsWidget; diff --git a/frontend/src/app/pages/AgentChat/tool-ui/ToolUiBubble.tsx b/frontend/src/app/pages/AgentChat/tool-ui/ToolUiBubble.tsx new file mode 100644 index 00000000..c8567566 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/tool-ui/ToolUiBubble.tsx @@ -0,0 +1,36 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import ToolCallBubble from '../tool-bubbles/ToolCallBubble'; +import type { ToolPair } from '../tool-bubbles/ToolCallBubble'; +import { parseShowUiPayload } from './showUiPayload'; +import WeatherWidget from './WeatherWidget'; +import PlanWidget from './PlanWidget'; +import StatsWidget from './StatsWidget'; +import LinksWidget from './LinksWidget'; + +interface ToolUiBubbleProps { + pair: ToolPair; + sessionId: string; + isPending: boolean; + suppressReveal: boolean; +} + +/** Renders a ShowUI call as its inline component; any schema mismatch falls back to the plain tool bubble. */ +function ToolUiBubble({ pair, sessionId, isPending, suppressReveal }: ToolUiBubbleProps): React.ReactElement { + const payload = parseShowUiPayload(pair); + if (!payload) { + return ( + + ); + } + return ( + + {payload.component === 'weather' && } + {payload.component === 'plan' && } + {payload.component === 'stats' && } + {payload.component === 'links' && } + + ); +} + +export default ToolUiBubble; diff --git a/frontend/src/app/pages/AgentChat/tool-ui/WeatherWidget.tsx b/frontend/src/app/pages/AgentChat/tool-ui/WeatherWidget.tsx new file mode 100644 index 00000000..e7a4fe30 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/tool-ui/WeatherWidget.tsx @@ -0,0 +1,92 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import WbSunnyOutlinedIcon from '@mui/icons-material/WbSunnyOutlined'; +import CloudOutlinedIcon from '@mui/icons-material/CloudOutlined'; +import GrainIcon from '@mui/icons-material/Grain'; +import AcUnitIcon from '@mui/icons-material/AcUnit'; +import ThunderstormOutlinedIcon from '@mui/icons-material/ThunderstormOutlined'; +import type { WeatherProps } from './showUiPayload'; + +function conditionIcon(condition: string | undefined, size: number): React.ReactElement { + const cond = (condition || '').toLowerCase(); + const sx = { fontSize: size, color: 'rgba(255,255,255,0.92)' }; + if (/thunder|storm/.test(cond)) return ; + if (/rain|drizzle|shower/.test(cond)) return ; + if (/snow|sleet|ice/.test(cond)) return ; + if (/cloud|overcast|fog|mist/.test(cond)) return ; + return ; +} + +/** iOS-style weather card: dusk-sky art, big thin temperature, five-day strip. */ +function WeatherWidget({ props }: { props: WeatherProps }): React.ReactElement { + const unit = props.unit || 'F'; + return ( + + + {props.location} + + + + {Math.round(props.temp)} + + °{unit} + + {(props.high != null || props.low != null) && ( + + {props.high != null && ( + + H {Math.round(props.high)}° + + )} + {props.low != null && ( + + L {Math.round(props.low)}° + + )} + + )} + {props.forecast && props.forecast.length > 0 && ( + + {props.forecast.slice(0, 5).map((d, i) => ( + + + {d.day} + + {conditionIcon(d.condition, 16)} + {Math.round(d.high)}° + {d.low != null && ( + {Math.round(d.low)}° + )} + + ))} + + )} + + ); +} + +export default WeatherWidget; diff --git a/frontend/src/app/pages/AgentChat/tool-ui/showUiPayload.ts b/frontend/src/app/pages/AgentChat/tool-ui/showUiPayload.ts new file mode 100644 index 00000000..4f13fe1a --- /dev/null +++ b/frontend/src/app/pages/AgentChat/tool-ui/showUiPayload.ts @@ -0,0 +1,151 @@ +import type { ToolPair } from '../tool-bubbles/ToolCallBubble'; + +export interface WeatherForecastDay { + day: string; + condition?: string; + high: number; + low?: number; +} + +export interface WeatherProps { + location: string; + temp: number; + unit?: 'F' | 'C'; + high?: number; + low?: number; + condition?: string; + forecast?: WeatherForecastDay[]; +} + +export interface PlanStep { + label: string; + status: 'pending' | 'in_progress' | 'completed'; +} + +export interface PlanProps { + title?: string; + steps: PlanStep[]; +} + +export interface StatItem { + label: string; + value: string; + delta?: string; + direction?: 'up' | 'down'; +} + +export interface StatsProps { + title?: string; + stats: StatItem[]; +} + +export interface LinkItem { + title: string; + url: string; + description?: string; +} + +export interface LinksProps { + links: LinkItem[]; +} + +export type ShowUiPayload = + | { component: 'weather'; props: WeatherProps } + | { component: 'plan'; props: PlanProps } + | { component: 'stats'; props: StatsProps } + | { component: 'links'; props: LinksProps }; + +function num(v: unknown): v is number { + return typeof v === 'number' && Number.isFinite(v); +} + +function str(v: unknown): v is string { + return typeof v === 'string' && v.length > 0; +} + +export function isShowUiPair(pair: ToolPair): boolean { + const tool = typeof pair.call.content === 'object' ? String(pair.call.content?.tool || '') : ''; + return /(^|__)ShowUI$/.test(tool); +} + +/** Strict parse of a ShowUI tool_call's input; null on any mismatch so the caller falls back to the plain bubble. */ +export function parseShowUiPayload(pair: ToolPair): ShowUiPayload | null { + const content = typeof pair.call.content === 'object' ? pair.call.content : null; + const input = content?.input; + if (!input || typeof input !== 'object') return null; + const component = String((input as { component?: unknown }).component || ''); + const props = (input as { props?: unknown }).props; + if (!props || typeof props !== 'object') return null; + const p = props as Record; + + if (component === 'weather') { + if (!str(p.location) || !num(p.temp)) return null; + const forecast = Array.isArray(p.forecast) + ? (p.forecast as Array>) + .filter((d) => str(d.day) && num(d.high)) + .slice(0, 7) + .map((d) => ({ + day: d.day as string, + condition: str(d.condition) ? d.condition : undefined, + high: d.high as number, + low: num(d.low) ? d.low : undefined, + })) + : undefined; + return { + component: 'weather', + props: { + location: p.location, + temp: p.temp, + unit: p.unit === 'C' ? 'C' : 'F', + high: num(p.high) ? p.high : undefined, + low: num(p.low) ? p.low : undefined, + condition: str(p.condition) ? p.condition : undefined, + forecast, + }, + }; + } + + if (component === 'plan') { + if (!Array.isArray(p.steps)) return null; + const steps = (p.steps as Array>) + .filter((s) => str(s.label)) + .slice(0, 20) + .map((s) => ({ + label: s.label as string, + status: (s.status === 'completed' || s.status === 'in_progress' ? s.status : 'pending') as PlanStep['status'], + })); + if (steps.length === 0) return null; + return { component: 'plan', props: { title: str(p.title) ? p.title : undefined, steps } }; + } + + if (component === 'stats') { + if (!Array.isArray(p.stats)) return null; + const stats = (p.stats as Array>) + .filter((s) => str(s.label) && str(s.value)) + .slice(0, 8) + .map((s) => ({ + label: s.label as string, + value: s.value as string, + delta: str(s.delta) ? s.delta : undefined, + direction: (s.direction === 'up' || s.direction === 'down' ? s.direction : undefined) as StatItem['direction'], + })); + if (stats.length === 0) return null; + return { component: 'stats', props: { title: str(p.title) ? p.title : undefined, stats } }; + } + + if (component === 'links') { + if (!Array.isArray(p.links)) return null; + const links = (p.links as Array>) + .filter((l) => str(l.title) && str(l.url) && /^https?:\/\//i.test(l.url as string)) + .slice(0, 10) + .map((l) => ({ + title: l.title as string, + url: l.url as string, + description: str(l.description) ? l.description : undefined, + })); + if (links.length === 0) return null; + return { component: 'links', props: { links } }; + } + + return null; +}