From ce2989ea1b6ab662ca02cd6e7d1d0368b7f02501 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Mon, 17 Aug 2026 16:04:21 -0700 Subject: [PATCH] [eric] agents: CanvasCommand lets an agent move, collapse, tile, tidy and close its own cards after spawn (ENG-334) --- backend/apps/agents/canvas_mcp_server.py | 150 ++++++++++++++++++ .../apps/agents/combined_meta_mcp_server.py | 1 + .../permissions/build_effective_tool_lists.py | 6 + .../manager/register_builtin_mcp_servers.py | 4 + backend/apps/tools_lib/models.py | 1 + backend/main.py | 43 +++++ backend/tests/test_canvas_command_route.py | 88 ++++++++++ backend/tests/test_publish_backend_bundle.py | 3 - frontend/src/shared/browserCommandHandler.ts | 6 + frontend/src/shared/canvasCommandHandler.ts | 102 ++++++++++++ 10 files changed, 401 insertions(+), 3 deletions(-) create mode 100644 backend/apps/agents/canvas_mcp_server.py create mode 100644 backend/tests/test_canvas_command_route.py create mode 100644 frontend/src/shared/canvasCommandHandler.ts diff --git a/backend/apps/agents/canvas_mcp_server.py b/backend/apps/agents/canvas_mcp_server.py new file mode 100644 index 00000000..5cae669b --- /dev/null +++ b/backend/apps/agents/canvas_mcp_server.py @@ -0,0 +1,150 @@ +#!/usr/bin/env python3 +"""Stdio MCP module letting an agent rearrange the canvas AFTER spawn (ENG-334). + +One tool, CanvasCommand, backed by /api/canvas/command, which relays to the +renderer over the browser-command bridge. Placement at spawn already existed; +this adds move/collapse/expand/tile/close/tidy so an agent can park its browser +next to its chat, shrink itself when done, or clean up a helper card it opened. +Close is enforced server-side to the caller's own card or cards it spawned.""" + +import json +import os +import sys +import urllib.error +import urllib.request + +BACKEND_PORT = os.environ.get("OPENSWARM_PORT", "8324") +BACKEND_AUTH = os.environ.get("OPENSWARM_AUTH_TOKEN", "") +BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/canvas/command" +PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "") +DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "") + +TOOLS = [ + { + "name": "CanvasCommand", + "description": ( + "Control cards on the user's canvas after they exist: move a card, collapse or " + "expand it, tile it to a screen zone, close it, or tidy the whole board. " + "card_id defaults to YOUR own chat card; use a browser card's id (from " + "CreateBrowserAgent) or a child session id to control cards you spawned. " + "Closing is limited to your own card and cards you spawned. Use this sparingly " + "and purposefully: the canvas belongs to the user, so rearrange only when it " + "clearly helps the task (e.g. tuck your browser beside your chat, close a " + "helper you no longer need, tidy after spawning several cards)." + ), + "inputSchema": { + "type": "object", + "properties": { + "action": { + "type": "string", + "enum": ["move", "collapse", "expand", "tile", "close", "tidy"], + "description": "What to do. tidy reflows every card on the board and ignores card_id.", + }, + "card_id": { + "type": "string", + "description": "Target card: an agent session id, browser card id, app/view card id, or workflow card id. Defaults to your own card.", + }, + "x": {"type": "number", "description": "move only: canvas x."}, + "y": {"type": "number", "description": "move only: canvas y."}, + "zone": { + "type": "string", + "enum": ["fill", "left", "right", "top", "bottom", "tl", "tr", "bl", "br", "fullscreen", "restore"], + "description": "tile only: which screen zone; restore un-tiles.", + }, + }, + "required": ["action"], + "additionalProperties": False, + }, + }, +] + + +def send_response(id_, result=None, error=None): + msg = {"jsonrpc": "2.0", "id": id_} + if error is not None: + msg["error"] = error + else: + msg["result"] = result + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def call_backend(payload: dict) -> dict: + full = {**payload, "parent_session_id": PARENT_SESSION_ID, "dashboard_id": DASHBOARD_ID} + body = json.dumps(full).encode() + headers = {"Content-Type": "application/json"} + if BACKEND_AUTH: + headers["Authorization"] = f"Bearer {BACKEND_AUTH}" + req = urllib.request.Request(BACKEND_URL, data=body, headers=headers, method="POST") + try: + with urllib.request.urlopen(req, timeout=30) as resp: + return json.loads(resp.read().decode()) + except urllib.error.HTTPError as e: + detail = e.read().decode() if e.fp else str(e) + return {"error": f"HTTP {e.code}: {detail}"} + except Exception as e: + return {"error": str(e)} + + +def handle_tool_call(tool_name: str, arguments: dict) -> dict: + if tool_name != "CanvasCommand": + return {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True} + action = str(arguments.get("action") or "").strip() + if not action: + return {"content": [{"type": "text", "text": "Error: `action` is required."}], "isError": True} + payload = { + "action": action, + "card_id": str(arguments.get("card_id") or ""), + } + if arguments.get("x") is not None: + payload["x"] = arguments.get("x") + if arguments.get("y") is not None: + payload["y"] = arguments.get("y") + if arguments.get("zone"): + payload["zone"] = str(arguments.get("zone")) + result = call_backend(payload) + if isinstance(result, dict) and result.get("error"): + return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True} + text = result.get("text") if isinstance(result, dict) else None + return {"content": [{"type": "text", "text": str(text or "Done.")}]} + + +def main(): + for line in sys.stdin: + line = line.strip() + if not line: + continue + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + method = msg.get("method") + id_ = msg.get("id") + params = msg.get("params", {}) + if method == "initialize": + send_response(id_, { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "openswarm-canvas", "version": "1.0.0"}, + }) + elif method == "notifications/initialized": + pass + elif method == "tools/list": + send_response(id_, {"tools": TOOLS}) + elif method == "tools/call": + tool_name = params.get("name", "") + arguments = params.get("arguments", {}) + try: + send_response(id_, handle_tool_call(tool_name, arguments)) + except Exception as e: + send_response(id_, error={"code": -32000, "message": str(e)}) + elif method == "resources/list": + send_response(id_, {"resources": []}) + elif method == "prompts/list": + send_response(id_, {"prompts": []}) + elif id_ is not None: + send_response(id_, error={"code": -32601, "message": f"Method not found: {method}"}) + + +if __name__ == "__main__": + main() diff --git a/backend/apps/agents/combined_meta_mcp_server.py b/backend/apps/agents/combined_meta_mcp_server.py index 6c841904..216b3ff9 100644 --- a/backend/apps/agents/combined_meta_mcp_server.py +++ b/backend/apps/agents/combined_meta_mcp_server.py @@ -32,6 +32,7 @@ P_MODULE_FILES = { "schedule": "schedule_mcp_server", "web": "web_mcp_server", "browser": "browser_agent_mcp_server", + "canvas": "canvas_mcp_server", } P_ENABLED = [m.strip() for m in os.environ.get("OSW_MCP_MODULES", "meta,settings,apps").split(",") if m.strip()] diff --git a/backend/apps/agents/manager/permissions/build_effective_tool_lists.py b/backend/apps/agents/manager/permissions/build_effective_tool_lists.py index a4527040..8df9f26e 100644 --- a/backend/apps/agents/manager/permissions/build_effective_tool_lists.py +++ b/backend/apps/agents/manager/permissions/build_effective_tool_lists.py @@ -104,6 +104,12 @@ def build_effective_tool_lists( effective_allowed.append(f"mcp__openswarm-core__{ui_tool}") else: effective_disallowed.append(f"mcp__openswarm-core__{ui_tool}") + if "canvas" in p_modules: + policy = builtin_perms.get("CanvasCommand", "always_allow") + if policy == "always_allow": + effective_allowed.append("mcp__openswarm-core__CanvasCommand") + elif policy == "deny": + effective_disallowed.append("mcp__openswarm-core__CanvasCommand") if "web" in p_modules: # Honor existing WebSearch/WebFetch permission policy, if the user disabled them in Settings, don't offer the MCP variants either. for wt in ("WebSearch", "WebFetch"): diff --git a/backend/apps/agents/manager/register_builtin_mcp_servers.py b/backend/apps/agents/manager/register_builtin_mcp_servers.py index df5ea4a0..6df65801 100644 --- a/backend/apps/agents/manager/register_builtin_mcp_servers.py +++ b/backend/apps/agents/manager/register_builtin_mcp_servers.py @@ -75,6 +75,10 @@ def register_builtin_mcp_servers( # Schedule module: ScheduleWorkflow + CRUD + step editing so the agent (and the workflow Edit Agent) can build and schedule recurring work via the native scheduler. The 4 scheduling tools are force-gated in path_gate; Cron* is denied in build_effective_tool_lists. modules.append("schedule") + # Canvas control after spawn (ENG-334): move/collapse/tile/close/tidy; close is scoped server-side to the caller's own cards. + if builtin_perms.get("CanvasCommand", "always_allow") != "deny": + modules.append("canvas") + # Only the card the user actually picked in select-mode gets claimed for the task, so the sub drives that one instead of opening its own duplicate. Passing EVERY dashboard card here (the old behavior) made the sub force-grab a random, usually-parked card and never navigate it, which broke the bulk of browser tasks. pre_selected_bids = [b for b in (selected_browser_ids or []) if b] # Apps the user selected this turn; the AppAgent tool may only target these (anti-hallucination gate in the MCP server, which reads this at startup). diff --git a/backend/apps/tools_lib/models.py b/backend/apps/tools_lib/models.py index bc658a52..7d2ce81e 100644 --- a/backend/apps/tools_lib/models.py +++ b/backend/apps/tools_lib/models.py @@ -38,6 +38,7 @@ BUILTIN_TOOLS: list[BuiltinTool] = [ # Agent tools BuiltinTool(name="Agent", display_name="CreateAgent", description="Spawn a sub-agent to handle a complex subtask", category="agents"), BuiltinTool(name="InvokeAgent", description="Invoke a copy of an existing agent with a new message, preserving full conversation context", category="agents"), + BuiltinTool(name="CanvasCommand", display_name="Canvas control", description="Move, collapse, tile, close, or tidy cards on the canvas after spawn", category="agents"), # Browser delegation tools (Layer 1, what the main agent calls) BuiltinTool(name="CreateBrowserAgent", description="Create a new browser and run a task on it", category="browser_delegation"), BuiltinTool(name="BrowserAgent", description="Delegate a browser task to an existing browser agent", category="browser_delegation"), diff --git a/backend/main.py b/backend/main.py index cd20a869..abd5a0f4 100644 --- a/backend/main.py +++ b/backend/main.py @@ -921,6 +921,49 @@ async def spawn_agent_run(request: Request): return JSONResponse({"error": str(e)}, status_code=500) +P_CANVAS_ACTIONS = {"move", "collapse", "expand", "tile", "close", "tidy"} + + +@app.post("/api/canvas/command") +async def canvas_command(request: Request): + """Relay a CanvasCommand tool call to the renderer over the browser-command bridge (ENG-334). + Called by the canvas module in the per-session sidecar. Close is enforced HERE, not in the + renderer: only the caller's own card, or a card the caller spawned, may be closed.""" + body = await request.json() + action = str(body.get("action") or "") + parent_session_id = str(body.get("parent_session_id") or "") + card_id = str(body.get("card_id") or "") or parent_session_id + if action not in P_CANVAS_ACTIONS: + return JSONResponse({"error": f"unknown action: {action}"}, status_code=400) + if not parent_session_id: + return JSONResponse({"error": "parent_session_id is required"}, status_code=400) + if action == "close" and card_id != parent_session_id: + from backend.apps.agents.agent_manager import agent_manager + p_sess = agent_manager.get_session(card_id) + owned = p_sess is not None and getattr(p_sess, "parent_session_id", None) == parent_session_id + if not owned: + p_dash_id = str(body.get("dashboard_id") or "") + try: + from backend.apps.dashboards.dashboards import load as p_dash_load + p_card = p_dash_load(p_dash_id).layout.browser_cards.get(card_id) if p_dash_id else None + owned = p_card is not None and getattr(p_card, "spawned_by", None) == parent_session_id + except Exception: + owned = False + if not owned: + return JSONResponse( + {"error": "close is limited to your own card or cards you spawned"}, status_code=403) + params = { + "action": action, + "card_id": card_id, + "x": body.get("x"), + "y": body.get("y"), + "zone": body.get("zone"), + } + result = await ws_manager.send_browser_command(uuid4().hex, "canvas_command", card_id, params) + ok = isinstance(result, dict) and not result.get("error") + return JSONResponse(result, status_code=200 if ok else 502) + + @app.post("/api/ui-requests/wait") async def ui_request_wait(request: Request): """AskUI's blocking half: parks until the user answers the interactive component in the diff --git a/backend/tests/test_canvas_command_route.py b/backend/tests/test_canvas_command_route.py new file mode 100644 index 00000000..d4aa9e8a --- /dev/null +++ b/backend/tests/test_canvas_command_route.py @@ -0,0 +1,88 @@ +"""ENG-334: the CanvasCommand route's close-guard is the security boundary. + +An agent may rearrange the canvas freely, but close is destructive, so the route (not the +renderer, not the tool list) must refuse closing anything the caller does not own: its own card, +a session it spawned, or a browser card it spawned. A client that skips the MCP tool and posts +straight here must hit the same wall.""" +import secrets +from unittest.mock import AsyncMock, patch + +import pytest +from fastapi.testclient import TestClient + + +@pytest.fixture() +def p_client(): + import backend.auth as auth_mod + from backend.main import app + + if not auth_mod.TOKEN: + auth_mod.TOKEN = secrets.token_urlsafe(32) + return TestClient(app, headers={"Authorization": f"Bearer {auth_mod.TOKEN}"}) + + +def test_unknown_action_is_a_400(p_client): + r = p_client.post("/api/canvas/command", json={"action": "yeet", "parent_session_id": "s1"}) + assert r.status_code == 400 + + +def test_missing_parent_session_is_a_400(p_client): + r = p_client.post("/api/canvas/command", json={"action": "move", "x": 1, "y": 2}) + assert r.status_code == 400 + + +def test_close_of_a_foreign_card_is_refused(p_client): + with patch("backend.apps.agents.agent_manager.agent_manager.get_session", return_value=None): + r = p_client.post("/api/canvas/command", json={ + "action": "close", "card_id": "someone-elses-card", "parent_session_id": "s1", + }) + assert r.status_code == 403 + assert "your own card" in r.json()["error"] + + +def test_close_of_own_card_relays_to_the_renderer(p_client): + relay = AsyncMock(return_value={"text": "Closed agent card s1."}) + with patch("backend.main.ws_manager.send_browser_command", relay): + r = p_client.post("/api/canvas/command", json={ + "action": "close", "parent_session_id": "s1", + }) + assert r.status_code == 200 + assert relay.await_count == 1 + p_args = relay.await_args.args + assert p_args[1] == "canvas_command" + assert p_args[3]["card_id"] == "s1" + + +def test_close_of_a_spawned_child_session_is_allowed(p_client): + class P_Child: + parent_session_id = "s1" + + relay = AsyncMock(return_value={"text": "Closed."}) + with patch("backend.apps.agents.agent_manager.agent_manager.get_session", return_value=P_Child()), \ + patch("backend.main.ws_manager.send_browser_command", relay): + r = p_client.post("/api/canvas/command", json={ + "action": "close", "card_id": "child-1", "parent_session_id": "s1", + }) + assert r.status_code == 200 + assert relay.await_count == 1 + + +def test_move_defaults_to_the_callers_own_card_and_relays(p_client): + relay = AsyncMock(return_value={"text": "Moved."}) + with patch("backend.main.ws_manager.send_browser_command", relay): + r = p_client.post("/api/canvas/command", json={ + "action": "move", "parent_session_id": "s1", "x": 120, "y": 80, + }) + assert r.status_code == 200 + p_params = relay.await_args.args[3] + assert p_params == {"action": "move", "card_id": "s1", "x": 120, "y": 80, "zone": None} + + +def test_renderer_error_comes_back_honest_not_200(p_client): + relay = AsyncMock(return_value={"error": "No dashboard is connected."}) + with patch("backend.main.ws_manager.send_browser_command", relay): + r = p_client.post("/api/canvas/command", json={ + "action": "tidy", "parent_session_id": "s1", + }) + assert r.status_code == 502 + assert "error" in r.json() diff --git a/backend/tests/test_publish_backend_bundle.py b/backend/tests/test_publish_backend_bundle.py index 407a3a77..30caa321 100644 --- a/backend/tests/test_publish_backend_bundle.py +++ b/backend/tests/test_publish_backend_bundle.py @@ -4,11 +4,8 @@ the cloud can trust has_backend without unpacking the tar.""" import io import json -import os import tarfile -import pytest - from backend.apps.outputs.models import Output from backend.apps.outputs import publish_build diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 753f5aa0..ab42a827 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -12,6 +12,7 @@ import { rankAndCapInteractives, type RankItem } from './interactiveRanking'; import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettle'; import { unwrapCdpEval } from './cdpEval'; import { navigationOutcome, sameDoc } from './navigationOutcome'; +import { handleCanvasCommand } from './canvasCommandHandler'; import { typeChars, type TypedKeys } from './typeChars'; let initialized = false; @@ -2370,6 +2371,11 @@ async function runBrowserCommand( dashboardWs.send('browser:result', { request_id, ...result }); return; } + if (action === 'canvas_command') { + const result = await handleCanvasCommand(params); + dashboardWs.send('browser:result', { request_id, ...result }); + return; + } const p_gateT0 = Date.now(); const wv = await awaitWebview(browser_id, tab_id || undefined, action); // A command that spends seconds before its handler even starts looks identical, from the backend, diff --git a/frontend/src/shared/canvasCommandHandler.ts b/frontend/src/shared/canvasCommandHandler.ts new file mode 100644 index 00000000..a2a64b7c --- /dev/null +++ b/frontend/src/shared/canvasCommandHandler.ts @@ -0,0 +1,102 @@ +// ENG-334: the renderer half of the CanvasCommand tool. An agent could place its card once at +// spawn and never touch the canvas again; this executes move/collapse/expand/tile/close/tidy +// against the live stores, answering over the same request/response bridge browser commands use. +import { store } from './state/store'; +import { + setCardPosition, + setBrowserCardPosition, + setViewCardPosition, + setWorkflowCardPosition, + setTiledCard, + clearTiledCard, + toggleMinimizeCard, + removeCard, + removeViewCard, + removeWorkflowCard, + recordClosedCard, + tidyLayout, + bringToFront, + type CardType, +} from './state/dashboardLayoutSlice'; +import { collapseSession, expandSession, closeSession } from './state/agentsSlice'; +import { removeBrowserCardCleanly } from './browserTeardown'; + +type CanvasKind = Extract; + +const ZONES = new Set(['fill', 'left', 'right', 'top', 'bottom', 'tl', 'tr', 'bl', 'br', 'fullscreen', 'restore']); + +function findCardKind(id: string): CanvasKind | null { + const s = store.getState().dashboardLayout; + if (s.cards[id]) return 'agent'; + if (s.browserCards[id]) return 'browser'; + if (s.viewCards[id]) return 'view'; + if (s.workflowCards[id]) return 'workflow'; + return null; +} + +export async function handleCanvasCommand(params: Record): Promise> { + const action = String(params.action || ''); + if (action === 'tidy') { + store.dispatch(tidyLayout({ expandedSessionIds: store.getState().agents.expandedSessionIds })); + return { text: 'Canvas tidied: every card reflowed onto the grid.' }; + } + const id = String(params.card_id || ''); + if (!id) return { error: 'card_id is required' }; + const kind = findCardKind(id); + if (!kind) { + return { error: `No card '${id}' is on the canvas. Agent chats use their session id, browser cards their browser id.` }; + } + if (action === 'move') { + const x = Number(params.x); + const y = Number(params.y); + if (!Number.isFinite(x) || !Number.isFinite(y)) return { error: 'move needs numeric x and y' }; + if (kind === 'agent') store.dispatch(setCardPosition({ sessionId: id, x, y })); + else if (kind === 'browser') store.dispatch(setBrowserCardPosition({ browserId: id, x, y })); + else if (kind === 'view') store.dispatch(setViewCardPosition({ outputId: id, x, y })); + else store.dispatch(setWorkflowCardPosition({ workflowId: id, x, y })); + store.dispatch(bringToFront({ id, type: kind })); + return { text: `Moved ${kind} card to (${Math.round(x)}, ${Math.round(y)}).` }; + } + if (action === 'collapse') { + if (kind === 'agent') store.dispatch(collapseSession(id)); + else if (!store.getState().dashboardLayout.minimizedCards[id]) store.dispatch(toggleMinimizeCard({ cardId: id })); + return { text: `Collapsed ${kind} card ${id}.` }; + } + if (action === 'expand') { + if (kind === 'agent') store.dispatch(expandSession(id)); + else if (store.getState().dashboardLayout.minimizedCards[id]) store.dispatch(toggleMinimizeCard({ cardId: id })); + store.dispatch(bringToFront({ id, type: kind })); + return { text: `Expanded ${kind} card ${id}.` }; + } + if (action === 'tile') { + const zone = String(params.zone || ''); + if (!ZONES.has(zone)) return { error: `zone must be one of: ${Array.from(ZONES).join(', ')}` }; + if (zone === 'restore') { + store.dispatch(clearTiledCard(id)); + return { text: `Restored ${kind} card ${id} from its tile.` }; + } + store.dispatch(setTiledCard({ cardId: id, zone })); + store.dispatch(bringToFront({ id, type: kind })); + return { text: `Tiled ${kind} card ${id} to ${zone}.` }; + } + if (action === 'close') { + if (kind === 'agent') { + // Mirrors the user's own close sequence (AgentCard.handleRemove) so undo keeps working. + store.dispatch(recordClosedCard({ kind: 'agent', id })); + store.dispatch(collapseSession(id)); + store.dispatch(removeCard(id)); + void store.dispatch(closeSession({ sessionId: id })); + } else if (kind === 'browser') { + store.dispatch(recordClosedCard({ kind: 'browser', id })); + await removeBrowserCardCleanly(id, store.dispatch); + } else if (kind === 'view') { + store.dispatch(recordClosedCard({ kind: 'view', id })); + store.dispatch(removeViewCard(id)); + } else { + store.dispatch(recordClosedCard({ kind: 'workflow', id })); + store.dispatch(removeWorkflowCard(id)); + } + return { text: `Closed ${kind} card ${id}.` }; + } + return { error: `Unknown canvas action '${action}'. Use move, collapse, expand, tile, close, or tidy.` }; +}