[eric] agents: CanvasCommand lets an agent move, collapse, tile, tidy and close its own cards after spawn (ENG-334)

backend/apps/agents/canvas_mcp_server.py

@@ -0,0 +1,150 @@
+#!/usr/bin/env python3
+"""Stdio MCP module letting an agent rearrange the canvas AFTER spawn (ENG-334).
+
+One tool, CanvasCommand, backed by /api/canvas/command, which relays to the
+renderer over the browser-command bridge. Placement at spawn already existed;
+this adds move/collapse/expand/tile/close/tidy so an agent can park its browser
+next to its chat, shrink itself when done, or clean up a helper card it opened.
+Close is enforced server-side to the caller's own card or cards it spawned."""
+
+import json
+import os
+import sys
+import urllib.error
+import urllib.request
+
+BACKEND_PORT = os.environ.get("OPENSWARM_PORT", "8324")
+BACKEND_AUTH = os.environ.get("OPENSWARM_AUTH_TOKEN", "")
+BACKEND_URL = f"http://127.0.0.1:{BACKEND_PORT}/api/canvas/command"
+PARENT_SESSION_ID = os.environ.get("OPENSWARM_PARENT_SESSION_ID", "")
+DASHBOARD_ID = os.environ.get("OPENSWARM_DASHBOARD_ID", "")
+
+TOOLS = [
+    {
+        "name": "CanvasCommand",
+        "description": (
+            "Control cards on the user's canvas after they exist: move a card, collapse or "
+            "expand it, tile it to a screen zone, close it, or tidy the whole board. "
+            "card_id defaults to YOUR own chat card; use a browser card's id (from "
+            "CreateBrowserAgent) or a child session id to control cards you spawned. "
+            "Closing is limited to your own card and cards you spawned. Use this sparingly "
+            "and purposefully: the canvas belongs to the user, so rearrange only when it "
+            "clearly helps the task (e.g. tuck your browser beside your chat, close a "
+            "helper you no longer need, tidy after spawning several cards)."
+        ),
+        "inputSchema": {
+            "type": "object",
+            "properties": {
+                "action": {
+                    "type": "string",
+                    "enum": ["move", "collapse", "expand", "tile", "close", "tidy"],
+                    "description": "What to do. tidy reflows every card on the board and ignores card_id.",
+                },
+                "card_id": {
+                    "type": "string",
+                    "description": "Target card: an agent session id, browser card id, app/view card id, or workflow card id. Defaults to your own card.",
+                },
+                "x": {"type": "number", "description": "move only: canvas x."},
+                "y": {"type": "number", "description": "move only: canvas y."},
+                "zone": {
+                    "type": "string",
+                    "enum": ["fill", "left", "right", "top", "bottom", "tl", "tr", "bl", "br", "fullscreen", "restore"],
+                    "description": "tile only: which screen zone; restore un-tiles.",
+                },
+            },
+            "required": ["action"],
+            "additionalProperties": False,
+        },
+    },
+]
+
+
+def send_response(id_, result=None, error=None):
+    msg = {"jsonrpc": "2.0", "id": id_}
+    if error is not None:
+        msg["error"] = error
+    else:
+        msg["result"] = result
+    sys.stdout.write(json.dumps(msg) + "\n")
+    sys.stdout.flush()
+
+
+def call_backend(payload: dict) -> dict:
+    full = {**payload, "parent_session_id": PARENT_SESSION_ID, "dashboard_id": DASHBOARD_ID}
+    body = json.dumps(full).encode()
+    headers = {"Content-Type": "application/json"}
+    if BACKEND_AUTH:
+        headers["Authorization"] = f"Bearer {BACKEND_AUTH}"
+    req = urllib.request.Request(BACKEND_URL, data=body, headers=headers, method="POST")
+    try:
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            return json.loads(resp.read().decode())
+    except urllib.error.HTTPError as e:
+        detail = e.read().decode() if e.fp else str(e)
+        return {"error": f"HTTP {e.code}: {detail}"}
+    except Exception as e:
+        return {"error": str(e)}
+
+
+def handle_tool_call(tool_name: str, arguments: dict) -> dict:
+    if tool_name != "CanvasCommand":
+        return {"content": [{"type": "text", "text": f"Unknown tool: {tool_name}"}], "isError": True}
+    action = str(arguments.get("action") or "").strip()
+    if not action:
+        return {"content": [{"type": "text", "text": "Error: `action` is required."}], "isError": True}
+    payload = {
+        "action": action,
+        "card_id": str(arguments.get("card_id") or ""),
+    }
+    if arguments.get("x") is not None:
+        payload["x"] = arguments.get("x")
+    if arguments.get("y") is not None:
+        payload["y"] = arguments.get("y")
+    if arguments.get("zone"):
+        payload["zone"] = str(arguments.get("zone"))
+    result = call_backend(payload)
+    if isinstance(result, dict) and result.get("error"):
+        return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "isError": True}
+    text = result.get("text") if isinstance(result, dict) else None
+    return {"content": [{"type": "text", "text": str(text or "Done.")}]}
+
+
+def main():
+    for line in sys.stdin:
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            msg = json.loads(line)
+        except json.JSONDecodeError:
+            continue
+        method = msg.get("method")
+        id_ = msg.get("id")
+        params = msg.get("params", {})
+        if method == "initialize":
+            send_response(id_, {
+                "protocolVersion": "2024-11-05",
+                "capabilities": {"tools": {}},
+                "serverInfo": {"name": "openswarm-canvas", "version": "1.0.0"},
+            })
+        elif method == "notifications/initialized":
+            pass
+        elif method == "tools/list":
+            send_response(id_, {"tools": TOOLS})
+        elif method == "tools/call":
+            tool_name = params.get("name", "")
+            arguments = params.get("arguments", {})
+            try:
+                send_response(id_, handle_tool_call(tool_name, arguments))
+            except Exception as e:
+                send_response(id_, error={"code": -32000, "message": str(e)})
+        elif method == "resources/list":
+            send_response(id_, {"resources": []})
+        elif method == "prompts/list":
+            send_response(id_, {"prompts": []})
+        elif id_ is not None:
+            send_response(id_, error={"code": -32601, "message": f"Method not found: {method}"})
+
+
+if __name__ == "__main__":
+    main()

backend/apps/agents/combined_meta_mcp_server.py

@@ -32,6 +32,7 @@ P_MODULE_FILES = {
     "schedule": "schedule_mcp_server",
     "web": "web_mcp_server",
     "browser": "browser_agent_mcp_server",
+    "canvas": "canvas_mcp_server",
 }
 
 P_ENABLED = [m.strip() for m in os.environ.get("OSW_MCP_MODULES", "meta,settings,apps").split(",") if m.strip()]

backend/apps/agents/manager/permissions/build_effective_tool_lists.py

@@ -104,6 +104,12 @@ def build_effective_tool_lists(
                             effective_allowed.append(f"mcp__openswarm-core__{ui_tool}")
                         else:
                             effective_disallowed.append(f"mcp__openswarm-core__{ui_tool}")
+                if "canvas" in p_modules:
+                    policy = builtin_perms.get("CanvasCommand", "always_allow")
+                    if policy == "always_allow":
+                        effective_allowed.append("mcp__openswarm-core__CanvasCommand")
+                    elif policy == "deny":
+                        effective_disallowed.append("mcp__openswarm-core__CanvasCommand")
                 if "web" in p_modules:
                     # Honor existing WebSearch/WebFetch permission policy, if the user disabled them in Settings, don't offer the MCP variants either.
                     for wt in ("WebSearch", "WebFetch"):

backend/apps/agents/manager/register_builtin_mcp_servers.py

@@ -75,6 +75,10 @@ def register_builtin_mcp_servers(
     # Schedule module: ScheduleWorkflow + CRUD + step editing so the agent (and the workflow Edit Agent) can build and schedule recurring work via the native scheduler. The 4 scheduling tools are force-gated in path_gate; Cron* is denied in build_effective_tool_lists.
     modules.append("schedule")
 
+    # Canvas control after spawn (ENG-334): move/collapse/tile/close/tidy; close is scoped server-side to the caller's own cards.
+    if builtin_perms.get("CanvasCommand", "always_allow") != "deny":
+        modules.append("canvas")
+
     # Only the card the user actually picked in select-mode gets claimed for the task, so the sub drives that one instead of opening its own duplicate. Passing EVERY dashboard card here (the old behavior) made the sub force-grab a random, usually-parked card and never navigate it, which broke the bulk of browser tasks.
     pre_selected_bids = [b for b in (selected_browser_ids or []) if b]
     # Apps the user selected this turn; the AppAgent tool may only target these (anti-hallucination gate in the MCP server, which reads this at startup).

backend/apps/tools_lib/models.py

@@ -38,6 +38,7 @@ BUILTIN_TOOLS: list[BuiltinTool] = [
     # Agent tools
     BuiltinTool(name="Agent", display_name="CreateAgent", description="Spawn a sub-agent to handle a complex subtask", category="agents"),
     BuiltinTool(name="InvokeAgent", description="Invoke a copy of an existing agent with a new message, preserving full conversation context", category="agents"),
+    BuiltinTool(name="CanvasCommand", display_name="Canvas control", description="Move, collapse, tile, close, or tidy cards on the canvas after spawn", category="agents"),
     # Browser delegation tools (Layer 1, what the main agent calls)
     BuiltinTool(name="CreateBrowserAgent", description="Create a new browser and run a task on it", category="browser_delegation"),
     BuiltinTool(name="BrowserAgent", description="Delegate a browser task to an existing browser agent", category="browser_delegation"),

backend/main.py

@@ -921,6 +921,49 @@ async def spawn_agent_run(request: Request):
         return JSONResponse({"error": str(e)}, status_code=500)
 
 
+P_CANVAS_ACTIONS = {"move", "collapse", "expand", "tile", "close", "tidy"}
+
+
+@app.post("/api/canvas/command")
+async def canvas_command(request: Request):
+    """Relay a CanvasCommand tool call to the renderer over the browser-command bridge (ENG-334).
+    Called by the canvas module in the per-session sidecar. Close is enforced HERE, not in the
+    renderer: only the caller's own card, or a card the caller spawned, may be closed."""
+    body = await request.json()
+    action = str(body.get("action") or "")
+    parent_session_id = str(body.get("parent_session_id") or "")
+    card_id = str(body.get("card_id") or "") or parent_session_id
+    if action not in P_CANVAS_ACTIONS:
+        return JSONResponse({"error": f"unknown action: {action}"}, status_code=400)
+    if not parent_session_id:
+        return JSONResponse({"error": "parent_session_id is required"}, status_code=400)
+    if action == "close" and card_id != parent_session_id:
+        from backend.apps.agents.agent_manager import agent_manager
+        p_sess = agent_manager.get_session(card_id)
+        owned = p_sess is not None and getattr(p_sess, "parent_session_id", None) == parent_session_id
+        if not owned:
+            p_dash_id = str(body.get("dashboard_id") or "")
+            try:
+                from backend.apps.dashboards.dashboards import load as p_dash_load
+                p_card = p_dash_load(p_dash_id).layout.browser_cards.get(card_id) if p_dash_id else None
+                owned = p_card is not None and getattr(p_card, "spawned_by", None) == parent_session_id
+            except Exception:
+                owned = False
+        if not owned:
+            return JSONResponse(
+                {"error": "close is limited to your own card or cards you spawned"}, status_code=403)
+    params = {
+        "action": action,
+        "card_id": card_id,
+        "x": body.get("x"),
+        "y": body.get("y"),
+        "zone": body.get("zone"),
+    }
+    result = await ws_manager.send_browser_command(uuid4().hex, "canvas_command", card_id, params)
+    ok = isinstance(result, dict) and not result.get("error")
+    return JSONResponse(result, status_code=200 if ok else 502)
+
+
 @app.post("/api/ui-requests/wait")
 async def ui_request_wait(request: Request):
     """AskUI's blocking half: parks until the user answers the interactive component in the

backend/tests/test_canvas_command_route.py

@@ -0,0 +1,88 @@
+"""ENG-334: the CanvasCommand route's close-guard is the security boundary.
+
+An agent may rearrange the canvas freely, but close is destructive, so the route (not the
+renderer, not the tool list) must refuse closing anything the caller does not own: its own card,
+a session it spawned, or a browser card it spawned. A client that skips the MCP tool and posts
+straight here must hit the same wall."""
+import secrets
+from unittest.mock import AsyncMock, patch
+
+import pytest
+from fastapi.testclient import TestClient
+
+
+@pytest.fixture()
+def p_client():
+    import backend.auth as auth_mod
+    from backend.main import app
+
+    if not auth_mod.TOKEN:
+        auth_mod.TOKEN = secrets.token_urlsafe(32)
+    return TestClient(app, headers={"Authorization": f"Bearer {auth_mod.TOKEN}"})
+
+
+def test_unknown_action_is_a_400(p_client):
+    r = p_client.post("/api/canvas/command", json={"action": "yeet", "parent_session_id": "s1"})
+    assert r.status_code == 400
+
+
+def test_missing_parent_session_is_a_400(p_client):
+    r = p_client.post("/api/canvas/command", json={"action": "move", "x": 1, "y": 2})
+    assert r.status_code == 400
+
+
+def test_close_of_a_foreign_card_is_refused(p_client):
+    with patch("backend.apps.agents.agent_manager.agent_manager.get_session", return_value=None):
+        r = p_client.post("/api/canvas/command", json={
+            "action": "close", "card_id": "someone-elses-card", "parent_session_id": "s1",
+        })
+    assert r.status_code == 403
+    assert "your own card" in r.json()["error"]
+
+
+def test_close_of_own_card_relays_to_the_renderer(p_client):
+    relay = AsyncMock(return_value={"text": "Closed agent card s1."})
+    with patch("backend.main.ws_manager.send_browser_command", relay):
+        r = p_client.post("/api/canvas/command", json={
+            "action": "close", "parent_session_id": "s1",
+        })
+    assert r.status_code == 200
+    assert relay.await_count == 1
+    p_args = relay.await_args.args
+    assert p_args[1] == "canvas_command"
+    assert p_args[3]["card_id"] == "s1"
+
+
+def test_close_of_a_spawned_child_session_is_allowed(p_client):
+    class P_Child:
+        parent_session_id = "s1"
+
+    relay = AsyncMock(return_value={"text": "Closed."})
+    with patch("backend.apps.agents.agent_manager.agent_manager.get_session", return_value=P_Child()), \
+         patch("backend.main.ws_manager.send_browser_command", relay):
+        r = p_client.post("/api/canvas/command", json={
+            "action": "close", "card_id": "child-1", "parent_session_id": "s1",
+        })
+    assert r.status_code == 200
+    assert relay.await_count == 1
+
+
+def test_move_defaults_to_the_callers_own_card_and_relays(p_client):
+    relay = AsyncMock(return_value={"text": "Moved."})
+    with patch("backend.main.ws_manager.send_browser_command", relay):
+        r = p_client.post("/api/canvas/command", json={
+            "action": "move", "parent_session_id": "s1", "x": 120, "y": 80,
+        })
+    assert r.status_code == 200
+    p_params = relay.await_args.args[3]
+    assert p_params == {"action": "move", "card_id": "s1", "x": 120, "y": 80, "zone": None}
+
+
+def test_renderer_error_comes_back_honest_not_200(p_client):
+    relay = AsyncMock(return_value={"error": "No dashboard is connected."})
+    with patch("backend.main.ws_manager.send_browser_command", relay):
+        r = p_client.post("/api/canvas/command", json={
+            "action": "tidy", "parent_session_id": "s1",
+        })
+    assert r.status_code == 502
+    assert "error" in r.json()

backend/tests/test_publish_backend_bundle.py

@@ -4,11 +4,8 @@ the cloud can trust has_backend without unpacking the tar."""
 
 import io
 import json
-import os
 import tarfile
 
-import pytest
-
 from backend.apps.outputs.models import Output
 from backend.apps.outputs import publish_build
 

frontend/src/shared/browserCommandHandler.ts

@@ -12,6 +12,7 @@ import { rankAndCapInteractives, type RankItem } from './interactiveRanking';
 import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettle';
 import { unwrapCdpEval } from './cdpEval';
 import { navigationOutcome, sameDoc } from './navigationOutcome';
+import { handleCanvasCommand } from './canvasCommandHandler';
 import { typeChars, type TypedKeys } from './typeChars';
 
 let initialized = false;
@@ -2370,6 +2371,11 @@ async function runBrowserCommand(
     dashboardWs.send('browser:result', { request_id, ...result });
     return;
   }
+  if (action === 'canvas_command') {
+    const result = await handleCanvasCommand(params);
+    dashboardWs.send('browser:result', { request_id, ...result });
+    return;
+  }
   const p_gateT0 = Date.now();
   const wv = await awaitWebview(browser_id, tab_id || undefined, action);
   // A command that spends seconds before its handler even starts looks identical, from the backend,

frontend/src/shared/canvasCommandHandler.ts

@@ -0,0 +1,102 @@
+// ENG-334: the renderer half of the CanvasCommand tool. An agent could place its card once at
+// spawn and never touch the canvas again; this executes move/collapse/expand/tile/close/tidy
+// against the live stores, answering over the same request/response bridge browser commands use.
+import { store } from './state/store';
+import {
+  setCardPosition,
+  setBrowserCardPosition,
+  setViewCardPosition,
+  setWorkflowCardPosition,
+  setTiledCard,
+  clearTiledCard,
+  toggleMinimizeCard,
+  removeCard,
+  removeViewCard,
+  removeWorkflowCard,
+  recordClosedCard,
+  tidyLayout,
+  bringToFront,
+  type CardType,
+} from './state/dashboardLayoutSlice';
+import { collapseSession, expandSession, closeSession } from './state/agentsSlice';
+import { removeBrowserCardCleanly } from './browserTeardown';
+
+type CanvasKind = Extract<CardType, 'agent' | 'browser' | 'view' | 'workflow'>;
+
+const ZONES = new Set(['fill', 'left', 'right', 'top', 'bottom', 'tl', 'tr', 'bl', 'br', 'fullscreen', 'restore']);
+
+function findCardKind(id: string): CanvasKind | null {
+  const s = store.getState().dashboardLayout;
+  if (s.cards[id]) return 'agent';
+  if (s.browserCards[id]) return 'browser';
+  if (s.viewCards[id]) return 'view';
+  if (s.workflowCards[id]) return 'workflow';
+  return null;
+}
+
+export async function handleCanvasCommand(params: Record<string, any>): Promise<Record<string, any>> {
+  const action = String(params.action || '');
+  if (action === 'tidy') {
+    store.dispatch(tidyLayout({ expandedSessionIds: store.getState().agents.expandedSessionIds }));
+    return { text: 'Canvas tidied: every card reflowed onto the grid.' };
+  }
+  const id = String(params.card_id || '');
+  if (!id) return { error: 'card_id is required' };
+  const kind = findCardKind(id);
+  if (!kind) {
+    return { error: `No card '${id}' is on the canvas. Agent chats use their session id, browser cards their browser id.` };
+  }
+  if (action === 'move') {
+    const x = Number(params.x);
+    const y = Number(params.y);
+    if (!Number.isFinite(x) || !Number.isFinite(y)) return { error: 'move needs numeric x and y' };
+    if (kind === 'agent') store.dispatch(setCardPosition({ sessionId: id, x, y }));
+    else if (kind === 'browser') store.dispatch(setBrowserCardPosition({ browserId: id, x, y }));
+    else if (kind === 'view') store.dispatch(setViewCardPosition({ outputId: id, x, y }));
+    else store.dispatch(setWorkflowCardPosition({ workflowId: id, x, y }));
+    store.dispatch(bringToFront({ id, type: kind }));
+    return { text: `Moved ${kind} card to (${Math.round(x)}, ${Math.round(y)}).` };
+  }
+  if (action === 'collapse') {
+    if (kind === 'agent') store.dispatch(collapseSession(id));
+    else if (!store.getState().dashboardLayout.minimizedCards[id]) store.dispatch(toggleMinimizeCard({ cardId: id }));
+    return { text: `Collapsed ${kind} card ${id}.` };
+  }
+  if (action === 'expand') {
+    if (kind === 'agent') store.dispatch(expandSession(id));
+    else if (store.getState().dashboardLayout.minimizedCards[id]) store.dispatch(toggleMinimizeCard({ cardId: id }));
+    store.dispatch(bringToFront({ id, type: kind }));
+    return { text: `Expanded ${kind} card ${id}.` };
+  }
+  if (action === 'tile') {
+    const zone = String(params.zone || '');
+    if (!ZONES.has(zone)) return { error: `zone must be one of: ${Array.from(ZONES).join(', ')}` };
+    if (zone === 'restore') {
+      store.dispatch(clearTiledCard(id));
+      return { text: `Restored ${kind} card ${id} from its tile.` };
+    }
+    store.dispatch(setTiledCard({ cardId: id, zone }));
+    store.dispatch(bringToFront({ id, type: kind }));
+    return { text: `Tiled ${kind} card ${id} to ${zone}.` };
+  }
+  if (action === 'close') {
+    if (kind === 'agent') {
+      // Mirrors the user's own close sequence (AgentCard.handleRemove) so undo keeps working.
+      store.dispatch(recordClosedCard({ kind: 'agent', id }));
+      store.dispatch(collapseSession(id));
+      store.dispatch(removeCard(id));
+      void store.dispatch(closeSession({ sessionId: id }));
+    } else if (kind === 'browser') {
+      store.dispatch(recordClosedCard({ kind: 'browser', id }));
+      await removeBrowserCardCleanly(id, store.dispatch);
+    } else if (kind === 'view') {
+      store.dispatch(recordClosedCard({ kind: 'view', id }));
+      store.dispatch(removeViewCard(id));
+    } else {
+      store.dispatch(recordClosedCard({ kind: 'workflow', id }));
+      store.dispatch(removeWorkflowCard(id));
+    }
+    return { text: `Closed ${kind} card ${id}.` };
+  }
+  return { error: `Unknown canvas action '${action}'. Use move, collapse, expand, tile, close, or tidy.` };
+}