diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 93fcd11d..50fd6773 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -46,6 +46,26 @@ def p_make_session_emitter(session_id: str) -> EventCallback: await ws.send_to_session(session_id, event.event, event.model_dump(mode="json")) return emit + +async def p_send_browser_command( + action: str, browser_id: str, tab_id: str, params: dict, +) -> dict: + """BrowserCommandFn implementation that routes through the browser FutureBridge.""" + request_id = uuid4().hex + if not ws.has_global_connections(): + return {"error": "No dashboard connected. Open the dashboard to use browser tools."} + return await ws.BROWSER_BRIDGE.request( + request_id=request_id, + send_fn=lambda: ws.broadcast_global("browser:command", { + "request_id": request_id, + "action": action, + "browser_id": browser_id, + "tab_id": tab_id, + "params": params, + }), + timeout=30.0, + ) + def get_agent(session_id: str) -> Agent: agent: Optional[Agent] = SESSIONS.get(session_id) if not agent: diff --git a/backend/core/tools/make_builtin_toolkit/make_builtin_toolkit.py b/backend/core/tools/make_builtin_toolkit/make_builtin_toolkit.py index 3499331c..359858b6 100644 --- a/backend/core/tools/make_builtin_toolkit/make_builtin_toolkit.py +++ b/backend/core/tools/make_builtin_toolkit/make_builtin_toolkit.py @@ -1,10 +1,10 @@ -# BUILTIN_TOOLKIT.py - from typing import Dict from typeguard import typechecked from backend.core.tools.shared_structs.Toolkit import Toolkit from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.agents_toolkit.make_agents_toolkit import make_agents_toolkit -from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_toolkit import make_browser_toolkit +from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.make_browser_delegation_toolkit import make_browser_delegation_toolkit +from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.make_browser_actions_toolkit import make_browser_actions_toolkit +from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import BrowserCommandFn from backend.core.tools.make_builtin_toolkit.pre_existing_toolkits.basic_toolkits.FILESYSTEM_TOOLKIT import FILESYSTEM_TOOLKIT from backend.core.tools.make_builtin_toolkit.pre_existing_toolkits.meta_toolkits.INTERACTION_TOOLKIT import INTERACTION_TOOLKIT @@ -18,13 +18,15 @@ from backend.core.Agent.Agent import Agent def make_builtin_toolkit( parent: Agent, agent_registry: Dict[str, Agent], + send_browser_command: BrowserCommandFn, ) -> Toolkit: return Toolkit( name="builtin", description="Builtin tools", nested_toolkits=[ make_agents_toolkit(parent, agent_registry), - make_browser_toolkit(parent), + make_browser_delegation_toolkit(parent, send_browser_command), + make_browser_actions_toolkit(parent, send_browser_command), FILESYSTEM_TOOLKIT, INTERACTION_TOOLKIT, PLANNING_TOOLKIT, diff --git a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_actions_toolkit/handlers/make_browser_action_handler.py b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_actions_toolkit/handlers/make_browser_action_handler.py index a81fc2b7..042e00df 100644 --- a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_actions_toolkit/handlers/make_browser_action_handler.py +++ b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_actions_toolkit/handlers/make_browser_action_handler.py @@ -1,22 +1,56 @@ -from typing import Dict, Any +import json +from typing import Dict, Any, Callable, Awaitable from typeguard import typechecked + from backend.core.shared_structs.agent.Message.agent_outputs import ToolResponse -from backend.OLDapps.agents.browser.executor import execute_browser_tool, _format_tool_result + +BrowserCommandFn = Callable[[str, str, str, Dict[str, Any]], Awaitable[dict]] + +ACTION_MAP: Dict[str, str] = { + "BrowserScreenshot": "screenshot", + "BrowserGetText": "get_text", + "BrowserNavigate": "navigate", + "BrowserClick": "click", + "BrowserType": "type", + "BrowserEvaluate": "evaluate", + "BrowserGetElements": "get_elements", + "BrowserScroll": "scroll", + "BrowserWait": "wait", +} + + +def _format_tool_result(result: dict, tool_name: str) -> list[dict]: + if "error" in result: + return [{"type": "text", "text": f"Error: {result['error']}"}] + if tool_name == "BrowserScreenshot" and result.get("image"): + return [ + { + "type": "image", + "source": {"type": "base64", "media_type": "image/png", "data": result["image"]}, + }, + {"type": "text", "text": f"Screenshot captured. URL: {result.get('url', 'unknown')}"}, + ] + text = result.get("text", json.dumps(result)) + return [{"type": "text", "text": str(text)}] + @typechecked -def make_browser_action_handler(tool_name: str, browser_id: str, tab_id: str): +def make_browser_action_handler( + tool_name: str, + browser_id: str, + tab_id: str, + send_command: BrowserCommandFn, +): + action = ACTION_MAP.get(tool_name) + async def handler(args: Dict[str, Any]) -> ToolResponse: - result = await execute_browser_tool(tool_name, args, browser_id, tab_id) + if not action: + return {"content": [{"type": "text", "text": f"Unknown browser tool: {tool_name}"}], "is_error": True} + + result = await send_command(action, browser_id, tab_id, args) + if "error" in result: - return { - "content": [ - { - "type": "text", - "text": f"Error: {result['error']}", - }, - ], - "is_error": True, - } - content_blocks = _format_tool_result(result, tool_name) - return {"content": content_blocks} + return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "is_error": True} + return {"content": _format_tool_result(result, tool_name)} + return handler diff --git a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_actions_toolkit/make_browser_actions_toolkit.py b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_actions_toolkit/make_browser_actions_toolkit.py index 44a02e30..338f5632 100644 --- a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_actions_toolkit/make_browser_actions_toolkit.py +++ b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_actions_toolkit/make_browser_actions_toolkit.py @@ -12,12 +12,14 @@ from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit BrowserScrollInput, BrowserWaitInput, ) -from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import make_browser_action_handler +from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import ( + make_browser_action_handler, BrowserCommandFn, +) SERVER_NAME = "openswarm-browser-actions" @typechecked -def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit: +def make_browser_actions_toolkit(browser_id: str, send_command: BrowserCommandFn, tab_id: str = "") -> Toolkit: return Toolkit( name="browser_actions", description="Low-level browser automation actions for a single browser tab", @@ -32,7 +34,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit: permission="allow", server_name=SERVER_NAME, input_schema=BrowserScreenshotInput, - handler=make_browser_action_handler("BrowserScreenshot", browser_id, tab_id), + handler=make_browser_action_handler("BrowserScreenshot", browser_id, tab_id, send_command), ), SDK_MCP_Tool( name="BrowserGetText", @@ -41,7 +43,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit: permission="allow", server_name=SERVER_NAME, input_schema=BrowserGetTextInput, - handler=make_browser_action_handler("BrowserGetText", browser_id, tab_id), + handler=make_browser_action_handler("BrowserGetText", browser_id, tab_id, send_command), ), SDK_MCP_Tool( name="BrowserNavigate", @@ -50,7 +52,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit: permission="allow", server_name=SERVER_NAME, input_schema=BrowserNavigateInput, - handler=make_browser_action_handler("BrowserNavigate", browser_id, tab_id), + handler=make_browser_action_handler("BrowserNavigate", browser_id, tab_id, send_command), ), SDK_MCP_Tool( name="BrowserClick", @@ -62,7 +64,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit: permission="allow", server_name=SERVER_NAME, input_schema=BrowserClickInput, - handler=make_browser_action_handler("BrowserClick", browser_id, tab_id), + handler=make_browser_action_handler("BrowserClick", browser_id, tab_id, send_command), ), SDK_MCP_Tool( name="BrowserType", @@ -71,7 +73,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit: permission="allow", server_name=SERVER_NAME, input_schema=BrowserTypeInput, - handler=make_browser_action_handler("BrowserType", browser_id, tab_id), + handler=make_browser_action_handler("BrowserType", browser_id, tab_id, send_command), ), SDK_MCP_Tool( name="BrowserEvaluate", @@ -80,7 +82,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit: permission="allow", server_name=SERVER_NAME, input_schema=BrowserEvaluateInput, - handler=make_browser_action_handler("BrowserEvaluate", browser_id, tab_id), + handler=make_browser_action_handler("BrowserEvaluate", browser_id, tab_id, send_command), ), SDK_MCP_Tool( name="BrowserGetElements", @@ -92,7 +94,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit: permission="allow", server_name=SERVER_NAME, input_schema=BrowserGetElementsInput, - handler=make_browser_action_handler("BrowserGetElements", browser_id, tab_id), + handler=make_browser_action_handler("BrowserGetElements", browser_id, tab_id, send_command), ), SDK_MCP_Tool( name="BrowserScroll", @@ -104,7 +106,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit: permission="allow", server_name=SERVER_NAME, input_schema=BrowserScrollInput, - handler=make_browser_action_handler("BrowserScroll", browser_id, tab_id), + handler=make_browser_action_handler("BrowserScroll", browser_id, tab_id, send_command), ), SDK_MCP_Tool( name="BrowserWait", @@ -116,7 +118,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit: permission="allow", server_name=SERVER_NAME, input_schema=BrowserWaitInput, - handler=make_browser_action_handler("BrowserWait", browser_id, tab_id), + handler=make_browser_action_handler("BrowserWait", browser_id, tab_id, send_command), ), ], ) \ No newline at end of file diff --git a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_create_browser_agent_handler.py b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_create_browser_agent_handler.py index 16206b36..114fbcd2 100644 --- a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_create_browser_agent_handler.py +++ b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_create_browser_agent_handler.py @@ -6,6 +6,7 @@ from backend.core.shared_structs.agent.Message.agent_outputs import ToolResponse from backend.core.Agent.Agent import Agent from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_agent import run_browser_agent from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.create_browser_card import create_browser_card +from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import BrowserCommandFn class CreateBrowserAgentInput(TypedDict): @@ -13,7 +14,7 @@ class CreateBrowserAgentInput(TypedDict): @typechecked -def make_create_browser_agent_handler(parent: Agent, dashboard_id: str): +def make_create_browser_agent_handler(parent: Agent, send_command: BrowserCommandFn, dashboard_id: str = ""): async def handler(args: CreateBrowserAgentInput) -> ToolResponse: task: str = args["task"] @@ -27,7 +28,7 @@ def make_create_browser_agent_handler(parent: Agent, dashboard_id: str): await asyncio.sleep(2.0) - response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task) + response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task, send_command=send_command) return { "content": [ diff --git a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_invoke_browser_agent_handler.py b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_invoke_browser_agent_handler.py index 751efffa..20474028 100644 --- a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_invoke_browser_agent_handler.py +++ b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_invoke_browser_agent_handler.py @@ -4,6 +4,7 @@ from typeguard import typechecked from backend.core.shared_structs.agent.Message.agent_outputs import ToolResponse from backend.core.Agent.Agent import Agent from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_agent import run_browser_agent +from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import BrowserCommandFn class InvokeBrowserAgentInput(TypedDict): @@ -12,13 +13,13 @@ class InvokeBrowserAgentInput(TypedDict): @typechecked -def make_invoke_browser_agent_handler(parent: Agent): +def make_invoke_browser_agent_handler(parent: Agent, send_command: BrowserCommandFn): async def handler(args: InvokeBrowserAgentInput) -> ToolResponse: browser_id: str = args["browser_id"] task: str = args["task"] - response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task) + response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task, send_command=send_command) return { "content": [ diff --git a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_agent.py b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_agent.py index f9bed9b3..36335f63 100644 --- a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_agent.py +++ b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_agent.py @@ -10,6 +10,7 @@ from backend.core.shared_structs.agent.Message.Message import ( ) from backend.core.tools.shared_structs.MCP_Tool import SDK_MCP_Tool from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.make_browser_actions_toolkit import make_browser_actions_toolkit +from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import BrowserCommandFn from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.constants import BROWSER_AGENT_SYSTEM_PROMPT @@ -18,11 +19,12 @@ async def run_browser_agent( parent: Agent, browser_id: str, task: str, + send_command: BrowserCommandFn, tab_id: str = "", ) -> str: """Spin up a child Agent with browser action tools, run a task, return the response text.""" - actions_toolkit = make_browser_actions_toolkit(browser_id=browser_id, tab_id=tab_id) + actions_toolkit = make_browser_actions_toolkit(browser_id=browser_id, send_command=send_command, tab_id=tab_id) mcp_servers: Dict[str, McpServerConfig] = {} tool_names: List[str] = [] diff --git a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/make_browser_delegation_toolkit.py b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/make_browser_delegation_toolkit.py index 306ac8dd..caf28aea 100644 --- a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/make_browser_delegation_toolkit.py +++ b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/make_browser_delegation_toolkit.py @@ -8,10 +8,11 @@ from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit make_invoke_browser_agent_handler, InvokeBrowserAgentInput, ) from backend.core.Agent.Agent import Agent +from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import BrowserCommandFn @typechecked -def make_browser_delegation_toolkit(parent: Agent) -> Toolkit: +def make_browser_delegation_toolkit(parent: Agent, send_command: BrowserCommandFn) -> Toolkit: return Toolkit( name="browser", description="Tools for browser automation", @@ -26,7 +27,7 @@ def make_browser_delegation_toolkit(parent: Agent) -> Toolkit: permission="allow", server_name="openswarm-browser", input_schema=CreateBrowserAgentInput, - handler=make_create_browser_agent_handler(parent), + handler=make_create_browser_agent_handler(parent, send_command), ), SDK_MCP_Tool( name="InvokeBrowserAgent", @@ -38,7 +39,7 @@ def make_browser_delegation_toolkit(parent: Agent) -> Toolkit: permission="allow", server_name="openswarm-browser", input_schema=InvokeBrowserAgentInput, - handler=make_invoke_browser_agent_handler(parent), + handler=make_invoke_browser_agent_handler(parent, send_command), ), ], ) \ No newline at end of file diff --git a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_toolkit.py b/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_toolkit.py deleted file mode 100644 index d6681a34..00000000 --- a/backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_toolkit.py +++ /dev/null @@ -1,18 +0,0 @@ -from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.make_browser_actions_toolkit import make_browser_actions_toolkit -from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.make_browser_delegation_toolkit import make_browser_delegation_toolkit -from backend.core.tools.shared_structs.Toolkit import Toolkit -from backend.core.Agent.Agent import Agent - -# # NOTE: The minor issue here is that with the current setup, any agent will prly have access to the browser actions toolkit, idk if this bad or good but for now its fine ig - -def make_browser_toolkit( - parent: Agent, -) -> Toolkit: - return Toolkit( - name="browser", - description="Tools for browser automation", - nested_toolkits=[ - make_browser_delegation_toolkit(parent), - make_browser_actions_toolkit(parent), - ], - ) \ No newline at end of file