[Haik]: initial implementation of browser bridge, this push used some ai so ima clean up a couple things

backend/apps/agents/agents.py

@@ -46,6 +46,26 @@ def p_make_session_emitter(session_id: str) -> EventCallback:
         await ws.send_to_session(session_id, event.event, event.model_dump(mode="json"))
     return emit
 
+
+async def p_send_browser_command(
+    action: str, browser_id: str, tab_id: str, params: dict,
+) -> dict:
+    """BrowserCommandFn implementation that routes through the browser FutureBridge."""
+    request_id = uuid4().hex
+    if not ws.has_global_connections():
+        return {"error": "No dashboard connected. Open the dashboard to use browser tools."}
+    return await ws.BROWSER_BRIDGE.request(
+        request_id=request_id,
+        send_fn=lambda: ws.broadcast_global("browser:command", {
+            "request_id": request_id,
+            "action": action,
+            "browser_id": browser_id,
+            "tab_id": tab_id,
+            "params": params,
+        }),
+        timeout=30.0,
+    )
+
 def get_agent(session_id: str) -> Agent:
     agent: Optional[Agent] = SESSIONS.get(session_id)
     if not agent:

backend/core/tools/make_builtin_toolkit/make_builtin_toolkit.py

@@ -1,10 +1,10 @@
-# BUILTIN_TOOLKIT.py
-
 from typing import Dict
 from typeguard import typechecked
 from backend.core.tools.shared_structs.Toolkit import Toolkit
 from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.agents_toolkit.make_agents_toolkit import make_agents_toolkit
-from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_toolkit import make_browser_toolkit
+from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.make_browser_delegation_toolkit import make_browser_delegation_toolkit
+from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.make_browser_actions_toolkit import make_browser_actions_toolkit
+from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import BrowserCommandFn
 
 from backend.core.tools.make_builtin_toolkit.pre_existing_toolkits.basic_toolkits.FILESYSTEM_TOOLKIT import FILESYSTEM_TOOLKIT
 from backend.core.tools.make_builtin_toolkit.pre_existing_toolkits.meta_toolkits.INTERACTION_TOOLKIT import INTERACTION_TOOLKIT
@@ -18,13 +18,15 @@ from backend.core.Agent.Agent import Agent
 def make_builtin_toolkit(
     parent: Agent,
     agent_registry: Dict[str, Agent],
+    send_browser_command: BrowserCommandFn,
 ) -> Toolkit:
     return Toolkit(
         name="builtin",
         description="Builtin tools",
         nested_toolkits=[
             make_agents_toolkit(parent, agent_registry),
-            make_browser_toolkit(parent),
+            make_browser_delegation_toolkit(parent, send_browser_command),
+            make_browser_actions_toolkit(parent, send_browser_command),
             FILESYSTEM_TOOLKIT,
             INTERACTION_TOOLKIT,
             PLANNING_TOOLKIT,

backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_actions_toolkit/handlers/make_browser_action_handler.py

@@ -1,22 +1,56 @@
-from typing import Dict, Any
+import json
+from typing import Dict, Any, Callable, Awaitable
 from typeguard import typechecked
+
 from backend.core.shared_structs.agent.Message.agent_outputs import ToolResponse
-from backend.OLDapps.agents.browser.executor import execute_browser_tool, _format_tool_result
+
+BrowserCommandFn = Callable[[str, str, str, Dict[str, Any]], Awaitable[dict]]
+
+ACTION_MAP: Dict[str, str] = {
+    "BrowserScreenshot": "screenshot",
+    "BrowserGetText": "get_text",
+    "BrowserNavigate": "navigate",
+    "BrowserClick": "click",
+    "BrowserType": "type",
+    "BrowserEvaluate": "evaluate",
+    "BrowserGetElements": "get_elements",
+    "BrowserScroll": "scroll",
+    "BrowserWait": "wait",
+}
+
+
+def _format_tool_result(result: dict, tool_name: str) -> list[dict]:
+    if "error" in result:
+        return [{"type": "text", "text": f"Error: {result['error']}"}]
+    if tool_name == "BrowserScreenshot" and result.get("image"):
+        return [
+            {
+                "type": "image",
+                "source": {"type": "base64", "media_type": "image/png", "data": result["image"]},
+            },
+            {"type": "text", "text": f"Screenshot captured. URL: {result.get('url', 'unknown')}"},
+        ]
+    text = result.get("text", json.dumps(result))
+    return [{"type": "text", "text": str(text)}]
+
 
 @typechecked
-def make_browser_action_handler(tool_name: str, browser_id: str, tab_id: str):
+def make_browser_action_handler(
+    tool_name: str,
+    browser_id: str,
+    tab_id: str,
+    send_command: BrowserCommandFn,
+):
+    action = ACTION_MAP.get(tool_name)
+
     async def handler(args: Dict[str, Any]) -> ToolResponse:
-        result = await execute_browser_tool(tool_name, args, browser_id, tab_id)
+        if not action:
+            return {"content": [{"type": "text", "text": f"Unknown browser tool: {tool_name}"}], "is_error": True}
+
+        result = await send_command(action, browser_id, tab_id, args)
+
         if "error" in result:
-            return {
-                "content": [
-                    {
-                        "type": "text",
-                        "text": f"Error: {result['error']}",
-                    },
-                ],
-                "is_error": True,
-            }
-        content_blocks = _format_tool_result(result, tool_name)
-        return {"content": content_blocks}
+            return {"content": [{"type": "text", "text": f"Error: {result['error']}"}], "is_error": True}
+        return {"content": _format_tool_result(result, tool_name)}
+
     return handler

backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_actions_toolkit/make_browser_actions_toolkit.py

@@ -12,12 +12,14 @@ from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit
     BrowserScrollInput,
     BrowserWaitInput,
 )
-from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import make_browser_action_handler
+from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import (
+    make_browser_action_handler, BrowserCommandFn,
+)
 
 SERVER_NAME = "openswarm-browser-actions"
 
 @typechecked
-def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit:
+def make_browser_actions_toolkit(browser_id: str, send_command: BrowserCommandFn, tab_id: str = "") -> Toolkit:
     return Toolkit(
         name="browser_actions",
         description="Low-level browser automation actions for a single browser tab",
@@ -32,7 +34,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit:
                 permission="allow",
                 server_name=SERVER_NAME,
                 input_schema=BrowserScreenshotInput,
-                handler=make_browser_action_handler("BrowserScreenshot", browser_id, tab_id),
+                handler=make_browser_action_handler("BrowserScreenshot", browser_id, tab_id, send_command),
             ),
             SDK_MCP_Tool(
                 name="BrowserGetText",
@@ -41,7 +43,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit:
                 permission="allow",
                 server_name=SERVER_NAME,
                 input_schema=BrowserGetTextInput,
-                handler=make_browser_action_handler("BrowserGetText", browser_id, tab_id),
+                handler=make_browser_action_handler("BrowserGetText", browser_id, tab_id, send_command),
             ),
             SDK_MCP_Tool(
                 name="BrowserNavigate",
@@ -50,7 +52,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit:
                 permission="allow",
                 server_name=SERVER_NAME,
                 input_schema=BrowserNavigateInput,
-                handler=make_browser_action_handler("BrowserNavigate", browser_id, tab_id),
+                handler=make_browser_action_handler("BrowserNavigate", browser_id, tab_id, send_command),
             ),
             SDK_MCP_Tool(
                 name="BrowserClick",
@@ -62,7 +64,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit:
                 permission="allow",
                 server_name=SERVER_NAME,
                 input_schema=BrowserClickInput,
-                handler=make_browser_action_handler("BrowserClick", browser_id, tab_id),
+                handler=make_browser_action_handler("BrowserClick", browser_id, tab_id, send_command),
             ),
             SDK_MCP_Tool(
                 name="BrowserType",
@@ -71,7 +73,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit:
                 permission="allow",
                 server_name=SERVER_NAME,
                 input_schema=BrowserTypeInput,
-                handler=make_browser_action_handler("BrowserType", browser_id, tab_id),
+                handler=make_browser_action_handler("BrowserType", browser_id, tab_id, send_command),
             ),
             SDK_MCP_Tool(
                 name="BrowserEvaluate",
@@ -80,7 +82,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit:
                 permission="allow",
                 server_name=SERVER_NAME,
                 input_schema=BrowserEvaluateInput,
-                handler=make_browser_action_handler("BrowserEvaluate", browser_id, tab_id),
+                handler=make_browser_action_handler("BrowserEvaluate", browser_id, tab_id, send_command),
             ),
             SDK_MCP_Tool(
                 name="BrowserGetElements",
@@ -92,7 +94,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit:
                 permission="allow",
                 server_name=SERVER_NAME,
                 input_schema=BrowserGetElementsInput,
-                handler=make_browser_action_handler("BrowserGetElements", browser_id, tab_id),
+                handler=make_browser_action_handler("BrowserGetElements", browser_id, tab_id, send_command),
             ),
             SDK_MCP_Tool(
                 name="BrowserScroll",
@@ -104,7 +106,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit:
                 permission="allow",
                 server_name=SERVER_NAME,
                 input_schema=BrowserScrollInput,
-                handler=make_browser_action_handler("BrowserScroll", browser_id, tab_id),
+                handler=make_browser_action_handler("BrowserScroll", browser_id, tab_id, send_command),
             ),
             SDK_MCP_Tool(
                 name="BrowserWait",
@@ -116,7 +118,7 @@ def make_browser_actions_toolkit(browser_id: str, tab_id: str = "") -> Toolkit:
                 permission="allow",
                 server_name=SERVER_NAME,
                 input_schema=BrowserWaitInput,
-                handler=make_browser_action_handler("BrowserWait", browser_id, tab_id),
+                handler=make_browser_action_handler("BrowserWait", browser_id, tab_id, send_command),
             ),
         ],
     )

backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_create_browser_agent_handler.py

@@ -6,6 +6,7 @@ from backend.core.shared_structs.agent.Message.agent_outputs import ToolResponse
 from backend.core.Agent.Agent import Agent
 from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_agent import run_browser_agent
 from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.create_browser_card import create_browser_card
+from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import BrowserCommandFn
 
 
 class CreateBrowserAgentInput(TypedDict):
@@ -13,7 +14,7 @@ class CreateBrowserAgentInput(TypedDict):
 
 
 @typechecked
-def make_create_browser_agent_handler(parent: Agent, dashboard_id: str):
+def make_create_browser_agent_handler(parent: Agent, send_command: BrowserCommandFn, dashboard_id: str = ""):
 
     async def handler(args: CreateBrowserAgentInput) -> ToolResponse:
         task: str = args["task"]
@@ -27,7 +28,7 @@ def make_create_browser_agent_handler(parent: Agent, dashboard_id: str):
 
         await asyncio.sleep(2.0)
 
-        response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task)
+        response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task, send_command=send_command)
 
         return {
             "content": [

backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_invoke_browser_agent_handler.py

@@ -4,6 +4,7 @@ from typeguard import typechecked
 from backend.core.shared_structs.agent.Message.agent_outputs import ToolResponse
 from backend.core.Agent.Agent import Agent
 from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_agent import run_browser_agent
+from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import BrowserCommandFn
 
 
 class InvokeBrowserAgentInput(TypedDict):
@@ -12,13 +13,13 @@ class InvokeBrowserAgentInput(TypedDict):
 
 
 @typechecked
-def make_invoke_browser_agent_handler(parent: Agent):
+def make_invoke_browser_agent_handler(parent: Agent, send_command: BrowserCommandFn):
 
     async def handler(args: InvokeBrowserAgentInput) -> ToolResponse:
         browser_id: str = args["browser_id"]
         task: str = args["task"]
 
-        response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task)
+        response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task, send_command=send_command)
 
         return {
             "content": [

backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_agent.py

@@ -10,6 +10,7 @@ from backend.core.shared_structs.agent.Message.Message import (
 )
 from backend.core.tools.shared_structs.MCP_Tool import SDK_MCP_Tool
 from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.make_browser_actions_toolkit import make_browser_actions_toolkit
+from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import BrowserCommandFn
 from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.constants import BROWSER_AGENT_SYSTEM_PROMPT
 
 
@@ -18,11 +19,12 @@ async def run_browser_agent(
     parent: Agent,
     browser_id: str,
     task: str,
+    send_command: BrowserCommandFn,
     tab_id: str = "",
 ) -> str:
     """Spin up a child Agent with browser action tools, run a task, return the response text."""
 
-    actions_toolkit = make_browser_actions_toolkit(browser_id=browser_id, tab_id=tab_id)
+    actions_toolkit = make_browser_actions_toolkit(browser_id=browser_id, send_command=send_command, tab_id=tab_id)
 
     mcp_servers: Dict[str, McpServerConfig] = {}
     tool_names: List[str] = []

backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/make_browser_delegation_toolkit.py

@@ -8,10 +8,11 @@ from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit
     make_invoke_browser_agent_handler, InvokeBrowserAgentInput,
 )
 from backend.core.Agent.Agent import Agent
+from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.handlers.make_browser_action_handler import BrowserCommandFn
 
 
 @typechecked
-def make_browser_delegation_toolkit(parent: Agent) -> Toolkit:
+def make_browser_delegation_toolkit(parent: Agent, send_command: BrowserCommandFn) -> Toolkit:
     return Toolkit(
         name="browser",
         description="Tools for browser automation",
@@ -26,7 +27,7 @@ def make_browser_delegation_toolkit(parent: Agent) -> Toolkit:
                 permission="allow",
                 server_name="openswarm-browser",
                 input_schema=CreateBrowserAgentInput,
-                handler=make_create_browser_agent_handler(parent),
+                handler=make_create_browser_agent_handler(parent, send_command),
             ),
             SDK_MCP_Tool(
                 name="InvokeBrowserAgent",
@@ -38,7 +39,7 @@ def make_browser_delegation_toolkit(parent: Agent) -> Toolkit:
                 permission="allow",
                 server_name="openswarm-browser",
                 input_schema=InvokeBrowserAgentInput,
-                handler=make_invoke_browser_agent_handler(parent),
+                handler=make_invoke_browser_agent_handler(parent, send_command),
             ),
         ],
     )

backend/core/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_toolkit.py

@@ -1,18 +0,0 @@
-from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.make_browser_actions_toolkit import make_browser_actions_toolkit
-from backend.core.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.make_browser_delegation_toolkit import make_browser_delegation_toolkit
-from backend.core.tools.shared_structs.Toolkit import Toolkit
-from backend.core.Agent.Agent import Agent
-
-# # NOTE: The minor issue here is that with the current setup, any agent will prly have access to the browser actions toolkit, idk if this bad or good but for now its fine ig
-
-def make_browser_toolkit(
-    parent: Agent,
-) -> Toolkit:
-    return Toolkit(
-        name="browser",
-        description="Tools for browser automation",
-        nested_toolkits=[
-            make_browser_delegation_toolkit(parent),
-            make_browser_actions_toolkit(parent),
-        ],
-    )