From 576f9d4d705b2c60231ab896d4f793b0da4758c4 Mon Sep 17 00:00:00 2001 From: haikdc Date: Thu, 2 Apr 2026 15:44:33 -0700 Subject: [PATCH] [Haik]: swapped run_browser_loop to run_browser_agent and it now uses the browser_actions_toolkit and the Agent class. Next push is gonna slightly rename some stuff, then idk what ill move on to yet --- backend/apps/agents/HaikFix/Agent/Agent.py | 3 + .../make_create_browser_agent_handler.py | 45 +++--- .../make_invoke_browser_agent_handler.py | 23 +-- .../handlers/utils/constants.py | 22 +++ .../create_browser_card.py | 0 .../handlers/utils/format_browser_result.py | 31 ---- .../handlers/utils/run_browser_agent.py | 60 ++++++++ .../handlers/utils/run_browser_loop.py | 92 ------------ .../utils/temp_sub_utils/model_ids.py | 30 ---- .../handlers/utils/temp_sub_utils/schemas.py | 137 ------------------ .../browser_toolkit/make_browser_toolkit.py | 3 +- .../HaikFix/tools/shared_structs/Tool.py | 15 +- 12 files changed, 120 insertions(+), 341 deletions(-) create mode 100644 backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/constants.py rename backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/{temp_sub_utils => }/create_browser_card.py (100%) delete mode 100644 backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/format_browser_result.py create mode 100644 backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_agent.py delete mode 100644 backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_loop.py delete mode 100644 backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/temp_sub_utils/model_ids.py delete mode 100644 backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/temp_sub_utils/schemas.py diff --git a/backend/apps/agents/HaikFix/Agent/Agent.py b/backend/apps/agents/HaikFix/Agent/Agent.py index 4923d49c..dc79743a 100644 --- a/backend/apps/agents/HaikFix/Agent/Agent.py +++ b/backend/apps/agents/HaikFix/Agent/Agent.py @@ -18,6 +18,9 @@ from backend.apps.agents.HaikFix.Agent.shared_structs.MessageLog import MessageL os.environ.setdefault("CLAUDE_CODE_STREAM_CLOSE_TIMEOUT", "3600000") + +# NOTE and TODO: we shld remove the ws streaming from this class bc it conflicts with the browser agent + class Agent(BaseModel): model: str mode: str diff --git a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_create_browser_agent_handler.py b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_create_browser_agent_handler.py index 92ab24b0..892c6253 100644 --- a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_create_browser_agent_handler.py +++ b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_create_browser_agent_handler.py @@ -3,46 +3,39 @@ from typing import TypedDict from typeguard import typechecked from backend.apps.agents.HaikFix.Agent.shared_structs.Message.agent_outputs import ToolResponse -from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_loop import run_browser_loop -from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.format_browser_result import format_browser_result from backend.apps.agents.HaikFix.Agent.Agent import Agent +from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_agent import run_browser_agent +from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.create_browser_card import create_browser_card -# NOTE: Legacy dependancy. TODO: fix this shit cuh -from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.temp_sub_utils.create_browser_card import create_browser_card class CreateBrowserAgentInput(TypedDict): task: str -@typechecked -def make_create_browser_agent_handler( - parent: Agent, - dashboard_id: str, -): - async def handler(args: CreateBrowserAgentInput) -> ToolResponse: - task = args["task"] - browser_id = await create_browser_card( - dashboard_id=dashboard_id, - ) +@typechecked +def make_create_browser_agent_handler(parent: Agent, dashboard_id: str): + + async def handler(args: CreateBrowserAgentInput) -> ToolResponse: + task: str = args["task"] + + browser_id: str = await create_browser_card(dashboard_id=dashboard_id) if not browser_id: return { - "content": [ - { - "type": "text", - "text": "Error: failed to create browser agent", - }, - ], + "content": [{"type": "text", "text": "Error: failed to create browser agent"}], "is_error": True, } await asyncio.sleep(2.0) - result = await run_browser_loop( - task=task, - browser_id=browser_id, - model=parent.model, - ) + response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task) - return format_browser_result(result, browser_id) + return { + "content": [ + { + "type": "text", + "text": f"**Browser Agent Result** (browser: {browser_id})\n\n{response}", + } + ], + } return handler \ No newline at end of file diff --git a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_invoke_browser_agent_handler.py b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_invoke_browser_agent_handler.py index 83bc1eeb..8c5728c3 100644 --- a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_invoke_browser_agent_handler.py +++ b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/make_invoke_browser_agent_handler.py @@ -2,9 +2,8 @@ from typing import TypedDict from typeguard import typechecked from backend.apps.agents.HaikFix.Agent.shared_structs.Message.agent_outputs import ToolResponse -from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_loop import run_browser_loop -from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.format_browser_result import format_browser_result from backend.apps.agents.HaikFix.Agent.Agent import Agent +from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_agent import run_browser_agent class InvokeBrowserAgentInput(TypedDict): @@ -14,16 +13,20 @@ class InvokeBrowserAgentInput(TypedDict): @typechecked def make_invoke_browser_agent_handler(parent: Agent): + async def handler(args: InvokeBrowserAgentInput) -> ToolResponse: - browser_id = args["browser_id"] - task = args["task"] + browser_id: str = args["browser_id"] + task: str = args["task"] - result = await run_browser_loop( - task=task, - browser_id=browser_id, - model=parent.model, - ) + response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task) - return format_browser_result(result, browser_id) + return { + "content": [ + { + "type": "text", + "text": f"**Browser Agent Result** (browser: {browser_id})\n\n{response}", + } + ], + } return handler \ No newline at end of file diff --git a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/constants.py b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/constants.py new file mode 100644 index 00000000..3d5009af --- /dev/null +++ b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/constants.py @@ -0,0 +1,22 @@ +BROWSER_AGENT_SYSTEM_PROMPT = ( + "You are a browser automation agent. You control a single browser tab and " + "execute the task you are given.\n\n" + "Strategy:\n" + "1. Start by taking a screenshot to understand the page.\n" + "2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n" + "3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with " + "window.scrollBy() as many sites use nested scroll containers that BrowserScroll " + "handles automatically.\n" + "4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n" + "5. After performing actions, take a screenshot to verify the result.\n" + "6. If an action fails, try alternative selectors or approaches.\n" + "7. When the task is complete, provide a clear summary of what you accomplished.\n\n" + "Important notes:\n" + "- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n" + "- BrowserScroll returns position info including atTop/atBottom — use this to know when " + "you've reached the end of the page.\n" + "- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n" + "- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n" + "You have access ONLY to browser tools. Do not ask the user questions — " + "complete the task autonomously to the best of your ability." +) \ No newline at end of file diff --git a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/temp_sub_utils/create_browser_card.py b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/create_browser_card.py similarity index 100% rename from backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/temp_sub_utils/create_browser_card.py rename to backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/create_browser_card.py diff --git a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/format_browser_result.py b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/format_browser_result.py deleted file mode 100644 index 8a13a939..00000000 --- a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/format_browser_result.py +++ /dev/null @@ -1,31 +0,0 @@ -import json -from typing import Dict, List, Any - -from backend.apps.agents.HaikFix.Agent.shared_structs.Message.agent_outputs import ToolResponse - -MAX_IMAGE_B64_BYTES = 400_000 - -def format_browser_result(result: Dict[str, Any], browser_id: str) -> ToolResponse: - """Format the browser loop result into a ToolResponse.""" - lines = [ - f"**Browser Agent Result** (browser: {browser_id})", - "", - f"**Summary:** {result['summary']}", - ] - - if result["action_log"]: - lines.append("") - lines.append("**Actions taken:**") - for i, entry in enumerate[Any](result["action_log"], 1): - tool = entry["tool"] - inp = json.dumps(entry.get("input", {}))[:120] - ms = entry.get("elapsed_ms", 0) - lines.append(f" {i}. {tool}({inp}) [{ms}ms]") - - content: List[Dict[str, str]] = [{"type": "text", "text": "\n".join(lines)}] - - screenshot = result.get("final_screenshot") - if screenshot and len(screenshot) <= MAX_IMAGE_B64_BYTES: - content.append({"type": "image", "data": screenshot, "mimeType": "image/png"}) - - return {"content": content} \ No newline at end of file diff --git a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_agent.py b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_agent.py new file mode 100644 index 00000000..b7ac84bc --- /dev/null +++ b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_agent.py @@ -0,0 +1,60 @@ +from typing import List, Dict +from typeguard import typechecked + +from claude_agent_sdk import ClaudeAgentOptions +from claude_agent_sdk.types import McpServerConfig + +from backend.apps.agents.HaikFix.Agent.Agent import Agent +from backend.apps.agents.HaikFix.Agent.shared_structs.Message.Message import ( + UserMessage, AssistantMessage, AnyMessage, +) +from backend.apps.agents.HaikFix.tools.shared_structs.MCP_Tool import SDK_MCP_Tool +from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.make_browser_actions_toolkit import make_browser_actions_toolkit +from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.constants import BROWSER_AGENT_SYSTEM_PROMPT + + +@typechecked +async def run_browser_agent( + parent: Agent, + browser_id: str, + task: str, + tab_id: str = "", +) -> str: + """Spin up a child Agent with browser action tools, run a task, return the response text.""" + + actions_toolkit = make_browser_actions_toolkit(browser_id=browser_id, tab_id=tab_id) + + mcp_servers: Dict[str, McpServerConfig] = {} + tool_names: List[str] = [] + for tool in actions_toolkit.tools: + assert isinstance(tool, SDK_MCP_Tool) + mcp_servers.update(tool.to_mcp_server_config()) + tool_names.append(tool.to_sdk_args()) + + browser_config = ClaudeAgentOptions( + model=parent.model, + system_prompt=BROWSER_AGENT_SYSTEM_PROMPT, + tools=tool_names, + mcp_servers=mcp_servers, + max_turns=25, + ) + + child = Agent( + model=parent.model, + mode="browser", + status="completed", + config=browser_config, + parent_id=parent.session_id, + ) + parent.sub_agents.append(child) + + msg = UserMessage(content=task, branch_id=child.branch_id) + await child.send_message(msg) + if child.task: + await child.task + + for m in reversed[AnyMessage](child.messages.messages): + if isinstance(m, AssistantMessage): + return m.content + + return "Task completed." \ No newline at end of file diff --git a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_loop.py b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_loop.py deleted file mode 100644 index cd4f9b36..00000000 --- a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/run_browser_loop.py +++ /dev/null @@ -1,92 +0,0 @@ -import time -from typing import Dict, List, Any - -# NOTE: Hella dependancies, this baddddddd -# TODO: fix this shit cuh -from backend.apps.agents.browser.executor import execute_browser_tool, _format_tool_result - -from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.temp_sub_utils.model_ids import is_valid_model_id -from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.temp_sub_utils.model_ids import get_anthropic_client -from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.temp_sub_utils.schemas import BROWSER_TOOLS_SCHEMA, SYSTEM_PROMPT, MAX_TURNS - -async def run_browser_loop( - task: str, - browser_id: str, - model_id: str, -) -> Dict[str, Any]: - """Run the browser agent tool loop against the Anthropic API directly.""" - - if not is_valid_model_id(model_id): - raise ValueError(f"Invalid model ID: {model_id}") - - client = get_anthropic_client() - - messages: List[Dict] = [{"role": "user", "content": task}] - action_log: List[Dict] = [] - final_screenshot: str | None = None - last_text_parts: List[str] = [] - - for _ in range(MAX_TURNS): - response = await client.messages.create( - model=f"cc/{model_id}", - max_tokens=4096, - system=SYSTEM_PROMPT, - tools=BROWSER_TOOLS_SCHEMA, - messages=messages, - ) - - assistant_content = [] - last_text_parts = [] - tool_uses = [] - - for block in response.content: - if block.type == "text": - last_text_parts.append(block.text) - assistant_content.append({"type": "text", "text": block.text}) - elif block.type == "tool_use": - tool_uses.append(block) - assistant_content.append({ - "type": "tool_use", "id": block.id, - "name": block.name, "input": block.input, - }) - - messages.append({"role": "assistant", "content": assistant_content}) - - if response.stop_reason != "tool_use": - break - - tool_results = [] - for tu in tool_uses: - start = time.time() - result = await execute_browser_tool(tu.name, tu.input, browser_id,) - elapsed_ms = int((time.time() - start) * 1000) - action_log.append({ - "tool": tu.name, - "input": tu.input, - "elapsed_ms": elapsed_ms, - }) - if tu.name == "BrowserScreenshot" and result.get("image"): - final_screenshot = result["image"] - content_blocks = _format_tool_result(result, tu.name) - tool_results.append({ - "type": "tool_result", - "tool_use_id": tu.id, - "content": content_blocks, - }) - - messages.append({"role": "user", "content": tool_results}) - - if not final_screenshot: - try: - ss = await execute_browser_tool("BrowserScreenshot", {}, browser_id) - if ss.get("image"): - final_screenshot = ss["image"] - except Exception: - pass - - return { - "summary": "\n".join(last_text_parts) if last_text_parts else "Task completed.", - "action_log": action_log, - "final_screenshot": final_screenshot, - } - diff --git a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/temp_sub_utils/model_ids.py b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/temp_sub_utils/model_ids.py deleted file mode 100644 index c82ed351..00000000 --- a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/temp_sub_utils/model_ids.py +++ /dev/null @@ -1,30 +0,0 @@ -from typeguard import typechecked -from backend.ports import NINE_ROUTER_PORT -import anthropic -import httpx - -SONNET: str = "claude-sonnet-4-6" -OPUS: str = "claude-opus-4-6" -HAIKU: str = "claude-haiku-4-5" - -@typechecked -def is_valid_model_id(model_id: str) -> bool: - return model_id in [SONNET, OPUS, HAIKU] - -@typechecked -def check_9router() -> bool: - """Check if 9Router is running locally.""" - try: - response: httpx.Response = httpx.get(f"http://localhost:{NINE_ROUTER_PORT}/v1/models", timeout=2.0) - return response.status_code == 200 - except Exception: - return False - -@typechecked -def get_anthropic_client() -> anthropic.AsyncAnthropic: - if not check_9router(): - raise ValueError("9Router is not running") - return anthropic.AsyncAnthropic( - api_key="9router", - base_url=f"http://localhost:{NINE_ROUTER_PORT}", - ) \ No newline at end of file diff --git a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/temp_sub_utils/schemas.py b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/temp_sub_utils/schemas.py deleted file mode 100644 index a68b0f22..00000000 --- a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_delegation_toolkit/handlers/utils/temp_sub_utils/schemas.py +++ /dev/null @@ -1,137 +0,0 @@ -"""Browser tool schemas, constants, and system prompt.""" - -BROWSER_TOOLS_SCHEMA = [ - { - "name": "BrowserScreenshot", - "description": ( - "Capture a screenshot of the browser page. Returns the screenshot as a " - "base64-encoded PNG image. Use this to see what is currently displayed." - ), - "input_schema": {"type": "object", "properties": {}, "required": []}, - }, - { - "name": "BrowserGetText", - "description": "Get the visible text content of the browser page. Returns up to 15000 characters.", - "input_schema": {"type": "object", "properties": {}, "required": []}, - }, - { - "name": "BrowserNavigate", - "description": "Navigate the browser to a URL.", - "input_schema": { - "type": "object", - "properties": {"url": {"type": "string", "description": "The URL to navigate to."}}, - "required": ["url"], - }, - }, - { - "name": "BrowserClick", - "description": "Click an element identified by a CSS selector. Use BrowserGetElements first to discover valid selectors.", - "input_schema": { - "type": "object", - "properties": {"selector": {"type": "string", "description": "CSS selector of the element to click."}}, - "required": ["selector"], - }, - }, - { - "name": "BrowserType", - "description": "Type text into an input element. Clears existing value first.", - "input_schema": { - "type": "object", - "properties": { - "selector": {"type": "string", "description": "CSS selector of the input element."}, - "text": {"type": "string", "description": "The text to type."}, - }, - "required": ["selector", "text"], - }, - }, - { - "name": "BrowserEvaluate", - "description": "Evaluate a JavaScript expression in the browser page and return the result.", - "input_schema": { - "type": "object", - "properties": {"expression": {"type": "string", "description": "JavaScript expression to evaluate."}}, - "required": ["expression"], - }, - }, - { - "name": "BrowserGetElements", - "description": ( - "Get a list of interactive elements on the page with CSS selectors. " - "Call this BEFORE clicking or typing so you know which selectors are valid." - ), - "input_schema": { - "type": "object", - "properties": { - "selector": { - "type": "string", - "description": "Optional CSS selector to scope the search (e.g. 'form', '#main'). Defaults to 'body'.", - }, - }, - "required": [], - }, - }, - { - "name": "BrowserScroll", - "description": ( - "Scroll the page up or down. Automatically finds the correct scrollable " - "container. Returns scroll position info including whether top/bottom has been reached." - ), - "input_schema": { - "type": "object", - "properties": { - "direction": {"type": "string", "enum": ["up", "down"], "description": "Scroll direction. Defaults to 'down'."}, - "amount": {"type": "number", "description": "Pixels to scroll. Defaults to 500."}, - }, - "required": [], - }, - }, - { - "name": "BrowserWait", - "description": ( - "Wait for a specified duration. Useful after navigation or actions that " - "trigger page loads. Min 100ms, max 10000ms." - ), - "input_schema": { - "type": "object", - "properties": {"milliseconds": {"type": "number", "description": "Duration to wait in milliseconds. Defaults to 1000."}}, - "required": [], - }, - }, -] - -ACTION_MAP = { - "BrowserScreenshot": "screenshot", - "BrowserGetText": "get_text", - "BrowserNavigate": "navigate", - "BrowserClick": "click", - "BrowserType": "type", - "BrowserEvaluate": "evaluate", - "BrowserGetElements": "get_elements", - "BrowserScroll": "scroll", - "BrowserWait": "wait", -} - -SYSTEM_PROMPT = ( - "You are a browser automation agent. You control a single browser tab and " - "execute the task you are given.\n\n" - "Strategy:\n" - "1. Start by taking a screenshot to understand the page.\n" - "2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n" - "3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with " - "window.scrollBy() as many sites use nested scroll containers that BrowserScroll " - "handles automatically.\n" - "4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n" - "5. After performing actions, take a screenshot to verify the result.\n" - "6. If an action fails, try alternative selectors or approaches.\n" - "7. When the task is complete, provide a clear summary of what you accomplished.\n\n" - "Important notes:\n" - "- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n" - "- BrowserScroll returns position info including atTop/atBottom — use this to know when " - "you've reached the end of the page.\n" - "- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n" - "- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n" - "You have access ONLY to browser tools. Do not ask the user questions — " - "complete the task autonomously to the best of your ability." -) - -MAX_TURNS = 25 diff --git a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_toolkit.py b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_toolkit.py index e416246c..21051d09 100644 --- a/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_toolkit.py +++ b/backend/apps/agents/HaikFix/tools/make_builtin_toolkit/open_swarm_toolkits/browser_toolkit/make_browser_toolkit.py @@ -1,10 +1,9 @@ from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.make_browser_actions_toolkit import make_browser_actions_toolkit from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.make_browser_delegation_toolkit import make_browser_delegation_toolkit from backend.apps.agents.HaikFix.tools.shared_structs.Toolkit import Toolkit -from backend.apps.agents.HaikFix.tools.shared_structs.MCP_Tool import SDK_MCP_Tool -from typing import Dict from backend.apps.agents.HaikFix.Agent.Agent import Agent +# # NOTE: The minor issue here is that with the current setup, any agent will prly have access to the browser actions toolkit, idk if this bad or good but for now its fine ig def make_browser_toolkit( parent: Agent, diff --git a/backend/apps/agents/HaikFix/tools/shared_structs/Tool.py b/backend/apps/agents/HaikFix/tools/shared_structs/Tool.py index c5d3d016..692ac3b8 100644 --- a/backend/apps/agents/HaikFix/tools/shared_structs/Tool.py +++ b/backend/apps/agents/HaikFix/tools/shared_structs/Tool.py @@ -1,17 +1,6 @@ -from typing import Optional, Dict, Any, Literal, Callable, Awaitable, List, Union -from pydantic import BaseModel, Field +from typing import Optional +from pydantic import BaseModel from backend.apps.agents.HaikFix.tools.shared_structs.TOOL_PERMISSIONS import TOOL_PERMISSIONS -from claude_agent_sdk import ( - create_sdk_mcp_server, - tool as sdk_tool -) -from claude_agent_sdk.types import ( - McpStdioServerConfig, - McpSSEServerConfig, - McpHttpServerConfig, - McpSdkServerConfig, - McpServerConfig -) from typeguard import typechecked class Tool(BaseModel):