[Haik]: swapped run_browser_loop to run_browser_agent and it now uses the browser_actions_toolkit and the Agent class. Next push is gonna slightly rename some stuff, then idk what ill move on to yet

This commit is contained in:
haikdc
2026-04-02 15:44:33 -07:00
parent d0a636cf7e
commit 576f9d4d70
12 changed files with 120 additions and 341 deletions
@@ -18,6 +18,9 @@ from backend.apps.agents.HaikFix.Agent.shared_structs.MessageLog import MessageL
os.environ.setdefault("CLAUDE_CODE_STREAM_CLOSE_TIMEOUT", "3600000")
# NOTE and TODO: we shld remove the ws streaming from this class bc it conflicts with the browser agent
class Agent(BaseModel):
model: str
mode: str
@@ -3,46 +3,39 @@ from typing import TypedDict
from typeguard import typechecked
from backend.apps.agents.HaikFix.Agent.shared_structs.Message.agent_outputs import ToolResponse
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_loop import run_browser_loop
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.format_browser_result import format_browser_result
from backend.apps.agents.HaikFix.Agent.Agent import Agent
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_agent import run_browser_agent
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.create_browser_card import create_browser_card
# NOTE: Legacy dependancy. TODO: fix this shit cuh
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.temp_sub_utils.create_browser_card import create_browser_card
class CreateBrowserAgentInput(TypedDict):
task: str
@typechecked
def make_create_browser_agent_handler(
parent: Agent,
dashboard_id: str,
):
async def handler(args: CreateBrowserAgentInput) -> ToolResponse:
task = args["task"]
browser_id = await create_browser_card(
dashboard_id=dashboard_id,
)
@typechecked
def make_create_browser_agent_handler(parent: Agent, dashboard_id: str):
async def handler(args: CreateBrowserAgentInput) -> ToolResponse:
task: str = args["task"]
browser_id: str = await create_browser_card(dashboard_id=dashboard_id)
if not browser_id:
return {
"content": [
{
"type": "text",
"text": "Error: failed to create browser agent",
},
],
"content": [{"type": "text", "text": "Error: failed to create browser agent"}],
"is_error": True,
}
await asyncio.sleep(2.0)
result = await run_browser_loop(
task=task,
browser_id=browser_id,
model=parent.model,
)
response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task)
return format_browser_result(result, browser_id)
return {
"content": [
{
"type": "text",
"text": f"**Browser Agent Result** (browser: {browser_id})\n\n{response}",
}
],
}
return handler
@@ -2,9 +2,8 @@ from typing import TypedDict
from typeguard import typechecked
from backend.apps.agents.HaikFix.Agent.shared_structs.Message.agent_outputs import ToolResponse
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_loop import run_browser_loop
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.format_browser_result import format_browser_result
from backend.apps.agents.HaikFix.Agent.Agent import Agent
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.run_browser_agent import run_browser_agent
class InvokeBrowserAgentInput(TypedDict):
@@ -14,16 +13,20 @@ class InvokeBrowserAgentInput(TypedDict):
@typechecked
def make_invoke_browser_agent_handler(parent: Agent):
async def handler(args: InvokeBrowserAgentInput) -> ToolResponse:
browser_id = args["browser_id"]
task = args["task"]
browser_id: str = args["browser_id"]
task: str = args["task"]
result = await run_browser_loop(
task=task,
browser_id=browser_id,
model=parent.model,
)
response = await run_browser_agent(parent=parent, browser_id=browser_id, task=task)
return format_browser_result(result, browser_id)
return {
"content": [
{
"type": "text",
"text": f"**Browser Agent Result** (browser: {browser_id})\n\n{response}",
}
],
}
return handler
@@ -0,0 +1,22 @@
BROWSER_AGENT_SYSTEM_PROMPT = (
"You are a browser automation agent. You control a single browser tab and "
"execute the task you are given.\n\n"
"Strategy:\n"
"1. Start by taking a screenshot to understand the page.\n"
"2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n"
"3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with "
"window.scrollBy() as many sites use nested scroll containers that BrowserScroll "
"handles automatically.\n"
"4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n"
"5. After performing actions, take a screenshot to verify the result.\n"
"6. If an action fails, try alternative selectors or approaches.\n"
"7. When the task is complete, provide a clear summary of what you accomplished.\n\n"
"Important notes:\n"
"- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n"
"- BrowserScroll returns position info including atTop/atBottom — use this to know when "
"you've reached the end of the page.\n"
"- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n"
"- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n"
"You have access ONLY to browser tools. Do not ask the user questions — "
"complete the task autonomously to the best of your ability."
)
@@ -1,31 +0,0 @@
import json
from typing import Dict, List, Any
from backend.apps.agents.HaikFix.Agent.shared_structs.Message.agent_outputs import ToolResponse
MAX_IMAGE_B64_BYTES = 400_000
def format_browser_result(result: Dict[str, Any], browser_id: str) -> ToolResponse:
"""Format the browser loop result into a ToolResponse."""
lines = [
f"**Browser Agent Result** (browser: {browser_id})",
"",
f"**Summary:** {result['summary']}",
]
if result["action_log"]:
lines.append("")
lines.append("**Actions taken:**")
for i, entry in enumerate[Any](result["action_log"], 1):
tool = entry["tool"]
inp = json.dumps(entry.get("input", {}))[:120]
ms = entry.get("elapsed_ms", 0)
lines.append(f" {i}. {tool}({inp}) [{ms}ms]")
content: List[Dict[str, str]] = [{"type": "text", "text": "\n".join(lines)}]
screenshot = result.get("final_screenshot")
if screenshot and len(screenshot) <= MAX_IMAGE_B64_BYTES:
content.append({"type": "image", "data": screenshot, "mimeType": "image/png"})
return {"content": content}
@@ -0,0 +1,60 @@
from typing import List, Dict
from typeguard import typechecked
from claude_agent_sdk import ClaudeAgentOptions
from claude_agent_sdk.types import McpServerConfig
from backend.apps.agents.HaikFix.Agent.Agent import Agent
from backend.apps.agents.HaikFix.Agent.shared_structs.Message.Message import (
UserMessage, AssistantMessage, AnyMessage,
)
from backend.apps.agents.HaikFix.tools.shared_structs.MCP_Tool import SDK_MCP_Tool
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.make_browser_actions_toolkit import make_browser_actions_toolkit
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.constants import BROWSER_AGENT_SYSTEM_PROMPT
@typechecked
async def run_browser_agent(
parent: Agent,
browser_id: str,
task: str,
tab_id: str = "",
) -> str:
"""Spin up a child Agent with browser action tools, run a task, return the response text."""
actions_toolkit = make_browser_actions_toolkit(browser_id=browser_id, tab_id=tab_id)
mcp_servers: Dict[str, McpServerConfig] = {}
tool_names: List[str] = []
for tool in actions_toolkit.tools:
assert isinstance(tool, SDK_MCP_Tool)
mcp_servers.update(tool.to_mcp_server_config())
tool_names.append(tool.to_sdk_args())
browser_config = ClaudeAgentOptions(
model=parent.model,
system_prompt=BROWSER_AGENT_SYSTEM_PROMPT,
tools=tool_names,
mcp_servers=mcp_servers,
max_turns=25,
)
child = Agent(
model=parent.model,
mode="browser",
status="completed",
config=browser_config,
parent_id=parent.session_id,
)
parent.sub_agents.append(child)
msg = UserMessage(content=task, branch_id=child.branch_id)
await child.send_message(msg)
if child.task:
await child.task
for m in reversed[AnyMessage](child.messages.messages):
if isinstance(m, AssistantMessage):
return m.content
return "Task completed."
@@ -1,92 +0,0 @@
import time
from typing import Dict, List, Any
# NOTE: Hella dependancies, this baddddddd
# TODO: fix this shit cuh
from backend.apps.agents.browser.executor import execute_browser_tool, _format_tool_result
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.temp_sub_utils.model_ids import is_valid_model_id
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.temp_sub_utils.model_ids import get_anthropic_client
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.handlers.utils.temp_sub_utils.schemas import BROWSER_TOOLS_SCHEMA, SYSTEM_PROMPT, MAX_TURNS
async def run_browser_loop(
task: str,
browser_id: str,
model_id: str,
) -> Dict[str, Any]:
"""Run the browser agent tool loop against the Anthropic API directly."""
if not is_valid_model_id(model_id):
raise ValueError(f"Invalid model ID: {model_id}")
client = get_anthropic_client()
messages: List[Dict] = [{"role": "user", "content": task}]
action_log: List[Dict] = []
final_screenshot: str | None = None
last_text_parts: List[str] = []
for _ in range(MAX_TURNS):
response = await client.messages.create(
model=f"cc/{model_id}",
max_tokens=4096,
system=SYSTEM_PROMPT,
tools=BROWSER_TOOLS_SCHEMA,
messages=messages,
)
assistant_content = []
last_text_parts = []
tool_uses = []
for block in response.content:
if block.type == "text":
last_text_parts.append(block.text)
assistant_content.append({"type": "text", "text": block.text})
elif block.type == "tool_use":
tool_uses.append(block)
assistant_content.append({
"type": "tool_use", "id": block.id,
"name": block.name, "input": block.input,
})
messages.append({"role": "assistant", "content": assistant_content})
if response.stop_reason != "tool_use":
break
tool_results = []
for tu in tool_uses:
start = time.time()
result = await execute_browser_tool(tu.name, tu.input, browser_id,)
elapsed_ms = int((time.time() - start) * 1000)
action_log.append({
"tool": tu.name,
"input": tu.input,
"elapsed_ms": elapsed_ms,
})
if tu.name == "BrowserScreenshot" and result.get("image"):
final_screenshot = result["image"]
content_blocks = _format_tool_result(result, tu.name)
tool_results.append({
"type": "tool_result",
"tool_use_id": tu.id,
"content": content_blocks,
})
messages.append({"role": "user", "content": tool_results})
if not final_screenshot:
try:
ss = await execute_browser_tool("BrowserScreenshot", {}, browser_id)
if ss.get("image"):
final_screenshot = ss["image"]
except Exception:
pass
return {
"summary": "\n".join(last_text_parts) if last_text_parts else "Task completed.",
"action_log": action_log,
"final_screenshot": final_screenshot,
}
@@ -1,30 +0,0 @@
from typeguard import typechecked
from backend.ports import NINE_ROUTER_PORT
import anthropic
import httpx
SONNET: str = "claude-sonnet-4-6"
OPUS: str = "claude-opus-4-6"
HAIKU: str = "claude-haiku-4-5"
@typechecked
def is_valid_model_id(model_id: str) -> bool:
return model_id in [SONNET, OPUS, HAIKU]
@typechecked
def check_9router() -> bool:
"""Check if 9Router is running locally."""
try:
response: httpx.Response = httpx.get(f"http://localhost:{NINE_ROUTER_PORT}/v1/models", timeout=2.0)
return response.status_code == 200
except Exception:
return False
@typechecked
def get_anthropic_client() -> anthropic.AsyncAnthropic:
if not check_9router():
raise ValueError("9Router is not running")
return anthropic.AsyncAnthropic(
api_key="9router",
base_url=f"http://localhost:{NINE_ROUTER_PORT}",
)
@@ -1,137 +0,0 @@
"""Browser tool schemas, constants, and system prompt."""
BROWSER_TOOLS_SCHEMA = [
{
"name": "BrowserScreenshot",
"description": (
"Capture a screenshot of the browser page. Returns the screenshot as a "
"base64-encoded PNG image. Use this to see what is currently displayed."
),
"input_schema": {"type": "object", "properties": {}, "required": []},
},
{
"name": "BrowserGetText",
"description": "Get the visible text content of the browser page. Returns up to 15000 characters.",
"input_schema": {"type": "object", "properties": {}, "required": []},
},
{
"name": "BrowserNavigate",
"description": "Navigate the browser to a URL.",
"input_schema": {
"type": "object",
"properties": {"url": {"type": "string", "description": "The URL to navigate to."}},
"required": ["url"],
},
},
{
"name": "BrowserClick",
"description": "Click an element identified by a CSS selector. Use BrowserGetElements first to discover valid selectors.",
"input_schema": {
"type": "object",
"properties": {"selector": {"type": "string", "description": "CSS selector of the element to click."}},
"required": ["selector"],
},
},
{
"name": "BrowserType",
"description": "Type text into an input element. Clears existing value first.",
"input_schema": {
"type": "object",
"properties": {
"selector": {"type": "string", "description": "CSS selector of the input element."},
"text": {"type": "string", "description": "The text to type."},
},
"required": ["selector", "text"],
},
},
{
"name": "BrowserEvaluate",
"description": "Evaluate a JavaScript expression in the browser page and return the result.",
"input_schema": {
"type": "object",
"properties": {"expression": {"type": "string", "description": "JavaScript expression to evaluate."}},
"required": ["expression"],
},
},
{
"name": "BrowserGetElements",
"description": (
"Get a list of interactive elements on the page with CSS selectors. "
"Call this BEFORE clicking or typing so you know which selectors are valid."
),
"input_schema": {
"type": "object",
"properties": {
"selector": {
"type": "string",
"description": "Optional CSS selector to scope the search (e.g. 'form', '#main'). Defaults to 'body'.",
},
},
"required": [],
},
},
{
"name": "BrowserScroll",
"description": (
"Scroll the page up or down. Automatically finds the correct scrollable "
"container. Returns scroll position info including whether top/bottom has been reached."
),
"input_schema": {
"type": "object",
"properties": {
"direction": {"type": "string", "enum": ["up", "down"], "description": "Scroll direction. Defaults to 'down'."},
"amount": {"type": "number", "description": "Pixels to scroll. Defaults to 500."},
},
"required": [],
},
},
{
"name": "BrowserWait",
"description": (
"Wait for a specified duration. Useful after navigation or actions that "
"trigger page loads. Min 100ms, max 10000ms."
),
"input_schema": {
"type": "object",
"properties": {"milliseconds": {"type": "number", "description": "Duration to wait in milliseconds. Defaults to 1000."}},
"required": [],
},
},
]
ACTION_MAP = {
"BrowserScreenshot": "screenshot",
"BrowserGetText": "get_text",
"BrowserNavigate": "navigate",
"BrowserClick": "click",
"BrowserType": "type",
"BrowserEvaluate": "evaluate",
"BrowserGetElements": "get_elements",
"BrowserScroll": "scroll",
"BrowserWait": "wait",
}
SYSTEM_PROMPT = (
"You are a browser automation agent. You control a single browser tab and "
"execute the task you are given.\n\n"
"Strategy:\n"
"1. Start by taking a screenshot to understand the page.\n"
"2. After navigation, use BrowserWait (1-3 seconds) to let the page finish loading.\n"
"3. Use BrowserScroll to scroll through pages — do NOT use BrowserEvaluate with "
"window.scrollBy() as many sites use nested scroll containers that BrowserScroll "
"handles automatically.\n"
"4. Use BrowserGetElements BEFORE clicking or typing to discover valid CSS selectors.\n"
"5. After performing actions, take a screenshot to verify the result.\n"
"6. If an action fails, try alternative selectors or approaches.\n"
"7. When the task is complete, provide a clear summary of what you accomplished.\n\n"
"Important notes:\n"
"- BrowserGetText returns up to 15000 chars of visible text — use it to read page content.\n"
"- BrowserScroll returns position info including atTop/atBottom — use this to know when "
"you've reached the end of the page.\n"
"- For complex SPAs (Notion, Gmail, etc.), prefer BrowserScroll over BrowserEvaluate for scrolling.\n"
"- Avoid looping: if scrolling shows no new content (scrolled 0px), you're at the boundary.\n\n"
"You have access ONLY to browser tools. Do not ask the user questions — "
"complete the task autonomously to the best of your ability."
)
MAX_TURNS = 25
@@ -1,10 +1,9 @@
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_actions_toolkit.make_browser_actions_toolkit import make_browser_actions_toolkit
from backend.apps.agents.HaikFix.tools.make_builtin_toolkit.open_swarm_toolkits.browser_toolkit.make_browser_delegation_toolkit.make_browser_delegation_toolkit import make_browser_delegation_toolkit
from backend.apps.agents.HaikFix.tools.shared_structs.Toolkit import Toolkit
from backend.apps.agents.HaikFix.tools.shared_structs.MCP_Tool import SDK_MCP_Tool
from typing import Dict
from backend.apps.agents.HaikFix.Agent.Agent import Agent
# # NOTE: The minor issue here is that with the current setup, any agent will prly have access to the browser actions toolkit, idk if this bad or good but for now its fine ig
def make_browser_toolkit(
parent: Agent,
@@ -1,17 +1,6 @@
from typing import Optional, Dict, Any, Literal, Callable, Awaitable, List, Union
from pydantic import BaseModel, Field
from typing import Optional
from pydantic import BaseModel
from backend.apps.agents.HaikFix.tools.shared_structs.TOOL_PERMISSIONS import TOOL_PERMISSIONS
from claude_agent_sdk import (
create_sdk_mcp_server,
tool as sdk_tool
)
from claude_agent_sdk.types import (
McpStdioServerConfig,
McpSSEServerConfig,
McpHttpServerConfig,
McpSdkServerConfig,
McpServerConfig
)
from typeguard import typechecked
class Tool(BaseModel):