diff --git a/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/BROWSER_TOOLKIT.py b/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/BROWSER_TOOLKIT.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/custom_handlers/browser_utils/format_browser_result.py b/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/custom_handlers/browser_utils/format_browser_result.py new file mode 100644 index 00000000..4a834e58 --- /dev/null +++ b/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/custom_handlers/browser_utils/format_browser_result.py @@ -0,0 +1,31 @@ +import json +from typing import Dict, List, Any + +from backend.apps.agents.HaikFix.Agent.shared_structs.Message.agent_outputs import ToolResponse + +MAX_IMAGE_B64_BYTES = 400_000 + +def format_browser_result(result: Dict[str, Any], browser_id: str) -> ToolResponse: + """Format the browser loop result into a ToolResponse.""" + lines = [ + f"**Browser Agent Result** (browser: {browser_id})", + "", + f"**Summary:** {result['summary']}", + ] + + if result["action_log"]: + lines.append("") + lines.append("**Actions taken:**") + for i, entry in enumerate(result["action_log"], 1): + tool = entry["tool"] + inp = json.dumps(entry.get("input", {}))[:120] + ms = entry.get("elapsed_ms", 0) + lines.append(f" {i}. {tool}({inp}) [{ms}ms]") + + content: List[Dict[str, str]] = [{"type": "text", "text": "\n".join(lines)}] + + screenshot = result.get("final_screenshot") + if screenshot and len(screenshot) <= MAX_IMAGE_B64_BYTES: + content.append({"type": "image", "data": screenshot, "mimeType": "image/png"}) + + return {"content": content} \ No newline at end of file diff --git a/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/custom_handlers/browser_utils/run_browser_loop.py b/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/custom_handlers/browser_utils/run_browser_loop.py new file mode 100644 index 00000000..78ff64e6 --- /dev/null +++ b/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/custom_handlers/browser_utils/run_browser_loop.py @@ -0,0 +1,98 @@ +import time +from typing import Dict, List, Any +from backend.apps.agents.browser.schemas import ( + BROWSER_TOOLS_SCHEMA, SYSTEM_PROMPT, MAX_TURNS, +) +# NOTE: Hella dependancies, this baddddddd +# TODO: fix this shit cuh +from backend.apps.agents.browser.executor import execute_browser_tool, _format_tool_result +from backend.apps.settings.settings import load_settings +from backend.apps.settings.credentials import get_anthropic_client +from backend.apps.common.model_registry import resolve_model_id +from backend.apps.common.llm_helpers import _resolve_model as _resolve_9r + +async def run_browser_loop( + task: str, + browser_id: str, + model: str, + initial_url: str | None = None, + tab_id: str = "", +) -> Dict[str, Any]: + """Run the browser agent tool loop against the Anthropic API directly.""" + + if initial_url: + await execute_browser_tool("BrowserNavigate", {"url": initial_url}, browser_id, tab_id) + + settings = load_settings() + api_model = _resolve_9r(resolve_model_id(model), settings) + client = get_anthropic_client(settings) + + messages: List[Dict] = [{"role": "user", "content": task}] + action_log: List[Dict] = [] + final_screenshot: str | None = None + last_text_parts: List[str] = [] + + for _ in range(MAX_TURNS): + response = await client.messages.create( + model=api_model, + max_tokens=4096, + system=SYSTEM_PROMPT, + tools=BROWSER_TOOLS_SCHEMA, + messages=messages, + ) + + assistant_content = [] + last_text_parts = [] + tool_uses = [] + + for block in response.content: + if block.type == "text": + last_text_parts.append(block.text) + assistant_content.append({"type": "text", "text": block.text}) + elif block.type == "tool_use": + tool_uses.append(block) + assistant_content.append({ + "type": "tool_use", "id": block.id, + "name": block.name, "input": block.input, + }) + + messages.append({"role": "assistant", "content": assistant_content}) + + if response.stop_reason != "tool_use": + break + + tool_results = [] + for tu in tool_uses: + start = time.time() + result = await execute_browser_tool(tu.name, tu.input, browser_id, tab_id) + elapsed_ms = int((time.time() - start) * 1000) + action_log.append({ + "tool": tu.name, + "input": tu.input, + "elapsed_ms": elapsed_ms, + }) + if tu.name == "BrowserScreenshot" and result.get("image"): + final_screenshot = result["image"] + content_blocks = _format_tool_result(result, tu.name) + tool_results.append({ + "type": "tool_result", + "tool_use_id": tu.id, + "content": content_blocks, + }) + + messages.append({"role": "user", "content": tool_results}) + + if not final_screenshot: + try: + ss = await execute_browser_tool("BrowserScreenshot", {}, browser_id, tab_id) + if ss.get("image"): + final_screenshot = ss["image"] + except Exception: + pass + + return { + "summary": "\n".join(last_text_parts) if last_text_parts else "Task completed.", + "action_log": action_log, + "final_screenshot": final_screenshot, + } + diff --git a/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/custom_handlers/make_create_browser_agent_handler.py b/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/custom_handlers/make_create_browser_agent_handler.py new file mode 100644 index 00000000..7f059211 --- /dev/null +++ b/backend/apps/agents/HaikFix/tools/BUILTIN_TOOLKIT/components/custom_handlers/make_create_browser_agent_handler.py @@ -0,0 +1,44 @@ +import asyncio +import logging +from typing import TypedDict +from typing_extensions import NotRequired +from typeguard import typechecked + +from backend.apps.agents.HaikFix.Agent.shared_structs.Message.agent_outputs import ToolResponse +from backend.apps.agents.HaikFix.tools.BUILTIN_TOOLKIT.components.custom_handlers.browser_utils.run_browser_loop import run_browser_loop +from backend.apps.agents.HaikFix.tools.BUILTIN_TOOLKIT.components.custom_handlers.browser_utils.format_browser_result import format_browser_result +from backend.apps.agents.HaikFix.Agent.Agent import Agent + +class CreateBrowserAgentInput(TypedDict): + task: str + url: NotRequired[str] + +@typechecked +def make_create_browser_agent_handler( + parent: Agent, + create_browser_card_fn, +): + async def handler(args: CreateBrowserAgentInput) -> ToolResponse: + task = args["task"] + url = args.get("url", "") + + browser_id = await create_browser_card_fn( + dashboard_id=parent.config.session_id or "", + url=url, + parent_session_id=parent.session_id, + ) + if not browser_id: + return {"content": [{"type": "text", "text": "Error: failed to create browser card"}], "is_error": True} + + await asyncio.sleep(2.0) + + result = await run_browser_loop( + task=task, + browser_id=browser_id, + model=parent.model, + initial_url=url, + ) + + return format_browser_result(result, browser_id) + + return handler \ No newline at end of file