From d594f2afb99deb63793a8b267dacbda72a128c05 Mon Sep 17 00:00:00 2001 From: Arnav Naval Date: Mon, 4 May 2026 18:02:42 -0500 Subject: [PATCH] [arnav] test(backend): add unit and integration tests for browser_agent.py --- .../tests/test_browser_agent_integration.py | 920 ++++++++++++++++++ backend/tests/test_browser_agent_unit.py | 600 ++++++++++++ 2 files changed, 1520 insertions(+) create mode 100644 backend/tests/test_browser_agent_integration.py create mode 100644 backend/tests/test_browser_agent_unit.py diff --git a/backend/tests/test_browser_agent_integration.py b/backend/tests/test_browser_agent_integration.py new file mode 100644 index 00000000..f0c4b4fa --- /dev/null +++ b/backend/tests/test_browser_agent_integration.py @@ -0,0 +1,920 @@ +"""Integration tests for `backend.apps.agents.browser_agent`. + +Drives the full `run_browser_agent` loop end-to-end with a fake +Anthropic SDK and a fully-stubbed `ws_manager`. The unit-helper +coverage lives in `test_browser_agent_unit.py`; this file focuses on +the orchestration + protocol behaviour: + + - happy path (text-only response → completed + screenshot fallback) + - ReportProgress brain-state recording + same-turn action tool + - ReportProgress violation: action tool without brain state is rejected + - loop detection: third identical (tool, input, result) gets a warning + - loop hard cap: 5 triggers force-exit before MAX_TURNS + - RequestHumanIntervention allow + deny-with-message branches + - cancellation via parent_session_id already stopped + - "no LLM connected" error branch (resolve_aux_model raises) + - prior-history resume + corrupted-cache drop + - initial_url navigation runs before the first API call + - token usage accumulates across turns + - history is persisted (and well-formed) on completion + - `_create_browser_card` writes to disk + broadcasts globally + - `run_browser_agents` parallel fanout (auto-create card, + exception-per-task surfacing) + +All tests: + - Patch the Anthropic client at the lazy-import boundary in + `backend.apps.settings.credentials.get_anthropic_client_for_model`. + - Patch `ws_manager.send_to_session` / `send_browser_command` / + `send_approval_request` / `broadcast_global` as AsyncMocks. + - Wipe `_browser_history` and `agent_manager.sessions/tasks` between + tests so module-level state never leaks. +""" + +from __future__ import annotations + +import asyncio +from types import SimpleNamespace +from typing import Any +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from backend.apps.agents import browser_agent as ba +from backend.apps.agents.browser_agent import ( + _LOOP_HARD_CAP, + MAX_TURNS, + _create_browser_card, + _validate_message_pairing, + run_browser_agent, + run_browser_agents, +) +from backend.apps.agents.models import AgentSession + + +# --------------------------------------------------------------------------- +# SDK fakes +# --------------------------------------------------------------------------- + + +def _text_block(text: str) -> SimpleNamespace: + """Anthropic-shaped text content block (dot-attribute access).""" + return SimpleNamespace(type="text", text=text) + + +def _tool_use_block(*, id: str, name: str, input: dict) -> SimpleNamespace: + """Anthropic-shaped tool_use content block.""" + return SimpleNamespace(type="tool_use", id=id, name=name, input=input) + + +def _make_response( + blocks: list[Any], + *, + stop_reason: str = "end_turn", + input_tokens: int = 0, + output_tokens: int = 0, +) -> SimpleNamespace: + """Anthropic-shaped Messages API response with a `usage` sub-object.""" + return SimpleNamespace( + content=blocks, + stop_reason=stop_reason, + usage=SimpleNamespace( + input_tokens=input_tokens, + output_tokens=output_tokens, + ), + ) + + +def _make_fake_client(responses: list[Any] | None = None) -> MagicMock: + """Build a duck-typed AsyncAnthropic client. + + `client.messages.create(...)` returns a coroutine resolving to the + next item from `responses`. If a callable is passed instead, it's + used as the AsyncMock side_effect (lets tests return the same + response indefinitely for loop-detection coverage). + + The fake also snapshots each call's `messages` kwarg into + `client.message_snapshots` because the agent loop mutates the + same `messages` list after every turn — `mock.await_args.kwargs` + only retains the reference, so without a snapshot we'd assert on + the post-mutation state.""" + client = MagicMock() + client.messages = MagicMock() + client.message_snapshots: list[list[dict]] = [] + + if callable(responses): + cb = responses + + async def _wrapped(*args, **kwargs): + client.message_snapshots.append( + [dict(m) for m in kwargs.get("messages", [])] + ) + return cb(*args, **kwargs) + + client.messages.create = AsyncMock(side_effect=_wrapped) + else: + queue = list(responses or []) + + async def _wrapped(*args, **kwargs): + client.message_snapshots.append( + [dict(m) for m in kwargs.get("messages", [])] + ) + if not queue: + raise StopAsyncIteration("ran out of scripted responses") + return queue.pop(0) + + client.messages.create = AsyncMock(side_effect=_wrapped) + return client + + +# --------------------------------------------------------------------------- +# Shared patching helpers +# --------------------------------------------------------------------------- + + +class _WSStub: + """Captures every (event, payload) emitted to `ws_manager.send_to_session`, + and counts approvals + browser command calls. Plug in via patch.object.""" + + def __init__(self) -> None: + self.events: list[tuple[str, str, dict]] = [] + self.browser_commands: list[dict] = [] + self.approval_requests: list[dict] = [] + self.global_broadcasts: list[tuple[str, dict]] = [] + + # Default behaviours; tests override to script approval flows. + self.send_to_session = AsyncMock(side_effect=self._on_send_to_session) + self.send_browser_command = AsyncMock(side_effect=self._on_browser_command) + self.send_approval_request = AsyncMock( + return_value={"behavior": "allow"} + ) + self.broadcast_global = AsyncMock(side_effect=self._on_broadcast_global) + + # Default "browser command result" — tests override to script + # specific tool results (e.g. {"image": "..."} for screenshots). + self.browser_result: dict | Any = {"text": "ok"} + + async def _on_send_to_session(self, session_id, event, payload): + self.events.append((session_id, event, payload)) + + async def _on_browser_command(self, request_id, action, browser_id, params, tab_id=""): + self.browser_commands.append({ + "request_id": request_id, + "action": action, + "browser_id": browser_id, + "params": params, + "tab_id": tab_id, + }) + if callable(self.browser_result): + return self.browser_result(action, params) + return dict(self.browser_result) + + async def _on_broadcast_global(self, event, data): + self.global_broadcasts.append((event, data)) + + def events_of_type(self, event_type: str) -> list[dict]: + return [p for (_sid, ev, p) in self.events if ev == event_type] + + +def _patch_browser_agent_deps( + monkeypatch, + *, + fake_client: MagicMock, + ws_stub: _WSStub, + builtin_perms: dict[str, str] | None = None, + is_builtin_model: bool = True, + aux_model_raises: bool = False, +): + """Apply the four canonical patches every integration test needs. + + - ws_manager methods → ws_stub's AsyncMocks + - get_anthropic_client_for_model → returns fake_client + - load_settings → no-op AppSettings() + - permissions / model resolution → defaults safe for tests + """ + monkeypatch.setattr(ba.ws_manager, "send_to_session", ws_stub.send_to_session) + monkeypatch.setattr(ba.ws_manager, "send_browser_command", ws_stub.send_browser_command) + monkeypatch.setattr(ba.ws_manager, "send_approval_request", ws_stub.send_approval_request) + monkeypatch.setattr(ba.ws_manager, "broadcast_global", ws_stub.broadcast_global) + + monkeypatch.setattr( + ba, "load_builtin_permissions", lambda: builtin_perms or {} + ) + + from backend.apps.settings import credentials as creds_mod + from backend.apps.settings import settings as settings_mod + from backend.apps.settings.models import AppSettings + from backend.apps.agents.providers import registry as registry_mod + + monkeypatch.setattr(settings_mod, "load_settings", lambda: AppSettings()) + monkeypatch.setattr( + creds_mod, "get_anthropic_client_for_model", + lambda settings, model: fake_client, + ) + + if is_builtin_model: + monkeypatch.setattr( + registry_mod, "_find_builtin_model", + lambda short: {"value": short, "api": "anthropic"}, + ) + monkeypatch.setattr( + registry_mod, "resolve_model_id_for_sdk", + lambda short, settings: f"claude-fake-{short}", + ) + else: + monkeypatch.setattr(registry_mod, "_find_builtin_model", lambda short: None) + if aux_model_raises: + async def _raises(*args, **kwargs): + raise ValueError("no provider connected") + monkeypatch.setattr(registry_mod, "resolve_aux_model", _raises) + else: + async def _ok(*args, **kwargs): + return ("claude-fake-aux", None) + monkeypatch.setattr(registry_mod, "resolve_aux_model", _ok) + + +@pytest.fixture(autouse=True) +def _reset_browser_agent_state(tmp_data_dirs): + """Module-level conversation cache only — `tmp_data_dirs` (from + conftest.py) already wipes the per-feature data dirs AND clears + `agent_manager.sessions/tasks` between tests, so the only piece + of state unique to browser_agent is `_browser_history`.""" + ba._browser_history.clear() + yield + ba._browser_history.clear() + + +# =========================================================================== +# run_browser_agent — happy paths +# =========================================================================== + + +async def test_happy_path_text_only_completes_and_persists_history(monkeypatch): + """One turn returning text only → status flips to completed, + summary echoes the model text, history is cached for next call.""" + ws = _WSStub() + ws.browser_result = {"image": "FALLBACK_PNG", "url": "https://x.com"} + fake = _make_fake_client([ + _make_response( + [_text_block("All done!")], + stop_reason="end_turn", + input_tokens=10, + output_tokens=2, + ), + ]) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + out = await run_browser_agent( + task="say hello", + browser_id="b-1", + model="sonnet", + ) + assert out["browser_id"] == "b-1" + assert out["summary"] == "All done!" + # Final screenshot fallback was triggered (no screenshot during the loop). + assert out["final_screenshot"] == "FALLBACK_PNG" + + # agent:status events fired for both running and completed. + statuses = [p["status"] for p in ws.events_of_type("agent:status")] + assert "running" in statuses + assert "completed" in statuses + + # History was cached and round-trips _validate_message_pairing. + assert "b-1" in ba._browser_history + assert _validate_message_pairing(ba._browser_history["b-1"]) + + +async def test_report_progress_plus_action_tool_records_brain_state(monkeypatch): + """Turn 1: ReportProgress + BrowserNavigate. Turn 2: text + end. + The brain-state assistant message must show up in session.messages + AND the navigate tool must be dispatched through ws_manager.""" + ws = _WSStub() + + def _result_for(action: str, params: dict) -> dict: + if action == "navigate": + return {"text": f"navigated to {params.get('url')}"} + return {"image": "PNG", "url": "https://x.com"} + + ws.browser_result = _result_for + + fake = _make_fake_client([ + _make_response( + [ + _tool_use_block( + id="rp-1", name="ReportProgress", + input={ + "evaluation_previous": "n/a (first turn)", + "working_memory": "fresh page", + "next_goal": "navigate to example.com", + }, + ), + _tool_use_block( + id="nav-1", name="BrowserNavigate", + input={"url": "https://example.com"}, + ), + ], + stop_reason="tool_use", + input_tokens=15, output_tokens=3, + ), + _make_response( + [_text_block("Navigation complete.")], + stop_reason="end_turn", + input_tokens=20, output_tokens=4, + ), + ]) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + out = await run_browser_agent(task="go", browser_id="b-1", model="sonnet") + assert out["summary"] == "Navigation complete." + + # Brain-state message ("📋 **Plan**") was persisted. + from backend.apps.agents.agent_manager import agent_manager + sess = agent_manager.sessions[out["session_id"]] + brain_msgs = [ + m for m in sess.messages + if m.role == "assistant" and isinstance(m.content, str) + and "**Plan**" in m.content + ] + assert len(brain_msgs) == 1 + assert "next goal" in brain_msgs[0].content.lower() \ + or "navigate to example.com" in brain_msgs[0].content + + # Navigate command flowed through ws_manager with the right params. + nav_calls = [c for c in ws.browser_commands if c["action"] == "navigate"] + assert len(nav_calls) >= 1 + assert nav_calls[0]["params"] == {"url": "https://example.com"} + + # action_log captured the tool execution. + nav_logs = [a for a in out["action_log"] if a["tool"] == "BrowserNavigate"] + assert len(nav_logs) == 1 + + +async def test_report_progress_violation_rejects_action_tool(monkeypatch): + """Action tool without ReportProgress in the same turn → tool_result + is is_error=True with REJECTED text; the underlying browser + command is NEVER dispatched to ws_manager.""" + ws = _WSStub() + fake = _make_fake_client([ + _make_response( + [ + _tool_use_block( + id="click-1", name="BrowserClick", + input={"selector": "#go"}, + ), + ], + stop_reason="tool_use", + ), + # Turn 2: model gives up and returns text. + _make_response([_text_block("aborted")], stop_reason="end_turn"), + ]) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + out = await run_browser_agent(task="x", browser_id="b-rp", model="sonnet") + assert out["summary"] == "aborted" + + # Click was rejected → no `click` action ever made it to ws_manager. + click_calls = [c for c in ws.browser_commands if c["action"] == "click"] + assert click_calls == [] + + # The rejection text shows up as a tool_result Message in the session. + from backend.apps.agents.agent_manager import agent_manager + sess = agent_manager.sessions[out["session_id"]] + rejection_msgs = [ + m for m in sess.messages + if m.role == "tool_result" and isinstance(m.content, dict) + and "REJECTED" in str(m.content.get("text", "")) + ] + assert len(rejection_msgs) == 1 + + +# =========================================================================== +# Loop detection +# =========================================================================== + + +async def test_loop_detection_third_identical_call_gets_warning(monkeypatch): + """Three identical (tool, input, result) tuples → on the third call + the tool_result includes the LOOP DETECTED warning (is_error=True).""" + ws = _WSStub() + # Always return the same result so the loop-detector key is identical. + ws.browser_result = {"text": "click did nothing"} + + def _looping_response(*args, **kwargs): + return _make_response( + [ + _tool_use_block( + id=f"rp-{kwargs.get('messages', [{}])[-1]}", + name="ReportProgress", + input={ + "evaluation_previous": "didn't help", + "working_memory": "site is hostile", + "next_goal": "try again", + }, + ), + _tool_use_block( + id=f"click-{len(kwargs.get('messages', []))}", + name="BrowserClick", + input={"selector": "#submit"}, + ), + ], + stop_reason="tool_use", + ) + + fake = _make_fake_client(_looping_response) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + out = await run_browser_agent(task="x", browser_id="b-loop", model="sonnet") + + # Loop hard cap eventually force-exits, so we get the canned summary. + assert isinstance(out["summary"], str) and out["summary"] + + # Inspect session tool_result messages: the 3rd+ click result must + # carry the LOOP DETECTED warning. (Earlier ones won't.) + from backend.apps.agents.agent_manager import agent_manager + sess = agent_manager.sessions[out["session_id"]] + click_results = [ + m for m in sess.messages + if m.role == "tool_result" + and isinstance(m.content, dict) + and m.content.get("tool_name") == "BrowserClick" + ] + assert len(click_results) >= 3 + # Note: the warning is appended to the API content_block, not the + # session.message text — so we verify via ws_manager send_browser_command + # call count + the loop-trigger side effect (hard cap force-exit). + # Verify hard cap took effect: fewer API calls than MAX_TURNS. + assert fake.messages.create.await_count < MAX_TURNS + + +async def test_loop_hard_cap_force_exits_before_max_turns(monkeypatch): + """Once `_LOOP_HARD_CAP` triggers fire, the loop must break early + rather than burning the entire MAX_TURNS budget.""" + ws = _WSStub() + ws.browser_result = {"text": "still failing"} + + call_count = {"n": 0} + + def _looping_response(*args, **kwargs): + call_count["n"] += 1 + n = call_count["n"] + return _make_response( + [ + _tool_use_block( + id=f"rp-{n}", name="ReportProgress", + input={ + "evaluation_previous": "stuck", + "working_memory": "...", + "next_goal": "click again", + }, + ), + _tool_use_block( + id=f"click-{n}", name="BrowserClick", + input={"selector": "#x"}, + ), + ], + stop_reason="tool_use", + ) + + fake = _make_fake_client(_looping_response) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + out = await run_browser_agent(task="x", browser_id="b-cap", model="sonnet") + + # Strict bound: hard cap ought to fire within ~2 * _LOOP_HARD_CAP turns + # (each trigger after threshold takes one turn). Assert we're well + # below MAX_TURNS — and well below it (no near-misses). + assert fake.messages.create.await_count <= _LOOP_HARD_CAP + 5 + assert fake.messages.create.await_count < MAX_TURNS + # Run still completed cleanly (status=completed, not error). + from backend.apps.agents.agent_manager import agent_manager + sess = agent_manager.sessions[out["session_id"]] + assert sess.status == "completed" + + +# =========================================================================== +# RequestHumanIntervention +# =========================================================================== + + +async def test_request_human_intervention_allow_resumes(monkeypatch): + """Approval allow → tool_result text reads 'User resolved the issue.' + AND a waiting_approval status event was emitted to the session.""" + ws = _WSStub() + ws.send_approval_request = AsyncMock(return_value={"behavior": "allow"}) + + fake = _make_fake_client([ + _make_response( + [ + _tool_use_block( + id="rhi-1", name="RequestHumanIntervention", + input={"problem": "captcha", "instruction": "solve it"}, + ), + ], + stop_reason="tool_use", + ), + _make_response([_text_block("done")], stop_reason="end_turn"), + ]) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + out = await run_browser_agent(task="x", browser_id="b-hi", model="sonnet") + assert out["summary"] == "done" + + statuses = [p["status"] for p in ws.events_of_type("agent:status")] + assert "waiting_approval" in statuses + + from backend.apps.agents.agent_manager import agent_manager + sess = agent_manager.sessions[out["session_id"]] + resolved = [ + m for m in sess.messages + if m.role == "tool_result" + and isinstance(m.content, dict) + and "User resolved" in str(m.content.get("text", "")) + ] + assert len(resolved) == 1 + + +async def test_request_human_intervention_deny_with_message_surfaces_user_text(monkeypatch): + """Approval deny with user-supplied message → tool_result text must + include the user message and the 'Address what the user said' prefix.""" + ws = _WSStub() + ws.send_approval_request = AsyncMock(return_value={ + "behavior": "deny", + "message": "use a different site", + }) + + fake = _make_fake_client([ + _make_response( + [ + _tool_use_block( + id="rhi-2", name="RequestHumanIntervention", + input={"problem": "blocked", "instruction": "?"}, + ), + ], + stop_reason="tool_use", + ), + _make_response([_text_block("ok")], stop_reason="end_turn"), + ]) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + out = await run_browser_agent(task="x", browser_id="b-hi2", model="sonnet") + + from backend.apps.agents.agent_manager import agent_manager + sess = agent_manager.sessions[out["session_id"]] + deny_msgs = [ + m for m in sess.messages + if m.role == "tool_result" and isinstance(m.content, dict) + and "use a different site" in str(m.content.get("text", "")) + ] + assert len(deny_msgs) == 1 + assert "Address what the user said" in str(deny_msgs[0].content["text"]) + + +# =========================================================================== +# Cancellation + error paths +# =========================================================================== + + +async def test_cancellation_via_stopped_parent_returns_stopped_summary(monkeypatch): + """Parent session already stopped before browser-agent registers → + cancel_event set immediately → no API call, status='stopped'.""" + from backend.apps.agents.agent_manager import agent_manager + + parent_id = "parent-stopped" + parent = AgentSession(id=parent_id, name="parent", model="sonnet") + parent.status = "stopped" + agent_manager.sessions[parent_id] = parent + + ws = _WSStub() + fake = _make_fake_client() # never actually called + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + out = await run_browser_agent( + task="x", + browser_id="b-cancel", + model="sonnet", + parent_session_id=parent_id, + ) + assert "stopped by the user" in out["summary"] + assert out["error"] == "Agent was stopped by the user." + assert agent_manager.sessions[out["session_id"]].status == "stopped" + # Loop never entered — API client was never called. + assert fake.messages.create.await_count == 0 + + +async def test_no_creds_branch_returns_error_summary(monkeypatch): + """Unknown model + resolve_aux_model raises ValueError → error + payload returned without ever making an API call.""" + ws = _WSStub() + fake = _make_fake_client() + _patch_browser_agent_deps( + monkeypatch, fake_client=fake, ws_stub=ws, + is_builtin_model=False, aux_model_raises=True, + ) + + out = await run_browser_agent( + task="x", browser_id="b-noc", model="random-unknown-model", + ) + assert out["summary"].startswith("Error: Browser agent requires") + assert out["action_log"] == [] + assert out["final_screenshot"] is None + + from backend.apps.agents.agent_manager import agent_manager + assert agent_manager.sessions[out["session_id"]].status == "error" + assert fake.messages.create.await_count == 0 + + +# =========================================================================== +# History resume / cache invalidation +# =========================================================================== + + +async def test_history_resume_prepends_cached_messages_to_first_call(monkeypatch): + """Cached prior messages must be prepended to the messages list + sent on turn 1 (otherwise the agent re-orients from scratch every + swipe).""" + cached = [ + {"role": "user", "content": "earlier task"}, + {"role": "assistant", "content": [{"type": "text", "text": "earlier reply"}]}, + ] + ba._browser_history["b-resume"] = list(cached) + + ws = _WSStub() + fake = _make_fake_client([ + _make_response([_text_block("ok")], stop_reason="end_turn"), + ]) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + await run_browser_agent(task="next", browser_id="b-resume", model="sonnet") + + # Snapshot of the messages list at first-call time (the loop mutates it + # in place between turns, so we can't read await_args after-the-fact). + sent_messages = fake.message_snapshots[0] + assert sent_messages[:2] == cached + assert sent_messages[2] == {"role": "user", "content": "next"} + + +async def test_corrupt_cached_history_is_dropped(monkeypatch): + """If the cached chain has an orphan tool_result, it's dropped and + only the new user task is sent.""" + bad = [ + {"role": "user", "content": "earlier"}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "ghost", "content": []}, + ]}, + ] + ba._browser_history["b-corrupt"] = list(bad) + + ws = _WSStub() + fake = _make_fake_client([ + _make_response([_text_block("clean start")], stop_reason="end_turn"), + ]) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + await run_browser_agent(task="fresh", browser_id="b-corrupt", model="sonnet") + + sent_messages = fake.message_snapshots[0] + assert sent_messages == [{"role": "user", "content": "fresh"}] + + +# =========================================================================== +# Initial URL navigation + token tracking +# =========================================================================== + + +async def test_initial_url_triggers_navigate_before_first_api_call(monkeypatch): + """When initial_url is provided, BrowserNavigate runs once before + the loop opens an API call.""" + ws = _WSStub() + + def _result_for(action: str, params: dict) -> dict: + if action == "navigate": + return {"text": "navigated"} + return {"image": "PNG", "url": "https://example.com"} + + ws.browser_result = _result_for + + fake = _make_fake_client([ + _make_response([_text_block("ack")], stop_reason="end_turn"), + ]) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + await run_browser_agent( + task="x", browser_id="b-init", model="sonnet", + initial_url="https://example.com", + ) + + nav_calls = [c for c in ws.browser_commands if c["action"] == "navigate"] + assert len(nav_calls) == 1 + assert nav_calls[0]["params"] == {"url": "https://example.com"} + assert nav_calls[0]["browser_id"] == "b-init" + + +async def test_token_usage_accumulates_across_turns(monkeypatch): + """Per-turn input/output tokens roll up into session.tokens.""" + ws = _WSStub() + ws.browser_result = {"text": "ok"} + + fake = _make_fake_client([ + _make_response( + [ + _tool_use_block( + id="rp-1", name="ReportProgress", + input={"evaluation_previous": "n/a", + "working_memory": "x", + "next_goal": "click"}, + ), + _tool_use_block( + id="c-1", name="BrowserClick", input={"selector": "#a"}, + ), + ], + stop_reason="tool_use", + input_tokens=5, output_tokens=3, + ), + _make_response( + [_text_block("done")], + stop_reason="end_turn", + input_tokens=2, output_tokens=1, + ), + ]) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + out = await run_browser_agent(task="x", browser_id="b-tok", model="sonnet") + from backend.apps.agents.agent_manager import agent_manager + sess = agent_manager.sessions[out["session_id"]] + assert sess.tokens == {"input": 7, "output": 4} + + +async def test_history_persisted_and_well_formed(monkeypatch): + """After a successful run, _browser_history[browser_id] is + populated AND passes the same orphan-pairing validator the resume + path uses on read.""" + ws = _WSStub() + ws.browser_result = {"text": "ok"} + + fake = _make_fake_client([ + _make_response( + [ + _tool_use_block( + id="rp-1", name="ReportProgress", + input={"evaluation_previous": "x", "working_memory": "y", "next_goal": "z"}, + ), + _tool_use_block( + id="c-1", name="BrowserClick", input={"selector": "#a"}, + ), + ], + stop_reason="tool_use", + ), + _make_response([_text_block("done")], stop_reason="end_turn"), + ]) + _patch_browser_agent_deps(monkeypatch, fake_client=fake, ws_stub=ws) + + await run_browser_agent(task="x", browser_id="b-hist", model="sonnet") + assert "b-hist" in ba._browser_history + cached = ba._browser_history["b-hist"] + assert len(cached) >= 2 + assert _validate_message_pairing(cached) is True + + +# =========================================================================== +# _create_browser_card +# =========================================================================== + + +async def test_create_browser_card_writes_disk_and_broadcasts(monkeypatch): + """Card is appended to the dashboard layout, persisted, and + broadcast as `dashboard:browser_card_added`.""" + from backend.apps.dashboards.dashboards import _save, _load + from backend.apps.dashboards.models import Dashboard + + dash = Dashboard(id="d-card", name="t") + _save(dash) + + captured: list[tuple[str, dict]] = [] + + async def _capture_broadcast(event: str, data: dict): + captured.append((event, data)) + + monkeypatch.setattr(ba.ws_manager, "broadcast_global", + AsyncMock(side_effect=_capture_broadcast)) + + browser_id = await _create_browser_card( + "d-card", "https://x.com", parent_session_id="p-1", + ) + assert browser_id.startswith("browser-") + + reloaded = _load("d-card") + assert browser_id in reloaded.layout.browser_cards + card = reloaded.layout.browser_cards[browser_id] + assert card.url == "https://x.com" + assert card.spawned_by == "p-1" + + # Single global broadcast for the new card. + new_card_events = [(ev, p) for (ev, p) in captured + if ev == "dashboard:browser_card_added"] + assert len(new_card_events) == 1 + payload = new_card_events[0][1] + assert payload["dashboard_id"] == "d-card" + assert payload["parent_session_id"] == "p-1" + assert payload["browser_card"]["browser_id"] == browser_id + + +async def test_create_browser_card_defaults_url_when_blank(monkeypatch): + """If url is empty, both the card and tab fall back to google.com.""" + from backend.apps.dashboards.dashboards import _save, _load + from backend.apps.dashboards.models import Dashboard + + _save(Dashboard(id="d-card2", name="t")) + monkeypatch.setattr(ba.ws_manager, "broadcast_global", AsyncMock()) + + browser_id = await _create_browser_card("d-card2", "") + card = _load("d-card2").layout.browser_cards[browser_id] + assert card.url == "https://www.google.com" + assert card.tabs[0].url == "https://www.google.com" + + +# =========================================================================== +# run_browser_agents — parallel fanout +# =========================================================================== + + +async def test_run_browser_agents_fanout_auto_creates_card_and_returns_results(monkeypatch): + """Two tasks: one with browser_id, one without. The latter should + auto-create a card. Both results returned in input order.""" + from backend.apps.dashboards.dashboards import _save + from backend.apps.dashboards.models import Dashboard + + _save(Dashboard(id="d-fanout", name="t")) + + # Don't actually wait 2s for "card settle". + monkeypatch.setattr(ba.asyncio, "sleep", AsyncMock()) + monkeypatch.setattr(ba.ws_manager, "broadcast_global", AsyncMock()) + # Patch analytics so we don't depend on its plumbing here. + monkeypatch.setattr( + "backend.apps.analytics.collector.record", + lambda *a, **kw: None, + ) + + call_log: list[dict] = [] + + async def _fake_run_one(**kwargs): + call_log.append(kwargs) + return { + "session_id": f"s-{kwargs['browser_id']}", + "browser_id": kwargs["browser_id"], + "summary": f"summary for {kwargs['browser_id']}", + "action_log": [], + "final_screenshot": None, + } + + monkeypatch.setattr(ba, "run_browser_agent", _fake_run_one) + + results = await run_browser_agents( + tasks=[ + {"browser_id": "b-existing", "task": "do A"}, + {"task": "do B", "url": "https://example.com"}, + ], + model="sonnet", + dashboard_id="d-fanout", + ) + + assert len(results) == 2 + assert results[0]["summary"] == "summary for b-existing" + # Auto-created browser_id matches the second call's argument. + auto_browser_id = call_log[1]["browser_id"] + assert auto_browser_id.startswith("browser-") + assert results[1]["summary"] == f"summary for {auto_browser_id}" + + +async def test_run_browser_agents_exception_per_task_surfaces_in_results(monkeypatch): + """If run_browser_agent raises for one task, the other still + completes and the failed slot becomes {summary: 'Error: ...'}.""" + monkeypatch.setattr(ba.asyncio, "sleep", AsyncMock()) + monkeypatch.setattr(ba.ws_manager, "broadcast_global", AsyncMock()) + monkeypatch.setattr( + "backend.apps.analytics.collector.record", + lambda *a, **kw: None, + ) + + async def _maybe_explode(**kwargs): + if kwargs["browser_id"] == "boom": + raise RuntimeError("kaboom") + return { + "session_id": "s-ok", + "browser_id": kwargs["browser_id"], + "summary": "ok-summary", + "action_log": [], + "final_screenshot": None, + } + + monkeypatch.setattr(ba, "run_browser_agent", _maybe_explode) + + results = await run_browser_agents( + tasks=[ + {"browser_id": "ok-1", "task": "fine"}, + {"browser_id": "boom", "task": "explodes"}, + ], + model="sonnet", + ) + assert results[0]["summary"] == "ok-summary" + assert results[1]["summary"].startswith("Error: ") + assert results[1]["action_log"] == [] + assert results[1]["final_screenshot"] is None diff --git a/backend/tests/test_browser_agent_unit.py b/backend/tests/test_browser_agent_unit.py new file mode 100644 index 00000000..f8195a3e --- /dev/null +++ b/backend/tests/test_browser_agent_unit.py @@ -0,0 +1,600 @@ +"""Unit tests for `backend.apps.agents.browser_agent` pure helpers. + +The browser-agent runner is a hefty module, but most of the cleverness is +in small helpers that can be exercised in isolation: + + - `_hash_tool_call` / `_detect_loop` — sliding-window loop detector. + - `_validate_message_pairing` — orphan-`tool_result` guard for cached + history (the last-line-of-defense against a 400 from Anthropic). + - `_is_fresh_user_message` / `_summarize_messages` / + `_trim_history_by_turns` — programmatic compaction of long + conversations. + - `_format_tool_result` — translate `ws_manager` result dicts into + Anthropic content blocks. + - `clear_browser_history` / `execute_browser_tool` — small async + state + helpers. + +The integration-level tests for `run_browser_agent` / +`run_browser_agents` / `_create_browser_card` live in the sister file +`test_browser_agent_integration.py`. This file deliberately contains NO +real API calls — it asserts only on pure-Python behaviour. +""" + +from __future__ import annotations + +import json +from unittest.mock import AsyncMock, patch + +import pytest + +from backend.apps.agents import browser_agent as ba +from backend.apps.agents.browser_agent import ( + _LOOP_HARD_CAP, + _LOOP_REPEAT_THRESHOLD, + _LOOP_WARNING_TEXT, + _LOOP_WINDOW_SIZE, + _detect_loop, + _format_tool_result, + _hash_tool_call, + _is_fresh_user_message, + _summarize_messages, + _trim_history_by_turns, + _validate_message_pairing, + clear_browser_history, + execute_browser_tool, +) + + +# --------------------------------------------------------------------------- +# Fixtures +# --------------------------------------------------------------------------- + + +@pytest.fixture(autouse=True) +def _clear_browser_history_module_state(): + """Wipe the module-level conversation cache between tests so a leaked + entry from a prior test never bleeds into the next.""" + ba._browser_history.clear() + yield + ba._browser_history.clear() + + +# --------------------------------------------------------------------------- +# clear_browser_history +# --------------------------------------------------------------------------- + + +def test_clear_browser_history_drops_only_target_entry(): + ba._browser_history["b-1"] = [{"role": "user", "content": "hi"}] + ba._browser_history["b-2"] = [{"role": "user", "content": "yo"}] + + clear_browser_history("b-1") + + assert "b-1" not in ba._browser_history + assert "b-2" in ba._browser_history + + +def test_clear_browser_history_missing_id_is_noop(): + """Calling clear on an unknown id must not raise (it's used in + cleanup paths where the cache may already be empty).""" + clear_browser_history("never-existed") + assert ba._browser_history == {} + + +# --------------------------------------------------------------------------- +# _hash_tool_call +# --------------------------------------------------------------------------- + + +def test_hash_tool_call_deterministic_for_same_inputs(): + """Same (tool, input, result) → same key, every time. This is what + lets the sliding-window loop detector compare across turns.""" + a = _hash_tool_call("BrowserClick", {"selector": "#x"}, {"text": "ok"}) + b = _hash_tool_call("BrowserClick", {"selector": "#x"}, {"text": "ok"}) + assert a == b + # Tuple shape: (tool_name, json_input, truncated_json_result) + assert isinstance(a, tuple) and len(a) == 3 + assert a[0] == "BrowserClick" + + +def test_hash_tool_call_input_key_uses_sorted_keys(): + """Dict ordering must NOT change the key — otherwise a model that + happens to emit input keys in different orders would dodge the + loop guard.""" + a = _hash_tool_call("BrowserType", {"selector": "#i", "text": "a"}, {}) + b = _hash_tool_call("BrowserType", {"text": "a", "selector": "#i"}, {}) + assert a == b + + +def test_hash_tool_call_different_results_yield_different_keys(): + """If the result changes (e.g. BrowserScroll on a long feed), the + key must change so legitimate progress isn't mistaken for a loop.""" + a = _hash_tool_call("BrowserScroll", {"direction": "down"}, {"text": "scrolled 500px"}) + b = _hash_tool_call("BrowserScroll", {"direction": "down"}, {"text": "scrolled 250px"}) + assert a != b + + +def test_hash_tool_call_truncates_long_result_to_300(): + """The result key is sliced at 300 chars to keep huge image blobs + (BrowserScreenshot returns base64 PNG) out of the in-memory key.""" + big = {"image": "A" * 5000} + key = _hash_tool_call("BrowserScreenshot", {}, big) + assert len(key[2]) <= 300 + + +def test_hash_tool_call_falls_back_to_repr_on_serialization_failure(): + """Self-referential dicts trip json.dumps even with default=str. + The except branch must still return a (str, str, str) tuple.""" + + class _Boom: + def __repr__(self) -> str: + return "" + + boom = _Boom() + # Self-referential dict — json.dumps raises ValueError with default=str + bad_input: dict = {"x": 1} + bad_input["self"] = bad_input + + bad_result: dict = {} + bad_result["self"] = bad_result + + key = _hash_tool_call("BrowserClick", bad_input, bad_result) + assert isinstance(key, tuple) and len(key) == 3 + assert key[0] == "BrowserClick" + # Both fallbacks should be strings (repr-derived), not raise. + assert isinstance(key[1], str) and key[1] + assert isinstance(key[2], str) and len(key[2]) <= 300 + + +# --------------------------------------------------------------------------- +# _detect_loop +# --------------------------------------------------------------------------- + + +def _key(name: str, suffix: str = "") -> tuple[str, str, str]: + """Convenience for building loop-detector keys without having to + re-derive the JSON input/result every time.""" + return (name, "{}", suffix or "{}") + + +def test_detect_loop_excluded_tool_never_triggers(): + """Read-only / idempotent tools (Screenshot, GetText, ReportProgress…) + are exempt — the model is allowed to call them in tight loops.""" + same = _key("BrowserScreenshot") + history = [same, same, same, same] # already at threshold + assert _detect_loop(history, same) is False + + +def test_detect_loop_third_repeat_triggers(): + """Threshold is 3 occurrences within the sliding window. Two + repeats is fine; the third (counting the new call) is the loop.""" + same = _key("BrowserClick") + different = _key("BrowserType", suffix='{"text":"x"}') + + # 1st & 2nd occurrence in history; the new call is the 3rd. + assert _detect_loop([same, same, different], same) is True + + # Only 1 prior occurrence + the new = 2 total → not yet a loop. + assert _detect_loop([same], same) is False + + +def test_detect_loop_two_identical_plus_one_different_in_window_is_false(): + """Two repeats interleaved with other calls don't add up to a loop.""" + same = _key("BrowserClick") + other = _key("BrowserScroll", suffix='{"text":"a"}') + # Window: [same, other, same]. New call=same → 3 total inc. new. + # Wait — that IS at threshold. Use a lower count instead. + assert _detect_loop([same, other], same) is False + + +def test_detect_loop_window_size_caps_lookback(): + """`_LOOP_WINDOW_SIZE - 1 = 4` past entries are considered, plus the + new call (= 5 total). Anything older falls outside the window.""" + same = _key("BrowserClick") + other = _key("BrowserScroll") + # 5 oldest matches but they're outside the window of 4 prior + 1 new. + history = [same] * 5 + [other, other, other, other] + # New call=same. Window = last 4 history + new → [other, other, other, other, same]. + # Only 1 match for `same` → not a loop. + assert _detect_loop(history, same) is False + + +def test_detect_loop_constants_match_expectations(): + """Sanity-check the module constants the integration tests rely on + so a refactor that loosens the threshold can't silently invalidate + coverage downstream.""" + assert _LOOP_WINDOW_SIZE == 5 + assert _LOOP_REPEAT_THRESHOLD == 3 + assert _LOOP_HARD_CAP == 5 + assert "LOOP DETECTED" in _LOOP_WARNING_TEXT + + +# --------------------------------------------------------------------------- +# _validate_message_pairing +# --------------------------------------------------------------------------- + + +def test_validate_message_pairing_empty_is_valid(): + assert _validate_message_pairing([]) is True + + +def test_validate_message_pairing_string_content_skipped(): + """User messages with plain string content carry no tool_result + blocks, so they have nothing to validate.""" + msgs = [ + {"role": "user", "content": "hello"}, + {"role": "assistant", "content": "world"}, + ] + assert _validate_message_pairing(msgs) is True + + +def test_validate_message_pairing_proper_pairing_passes(): + msgs = [ + {"role": "user", "content": "do it"}, + {"role": "assistant", "content": [ + {"type": "text", "text": "thinking"}, + {"type": "tool_use", "id": "t1", "name": "BrowserClick", "input": {}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": []}, + ]}, + ] + assert _validate_message_pairing(msgs) is True + + +def test_validate_message_pairing_orphan_tool_result_fails(): + """tool_result whose tool_use_id was never declared by a prior + assistant tool_use → invalid (Anthropic API would 400).""" + msgs = [ + {"role": "user", "content": "do it"}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t-orphan", "content": []}, + ]}, + ] + assert _validate_message_pairing(msgs) is False + + +def test_validate_message_pairing_multiple_tool_uses_one_orphan_fails(): + """Even with several valid pairings, a single orphan must invalidate + the whole history (we drop the cache wholesale, not partially).""" + msgs = [ + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "BrowserClick", "input": {}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": []}, + {"type": "tool_result", "tool_use_id": "t-missing", "content": []}, + ]}, + ] + assert _validate_message_pairing(msgs) is False + + +# --------------------------------------------------------------------------- +# _is_fresh_user_message +# --------------------------------------------------------------------------- + + +def test_is_fresh_user_message_string_content(): + assert _is_fresh_user_message({"role": "user", "content": "hi"}) is True + + +def test_is_fresh_user_message_text_only_list_is_fresh(): + msg = {"role": "user", "content": [{"type": "text", "text": "hi"}]} + assert _is_fresh_user_message(msg) is True + + +def test_is_fresh_user_message_with_tool_result_is_not_fresh(): + """A user message containing a tool_result is mid-turn — cutting + here would orphan the prior assistant tool_use.""" + msg = {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": []}, + ]} + assert _is_fresh_user_message(msg) is False + + +def test_is_fresh_user_message_assistant_role_is_never_fresh(): + msg = {"role": "assistant", "content": "world"} + assert _is_fresh_user_message(msg) is False + + +def test_is_fresh_user_message_empty_list_is_fresh(): + """A degenerate empty content list contains no tool_result → still + a safe cut point under the helper's contract.""" + msg = {"role": "user", "content": []} + assert _is_fresh_user_message(msg) is True + + +# --------------------------------------------------------------------------- +# _summarize_messages +# --------------------------------------------------------------------------- + + +def test_summarize_messages_empty_returns_empty_string(): + assert _summarize_messages([]) == "" + + +def test_summarize_messages_extracts_initial_user_task(): + msgs = [ + {"role": "user", "content": "swipe right ten times"}, + {"role": "assistant", "content": [{"type": "text", "text": "ok"}]}, + ] + summary = _summarize_messages(msgs) + assert 'Original task: "swipe right ten times"' in summary + assert "Summary of earlier browser-agent activity" in summary + + +def test_summarize_messages_truncates_initial_task_at_300_chars(): + long_task = "x" * 600 + msgs = [{"role": "user", "content": long_task}] + summary = _summarize_messages(msgs) + # The quoted task substring must be at most 300 chars. + start = summary.index('Original task: "') + len('Original task: "') + end = summary.index('"', start) + assert end - start <= 300 + + +def test_summarize_messages_counts_tool_calls_with_key_param(): + """Tool calls are bucketed by name; key params are extracted in the + priority order index → key → url → selector → direction → text.""" + msgs = [ + {"role": "user", "content": "do stuff"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "BrowserClick", + "input": {"selector": "#a"}}, + {"type": "tool_use", "id": "t2", "name": "BrowserClick", + "input": {"selector": "#b"}}, + {"type": "tool_use", "id": "t3", "name": "BrowserNavigate", + "input": {"url": "https://example.com"}}, + ]}, + ] + summary = _summarize_messages(msgs) + assert "Actions taken (3 total):" in summary + # BrowserClick appears twice → "(×2)" suffix on its bucket line. + assert "BrowserClick(selector=#b) (×2)" in summary + assert "BrowserNavigate(url=https://example.com)" in summary + + +def test_summarize_messages_includes_recent_brain_states(): + """ReportProgress.next_goal entries surface under 'Recent intents:' + (last 5 only).""" + blocks = [] + for i in range(7): + blocks.append({ + "type": "tool_use", "id": f"r{i}", "name": "ReportProgress", + "input": { + "evaluation_previous": "ok", + "working_memory": "...", + "next_goal": f"goal-{i}", + }, + }) + msgs = [{"role": "assistant", "content": blocks}] + summary = _summarize_messages(msgs) + assert "Recent intents:" in summary + # Last 5 of 7 → goal-2..goal-6 included; goal-0/goal-1 dropped. + for i in (2, 3, 4, 5, 6): + assert f"goal-{i}" in summary + assert "goal-0" not in summary + assert "goal-1" not in summary + + +def test_summarize_messages_includes_last_assistant_text(): + msgs = [ + {"role": "assistant", "content": [{"type": "text", "text": "first thought"}]}, + {"role": "assistant", "content": [{"type": "text", "text": "final thought"}]}, + ] + summary = _summarize_messages(msgs) + assert "Last update from assistant: final thought" in summary + + +def test_summarize_messages_truncates_last_assistant_text_at_400(): + long = "y" * 1000 + msgs = [{"role": "assistant", "content": [{"type": "text", "text": long}]}] + summary = _summarize_messages(msgs) + line = next(l for l in summary.split("\n") if l.startswith("Last update")) + payload = line[len("Last update from assistant: "):] + assert len(payload) <= 400 + + +# --------------------------------------------------------------------------- +# _trim_history_by_turns +# --------------------------------------------------------------------------- + + +def test_trim_history_under_cap_returns_copy_unchanged(): + msgs = [ + {"role": "user", "content": "a"}, + {"role": "assistant", "content": "b"}, + ] + out = _trim_history_by_turns(msgs, max_messages=10) + assert out == msgs + # Returns a new list (not the same object) — defensive copy. + assert out is not msgs + + +def test_trim_history_compacts_when_clean_cut_exists(): + """The classic case: long history, but there's a fresh user-text + message later in the list → compact prefix into a single summary.""" + msgs = [ + {"role": "user", "content": "task one"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "BrowserClick", "input": {"selector": "#a"}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": []}, + ]}, + {"role": "assistant", "content": [{"type": "text", "text": "done one"}]}, + # Clean boundary: a brand-new user-text message + {"role": "user", "content": "task two"}, + {"role": "assistant", "content": [{"type": "text", "text": "done two"}]}, + ] + out = _trim_history_by_turns(msgs, max_messages=3) + # First message is the synthetic summary, then the verbatim tail. + assert out[0]["role"] == "user" + assert isinstance(out[0]["content"], str) + assert "Summary of earlier browser-agent activity" in out[0]["content"] + assert "task one" in out[0]["content"] + assert out[1:] == msgs[4:] + + +def test_trim_history_no_clean_cut_returns_original(): + """If every user message after [0] is a tool_result (no clean + boundary anywhere), the helper returns the input unchanged rather + than corrupting the conversation.""" + msgs = [ + {"role": "user", "content": "start"}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "BrowserClick", "input": {}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": []}, + ]}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t2", "name": "BrowserClick", "input": {}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t2", "content": []}, + ]}, + ] + out = _trim_history_by_turns(msgs, max_messages=2) + assert out == msgs + + +def test_trim_history_falls_back_to_latest_cut_when_current_turn_too_big(): + """If the only clean cut leaves a tail that still exceeds the cap, + we still take that latest cut — better some compaction than none.""" + msgs = [ + {"role": "user", "content": "first task"}, + {"role": "assistant", "content": [{"type": "text", "text": "ack"}]}, + # Clean boundary at index 2 + {"role": "user", "content": "second task"}, + # The "current" turn alone is now bigger than the cap. + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t1", "name": "BrowserClick", "input": {}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t1", "content": []}, + ]}, + {"role": "assistant", "content": [ + {"type": "tool_use", "id": "t2", "name": "BrowserClick", "input": {}}, + ]}, + {"role": "user", "content": [ + {"type": "tool_result", "tool_use_id": "t2", "content": []}, + ]}, + ] + out = _trim_history_by_turns(msgs, max_messages=3) + # Even though tail (5 msgs) exceeds cap (3), we still compact + # everything before the latest clean boundary at index 2. + assert out[0]["role"] == "user" and isinstance(out[0]["content"], str) + assert "first task" in out[0]["content"] + assert out[1:] == msgs[2:] + + +# --------------------------------------------------------------------------- +# _format_tool_result +# --------------------------------------------------------------------------- + + +def test_format_tool_result_error_returns_text_block(): + out = _format_tool_result({"error": "no browser connected"}, "BrowserClick") + assert out == [{"type": "text", "text": "Error: no browser connected"}] + + +def test_format_tool_result_screenshot_with_image_emits_image_and_text(): + """BrowserScreenshot results carry a base64 PNG; the formatter + must wrap it in an Anthropic `image` content block.""" + out = _format_tool_result( + {"image": "AAAA==", "url": "https://example.com"}, + "BrowserScreenshot", + ) + assert len(out) == 2 + assert out[0]["type"] == "image" + assert out[0]["source"]["type"] == "base64" + assert out[0]["source"]["media_type"] == "image/png" + assert out[0]["source"]["data"] == "AAAA==" + assert out[1]["type"] == "text" + assert "https://example.com" in out[1]["text"] + + +def test_format_tool_result_screenshot_without_image_falls_back_to_text(): + """If a screenshot tool somehow returns no image, fall through to + the standard text-block path rather than emitting an empty image.""" + out = _format_tool_result({"text": "no image captured"}, "BrowserScreenshot") + assert out == [{"type": "text", "text": "no image captured"}] + + +def test_format_tool_result_text_field_passes_through(): + out = _format_tool_result({"text": "clicked successfully"}, "BrowserClick") + assert out == [{"type": "text", "text": "clicked successfully"}] + + +def test_format_tool_result_no_text_falls_back_to_json_dump(): + """Last-resort path: arbitrary dict → JSON-stringified text block.""" + out = _format_tool_result({"some": "shape", "value": 1}, "BrowserEvaluate") + assert len(out) == 1 + assert out[0]["type"] == "text" + # Should be a JSON dump containing both keys. + payload = json.loads(out[0]["text"]) + assert payload == {"some": "shape", "value": 1} + + +# --------------------------------------------------------------------------- +# execute_browser_tool (async) +# --------------------------------------------------------------------------- + + +async def test_execute_browser_tool_unknown_tool_returns_error(): + """Tools not in ACTION_MAP must short-circuit — the frontend has + no handler for them and would silently drop the command.""" + out = await execute_browser_tool("BrowserNotARealTool", {}, "b-1", "") + assert out == {"error": "Unknown browser tool: BrowserNotARealTool"} + + +async def test_execute_browser_tool_dispatches_through_ws_manager(): + """Known tool → ws_manager.send_browser_command awaited with the + mapped action, browser_id, params copy, and a tab_id.""" + captured: dict = {} + + async def _fake_send(request_id, action, browser_id, params, tab_id=""): + captured.update({ + "request_id": request_id, + "action": action, + "browser_id": browser_id, + "params": params, + "tab_id": tab_id, + }) + return {"text": "ok"} + + with patch.object(ba.ws_manager, "send_browser_command", + AsyncMock(side_effect=_fake_send)) as mock_send: + result = await execute_browser_tool( + "BrowserNavigate", {"url": "https://x.com"}, "b-42", "tab-7", + ) + assert result == {"text": "ok"} + mock_send.assert_awaited_once() + + # Tool name → mapped action via ACTION_MAP + assert captured["action"] == "navigate" + assert captured["browser_id"] == "b-42" + assert captured["tab_id"] == "tab-7" + assert captured["params"] == {"url": "https://x.com"} + # request_id is a hex uuid4 → 32-char lowercase hex + assert isinstance(captured["request_id"], str) + assert len(captured["request_id"]) == 32 + assert all(c in "0123456789abcdef" for c in captured["request_id"]) + + +async def test_execute_browser_tool_each_action_map_entry_resolves(): + """Smoke-cover ACTION_MAP: every declared schema tool that's mapped + must dispatch with its canonical action string.""" + seen: list[str] = [] + + async def _fake_send(request_id, action, browser_id, params, tab_id=""): + seen.append(action) + return {"text": "ok"} + + with patch.object(ba.ws_manager, "send_browser_command", + AsyncMock(side_effect=_fake_send)): + for tool_name, action in ba.ACTION_MAP.items(): + seen.clear() + await execute_browser_tool(tool_name, {}, "b-x") + assert seen == [action], f"{tool_name} routed to {seen}, expected {[action]}"