From 3aa9cf6ba08b14521b7cc26afeec92d6c7dbc96b Mon Sep 17 00:00:00 2001 From: ciregenz Date: Tue, 2 Jun 2026 14:02:47 -0700 Subject: [PATCH] [eric] browser: front-load page perception into turn 1 to cut orientation round-trips --- backend/apps/agents/browser/browser_agent.py | 26 +++++++++++++++++++- backend/tests/test_browser_agent_loop.py | 18 ++++++++++++++ 2 files changed, 43 insertions(+), 1 deletion(-) diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 7060d1f0..2d71bb5c 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -194,11 +194,31 @@ async def run_browser_agent( "session": session.model_dump(mode="json"), }) + # Perception we prefetch on a known starting page so the model can ACT on + # turn 1 instead of spending turns 0-2 orienting (screenshot/get_elements). + # Pure speed: it's the same reads the agent would do anyway, just front-loaded. + preloaded_perception = "" if initial_url: nav_result = await execute_browser_tool( "BrowserNavigate", {"url": initial_url}, browser_id, tab_id, ) logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}") + try: + li = await execute_browser_tool("BrowserListInteractives", {}, browser_id, tab_id) + gt = await execute_browser_tool("BrowserGetText", {}, browser_id, tab_id) + parts = [] + if li.get("text") and "error" not in li: + parts.append("Interactive elements already on the page:\n" + str(li["text"])) + if gt.get("text") and "error" not in gt: + parts.append("Visible page text (truncated):\n" + str(gt["text"])[:2000]) + if parts: + preloaded_perception = ( + "\n\n[Page already loaded and inspected for you, act directly; " + "no need to screenshot or list elements again unless it changes]\n" + + "\n\n".join(parts) + ) + except Exception as e: + logger.debug(f"[browser-perf] perception prefetch skipped: {e}") from backend.apps.settings.settings import load_settings from backend.apps.settings.credentials import get_anthropic_client_for_model @@ -269,7 +289,11 @@ async def run_browser_agent( ) clear_browser_history(browser_id) prior_messages = [] - messages: list[dict] = list(prior_messages) + [{"role": "user", "content": task}] + # Front-load the prefetched perception into the first user turn so the model + # can act immediately (only when this is a fresh conversation; a resumed one + # already knows the page). The visible task text stays clean. + first_user_content = task + preloaded_perception if (preloaded_perception and not prior_messages) else task + messages: list[dict] = list(prior_messages) + [{"role": "user", "content": first_user_content}] action_log: list[dict] = [] final_screenshot: str | None = None metrics_started_at = time.time() # wall-clock start for per-task timing diff --git a/backend/tests/test_browser_agent_loop.py b/backend/tests/test_browser_agent_loop.py index d40ee713..0287c049 100644 --- a/backend/tests/test_browser_agent_loop.py +++ b/backend/tests/test_browser_agent_loop.py @@ -268,6 +268,24 @@ def test_replay_falls_back_to_full_agent_when_a_step_fails(monkeypatch): assert len(primary.calls) > 0, "fell back to the full LLM agent" +def test_perception_is_frontloaded_into_first_turn(monkeypatch): + # With a known start URL, the agent should prefetch the element list + page + # text and put them in the FIRST user message, so the model can act on turn 1 + # instead of spending early turns orienting. + BH._browser_history.clear(); BH._domain_notes.clear() + primary = FakeLLM([Resp([Blk("text", "done")], stop_reason="end_turn")]) + aux = FakeAux() + _install(monkeypatch, primary, aux) + asyncio.run(BA.run_browser_agent( + task="click submit", browser_id="bp", model="sonnet", initial_url=DOC_URL, + )) + first_user = primary.calls[0]["messages"][0]["content"] + text = first_user if isinstance(first_user, str) else json.dumps(first_user) + # the fake list_interactives returns a "[1]