mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-08 02:37:45 +02:00
[eric] browser: front-load page perception into turn 1 to cut orientation round-trips
This commit is contained in:
@@ -194,11 +194,31 @@ async def run_browser_agent(
|
||||
"session": session.model_dump(mode="json"),
|
||||
})
|
||||
|
||||
# Perception we prefetch on a known starting page so the model can ACT on
|
||||
# turn 1 instead of spending turns 0-2 orienting (screenshot/get_elements).
|
||||
# Pure speed: it's the same reads the agent would do anyway, just front-loaded.
|
||||
preloaded_perception = ""
|
||||
if initial_url:
|
||||
nav_result = await execute_browser_tool(
|
||||
"BrowserNavigate", {"url": initial_url}, browser_id, tab_id,
|
||||
)
|
||||
logger.info(f"Browser agent {session_id}: navigated to {initial_url}: {nav_result.get('text', nav_result.get('error', ''))}")
|
||||
try:
|
||||
li = await execute_browser_tool("BrowserListInteractives", {}, browser_id, tab_id)
|
||||
gt = await execute_browser_tool("BrowserGetText", {}, browser_id, tab_id)
|
||||
parts = []
|
||||
if li.get("text") and "error" not in li:
|
||||
parts.append("Interactive elements already on the page:\n" + str(li["text"]))
|
||||
if gt.get("text") and "error" not in gt:
|
||||
parts.append("Visible page text (truncated):\n" + str(gt["text"])[:2000])
|
||||
if parts:
|
||||
preloaded_perception = (
|
||||
"\n\n[Page already loaded and inspected for you, act directly; "
|
||||
"no need to screenshot or list elements again unless it changes]\n"
|
||||
+ "\n\n".join(parts)
|
||||
)
|
||||
except Exception as e:
|
||||
logger.debug(f"[browser-perf] perception prefetch skipped: {e}")
|
||||
|
||||
from backend.apps.settings.settings import load_settings
|
||||
from backend.apps.settings.credentials import get_anthropic_client_for_model
|
||||
@@ -269,7 +289,11 @@ async def run_browser_agent(
|
||||
)
|
||||
clear_browser_history(browser_id)
|
||||
prior_messages = []
|
||||
messages: list[dict] = list(prior_messages) + [{"role": "user", "content": task}]
|
||||
# Front-load the prefetched perception into the first user turn so the model
|
||||
# can act immediately (only when this is a fresh conversation; a resumed one
|
||||
# already knows the page). The visible task text stays clean.
|
||||
first_user_content = task + preloaded_perception if (preloaded_perception and not prior_messages) else task
|
||||
messages: list[dict] = list(prior_messages) + [{"role": "user", "content": first_user_content}]
|
||||
action_log: list[dict] = []
|
||||
final_screenshot: str | None = None
|
||||
metrics_started_at = time.time() # wall-clock start for per-task timing
|
||||
|
||||
@@ -268,6 +268,24 @@ def test_replay_falls_back_to_full_agent_when_a_step_fails(monkeypatch):
|
||||
assert len(primary.calls) > 0, "fell back to the full LLM agent"
|
||||
|
||||
|
||||
def test_perception_is_frontloaded_into_first_turn(monkeypatch):
|
||||
# With a known start URL, the agent should prefetch the element list + page
|
||||
# text and put them in the FIRST user message, so the model can act on turn 1
|
||||
# instead of spending early turns orienting.
|
||||
BH._browser_history.clear(); BH._domain_notes.clear()
|
||||
primary = FakeLLM([Resp([Blk("text", "done")], stop_reason="end_turn")])
|
||||
aux = FakeAux()
|
||||
_install(monkeypatch, primary, aux)
|
||||
asyncio.run(BA.run_browser_agent(
|
||||
task="click submit", browser_id="bp", model="sonnet", initial_url=DOC_URL,
|
||||
))
|
||||
first_user = primary.calls[0]["messages"][0]["content"]
|
||||
text = first_user if isinstance(first_user, str) else json.dumps(first_user)
|
||||
# the fake list_interactives returns a "[1]<button ...>" listing
|
||||
assert "Interactive elements already on the page" in text
|
||||
assert "act directly" in text
|
||||
|
||||
|
||||
def test_prompt_caching_markers_present(monkeypatch):
|
||||
# The fixed system+tools prefix must carry cache_control so it's cached
|
||||
# across turns (the first-run speed/cost win). Without the marker the
|
||||
|
||||
Reference in New Issue
Block a user