diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index b7748d25..29d75b5a 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -59,6 +59,7 @@ from backend.apps.agents.browser import browser_metrics from backend.apps.agents.browser import browser_playbook from backend.apps.agents.browser import browser_skills from backend.apps.agents.browser import browser_wait +from backend.apps.agents.browser import browser_schema from backend.apps.agents.browser.browser_schema import ( _ACTION_TOOLS_REQUIRING_REPORT, ACTION_MAP, @@ -504,7 +505,7 @@ async def run_browser_agent( "type": "text", "text": run_system_prompt, "cache_control": {"type": "ephemeral"}, }] - _cached_tools = [dict(t) for t in BROWSER_TOOLS_SCHEMA] + _cached_tools = [dict(t) for t in browser_schema.MODEL_VISIBLE_TOOLS] if _cached_tools: _cached_tools[-1] = {**_cached_tools[-1], "cache_control": {"type": "ephemeral"}} @@ -766,6 +767,9 @@ async def run_browser_agent( # under-batching telemetry + nudge state single_action_streak = 0 batching_nudges = 0 + redundant_read_nudges = 0 + # True after a mutating action attaches fresh state; a solo read next is waste + fresh_state_pending = False multi_action_turns = 0 batch_calls = 0 try: @@ -1227,6 +1231,9 @@ async def run_browser_agent( continue start = time.time() + # did the PREVIOUS action already hand us fresh page state? a solo + # re-read now is a wasted round-trip; remembered before we overwrite it + _had_fresh_state = fresh_state_pending tool_input = tu.input if tu.name == "BrowserListInteractives" and current_next_goal: tool_input = {**tu.input, "goal": current_next_goal} @@ -1306,6 +1313,10 @@ async def run_browser_agent( ) if _auto_state: result["text"] = f"{result.get('text') or ''}{_auto_state}" + # a mutation attached fresh state; it stays "available" through + # intervening reads (Wait/Extract don't invalidate it), so a + # later solo re-list is still caught as redundant. + fresh_state_pending = True # Deferred replay re-check: the orchestrator often opens a fresh # card on the wrong host, so the dispatch-time replay missed. Once @@ -1418,6 +1429,24 @@ async def run_browser_agent( "conservative batch costs nothing. Keep irreversible steps " "(Send/Submit/Pay/Post) solo." )}] + + # Redundant-read nudge: the previous action already attached a fresh + # element list, and this turn spent a whole round-trip re-reading it. + # Reads are the biggest turn sink (measured ~16 of 25 turns). + if (tu.name in ("BrowserListInteractives", "BrowserGetText") + and _had_fresh_state and "error" not in result): + redundant_read_nudges += 1 + fresh_state_pending = False # nudge once per attached-state cluster + logger.info( + f"[browser-batching {session_id}] redundant-read nudge #{redundant_read_nudges} " + f"({tu.name}) at turn {turn}" + ) + content_blocks = content_blocks + [{"type": "text", "text": ( + "\n\n⚡ Your PREVIOUS action already ended with a fresh '[page state after " + "action]' element list, so this re-read cost a round-trip for nothing. Act " + "straight from the attached state; only re-read after it says it was truncated " + "or you genuinely changed the page in a way that list wouldn't reflect." + )}] # Deterministic nudging exhausted: ONE cheap aux adjudication # to suggest a concrete next step before we keep failing. if stagnation_exhausted(stagnation_streak) and not aux_adjudicated: @@ -1549,7 +1578,7 @@ async def run_browser_agent( logger.info( f"[browser-batching {session_id}] run summary: turns={turn + 1} " f"multi_action_turns={multi_action_turns} batch_calls={batch_calls} " - f"nudges={batching_nudges}" + f"nudges={batching_nudges} redundant_reads={redundant_read_nudges}" ) browser_metrics.record_task(session_id, browser_id, task, final_status, metrics_started_at, turn + 1, action_log, session.tokens, diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index 0d82174a..ddd70743 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -264,14 +264,14 @@ BROWSER_TOOLS_SCHEMA = [ { "name": "BrowserClickIndex", "description": ( - "Click an element by its numeric index from BrowserListInteractives. " + "SOLO click, reserved for two cases: (1) the final IRREVERSIBLE step " + "(Send/Submit/Pay/Post), always alone with `expect` proof, and (2) " + "filling a text box via `text` (focused directly by node, so it works " + "inside messaging/compose overlays where coordinate clicks miss). " + "Routine clicks belong inside BrowserBatch as click_index sub-actions. " "Uses native OS-level mouse events (event.isTrusted=true) so it works " - "on sites that filter out synthetic JS events. Always call " - "BrowserListInteractives first to get a fresh index list. If the click " - "returns 'index no longer valid', the page changed; re-list and retry. " - "If the index is a TEXT BOX (role textbox or searchbox), it is " - "focused directly by node (no coordinates, so it works inside messaging/" - "compose overlays where clicks miss); pass `text` to fill it in one call." + "on sites that filter synthetic JS events. If the click returns " + "'index no longer valid', the page changed; re-list and retry." ), "input_schema": { "type": "object", @@ -296,12 +296,13 @@ BROWSER_TOOLS_SCHEMA = [ { "name": "BrowserBatch", "description": ( - "Run a sequence of browser actions in one tool call. Each sub-action " - "is executed in order, with the URL captured before/after each one. " - "If the URL changes mid-batch (the page navigated), the rest of the " - "batch is aborted and you get a partial result. Use this when you " - "have a known sequence; typing then pressing Enter, swiping multiple " - "times, clicking through pagination. Max 5 actions per batch.\n\n" + "Your standard way to ACT on the page. Every mutation (navigate, " + "click, type, press, scroll) is a sub-action in this array, 1-5 per " + "call; an array of one is fine when that is genuinely all you know. " + "Each sub-action executes in order with the URL captured before/after. " + "If the URL changes mid-batch (the page navigated), the rest is " + "aborted and you get a partial result plus fresh state, so a " + "conservative batch costs nothing.\n\n" "Sub-action types and their params:\n" "- click_index: { index: int }\n" "- press_key: { key: str }\n" @@ -547,6 +548,14 @@ BROWSER_TOOLS_SCHEMA = [ }, ] +# Schema-forced batching: the model ignored every prompt-level batching +# invitation (0 adoptions across 8 measured runs), so the single-step mutating +# tools are not offered to it at all; acting means a BrowserBatch array, and +# the one deliberate solo path is BrowserClickIndex (irreversible step with +# expect, or a text-box fill). Executors and replay still support everything. +_SOLO_MUTATORS_HIDDEN = {"BrowserNavigate", "BrowserClick", "BrowserType", "BrowserScroll", "BrowserPressKey"} +MODEL_VISIBLE_TOOLS = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] not in _SOLO_MUTATORS_HIDDEN] + ACTION_MAP = { "BrowserScreenshot": "screenshot", "BrowserGetText": "get_text", @@ -598,9 +607,8 @@ SYSTEM_PROMPT = ( "waits. Fall back to the UI only when you don't know the site's URL pattern.\n\n" "## Required output structure: ReportProgress before every action\n" - "Before ANY action tool (BrowserClick, BrowserType, BrowserNavigate, " - "BrowserPressKey, BrowserScroll, BrowserEvaluate, BrowserClickIndex, " - "BrowserBatch), you MUST call the ReportProgress tool in the SAME turn. " + "Before ANY action tool (BrowserBatch, BrowserClickIndex, BrowserEvaluate), " + "you MUST call the ReportProgress tool in the SAME turn. " "ReportProgress takes three short fields:\n" "- evaluation_previous: did your last action work? what changed on the page?\n" "- working_memory: what have you learned about this site? what worked, what didn't?\n" @@ -648,7 +656,7 @@ SYSTEM_PROMPT = ( "## Try multiple strategies, learn from failures\n" "Sites vary wildly. When one approach fails, switch tactics; don't retry the same " "thing. The escalation ladder, fastest to slowest:\n" - "1. **Keyboard shortcuts via BrowserPressKey**; fastest and most reliable on sites " + "1. **Keyboard shortcuts via press_key sub-actions**; fastest and most reliable on sites " "that support them (Tinder swipes, Gmail navigation, Slack message jump, etc.). " "Always check if the site shows keyboard hints in the UI before falling back to clicks. " "BrowserPressKey sends real native events that pass the `event.isTrusted` check, so " @@ -672,9 +680,9 @@ SYSTEM_PROMPT = ( "To FILL a text box (a `` like a message/compose field, including ones inside " "a messaging overlay where clicks miss), call BrowserClickIndex on it with a `text` arg: " "it focuses the box by node and types the whole string in ONE call, no coordinates, no " - "character-by-character. Then send with BrowserPressKey 'Enter' or the Send button.\n" + "character-by-character. Then send with a press_key 'Enter' sub-action or the Send button.\n" "3. **Semantic CSS selectors**; `button[aria-label='X']`, `[role='button']`, " - "`a[href*='...']`. Try these via BrowserGetElements + BrowserClick when the site " + "`a[href*='...']`. Try these via BrowserGetElements + a click sub-action when the site " "actually has semantic HTML.\n" "4. **Text-based JS query**; when both of the above fail, use BrowserEvaluate to " "find elements by visible text: `Array.from(document.querySelectorAll('*')).find(el => el.textContent.trim() === 'Like')`.\n" @@ -682,27 +690,21 @@ SYSTEM_PROMPT = ( "button visually, then click by approximate coords.\n\n" "## Speed: fewer turns is the #1 driver\n" - "Turns are slow model round-trips; tools are fast, so go fast by taking FEWER " - "turns. BATCH BY DEFAULT: every action result already ends with fresh page " - "state, so when that state shows you the next 2-3 steps, emit them as ONE " - "BrowserBatch instead of one tool per turn; single-action turns are for when " - "the next step genuinely depends on something you haven't seen. Good batches: " - "filling a form (type, type, click Next); a repeated action (5 swipes, 3 " - "scrolls); a deterministic flow (type query, press Enter, click first result); " - "or act-then-read by ending the batch with list_interactives, so click, wait, " - "list_interactives is ONE turn not three. Max 5 sub-actions. The batch runs " - "them in order and STOPS at the first that fails or if the URL changes, so " - "order them safely (never a maybe-failing step before a must-do one); a safe " - "stop means the rest is skipped and you just continue from the fresh state, so " - "a conservative batch costs you nothing. Go solo ONLY when you cannot yet NAME " - "the next action's target: its element is not in the state you are reading and " - "its name is not predictable. Needing to see a result AFTER acting is never a " - "reason to go solo (results auto-attach fresh state, and a failed or page-" - "changing step stops the batch safely). If your last 2 turns were single " - "actions whose targets were already visible beforehand, you are under-" - "batching: batch the next sequence. EXCEPTION: an irreversible step " - "(Send/Submit/Pay/Post) is always its own single action with `expect` proof, " - "never inside a batch.\n\n" + "Turns are slow model round-trips; tools are fast. You act through " + "BrowserBatch: an array of 1-5 actions per call (navigate, click_index, " + "type, press_key, scroll), executed in order with settle between steps. " + "When the state you are reading already names your next 2-3 targets, put " + "them in ONE array; an array of one is fine when the next step genuinely " + "depends on something you haven't seen. Good arrays: a deterministic flow " + "(type query, press Enter, click first result); a repeated action (5 " + "swipes); act-then-read by ending with list_interactives so click, wait, " + "read is ONE turn not three. The batch STOPS at the first failing or URL-" + "changing step and you continue from the fresh attached state, so a " + "conservative array costs nothing; order steps safely (never a maybe-" + "failing step before a must-do one). TWO solo exceptions, both via " + "BrowserClickIndex: the final irreversible step (Send/Submit/Pay/Post) " + "always alone with `expect` proof, and text-box fills via its `text` " + "param.\n\n" "## Doing the SAME flow for many inputs? Use BrowserRepeatFlow\n" "If you're about to repeat the same mechanical flow for a list (read 10 " @@ -719,7 +721,7 @@ SYSTEM_PROMPT = ( "- Do NOT screenshot after every single action. Screenshot ONLY when you genuinely " "don't know the page state (start of task, after navigation, after a failure).\n" "- NEVER call BrowserWait or BrowserListInteractives right after an action. Every " - "mutating action (navigate, click, type, press, scroll, batch) already settles on its " + "mutating action (every batch and every solo click) already settles on its " "own (waits for network/DOM quiet) and its result ends with '[page state after " "action]': a fresh numbered element list. To keep results lean it may show ONLY the " "rows that changed plus a count of unchanged ones; unchanged rows keep their numbers, " @@ -727,14 +729,14 @@ SYSTEM_PROMPT = ( "re-listing after an action is a wasted turn. Reserve BrowserWait for content you " "KNOW arrives late (pass `until` with the exact thing you expect), and " "BrowserListInteractives for re-orienting or when the attached list was truncated.\n" - "- When scrolling, stop as soon as BrowserScroll reports atTop/atBottom or a 0 delta; " + "- When scrolling, stop as soon as a scroll sub-action reports atTop/atBottom or a 0 delta; " "don't loop past the end.\n" "- Do NOT call BrowserGetElements on the entire body if you already know roughly " "where the target is. Scope it: `BrowserGetElements({selector: 'nav'})`.\n" "- Do NOT call the same failing tool twice with identical parameters. If selector " "X failed, try a DIFFERENT selector or a DIFFERENT strategy.\n" "- For repeated actions (swiping through profiles, going through inbox messages), " - "use BrowserPressKey if available; it's an order of magnitude faster than DOM clicks.\n" + "use press_key sub-actions if available; an order of magnitude faster than DOM clicks.\n" "- To RE-READ data you already loaded once (search results, a list, a detail page), " "check BrowserListRoutes and use BrowserReplayRoute to fetch it straight from the " "site's API instead of re-navigating and re-scraping; it's much faster. This is for "