WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-08-22 20:52:23 +02:00
Code Issues Packages Projects Releases Wiki Activity

[eric] browser: re-land schema-forced batching + redundant-read nudge (zombie forensics exonerated both)

Browse Source
This commit is contained in:
ciregenz
2026-06-05 22:16:40 -07:00
parent e7099cc888
commit 9af337585b
2 changed files with 76 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/agents/browser/browser_agent.py
+31 -2
View File
@@ -59,6 +59,7 @@ from backend.apps.agents.browser import browser_metrics
from backend.apps.agents.browser import browser_playbook
from backend.apps.agents.browser import browser_skills
from backend.apps.agents.browser import browser_wait
from backend.apps.agents.browser import browser_schema
from backend.apps.agents.browser.browser_schema import (
_ACTION_TOOLS_REQUIRING_REPORT,
ACTION_MAP,
@@ -504,7 +505,7 @@ async def run_browser_agent(
"type": "text", "text": run_system_prompt,
"cache_control": {"type": "ephemeral"},
}]
_cached_tools = [dict(t) for t in BROWSER_TOOLS_SCHEMA]
_cached_tools = [dict(t) for t in browser_schema.MODEL_VISIBLE_TOOLS]
if _cached_tools:
_cached_tools[-1] = {**_cached_tools[-1], "cache_control": {"type": "ephemeral"}}
@@ -766,6 +767,9 @@ async def run_browser_agent(
# under-batching telemetry + nudge state
single_action_streak = 0
batching_nudges = 0
redundant_read_nudges = 0
# True after a mutating action attaches fresh state; a solo read next is waste
fresh_state_pending = False
multi_action_turns = 0
batch_calls = 0
try:
@@ -1227,6 +1231,9 @@ async def run_browser_agent(
continue
start = time.time()
# did the PREVIOUS action already hand us fresh page state? a solo
# re-read now is a wasted round-trip; remembered before we overwrite it
_had_fresh_state = fresh_state_pending
tool_input = tu.input
if tu.name == "BrowserListInteractives" and current_next_goal:
tool_input = {**tu.input, "goal": current_next_goal}
@@ -1306,6 +1313,10 @@ async def run_browser_agent(
)
if _auto_state:
result["text"] = f"{result.get('text') or ''}{_auto_state}"
# a mutation attached fresh state; it stays "available" through
# intervening reads (Wait/Extract don't invalidate it), so a
# later solo re-list is still caught as redundant.
fresh_state_pending = True
# Deferred replay re-check: the orchestrator often opens a fresh
# card on the wrong host, so the dispatch-time replay missed. Once
@@ -1418,6 +1429,24 @@ async def run_browser_agent(
"conservative batch costs nothing. Keep irreversible steps "
"(Send/Submit/Pay/Post) solo."
)}]
# Redundant-read nudge: the previous action already attached a fresh
# element list, and this turn spent a whole round-trip re-reading it.
# Reads are the biggest turn sink (measured ~16 of 25 turns).
if (tu.name in ("BrowserListInteractives", "BrowserGetText")
and _had_fresh_state and "error" not in result):
redundant_read_nudges += 1
fresh_state_pending = False # nudge once per attached-state cluster
logger.info(
f"[browser-batching {session_id}] redundant-read nudge #{redundant_read_nudges} "
f"({tu.name}) at turn {turn}"
)
content_blocks = content_blocks + [{"type": "text", "text": (
"\n\n⚡ Your PREVIOUS action already ended with a fresh '[page state after "
"action]' element list, so this re-read cost a round-trip for nothing. Act "
"straight from the attached state; only re-read after it says it was truncated "
"or you genuinely changed the page in a way that list wouldn't reflect."
)}]
# Deterministic nudging exhausted: ONE cheap aux adjudication
# to suggest a concrete next step before we keep failing.
if stagnation_exhausted(stagnation_streak) and not aux_adjudicated:
@@ -1549,7 +1578,7 @@ async def run_browser_agent(
logger.info(
f"[browser-batching {session_id}] run summary: turns={turn + 1} "
f"multi_action_turns={multi_action_turns} batch_calls={batch_calls} "
f"nudges={batching_nudges}"
f"nudges={batching_nudges} redundant_reads={redundant_read_nudges}"
)
browser_metrics.record_task(session_id, browser_id, task, final_status,
metrics_started_at, turn + 1, action_log, session.tokens,
backend/apps/agents/browser/browser_schema.py
+45 -43
View File
@@ -264,14 +264,14 @@ BROWSER_TOOLS_SCHEMA = [
{
"name": "BrowserClickIndex",
"description": (
"Click an element by its numeric index from BrowserListInteractives. "
"SOLO click, reserved for two cases: (1) the final IRREVERSIBLE step "
"(Send/Submit/Pay/Post), always alone with `expect` proof, and (2) "
"filling a text box via `text` (focused directly by node, so it works "
"inside messaging/compose overlays where coordinate clicks miss). "
"Routine clicks belong inside BrowserBatch as click_index sub-actions. "
"Uses native OS-level mouse events (event.isTrusted=true) so it works "
"on sites that filter out synthetic JS events. Always call "
"BrowserListInteractives first to get a fresh index list. If the click "
"returns 'index no longer valid', the page changed; re-list and retry. "
"If the index is a TEXT BOX (role textbox or searchbox), it is "
"focused directly by node (no coordinates, so it works inside messaging/"
"compose overlays where clicks miss); pass `text` to fill it in one call."
"on sites that filter synthetic JS events. If the click returns "
"'index no longer valid', the page changed; re-list and retry."
),
"input_schema": {
"type": "object",
@@ -296,12 +296,13 @@ BROWSER_TOOLS_SCHEMA = [
{
"name": "BrowserBatch",
"description": (
"Run a sequence of browser actions in one tool call. Each sub-action "
"is executed in order, with the URL captured before/after each one. "
"If the URL changes mid-batch (the page navigated), the rest of the "
"batch is aborted and you get a partial result. Use this when you "
"have a known sequence; typing then pressing Enter, swiping multiple "
"times, clicking through pagination. Max 5 actions per batch.\n\n"
"Your standard way to ACT on the page. Every mutation (navigate, "
"click, type, press, scroll) is a sub-action in this array, 1-5 per "
"call; an array of one is fine when that is genuinely all you know. "
"Each sub-action executes in order with the URL captured before/after. "
"If the URL changes mid-batch (the page navigated), the rest is "
"aborted and you get a partial result plus fresh state, so a "
"conservative batch costs nothing.\n\n"
"Sub-action types and their params:\n"
"- click_index: { index: int }\n"
"- press_key: { key: str }\n"
@@ -547,6 +548,14 @@ BROWSER_TOOLS_SCHEMA = [
},
]
# Schema-forced batching: the model ignored every prompt-level batching
# invitation (0 adoptions across 8 measured runs), so the single-step mutating
# tools are not offered to it at all; acting means a BrowserBatch array, and
# the one deliberate solo path is BrowserClickIndex (irreversible step with
# expect, or a text-box fill). Executors and replay still support everything.
_SOLO_MUTATORS_HIDDEN = {"BrowserNavigate", "BrowserClick", "BrowserType", "BrowserScroll", "BrowserPressKey"}
MODEL_VISIBLE_TOOLS = [t for t in BROWSER_TOOLS_SCHEMA if t["name"] not in _SOLO_MUTATORS_HIDDEN]
ACTION_MAP = {
"BrowserScreenshot": "screenshot",
"BrowserGetText": "get_text",
@@ -598,9 +607,8 @@ SYSTEM_PROMPT = (
"waits. Fall back to the UI only when you don't know the site's URL pattern.\n\n"
"## Required output structure: ReportProgress before every action\n"
"Before ANY action tool (BrowserClick, BrowserType, BrowserNavigate, "
"BrowserPressKey, BrowserScroll, BrowserEvaluate, BrowserClickIndex, "
"BrowserBatch), you MUST call the ReportProgress tool in the SAME turn. "
"Before ANY action tool (BrowserBatch, BrowserClickIndex, BrowserEvaluate), "
"you MUST call the ReportProgress tool in the SAME turn. "
"ReportProgress takes three short fields:\n"
"- evaluation_previous: did your last action work? what changed on the page?\n"
"- working_memory: what have you learned about this site? what worked, what didn't?\n"
@@ -648,7 +656,7 @@ SYSTEM_PROMPT = (
"## Try multiple strategies, learn from failures\n"
"Sites vary wildly. When one approach fails, switch tactics; don't retry the same "
"thing. The escalation ladder, fastest to slowest:\n"
"1. **Keyboard shortcuts via BrowserPressKey**; fastest and most reliable on sites "
"1. **Keyboard shortcuts via press_key sub-actions**; fastest and most reliable on sites "
"that support them (Tinder swipes, Gmail navigation, Slack message jump, etc.). "
"Always check if the site shows keyboard hints in the UI before falling back to clicks. "
"BrowserPressKey sends real native events that pass the `event.isTrusted` check, so "
@@ -672,9 +680,9 @@ SYSTEM_PROMPT = (
"To FILL a text box (a `<textbox>` like a message/compose field, including ones inside "
"a messaging overlay where clicks miss), call BrowserClickIndex on it with a `text` arg: "
"it focuses the box by node and types the whole string in ONE call, no coordinates, no "
"character-by-character. Then send with BrowserPressKey 'Enter' or the Send button.\n"
"character-by-character. Then send with a press_key 'Enter' sub-action or the Send button.\n"
"3. **Semantic CSS selectors**; `button[aria-label='X']`, `[role='button']`, "
"`a[href*='...']`. Try these via BrowserGetElements + BrowserClick when the site "
"`a[href*='...']`. Try these via BrowserGetElements + a click sub-action when the site "
"actually has semantic HTML.\n"
"4. **Text-based JS query**; when both of the above fail, use BrowserEvaluate to "
"find elements by visible text: `Array.from(document.querySelectorAll('*')).find(el => el.textContent.trim() === 'Like')`.\n"
@@ -682,27 +690,21 @@ SYSTEM_PROMPT = (
"button visually, then click by approximate coords.\n\n"
"## Speed: fewer turns is the #1 driver\n"
"Turns are slow model round-trips; tools are fast, so go fast by taking FEWER "
"turns. BATCH BY DEFAULT: every action result already ends with fresh page "
"state, so when that state shows you the next 2-3 steps, emit them as ONE "
"BrowserBatch instead of one tool per turn; single-action turns are for when "
"the next step genuinely depends on something you haven't seen. Good batches: "
"filling a form (type, type, click Next); a repeated action (5 swipes, 3 "
"scrolls); a deterministic flow (type query, press Enter, click first result); "
"or act-then-read by ending the batch with list_interactives, so click, wait, "
"list_interactives is ONE turn not three. Max 5 sub-actions. The batch runs "
"them in order and STOPS at the first that fails or if the URL changes, so "
"order them safely (never a maybe-failing step before a must-do one); a safe "
"stop means the rest is skipped and you just continue from the fresh state, so "
"a conservative batch costs you nothing. Go solo ONLY when you cannot yet NAME "
"the next action's target: its element is not in the state you are reading and "
"its name is not predictable. Needing to see a result AFTER acting is never a "
"reason to go solo (results auto-attach fresh state, and a failed or page-"
"changing step stops the batch safely). If your last 2 turns were single "
"actions whose targets were already visible beforehand, you are under-"
"batching: batch the next sequence. EXCEPTION: an irreversible step "
"(Send/Submit/Pay/Post) is always its own single action with `expect` proof, "
"never inside a batch.\n\n"
"Turns are slow model round-trips; tools are fast. You act through "
"BrowserBatch: an array of 1-5 actions per call (navigate, click_index, "
"type, press_key, scroll), executed in order with settle between steps. "
"When the state you are reading already names your next 2-3 targets, put "
"them in ONE array; an array of one is fine when the next step genuinely "
"depends on something you haven't seen. Good arrays: a deterministic flow "
"(type query, press Enter, click first result); a repeated action (5 "
"swipes); act-then-read by ending with list_interactives so click, wait, "
"read is ONE turn not three. The batch STOPS at the first failing or URL-"
"changing step and you continue from the fresh attached state, so a "
"conservative array costs nothing; order steps safely (never a maybe-"
"failing step before a must-do one). TWO solo exceptions, both via "
"BrowserClickIndex: the final irreversible step (Send/Submit/Pay/Post) "
"always alone with `expect` proof, and text-box fills via its `text` "
"param.\n\n"
"## Doing the SAME flow for many inputs? Use BrowserRepeatFlow\n"
"If you're about to repeat the same mechanical flow for a list (read 10 "
@@ -719,7 +721,7 @@ SYSTEM_PROMPT = (
"- Do NOT screenshot after every single action. Screenshot ONLY when you genuinely "
"don't know the page state (start of task, after navigation, after a failure).\n"
"- NEVER call BrowserWait or BrowserListInteractives right after an action. Every "
"mutating action (navigate, click, type, press, scroll, batch) already settles on its "
"mutating action (every batch and every solo click) already settles on its "
"own (waits for network/DOM quiet) and its result ends with '[page state after "
"action]': a fresh numbered element list. To keep results lean it may show ONLY the "
"rows that changed plus a count of unchanged ones; unchanged rows keep their numbers, "
@@ -727,14 +729,14 @@ SYSTEM_PROMPT = (
"re-listing after an action is a wasted turn. Reserve BrowserWait for content you "
"KNOW arrives late (pass `until` with the exact thing you expect), and "
"BrowserListInteractives for re-orienting or when the attached list was truncated.\n"
"- When scrolling, stop as soon as BrowserScroll reports atTop/atBottom or a 0 delta; "
"- When scrolling, stop as soon as a scroll sub-action reports atTop/atBottom or a 0 delta; "
"don't loop past the end.\n"
"- Do NOT call BrowserGetElements on the entire body if you already know roughly "
"where the target is. Scope it: `BrowserGetElements({selector: 'nav'})`.\n"
"- Do NOT call the same failing tool twice with identical parameters. If selector "
"X failed, try a DIFFERENT selector or a DIFFERENT strategy.\n"
"- For repeated actions (swiping through profiles, going through inbox messages), "
"use BrowserPressKey if available; it's an order of magnitude faster than DOM clicks.\n"
"use press_key sub-actions if available; an order of magnitude faster than DOM clicks.\n"
"- To RE-READ data you already loaded once (search results, a list, a detail page), "
"check BrowserListRoutes and use BrowserReplayRoute to fetch it straight from the "
"site's API instead of re-navigating and re-scraping; it's much faster. This is for "