diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 5c622a87..a670bfaf 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -206,6 +206,37 @@ def _delta_state(text: str, seen_lines: set[str]) -> str: ) +# A button row whose name is exactly a Send control (not "Send InMail credit" or +# "Send a message to X"); used to hand the model the Send button after it types, +# so it never burns turns hunting a button that's right there. +_SEND_ROW_RE = re.compile(r'\[(\d+)\]\*?<\s*button\s+"([^"]*)"', re.I) + + +def _is_composer_fill(tool_name: str, tool_input: dict) -> bool: + """True if this action typed a message into a composer (the moment the Send + button is about to matter). Covers the solo fill, BrowserType, and a batched + fill, the three ways the model composes.""" + ti = tool_input or {} + if tool_name in ("BrowserClickIndex", "BrowserType"): + return bool(str(ti.get("text") or "").strip()) + if tool_name == "BrowserBatch": + for a in (ti.get("actions") or []): + p = a.get("params") or {} + if a.get("type") in ("type", "click_index") and str(p.get("text") or "").strip(): + return True + return False + + +def _send_index_in_state(state_text: str): + """(index, name) of a real Send button in an interactives list, or None. + Strict exact match so it never grabs an upsell or a profile 'Send a message' link.""" + for line in (state_text or "").splitlines(): + m = _SEND_ROW_RE.search(line) + if m and m.group(2).strip().lower() in ("send", "send now", "send message"): + return int(m.group(1)), m.group(2) + return None + + async def _post_action_state( tool_name: str, tool_input: dict, result: dict, browser_id: str, tab_id: str, wait_exec, goal: str, @@ -224,6 +255,16 @@ async def _post_action_state( ) if settle.get("hung"): return "" + # Composer fill: the Send button renders a beat LATER than the text commits, so + # a re-list right now misses it and the model wastes turns hunting (measured ~54s + # on one run). Wait for Send to paint first, then it's in the list we hand back. + _composer_fill = _is_composer_fill(tool_name, tool_input) + if _composer_fill: + try: + await browser_wait.smart_wait(wait_exec, browser_id, tab_id, 2500, + until="Send", target_only=True) + except Exception: + pass try: params = {"goal": goal} if goal else {} lst = await asyncio.wait_for( @@ -234,7 +275,16 @@ async def _post_action_state( if not isinstance(lst, dict) or "error" in lst or not lst.get("text"): return "" state = lst["text"] if seen_lines is None else _delta_state(lst["text"], seen_lines) - return f"\n\n{PAGE_STATE_MARKER}\n{_truncate_state(state)}" + out = f"\n\n{PAGE_STATE_MARKER}\n{_truncate_state(state)}" + # Hand the Send button over so the model clicks it instead of hunting via CSS/JS. + if _composer_fill: + _si = _send_index_in_state(lst["text"]) + if _si: + out = (f"\n\n[send-ready] Your message is typed and the Send button is index " + f"{_si[0]} below. To deliver, click it SOLO with BrowserClickIndex + an " + f"`expect` proof. Do NOT hunt for it with CSS/JS/screenshots, it is right here." + ) + out + return out async def _request_browser_approval( diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index 9e9a6b06..b27494d7 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -659,11 +659,13 @@ SYSTEM_PROMPT = ( "second time unless you have verified the first did NOT go through. This is how you " "avoid both ghost-successes and double-sends.\n" "When you arrive (or wake) with the target's message THREAD or composer ALREADY OPEN, " - "that IS your thread: confirm the recipient ONCE from what's already on screen (the " - "header name, or your own earlier messages to them sitting in it, those are proof the " - "thread is right), then type and send. Do NOT close it to reopen a 'fresh' one, and do " - "NOT run repeated screenshots or DOM/JS probes to re-confirm what the open thread " - "already shows; that re-verification just burns turns.\n" + "that IS your thread, commit to it. The open thread's header name (and your own earlier " + "messages to that person sitting in it) ARE the recipient proof; do NOT navigate away to " + "open their profile or re-search 'just to be sure', that round-trips for nothing and can " + "even land you on the wrong surface. The plan from an open composer is fixed and short: " + "type the message, click Send, then verify ONCE. Execute it; do not re-derive it, re-open " + "anything, or re-confirm with extra screenshots/DOM/JS probes what the open thread already " + "shows. That re-verification is the single biggest waste of turns.\n" "If you already saw the Send button (or Submit/Post) at an index, REMEMBER that " "number: typing into the composer does NOT move it, so after you type, click that " "same remembered index directly. Do NOT decide it vanished and hunt for it with JS, " diff --git a/backend/tests/test_browser_agent_loop.py b/backend/tests/test_browser_agent_loop.py index 36653ca9..87102e6e 100644 --- a/backend/tests/test_browser_agent_loop.py +++ b/backend/tests/test_browser_agent_loop.py @@ -1411,6 +1411,24 @@ def test_recoverable_tool_error_classifier(): assert not recoverable_tool_error("some unrelated failure") +def test_composer_fill_detection_and_send_handoff(): + from backend.apps.agents.browser.browser_agent import _is_composer_fill, _send_index_in_state + # a composer fill is detected across the three ways the model types + assert _is_composer_fill("BrowserClickIndex", {"index": 4, "text": "hello world"}) + assert _is_composer_fill("BrowserType", {"selector": "#m", "text": "hi"}) + assert _is_composer_fill("BrowserBatch", {"actions": [ + {"type": "click_index", "params": {"index": 4, "text": "hi there"}}]}) + # a plain click (no text) is NOT a fill + assert not _is_composer_fill("BrowserClickIndex", {"index": 4}) + assert not _is_composer_fill("BrowserScroll", {}) + # the real Send button is handed over; upsells / profile links are never mistaken for it + page = '[1]\n[33]\n[44]