[eric] browser: hand the Send button to the model after a composer fill, and commit to an already-open thread instead of re-verifying

backend/apps/agents/browser/browser_agent.py

@@ -206,6 +206,37 @@ def _delta_state(text: str, seen_lines: set[str]) -> str:
     )
 
 
+# A button row whose name is exactly a Send control (not "Send InMail credit" or
+# "Send a message to X"); used to hand the model the Send button after it types,
+# so it never burns turns hunting a button that's right there.
+_SEND_ROW_RE = re.compile(r'\[(\d+)\]\*?<\s*button\s+"([^"]*)"', re.I)
+
+
+def _is_composer_fill(tool_name: str, tool_input: dict) -> bool:
+    """True if this action typed a message into a composer (the moment the Send
+    button is about to matter). Covers the solo fill, BrowserType, and a batched
+    fill, the three ways the model composes."""
+    ti = tool_input or {}
+    if tool_name in ("BrowserClickIndex", "BrowserType"):
+        return bool(str(ti.get("text") or "").strip())
+    if tool_name == "BrowserBatch":
+        for a in (ti.get("actions") or []):
+            p = a.get("params") or {}
+            if a.get("type") in ("type", "click_index") and str(p.get("text") or "").strip():
+                return True
+    return False
+
+
+def _send_index_in_state(state_text: str):
+    """(index, name) of a real Send button in an interactives list, or None.
+    Strict exact match so it never grabs an upsell or a profile 'Send a message' link."""
+    for line in (state_text or "").splitlines():
+        m = _SEND_ROW_RE.search(line)
+        if m and m.group(2).strip().lower() in ("send", "send now", "send message"):
+            return int(m.group(1)), m.group(2)
+    return None
+
+
 async def _post_action_state(
     tool_name: str, tool_input: dict, result: dict,
     browser_id: str, tab_id: str, wait_exec, goal: str,
@@ -224,6 +255,16 @@ async def _post_action_state(
         )
         if settle.get("hung"):
             return ""
+    # Composer fill: the Send button renders a beat LATER than the text commits, so
+    # a re-list right now misses it and the model wastes turns hunting (measured ~54s
+    # on one run). Wait for Send to paint first, then it's in the list we hand back.
+    _composer_fill = _is_composer_fill(tool_name, tool_input)
+    if _composer_fill:
+        try:
+            await browser_wait.smart_wait(wait_exec, browser_id, tab_id, 2500,
+                                          until="Send", target_only=True)
+        except Exception:
+            pass
     try:
         params = {"goal": goal} if goal else {}
         lst = await asyncio.wait_for(
@@ -234,7 +275,16 @@ async def _post_action_state(
     if not isinstance(lst, dict) or "error" in lst or not lst.get("text"):
         return ""
     state = lst["text"] if seen_lines is None else _delta_state(lst["text"], seen_lines)
-    return f"\n\n{PAGE_STATE_MARKER}\n{_truncate_state(state)}"
+    out = f"\n\n{PAGE_STATE_MARKER}\n{_truncate_state(state)}"
+    # Hand the Send button over so the model clicks it instead of hunting via CSS/JS.
+    if _composer_fill:
+        _si = _send_index_in_state(lst["text"])
+        if _si:
+            out = (f"\n\n[send-ready] Your message is typed and the Send button is index "
+                   f"{_si[0]} below. To deliver, click it SOLO with BrowserClickIndex + an "
+                   f"`expect` proof. Do NOT hunt for it with CSS/JS/screenshots, it is right here."
+                   ) + out
+    return out
 
 
 async def _request_browser_approval(

backend/apps/agents/browser/browser_schema.py

@@ -659,11 +659,13 @@ SYSTEM_PROMPT = (
     "second time unless you have verified the first did NOT go through. This is how you "
     "avoid both ghost-successes and double-sends.\n"
     "When you arrive (or wake) with the target's message THREAD or composer ALREADY OPEN, "
-    "that IS your thread: confirm the recipient ONCE from what's already on screen (the "
-    "header name, or your own earlier messages to them sitting in it, those are proof the "
-    "thread is right), then type and send. Do NOT close it to reopen a 'fresh' one, and do "
-    "NOT run repeated screenshots or DOM/JS probes to re-confirm what the open thread "
-    "already shows; that re-verification just burns turns.\n"
+    "that IS your thread, commit to it. The open thread's header name (and your own earlier "
+    "messages to that person sitting in it) ARE the recipient proof; do NOT navigate away to "
+    "open their profile or re-search 'just to be sure', that round-trips for nothing and can "
+    "even land you on the wrong surface. The plan from an open composer is fixed and short: "
+    "type the message, click Send, then verify ONCE. Execute it; do not re-derive it, re-open "
+    "anything, or re-confirm with extra screenshots/DOM/JS probes what the open thread already "
+    "shows. That re-verification is the single biggest waste of turns.\n"
     "If you already saw the Send button (or Submit/Post) at an index, REMEMBER that "
     "number: typing into the composer does NOT move it, so after you type, click that "
     "same remembered index directly. Do NOT decide it vanished and hunt for it with JS, "

backend/tests/test_browser_agent_loop.py

@@ -1411,6 +1411,24 @@ def test_recoverable_tool_error_classifier():
     assert not recoverable_tool_error("some unrelated failure")
 
 
+def test_composer_fill_detection_and_send_handoff():
+    from backend.apps.agents.browser.browser_agent import _is_composer_fill, _send_index_in_state
+    # a composer fill is detected across the three ways the model types
+    assert _is_composer_fill("BrowserClickIndex", {"index": 4, "text": "hello world"})
+    assert _is_composer_fill("BrowserType", {"selector": "#m", "text": "hi"})
+    assert _is_composer_fill("BrowserBatch", {"actions": [
+        {"type": "click_index", "params": {"index": 4, "text": "hi there"}}]})
+    # a plain click (no text) is NOT a fill
+    assert not _is_composer_fill("BrowserClickIndex", {"index": 4})
+    assert not _is_composer_fill("BrowserScroll", {})
+    # the real Send button is handed over; upsells / profile links are never mistaken for it
+    page = '[1]<link "Tyler Chen">\n[33]<textbox "Write a message">\n[44]<button "Send">'
+    assert _send_index_in_state(page) == (44, "Send")
+    assert _send_index_in_state('[12]<button "Send InMail credit">') is None
+    assert _send_index_in_state('[5]<button "Send a message to Maya">') is None
+    assert _send_index_in_state("") is None
+
+
 def test_strip_lone_surrogates():
     from backend.apps.agents.browser.browser_agent import _strip_lone_surrogates, _format_tool_result
     # an orphan UTF-16 surrogate (half an emoji from the webview) is what crashes