From f03dde7e0c43a064dadffbf671f5823b6c090587 Mon Sep 17 00:00:00 2001
From: ciregenz <ciregenz@berkeley.edu>
Date: Thu, 4 Jun 2026 16:38:56 -0700
Subject: [PATCH] [eric] browser: focus text boxes by node and insert text
 directly, reaches compose iframes clicks miss

---
 backend/apps/agents/browser/browser_schema.py | 19 ++++++++++--
 frontend/src/shared/browserCommandHandler.ts  | 30 +++++++++++++++++--
 2 files changed, 45 insertions(+), 4 deletions(-)

diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py
index e1d318b2..255caadf 100644
--- a/backend/apps/agents/browser/browser_schema.py
+++ b/backend/apps/agents/browser/browser_schema.py
@@ -216,7 +216,10 @@ BROWSER_TOOLS_SCHEMA = [
             "Uses native OS-level mouse events (event.isTrusted=true) so it works "
             "on sites that filter out synthetic JS events. Always call "
             "BrowserListInteractives first to get a fresh index list. If the click "
-            "returns 'index no longer valid', the page changed; re-list and retry."
+            "returns 'index no longer valid', the page changed; re-list and retry. "
+            "If the index is a TEXT BOX (role textbox or searchbox), it is "
+            "focused directly by node (no coordinates, so it works inside messaging/"
+            "compose overlays where clicks miss); pass `text` to fill it in one call."
         ),
         "input_schema": {
             "type": "object",
@@ -225,6 +228,14 @@ BROWSER_TOOLS_SCHEMA = [
                     "type": "integer",
                     "description": "The numeric index from BrowserListInteractives (1-based).",
                 },
+                "text": {
+                    "type": "string",
+                    "description": (
+                        "Optional. If the index is a text box, focus it and type this "
+                        "whole string in one call (no character-by-character). Use this "
+                        "to fill a compose/message box reliably."
+                    ),
+                },
                 "expect": _EXPECT_DESC,
             },
             "required": ["index"],
@@ -586,7 +597,11 @@ SYSTEM_PROMPT = (
     "Call BrowserListInteractives to get a numbered list (`[1]<button \"Like\">`, "
     "`[2]<link \"Settings\">`), then BrowserClickIndex with the number. The click uses "
     "native OS-level mouse events so it works where DOM .click() doesn't. THIS IS YOUR "
-    "GO-TO STRATEGY for unlabeled or hostile sites; try this BEFORE BrowserGetElements.\n"
+    "GO-TO STRATEGY for unlabeled or hostile sites; try this BEFORE BrowserGetElements. "
+    "To FILL a text box (a `<textbox>` like a message/compose field, including ones inside "
+    "a messaging overlay where clicks miss), call BrowserClickIndex on it with a `text` arg: "
+    "it focuses the box by node and types the whole string in ONE call, no coordinates, no "
+    "character-by-character. Then send with BrowserPressKey 'Enter' or the Send button.\n"
     "3. **Semantic CSS selectors**; `button[aria-label='X']`, `[role='button']`, "
     "`a[href*='...']`. Try these via BrowserGetElements + BrowserClick when the site "
     "actually has semantic HTML.\n"
diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts
index 4b99f569..15711021 100644
--- a/frontend/src/shared/browserCommandHandler.ts
+++ b/frontend/src/shared/browserCommandHandler.ts
@@ -367,12 +367,36 @@ async function enumerateCandidates(wv: BrowserWebview): Promise<RankItem[]> {
 // Shared by click_index (cache lookup) and click_by_name (fresh resolution).
 async function clickBackendNode(
   wv: BrowserWebview, backendNodeId: number, sessionId: string | undefined, label: string,
+  opts: { role?: string; text?: string } = {},
 ): Promise<Record<string, any>> {
   try {
     await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
   } catch (err: any) {
     return { error: `${label} is no longer valid (${err.message || 'node not found'}). The page may have changed.` };
   }
+
+  // A text box is focused DIRECTLY by node id, never by screen coordinates. Inside an
+  // about:blank compose iframe (LinkedIn/Gmail messaging) the coordinate path lands on
+  // the wrong element (the box model is frame-local but the click dispatches in the root
+  // frame), while DOM.focus reaches the node in any frame. With a `text` arg we then
+  // insert the whole string at once, no clicking, no character-by-character typing.
+  if (/\b(textbox|searchbox)\b/i.test(opts.role || '')) {
+    try {
+      await sendCdp(wv, 'DOM.focus', { backendNodeId }, sessionId);
+    } catch (err: any) {
+      return { error: `${label} could not be focused (${err?.message || 'focus failed'}); it may be disabled or hidden.` };
+    }
+    if (typeof opts.text === 'string' && opts.text.length > 0) {
+      try {
+        await sendCdp(wv, 'Input.insertText', { text: opts.text });
+      } catch (err: any) {
+        return { error: `Focused ${label} but could not type into it: ${err?.message || String(err)}` };
+      }
+      return { text: `Focused ${label} and typed the text in.` };
+    }
+    return { text: `Focused ${label}; the cursor is in it now (type with BrowserPressKey, or pass a text arg to fill it in one call).` };
+  }
+
   let boxModel;
   try {
     boxModel = await sendCdp(wv, 'DOM.getBoxModel', { backendNodeId }, sessionId);
@@ -493,7 +517,8 @@ async function handleClickIndex(wv: BrowserWebview, params: Record<string, any>)
     };
   }
 
-  const result = await clickBackendNode(wv, backendNodeId, sessionId, `index ${idx}`);
+  const result = await clickBackendNode(wv, backendNodeId, sessionId, `index ${idx}`,
+    { role, text: typeof params.text === 'string' ? params.text : undefined });
   // Surface what was clicked so the agent loop can record a stable,
   // replayable click-by-name step (indices are ephemeral; names aren't).
   if (!result.error) {
@@ -523,7 +548,8 @@ async function handleClickByName(wv: BrowserWebview, params: Record<string, any>
   if (!match) {
     return { error: `No element matching role="${wantRole}" name="${wantName}" on this page.` };
   }
-  return clickBackendNode(wv, match.backendNodeId, match.sessionId, `${match.role} "${match.name}"`);
+  return clickBackendNode(wv, match.backendNodeId, match.sessionId, `${match.role} "${match.name}"`,
+    { role: match.role });
 }
 
 // Sequential sub-actions; aborts mid-batch if URL changes (indices/selectors go stale on navigation).