From 4c85dd63142e55f40f85f3a5d0337e03d578f529 Mon Sep 17 00:00:00 2001
From: ciregenz <ciregenz@berkeley.edu>
Date: Fri, 10 Jul 2026 00:13:44 -0700
Subject: [PATCH] [eric] browser: structural composer finder
 (OSW_COMPOSER_STRUCT, default off) = an in-page primitive that ranks editable
 regions (contenteditable/textarea/text-input near a Send/Post/Reply control,
 excluding search boxes) instead of matching AX names, marks the winner +
 fills+reads-back in-page (the only reliable commit-check for a React
 contenteditable whose text never reaches the AX tree); send-script uses it as
 a fallback when the name detector misses, unblocking unnamed/non-standard
 composers (Reddit Lexical, X DM); 3 tests, all safety gates + dry-run stop
 unchanged

---
 backend/apps/agents/browser/browser_schema.py |  2 +
 .../agents/browser/browser_send_script.py     | 95 ++++++++++++-------
 backend/tests/test_browser_send_script.py     | 65 +++++++++++++
 frontend/src/shared/browserCommandHandler.ts  | 65 ++++++++++++-
 4 files changed, 190 insertions(+), 37 deletions(-)

diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py
index 0b94b5b8..f2455458 100644
--- a/backend/apps/agents/browser/browser_schema.py
+++ b/backend/apps/agents/browser/browser_schema.py
@@ -752,6 +752,8 @@ ACTION_MAP = {
     "BrowserReplayRoute": "replay_route",
     # Internal replay primitive (skill replay calls it directly; not in the LLM-facing schema). Re-resolves a click target by role+name.
     "BrowserClickByName": "click_by_name",
+    # Internal: structural composer finder (send-script fallback; not LLM-facing). Ranks editable regions, marks the winner, optionally fills+verifies in-page.
+    "BrowserFindComposer": "find_composer",
 }
 
 # --- App agent: driving an OpenSwarm-built app via its native bridge ---------
diff --git a/backend/apps/agents/browser/browser_send_script.py b/backend/apps/agents/browser/browser_send_script.py
index f3d23a67..a214c279 100644
--- a/backend/apps/agents/browser/browser_send_script.py
+++ b/backend/apps/agents/browser/browser_send_script.py
@@ -124,7 +124,10 @@ async def run_send_script(
     prompt; the composed task carries the routing brief whose own quoted strings
     made every real payload look ambiguous (r242/r243)."""
     t0 = time.monotonic()
-    if not surface_supports_script(current_url, state_text):
+    p_struct = os.environ.get("OSW_COMPOSER_STRUCT") == "1"
+    # The name-based surface gate can't see an unnamed/non-standard composer; under the
+    # structural flag, don't early-decline on it, the in-page finder gets a chance below.
+    if not surface_supports_script(current_url, state_text) and not p_struct:
         logger.info(f"[browser-sendscript] decline: no composer or opener in the perception ({current_url[:50]!r})")
         return None
     if P_READONLY_RE.search(task) or P_READONLY_RE.search(payload_source or ""):
@@ -154,50 +157,70 @@ async def run_send_script(
             if composer:
                 state_text = fresh
                 break
+    p_struct_selector: str = ""
     if not composer:
         # Reversible-opener hop: prestage often stops on the profile with the "Message" opener visible (its settle raced the overlay). Opening a composer is the allowed opener class; the irreversible bar is unchanged.
         opener = opener_index_in_state(state_text)
-        if not opener:
-            logger.info("[browser-sendscript] decline: no composer and no single exact-named opener in staged or fresh state")
-            return None
-        logger.info(f"[browser-sendscript] firing via opener {opener[1]!r} [{opener[0]}]")
-        r_open = await execute_tool("BrowserClickIndex", {"index": opener[0]}, browser_id, tab_id)
-        if not (isinstance(r_open, dict) and "error" not in r_open):
-            return None
-        log.append({"tool": "BrowserClickIndex", "input": {"index": opener[0]}, "ok": True,
-                    "result_summary": f"script opened composer via {opener[1]!r}"[:200], "elapsed_ms": 0})
-        for wait_s in (0.6, 1.2):
-            await asyncio.sleep(wait_s)
-            state_text = await fresh_list()
-            composer = composer_index_in_state(state_text)
-            if composer:
-                break
+        if opener:
+            logger.info(f"[browser-sendscript] firing via opener {opener[1]!r} [{opener[0]}]")
+            r_open = await execute_tool("BrowserClickIndex", {"index": opener[0]}, browser_id, tab_id)
+            if not (isinstance(r_open, dict) and "error" not in r_open):
+                return None
+            log.append({"tool": "BrowserClickIndex", "input": {"index": opener[0]}, "ok": True,
+                        "result_summary": f"script opened composer via {opener[1]!r}"[:200], "elapsed_ms": 0})
+            for wait_s in (0.6, 1.2):
+                await asyncio.sleep(wait_s)
+                state_text = await fresh_list()
+                composer = composer_index_in_state(state_text)
+                if composer:
+                    break
+        # Structural fallback: the AX-name detector missed it (an unnamed contenteditable, a
+        # non-standard rich editor, or two textboxes it couldn't disambiguate). Ask the page to
+        # rank its editable regions and fill+read-back the winner IN-PAGE (the only reliable
+        # commit-check for a React contenteditable, whose text never reaches the AX value).
+        # Flag-gated so the proven name path stays the default.
+        if not composer and p_struct:
+            fc = await execute_tool("BrowserFindComposer", {"fill": payload}, browser_id, tab_id)
+            if isinstance(fc, dict) and fc.get("found") and fc.get("filled"):
+                p_struct_selector = str(fc.get("selector") or "")
+                logger.info(f"[browser-sendscript] structural composer role={fc.get('role')!r} "
+                            f"score={fc.get('score')} nearSubmit={fc.get('nearSubmit')} filled+verified")
+                log.append({"tool": "BrowserFindComposer", "input": {"fill": "<payload>"}, "ok": True,
+                            "result_summary": f"structural composer {fc.get('role')!r} filled+verified"[:200], "elapsed_ms": 0})
+                composer = (-1, str(fc.get("role") or "composer"))
+            else:
+                logger.info(f"[browser-sendscript] structural finder: no usable composer ({str(fc)[:120]})")
         if not composer:
-            logger.info("[browser-sendscript] opener clicked but no composer appeared; handing to model")
+            logger.info("[browser-sendscript] decline: no composer, opener, or structural editable")
             return None
     # No Send-button precondition: composer sites (LinkedIn) lazy-render Send only AFTER text commits, so it's resolved post-fill; never appearing = clean pre-click abort.
     logger.info(f"[browser-sendscript] fill target {composer[1]!r} [{composer[0]}]")
 
-    # 1. fill (focused by node, the composer overlay path coordinate clicks miss)
-    r_fill = await execute_tool("BrowserClickIndex", {"index": composer[0], "text": payload}, browser_id, tab_id)
-    fill_ok = isinstance(r_fill, dict) and "error" not in r_fill
-    log.append({"tool": "BrowserClickIndex", "input": {"index": composer[0], "text": payload},
-                "ok": fill_ok, "result_summary": f"script fill into {composer[1]!r}"[:200], "elapsed_ms": 0})
-    if not fill_ok:
-        logger.info("[browser-sendscript] fill errored; handing to model untouched")
-        return None
-    # 2. verify the fill committed. Send is resolved AFTER, two ways: LinkedIn enables Send only once its JS digests the input (beats later than the text is visible), so the scan waits a little.
-    state2 = ""
-    committed = False
-    for wait_s in (0.4, 0.8, 1.2, 1.6):
-        await asyncio.sleep(wait_s)
+    if p_struct_selector:
+        # The finder already filled + read-back-verified in-page; nothing to re-fill or re-check.
         state2 = await fresh_list()
-        committed = bool(state2 and payload_in_textbox(state2, payload))
-        if committed:
-            break
-    if not committed:
-        logger.info("[browser-sendscript] fill not seen committed; aborting pre-click")
-        return None
+        committed = True
+    else:
+        # 1. fill (focused by node, the composer overlay path coordinate clicks miss)
+        r_fill = await execute_tool("BrowserClickIndex", {"index": composer[0], "text": payload}, browser_id, tab_id)
+        fill_ok = isinstance(r_fill, dict) and "error" not in r_fill
+        log.append({"tool": "BrowserClickIndex", "input": {"index": composer[0], "text": payload},
+                    "ok": fill_ok, "result_summary": f"script fill into {composer[1]!r}"[:200], "elapsed_ms": 0})
+        if not fill_ok:
+            logger.info("[browser-sendscript] fill errored; handing to model untouched")
+            return None
+        # 2. verify the fill committed. Send is resolved AFTER, two ways: LinkedIn enables Send only once its JS digests the input (beats later than the text is visible), so the scan waits a little.
+        state2 = ""
+        committed = False
+        for wait_s in (0.4, 0.8, 1.2, 1.6):
+            await asyncio.sleep(wait_s)
+            state2 = await fresh_list()
+            committed = bool(state2 and payload_in_textbox(state2, payload))
+            if committed:
+                break
+        if not committed:
+            logger.info("[browser-sendscript] fill not seen committed; aborting pre-click")
+            return None
     # Dry-run probe: prove the script FIRES + fills on a NON-LinkedIn site without ever
     # doing the outward send. Everything up to here ran (surface gate passed, composer
     # found, fill committed); we stop before the irreversible click and report readiness.
diff --git a/backend/tests/test_browser_send_script.py b/backend/tests/test_browser_send_script.py
index bd19ec9f..6cc27e78 100644
--- a/backend/tests/test_browser_send_script.py
+++ b/backend/tests/test_browser_send_script.py
@@ -186,6 +186,71 @@ async def test_composed_task_brief_quotes_fire_via_payload_source():
     assert r["payload"] == "[test] hello world r9-os"
 
 
+NAMELESS = '[1]<link "Home">\n[2]<button "Search">'  # no name-matched composer, no opener
+
+
+def make_struct_exec(fc_result, list_states=None):
+    """execute_tool mock for the structural-finder path: list calls return no-composer
+    states; a BrowserFindComposer call returns the scripted structural result."""
+    calls = {"clicks": [], "find": 0}
+    states = list(list_states or [NAMELESS] * 6)
+    i = {"n": 0}
+
+    async def execute(tool, params, bid, tid):
+        if tool == "BrowserListInteractives":
+            s = states[min(i["n"], len(states) - 1)]
+            i["n"] += 1
+            return {"text": s}
+        if tool == "BrowserFindComposer":
+            calls["find"] += 1
+            calls["clicks"].append((tool, params))
+            return fc_result
+        calls["clicks"].append((tool, params))
+        return {"ok": True}
+    return execute, calls
+
+
+@pytest.mark.asyncio
+async def test_structural_finder_fills_when_name_detector_misses(monkeypatch):
+    """The generalization: no AX-named composer and no opener, but the in-page finder
+    ranks an editable region, fills + reads it back, and the script reports filled-ready
+    (dry-run stops before the send). This is what unblocks Reddit-style contenteditables."""
+    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
+    monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
+    ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
+                                  "selector": '[data-osw-composer="1"]', "score": 6.2, "nearSubmit": True})
+    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
+                                 payload_in_textbox, payload_source=TASK,
+                                 current_url="https://www.reddit.com/r/test/comments/x/")
+    assert r is not None and r["sent"] is False           # filled, stopped before the send
+    assert calls["find"] == 1
+    assert any(c[0] == "BrowserFindComposer" for c in calls["clicks"])
+
+
+@pytest.mark.asyncio
+async def test_structural_finder_declines_when_no_editable(monkeypatch):
+    """The finder honestly finds nothing usable (a page with only a search box) -> decline
+    to the model path, never a false fire."""
+    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
+    ex, calls = make_struct_exec({"found": False})
+    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
+                                 payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
+    assert r is None
+    assert calls["find"] == 1
+
+
+@pytest.mark.asyncio
+async def test_structural_off_by_default_never_calls_finder(monkeypatch):
+    """Flag off = the proven name path only; a name-less perception declines and the finder
+    is never invoked (the structural path can't perturb the default)."""
+    monkeypatch.delenv("OSW_COMPOSER_STRUCT", raising=False)
+    ex, calls = make_struct_exec({"found": True, "filled": True, "selector": "x", "role": "textarea"})
+    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
+                                 payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
+    assert r is None
+    assert calls["find"] == 0
+
+
 def test_dryrun_report_encodes_the_gate_funnel():
     """The coverage harness greps this one line for gate attribution: a staged composer
     reports composer=1, a bare page reports zeros, and armed/filled ride the booleans."""
diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts
index 235f61c8..694d56b2 100644
--- a/frontend/src/shared/browserCommandHandler.ts
+++ b/frontend/src/shared/browserCommandHandler.ts
@@ -10,7 +10,7 @@ import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettl
 
 let initialized = false;
 
-export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'click_point' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name';
+export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'click_point' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name' | 'find_composer';
 
 export interface BrowserActivity {
   action: BrowserAction;
@@ -338,6 +338,66 @@ async function handleType(wv: BrowserWebview, params: Record<string, any>): Prom
   return result;
 }
 
+// Find the page's composer STRUCTURALLY, not by accessible name: the biggest editable
+// region (contenteditable / textarea / text input) that is a real writing surface, not a
+// search box. Ranks by editable-richness + proximity to a Send/Post/Reply control + size,
+// so it picks the comment box out of a page that also has a search field. Marks the winner
+// with data-osw-composer so a follow-up type/click can target it by a stable selector.
+// With {fill}, it also types + reads back in the SAME in-page context (the only reliable
+// commit-check for a React-controlled contenteditable, whose value never reaches the AX tree).
+async function handleFindComposer(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
+  const fill = params.fill != null ? String(params.fill) : null;
+  const safeFill = JSON.stringify(fill);
+  const code = `(() => {
+    const SUBMIT = /\\b(send|post|reply|comment|tweet|publish|share|message)\\b/i;
+    const SEARCH = /search|find\\b|filter|query|lookup|explore|jump to/i;
+    const vis = (el) => {
+      const r = el.getBoundingClientRect(); const s = getComputedStyle(el);
+      return r.width >= 80 && r.height >= 16 && s.visibility !== 'hidden'
+        && s.display !== 'none' && s.opacity !== '0';
+    };
+    const cands = [...document.querySelectorAll(
+      'textarea, [contenteditable="true"], [role="textbox"], input[type="text"]')];
+    let best = null, bestScore = 0, bestNear = false;
+    for (const el of cands) {
+      if (!vis(el) || el.readOnly || el.disabled) continue;
+      const label = ((el.getAttribute('aria-label')||'') + ' ' + (el.getAttribute('placeholder')||'')
+        + ' ' + (el.getAttribute('data-placeholder')||'') + ' ' + (el.getAttribute('name')||'')).trim();
+      if (el.type === 'search' || SEARCH.test(label)) continue;
+      const rich = el.tagName === 'TEXTAREA' || el.isContentEditable || el.getAttribute('role') === 'textbox';
+      const area = el.closest('form, [role="dialog"], [role="group"], section, main, [role="main"]') || document.body;
+      let near = false;
+      const btns = area.querySelectorAll('button, [role="button"], input[type="submit"]');
+      for (const b of btns) {
+        const bt = ((b.textContent||'') + ' ' + (b.getAttribute('aria-label')||'')).trim();
+        if (bt.length < 40 && SUBMIT.test(bt)) { near = true; break; }
+      }
+      if (!rich && !near) continue;                       // a bare form input near nothing is not a composer
+      const r = el.getBoundingClientRect();
+      const score = (el.isContentEditable ? 2 : 0) + (el.tagName === 'TEXTAREA' ? 2 : 0)
+        + (near ? 3 : 0) + Math.min((r.width * r.height) / 40000, 3) + (SUBMIT.test(label) ? 1 : 0);
+      if (score > bestScore) { bestScore = score; best = el; bestNear = near; }
+    }
+    if (!best || bestScore < 2) return { found: false };
+    best.setAttribute('data-osw-composer', '1');
+    let filled = false;
+    if (${safeFill} != null) {
+      best.scrollIntoView({ block: 'center', behavior: 'instant' }); best.focus();
+      if (best.select) best.select();
+      document.execCommand('selectAll', false); document.execCommand('delete', false);
+      document.execCommand('insertText', false, ${safeFill});
+      best.dispatchEvent(new InputEvent('input', { bubbles: true, cancelable: true, inputType: 'insertText', data: ${safeFill} }));
+      best.dispatchEvent(new Event('change', { bubbles: true }));
+      const now = (best.value != null ? best.value : (best.textContent || ''));
+      filled = now.includes(${safeFill});
+    }
+    return { found: true, selector: '[data-osw-composer="1"]', tag: best.tagName.toLowerCase(),
+      role: best.isContentEditable ? 'contenteditable' : (best.getAttribute('role') || best.tagName.toLowerCase()),
+      score: Math.round(bestScore * 10) / 10, nearSubmit: bestNear, filled };
+  })()`;
+  return await evalInPage(wv, code);
+}
+
 // Electron sendInputEvent expects names like 'Up', 'Enter', 'Space', not 'ArrowUp'/' '/'Esc'.
 const KEY_NAME_MAP: Record<string, string> = {
   ArrowUp: 'Up',
@@ -1570,6 +1630,9 @@ async function runBrowserCommand(
       case 'type':
         result = await handleType(wv, params);
         break;
+      case 'find_composer':
+        result = await handleFindComposer(wv, params);
+        break;
       case 'evaluate':
         result = await handleEvaluate(wv, params);
         break;