From 19a3e8ad1aca4c5668ef87543093ccffdeecb88f Mon Sep 17 00:00:00 2001
From: ciregenz <ciregenz@berkeley.edu>
Date: Sun, 12 Jul 2026 20:52:15 -0700
Subject: [PATCH] [eric] browser: keystroke-fill fallback + cross-nav reveal
 retry (both flag-gated, default off)

Item 1: when the finder's execCommand insertText read-back fails (editors that reject synthetic input), retype the same text as real OS-level key events via wv.sendInputEvent, then verify from the marked element OR document.activeElement (editors like Reddit swap the node on activation, staling the selector). Item 3: when an open-first reveal navigates to a new document (Reddit thread, TikTok video, GitHub issue), the send-script re-perceives after a load beat and calls the finder once more on the destination, bounded to 2 tries. 4 new tests (cross-nav retry fires / no wasted retry) + structural tests split into test_browser_reveal.py to stay under the 300-line cap. Live real-path validation of the sendInputEvent leg is owed on a healthy rig: the eric/browser agent loop wedges at fresh-card webview mount (15.25s cap, renderer starvation, predates the suspend fix) at low load too
---
 .../agents/browser/browser_send_script.py     |  23 ++-
 backend/tests/test_browser_reveal.py          | 143 ++++++++++++++++++
 backend/tests/test_browser_send_script.py     |  96 ------------
 frontend/src/shared/browserCommandHandler.ts  |  36 ++++-
 4 files changed, 198 insertions(+), 100 deletions(-)
 create mode 100644 backend/tests/test_browser_reveal.py

diff --git a/backend/apps/agents/browser/browser_send_script.py b/backend/apps/agents/browser/browser_send_script.py
index 82403daf..6abe17bb 100644
--- a/backend/apps/agents/browser/browser_send_script.py
+++ b/backend/apps/agents/browser/browser_send_script.py
@@ -17,7 +17,7 @@ import logging
 import os
 import re
 import time
-from typing import Awaitable, Callable
+from typing import Awaitable, Callable, Dict
 
 from backend.apps.agents.browser import browser_verified_action
 
@@ -184,12 +184,29 @@ async def run_send_script(
             # compose surface: a modal trigger, the first conversation, or a scroll) when the
             # composer isn't painted yet. It never commits a send, only opens a surface.
             p_reveal = os.environ.get("OSW_COMPOSER_REVEAL") == "1"
-            fc = await execute_tool("BrowserFindComposer", {"fill": payload, "reveal": p_reveal}, browser_id, tab_id)
+            # A reveal that OPENS the first list item (a Reddit thread, a TikTok video, a GitHub
+            # issue) is a full-page NAVIGATION: it kills the finder's own JS context, so that one
+            # call can't reach the composer that only exists on the destination. When the finder
+            # reports it fired `open-first` but found nothing, the page is now loading the item;
+            # give it a beat and run the finder ONCE more on the destination. Bounded to 2 tries so
+            # a feed-of-feeds can't walk forever.
+            fc: Dict[str, object] = {}
+            for attempt in range(2):
+                fc = await execute_tool("BrowserFindComposer", {"fill": payload, "reveal": p_reveal}, browser_id, tab_id)
+                if isinstance(fc, dict) and fc.get("found") and fc.get("filled"):
+                    break
+                revs = fc.get("reveals") if isinstance(fc, dict) else None
+                navigated = p_reveal and isinstance(revs, list) and "open-first" in revs
+                if not navigated:
+                    break
+                logger.info("[browser-sendscript] reveal navigated (open-first); re-perceiving the destination")
+                await asyncio.sleep(1.5)
+                await fresh_list()
             if isinstance(fc, dict) and fc.get("found") and fc.get("filled"):
                 p_struct_selector = str(fc.get("selector") or "")
                 logger.info(f"[browser-sendscript] structural composer role={fc.get('role')!r} "
                             f"score={fc.get('score')} nearSubmit={fc.get('nearSubmit')} "
-                            f"reveals={fc.get('reveals')} filled+verified")
+                            f"reveals={fc.get('reveals')} fillMode={fc.get('fillMode')} filled+verified")
                 log.append({"tool": "BrowserFindComposer", "input": {"fill": "<payload>"}, "ok": True,
                             "result_summary": f"structural composer {fc.get('role')!r} filled+verified"[:200], "elapsed_ms": 0})
                 composer = (-1, str(fc.get("role") or "composer"))
diff --git a/backend/tests/test_browser_reveal.py b/backend/tests/test_browser_reveal.py
new file mode 100644
index 00000000..13441493
--- /dev/null
+++ b/backend/tests/test_browser_reveal.py
@@ -0,0 +1,143 @@
+"""Structural composer finder + reveal-and-find reach + cross-nav retry, verified without a
+live webview: a mock executor scripts BrowserFindComposer results so the send-script's
+structural fallback path (find -> reveal -> re-perceive-on-nav -> fill) is exercised end to
+end, including the flag gating and the safety that reveal only forwards under the flag."""
+import pytest
+
+from backend.apps.agents.browser import browser_send_script as ss
+from backend.apps.agents.browser.browser_agent import send_index_in_state, payload_in_textbox
+
+TASK = "go to tyler chen's linkedin hes in entrepreneurs first and text him '[test] hello world r9-os'"
+NAMELESS = '[1]<link "Home">\n[2]<button "Search">'  # no name-matched composer, no opener
+
+
+def make_struct_exec(fc_result, list_states=None):
+    """execute_tool mock for the structural-finder path: list calls return no-composer
+    states; a BrowserFindComposer call returns the scripted structural result."""
+    calls = {"clicks": [], "find": 0}
+    states = list(list_states or [NAMELESS] * 6)
+    i = {"n": 0}
+
+    # fc_result may be a single dict (returned every call) or a list of dicts (returned in
+    # sequence, last one repeating) to script the cross-nav retry.
+    fc_seq = fc_result if isinstance(fc_result, list) else [fc_result]
+
+    async def execute(tool, params, bid, tid):
+        if tool == "BrowserListInteractives":
+            s = states[min(i["n"], len(states) - 1)]
+            i["n"] += 1
+            return {"text": s}
+        if tool == "BrowserFindComposer":
+            r = fc_seq[min(calls["find"], len(fc_seq) - 1)]
+            calls["find"] += 1
+            calls["clicks"].append((tool, params))
+            return r
+        calls["clicks"].append((tool, params))
+        return {"ok": True}
+    return execute, calls
+
+
+@pytest.mark.asyncio
+async def test_structural_finder_fills_when_name_detector_misses(monkeypatch):
+    """The generalization: no AX-named composer and no opener, but the in-page finder
+    ranks an editable region, fills + reads it back, and the script reports filled-ready
+    (dry-run stops before the send). This is what unblocks Reddit-style contenteditables."""
+    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
+    monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
+    ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
+                                  "selector": '[data-osw-composer="1"]', "score": 6.2, "nearSubmit": True})
+    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
+                                 payload_in_textbox, payload_source=TASK,
+                                 current_url="https://www.reddit.com/r/test/comments/x/")
+    assert r is not None and r["sent"] is False           # filled, stopped before the send
+    assert calls["find"] == 1
+    assert any(c[0] == "BrowserFindComposer" for c in calls["clicks"])
+
+
+@pytest.mark.asyncio
+async def test_structural_finder_declines_when_no_editable(monkeypatch):
+    """The finder honestly finds nothing usable (a page with only a search box) -> decline
+    to the model path, never a false fire."""
+    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
+    ex, calls = make_struct_exec({"found": False})
+    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
+                                 payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
+    assert r is None
+    assert calls["find"] == 1
+
+
+@pytest.mark.asyncio
+async def test_structural_off_by_default_never_calls_finder(monkeypatch):
+    """Flag off = the proven name path only; a name-less perception declines and the finder
+    is never invoked (the structural path can't perturb the default)."""
+    monkeypatch.delenv("OSW_COMPOSER_STRUCT", raising=False)
+    ex, calls = make_struct_exec({"found": True, "filled": True, "selector": "x", "role": "textarea"})
+    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
+                                 payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
+    assert r is None
+    assert calls["find"] == 0
+
+
+@pytest.mark.asyncio
+async def test_reveal_flag_passed_to_finder_when_enabled(monkeypatch):
+    """OSW_COMPOSER_REVEAL=1 lets the finder take a reversible reveal action: the send-script
+    must forward reveal=True to BrowserFindComposer (the composer isn't painted yet)."""
+    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
+    monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
+    monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
+    ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
+                                  "selector": '[data-osw-composer="1"]', "score": 6.0,
+                                  "nearSubmit": True, "reveals": ["trigger"]})
+    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
+                                 payload_in_textbox, payload_source=TASK,
+                                 current_url="https://www.linkedin.com/feed/")
+    assert r is not None and r["sent"] is False
+    find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
+    assert find_call[1].get("reveal") is True
+
+
+@pytest.mark.asyncio
+async def test_cross_nav_retry_after_open_first_navigates(monkeypatch):
+    """open-first reveal navigates to the item page (killing the finder's context) so the first
+    call finds nothing; the send-script re-perceives and calls the finder a SECOND time on the
+    destination, where the composer now fills. Proves the cross-nav retry."""
+    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
+    monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
+    monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
+    seq = [
+        {"found": False, "reveals": ["trigger:noop", "open-first", "scroll"]},   # navigated away
+        {"found": True, "filled": True, "role": "contenteditable",
+         "selector": '[data-osw-composer="1"]', "score": 6.0, "reveals": ["scroll"]},
+    ]
+    ex, calls = make_struct_exec(seq)
+    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
+                                 payload_in_textbox, payload_source=TASK,
+                                 current_url="https://www.reddit.com/r/test")
+    assert r is not None and r["sent"] is False
+    assert calls["find"] == 2                       # retried on the destination
+
+
+@pytest.mark.asyncio
+async def test_cross_nav_no_retry_when_reveal_did_not_navigate(monkeypatch):
+    """If the finder did NOT fire open-first (a same-doc trigger/scroll that just failed), there
+    is no destination to re-perceive, so it declines after ONE call, no wasted retry."""
+    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
+    monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
+    ex, calls = make_struct_exec({"found": False, "reveals": ["trigger:noop", "open-first:noop", "scroll"]})
+    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
+                                 payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
+    assert r is None
+    assert calls["find"] == 1
+
+
+@pytest.mark.asyncio
+async def test_reveal_flag_off_by_default(monkeypatch):
+    """Struct on but reveal unset: the finder is still called, but reveal=False, so it only
+    scans what's painted and never clicks a trigger (the safe default)."""
+    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
+    monkeypatch.delenv("OSW_COMPOSER_REVEAL", raising=False)
+    ex, calls = make_struct_exec({"found": False})
+    await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
+                             payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
+    find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
+    assert find_call[1].get("reveal") is False
diff --git a/backend/tests/test_browser_send_script.py b/backend/tests/test_browser_send_script.py
index 3a928d18..bd19ec9f 100644
--- a/backend/tests/test_browser_send_script.py
+++ b/backend/tests/test_browser_send_script.py
@@ -186,102 +186,6 @@ async def test_composed_task_brief_quotes_fire_via_payload_source():
     assert r["payload"] == "[test] hello world r9-os"
 
 
-NAMELESS = '[1]<link "Home">\n[2]<button "Search">'  # no name-matched composer, no opener
-
-
-def make_struct_exec(fc_result, list_states=None):
-    """execute_tool mock for the structural-finder path: list calls return no-composer
-    states; a BrowserFindComposer call returns the scripted structural result."""
-    calls = {"clicks": [], "find": 0}
-    states = list(list_states or [NAMELESS] * 6)
-    i = {"n": 0}
-
-    async def execute(tool, params, bid, tid):
-        if tool == "BrowserListInteractives":
-            s = states[min(i["n"], len(states) - 1)]
-            i["n"] += 1
-            return {"text": s}
-        if tool == "BrowserFindComposer":
-            calls["find"] += 1
-            calls["clicks"].append((tool, params))
-            return fc_result
-        calls["clicks"].append((tool, params))
-        return {"ok": True}
-    return execute, calls
-
-
-@pytest.mark.asyncio
-async def test_structural_finder_fills_when_name_detector_misses(monkeypatch):
-    """The generalization: no AX-named composer and no opener, but the in-page finder
-    ranks an editable region, fills + reads it back, and the script reports filled-ready
-    (dry-run stops before the send). This is what unblocks Reddit-style contenteditables."""
-    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
-    monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
-    ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
-                                  "selector": '[data-osw-composer="1"]', "score": 6.2, "nearSubmit": True})
-    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
-                                 payload_in_textbox, payload_source=TASK,
-                                 current_url="https://www.reddit.com/r/test/comments/x/")
-    assert r is not None and r["sent"] is False           # filled, stopped before the send
-    assert calls["find"] == 1
-    assert any(c[0] == "BrowserFindComposer" for c in calls["clicks"])
-
-
-@pytest.mark.asyncio
-async def test_structural_finder_declines_when_no_editable(monkeypatch):
-    """The finder honestly finds nothing usable (a page with only a search box) -> decline
-    to the model path, never a false fire."""
-    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
-    ex, calls = make_struct_exec({"found": False})
-    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
-                                 payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
-    assert r is None
-    assert calls["find"] == 1
-
-
-@pytest.mark.asyncio
-async def test_structural_off_by_default_never_calls_finder(monkeypatch):
-    """Flag off = the proven name path only; a name-less perception declines and the finder
-    is never invoked (the structural path can't perturb the default)."""
-    monkeypatch.delenv("OSW_COMPOSER_STRUCT", raising=False)
-    ex, calls = make_struct_exec({"found": True, "filled": True, "selector": "x", "role": "textarea"})
-    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
-                                 payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
-    assert r is None
-    assert calls["find"] == 0
-
-
-@pytest.mark.asyncio
-async def test_reveal_flag_passed_to_finder_when_enabled(monkeypatch):
-    """OSW_COMPOSER_REVEAL=1 lets the finder take a reversible reveal action: the send-script
-    must forward reveal=True to BrowserFindComposer (the composer isn't painted yet)."""
-    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
-    monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
-    monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
-    ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
-                                  "selector": '[data-osw-composer="1"]', "score": 6.0,
-                                  "nearSubmit": True, "reveals": ["trigger"]})
-    r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
-                                 payload_in_textbox, payload_source=TASK,
-                                 current_url="https://www.linkedin.com/feed/")
-    assert r is not None and r["sent"] is False
-    find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
-    assert find_call[1].get("reveal") is True
-
-
-@pytest.mark.asyncio
-async def test_reveal_flag_off_by_default(monkeypatch):
-    """Struct on but reveal unset: the finder is still called, but reveal=False, so it only
-    scans what's painted and never clicks a trigger (the safe default)."""
-    monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
-    monkeypatch.delenv("OSW_COMPOSER_REVEAL", raising=False)
-    ex, calls = make_struct_exec({"found": False})
-    await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
-                             payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
-    find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
-    assert find_call[1].get("reveal") is False
-
-
 def test_dryrun_report_encodes_the_gate_funnel():
     """The coverage harness greps this one line for gate attribution: a staged composer
     reports composer=1, a bare page reports zeros, and armed/filled ride the booleans."""
diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts
index 77a4d2ee..c34745bf 100644
--- a/frontend/src/shared/browserCommandHandler.ts
+++ b/frontend/src/shared/browserCommandHandler.ts
@@ -510,7 +510,41 @@ async function handleFindComposer(wv: BrowserWebview, params: Record<string, any
       role: best.isContentEditable ? 'contenteditable' : (best.getAttribute('role') || best.tagName.toLowerCase()),
       score: Math.round(hit.score * 10) / 10, nearSubmit: hit.near, filled, reveals: acts };
   })()`;
-  return await evalInPage(wv, code);
+  const result = await evalInPage(wv, code);
+  // Real-keystroke fill fallback: some editors (Reddit's Lexical, strict React contenteditables)
+  // manage their own state through beforeinput and ignore execCommand insertText, so the in-page
+  // fill reads back empty. Retype the SAME text as OS-level key events (isTrusted, the way a human
+  // types), which those editors DO honor, then re-read to confirm. Only fires when the cheap
+  // in-page fill already failed, so the fast path is untouched.
+  if (fill != null && result && result.found && !result.filled && result.selector) {
+    const ok = await keystrokeFill(wv, String(result.selector), fill);
+    result.filled = ok;
+    result.fillMode = ok ? 'keystroke' : 'keystroke-failed';
+  }
+  return result;
+}
+
+// Type `text` into the element at `selector` as real OS-level key events (Electron
+// sendInputEvent 'char'), for editors that reject synthetic execCommand insertText. Focuses +
+// clears in-page first, types each char natively, then reads the value back in-page to verify.
+async function keystrokeFill(wv: BrowserWebview, selector: string, text: string): Promise<boolean> {
+  const safeSel = JSON.stringify(selector);
+  await evalInPage(wv, `(() => { const el = document.querySelector(${safeSel});
+    if (!el) return false; el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus();
+    if (el.select) el.select(); document.execCommand('selectAll', false); document.execCommand('delete', false);
+    return true; })()`);
+  for (const ch of text) {
+    wv.sendInputEvent({ type: 'char', keyCode: ch });
+  }
+  // Read back from the marked element OR the active element: editors like Reddit's swap the
+  // node on activation, so the original selector can go stale even though the keystrokes landed
+  // in whatever now holds focus. Checking both survives that re-render.
+  const readBack = `(() => {
+    const check = (el) => { if (!el) return false;
+      el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true }));
+      const now = (el.value != null ? el.value : (el.textContent || '')); return now.includes(${JSON.stringify(text)}); };
+    return check(document.querySelector(${safeSel})) || check(document.activeElement); })()`;
+  return Boolean(await evalInPage(wv, readBack));
 }
 
 // Electron sendInputEvent expects names like 'Up', 'Enter', 'Space', not 'ArrowUp'/' '/'Esc'.