mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-12 20:57:42 +02:00
[eric] browser: keystroke-fill fallback + cross-nav reveal retry (both flag-gated, default off)
Item 1: when the finder's execCommand insertText read-back fails (editors that reject synthetic input), retype the same text as real OS-level key events via wv.sendInputEvent, then verify from the marked element OR document.activeElement (editors like Reddit swap the node on activation, staling the selector). Item 3: when an open-first reveal navigates to a new document (Reddit thread, TikTok video, GitHub issue), the send-script re-perceives after a load beat and calls the finder once more on the destination, bounded to 2 tries. 4 new tests (cross-nav retry fires / no wasted retry) + structural tests split into test_browser_reveal.py to stay under the 300-line cap. Live real-path validation of the sendInputEvent leg is owed on a healthy rig: the eric/browser agent loop wedges at fresh-card webview mount (15.25s cap, renderer starvation, predates the suspend fix) at low load too
This commit is contained in:
ciregenz
@@ -17,7 +17,7 @@ import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from typing import Awaitable, Callable
|
||||
from typing import Awaitable, Callable, Dict
|
||||
|
||||
from backend.apps.agents.browser import browser_verified_action
|
||||
|
||||
@@ -184,12 +184,29 @@ async def run_send_script(
|
||||
# compose surface: a modal trigger, the first conversation, or a scroll) when the
|
||||
# composer isn't painted yet. It never commits a send, only opens a surface.
|
||||
p_reveal = os.environ.get("OSW_COMPOSER_REVEAL") == "1"
|
||||
fc = await execute_tool("BrowserFindComposer", {"fill": payload, "reveal": p_reveal}, browser_id, tab_id)
|
||||
# A reveal that OPENS the first list item (a Reddit thread, a TikTok video, a GitHub
|
||||
# issue) is a full-page NAVIGATION: it kills the finder's own JS context, so that one
|
||||
# call can't reach the composer that only exists on the destination. When the finder
|
||||
# reports it fired `open-first` but found nothing, the page is now loading the item;
|
||||
# give it a beat and run the finder ONCE more on the destination. Bounded to 2 tries so
|
||||
# a feed-of-feeds can't walk forever.
|
||||
fc: Dict[str, object] = {}
|
||||
for attempt in range(2):
|
||||
fc = await execute_tool("BrowserFindComposer", {"fill": payload, "reveal": p_reveal}, browser_id, tab_id)
|
||||
if isinstance(fc, dict) and fc.get("found") and fc.get("filled"):
|
||||
break
|
||||
revs = fc.get("reveals") if isinstance(fc, dict) else None
|
||||
navigated = p_reveal and isinstance(revs, list) and "open-first" in revs
|
||||
if not navigated:
|
||||
break
|
||||
logger.info("[browser-sendscript] reveal navigated (open-first); re-perceiving the destination")
|
||||
await asyncio.sleep(1.5)
|
||||
await fresh_list()
|
||||
if isinstance(fc, dict) and fc.get("found") and fc.get("filled"):
|
||||
p_struct_selector = str(fc.get("selector") or "")
|
||||
logger.info(f"[browser-sendscript] structural composer role={fc.get('role')!r} "
|
||||
f"score={fc.get('score')} nearSubmit={fc.get('nearSubmit')} "
|
||||
f"reveals={fc.get('reveals')} filled+verified")
|
||||
f"reveals={fc.get('reveals')} fillMode={fc.get('fillMode')} filled+verified")
|
||||
log.append({"tool": "BrowserFindComposer", "input": {"fill": "<payload>"}, "ok": True,
|
||||
"result_summary": f"structural composer {fc.get('role')!r} filled+verified"[:200], "elapsed_ms": 0})
|
||||
composer = (-1, str(fc.get("role") or "composer"))
|
||||
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Structural composer finder + reveal-and-find reach + cross-nav retry, verified without a
|
||||
live webview: a mock executor scripts BrowserFindComposer results so the send-script's
|
||||
structural fallback path (find -> reveal -> re-perceive-on-nav -> fill) is exercised end to
|
||||
end, including the flag gating and the safety that reveal only forwards under the flag."""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_send_script as ss
|
||||
from backend.apps.agents.browser.browser_agent import send_index_in_state, payload_in_textbox
|
||||
|
||||
TASK = "go to tyler chen's linkedin hes in entrepreneurs first and text him '[test] hello world r9-os'"
|
||||
NAMELESS = '[1]<link "Home">\n[2]<button "Search">' # no name-matched composer, no opener
|
||||
|
||||
|
||||
def make_struct_exec(fc_result, list_states=None):
|
||||
"""execute_tool mock for the structural-finder path: list calls return no-composer
|
||||
states; a BrowserFindComposer call returns the scripted structural result."""
|
||||
calls = {"clicks": [], "find": 0}
|
||||
states = list(list_states or [NAMELESS] * 6)
|
||||
i = {"n": 0}
|
||||
|
||||
# fc_result may be a single dict (returned every call) or a list of dicts (returned in
|
||||
# sequence, last one repeating) to script the cross-nav retry.
|
||||
fc_seq = fc_result if isinstance(fc_result, list) else [fc_result]
|
||||
|
||||
async def execute(tool, params, bid, tid):
|
||||
if tool == "BrowserListInteractives":
|
||||
s = states[min(i["n"], len(states) - 1)]
|
||||
i["n"] += 1
|
||||
return {"text": s}
|
||||
if tool == "BrowserFindComposer":
|
||||
r = fc_seq[min(calls["find"], len(fc_seq) - 1)]
|
||||
calls["find"] += 1
|
||||
calls["clicks"].append((tool, params))
|
||||
return r
|
||||
calls["clicks"].append((tool, params))
|
||||
return {"ok": True}
|
||||
return execute, calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structural_finder_fills_when_name_detector_misses(monkeypatch):
|
||||
"""The generalization: no AX-named composer and no opener, but the in-page finder
|
||||
ranks an editable region, fills + reads it back, and the script reports filled-ready
|
||||
(dry-run stops before the send). This is what unblocks Reddit-style contenteditables."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
|
||||
ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
|
||||
"selector": '[data-osw-composer="1"]', "score": 6.2, "nearSubmit": True})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK,
|
||||
current_url="https://www.reddit.com/r/test/comments/x/")
|
||||
assert r is not None and r["sent"] is False # filled, stopped before the send
|
||||
assert calls["find"] == 1
|
||||
assert any(c[0] == "BrowserFindComposer" for c in calls["clicks"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structural_finder_declines_when_no_editable(monkeypatch):
|
||||
"""The finder honestly finds nothing usable (a page with only a search box) -> decline
|
||||
to the model path, never a false fire."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
ex, calls = make_struct_exec({"found": False})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
assert r is None
|
||||
assert calls["find"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structural_off_by_default_never_calls_finder(monkeypatch):
|
||||
"""Flag off = the proven name path only; a name-less perception declines and the finder
|
||||
is never invoked (the structural path can't perturb the default)."""
|
||||
monkeypatch.delenv("OSW_COMPOSER_STRUCT", raising=False)
|
||||
ex, calls = make_struct_exec({"found": True, "filled": True, "selector": "x", "role": "textarea"})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
assert r is None
|
||||
assert calls["find"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reveal_flag_passed_to_finder_when_enabled(monkeypatch):
|
||||
"""OSW_COMPOSER_REVEAL=1 lets the finder take a reversible reveal action: the send-script
|
||||
must forward reveal=True to BrowserFindComposer (the composer isn't painted yet)."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
|
||||
monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
|
||||
ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
|
||||
"selector": '[data-osw-composer="1"]', "score": 6.0,
|
||||
"nearSubmit": True, "reveals": ["trigger"]})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK,
|
||||
current_url="https://www.linkedin.com/feed/")
|
||||
assert r is not None and r["sent"] is False
|
||||
find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
|
||||
assert find_call[1].get("reveal") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_nav_retry_after_open_first_navigates(monkeypatch):
|
||||
"""open-first reveal navigates to the item page (killing the finder's context) so the first
|
||||
call finds nothing; the send-script re-perceives and calls the finder a SECOND time on the
|
||||
destination, where the composer now fills. Proves the cross-nav retry."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
|
||||
monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
|
||||
seq = [
|
||||
{"found": False, "reveals": ["trigger:noop", "open-first", "scroll"]}, # navigated away
|
||||
{"found": True, "filled": True, "role": "contenteditable",
|
||||
"selector": '[data-osw-composer="1"]', "score": 6.0, "reveals": ["scroll"]},
|
||||
]
|
||||
ex, calls = make_struct_exec(seq)
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK,
|
||||
current_url="https://www.reddit.com/r/test")
|
||||
assert r is not None and r["sent"] is False
|
||||
assert calls["find"] == 2 # retried on the destination
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_nav_no_retry_when_reveal_did_not_navigate(monkeypatch):
|
||||
"""If the finder did NOT fire open-first (a same-doc trigger/scroll that just failed), there
|
||||
is no destination to re-perceive, so it declines after ONE call, no wasted retry."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
|
||||
ex, calls = make_struct_exec({"found": False, "reveals": ["trigger:noop", "open-first:noop", "scroll"]})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
assert r is None
|
||||
assert calls["find"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reveal_flag_off_by_default(monkeypatch):
|
||||
"""Struct on but reveal unset: the finder is still called, but reveal=False, so it only
|
||||
scans what's painted and never clicks a trigger (the safe default)."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.delenv("OSW_COMPOSER_REVEAL", raising=False)
|
||||
ex, calls = make_struct_exec({"found": False})
|
||||
await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
|
||||
assert find_call[1].get("reveal") is False
|
||||
@@ -186,102 +186,6 @@ async def test_composed_task_brief_quotes_fire_via_payload_source():
|
||||
assert r["payload"] == "[test] hello world r9-os"
|
||||
|
||||
|
||||
NAMELESS = '[1]<link "Home">\n[2]<button "Search">' # no name-matched composer, no opener
|
||||
|
||||
|
||||
def make_struct_exec(fc_result, list_states=None):
|
||||
"""execute_tool mock for the structural-finder path: list calls return no-composer
|
||||
states; a BrowserFindComposer call returns the scripted structural result."""
|
||||
calls = {"clicks": [], "find": 0}
|
||||
states = list(list_states or [NAMELESS] * 6)
|
||||
i = {"n": 0}
|
||||
|
||||
async def execute(tool, params, bid, tid):
|
||||
if tool == "BrowserListInteractives":
|
||||
s = states[min(i["n"], len(states) - 1)]
|
||||
i["n"] += 1
|
||||
return {"text": s}
|
||||
if tool == "BrowserFindComposer":
|
||||
calls["find"] += 1
|
||||
calls["clicks"].append((tool, params))
|
||||
return fc_result
|
||||
calls["clicks"].append((tool, params))
|
||||
return {"ok": True}
|
||||
return execute, calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structural_finder_fills_when_name_detector_misses(monkeypatch):
|
||||
"""The generalization: no AX-named composer and no opener, but the in-page finder
|
||||
ranks an editable region, fills + reads it back, and the script reports filled-ready
|
||||
(dry-run stops before the send). This is what unblocks Reddit-style contenteditables."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
|
||||
ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
|
||||
"selector": '[data-osw-composer="1"]', "score": 6.2, "nearSubmit": True})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK,
|
||||
current_url="https://www.reddit.com/r/test/comments/x/")
|
||||
assert r is not None and r["sent"] is False # filled, stopped before the send
|
||||
assert calls["find"] == 1
|
||||
assert any(c[0] == "BrowserFindComposer" for c in calls["clicks"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structural_finder_declines_when_no_editable(monkeypatch):
|
||||
"""The finder honestly finds nothing usable (a page with only a search box) -> decline
|
||||
to the model path, never a false fire."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
ex, calls = make_struct_exec({"found": False})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
assert r is None
|
||||
assert calls["find"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structural_off_by_default_never_calls_finder(monkeypatch):
|
||||
"""Flag off = the proven name path only; a name-less perception declines and the finder
|
||||
is never invoked (the structural path can't perturb the default)."""
|
||||
monkeypatch.delenv("OSW_COMPOSER_STRUCT", raising=False)
|
||||
ex, calls = make_struct_exec({"found": True, "filled": True, "selector": "x", "role": "textarea"})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
assert r is None
|
||||
assert calls["find"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reveal_flag_passed_to_finder_when_enabled(monkeypatch):
|
||||
"""OSW_COMPOSER_REVEAL=1 lets the finder take a reversible reveal action: the send-script
|
||||
must forward reveal=True to BrowserFindComposer (the composer isn't painted yet)."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
|
||||
monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
|
||||
ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
|
||||
"selector": '[data-osw-composer="1"]', "score": 6.0,
|
||||
"nearSubmit": True, "reveals": ["trigger"]})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK,
|
||||
current_url="https://www.linkedin.com/feed/")
|
||||
assert r is not None and r["sent"] is False
|
||||
find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
|
||||
assert find_call[1].get("reveal") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reveal_flag_off_by_default(monkeypatch):
|
||||
"""Struct on but reveal unset: the finder is still called, but reveal=False, so it only
|
||||
scans what's painted and never clicks a trigger (the safe default)."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.delenv("OSW_COMPOSER_REVEAL", raising=False)
|
||||
ex, calls = make_struct_exec({"found": False})
|
||||
await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
|
||||
assert find_call[1].get("reveal") is False
|
||||
|
||||
|
||||
def test_dryrun_report_encodes_the_gate_funnel():
|
||||
"""The coverage harness greps this one line for gate attribution: a staged composer
|
||||
reports composer=1, a bare page reports zeros, and armed/filled ride the booleans."""
|
||||
|
||||
@@ -510,7 +510,41 @@ async function handleFindComposer(wv: BrowserWebview, params: Record<string, any
|
||||
role: best.isContentEditable ? 'contenteditable' : (best.getAttribute('role') || best.tagName.toLowerCase()),
|
||||
score: Math.round(hit.score * 10) / 10, nearSubmit: hit.near, filled, reveals: acts };
|
||||
})()`;
|
||||
return await evalInPage(wv, code);
|
||||
const result = await evalInPage(wv, code);
|
||||
// Real-keystroke fill fallback: some editors (Reddit's Lexical, strict React contenteditables)
|
||||
// manage their own state through beforeinput and ignore execCommand insertText, so the in-page
|
||||
// fill reads back empty. Retype the SAME text as OS-level key events (isTrusted, the way a human
|
||||
// types), which those editors DO honor, then re-read to confirm. Only fires when the cheap
|
||||
// in-page fill already failed, so the fast path is untouched.
|
||||
if (fill != null && result && result.found && !result.filled && result.selector) {
|
||||
const ok = await keystrokeFill(wv, String(result.selector), fill);
|
||||
result.filled = ok;
|
||||
result.fillMode = ok ? 'keystroke' : 'keystroke-failed';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Type `text` into the element at `selector` as real OS-level key events (Electron
|
||||
// sendInputEvent 'char'), for editors that reject synthetic execCommand insertText. Focuses +
|
||||
// clears in-page first, types each char natively, then reads the value back in-page to verify.
|
||||
async function keystrokeFill(wv: BrowserWebview, selector: string, text: string): Promise<boolean> {
|
||||
const safeSel = JSON.stringify(selector);
|
||||
await evalInPage(wv, `(() => { const el = document.querySelector(${safeSel});
|
||||
if (!el) return false; el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus();
|
||||
if (el.select) el.select(); document.execCommand('selectAll', false); document.execCommand('delete', false);
|
||||
return true; })()`);
|
||||
for (const ch of text) {
|
||||
wv.sendInputEvent({ type: 'char', keyCode: ch });
|
||||
}
|
||||
// Read back from the marked element OR the active element: editors like Reddit's swap the
|
||||
// node on activation, so the original selector can go stale even though the keystrokes landed
|
||||
// in whatever now holds focus. Checking both survives that re-render.
|
||||
const readBack = `(() => {
|
||||
const check = (el) => { if (!el) return false;
|
||||
el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
const now = (el.value != null ? el.value : (el.textContent || '')); return now.includes(${JSON.stringify(text)}); };
|
||||
return check(document.querySelector(${safeSel})) || check(document.activeElement); })()`;
|
||||
return Boolean(await evalInPage(wv, readBack));
|
||||
}
|
||||
|
||||
// Electron sendInputEvent expects names like 'Up', 'Enter', 'Space', not 'ArrowUp'/' '/'Esc'.
|
||||
|
||||
Reference in New Issue
Block a user