WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-08-17 18:25:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

[eric] browser: scope broadened submit vocab to the receipt-gated send-script; keep the always-on Send hint tight (broadening it regressed the default config with the send-script off)

Browse Source
This commit is contained in:
ciregenz
2026-07-15 16:22:10 -07:00
parent 5196ba77e3
commit 449f66057d
4 changed files with 56 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
backend/apps/agents/browser/browser_agent.py
+22 -8
View File
@@ -473,11 +473,14 @@ def delta_state(text: str, seen_lines: set[str]) -> str:
# A button row whose name is exactly a Send control (not "Send InMail credit" or "Send a message to X"); used to hand the model the Send button after it types, so it never burns turns hunting a button that's right there.
P_SEND_ROW_RE = re.compile(r'\[(\d+)\]\*?<\s*button\s+"([^"]*)"', re.I)
# The submit-button labels across the popular composers, so the fast send-path COMPLETES on
# X/IG/FB/Threads/YouTube, not just LinkedIn. EXACT match + button-only (P_SEND_ROW_RE) is the
# safety: "Post" never matches "Post a job", a "Share" menuitem isn't a <button so it's excluded,
# and every caller is post-fill (a hint, or the send-script's receipt-gated click), so a rare
# opener-vs-submit mismatch fails safe (the composer just doesn't clear), never a false send.
# TIGHT set for the ALWAYS-ON model hint (post_action_state). Kept to the unambiguous "Send" family
# on purpose: this hint fires after ANY text fill, so a broad match ("Reply"/"Share"/"Comment"/
# "Post") would mislabel a stray feed button as the Send button after an unrelated search fill.
P_HINT_SEND_LABELS = frozenset({"send", "send now", "send message"})
# BROAD submit vocabulary for the SEND-SCRIPT only (flag-gated + receipt-gated): lets the fast
# send-path COMPLETE on X/IG/FB/Threads/YouTube. Safe ONLY there because the send-script re-verifies
# the composer cleared the exact payload, so an opener-vs-submit mismatch fails safe, never a false
# send. button-only (P_SEND_ROW_RE) + exact match keeps "Post" from matching "Post a job" etc.
P_SEND_LABELS = frozenset({
"send", "send now", "send message", # LinkedIn / Gmail / DMs
"post", "post all", "tweet", "reply", # X / Threads compose + reply
@@ -486,8 +489,19 @@ P_SEND_LABELS = frozenset({
def send_index_in_state(state_text: str):
"""(index, name) of a real submit button in an interactives list, or None. Strict exact
match against P_SEND_LABELS so it never grabs an upsell or a profile 'Send a message' link."""
"""(index, name) of a real Send button for the ALWAYS-ON model hint, or None. TIGHT exact match
(Send family only) so it never mislabels a common feed 'Reply'/'Share'/'Comment' button."""
for line in (state_text or "").splitlines():
m = P_SEND_ROW_RE.search(line)
if m and m.group(2).strip().lower() in P_HINT_SEND_LABELS:
return int(m.group(1)), m.group(2)
return None
def send_submit_index_in_state(state_text: str):
"""(index, name) of a submit button across the popular composers (Post/Reply/Tweet/...), for the
receipt-gated SEND-SCRIPT only. Broader than the hint matcher; safe because the send-script
verifies the composer cleared afterward, so a wrong match aborts, never sends."""
for line in (state_text or "").splitlines():
m = P_SEND_ROW_RE.search(line)
if m and m.group(2).strip().lower() in P_SEND_LABELS:
@@ -1340,7 +1354,7 @@ async def run_browser_agent(
try:
p_script = await asyncio.wait_for(browser_send_script.run_send_script(
task, browser_id, tab_id, preloaded_perception,
execute_browser_tool, send_index_in_state, payload_in_textbox,
execute_browser_tool, send_submit_index_in_state, payload_in_textbox,
payload_source=user_prompt, current_url=current_url,
), timeout=30.0)
except Exception as p_se:
backend/tests/test_browser_agent_loop.py
+18 -12
View File
@@ -1524,18 +1524,24 @@ def test_send_index_handoff_points_only_at_a_real_send_button():
assert send_index_in_state("") is None
def test_send_index_finds_the_popular_composers_submit_buttons():
# the generalization: the fast send-path must COMPLETE on X/IG/FB/YouTube, so the submit
# finder knows Post/Reply/Tweet/etc, not just LinkedIn's "Send".
from backend.apps.agents.browser.browser_agent import send_index_in_state
assert send_index_in_state('[3]<textbox "Post your reply">\n[8]<button "Reply">') == (8, "Reply")
assert send_index_in_state('[2]<textbox "What is happening?">\n[9]<button "Post">') == (9, "Post")
assert send_index_in_state('[4]<button "Tweet">') == (4, "Tweet")
assert send_index_in_state('[7]<button "Comment">') == (7, "Comment")
# exact + button-only keeps the safety: not a look-alike, not a menuitem, not an opener phrase
assert send_index_in_state('[5]<button "Post a job">') is None
assert send_index_in_state('[6]<menuitem "Share">') is None # button-only excludes the Drive-Share false positive
assert send_index_in_state('[8]<button "Reply to Maya">') is None
def test_send_submit_matcher_broad_but_hint_matcher_tight():
# SCOPING: the send-script's submit finder must know Post/Reply/Tweet/etc so the fast path
# COMPLETES on the giants; the ALWAYS-ON model hint must STAY tight so it never mislabels a
# stray feed 'Reply'/'Share'/'Comment' button as the Send button after an unrelated fill.
from backend.apps.agents.browser.browser_agent import send_index_in_state, send_submit_index_in_state
# broad (send-script) finds the popular composers' submit buttons
assert send_submit_index_in_state('[3]<textbox "Post your reply">\n[8]<button "Reply">') == (8, "Reply")
assert send_submit_index_in_state('[2]<textbox "What is happening?">\n[9]<button "Post">') == (9, "Post")
assert send_submit_index_in_state('[4]<button "Tweet">') == (4, "Tweet")
# exact + button-only keeps its own safety
assert send_submit_index_in_state('[5]<button "Post a job">') is None
assert send_submit_index_in_state('[6]<menuitem "Share">') is None
# TIGHT hint matcher: Send family only, and NOT the common feed buttons (the regression guard)
assert send_index_in_state('[44]<button "Send">') == (44, "Send")
assert send_index_in_state('[8]<button "Reply">') is None
assert send_index_in_state('[9]<button "Post">') is None
assert send_index_in_state('[7]<button "Comment">') is None
assert send_index_in_state('[3]<button "Share">') is None
def test_strip_lone_surrogates():
backend/tests/test_browser_reveal.py
+8 -8
View File
@@ -5,7 +5,7 @@ end, including the flag gating and the safety that reveal only forwards under th
import pytest
from backend.apps.agents.browser import browser_send_script as ss
from backend.apps.agents.browser.browser_agent import send_index_in_state, payload_in_textbox
from backend.apps.agents.browser.browser_agent import send_submit_index_in_state, payload_in_textbox
TASK = "go to tyler chen's linkedin hes in entrepreneurs first and text him '[test] hello world r9-os'"
NAMELESS = '[1]<link "Home">\n[2]<button "Search">' # no name-matched composer, no opener
@@ -46,7 +46,7 @@ async def test_structural_finder_fills_when_name_detector_misses(monkeypatch):
monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
"selector": '[data-osw-composer="1"]', "score": 6.2, "nearSubmit": True})
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK,
current_url="https://www.reddit.com/r/test/comments/x/")
assert r is not None and r["sent"] is False # filled, stopped before the send
@@ -60,7 +60,7 @@ async def test_structural_finder_declines_when_no_editable(monkeypatch):
to the model path, never a false fire."""
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
ex, calls = make_struct_exec({"found": False})
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
assert r is None
assert calls["find"] == 1
@@ -72,7 +72,7 @@ async def test_structural_off_by_default_never_calls_finder(monkeypatch):
is never invoked (the structural path can't perturb the default)."""
monkeypatch.delenv("OSW_COMPOSER_STRUCT", raising=False)
ex, calls = make_struct_exec({"found": True, "filled": True, "selector": "x", "role": "textarea"})
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
assert r is None
assert calls["find"] == 0
@@ -88,7 +88,7 @@ async def test_reveal_flag_passed_to_finder_when_enabled(monkeypatch):
ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
"selector": '[data-osw-composer="1"]', "score": 6.0,
"nearSubmit": True, "reveals": ["trigger"]})
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK,
current_url="https://www.linkedin.com/feed/")
assert r is not None and r["sent"] is False
@@ -110,7 +110,7 @@ async def test_cross_nav_retry_after_open_first_navigates(monkeypatch):
"selector": '[data-osw-composer="1"]', "score": 6.0, "reveals": ["scroll"]},
]
ex, calls = make_struct_exec(seq)
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK,
current_url="https://www.reddit.com/r/test")
assert r is not None and r["sent"] is False
@@ -124,7 +124,7 @@ async def test_cross_nav_no_retry_when_reveal_did_not_navigate(monkeypatch):
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
ex, calls = make_struct_exec({"found": False, "reveals": ["trigger:noop", "open-first:noop", "scroll"]})
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
assert r is None
assert calls["find"] == 1
@@ -137,7 +137,7 @@ async def test_reveal_flag_off_by_default(monkeypatch):
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
monkeypatch.delenv("OSW_COMPOSER_REVEAL", raising=False)
ex, calls = make_struct_exec({"found": False})
await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state,
await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
assert find_call[1].get("reveal") is False
backend/tests/test_browser_send_script.py
+8 -8
View File
@@ -6,7 +6,7 @@ the correctness the live run would prove is."""
import pytest
from backend.apps.agents.browser import browser_send_script as ss
from backend.apps.agents.browser.browser_agent import send_index_in_state, payload_in_textbox
from backend.apps.agents.browser.browser_agent import send_submit_index_in_state, payload_in_textbox
PROFILE = '[22]*<link "Tyler Chen Premium 1st">\n[50]*<link "Message">\n[51]<button "Follow">'
COMPOSER_EMPTY = '[2]<textbox "Write a message">\n[9]<button "Attach">'
@@ -45,7 +45,7 @@ PROFILE_URL = "https://www.linkedin.com/in/tylerchen1200/"
async def run(task, state0, list_script, url=THREAD_URL):
ex, calls = make_exec(list_script)
r = await ss.run_send_script(task, "b1", "", state0, ex, send_index_in_state,
r = await ss.run_send_script(task, "b1", "", state0, ex, send_submit_index_in_state,
payload_in_textbox, current_url=url)
return r, calls
@@ -61,7 +61,7 @@ async def test_surface_gate_declines_when_no_composer_in_perception():
messaging opener declines UNTOUCHED, regardless of URL, so the script never fires
where a fill would land nowhere useful."""
ex, calls = make_exec([COMPOSER_FILLED, COMPOSER_FILLED, COMPOSER_SENT])
r = await ss.run_send_script(TASK, "b1", "", NO_COMPOSER, ex, send_index_in_state,
r = await ss.run_send_script(TASK, "b1", "", NO_COMPOSER, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK, current_url=FEED_URL)
assert r is None
assert not calls["clicks"]
@@ -77,7 +77,7 @@ async def test_surface_gate_fires_on_a_NON_linkedin_composer():
X_FILLED = '[3]<textbox "Post your reply" value="[test] hello world r9-os">\n[8]<button "Reply">'
X_SENT = '[3]<textbox "Post your reply">\n[1]<link "Home">'
ex, calls = make_exec([X_FILLED, X_FILLED, X_SENT])
r = await ss.run_send_script(TASK, "b1", "", X_COMPOSER, ex, send_index_in_state,
r = await ss.run_send_script(TASK, "b1", "", X_COMPOSER, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK,
current_url="https://x.com/messages/123")
assert r is not None and r["sent"] is True
@@ -91,7 +91,7 @@ async def test_surface_gate_allows_profile_overlay():
"""The profile /in/ overlay is winnable via click-by-name (ground truth: its Send
ranks out of the list but is a real button), so it's back in scope, not declined."""
ex, calls = make_exec([COMPOSER_FILLED_NO_SEND, COMPOSER_FILLED_NO_SEND, COMPOSER_SENT])
r = await ss.run_send_script(TASK, "b1", "", COMPOSER_EMPTY, ex, send_index_in_state,
r = await ss.run_send_script(TASK, "b1", "", COMPOSER_EMPTY, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK, current_url=PROFILE_URL)
assert r is not None and r["sent"] is True
@@ -102,7 +102,7 @@ async def test_send_via_click_by_name_when_send_absent_from_ranked_list():
box but NO Send in the capped numbered list. The script falls back to
click-by-name (the model's own send path there) and the receipt still passes."""
ex, calls = make_exec([COMPOSER_FILLED_NO_SEND, COMPOSER_FILLED_NO_SEND, COMPOSER_SENT])
r = await ss.run_send_script(TASK, "b1", "", COMPOSER_EMPTY, ex, send_index_in_state,
r = await ss.run_send_script(TASK, "b1", "", COMPOSER_EMPTY, ex, send_submit_index_in_state,
payload_in_textbox, payload_source=TASK, current_url=THREAD_URL)
assert r is not None and r["sent"] is True
# the send went through click-by-name, not an index click
@@ -186,7 +186,7 @@ async def test_composed_task_brief_quotes_fire_via_payload_source():
real dispatch (r242/r243 declined live); the raw user prompt rides separately."""
ex, calls = make_exec([COMPOSER_FILLED, COMPOSER_FILLED, COMPOSER_SENT])
r = await ss.run_send_script(COMPOSED, "b1", "", COMPOSER_EMPTY, ex,
send_index_in_state, payload_in_textbox,
send_submit_index_in_state, payload_in_textbox,
payload_source=TASK, current_url=THREAD_URL)
assert r is not None and r["sent"] is True
assert r["payload"] == "[test] hello world r9-os"
@@ -220,7 +220,7 @@ async def test_readonly_probe_never_fires():
)
ex, calls = make_exec([COMPOSER_FILLED, COMPOSER_FILLED, COMPOSER_SENT])
r = await ss.run_send_script(probe, "b1", "", COMPOSER_EMPTY, ex,
send_index_in_state, payload_in_textbox,
send_submit_index_in_state, payload_in_textbox,
payload_source=TASK, current_url=THREAD_URL)
assert r is None
assert not calls["clicks"]