mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-20 03:35:40 +02:00
[eric] browser: send-probe gate catches focus-type payloads, batched Enter blocked while composer pending
This commit is contained in:
@@ -3401,7 +3401,7 @@ class AgentManager:
|
||||
text = browser_fast_path.NO_DASHBOARD_REPLY
|
||||
else:
|
||||
from backend.apps.agents.browser import browser_batch_replay
|
||||
payload = browser_batch_replay.send_payload_from_log(first.get("action_log"))
|
||||
payload = browser_batch_replay.send_payload_from_log(first.get("action_log"), prompt)
|
||||
if payload:
|
||||
# The dead attempt had already typed into a composer, so a
|
||||
# blind retry risks a double-send: a read-only probe's
|
||||
|
||||
@@ -1246,7 +1246,8 @@ async def run_browser_agent(
|
||||
# Hard send-guard: an irreversible step physically cannot ride in a
|
||||
# batch (the solo-send rule was prompt-only before; prompts drift).
|
||||
_guard_why = (browser_batch_replay.live_batch_guard(
|
||||
(tool_input or {}).get("actions"), attached_state_seen)
|
||||
(tool_input or {}).get("actions"), attached_state_seen,
|
||||
composer_pending=bool(browser_batch_replay.send_payload_from_log(action_log)))
|
||||
if tu.name == "BrowserBatch" else "")
|
||||
if _guard_why:
|
||||
batch_guard_blocks += 1
|
||||
|
||||
@@ -115,14 +115,16 @@ _LIVE_IRREVERSIBLE_RE = re.compile(
|
||||
)
|
||||
|
||||
|
||||
def live_batch_guard(actions, seen_lines) -> str:
|
||||
def live_batch_guard(actions, seen_lines, composer_pending: bool = False) -> str:
|
||||
"""Reason string if a live BrowserBatch carries an irreversible step, else ''.
|
||||
|
||||
The solo-send rule was prompt-only until now; this makes it physical. A
|
||||
click_index resolves to its element line from the last attached state (an
|
||||
unresolvable index passes: it fails at execution anyway), and Enter after
|
||||
typing into a composer counts as the send it is."""
|
||||
typed_composer = False
|
||||
typing into a composer counts as the send it is. composer_pending arms the
|
||||
Enter check across turns: r47 typed solo then batched [Enter, wait], which
|
||||
slid past the within-batch check."""
|
||||
typed_composer = composer_pending
|
||||
for i, a in enumerate(actions or []):
|
||||
if not isinstance(a, dict):
|
||||
continue
|
||||
@@ -153,7 +155,7 @@ def live_batch_guard(actions, seen_lines) -> str:
|
||||
return ""
|
||||
|
||||
|
||||
def send_payload_from_log(action_log) -> str:
|
||||
def send_payload_from_log(action_log, prompt: str = "") -> str:
|
||||
"""The text a failed run typed into a composer-ish field, '' if it never
|
||||
reached the send zone. Gates the recovery verify-first probe: r44's retry
|
||||
SAID it would verify first then didn't, so the check must be code, not prose."""
|
||||
@@ -169,11 +171,15 @@ def send_payload_from_log(action_log) -> str:
|
||||
if tool == "BrowserClickIndex":
|
||||
name = str(a.get("clicked_name") or "")
|
||||
role = str(a.get("clicked_role") or "")
|
||||
# a filter/search box also has role textbox; real messages are longer
|
||||
if _COMPOSE_SEL_RE.search(name) or (role == "textbox" and len(text) >= 20):
|
||||
summ = str(a.get("result_summary") or "")
|
||||
# focus+type results carry no clicked fields (r47's live miss); the
|
||||
# executor's own "typed the text" wording is the surviving signal
|
||||
if _COMPOSE_SEL_RE.search(name) or (len(text) >= 20 and (
|
||||
role == "textbox" or "typed the text" in summ.lower())):
|
||||
typed.append(text)
|
||||
elif tool == "BrowserType":
|
||||
if _COMPOSE_SEL_RE.search(str(inp.get("selector") or "")):
|
||||
sel = str(inp.get("selector") or "")
|
||||
if _COMPOSE_SEL_RE.search(sel) or (not sel and len(text) >= 20):
|
||||
typed.append(text)
|
||||
elif tool == "BrowserBatch":
|
||||
for sub in (inp.get("actions") or []):
|
||||
@@ -181,10 +187,19 @@ def send_payload_from_log(action_log) -> str:
|
||||
continue
|
||||
p = sub.get("params") if isinstance(sub.get("params"), dict) else {}
|
||||
sub_text = str(p.get("text") or "").strip()
|
||||
if (sub.get("type") == "type" and sub_text
|
||||
and _COMPOSE_SEL_RE.search(str(p.get("selector") or ""))):
|
||||
sub_sel = str(p.get("selector") or "")
|
||||
if sub.get("type") == "type" and sub_text and (
|
||||
_COMPOSE_SEL_RE.search(sub_sel)
|
||||
or (not sub_sel and len(sub_text) >= 20)):
|
||||
typed.append(sub_text)
|
||||
return max(typed, key=len) if typed else ""
|
||||
if not typed:
|
||||
return ""
|
||||
# the task usually quotes the message; a candidate echoed there beats a
|
||||
# longer search query or a garbled retype
|
||||
for t in reversed(typed):
|
||||
if t in (prompt or ""):
|
||||
return t
|
||||
return typed[-1]
|
||||
|
||||
|
||||
def _sub(val, value: str):
|
||||
|
||||
@@ -252,3 +252,57 @@ def test_payload_empty_log_and_garbage_safe():
|
||||
assert br.send_payload_from_log([]) == ""
|
||||
assert br.send_payload_from_log(None) == ""
|
||||
assert br.send_payload_from_log([{"tool": "BrowserClickIndex"}, "junk"]) == ""
|
||||
|
||||
|
||||
def test_payload_extracted_from_focus_type_click_without_clicked_fields():
|
||||
# r47's live miss: focus+type results carry no clickedRole/clickedName
|
||||
log = [{
|
||||
"tool": "BrowserClickIndex",
|
||||
"input": {"index": 1, "text": "[test] hello world r47-os"},
|
||||
"result_summary": 'Focused index 1 and typed the text in (via editor command). Verified: the box now contains "[test] hello world r47-os". Do NOT type it again.',
|
||||
"clicked_role": None, "clicked_name": None,
|
||||
}]
|
||||
assert br.send_payload_from_log(log) == "[test] hello world r47-os"
|
||||
|
||||
|
||||
def test_payload_prefers_prompt_quoted_candidate_over_garbled_retype():
|
||||
clean = "[test] hello world r47-os"
|
||||
garbled = "[test] hello world r47-os\n[test] hello world r47-os"
|
||||
log = [
|
||||
{"tool": "BrowserClickIndex", "input": {"index": 1, "text": clean},
|
||||
"result_summary": "typed the text in", "clicked_role": "textbox", "clicked_name": ""},
|
||||
{"tool": "BrowserClickIndex", "input": {"index": 1, "text": garbled},
|
||||
"result_summary": "typed the text in", "clicked_role": "textbox", "clicked_name": ""},
|
||||
]
|
||||
prompt = f"go to tyler chen's linkedin and text him '{clean}'"
|
||||
assert br.send_payload_from_log(log, prompt) == clean
|
||||
assert br.send_payload_from_log(log) == garbled
|
||||
|
||||
|
||||
def test_payload_from_index_based_batch_type():
|
||||
log = [{
|
||||
"tool": "BrowserBatch",
|
||||
"input": {"actions": [
|
||||
{"type": "click_index", "params": {"index": 4}},
|
||||
{"type": "type", "params": {"index": 4, "text": "[test] hello world long enough"}},
|
||||
]},
|
||||
}]
|
||||
assert br.send_payload_from_log(log) == "[test] hello world long enough"
|
||||
|
||||
|
||||
def test_guard_blocks_batched_enter_when_composer_pending():
|
||||
actions = [
|
||||
{"type": "press_key", "params": {"key": "Enter"}},
|
||||
{"type": "wait", "params": {"milliseconds": 3000}},
|
||||
]
|
||||
why = br.live_batch_guard(actions, [], composer_pending=True)
|
||||
assert "Enter" in why
|
||||
assert br.live_batch_guard(actions, [], composer_pending=False) == ""
|
||||
|
||||
|
||||
def test_guard_still_allows_search_type_enter_without_pending_composer():
|
||||
actions = [
|
||||
{"type": "type", "params": {"selector": "input[name=q]", "text": "tyler chen"}},
|
||||
{"type": "press_key", "params": {"key": "Enter"}},
|
||||
]
|
||||
assert br.live_batch_guard(actions, [], composer_pending=False) == ""
|
||||
|
||||
Reference in New Issue
Block a user