From 10f9c12a8a12cbd20d3e71bc8a393a61b4da8707 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Tue, 21 Jul 2026 21:02:55 -0700 Subject: [PATCH] [eric] browser: submit picker below-composer scope + container tier; readonly guard sealed --- backend/apps/agents/browser/browser_agent.py | 41 ++++++++--- .../apps/agents/browser/browser_fast_path.py | 7 +- .../agents/browser/browser_send_script.py | 44 ++++++++---- .../agents/browser/browser_submit_click.py | 69 +++++++++++++++++++ backend/tests/test_browser_agent_loop.py | 13 ++++ backend/tests/test_browser_send_script.py | 16 +++++ backend/tests/test_browser_submit_click.py | 64 +++++++++++++++++ 7 files changed, 231 insertions(+), 23 deletions(-) create mode 100644 backend/apps/agents/browser/browser_submit_click.py create mode 100644 backend/tests/test_browser_submit_click.py diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index f2e0b596..e9b92440 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -62,6 +62,8 @@ P_WRAPUP_NUDGE = ( from backend.apps.agents.browser import browser_batch_replay from backend.apps.agents.browser import browser_extract from backend.apps.agents.browser import browser_metrics +from backend.apps.agents.browser import browser_send_script +from backend.apps.agents.browser import browser_submit_click from backend.apps.agents.browser import browser_playbook from backend.apps.agents.browser import browser_save from backend.apps.agents.browser import browser_meta_playbook @@ -525,11 +527,8 @@ P_HINT_SEND_LABELS = frozenset({"send", "send now", "send message"}) # send-path COMPLETE on X/IG/FB/Threads/YouTube. Safe ONLY there because the send-script re-verifies # the composer cleared the exact payload, so an opener-vs-submit mismatch fails safe, never a false # send. button-only (P_SEND_ROW_RE) + exact match keeps "Post" from matching "Post a job" etc. -P_SEND_LABELS = frozenset({ - "send", "send now", "send message", # LinkedIn / Gmail / DMs - "post", "post all", "tweet", "reply", # X / Threads compose + reply - "publish", "comment", "share", # articles / YouTube+FB comments / shares -}) +# Defined in browser_submit_click so the container-scoped JS tier shares the exact same set. +P_SEND_LABELS = browser_submit_click.SEND_LABELS def send_index_in_state(state_text: str): @@ -542,13 +541,16 @@ def send_index_in_state(state_text: str): return None -def send_submit_index_in_state(state_text: str): +def send_submit_index_in_state(state_text: str, after_index: int = -1): """(index, name) of a submit button across the popular composers (Post/Reply/Tweet/...), for the receipt-gated SEND-SCRIPT only. Broader than the hint matcher; safe because the send-script - verifies the composer cleared afterward, so a wrong match aborts, never sends.""" + verifies the composer cleared afterward, so a wrong match aborts, never sends. `after_index` + scopes the scan to buttons BELOW the filled composer: every real submit follows its editor in + the listing, while a compose OPENER (X's sidebar "Post") sits above it, and clicking the opener + posts nothing (measured live: 0/2 X deliveries the day X shipped its opener as a button).""" for line in (state_text or "").splitlines(): m = P_SEND_ROW_RE.search(line) - if m and m.group(2).strip().lower() in P_SEND_LABELS: + if m and m.group(2).strip().lower() in P_SEND_LABELS and int(m.group(1)) > after_index: return int(m.group(1)), m.group(2) return None @@ -615,6 +617,25 @@ def fill_text_of(tool_name: str, tool_input: dict) -> str: return "" +def fill_index_of(tool_name: str, tool_input: dict) -> int: + """The listing index a composer-fill action typed into, -1 when unknown. Mirrors fill_text_of.""" + ti = tool_input or {} + if tool_name in ("BrowserClickIndex", "BrowserType"): + try: + return int(ti.get("index", -1)) + except (TypeError, ValueError): + return -1 + if tool_name == "BrowserBatch": + for a in (ti.get("actions") or []): + p = a.get("params") or {} + if a.get("type") in ("type", "click_index") and str(p.get("text") or "").strip(): + try: + return int(p.get("index", -1)) + except (TypeError, ValueError): + return -1 + return -1 + + def payload_in_textbox(state_text: str, payload: str) -> bool: """True if any listed textbox VALUE carries the typed payload (fill committed). Matches on a prefix because long payloads truncate in the list.""" @@ -1446,7 +1467,6 @@ async def run_browser_agent( batch_guard_blocks = 0 # Staged-send script: prestage left a ready composer + the task quotes its payload -> code runs the fill/verify/send/verify tail the model spends 4-5 turns on. Success skips the loop entirely (turns=0); any pre-click ambiguity falls through untouched. - from backend.apps.agents.browser import browser_send_script p_script = None # A removal task ("delete the post that says 'X'") is also task_is_send (the classifier keys # on the verb), so the send-script must stand down or it TYPES the target into a composer and @@ -2335,7 +2355,8 @@ async def run_browser_agent( and browser_send_script.autosend_enabled()): p_cs = await browser_send_script.complete_send( composer_committed_payload, p_auto_state or "", browser_id, tab_id, - execute_browser_tool, send_submit_index_in_state) + execute_browser_tool, send_submit_index_in_state, + composer_index=fill_index_of(tu.name, tool_input)) if p_cs.get("clicked"): send_confirmed = True action_log.extend(p_cs.get("log") or []) diff --git a/backend/apps/agents/browser/browser_fast_path.py b/backend/apps/agents/browser/browser_fast_path.py index c52a32d2..fbfe9b9b 100644 --- a/backend/apps/agents/browser/browser_fast_path.py +++ b/backend/apps/agents/browser/browser_fast_path.py @@ -117,6 +117,11 @@ def entry_url_from_brief(brief: str) -> str: return m.group(1).rstrip(".,;)") if m else "" +# Opens the advisory-brief section of a composed task; consumers strip everything after it when a +# check must apply only to the human's words (the brief once false-flagged a real send read-only). +BRIEF_MARKER = "[routing brief" + + def compose_task(prompt: str, brief: str) -> str: """User's words first and authoritative; the brief is advisory routing. Skill replay keys on the parent's user message, so brief variance is safe.""" @@ -124,7 +129,7 @@ def compose_task(prompt: str, brief: str) -> str: return prompt return ( f"{prompt}\n\n" - "[routing brief from a fast pre-pass; follow it unless the live page disagrees]\n" + f"{BRIEF_MARKER} from a fast pre-pass; follow it unless the live page disagrees]\n" f"{brief}" ) diff --git a/backend/apps/agents/browser/browser_send_script.py b/backend/apps/agents/browser/browser_send_script.py index 15c56071..68663efd 100644 --- a/backend/apps/agents/browser/browser_send_script.py +++ b/backend/apps/agents/browser/browser_send_script.py @@ -13,16 +13,18 @@ the untouched model path; ambiguity AFTER the click hands the model a truthful """ import asyncio +import json import logging import os import re import time from typing import Awaitable, Callable, Dict -from backend.apps.agents.browser import browser_verified_action +from backend.apps.agents.browser import browser_fast_path, browser_submit_click, browser_verified_action logger = logging.getLogger(__name__) + # Double quotes are unambiguous. Single quotes only delimit when the opener is at a word boundary (start/space/colon), so an in-word apostrophe like "chen's" is never mistaken for a payload quote, that mispairing was silently corrupting the canonical "text him '...'" errand. P_QUOTED_DQ_RE = re.compile(r'"([^"]{4,300})"') P_QUOTED_SQ_RE = re.compile(r"(?:^|[\s:>])'([^']{4,300})'") @@ -79,7 +81,8 @@ def autosend_enabled() -> bool: async def complete_send( payload: str, state_committed: str, browser_id: str, tab_id: str, - execute_tool: ToolRunner, send_index_in_state: Callable[[str], object], + execute_tool: ToolRunner, send_index_in_state: Callable[[str, int], object], + composer_index: int = -1, ) -> Dict[str, object]: """Send tail for a composer that ALREADY holds `payload` (visible in state_committed): find the Send control (ranked index first, else click-by-name over the full DOM), click it once, and @@ -97,15 +100,29 @@ async def complete_send( except Exception: return "" - send_btn = send_index_in_state(state_committed) + send_btn = send_index_in_state(state_committed, composer_index) + via = "index" if send_btn: r_send = await execute_tool("BrowserClickIndex", {"index": send_btn[0]}, browser_id, tab_id) send_name = send_btn[1] else: - r_send = await execute_tool("BrowserClickByName", {"name": "Send", "role": "button"}, browser_id, tab_id) - send_name = "Send (by-name)" + # No submit listed below the composer (the capped listing can starve a modal of its own + # button): click the submit inside the composer's OWN container, then last-resort by-name. + r_send = await execute_tool( + "BrowserEvaluate", + {"expression": browser_submit_click.container_submit_expression(payload)}, browser_id, tab_id) + p_v = browser_submit_click.parse_eval_value(r_send) + if isinstance(p_v, dict) and p_v.get("ok"): + send_name = str(p_v.get("name") or "submit") + via = "container" + else: + p_why = p_v.get("why") if isinstance(p_v, dict) else "unreadable eval" + logger.info(f"[browser-sendscript] container submit miss ({p_why}); by-name fallback") + r_send = await execute_tool("BrowserClickByName", {"name": "Send", "role": "button"}, browser_id, tab_id) + send_name = "Send (by-name)" + via = "by-name" clicked = isinstance(r_send, dict) and "error" not in r_send - log.append({"tool": "send click", "input": {"via": "index" if send_btn else "by-name"}, + log.append({"tool": "send click", "input": {"via": via}, "ok": clicked, "result_summary": f"send click {send_name!r}"[:200], "elapsed_ms": 0, "clicked_role": "button", "clicked_name": send_name}) if not clicked: @@ -228,10 +245,12 @@ async def run_send_script( else: logger.info(f"[browser-sendscript] decline: no composer or opener after poll ({current_url[:50]!r})") return None - # Key read-only on the user's OWN words, not the advisory brief composed into `task`: the aux - # brief wrote "do not submit/post it" for a plain "start a post", which falsely read-only-flagged - # a real send. Fall back to task only when the raw prompt isn't threaded through. - if P_READONLY_RE.search(payload_source or task): + # Key read-only on words a HUMAN wrote: the task minus the aux routing brief (the brief wrote + # "do not submit it" for a plain "start a post", falsely read-only-flagging a real send) PLUS + # the raw prompt when threaded through. The task text itself must keep declining regardless: a + # read-only VERIFY probe arrives as the task, and one once delivered a real message (r243). + task_sans_brief = task.split(browser_fast_path.BRIEF_MARKER, 1)[0] + if P_READONLY_RE.search(task_sans_brief) or (payload_source and P_READONLY_RE.search(payload_source)): logger.info("[browser-sendscript] decline: read-only directive in user request") return None if looks_like_login_wall(current_url, state_text): @@ -349,12 +368,13 @@ async def run_send_script( # doing the outward send. Everything up to here ran (surface gate passed, composer # found, fill committed); we stop before the irreversible click and report readiness. if os.environ.get("OSW_SENDSCRIPT_DRYRUN") == "1": - send_ready = bool(send_index_in_state(state2)) + send_ready = bool(send_index_in_state(state2, composer[0])) logger.info(f"[browser-sendscript] DRYRUN: WOULD send (fill committed, send_button_listed={send_ready}); not clicking") return {"sent": False, "payload": payload, "log": log, "note": "DRYRUN: filled + ready to send, stopped before the irreversible click"} # 3+4: the irreversible click + two-sided receipt, shared with the mid-loop takeover. A click error hands back to the model (fill committed, not sent); a clicked-but-unverified send returns sent=False so the caller never claims delivery. - r = await complete_send(payload, state2, browser_id, tab_id, execute_tool, send_index_in_state) + r = await complete_send(payload, state2, browser_id, tab_id, execute_tool, send_index_in_state, + composer_index=composer[0]) log.extend(r["log"]) if not r["clicked"]: logger.info("[browser-sendscript] send click errored; handing to model (fill committed, NOT sent)") diff --git a/backend/apps/agents/browser/browser_submit_click.py b/backend/apps/agents/browser/browser_submit_click.py new file mode 100644 index 00000000..dae0ee9d --- /dev/null +++ b/backend/apps/agents/browser/browser_submit_click.py @@ -0,0 +1,69 @@ +"""Container-scoped submit click for the receipt-gated send path. Exists because the ranked +interactives listing caps at 60 rows and a composer's own submit can fall off it (X's compose +modal: covered feed rows behind the overlay ate the cap, so no "Post" row ever reached the index +picker, measured live 0/2 deliveries). Scope = the dialog/form ancestor of the editable holding +the payload when there is one, else a bounded nearest-scope-first upward walk, so a page-level +opener with the same label can never be chosen. A wrong resolution still fails the send receipt +downstream, never a false delivery claim.""" + +import json +from typing import Any, Dict, Optional + +# BROAD submit vocabulary shared by the index picker (browser_agent) and the JS below, one source +# so the two tiers can never drift apart. +SEND_LABELS = frozenset({ + "send", "send now", "send message", # LinkedIn / Gmail / DMs + "post", "post all", "tweet", "reply", # X / Threads compose + reply + "publish", "comment", "share", # articles / YouTube+FB comments / shares +}) + +P_CONTAINER_SUBMIT_JS = r"""(() => { + const PAYLOAD = %s; + const LABELS = new Set(%s); + const norm = (s) => (s || '').replace(/\s+/g, ' ').trim().toLowerCase(); + const vis = (el) => !!el && el.getClientRects().length > 0 && el.offsetParent !== null; + const enabled = (el) => !el.disabled && el.getAttribute('aria-disabled') !== 'true'; + const labelOf = (el) => norm(el.getAttribute('aria-label') || el.innerText || ''); + const holds = (el) => ((el.value || el.textContent || '').indexOf(PAYLOAD) !== -1); + const ed = [...document.querySelectorAll('[contenteditable="true"],textarea,input')] + .find((e) => vis(e) && holds(e)); + if (!ed) return { ok: false, why: 'no editable holding the payload' }; + const submitIn = (root) => [...root.querySelectorAll('button,[role="button"]')] + .find((b) => vis(b) && enabled(b) && LABELS.has(labelOf(b))); + const scope = ed.closest('[role="dialog"],[role="alertdialog"],form'); + let btn = null; + if (scope) { + btn = submitIn(scope); + } else { + // Nearest-scope-first walk: X's inline submit shares an ancestor 20 hops above the Draft.js + // editable while foreign tweets' buttons only enter at 28 (measured live), so 24 finds the + // composer's own submit and stops before any wider scope could. + let node = ed.parentElement; + for (let hop = 0; node && node !== document.body && hop < 24; hop++, node = node.parentElement) { + btn = submitIn(node); + if (btn) break; + } + } + if (!btn) return { ok: false, why: 'no submit control in the composer container' }; + btn.click(); + return { ok: true, name: labelOf(btn) }; +})()""" + + +def container_submit_expression(payload: str) -> str: + """The container-scoped submit click for a composer holding `payload` (prefix-matched, same + 24-char truncation the fill verifier uses).""" + return P_CONTAINER_SUBMIT_JS % (json.dumps((payload or "")[:24]), json.dumps(sorted(SEND_LABELS))) + + +def parse_eval_value(res: object) -> Optional[Dict[str, Any]]: + """The dict a BrowserEvaluate returned, or None. Unreadable shapes are None (honest miss).""" + val: object = None + if isinstance(res, dict) and "error" not in res: + val = res.get("value") + if val is None and isinstance(res.get("text"), str): + try: + val = json.loads(res["text"]) + except (json.JSONDecodeError, ValueError): + val = None + return val if isinstance(val, dict) else None diff --git a/backend/tests/test_browser_agent_loop.py b/backend/tests/test_browser_agent_loop.py index 5a3735dd..39d65906 100644 --- a/backend/tests/test_browser_agent_loop.py +++ b/backend/tests/test_browser_agent_loop.py @@ -1570,6 +1570,19 @@ def test_send_submit_matcher_broad_but_hint_matcher_tight(): assert send_index_in_state('[3]