diff --git a/.github/workflows/windows-session-import.yml b/.github/workflows/windows-session-import.yml new file mode 100644 index 00000000..a6d40433 --- /dev/null +++ b/.github/workflows/windows-session-import.yml @@ -0,0 +1,42 @@ +name: windows-session-import + +# The Windows half of the browser session-import path can only be proven on Windows: it talks to +# DPAPI through crypt32, and every mock of that agrees with whatever we already believed. This job +# runs the round-trip tests on a real windows runner, so a layout mistake fails here instead of in +# a user's onboarding. + +on: + push: + paths: + - 'backend/apps/onboarding/usage/browser_cookies.py' + - 'backend/tests/test_browser_cookies_windows_live.py' + - '.github/workflows/windows-session-import.yml' + workflow_dispatch: + +jobs: + verify: + runs-on: windows-latest + steps: + - uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + + - name: Install the two things the module actually needs + run: pip install cryptography typeguard pytest + + - name: Prove the DPAPI + AES-GCM round trip on real Windows + run: python -m pytest backend/tests/test_browser_cookies_windows_live.py -v + + - name: Fail if the suite skipped itself + shell: pwsh + run: | + # A skip reads as green. On a windows runner these tests MUST run, so an all-skipped + # result means the platform gate is wrong and the job is proving nothing. + $out = python -m pytest backend/tests/test_browser_cookies_windows_live.py -q 2>&1 | Out-String + if ($out -match 'skipped') { + Write-Error "Windows tests skipped on a Windows runner; the platform gate is broken:`n$out" + exit 1 + } + Write-Host "Ran on real Windows, no skips." diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 8c40003f..2b21d5cb 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -4,7 +4,7 @@ import time from contextlib import asynccontextmanager from typing import Any, Dict -from fastapi import HTTPException +from fastapi import HTTPException, Request from typeguard import typechecked from backend.apps.agents.agent_manager import agent_manager @@ -425,7 +425,7 @@ async def subscriptions_status(): @agents.router.post("/subscriptions/connect") -async def subscriptions_connect(body: dict): +async def subscriptions_connect(body: dict, request: Request): """Start OAuth flow for a subscription provider.""" from backend.apps.nine_router import is_running, ensure_running, start_oauth provider = body.get("provider", "") @@ -446,7 +446,9 @@ async def subscriptions_connect(body: dict): pass try: - result = await start_oauth(provider) + # The port the user's app actually reached us on beats guessing the default; only consulted + # when OPENSWARM_PORT is unset (dev uvicorn launches), never in packaged builds. + result = await start_oauth(provider, request.url.port) if result.get("flow") == "authorization_code" and result.get("state"): from backend.apps.oauth_state import pending_oauth diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 071b327c..bbed1706 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -44,6 +44,7 @@ from backend.apps.agents.browser.browser_loop import ( completion_is_honest, deliverable_is_informational, interstitial_dismiss_target, + is_removal_task, recoverable_tool_error, replay_recheck_is_safe, stagnation_exhausted, @@ -67,6 +68,12 @@ P_WRAPUP_NUDGE = ( from backend.apps.agents.browser import browser_batch_replay from backend.apps.agents.browser import browser_extract from backend.apps.agents.browser import browser_metrics +from backend.apps.agents.browser import browser_send_script +from backend.apps.agents.browser import browser_send_parse +from backend.apps.agents.browser import browser_login_handoff +from backend.apps.agents.browser import browser_session_import +from backend.apps.agents.browser import browser_delivery_check +from backend.apps.agents.browser import browser_submit_click from backend.apps.agents.browser import browser_playbook from backend.apps.agents.browser import browser_save from backend.apps.agents.browser import browser_meta_playbook @@ -81,7 +88,6 @@ from backend.apps.agents.browser.browser_schema import ( APP_VISIBLE_TOOLS, BROWSER_TOOLS_SCHEMA, MAX_TURNS, - MODEL_MAP, SYSTEM_PROMPT, ) from backend.apps.agents.core.models import AgentSession, ApprovalRequest, Message @@ -126,6 +132,11 @@ P_BRIDGE_READY_WAIT_MS = 8000 P_BRIDGE_POLL_INTERVAL_MS = 400 p_bridge_known_absent: set[str] = set() +# Sites whose sign-in we already borrowed into the partition. Only SUCCESSFUL borrows are recorded, +# so a site the user signs into later still gets picked up without a restart; the re-probe that +# costs is a keychain-free row count, so re-asking is cheap. +signin_borrowed: set[str] = set() + def parse_bridge_result(result: dict) -> object: """Decode the JSON string an app-bridge evaluate returns (it always returns @@ -249,10 +260,37 @@ def p_summarize_action(tool_name: str, tool_input: dict) -> str: return p_summ_step(stype, ti) if stype else "" +async def borrow_signin_before_nav(url: str, browser_id: str) -> None: + """Load the user's own sign-in for a site BEFORE its first page load, so the very first request + already carries it. + + Borrowing only at a detected wall was too late in two ways: a task the model answers in one turn + calls Done, which breaks the loop before the handoff block ever runs, so short tasks never got it + at all; and even a long task had to render a logged-out page, notice, and throw it away. Doing it + at the door costs the user nothing and skips both. Silent and best-effort; the wall handoff is + still there as the backstop when this finds nothing.""" + from backend.apps.settings.settings import load_settings + + try: + if not browser_session_import.is_enabled(load_settings()): + return + domain = browser_session_import.site_domain(url) + if not domain or domain in signin_borrowed: + return + if not browser_session_import.has_importable_session(domain): + return + signin_borrowed.add(domain) + await browser_session_import.import_signin(domain, browser_id) + except Exception as exc: + logger.info(f"[session-import] pre-nav borrow skipped: {type(exc).__name__}") + + async def execute_browser_tool( tool_name: str, tool_input: dict, browser_id: str, tab_id: str = "", ) -> dict: """Execute a browser tool via ws_manager directly (no MCP/HTTP round-trip).""" + if tool_name == "BrowserNavigate": + await borrow_signin_before_nav(str((tool_input or {}).get("url") or ""), browser_id) # One greppable line naming the actual buttons/keys/selectors this call drives, # so a run reads as "key:ArrowRight x5" rather than an opaque tool name. Fires # for action tools only (reads stay quiet) and ungated so web runs get it too. @@ -260,6 +298,21 @@ async def execute_browser_tool( if p_action: logger.info(f"[browser-action] {tool_name}: {p_action} -> {browser_id}") + # BrowserDeleteItem: a model-invoked remove, translated to one BrowserEvaluate that runs the + # site's own delete flow scoped to the named item, then verifies it's gone (App-bridge pattern). + if tool_name == "BrowserDeleteItem": + from backend.apps.agents.browser import browser_delete_script + p_target = str((tool_input or {}).get("target_text") or "") + if len(p_target) < browser_delete_script.MIN_TARGET_CHARS: + return {"error": "target_text too short; give a longer distinctive snippet of the item's own text"} + p_parsed = await browser_delete_script.run_delete(p_target, browser_id, tab_id, execute_browser_tool) + logger.info(f"[browser-deleteitem] target={p_target[:40]!r} removed={p_parsed['removed']} stage={p_parsed['stage']}") + if p_parsed["removed"]: + return {"text": f'Removed the item containing "{p_target[:60]}" (verified gone).', "removed": True} + return {"text": (f'Did NOT remove it (stage={p_parsed["stage"]}): {p_parsed["msg"]}. If the item ' + "is not on this page, navigate to where it lives (e.g. your profile) and retry."), + "removed": False} + # App bridge tools translate to a single BrowserEvaluate against the app's # window.OPENSWARM_APP, so they need no frontend command-handler changes. if tool_name in APP_BRIDGE_TOOLS: @@ -299,13 +352,58 @@ async def execute_browser_tool( return {"error": f"Unknown browser tool: {tool_name}"} params = {k: v for k, v in tool_input.items()} + # Self-healing click toggle + click-effect metric. Threaded for solo clicks AND batches (most clicks are batched, so gating on click_index alone misses them). handleBatch propagates these into its click_index sub-actions. + if action in ("click_index", "batch"): + params["selfheal"] = os.environ.get("OSW_SELFHEAL_CLICK", "1") != "0" + if os.environ.get("OSW_CLICK_EFFECT_PROBE") == "1": + params["effectProbe"] = True + # Document-order interactives display (default on); OSW_DOC_ORDER=0 = legacy rank-order, for the A/B off-arm. + if action == "list_interactives": + params["docOrder"] = os.environ.get("OSW_DOC_ORDER", "1") != "0" request_id = uuid4().hex result = await ws_manager.send_browser_command( request_id, action, browser_id, params, tab_id=tab_id, ) + if os.environ.get("OSW_DEBUG_LIST") == "1" and action == "list_interactives" and isinstance(result, dict): + logger.info(f"[debug-list] {str(result.get('text') or '')[:2400]}") + # Click telemetry lives at the top level for a solo click and inside `results[]` for a batched one; scan both. + p_click_parts = [result] if isinstance(result, dict) else [] + if isinstance(result, dict) and isinstance(result.get("results"), list): + p_click_parts += [r for r in result["results"] if isinstance(r, dict)] + p_target_probe = os.environ.get("OSW_CLICK_EFFECT_PROBE") == "1" + for p_cr in p_click_parts: + if p_cr.get("selfHealed"): + logger.info(f"[browser-selfheal] recovered a stale-index click via {p_cr['selfHealed']} -> {browser_id}") + if p_cr.get("clickEffect"): + logger.info(f"[click-effect] {p_cr['clickEffect']} -> {browser_id}") + # The wrong-target signal a page-change metric misses: landed=False means the click point was NOT on the intended element (occluded/stale/moved). + if p_target_probe and "clickLanded" in p_cr: + logger.info(f"[click-target] landed={p_cr['clickLanded']} hit={str(p_cr.get('clickHit'))[:40]!r} -> {browser_id}") return result +async def p_learn_write_recipe(execute_tool, browser_id: str, tab_id: str, current_url: str, payload: str) -> None: + """After a receipt-VERIFIED DOM write, distill the site's own write route into a replayable + recipe so the NEXT write on this host can skip the DOM (the repeated-write tier). Inline + + bounded (the card is still alive here; a fire-and-forget task would race the finish teardown) + and fully fail-open: no routes / no payload leaf / any error just means no recipe learned, and + the DOM path stays the default. Gated OFF until soaked (OSW_WRITE_RECIPES=1).""" + if os.environ.get("OSW_WRITE_RECIPES", "0") == "0": + return + try: + from backend.apps.agents.browser import browser_write_recipes, browser_skills + host = browser_skills.host_of(current_url or "") + if not host or len(payload or "") < 4: + return + listed = await asyncio.wait_for( + execute_tool("BrowserListRoutes", {"writes": True}, browser_id, tab_id), timeout=4.0) + routes = listed.get("routes") if isinstance(listed, dict) else None + if routes and browser_write_recipes.learn_recipe(host, payload, routes): + logger.info(f"[write-recipe] learned a replayable {host} write from a verified DOM send") + except Exception: + pass + + def p_extract_domain(url: str) -> str | None: """Extract the apex domain from a URL (acme-corp.notion.so → notion.so). Returns None for non-http URLs.""" @@ -323,6 +421,57 @@ def p_extract_domain(url: str) -> str | None: return None +def p_api_write_result(res) -> dict: + """Shape a registry WriteResult into a loop result: a truthful receipt on success (with + send_confirmed set by the caller), or an `error` on a miss so the model does the write via the + UI and the run never distills it as a false success.""" + if res.ok: + return {"ok": True, "text": ( + f"Done via the {res.domain} API in {res.latency_ms}ms. Receipt: {res.receipt}. " + "The write landed, that receipt is your proof; you're finished with this step." + )} + return {"error": f"API write not used ({res.error}). Do this action through the UI instead."} + + +async def run_api_write(tool_input: dict, current_url: str, browser_id: str = "", tab_id: str = "") -> dict: + """Route a BrowserApiWrite to the API-first write tier: a deterministic built-in adapter + (Reddit) when one exists, else the GENERAL capture-replay tier (action='route': replay a + mutating route the site's own UI fired, verified same-origin + captured, behind OSW_ROUTE_WRITE). + Never raises: a missing adapter / disarmed tier / site-reject is a typed miss, so the model + falls back to the UI path, never a crash and never a false claim of success.""" + from urllib.parse import urlparse + from backend.apps.agents.browser import route_write, site_write_registry + action = str((tool_input or {}).get("action") or "").strip() + if not action: + return {"error": "BrowserApiWrite needs an 'action' (comment, reply, post, edit, delete, or route)."} + domain = p_extract_domain(current_url or "") + if not domain: + return {"error": "Can't tell what site you're on yet; navigate to the site first, then do the write through the UI or retry."} + + if action == "route": + # General tier: replay a captured mutating route. The captured set is fetched live from the + # page (the safety wall: only a route the UI actually fired can be replayed), and the replay + # itself is same-origin + flag-gated + session-borrowed in route_write. + method = str(tool_input.get("method") or "POST").strip() + url = str(tool_input.get("url") or "").strip() + body = tool_input.get("body") if isinstance(tool_input.get("body"), dict) else {} + if not url: + return {"error": "BrowserApiWrite route needs the 'url' of a captured write endpoint (see BrowserListRoutes)."} + try: + origin = f"{urlparse(current_url).scheme}://{urlparse(current_url).netloc}" + except Exception: + return {"error": "Can't resolve the current site's origin; do the write through the UI."} + listed = await execute_browser_tool("BrowserListRoutes", {"writes": True}, browser_id, tab_id) + captured = [route_write.CapturedRoute(method=str(r.get("method", "")), template=str(r.get("template", ""))) + for r in (listed.get("routes") or []) if isinstance(r, dict) and r.get("template")] + res = await site_write_registry.api_route_write(origin, method, url, body, captured) + return p_api_write_result(res) + + params = {k: v for k, v in (tool_input or {}).items() if k not in ("action", "expect")} + res = await site_write_registry.api_write(domain, action, params) + return p_api_write_result(res) + + def strip_lone_surrogates(s: str) -> str: # The JS/webview hands us page text as UTF-16, so an emoji can arrive as half of its surrogate pair; Python carries the orphan but .encode('utf-8') later (the SDK serializing the request to the LLM) detonates with "surrogates not allowed" and kills the turn. Swap any orphan for the replacement char. return re.sub(r"[\ud800-\udfff]", "�", s) if s else s @@ -356,14 +505,18 @@ P_AUTO_STATE_TOOLS = { "BrowserNavigate", "BrowserClick", "BrowserClickIndex", "BrowserClickByName", "BrowserType", "BrowserPressKey", "BrowserScroll", "BrowserBatch", } -P_AUTO_STATE_MAX_LINES = 35 +# Matches the frontend's DEFAULT_INTERACTIVE_CAP (interactiveRanking.ts): a shorter cap here silently hid rows 36-60 that an explicit BrowserListInteractives would show, forcing the model to re-list the very elements it just acted on. Delta compression keeps the common attach small, so the worst case (a full 60-row attach) is bounded and rare. +P_AUTO_STATE_MAX_LINES = 60 P_AUTO_SETTLE_CAPS_MS = {"BrowserNavigate": 2500, "BrowserBatch": 1500} -# URL shapes that mean "a list of candidates to pick from" (auto candidate scan) -RESULTS_URL_RE = re.compile( - r"[?&](q|query|keywords|search|search_query|find|term)=|/search\b|/results\b", re.I, -) +# RESULTS_URL_RE moved to browser_prestage (its READY overrule uses it too); re-exported here for the scan sites. +from backend.apps.agents.browser.browser_prestage import RESULTS_URL_RE P_AUTO_SCAN_MAX_PER_RUN = 2 +# The candidate scan is a fast HINT, not the critical path. Cap the aux input to the top of the +# page (matches are first) and bail quickly, so a big page can't turn the scan into dead idle time +# (a real 8s stall was measured on LinkedIn search). Head slice + short timeout = finishes-or-skips. +P_SCAN_TEXT_CAP = 8000 +P_SCAN_TIMEOUT_S = 4.0 def p_batch_ends_with_read(tool_input: dict) -> bool: @@ -406,17 +559,211 @@ def delta_state(text: str, seen_lines: set[str]) -> str: # A button row whose name is exactly a Send control (not "Send InMail credit" or "Send a message to X"); used to hand the model the Send button after it types, so it never burns turns hunting a button that's right there. P_SEND_ROW_RE = re.compile(r'\[(\d+)\]\*?<\s*button\s+"([^"]*)"', re.I) +# TIGHT set for the ALWAYS-ON model hint (post_action_state). Kept to the unambiguous "Send" family +# on purpose: this hint fires after ANY text fill, so a broad match ("Reply"/"Share"/"Comment"/ +# "Post") would mislabel a stray feed button as the Send button after an unrelated search fill. +P_HINT_SEND_LABELS = frozenset({"send", "send now", "send message"}) +# BROAD submit vocabulary for the SEND-SCRIPT only (flag-gated + receipt-gated): lets the fast +# send-path COMPLETE on X/IG/FB/Threads/YouTube. Safe ONLY there because the send-script re-verifies +# the composer cleared the exact payload, so an opener-vs-submit mismatch fails safe, never a false +# send. button-only (P_SEND_ROW_RE) + exact match keeps "Post" from matching "Post a job" etc. +# Defined in browser_submit_click so the container-scoped JS tier shares the exact same set. +P_SEND_LABELS = browser_submit_click.SEND_LABELS + def send_index_in_state(state_text: str): - """(index, name) of a real Send button in an interactives list, or None. - Strict exact match so it never grabs an upsell or a profile 'Send a message' link.""" + """(index, name) of a real Send button for the ALWAYS-ON model hint, or None. TIGHT exact match + (Send family only) so it never mislabels a common feed 'Reply'/'Share'/'Comment' button.""" for line in (state_text or "").splitlines(): m = P_SEND_ROW_RE.search(line) - if m and m.group(2).strip().lower() in ("send", "send now", "send message"): + if m and browser_submit_click.clean_button_name(m.group(2)) in P_HINT_SEND_LABELS: return int(m.group(1)), m.group(2) return None +def send_submit_index_in_state(state_text: str, after_index: int = -1): + """(index, name) of a submit button across the popular composers (Post/Reply/Tweet/...), for the + receipt-gated SEND-SCRIPT only. Broader than the hint matcher; safe because the send-script + verifies the composer cleared afterward, so a wrong match aborts, never sends. `after_index` + scopes the scan to buttons BELOW the filled composer: every real submit follows its editor in + the listing, while a compose OPENER (X's sidebar "Post") sits above it, and clicking the opener + posts nothing (measured live: 0/2 X deliveries the day X shipped its opener as a button).""" + for line in (state_text or "").splitlines(): + m = P_SEND_ROW_RE.search(line) + if m and browser_submit_click.clean_button_name(m.group(2)) in P_SEND_LABELS and int(m.group(1)) > after_index: + return int(m.group(1)), m.group(2) + return None + + +# Tokens that mean the aux wrote machinery, not a user sentence: reject and fall back to a template. +P_NOT_A_REPLY = ("browser", "clickindex", "composer", "textbox", "```", "{", "index", "http") + + +async def compose_send_confirmation(aux_client, aux_model, task: str, payload: str) -> str: + """The final 'done' line in the model's OWN voice, via one cheap aux call, so it isn't a + hardcoded template. The SEND already happened in code; this only writes the words. Fail-open: + returns '' on any error OR if the output doesn't read like a plain user sentence (tool names, + JSON, a URL), so the caller falls back to a simple template. Aux tier = cheap + fast; a + one-sentence confirmation needs no frontier model, and it never re-does the mechanical work.""" + if not aux_client or not aux_model or not payload: + return "" + prompt = ( + "You just finished a task for the user by controlling their web browser, and it SUCCEEDED.\n" + f"The user asked: {task[:280]}\n" + f"What you sent: \"{payload[:280]}\"\n" + "Reply with ONE short, warm, first-person sentence confirming it's done, the way a helpful " + 'friend would (e.g. "Done, I messaged Tyler and said hi."). No technical words, no quotes ' + "wrapping the whole sentence, no preamble, just the sentence." + ) + try: + resp = await aux_client.messages.create( + model=aux_model, max_tokens=80, + messages=[{"role": "user", "content": prompt}], + ) + text = "".join(getattr(b, "text", "") for b in (resp.content or [])).strip().strip('"').strip() + except Exception: + return "" + low = text.lower() + if not text or len(text) > 220 or any(t in low for t in P_NOT_A_REPLY): + return "" + return text + + +async def compose_partial_result(aux_client, aux_model, task: str, action_log: list) -> str: + """What the run actually reached, in the model's OWN voice, when the loop is cut off before it + can answer. The stock "that's as far as I could get" tells the user nothing about what WAS + found, which is often most of what they asked for. Fail-open like the others; must not invent + an outcome, so it is fed only the pages and actions that really happened.""" + if not aux_client or not aux_model: + return "" + steps = [f"{e.get('tool', '?')} {str(e.get('input', ''))[:70]}" for e in (action_log or [])[-12:]] + if not steps: + return "" + prompt = ( + "You were controlling the user's web browser and ran out of turns before you could give a " + "final answer. Here is what you actually did, most recent last:\n" + + "\n".join(steps) + + f"\n\nThe user asked: {task[:280]}\n" + "Reply with ONE or TWO short first-person sentences saying how far you got and what you " + "did or did not manage to find. Do NOT invent any result you cannot see above. If you got " + "nowhere useful, say that plainly. No technical words, no tool names, no preamble." + ) + try: + resp = await aux_client.messages.create( + model=aux_model, max_tokens=120, + messages=[{"role": "user", "content": prompt}], + ) + text = "".join(getattr(b, "text", "") for b in (resp.content or [])).strip().strip('"').strip() + except Exception: + return "" + low = text.lower() + if not text or len(text) > 320 or any(t in low for t in P_NOT_A_REPLY): + return "" + return text + + +async def compose_removal_confirmation(aux_client, aux_model, task: str, target: str) -> str: + """The 'I deleted it' line in the model's OWN voice. The removal is already verified gone in + code before this runs, so this only chooses words. Same fail-open contract as the others.""" + if not aux_client or not aux_model or not target: + return "" + prompt = ( + "You just finished a task for the user by controlling their web browser: you DELETED an " + "item for them, and you confirmed it is gone from the page.\n" + f"The user asked: {task[:280]}\n" + f"The item you removed contained: \"{target[:200]}\"\n" + "Reply with ONE short, plain first-person sentence confirming it's deleted, the way a " + "helpful friend would. No technical words, no preamble, just the sentence." + ) + try: + resp = await aux_client.messages.create( + model=aux_model, max_tokens=80, + messages=[{"role": "user", "content": prompt}], + ) + text = "".join(getattr(b, "text", "") for b in (resp.content or [])).strip().strip('"').strip() + except Exception: + return "" + low = text.lower() + if not text or len(text) > 220 or any(t in low for t in P_NOT_A_REPLY): + return "" + return text + + +async def compose_unverified_send(aux_client, aux_model, task: str, payload: str, url: str) -> str: + """The 'I clicked send but never saw it confirm' line in the model's OWN voice. + + Weaker than compose_delivery_warning by design: there the composer cleared, so we know the thing + was submitted and only its survival is in doubt. Here we never got the clear at all, so the only + thing we know is that a click ran. Same fail-open contract, and it must never claim success.""" + if not aux_client or not aux_model or not payload: + return "" + from urllib.parse import urlparse + host = urlparse(url or "").hostname or "the site" + if host.startswith("www."): + host = host[4:] + prompt = ( + "You tried to post or send something for the user by controlling their web browser. You " + "typed it and clicked send, but you never saw the confirmation you rely on: the compose box " + f"did not clear, so you do NOT know whether {host} accepted it. This is NOT a success.\n" + f"The user asked: {task[:280]}\n" + f"What you typed: \"{payload[:280]}\"\n" + "Reply with ONE or TWO short first-person sentences that say plainly you could not confirm " + "it posted, tell them to check, and mention you did not try again so they don't end up with " + "a duplicate. Never say it went through, was sent, or is showing. No technical words, no " + "preamble, just the sentences." + ) + try: + resp = await aux_client.messages.create( + model=aux_model, max_tokens=120, + messages=[{"role": "user", "content": prompt}], + ) + text = "".join(getattr(b, "text", "") for b in (resp.content or [])).strip().strip('"').strip() + except Exception: + return "" + low = text.lower() + if not text or len(text) > 320 or any(t in low for t in P_NOT_A_REPLY): + return "" + # A composed line that still claims delivery is worse than the template, so refuse it. + if any(p in low for p in ("went through", "was sent", "i sent", "it's showing", "is showing", "posted it")): + return "" + return text + + +async def compose_delivery_warning(aux_client, aux_model, task: str, payload: str, url: str) -> str: + """The 'I sent it but couldn't confirm it stayed live' line in the model's OWN voice, via one + cheap aux call, for a ghost-drop host where the composer cleared but the post did NOT persist. + Same fail-open contract as compose_send_confirmation: '' on any error or a non-sentence output, + so the caller falls back to the plain honest template. Never claims success.""" + if not aux_client or not aux_model or not payload: + return "" + from urllib.parse import urlparse + host = urlparse(url or "").hostname or "the site" + if host.startswith("www."): + host = host[4:] + prompt = ( + "You tried to post something for the user by controlling their web browser. You submitted " + "it and the compose box cleared, BUT when you re-checked the page the post did NOT stay up: " + f"{host} appears to have silently dropped it. This is NOT a confirmed success.\n" + f"The user asked: {task[:280]}\n" + f"What you tried to post: \"{payload[:200]}\"\n" + "Reply with ONE or TWO short, honest, first-person sentences: you submitted it but could NOT " + "confirm it actually went live, so they should check their posts. Warm and plain, no " + "technical words, no false reassurance that it worked." + ) + try: + resp = await aux_client.messages.create( + model=aux_model, max_tokens=120, + messages=[{"role": "user", "content": prompt}], + ) + text = "".join(getattr(b, "text", "") for b in (resp.content or [])).strip().strip('"').strip() + except Exception: + return "" + low = text.lower() + if not text or len(text) > 400 or any(t in low for t in P_NOT_A_REPLY): + return "" + return text + + def is_composer_fill(tool_name: str, tool_input: dict) -> bool: """True if this action typed a message into a composer (the moment the Send button is about to matter). Covers the solo fill, BrowserType, and a batched @@ -432,6 +779,50 @@ def is_composer_fill(tool_name: str, tool_input: dict) -> bool: return False +def fill_text_of(tool_name: str, tool_input: dict) -> str: + """The text a composer-fill action typed, '' when it isn't one.""" + ti = tool_input or {} + if tool_name in ("BrowserClickIndex", "BrowserType"): + return str(ti.get("text") or "").strip() + if tool_name == "BrowserBatch": + for a in (ti.get("actions") or []): + p = a.get("params") or {} + if a.get("type") in ("type", "click_index") and str(p.get("text") or "").strip(): + return str(p.get("text")).strip() + return "" + + +def fill_index_of(tool_name: str, tool_input: dict) -> int: + """The listing index a composer-fill action typed into, -1 when unknown. Mirrors fill_text_of.""" + ti = tool_input or {} + if tool_name in ("BrowserClickIndex", "BrowserType"): + try: + return int(ti.get("index", -1)) + except (TypeError, ValueError): + return -1 + if tool_name == "BrowserBatch": + for a in (ti.get("actions") or []): + p = a.get("params") or {} + if a.get("type") in ("type", "click_index") and str(p.get("text") or "").strip(): + try: + return int(p.get("index", -1)) + except (TypeError, ValueError): + return -1 + return -1 + + +def payload_in_textbox(state_text: str, payload: str) -> bool: + """True if any listed textbox VALUE carries the typed payload (fill committed). + Matches on a prefix because long payloads truncate in the list.""" + probe = (payload or "")[:24] + if not probe: + return False + for line in (state_text or "").splitlines(): + if "2.4s: recent sends ran the FULL 6s and still found nothing (send_button_found=False 3/3), so the long tail bought pure wall time; two polls catch the lazy-render case, the model finds Send itself past that. + p_deadline = time.monotonic() + 2.4 while True: try: p_l = await asyncio.wait_for( @@ -482,6 +873,16 @@ async def post_action_state( return "" state = lst["text"] if seen_lines is None else delta_state(lst["text"], seen_lines) out = f"\n\n{PAGE_STATE_MARKER}\n{p_truncate_state(state)}" + # Fold the page's READABLE TEXT in alongside the clickable elements (flag-gated). The model perceives nearly every page TWICE, once via list_interactives and once via GetText (measured: ~52% of all tool calls are perception, half of that redundant list+text pairs). Attaching a trimmed text excerpt here means after any action it already has BOTH views, so it never spends a separate GetText turn. A cheap code-side read (ms) trades for a ~4s model turn. + if os.environ.get("OSW_FOLD_TEXT", "0") == "1": + try: + p_gt = await asyncio.wait_for( + wait_exec("BrowserGetText", {}, browser_id, tab_id), timeout=5.0) + p_txt = str(p_gt.get("text") or "") if isinstance(p_gt, dict) and "error" not in p_gt else "" + if p_txt: + out += f"\n\n[Page text (you have this already, no need to GetText):]\n{p_txt[:1800]}" + except Exception: + pass # Hand the Send button's index over so the model clicks it directly instead of scanning the list or hunting via CSS/JS/screenshots (the polled list above is what makes Send actually present to point at, the two work together). if p_send_si: out = (f"\n\n[send-ready] Your message is typed and the Send button is index " @@ -531,6 +932,45 @@ async def p_request_browser_approval( return decision +async def try_borrow_signin(domain: str, browser_id: str, tab_id: str, url: str) -> bool: + """Sign in by borrowing the session the user's everyday browser already holds, rather than + interrupting them to type a password we would rather never see. + + True only when the partition genuinely carries their session now, so a False always falls back + to the pause that existed before this did. Off unless the user opted in. + + Swallows everything: this is a convenience bolted onto the critical path, and the worst it may + ever cost is the pause we were going to show anyway. Cancellation still propagates (it is a + BaseException), so a stopped run still stops.""" + from backend.apps.settings.settings import load_settings + + try: + if not browser_session_import.is_enabled(load_settings()): + return False + # Already borrowed at the door and STILL standing at a wall, so the session did not take. + # Re-importing the same values would change nothing; this one needs a human. + if domain in signin_borrowed: + return False + if not browser_session_import.has_importable_session(domain): + return False + signin_borrowed.add(domain) + result = await browser_session_import.import_signin(domain, browser_id) + if not result.ok: + return False + # A borrowed session only takes on the next load, so send the page back through the door. + if url: + await execute_browser_tool("BrowserNavigate", {"url": url}, browser_id, tab_id) + except Exception as exc: + logger.info(f"[session-import] borrow skipped for {domain}: {type(exc).__name__}") + return False + logger.info(f"[session-import] continued on {domain} without interrupting the user") + return True + + +# Background learning tasks (playbook distill) held by strong ref; asyncio only weak-refs tasks, and a GC'd task dies silently mid-distill. +learn_tasks: set[asyncio.Task] = set() + + async def run_browser_agent( task: str, browser_id: str, @@ -541,6 +981,7 @@ async def run_browser_agent( initial_url: str | None = None, parent_session_id: str | None = None, app_mode: bool = False, + user_prompt: str = "", ) -> dict: """Run a browser sub-agent loop for a single browser card. @@ -593,8 +1034,16 @@ async def run_browser_agent( whole point of front-loading). Best-effort; never raises.""" recs = [] try: - li = await execute_browser_tool("BrowserListInteractives", {}, browser_id, tab_id) - gt = await execute_browser_tool("BrowserGetText", {}, browser_id, tab_id) + # The two front-load reads are independent, so fire them together: the AX-tree list (slow, occlusion-filtered) and the text read overlap instead of adding up. return_exceptions keeps it best-effort, one read failing no longer discards the other. + li, gt = await asyncio.gather( + execute_browser_tool("BrowserListInteractives", {}, browser_id, tab_id), + execute_browser_tool("BrowserGetText", {}, browser_id, tab_id), + return_exceptions=True, + ) + if not isinstance(li, dict): + li = {} + if not isinstance(gt, dict): + gt = {} url = li.get("url") or gt.get("url") or label_url or "" parts = [] if li.get("text") and "error" not in li: @@ -636,6 +1085,7 @@ async def run_browser_agent( from backend.apps.settings.credentials import get_anthropic_client_for_model from backend.apps.agents.providers.registry import ( find_builtin_model, + get_api_type, resolve_model_id_for_sdk, resolve_aux_model, ) @@ -644,9 +1094,9 @@ async def run_browser_agent( if find_builtin_model(model) is not None: api_model = resolve_model_id_for_sdk(model, browser_settings) else: - # Unknown model string; fall back to whatever aux model is available + # Unknown model string (custom provider, unrecognized id): fall back to a CAPABLE aux tier, not the cheapest. Browser work is multi-step agentic, and the cheap tier is far weaker at it (OSWorld: Haiku 4.5 50.7% vs Sonnet 4.6 72.5%), so a sonnet-class fallback is worth the cost. resolve_aux_model is provider-agnostic (picks the sonnet tier of whatever the user has connected). try: - api_model, _ = await resolve_aux_model(browser_settings, preferred_tier="haiku") + api_model, _ = await resolve_aux_model(browser_settings, preferred_tier="sonnet") except ValueError: # Nothing connected at all; surface a clear error so the caller (parent agent) sees it in the tool result instead of crashing on a 400 from 9Router. session.status = "error" @@ -672,9 +1122,100 @@ async def run_browser_agent( "action_log": [], "final_screenshot": None, } + # A/B lever (flag-gated, default OFF, and it should STAY off, see below). The idea was to pin the loop to a cheap tier since mechanical browsing "rarely needs frontier reasoning". Two measurements killed it: (1) latency, n=10 live cc/ lanes 2026-07-15, opus-4-6 ~= sonnet-4-6 (~2.3s, the old "Opus->Sonnet 2x" is stale), only Haiku is ~2x/turn faster (1.2s); BUT (2) accuracy, OSWorld computer-use benchmark, Haiku 4.5 = 50.7% vs Sonnet 4.6 = 72.5% (~22pp worse). Per-turn speed is the WRONG metric: a cheap tier fails / needs more recovery turns on multi-step browser tasks, so a 2x-faster turn nets SLOWER + less reliable. Same cliff for non-Claude users (gpt-mini/gemini-flash are weaker than their frontier siblings too), so downgrading the tier hurts everyone. The real cold-start lever is FEWER TURNS on the user's OWN chosen model (plan-dispatch, batching), which preserves accuracy AND is provider-agnostic. Kept as a flag (fail-open to the inherited model) only for explicit experiments; not a default. + p_loop_tier = os.environ.get("OPENSWARM_BROWSER_LOOP_TIER", "").strip().lower() + if p_loop_tier in ("haiku", "sonnet"): + try: + p_pinned, _ = await resolve_aux_model( + browser_settings, preferred_tier=p_loop_tier, primary_api=get_api_type(model)) + if p_pinned and p_pinned != api_model: + logger.info(f"[browser-agent {session_id}] loop-tier pin: {api_model} -> {p_pinned} (tier={p_loop_tier})") + api_model = p_pinned + except Exception as e: + logger.info(f"[browser-agent {session_id}] loop-tier pin skipped ({e}); inheriting {api_model}") # Route the client based on the resolved model id, not just connection_mode. Without this, a pinned-route value like "sonnet-cc" resolves to "cc/claude-sonnet-4-6" but the old get_anthropic_client() still returned an OpenSwarm-proxy client (because connection_mode was openswarm-pro), which then rejected the cc/ prefix and surfaced as a misleading "OpenSwarm servers are busy" error. client = get_anthropic_client_for_model(browser_settings, api_model) + # Skill key, derived EARLY so both the prestage-skip below and the replay lookup share it. Prefer the USER's original request over the orchestrator's reformulation (reformulations vary run-to-run and silently break exact-key replay); multi-quoted messages fall back to the differentiated task. + skill_key_task = task + if parent_session_id: + try: + p_psess = agent_manager.get_session(parent_session_id) + if p_psess: + for p_m in reversed(p_psess.messages): + if p_m.role == "user" and isinstance(p_m.content, str) and p_m.content.strip(): + p_orig = p_m.content.strip() + if len(browser_skills.template_task(p_orig)[1]) <= 1: + skill_key_task = p_orig + break + except Exception: + pass + # A learned skill's replayed prefix does the same navigation prestage would aux-drive (~8-12s): when one exists for this host+task, skip prestage and let the replay own the nav. + # Removal tasks never replay a skill (see the record gate): a delete is a destructive one-shot, + # not a replayable nav prefix, so a stale delete-"skill" of scrolls must not hijack it. + p_task_is_removal = is_removal_task(skill_key_task) + # Computed here (pure, task-only) so the skill gate, prestage, and every dispatch tier below + # share one verdict. `task` here is prompt + an aux-written routing brief (compose_task), and + # the brief's prose can read informational and wrongly disarm a real send (facebook: the brief + # tripped the info-ask gate while the user's own "start a post" is plainly an action). The + # user's words are authoritative, so a send stands if EITHER the composed task OR the raw + # prompt says action. + task_is_send = not (deliverable_is_informational("", task) + and deliverable_is_informational("", user_prompt or task)) + p_early_host = browser_skills.host_of(initial_url or current_url or next(iter(re.findall(r"https?://\S+", task)), "")) + p_skip_prestage_for_skill = browser_skills.replay_owns_nav( + p_early_host, bool(browser_skills.find_skill(p_early_host, skill_key_task)), + p_task_is_removal, task_is_send) + if p_skip_prestage_for_skill: + logger.info(f"[browser-skills] skill exists for {p_early_host}; skipping prestage (replay owns the nav)") + + from backend.apps.agents.browser import browser_prestage + from backend.apps.agents.browser import browser_plan_dispatch + if (browser_prestage.prestage_enabled() and not app_mode and not cancel_event.is_set() + and not p_skip_prestage_for_skill): + try: + p_ps_block, p_ps_url, p_ps_recs = await asyncio.wait_for( + browser_prestage.run_prestage( + task, browser_id, tab_id, current_url, browser_settings, + get_api_type(model), execute_browser_tool, + perceive_only=(not task_is_send and browser_plan_dispatch.plan_dispatch_enabled()), + task_is_send=task_is_send, + ), + timeout=browser_prestage.TOTAL_TIMEOUT_S + 10, + ) + if p_ps_block: + preloaded_perception = p_ps_block + current_url = p_ps_url or current_url + preloaded_reads.extend(p_ps_recs) + except Exception as e: + logger.info(f"[browser-prestage] outer skip ({e})") + + # Early dead-card catch: a wedged or unmounted webview perceives as nothing (no url, no + # elements) even after prestage's warmup. Left alone the model burns a whole run piling up + # card_gone_streak before the late gate evicts it (measured ~70s of ghosting). One confirming + # probe here separates a genuinely dead card from a merely slow one; a dead one is evicted + + # recovered at the top of the loop instead, turning the wasted run into a ~10s honest bail. + p_card_dead_early = False + # Gate on the empty perception BLOCK, not current_url: prestage falls current_url back to + # start_url when the live perceive is empty, so a dead card still reports a url (measured). + # An empty block means the perceive got no live elements/text (dead OR merely thin); the + # confirming probe below is what separates the two. + if (browser_prestage.prestage_enabled() and not app_mode and not p_skip_prestage_for_skill + and not cancel_event.is_set() and not preloaded_perception + and os.environ.get("OSW_DEADCARD_EVICT", "1") != "0"): + try: + p_dead_probe = await asyncio.wait_for( + execute_browser_tool("BrowserGetText", {}, browser_id, tab_id), timeout=6.0) + p_card_dead_early = not (isinstance(p_dead_probe, dict) + and (str(p_dead_probe.get("url") or "") + or str(p_dead_probe.get("text") or ""))) + except Exception: + p_card_dead_early = True + if p_card_dead_early: + logger.warning( + f"[browser-agent {session_id}] card {browser_id} perceives dead after prestage " + "(no url/elements, confirming probe empty); evicting + recovering early") + # Resume prior conversation on this browser if we have one cached. This lets the sub-agent skip the "take a screenshot to figure out where I am" cycle every time the parent issues a new task. Defensively validate the cache; if it's somehow corrupted (orphaned tool_use_ids), drop it and start fresh rather than crash on the next API call. prior_messages = browser_history.BROWSER_HISTORY.get(browser_id) or [] if prior_messages and not validate_message_pairing(prior_messages): @@ -751,6 +1292,7 @@ async def run_browser_agent( loop_trigger_count = 0 card_gone_streak = 0 # consecutive "card is gone" results -> fail fast, don't spin route_hinted_hosts: set[str] = set() # surface the fast network tier once per host + p_login_prompted: set[str] = set() # login-once handoff: at most one sign-in pause per domain per run # Stagnation state: busy-but-stuck detection (no URL change + failures across a run of actions), distinct from the exact-repeat loop above. stagnation_streak = 0 @@ -765,7 +1307,10 @@ async def run_browser_agent( if not p_aux_state["resolved"]: p_aux_state["resolved"] = True try: - aux_model, _ = await resolve_aux_model(browser_settings, preferred_tier="haiku") + # primary_api unlocks the registry's family-match + API-key branches; without it an OpenAI/Google key-only user gets a raise here and auto-scan + playbook learning silently die. + aux_model, _ = await resolve_aux_model( + browser_settings, preferred_tier="haiku", primary_api=get_api_type(model), + ) p_aux_state["model"] = aux_model p_aux_state["client"] = get_anthropic_client_for_model(browser_settings, aux_model) except Exception as e: @@ -793,15 +1338,21 @@ async def run_browser_agent( if not isinstance(page, dict) or page.get("error") or not page.get("text"): return "" aux_client, aux_model = await p_get_aux_client() + # Cap the aux input to the top of the page: results pages put the matches first, and + # feeding a huge LinkedIn/Amazon page to the aux was blowing the whole time budget so + # the scan timed out with NOTHING (measured: 8001ms idle, empty). A head slice lets + # the aux actually FINISH fast (useful hint) instead of stalling. return await browser_extract.extract_structured( - aux_client, aux_model, str(page["text"]), + aux_client, aux_model, str(page["text"])[:P_SCAN_TEXT_CAP], "These are search results. Identify which result(s) match this task: " f"{scan_for[:400]}\nFor each plausible candidate give its exact displayed name, " "the distinguishing details shown (role, company, location, etc), and why it " "does or does not match. If none clearly match, say so in `best`.", {"candidates": [{"name": "", "details": "", "match": ""}], "best": ""}, ) - out = await asyncio.wait_for(p_inner(), timeout=8.0) + # Bail fast: a scan that can't produce a hint quickly is worse than none (the model reads + # the page itself next turn anyway), so don't sit idle on it. 8s -> P_SCAN_TIMEOUT_S. + out = await asyncio.wait_for(p_inner(), timeout=P_SCAN_TIMEOUT_S) except Exception: out = "" return out or "", int((time.time() - p_t0) * 1000) @@ -846,7 +1397,10 @@ async def run_browser_agent( "type": "text", "text": run_system_prompt, "cache_control": {"type": "ephemeral"}, }] + from backend.apps.agents.browser import browser_delete_script p_cached_tools = [dict(t) for t in (APP_VISIBLE_TOOLS if app_mode else browser_schema.MODEL_VISIBLE_TOOLS)] + if not browser_delete_script.delete_tool_enabled(): + p_cached_tools = [t for t in p_cached_tools if t["name"] != "BrowserDeleteItem"] if p_cached_tools: p_cached_tools[-1] = {**p_cached_tools[-1], "cache_control": {"type": "ephemeral"}} @@ -857,8 +1411,8 @@ async def run_browser_agent( "message": user_msg.model_dump(mode="json"), }) - # Perceived value, zero clicks: one calm line so the user FEELS the agent is picking up where it left off, not figuring the site out cold again. Only when strategy was actually seeded, so it's honest, never noise. - if pb_seeded and p_pb_host: + # Perceived value, zero clicks: one calm line so the user FEELS the agent is picking up where it left off. It claims "from a previous visit", so it must fire ONLY on a genuinely LEARNED playbook, never a shipped SEED (else a first-ever visit to any seeded popular site would lie about a visit that never happened). Seeds still inject their head-start into the prompt above; they just don't trigger this line. + if p_pb_host and browser_playbook.load(p_pb_host): session.memory_recalled = True # drives the subtle "Remembered" card chip p_where = "this app" if app_mode else p_pb_host p_recall_msg = Message(role="assistant", @@ -886,20 +1440,7 @@ async def run_browser_agent( return None return task.result() - # Skill key: prefer the USER's original request over the orchestrator's reformulation. The reformulation varies run-to-run ("click the search box" vs "find the search box") and that variance silently breaks exact-key replay (measured: two issuances of one request produced two skills). The user's words are stable across repeats. Guard: if the message carries multiple quoted values, several same-host sub-tasks could collide on one key, so fall back to the (differentiated) delegated task; the verify gate backs this up if a key is ever too loose. - skill_key_task = task - if parent_session_id: - try: - p_psess = agent_manager.get_session(parent_session_id) - if p_psess: - for p_m in reversed(p_psess.messages): - if p_m.role == "user" and isinstance(p_m.content, str) and p_m.content.strip(): - p_orig = p_m.content.strip() - if len(browser_skills.template_task(p_orig)[1]) <= 1: - skill_key_task = p_orig - break - except Exception: - pass + # (skill_key_task derived earlier, above the prestage gate, so the skip check and this lookup share one key) # --- Fast path: replay a previously-learned skill with NO LLM round-trips. This is what gets a REPEAT task from ~50s (full agent loop) down to ~1s, i.e. faster than a human. Robust by construction: clicks re-resolve by (role,name), every step is verified, and ANY miss aborts to the full LLM agent below (which re-records), so a changed page can never ghost-succeed. replay_attempted = False @@ -917,6 +1458,7 @@ async def run_browser_agent( allow_prefix, a send-gated skill replays its safe navigation prefix mechanically and hands the live agent just the irreversible tail.""" nonlocal final_screenshot, last_seen_url, replay_attempted, replay_prefix_note + nonlocal preloaded_perception, current_url if not host: return None @@ -954,6 +1496,11 @@ async def run_browser_agent( logger.info(f"[browser-skills] skill on {host} not replayed: {why}; running the full agent so the send is confirmed") return None prefix = steps[:unsafe_i] + # Marriage mode: the send-script owns the composer (it polls for the lazy overlay + fills + sends). Replaying a recorded composer-textbox click races that render and misses (v903/v906), so truncate the prefix to NAV + opener only and let the script take it from the navigated page. Keep >=1 step or there's nothing to replay. + if os.environ.get("OSW_REPLAY_SENDTAIL", "0") == "1": + p_nav_prefix = [s for s in prefix if not browser_skills.step_touches_composer(s)] + if p_nav_prefix: + prefix = p_nav_prefix logger.info( f"[browser-skills] PREFIX replay: {len(prefix)}/{len(steps)} steps on {host}, " f"live agent confirms the tail ({why})" @@ -981,6 +1528,11 @@ async def run_browser_agent( f"full agent from scratch (trust verdict: {verdict})" ) return None + # The end-of-run record_skill distills from action_log; without the replayed prefix in it, a warm run re-records a TAIL-ONLY skill (navigation missing) and clobbers the good one. + action_log.append({ + "tool": step["tool"], "input": step.get("params", {}), "ok": True, + "result_summary": str(res.get("text", ""))[:200], "elapsed_ms": el_ms, + }) if res.get("url"): last_seen_url = res["url"] replay_attempted = True @@ -989,6 +1541,12 @@ async def run_browser_agent( lst = await execute_browser_tool("BrowserListInteractives", {}, browser_id, tab_id) if isinstance(lst, dict) and lst.get("text") and "error" not in lst: p_fresh = f"\nCurrent page state after the replayed prefix:\n{p_truncate_state(lst['text'])}" + # THE MARRIAGE (flag-gated): hand the post-prefix state to the verified send-script slot, the proven code tail (fill -> verify -> send -> two-sided receipt). A warm write then completes with ZERO model turns: replayed prefix + verified tail. Fail-open: if the script declines, the model gets the existing handoff note, today's behavior. + if os.environ.get("OSW_REPLAY_SENDTAIL", "0") == "1": + preloaded_perception = str(lst["text"]) + if lst.get("url"): + current_url = str(lst["url"]) + logger.info("[browser-skills] prefix handoff -> send-script slot armed (perception + url set)") except Exception: pass remaining = "; ".join(f"{s['tool']}({str(s.get('params', {}))[:80]})" for s in steps[unsafe_i:]) @@ -1072,8 +1630,8 @@ async def run_browser_agent( replay_host = browser_skills.host_of(m.group(0)) # The card might have started on the WRONG host (the orchestrator often opens a fresh card on google and only navigates to the target later); if so, this dispatch check misses and the deferred re-check inside the loop catches it after the first navigation. replay_rechecked = False - logger.info(f"[browser-skills] dispatch replay check: host={replay_host!r}") - p_dispatch_replay = await p_try_replay(replay_host, 0, allow_prefix=True) + logger.info(f"[browser-skills] dispatch replay check: host={replay_host!r} removal={p_task_is_removal}") + p_dispatch_replay = None if p_task_is_removal else await p_try_replay(replay_host, 0, allow_prefix=True) if p_dispatch_replay is not None: return p_dispatch_replay if replay_prefix_note: @@ -1092,9 +1650,163 @@ async def run_browser_agent( f"sim={p_h_score:.2f} steps={len(route_hint_keys)} state={p_h_skill.get('state')}" ) - # Pre-nav landed on a results page (the cold entry case): scan it NOW so the model's very first turn can pick a candidate instead of read-then-decide. + text_parts = [] # initialized before loop so post-loop summary (line ~1294) has a default + rp_violations = 0 # turns the model acted without ReportProgress (now accepted + reminded, not rejected) + # The model finishes by calling the Done tool; `message` is the clean human reply, `success` whether the goal was met. Falls back to terminal text on the rare run that stops without calling Done. + done_called = False + done_message = "" + done_success = True + done_keep_open = False + # `send_confirmed` means "the click ran, so never fire another one" and NOTHING more. It was + # doing double duty as permission to claim success, which is how a run that clicked send with no + # receipt still reported "your message went through and it's showing in the conversation now" + # (measured live on X 2026-07-28; nothing had been posted). Not repeating an action and being + # allowed to claim it worked are different facts about the world, so they get different flags. + delivery_verified = False + # Completion detection uses task_is_send (computed above, before the candidate scan): once an irreversible SEND has confirmed, the goal is met. The model otherwise stalls re-verifying what the confirm already proved (measured: send done at turn ~11, then ~12 wasted perception turns). We drive it to the OUTCOME and, if it keeps re-perceiving, end the run. A genuine multi-send task issues its NEXT send (an action) which resets the stall, so only true spinning ends here. Meaningless for a gather/read task, and arming it there let a cookie 'Accept all' click masquerade as the task's send, so we gate it on intent. + send_confirmed = False + # Two-sided receipt evidence: the fill must have VISIBLY committed its text to a textbox before a send-class click may end the run in code. r228 clicked a send-labeled control after an uncommitted fill and the old click-name-only receipt claimed a send that never happened. + composer_committed_payload = "" + perception_stall = 0 # consecutive turns the model only LOOKED (no action) + P_POST_SEND_STALL_LIMIT = 2 # once the send registered, finish fast + P_PERCEPTION_STALL_LIMIT = 6 # backstop when we couldn't detect the send (e.g. Enter): bound the spin + # When the spin backstop trips we don't guillotine the run (that leaks the model's half-finished sentence as the reply). We nudge it to wrap up ONCE, so it summarizes what it has via Done; a second trip then stops for real. + wrapup_nudged = False + # Distinct read results seen, so the backstop tells a productive page-by-page gather (new data each turn) from genuine spinning (re-reading the same thing). + seen_read_sigs: set[str] = set() + # rows already shown to the model; attached state shrinks to the delta + attached_state_seen: set[str] = set() + # under-batching telemetry + nudge state + single_action_streak = 0 + batching_nudges = 0 + redundant_read_nudges = 0 + # True after a mutating action attaches fresh state; a solo read next is waste + fresh_state_pending = False + multi_action_turns = 0 + batch_calls = 0 + batch_guard_blocks = 0 + + # Staged-send script: prestage left a ready composer + the task quotes its payload -> code runs the fill/verify/send/verify tail the model spends 4-5 turns on. Success skips the loop entirely (turns=0); any pre-click ambiguity falls through untouched. + p_script = None + # A removal task ("delete the post that says 'X'") is also task_is_send (the classifier keys + # on the verb), so the send-script must stand down or it TYPES the target into a composer and + # POSTS it (measured live: delete tasks re-posted the marker). BrowserDeleteItem owns removals. + if (browser_send_script.script_enabled() and task_is_send and not is_removal_task(task) + and not app_mode and preloaded_perception and not cancel_event.is_set()): + try: + p_script = await asyncio.wait_for(browser_send_script.run_send_script( + task, browser_id, tab_id, preloaded_perception, + execute_browser_tool, send_submit_index_in_state, payload_in_textbox, + payload_source=user_prompt, current_url=current_url, + ), timeout=browser_send_script.WORST_CASE_BUDGET_S) + except Exception as p_se: + # Name the class: a bare TimeoutError stringifies to nothing, so this used to log + # "outer skip ()" and a starved send looked identical to a page we chose not to touch. + logger.info(f"[browser-sendscript] outer skip ({type(p_se).__name__}: {p_se})") + p_script = None + if isinstance(p_script, dict): + action_log.extend(p_script["log"]) + if p_script["sent"]: + send_confirmed = True + done_called = True + p_aux_c, p_aux_m = await p_get_aux_client() + if p_script.get("delivered") is False: + # ghost-drop host: composer cleared but the post did NOT persist. Tell the + # truth and don't learn a recipe for a send that never actually landed. + done_success = False + done_message = (await compose_delivery_warning(p_aux_c, p_aux_m, task, p_script["payload"], current_url) + or browser_delivery_check.unconfirmed_delivery_note(current_url, p_script["payload"])) + else: + # receipt verified (composer cleared): the send is DONE, end the run + done_success = True + delivery_verified = True + await p_learn_write_recipe(execute_browser_tool, browser_id, tab_id, current_url, p_script["payload"]) + done_message = (await compose_send_confirmation(p_aux_c, p_aux_m, task, p_script["payload"]) + or f'Done, I sent "{p_script["payload"]}" for you.') + else: + # Clicked but the composer did NOT clear: the send is UNVERIFIED. Leave send_confirmed False so the loop can't shortcut to a "done" it never earned (r264 set it True here and the model then FALSELY claimed delivery). The model gets ONE truthful verify pass, never a blind resend. + task = f"{task}\n\n[{p_script['note']}]" + + # Code-side DELETE dispatch: the removal analog of the send-script. A "delete the post that + # says X" task auto-fires the same verified remove flow (find the item by its text, open its + # overflow menu, Delete, confirm, verify-gone) on the page prestage landed, instead of waiting + # for the model to reach for BrowserDeleteItem, which it doesn't: live, it read the post then + # claimed it "can only read" and gave up. If the item isn't on the landed page it declines and + # the model loop navigates to where it lives. Skipped in dry-run (this is a real destructive act). + from backend.apps.agents.browser import browser_delete_script + if (browser_delete_script.delete_tool_enabled() and is_removal_task(task) and not app_mode + and not done_called and preloaded_perception and not cancel_event.is_set() + and os.environ.get("OSW_SENDSCRIPT_DRYRUN") != "1"): + p_del_target = browser_send_parse.quoted_payload(user_prompt or task) + if p_del_target and len(p_del_target) >= browser_delete_script.MIN_TARGET_CHARS: + try: + p_del = await browser_delete_script.run_delete(p_del_target, browser_id, tab_id, execute_browser_tool) + except Exception as p_de: + logger.info(f"[browser-deletedispatch] outer skip ({p_de})") + p_del = {"removed": False, "stage": "eval", "msg": str(p_de)} + logger.info(f"[browser-deletedispatch] target={p_del_target[:40]!r} " + f"removed={p_del['removed']} stage={p_del['stage']}") + if p_del["removed"]: + done_called = True + done_success = True + # The deletion is already verified gone in code; this only writes the words, and it + # writes them in the model's voice rather than a template. + p_aux_c, p_aux_m = await p_get_aux_client() + done_message = (await compose_removal_confirmation(p_aux_c, p_aux_m, task, p_del_target) + or f'Done, I deleted the item containing "{p_del_target[:60]}" (verified gone).') + # not removed: fall through to the model loop, which navigates to where the item lives. + + # Dry-run is a measurement mode, so the run ENDS here either way: letting the model loop run would both risk the REAL send the flag exists to avoid and rescue declines the flag exists to attribute. Inert when the flag is off. + if os.environ.get("OSW_SENDSCRIPT_DRYRUN") == "1" and not done_called: + p_dr = browser_send_parse.dryrun_report( + preloaded_perception or "", bool(task_is_send and preloaded_perception), + isinstance(p_script, dict), current_url) + logger.info(p_dr) + done_called = True + done_success = True + done_message = ("DRY-RUN coverage probe: no send was performed and none should be " + "retried; report this outcome verbatim. " + p_dr) + + # Code-side plan dispatch (the turn-collapser that doesn't wait for the model to adopt a tool): one aux call compiles the task's mechanical prefix into verified steps, code executes them, and the big model starts with that work DONE. Fail-open: no plan/steps = today's loop untouched. + # A send task whose composer is ALREADY staged has no mechanical prefix left (the model's one fill turn + autosend own the rest), so skip the aux call instead of letting it poke the composer (measured 4.7s of nothing). + from backend.apps.agents.browser import browser_plan_dispatch + p_composer_staged = bool(task_is_send and browser_send_parse.composer_index_in_state(preloaded_perception or "")) + if (browser_plan_dispatch.plan_dispatch_enabled() and not app_mode and not done_called + and preloaded_perception and not p_composer_staged and not cancel_event.is_set()): + try: + p_plan_note = await asyncio.wait_for(browser_plan_dispatch.run_plan_dispatch( + task, preloaded_perception, browser_id, tab_id, + load_settings(), get_api_type(model), execute_browser_tool, + current_url=current_url, + ), timeout=45.0) + except Exception as p_pe: + logger.info(f"[plan-dispatch] outer skip ({p_pe})") + p_plan_note = "" + if p_plan_note: + task = f"{task}\n\n{p_plan_note}" + + # READ leg for authed pages (the extraction turn-collapser): prestage landed the logged-in card on the target page; one aux read over the live page text answers a read task without the big-model loop. Fail-open: decline = the loop runs as today. + from backend.apps.agents.browser import browser_read_script + if (browser_read_script.read_script_enabled() and not task_is_send and not app_mode + and not done_called and preloaded_perception and not cancel_event.is_set()): + try: + p_aux_c, p_aux_m = await p_get_aux_client() + p_read_answer = await asyncio.wait_for(browser_read_script.run_read_script( + p_aux_c, p_aux_m, task, browser_id, tab_id, execute_browser_tool, + current_url=current_url, + ), timeout=25.0) + except Exception as p_re: + logger.info(f"[browser-readscript] outer skip ({p_re})") + p_read_answer = None + if p_read_answer: + done_called = True + done_success = True + done_message = p_read_answer + + # Candidate scan on a results-page entry, MOVED below the collapse tiers: it exists to hint the MODEL's first turn, so on a run the tiers already finished it was a measured ~4s of blocking aux for nothing. p_start_url = (current_url or initial_url or "").split("#")[0] - if p_start_url and RESULTS_URL_RE.search(p_start_url): + if (p_start_url and RESULTS_URL_RE.search(p_start_url) and not task_is_send + and not done_called and not cancel_event.is_set()): auto_scanned_urls.add(p_start_url) p_scan_json, p_sc_ms = await p_scan_results(task) if p_scan_json: @@ -1118,56 +1830,75 @@ async def run_browser_agent( f"{p_start_url[:90]} after {p_sc_ms}ms" ) - text_parts = [] # initialized before loop so post-loop summary (line ~1294) has a default - rp_violations = 0 # turns the model acted without ReportProgress (now accepted + reminded, not rejected) - # The model finishes by calling the Done tool; `message` is the clean human reply, `success` whether the goal was met. Falls back to terminal text on the rare run that stops without calling Done. - done_called = False - done_message = "" - done_success = True - done_keep_open = False - # Completion detection: once an irreversible SEND has confirmed, the goal is met. The model otherwise stalls re-verifying what the confirm already proved (measured: send done at turn ~11, then ~12 wasted perception turns). We drive it to the OUTCOME and, if it keeps re-perceiving, end the run. A genuine multi-send task issues its NEXT send (an action) which resets the stall, so only true spinning ends here. This whole shortcut is meaningless for a gather/read task (no send to confirm), and arming it there let a cookie 'Accept all' click masquerade as the task's send, so we gate it on intent. - task_is_send = not deliverable_is_informational("", task) - send_confirmed = False - perception_stall = 0 # consecutive turns the model only LOOKED (no action) - P_POST_SEND_STALL_LIMIT = 2 # once the send registered, finish fast - P_PERCEPTION_STALL_LIMIT = 6 # backstop when we couldn't detect the send (e.g. Enter): bound the spin - # When the spin backstop trips we don't guillotine the run (that leaks the model's half-finished sentence as the reply). We nudge it to wrap up ONCE, so it summarizes what it has via Done; a second trip then stops for real. - wrapup_nudged = False - # Distinct read results seen, so the backstop tells a productive page-by-page gather (new data each turn) from genuine spinning (re-reading the same thing). - seen_read_sigs: set[str] = set() - # rows already shown to the model; attached state shrinks to the delta - attached_state_seen: set[str] = set() - # under-batching telemetry + nudge state - single_action_streak = 0 - batching_nudges = 0 - redundant_read_nudges = 0 - # True after a mutating action attaches fresh state; a solo read next is waste - fresh_state_pending = False - multi_action_turns = 0 - batch_calls = 0 - batch_guard_blocks = 0 try: for turn in range(MAX_TURNS): - if cancel_event.is_set(): + if p_card_dead_early: + # Prestage already proved this webview dead; evict + report unresponsive via the + # exact same path the late gate uses, so recovery re-dispatches a fresh card + # without burning a run. turn is 0 here, so terminal handling stays well-defined. + DEAD_CARDS.add(browser_id) + await evict_dead_card(dashboard_id, browser_id) + card_gone_streak = CARD_GONE_LIMIT break + if done_called or cancel_event.is_set(): + break + + # Login-once handoff: if we've landed on a login wall, pause so the user can sign in ONCE + # in this card (the persistent partition keeps the session, so future runs won't ask + # again), then continue. At most one pause per domain per run; the model's own + # RequestHumanIntervention stays as the fallback for walls this detector misses. + # Soft signed-out (composer withheld behind a "Sign in") only counts once the agent has + # actually tried and is still stuck, so a first-turn glance can't raise a false prompt. + p_wall_dom = browser_login_handoff.login_wall_domain( + last_seen_url, "\n".join(attached_state_seen), allow_soft=(turn >= 2)) + if p_wall_dom and p_wall_dom not in p_login_prompted: + p_login_prompted.add(p_wall_dom) + # Borrow the sign-in the user already has in their everyday browser first: when it + # lands nobody is interrupted at all. Anything less falls through to the pause. + p_signed_in = await try_borrow_signin(p_wall_dom, browser_id, tab_id, last_seen_url) + if not p_signed_in: + p_login_problem, p_login_instruction = browser_login_handoff.prompt_copy(p_wall_dom) + p_login_decision = await p_request_browser_approval( + session, "RequestHumanIntervention", + {"problem": p_login_problem, "instruction": p_login_instruction}) + if cancel_event.is_set(): + break + p_signed_in = p_login_decision.get("behavior") != "deny" + if p_signed_in: + browser_login_handoff.record_login(p_wall_dom) + p_signed_note = (f"You are now signed in to {p_wall_dom}. The page has changed; " + "look at it fresh and continue the task.") + if messages and messages[-1].get("role") == "user": + p_prev = messages[-1]["content"] + if isinstance(p_prev, list): + p_prev.append({"type": "text", "text": p_signed_note}) + else: + messages[-1]["content"] = f"{p_prev}\n\n{p_signed_note}" + else: + messages.append({"role": "user", "content": p_signed_note}) # Drop stale screenshots before each call: keep first + previous + current, stub the rest. Images are ~1.3-2k tokens each and get re-read every turn, so this is the biggest per-turn context win on any visual task (measured ~2.9x fewer image tokens, ~5x less upload). browser_history.prune_old_screenshots(messages) browser_history.prune_stale_page_state(messages) browser_history.place_cache_marker(messages) p_llm_t0 = time.monotonic() + # STREAM, don't .create(): 9Router returns non-Anthropic lanes (Gemini/OpenRouter/Antigravity) as a REAL multi-event SSE stream that the non-streaming client parses to empty content, silently breaking tool-use on every non-Claude provider (measured: only cc/ emitted tool_use before this). The streaming parser reconstructs tool_use identically for ALL providers, so this is the model-independence fix, not a UX tweak. Cache marker still rides p_cached_system (Anthropic keys on it; other routes ignore it harmlessly). + async def p_stream_turn(): + async with client.messages.stream( + model=api_model, + max_tokens=4096, + # Cache the ~4k-token fixed prefix (system + tool schema) so it's reprocessed once, not on every turn: big TTFT + cost win on the first run, which is dominated by turns x per-turn prefill. The trailing cache_control marker is what Anthropic keys on; on non-Anthropic routes (9router) the marker is harmlessly ignored. + system=p_cached_system, + tools=p_cached_tools, + messages=messages, + ) as p_s: + return await p_s.get_final_message() + # Transient-capacity retry (free pool busy / 429 / overload): same backoff budget as chat turns, so a busy free tier waits instead of erroring the whole task. The sleep watches cancel_event so Stop stays instant. p_capacity_attempt = 0 while True: try: - response = await p_cancellable(client.messages.create( - model=api_model, - max_tokens=4096, - # Cache the ~4k-token fixed prefix (system + tool schema) so it's reprocessed once, not on every turn: big TTFT + cost win on the first run, which is dominated by turns x per-turn prefill. The trailing cache_control marker is what Anthropic keys on; on non-Anthropic routes (9router) the marker is harmlessly ignored. - system=p_cached_system, - tools=p_cached_tools, - messages=messages, - )) + response = await p_cancellable(p_stream_turn()) break except Exception as p_api_err: p_wait = capacity_retry_wait(p_api_err, p_capacity_attempt) @@ -1330,10 +2061,23 @@ async def run_browser_agent( else (P_PERCEPTION_STALL_LIMIT if p_acted else 10 ** 9)) if perception_stall >= p_stall_limit: if send_confirmed: - # the send registered: hand the parent a real done. The raw action-log proof (indices/coords) is machine-speak, kept out. + # The send CLICK ran; whether it landed is a separate question and the + # message has to answer the one we actually have evidence for. The raw + # action-log proof (indices/coords) is machine-speak, kept out. done_called = True - done_message = "All set, your message went through and it's showing in the conversation now." - logger.info(f"[browser-agent {session_id}] ending: {perception_stall} post-send perception turns") + done_success = delivery_verified + p_aux_c, p_aux_m = await p_get_aux_client() + p_pay = composer_committed_payload or "" + if delivery_verified: + p_nice = await compose_send_confirmation(p_aux_c, p_aux_m, task, p_pay) + done_message = p_nice or ( + f'Done, I sent "{p_pay}" for you.' if p_pay else "Done, that's sent.") + else: + done_message = (await compose_unverified_send( + p_aux_c, p_aux_m, task, p_pay, current_url) + or browser_delivery_check.unverified_send_note(current_url, p_pay)) + logger.info(f"[browser-agent {session_id}] ending: {perception_stall} post-send " + f"perception turns, delivery_verified={delivery_verified}") break if not wrapup_nudged: # don't cut it off mid-thought: ride a wrap-up nudge out on this turn's tool_results so next turn it answers via Done. @@ -1346,7 +2090,13 @@ async def run_browser_agent( logger.info(f"[browser-agent {session_id}] ending: wrap-up nudge ignored, stopping") if not done_called: done_called = True - done_message = "That's as far as I could get gathering this one." + # The run is being cut off mid-thought, so say what was actually + # reached in the model's own words instead of a stock apology that + # tells the user nothing about what it did or did not get. + p_aux_c, p_aux_m = await p_get_aux_client() + done_message = (await compose_partial_result( + p_aux_c, p_aux_m, task, action_log) + or "That's as far as I could get gathering this one.") break else: perception_stall = 0 @@ -1505,6 +2255,65 @@ async def run_browser_agent( continue # Intra-run batch replay: run a learned mechanical flow for many inputs at machine speed, verify every step, gate sends, never ghost. Reads/searches loop freely; irreversible steps refuse. + if tu.name == "BrowserActVerified": + from backend.apps.agents.browser import browser_verified_step + from backend.apps.agents.browser.browser_prestage import BLOCKED_CLICK_RE + p_steps_in = tu.input.get("steps") or [] + p_step_lines: list[str] = [] + p_all_ok = True + if not p_steps_in: + p_va_text = "No steps given; nothing to do." + else: + for p_si, p_raw in enumerate(p_steps_in[:4], start=1): + p_tgt = str((p_raw or {}).get("target") or "") + # the solo-send rule holds here in CODE: an irreversible-smelling target is refused, exactly like a batch + if BLOCKED_CLICK_RE.search(p_tgt): + p_step_lines.append(f"{p_si}. REFUSED: {p_tgt!r} looks irreversible; do it as a SOLO click with an `expect` proof.") + p_all_ok = False + break + p_vstep = browser_verified_step.VerifiedStep( + kind=str(p_raw.get("action") or "click"), target=p_tgt, + role=str(p_raw.get("role") or ""), text=str(p_raw.get("text") or ""), + expect=str(p_raw.get("expect") or "")) + p_st = time.time() + p_vr = await p_cancellable(browser_verified_step.run_verified_step( + p_vstep, browser_id, tab_id, execute_browser_tool)) + if p_vr is None: + p_step_lines.append(f"{p_si}. cancelled"); p_all_ok = False; break + p_el = int((time.time() - p_st) * 1000) + action_log.append({ + "tool": "BrowserActVerified", "input": p_raw, "ok": p_vr["ok"], + "result_summary": (f"{p_vstep.kind} {p_tgt!r} verified" if p_vr["ok"] + else str(p_vr["note"]))[:200], + "elapsed_ms": p_el, + }) + browser_metrics.record_tool( + session_id, browser_id, turn, "BrowserActVerified", p_el, ok=p_vr["ok"], + error="" if p_vr["ok"] else str(p_vr["note"]), is_loop=False, + stagnation_streak=0, result_len=0) + if p_vr["ok"]: + p_step_lines.append(f"{p_si}. {p_vstep.kind} {p_tgt!r}: OK (verified)") + else: + p_step_lines.append(f"{p_si}. {p_vstep.kind} {p_tgt!r}: FAILED ({p_vr['note']}); remaining steps skipped") + p_all_ok = False + break + p_va_text = ("All steps verified:\n" if p_all_ok else "Stopped early:\n") + "\n".join(p_step_lines) + # fold the post-plan page state in so the model's next turn already sees the result + p_va_state = await post_action_state( + "BrowserBatch", {}, {"ok": True}, browser_id, tab_id, + wait_exec=execute_browser_tool, goal=current_next_goal or "", + seen_lines=attached_state_seen) + if p_va_state: + p_va_text += p_va_state + fresh_state_pending = True + tool_results.append({"type": "tool_result", "tool_use_id": tu.id, "content": [{"type": "text", "text": p_va_text}]}) + result_msg = Message(role="tool_result", content={"text": p_va_text, "tool_name": tu.name, "elapsed_ms": 0}) + session.messages.append(result_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, "message": result_msg.model_dump(mode="json"), + }) + continue + if tu.name == "BrowserRepeatFlow": steps_tmpl = tu.input.get("steps") or [] values = [str(v) for v in (tu.input.get("values") or [])] @@ -1673,6 +2482,12 @@ async def run_browser_agent( "do that step SOLO with BrowserClickIndex + `expect` proof, and " "batch only the routine steps around it." )} + elif tu.name == "BrowserApiWrite": + # API-first write tier: the site's own write API via the borrowed session, deterministic + a real receipt. A miss is a typed "use the UI" (never a crash), so the loop falls back cleanly. + result = await p_cancellable(run_api_write(tu.input, current_url, browser_id, tab_id)) + if result is None: + cancelled = True + break elif tu.name == "BrowserWait": # Smart wait: return as soon as the page is ready (target or DOM settle), not on a blind timer (the audit's 42%-of-time hog). result = await browser_wait.smart_wait( @@ -1688,10 +2503,27 @@ async def run_browser_agent( break elapsed_ms = int((time.time() - start) * 1000) + # An API-first write that returned ok carries its own typed receipt, so it IS the confirmation: mark the send done so the loop drives to Done without a redundant UI re-verify (and never re-fires it). + if tu.name == "BrowserApiWrite" and isinstance(result, dict) and result.get("ok"): + send_confirmed = True + delivery_verified = True # a typed API receipt IS the evidence, not a proxy for it + # Act-and-confirm: if the agent declared the change it expects, VERIFY it actually happened, success is observed, never assumed. A hit returns fast (act + confirm in one turn); a miss is a clear "may not have worked" (and a wedge surfaces as a clean not-confirmed, not a blind 20s timeout), so the agent never claims a success it didn't see or re-fires blindly. p_expect = (str(tu.input.get("expect") or "").strip() if isinstance(tu.input, dict) else "") - if p_expect and "error" not in result and tu.name in P_CONFIRM_TOOLS: + # A send-class click's text-probe is documented-unreliable (sent text renders late/split/scrolled off) and the composer-clear receipt supersedes it, so don't burn the 4s probe timeout on exactly the clicks that never confirm by text. + p_is_send_click = (task_is_send and "error" not in result and tu.name in P_CONFIRM_TOOLS + and (browser_batch_replay.is_send_completed( + {"action": "click", "name": result.get("clickedName") or "", + "role": result.get("clickedRole") or ""}) or any( + browser_batch_replay.is_send_completed( + {"action": "click", "name": r.get("clickedName") or "", + "role": r.get("clickedRole") or ""}) + for r in (result.get("results") or [])))) + if p_is_send_click and p_expect: + # Click registration is NOT delivery proof (curve2: X swallowed a clean click); this branch only skips the unreliable 4s text-probe, the composer receipt below is the arbiter. + result["text"] = f"{result.get('text') or ''}\nThe send-class click registered; delivery is checked by the composer receipt." + elif p_expect and "error" not in result and tu.name in P_CONFIRM_TOOLS: # target_only: wait for the expected text to actually appear, don't call it 'not confirmed' just because the page settled first (a sent message lands in the thread a beat after settle, esp. under load) p_conf = await browser_wait.smart_wait(p_wait_exec, browser_id, tab_id, 4000, until=p_expect, target_only=True) @@ -1721,9 +2553,49 @@ async def run_browser_agent( for r in (result.get("results") or [])) if p_send_click: send_confirmed = True - result["text"] = (f"{result.get('text') or ''}\n\n[task complete] The send " - "went through (the composer cleared). Don't re-check it. Finish now by " - "calling Done with your reply to the user.") + p_receipt_ok = False + if os.environ.get("OSW_RECEIPT_DONE", "1") != "0" and composer_committed_payload: + # Deterministic receipt, two-sided: the fill was SEEN committed to a textbox earlier, and the box must now be SEEN empty of it. Click-name alone is not proof (r228: send-labeled click after an uncommitted fill = false success); missing evidence falls through to the old model-verified path, so the failure mode costs turns, never a lie. + try: + for p_rw in (0.4, 1.0): + await asyncio.sleep(p_rw) + p_rl2 = await asyncio.wait_for( + p_wait_exec("BrowserListInteractives", {}, browser_id, tab_id), + timeout=5.0) + if isinstance(p_rl2, dict) and "error" not in p_rl2 and p_rl2.get("text"): + if not payload_in_textbox(str(p_rl2["text"]), composer_committed_payload): + p_receipt_ok = True + break + except Exception: + p_receipt_ok = False + if p_receipt_ok: + done_called = True + done_success = True + delivery_verified = True # two-sided receipt: fill committed AND box now empty + p_payload = browser_batch_replay.send_payload_from_log(action_log, task) + await p_learn_write_recipe(execute_browser_tool, browser_id, tab_id, current_url, p_payload) + p_aux_c, p_aux_m = await p_get_aux_client() + p_nice = (await compose_send_confirmation(p_aux_c, p_aux_m, task, p_payload) + if p_payload else "") + done_message = p_nice or ( + f'Done, I sent "{p_payload}" for you.' + if p_payload else + "Done, I sent your message." + ) + logger.info(f"[browser-receipt {session_id}] two-sided receipt passed (fill committed + composer cleared); run ends in code") + else: + logger.info(f"[browser-receipt {session_id}] receipt WITHHELD (composer state unverified); model verifies") + # The completion text must match the evidence: the old unconditional "went through, don't re-check" rode along even when the receipt was WITHHELD, and the model repeated it to the user as fact while nothing had posted (curve2 X, live). Verified = drive to Done; unverified = one truthful verify pass, never a resend. + if p_receipt_ok: + result["text"] = (f"{result.get('text') or ''}\n\n[task complete] The send " + "went through (the composer cleared). Don't re-check it. Finish now by " + "calling Done with your reply to the user.") + else: + result["text"] = (f"{result.get('text') or ''}\n\n[send clicked, NOT verified] " + "The send click registered but delivery was NOT confirmed (no composer-cleared " + "receipt). Do NOT click send again, a resend risks a double-post. Verify ON THE " + "PAGE that the message actually posted (find it in the thread/feed/sent items); " + "if you cannot see it, report honestly that the send did not confirm.") action_log.append({ "tool": tu.name, @@ -1780,6 +2652,44 @@ async def run_browser_agent( result["text"] = f"{result.get('text') or ''}{p_auto_state}" # a mutation attached fresh state; it stays "available" through intervening reads (Wait/Extract don't invalidate it), so a later solo re-list is still caught as redundant. fresh_state_pending = True + p_fill_text = fill_text_of(tu.name, tool_input) + if p_fill_text and payload_in_textbox(p_auto_state or "", p_fill_text): + composer_committed_payload = p_fill_text + # B: the model just TYPED the message into a composer on a send task, so finish the send in CODE (find Send, click, verify the composer cleared) instead of it burning ~3-4 turns on a Send button whose index goes stale after the fill. Uses what the MODEL typed, so un-quoted phrasings ("say hi" -> "hi") work; fails safe, an unverified click never claims delivery and send_confirmed blocks a resend. + if (task_is_send and not send_confirmed and tu.name in P_CONFIRM_TOOLS + and browser_send_script.autosend_enabled()): + p_cs = await browser_send_script.complete_send( + composer_committed_payload, p_auto_state or "", browser_id, tab_id, + execute_browser_tool, send_submit_index_in_state, + composer_index=fill_index_of(tu.name, tool_input), current_url=current_url) + if p_cs.get("clicked"): + # True the moment the click RUNS, because its job here is to stop a + # second one: autosend rides the model's fill turns, so unlike the + # send-script path (which runs once and can safely leave this False) a + # re-fill would fire another send. It is a resend guard, not evidence. + send_confirmed = True + action_log.extend(p_cs.get("log") or []) + if p_cs.get("sent"): + done_called = True + p_aux_c, p_aux_m = await p_get_aux_client() + if p_cs.get("delivered") is False: + # ghost-drop host: cleared but the post did NOT persist. Truthful hedge, no recipe. + done_success = False + done_message = (await compose_delivery_warning( + p_aux_c, p_aux_m, task, composer_committed_payload, current_url) + or browser_delivery_check.unconfirmed_delivery_note(current_url, composer_committed_payload)) + logger.info(f"[browser-autosend {session_id}] post-fill code-send NOT confirmed (ghost-drop host; composer cleared, post did not persist)") + else: + done_success = True + delivery_verified = True + await p_learn_write_recipe(execute_browser_tool, browser_id, tab_id, current_url, composer_committed_payload) + done_message = (await compose_send_confirmation( + p_aux_c, p_aux_m, task, composer_committed_payload) + or f'Done, I sent "{composer_committed_payload}" for you.') + logger.info(f"[browser-autosend {session_id}] post-fill code-send delivered (receipt verified)") + else: + task = f"{task}\n\n[{p_cs.get('note')}]" + logger.info(f"[browser-autosend {session_id}] post-fill send click ran, receipt unverified; model verifies") # One gentle nudge per violating turn, folded onto the action that ran, so the model self-corrects next turn without us costing it one. if rp_reminder_pending and tu.name in ACTION_TOOLS_REQUIRING_REPORT: @@ -1840,7 +2750,7 @@ async def run_browser_agent( ) # Deferred replay re-check: the orchestrator often opens a fresh card on the wrong host, so the dispatch-time replay missed. Once a navigation lands us on a host that DOES have a matching skill, and nothing has dirtied the page yet, switch to replay (still verified per-step, still trust-gated). Fires at most once. - if (not replay_rechecked and tu.name == "BrowserNavigate" + if (not replay_rechecked and not p_task_is_removal and tu.name == "BrowserNavigate" and replay_recheck_is_safe(action_log)): cur_host = browser_skills.host_of(last_seen_url) if cur_host and cur_host != replay_host: @@ -2139,8 +3049,14 @@ async def run_browser_agent( playbook_seeded=pb_seeded) # Learn this task ONLY from a genuinely successful run whose deliverable a deterministic replay can actually reproduce. We skip recording when the run was dishonest (ghost) OR when its answer was gathered/judged content (a list/report): replay can redo the clicks but not regenerate the judgment, so recording it would create a thin shortcut that later ghosts. informational = deliverable_is_informational(summary, skill_key_task) - logger.info(f"[browser-skills] record gate: honest={honest} informational={informational}") - if honest and not informational: + # A removal run is NOT a recordable skill: the actual delete is a one-shot destructive + # dispatch (BrowserEvaluate), not in the replayable action_log, so what gets distilled is + # the meaningless scrolling around it. Replaying that bogus "skill" later reports done in 0 + # turns while deleting nothing (measured live: repeat deletes all ghost-succeeded). Deletes + # always run fresh through the delete-dispatch. + p_is_removal = is_removal_task(skill_key_task) + logger.info(f"[browser-skills] record gate: honest={honest} informational={informational} removal={p_is_removal}") + if honest and not informational and not p_is_removal: try: rec_host = browser_skills.host_of(last_seen_url) p_distilled = browser_skills.distill_steps(action_log) @@ -2161,10 +3077,12 @@ async def run_browser_agent( # Tier-2 memory: on a substantive verified success, distill this run into the DURABLE strategy playbook (one cheap aux call, mem0-style distill+ reconcile). Fires for BOTH mechanical and judgment tasks, it's how the judgment ones (which can't be skills) still get faster/wiser next time. if browser_playbook.should_learn(honest, turn + 1): - try: - # App mode keys by the stable app id (the run had no URL host); web keys by the final host, which navigation may have changed. - rec_pb_host = browser_id if app_mode else browser_skills.host_of(last_seen_url) - if rec_pb_host: + async def p_distill_learning() -> None: + try: + # App mode keys by the stable app id (the run had no URL host); web keys by the final host, which navigation may have changed. + rec_pb_host = browser_id if app_mode else browser_skills.host_of(last_seen_url) + if not rec_pb_host: + return aux_client, aux_model = await p_get_aux_client() changed = await browser_playbook.distill_and_store( rec_pb_host, skill_key_task, latest_working_mem, summary, @@ -2180,8 +3098,12 @@ async def run_browser_agent( await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": p_learn_msg.model_dump(mode="json"), }) - except Exception as e: - logger.debug(f"[browser-playbook] distill skipped: {e}") + except Exception as e: + logger.debug(f"[browser-playbook] distill skipped: {e}") + # Learning is advisory to FUTURE runs, so the user's reply must not wait on this aux call; it used to sit between "done" and the reply. + p_lt = asyncio.create_task(p_distill_learning()) + learn_tasks.add(p_lt) + p_lt.add_done_callback(learn_tasks.discard) # The model asked to leave the browser open because the deliverable lives on the page (a video playing, a page to read). Pin the card so the auto-close on parent finish skips it. Only on honest success: never pin a broken or ghost run open. The keep broadcast lands before the parent reaches terminal state (it awaits this run), so the frontend has the flag set before any close path runs. if honest and done_keep_open and dashboard_id: try: @@ -2372,6 +3294,26 @@ async def p_create_browser_card(dashboard_id: str, url: str, parent_session_id: return browser_id +async def p_rebroadcast_card(dashboard_id: str | None, browser_id: str) -> None: + """Re-send a card's card_added broadcast from the persisted layout. The spawn broadcast is + fire-and-forget with no ack, so a renderer that misses it (rAF stall, WS blip) leaves the card + unmounted forever = the dead-run wedge. Idempotent: the renderer no-ops on a card it already has.""" + if not dashboard_id: + return + try: + from backend.apps.dashboards.dashboards import load + card = load(dashboard_id).layout.browser_cards.get(browser_id) + if card is None: + return + await ws_manager.broadcast_global("dashboard:browser_card_added", { + "dashboard_id": dashboard_id, + "browser_card": card.model_dump(mode="json"), + "parent_session_id": getattr(card, "spawned_by", "") or "", + }) + except Exception as e: + logger.info(f"[browser-spawn-ack] rebroadcast failed for {browser_id}: {e}") + + async def run_browser_agents( tasks: list[dict], model: str, @@ -2418,6 +3360,11 @@ async def run_browser_agents( if not browser_id and dashboard_id: # the url param is often empty with the target buried in the task text; a url there still names the host we must not duplicate host_src = url or entry_url or next(iter(re.findall(r"https?://[^\s)\"'<>]+", task_text)), "") + # Borrow the user's sign-in BEFORE any card exists. A new card loads its entry URL the + # instant it mounts, with no BrowserNavigate to hook, so this is the only moment the + # very first request can already carry the session. Outside the pick lock: it can touch + # the keychain, and holding a lock across that would serialize every card creation. + await borrow_signin_before_nav(host_src, "") async with p_card_pick_lock: browser_id = find_reusable_card(dashboard_id, host_src, parent_session_id) if browser_id: @@ -2435,6 +3382,35 @@ async def run_browser_agents( await execute_browser_tool("BrowserNavigate", {"url": url}, browser_id) except Exception: pass + elif os.environ.get("OSW_PRELUDE_TRIM", "1") != "0": + # Poll until the mounting card serves real page text instead of a blind 2s; capped, so the worst case is the old wait plus one probe. + p_mount_t0 = time.monotonic() + p_mounted = False + while time.monotonic() - p_mount_t0 < 2.5: + try: + p_probe = await execute_browser_tool("BrowserGetText", {}, browser_id) + if (isinstance(p_probe, dict) and not p_probe.get("error") + and len(str(p_probe.get("text") or "")) > 200): + p_mounted = True + break + except Exception: + pass + await asyncio.sleep(0.25) + if not p_mounted: + # Spawn ack: no response yet could be a slow page OR a renderer that missed the card_added broadcast entirely (no ack exists). Re-broadcast (idempotent) and give it one more bounded window; a card that's still dead after this hits the prestage dead-card catch instead of a wasted run. + await p_rebroadcast_card(dashboard_id, browser_id) + p_ack_t0 = time.monotonic() + while time.monotonic() - p_ack_t0 < 4.0: + try: + p_probe = await execute_browser_tool("BrowserGetText", {}, browser_id) + if isinstance(p_probe, dict) and not p_probe.get("error"): + p_mounted = True + break + except Exception: + pass + await asyncio.sleep(0.4) + logger.info(f"[browser-spawn-ack] {browser_id} rebroadcast after silent mount; alive={p_mounted}") + logger.info(f"[browser-cold] mount poll {int((time.monotonic() - p_mount_t0) * 1000)}ms for {browser_id}") else: await asyncio.sleep(2.0) elif browser_id and not app_mode: @@ -2442,6 +3418,10 @@ async def run_browser_agents( is_pre_selected = browser_id in pre_selected p_nav_url = url or ("" if reused else entry_url) + # The fresh card already opened AT entry_url, so the pre-loop nav is a full second page load of the same page; trim mode drops it (perceive reads the mounting page, and the loop can still navigate itself if that read comes up empty). + if (os.environ.get("OSW_PRELUDE_TRIM", "1") != "0" and not reused and not url + and entry_url and p_nav_url == entry_url): + p_nav_url = "" try: return await run_browser_agent( task=task_text, @@ -2453,6 +3433,7 @@ async def run_browser_agents( initial_url=None if app_mode else (p_nav_url if p_nav_url and (url or browser_id not in pre_selected) else None), parent_session_id=parent_session_id, app_mode=app_mode, + user_prompt=str(task_def.get("user_prompt") or ""), ) finally: if not app_mode: diff --git a/backend/apps/agents/browser/browser_batch_replay.py b/backend/apps/agents/browser/browser_batch_replay.py index e9f74abf..28b3e62d 100644 --- a/backend/apps/agents/browser/browser_batch_replay.py +++ b/backend/apps/agents/browser/browser_batch_replay.py @@ -111,17 +111,24 @@ P_LIVE_IRREVERSIBLE_RE = re.compile( r"confirm|apply|accept|decline|delete|remove|unsend|withdraw|endorse)\b", re.I, ) +# Composer OPENERS phrased with a send-word: LinkedIn's profile button is literally +# named "Send a message to ", which opens the compose box (reversible), not +# a real Send. A true Send control is short and exact ("Send", "Send now"); these +# describe opening a conversation, so they must NOT trip the irreversible boundary. +P_SEND_OPENER_RE = re.compile(r"send (a |an |the )?(message|note|inmail|dm) to\b", re.I) def is_replay_boundary(step: dict) -> bool: """The genuinely irreversible step where a learned skill's mechanical replay must STOP and hand to the live agent. Same as is_send_step EXCEPT a composer - OPENER ('Message'/'DM' click) is reversible and NOT a boundary: the prefix can - mechanically open the composer, and only the real Send (and composer typing) - crosses to the live model. Uses the same opener-excluded wordlist the live - send-guard already trusts, so a recorded Send still stops the prefix.""" + OPENER ('Message'/'DM' click, incl. 'Send a message to X') is reversible and NOT + a boundary: the prefix can mechanically open the composer, and only the real Send + (and composer typing) crosses to the live model.""" action = step.get("action") - if action == "click" and P_LIVE_IRREVERSIBLE_RE.search(str(step.get("name") or "")): + name = str(step.get("name") or "") + if action == "click" and P_SEND_OPENER_RE.search(name): + return False # opener phrasing, not a real send + if action == "click" and P_LIVE_IRREVERSIBLE_RE.search(name): return True if action == "type" and P_COMPOSE_SEL_RE.search(str(step.get("selector") or "")): return True diff --git a/backend/apps/agents/browser/browser_delete_script.py b/backend/apps/agents/browser/browser_delete_script.py new file mode 100644 index 00000000..4de4647b --- /dev/null +++ b/backend/apps/agents/browser/browser_delete_script.py @@ -0,0 +1,260 @@ +"""BrowserDeleteItem: a model-invoked tool that removes ONE on-page item the model names by +text, deterministically. The model handles getting to the item's page (its strength); this runs +the site's own remove flow (open that item's overflow menu -> Delete -> confirm -> verify-gone), +which the model fails at by hand (measured live on X: 4 aborts on the tiny caret menu). + +Resolve-in-JS, click-with-real-input: each step RESOLVES the next control's viewport position in +a pierced-shadow DOM query, and the click is dispatched through the OS-level input path +(BrowserClickPoint). Synthetic el.click() is ignored by web-component sites (shreddit live: +the flow reached Delete yet nothing happened; the model's trusted clicks on the same controls +worked), and a real click also lands on whatever is topmost, so overlays can't be mis-clicked. + +Safety, in code: +- Resolution happens ONLY inside the element that contains the target text, so it can never + touch another item. +- The site enforces ownership (only your own item exposes Delete), so a target you don't own has + no menu entry and the tool reports that, it never forces one. +- Success REQUIRES verify-gone (the target text left the page). One destructive confirm click. +- Flag-gated (OSW_DELETE_SCRIPT): the tool is hidden from the model until Eric flips it. +""" + +import asyncio +import json +import logging +import os +from typing import Any, Awaitable, Callable, Dict + +from backend.apps.agents.browser import browser_submit_click + +logger = logging.getLogger(__name__) + +MIN_TARGET_CHARS = 6 + +ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]] + + +def delete_tool_enabled() -> bool: + return os.environ.get("OSW_DELETE_SCRIPT", "0") != "0" + + +# One resolver, four steps. Each call re-queries the live DOM (pierced), scrolls the control into +# view when needed, and returns the control's viewport center as percents for BrowserClickPoint. +# 'verify' returns gone-ness instead of a position. Controls render a beat late, so steps poll. +P_RESOLVE_JS = r"""(async () => { + const STEP = %s; + const TARGET = %s; + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); + const norm = (s) => (s || '').replace(/\s+/g, ' ').trim(); + // Web-component sites (shreddit, YouTube) hide the item AND its menus in shadow roots, so every lookup pierces. + const deep = (root, sel, out, depth) => { + if (depth > 10 || out.length > 4000) return out; + let hits; try { hits = root.querySelectorAll(sel); } catch (e) { hits = []; } + for (const el of hits) out.push(el); + let all; try { all = root.querySelectorAll('*'); } catch (e) { return out; } + for (const el of all) { if (el.shadowRoot) deep(el.shadowRoot, sel, out, depth + 1); } + return out; + }; + const vis = (el) => !!el && el.getClientRects().length > 0; + const center = (el) => { + const r0 = el.getBoundingClientRect(); + if (r0.top < 0 || r0.bottom > window.innerHeight) el.scrollIntoView({ block: 'center' }); + const r = el.getBoundingClientRect(); + return { xPct: (r.left + r.width / 2) / window.innerWidth * 100, + yPct: (r.top + r.height / 2) / window.innerHeight * 100, + label: norm(el.getAttribute('aria-label') || el.textContent || '').slice(0, 40) }; + }; + const CONTAINERS = 'article,[role="article"],li,[role="listitem"],tr,[role="row"],shreddit-post,' + + '[data-testid*="tweet"],[data-testid*="post"],[data-testid*="comment"],[data-testid*="Post"],[id^="t3_"],[id^="t1_"]'; + const MORE = 'button[aria-label*="More" i],button[aria-label*="option" i],' + + '[data-testid="caret"],button[aria-haspopup="menu"],button[aria-haspopup="true"],[aria-label*="menu" i]'; + const holders = () => deep(document, CONTAINERS, [], 0) + .filter((el) => el.textContent && el.textContent.includes(TARGET)) + .sort((a, b) => (a.textContent || '').length - (b.textContent || '').length); // tightest match first + + if (STEP === 'verify') { + // Gone = the text left the page, OR every remaining holder is a deletion TOMBSTONE (reddit + // swaps the tile for a "Post deleted" placeholder that keeps the title text). The substance + // gate stops a still-loading page (post-refresh) from reading as gone: never a false removed. + const TOMB = /post deleted|comment deleted|\[deleted\]|deleted by|removed by/i; + for (let attempt = 0; attempt < 7; attempt++) { + if (attempt) await sleep(1000); + if ((document.body.innerText || '').length < 500) continue; + const hs = holders(); + if (!hs.length || hs.every((h) => TOMB.test(h.textContent || ''))) return { ok: true, stage: 'verify' }; + } + return { ok: false, stage: 'verify' }; + } + if (STEP === 'direct') { + // Row-action sites (Gmail) put a literal Delete control ON the item, no kebab menu; click it + // straight and let verify arbitrate. Exact cleaned-label match so 'Delete row' etc never fires. + const clean = (s) => norm((s || '').replace(/[‪‬‎‏⁦-⁩]|\([^)]*\)/g, '')).toLowerCase(); + const DIRECT = new Set(['delete', 'move to trash', 'trash']); + if (!holders().length) return { ok: false, stage: 'find', msg: 'target text not on this page' }; + for (const h of holders()) { + const cands = deep(h, 'button,[role="button"],li,span[role]', [], 0) + .filter((b) => DIRECT.has(clean(b.getAttribute('aria-label') || b.textContent || ''))); + const v = cands.find(vis); + if (v) { h.scrollIntoView({ block: 'center' }); return { ok: true, stage: 'direct', ...center(v) }; } + if (cands.length) { + h.scrollIntoView({ block: 'center' }); + const hr = h.getBoundingClientRect(); + return { ok: false, stage: 'direct', hoverFirst: true, + xPct: (hr.left + hr.width / 2) / window.innerWidth * 100, + yPct: (hr.top + Math.min(20, hr.height / 2)) / window.innerHeight * 100, + msg: 'direct delete control hidden until hover' }; + } + } + return { ok: false, stage: 'direct', optional: true, msg: 'no direct delete control on the item' }; + } + if (STEP === 'more') { + if (!holders().length) return { ok: false, stage: 'find', msg: 'target text not on this page' }; + // A post tile carries OTHER kebabs too (reddit's user-attribution row: 'Open user actions', + // measured live opening the wrong menu). Rank: named overflow first, then reddit's unlabeled + // haspopup kebab, then generic; user/share/moderation controls never. + const rank = (el) => { + // Reddit quirks, all measured live: the post kebab is LABELED 'Open user actions' (inside + // shreddit-post-overflow-menu, so the host outranks the misleading label), a 0x0 DECOY + // lives in mod-content-state-indicators, and Share/mod controls also carry haspopup. + const host = (el.getRootNode() && el.getRootNode().host) ? el.getRootNode().host.tagName : ''; + if (/^MOD-/.test(host)) return 4; + const l = norm(el.getAttribute('aria-label') || el.textContent || '').toLowerCase(); + if (/overflow|more/.test(l) || /OVERFLOW/.test(host)) return 0; + if (!l && el.getAttribute('aria-haspopup')) return 1; + if (/share|award|vote|join|follow|moderat|approve/.test(l)) return 4; + if (/user|profile|author/.test(l)) return 3; + return 2; + }; + for (let attempt = 0; attempt < 6; attempt++) { + if (attempt) await sleep(500); + for (const h of holders()) { + const cands = deep(h, MORE, [], 0).filter((c) => rank(c) < 4); + const visBest = cands.filter(vis).sort((a, b) => rank(a) - rank(b))[0]; + if (visBest) { h.scrollIntoView({ block: 'center' }); return { ok: true, stage: 'more', ...center(visBest) }; } + // Hover-revealed kebab: exists but zero rects until the tile is hovered with REAL input. + // Only when NOTHING visible qualifies; hand the tile position back for a hover + retry. + const hid = cands.find((c) => !vis(c) && rank(c) <= 1); + if (hid) { + h.scrollIntoView({ block: 'center' }); + const hr = h.getBoundingClientRect(); + return { ok: false, stage: 'more', hoverFirst: true, + xPct: (hr.left + hr.width / 2) / window.innerWidth * 100, + yPct: (hr.top + Math.min(40, hr.height / 2)) / window.innerHeight * 100, + msg: 'overflow control hidden until hover' }; + } + } + } + return { ok: false, stage: 'more', msg: 'found the item but no overflow/More control on it' }; + } + if (STEP === 'menuitem') { + const DEL = /^\s*(delete|remove)\b/i; // exact-ish: not "delete row" or "remove filter" + const MENUS = '[role="menu"],[role="listbox"],faceplate-menu,faceplate-dropdown-menu,[data-testid="Dropdown"]'; + // Long poll: shreddit's menu content arrives through an async loader, measured up to ~20s + // after the kebab click on a cold page; X resolves on the first attempt so the tail is free. + for (let attempt = 0; attempt < 12; attempt++) { + if (attempt) await sleep(900); + // Only entries inside an OPEN menu count; a page-wide 'Delete' from another context must + // never be clicked (measured live after the wrong kebab opened). + const menus = deep(document, MENUS, [], 0).filter(vis); + for (const s of (menus.length ? menus : [document])) { + const del = deep(s, '[role="menuitem"],[role="option"],button,a,li', [], 0) + .find((m) => DEL.test(norm(m.textContent)) && vis(m)); + if (del) return { ok: true, stage: 'menuitem', ...center(del) }; + } + } + return { ok: false, stage: 'menuitem', + msg: 'no Delete/Remove entry in the menu (is this your own item?)', + menu: deep(document, '[role="menuitem"],[role="option"]', [], 0) + .map((m) => norm(m.textContent)).filter(Boolean).slice(0, 10) }; + } + if (STEP === 'confirm') { + const CONF = /^\s*(delete|remove|yes|confirm)\b/i; + // Scan EVERY visible dialog candidate then the whole document; taking [0] once grabbed a + // random lazy-loader wrapper and looked straight past the real open dialog (measured live). + const DLG = '[role="dialog"],[role="alertdialog"],[data-testid="confirmationSheetDialog"],faceplate-dialog'; + for (let attempt = 0; attempt < 7; attempt++) { + if (attempt) await sleep(700); + const scopes = [...deep(document, DLG, [], 0).filter(vis), document]; + for (const dlg of scopes) { + const conf = deep(dlg, '[data-testid="confirmationSheetConfirm"]', [], 0).find(vis) + || deep(dlg, 'button', [], 0).find((b) => CONF.test(norm(b.textContent)) && vis(b)); + if (conf) return { ok: true, stage: 'confirm', fromDialog: dlg !== document, ...center(conf) }; + } + } + return { ok: false, stage: 'confirm', optional: true, msg: 'no confirm dialog appeared' }; + } + return { ok: false, stage: 'eval', msg: 'unknown step' }; +})()""" + + +def resolve_expression(step: str, target_text: str) -> str: + return P_RESOLVE_JS % (json.dumps(step), json.dumps(target_text)) + + +async def run_delete(target_text: str, browser_id: str, tab_id: str, + execute_tool: ToolRunner) -> Dict[str, Any]: + """The full remove flow: resolve each control in-page, click it with REAL input, verify gone. + Any unreadable resolve is an honest failure at that stage (never a false 'removed').""" + + async def resolve(step: str) -> Dict[str, Any]: + res = await execute_tool( + "BrowserEvaluate", {"expression": resolve_expression(step, target_text)}, browser_id, tab_id) + return browser_submit_click.parse_eval_value(res) or {"ok": False, "stage": "eval", + "msg": "the remove flow returned no readable result"} + + async def hover_then_retry(step: str, r: Dict[str, Any]) -> Dict[str, Any]: + # Reveal a hover-only control with a real mouse move over the tile, then re-resolve once. + logger.info(f"[browser-deletescript] {step} control hidden; hovering the tile to reveal it") + await execute_tool("BrowserClickPoint", + {"xPercent": float(r["xPct"]), "yPercent": float(r["yPct"]), + "hoverOnly": True}, browser_id, tab_id) + await asyncio.sleep(0.7) + return await resolve(step) + + async def click_step(step: str, r: Dict[str, Any], settle_s: float) -> None: + logger.info(f"[browser-deletescript] step={step} label={str(r.get('label') or '')[:40]!r} " + f"at {float(r['xPct']):.1f},{float(r['yPct']):.1f}") + await execute_tool("BrowserClickPoint", + {"xPercent": float(r["xPct"]), "yPercent": float(r["yPct"])}, browser_id, tab_id) + await asyncio.sleep(settle_s) + + # Row-action shortcut first (Gmail): a literal Delete control on the item skips the menu walk; + # a confirm may still follow (Drive-style), so that step runs either way and stays optional. + p_direct = await resolve("direct") + if not p_direct.get("ok") and p_direct.get("hoverFirst"): + p_direct = await hover_then_retry("direct", p_direct) + p_via_direct = False + if p_direct.get("ok"): + await click_step("direct", p_direct, 1.5) + p_via_direct = True + steps = (("confirm", 1.8),) + elif str(p_direct.get("stage")) == "find": + return {"removed": False, "stage": "find", "msg": str(p_direct.get("msg") or "")} + else: + steps = (("more", 1.0), ("menuitem", 1.0), ("confirm", 1.8)) + for step, settle_s in steps: + r = await resolve(step) + if not r.get("ok") and r.get("hoverFirst"): + r = await hover_then_retry(step, r) + if not r.get("ok"): + if step == "confirm" and r.get("optional"): + break # some sites delete without a confirm; the verify below is the arbiter + return {"removed": False, "stage": str(r.get("stage") or step), "msg": str(r.get("msg") or "")} + if step == "confirm" and p_via_direct and not r.get("fromDialog"): + break # after a direct delete, only a REAL dialog earns a confirm click + await click_step(step, r, settle_s) + v = await resolve("verify") + if not v.get("ok"): + # Some clients keep the dead tile mounted until a reload (shreddit, measured live: the + # delete lands server-side while the DOM never flips). Refresh once and re-verify; the + # verify's substance gate keeps a half-loaded page from reading as gone. + p_loc = browser_submit_click.parse_eval_value( + await execute_tool("BrowserEvaluate", {"expression": "({href: location.href})"}, browser_id, tab_id)) or {} + p_href = str(p_loc.get("href") or "") + if p_href.startswith("http"): + logger.info("[browser-deletescript] tile still mounted; refreshing to re-verify") + await execute_tool("BrowserNavigate", {"url": p_href}, browser_id, tab_id) + await asyncio.sleep(3.0) + v = await resolve("verify") + removed = bool(v.get("ok")) + return {"removed": removed, "stage": "done", + "msg": "item removed" if removed else "clicked delete but the item is still on the page"} diff --git a/backend/apps/agents/browser/browser_delivery_check.py b/backend/apps/agents/browser/browser_delivery_check.py new file mode 100644 index 00000000..963fce0c --- /dev/null +++ b/backend/apps/agents/browser/browser_delivery_check.py @@ -0,0 +1,180 @@ +"""Delivery ground-truth for a write: did the post ACTUALLY land, or did the site clear the +composer and silently eat it? + +A cleared composer proves delivery everywhere EXCEPT the ghost-drop hosts (YouTube-class), which +accept an automated post, clear the box, maybe render it for a beat, then drop it server-side. On +those we re-read the live page to confirm the post PERSISTS before anyone claims success; +everywhere else the cleared composer stays the trusted proxy (proven across X/Reddit/LinkedIn/ +Gmail) and this module is never consulted, so proven sends keep their exact speed. +""" + +import asyncio +import json +import re +from typing import Awaitable, Callable, Optional +from urllib.parse import urlparse + +from typeguard import typechecked + +from backend.apps.agents.browser import browser_submit_click + +ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]] + +# Hosts known to accept-then-silently-drop an automated post. A newly-found one is a one-line add. +GHOST_DROP_HOSTS = ("youtube.com",) + +# What a site says when it REFUSED the write. Only ever consulted inside a live announcement +# region, never against the whole page, so an unrelated "failed" in an article body can't match. +P_REJECTION_RE = re.compile( + r"something went wrong|went wrong|couldn'?t\s|could not\s|unable to|failed to|" + r"\bfailed\b|try again|too many|rate.?limit|limit exceeded|not allowed|" + r"blocked|error occurred|wasn'?t (?:sent|posted)|was not (?:sent|posted)", + re.I, +) + + +@typechecked +def is_ghost_drop_host(url: str) -> bool: + host = (urlparse(url or "").hostname or "").lower().lstrip(".") + return any(host == g or host.endswith("." + g) for g in GHOST_DROP_HOSTS) + + +@typechecked +def delivery_probe_expression(payload: str) -> str: + """JS reporting whether a distinctive chunk of `payload` is rendered in the page's VISIBLE + text. Run only AFTER the composer cleared, so a hit means the text lives in real page content + (the posted item / a confirmation), not the emptied composer.""" + needle = " ".join((payload or "").split())[:80] + return ("(()=>{try{var n=" + json.dumps(needle) + ";" + "var t=(document.body&&document.body.innerText)||'';" + "return {visible: n.length>0 && t.indexOf(n)!==-1};}" + "catch(e){return {visible:false};}})()") + + +@typechecked +async def payload_visible( + payload: str, browser_id: str, tab_id: str, execute_tool: ToolRunner +) -> Optional[bool]: + """True = seen on the page, False = looked and it is NOT there, None = could not look. + + The third case is not pedantry. Returning False for a probe that timed out or came back + unreadable is asserting absence from a failed observation, and that is the same mistake as a + receipt claiming delivery it never saw, pointed the other way: it tells the user a post did not + land when nobody actually checked. Measured tonight, the identical shape in the test harness + scored every unreadable verification as a successful delete and left six posts on a real + account while reporting them cleaned. + """ + try: + r = await asyncio.wait_for(execute_tool( + "BrowserEvaluate", {"expression": delivery_probe_expression(payload)}, + browser_id, tab_id), timeout=6.0) + except Exception: + return None + v = browser_submit_click.parse_eval_value(r) + if not isinstance(v, dict) or "visible" not in v: + return None + return bool(v.get("visible")) + + +@typechecked +def rejection_probe_expression() -> str: + """JS returning the text of the page's live ANNOUNCEMENT regions only. + + role="alert" and aria-live are how sites are required to announce a transient result to + assistive tech, so error toasts land here on every major site without us naming any of them. + Scoped deliberately: reading whole-page text for the word "failed" would match articles, + changelogs and half the internet.""" + return ("(()=>{try{var out=[];" + "var sel='[role=alert],[role=alertdialog],[aria-live=assertive],[aria-live=polite]';" + "document.querySelectorAll(sel).forEach(function(e){" + "var s=(e.innerText||'').trim(); if(s) out.push(s);});" + "return {text: out.join(' | ').slice(0,600)};}" + "catch(e){return {text:''};}})()") + + +@typechecked +async def send_rejected(browser_id: str, tab_id: str, execute_tool: ToolRunner) -> bool: + """Did the site announce that the write FAILED, right after the composer cleared? + + A cleared composer is the receipt this whole fast path rests on, and the code has long admitted + it "cannot tell submitted from dismissed". The realistic way that bites is not a mis-click: it + is the site accepting the click, clearing the box, and popping "Something went wrong" or a rate + limit. The receipt then reads as success and the agent tells the user it posted. + + This only ever DEMOTES a claim, and only on an explicit failure announcement, so a normal send + (no alert region, or a success toast) is untouched and keeps its measured speed. Any read + failure returns False, because refusing to claim delivery on the basis of a broken probe would + invent failures that did not happen.""" + try: + r = await asyncio.wait_for(execute_tool( + "BrowserEvaluate", {"expression": rejection_probe_expression()}, + browser_id, tab_id), timeout=4.0) + except Exception: + return False + v = browser_submit_click.parse_eval_value(r) + if not isinstance(v, dict): + return False + return bool(P_REJECTION_RE.search(str(v.get("text") or ""))) + + +@typechecked +def rejected_send_note(url: str, payload: str) -> str: + """Honest line for a send the SITE said no to. Distinct from the unverified case: here we are + not guessing, the page told us, so the user should be told plainly rather than asked to check.""" + host = urlparse(url or "").hostname or "the site" + if host.startswith("www."): + host = host[4:] + clip = payload if len(payload) <= 80 else payload[:77] + "..." + return (f'I typed "{clip}" and clicked send, but {host} rejected it: the composer cleared and ' + f'the page showed an error instead of posting. It did NOT go through. I did not retry, ' + f'since whatever the site refused is likely to be refused again.') + + +@typechecked +async def ghost_delivery_confirmed( + payload: str, browser_id: str, tab_id: str, execute_tool: ToolRunner +) -> bool: + """For a ghost-drop host: did the post render AND survive the server-side drop window? True + only if the payload is visible now and STILL visible a few seconds later. A post that never + rendered, or rendered then vanished, returns False, so we never claim a delivery the site ate. + Pure page reads (no navigation), invisible to the site.""" + # `is not True` deliberately: an unknown must NOT confirm. This is the one place where + # collapsing unknown into "no" is right, because the caller is deciding whether to CLAIM a + # delivery, and withholding an uncertain claim is the safe direction. + if await payload_visible(payload, browser_id, tab_id, execute_tool) is not True: + return False + await asyncio.sleep(3.5) + return await payload_visible(payload, browser_id, tab_id, execute_tool) is True + + +@typechecked +def unverified_send_note(url: str, payload: str) -> str: + """Honest line for a send whose click RAN but whose two-sided receipt never arrived. + + Deliberately weaker than unconfirmed_delivery_note: there the composer cleared and the post + later vanished, so we know it was submitted. Here we never got the clear at all, so we know + strictly less and must claim strictly less. Measured 2026-07-28 on X: the agent reported "your + message went through and it's showing in the conversation now" on exactly this evidence and + nothing had been posted. Overclaiming here is the worst failure this agent has, because the + user stops checking. + """ + host = urlparse(url or "").hostname or "the site" + if host.startswith("www."): + host = host[4:] + clip = payload if len(payload) <= 80 else payload[:77] + "..." + return (f'I typed "{clip}" and clicked send on {host}, but I could NOT confirm it actually ' + f'posted: the composer never cleared, which is the signal I rely on. It may or may not ' + f'have gone through, so please check before relying on it. I did not try again, because ' + f'a blind retry is how you end up posting twice.') + + +def unconfirmed_delivery_note(url: str, payload: str) -> str: + """Plain honest fallback line when a ghost-drop send can't be confirmed (the aux-composed + version in browser_agent is preferred; this is the never-fails template behind it).""" + host = urlparse(url or "").hostname or "the site" + if host.startswith("www."): + host = host[4:] + clip = payload if len(payload) <= 80 else payload[:77] + "..." + return (f'I submitted "{clip}" and the composer cleared, but I could NOT confirm it stayed ' + f'live: {host} sometimes accepts an automated post and then drops it without an error. ' + f'Please check your posts to verify it actually went through before relying on it.') diff --git a/backend/apps/agents/browser/browser_fast_path.py b/backend/apps/agents/browser/browser_fast_path.py index eae29c0d..6398a93a 100644 --- a/backend/apps/agents/browser/browser_fast_path.py +++ b/backend/apps/agents/browser/browser_fast_path.py @@ -65,7 +65,9 @@ P_CLASSIFIER_SYSTEM = ( "If line 1 is READ or ACT, follow it with a short browsing brief:\n" "ENTRY: the best starting URL; use a direct deep/search URL when the site's " "pattern is well known (LinkedIn people search is " - "https://www.linkedin.com/search/results/people/?keywords=NAME).\n" + "https://www.linkedin.com/search/results/people/?keywords=NAME). Always a normal " + "page a person would see, never a raw JSON/API endpoint (e.g. Instagram's " + "web/search/topsearch), which renders as an unreadable data wall.\n" "Then 3-6 numbered steps, one short action each.\n" "Copy any text the user wants typed, sent, or posted EXACTLY, character for " "character. Never invent names, values, or wording the user did not give." @@ -115,6 +117,11 @@ def entry_url_from_brief(brief: str) -> str: return m.group(1).rstrip(".,;)") if m else "" +# Opens the advisory-brief section of a composed task; consumers strip everything after it when a +# check must apply only to the human's words (the brief once false-flagged a real send read-only). +BRIEF_MARKER = "[routing brief" + + def compose_task(prompt: str, brief: str) -> str: """User's words first and authoritative; the brief is advisory routing. Skill replay keys on the parent's user message, so brief variance is safe.""" @@ -122,7 +129,7 @@ def compose_task(prompt: str, brief: str) -> str: return prompt return ( f"{prompt}\n\n" - "[routing brief from a fast pre-pass; follow it unless the live page disagrees]\n" + f"{BRIEF_MARKER} from a fast pre-pass; follow it unless the live page disagrees]\n" f"{brief}" ) @@ -219,6 +226,22 @@ def normalize_for_classifier(prompt: str) -> str: return re.sub(r"\btext(ing|ed|s)?\b", "message", prompt, flags=re.I) +def seed_hints_for_task(prompt: str) -> str: + """Documented facts for sites the task names, fed to the classifier so its ENTRY + uses the site's real search-URL pattern instead of the homepage (measured: the aux + sent walmart to the homepage while the seed had the exact /search?q= pattern).""" + from backend.apps.agents.browser.seed_playbooks import SEED_PLAYBOOKS + low = f" {prompt.lower()} " + lines: list[str] = [] + for domain, facts in SEED_PLAYBOOKS.items(): + name = domain.split(".")[0] + if len(name) >= 4 and f" {name}" in low and facts: + lines.append(f"{domain}: {facts[0][:180]}") + if len(lines) >= 2: + break + return ("\n\nKnown site facts (use their URL patterns for ENTRY):\n" + "\n".join(lines)) if lines else "" + + async def classify_and_brief(prompt: str, settings, primary_api: str | None) -> tuple[str, str]: """One cheap aux call returns a READ/ACT/NO verdict plus a routing brief (entry URL + step outline), timeboxed; any failure means NO (normal path).""" @@ -227,22 +250,35 @@ async def classify_and_brief(prompt: str, settings, primary_api: str | None) -> from backend.apps.settings.credentials import get_anthropic_client_for_model from backend.apps.agents.providers.registry import resolve_aux_model - aux_model, _ = await resolve_aux_model( - settings, preferred_tier="haiku", primary_api=primary_api, - ) - client = get_anthropic_client_for_model(settings, aux_model) - resp = await asyncio.wait_for( - client.messages.create( - model=aux_model, - max_tokens=250, - temperature=0, - system=P_CLASSIFIER_SYSTEM, - messages=[{"role": "user", "content": normalize_for_classifier(prompt[:2000])}], - ), - timeout=8.0, - ) from backend.apps.agents.core.aux_llm import safe_resp_text - verdict, brief = parse_verdict_and_brief(safe_resp_text(resp)) + + async def p_ask(api: str | None) -> tuple[str, str, str]: + aux_model, _ = await resolve_aux_model( + settings, preferred_tier="haiku", primary_api=api, + ) + client = get_anthropic_client_for_model(settings, aux_model) + resp = await asyncio.wait_for( + client.messages.create( + model=aux_model, + max_tokens=250, + temperature=0, + system=P_CLASSIFIER_SYSTEM, + messages=[{"role": "user", "content": ( + normalize_for_classifier(prompt[:2000]) + seed_hints_for_task(prompt))}], + ), + timeout=8.0, + ) + return safe_resp_text(resp), aux_model, "" + + text, aux_model, _ = await p_ask(primary_api) + # An EMPTY body is a broken lane, not a verdict. Measured live: cx/gpt-5.4-mini returns '' + # for this call, which parsed to "no" and silently switched the whole browser fast path off + # for every GPT user, with no error to show for it. Fall back once to the provider-agnostic + # cheap tier (same cure as the distill fix) so a mute aux can't disable a working feature. + if not text.strip() and primary_api: + logger.info(f"[browser-fast-path] classifier empty on {aux_model}; retrying provider-agnostic") + text, aux_model, _ = await p_ask(None) + verdict, brief = parse_verdict_and_brief(text) logger.info( f"[browser-fast-path] classifier: {verdict.upper()} brief={len(brief)}ch " f"model={aux_model} in {int((time.monotonic() - t0) * 1000)}ms" diff --git a/backend/apps/agents/browser/browser_fast_read.py b/backend/apps/agents/browser/browser_fast_read.py index 6968c872..13e0cbff 100644 --- a/backend/apps/agents/browser/browser_fast_read.py +++ b/backend/apps/agents/browser/browser_fast_read.py @@ -8,17 +8,19 @@ old path, never a wrong answer from a thin read. import asyncio import logging +import os import re import time +from urllib.parse import urljoin logger = logging.getLogger(__name__) P_ENTRY_RE = re.compile(r"^ENTRY:\s*(https?://\S+)", re.I | re.M) P_MIN_PAGE_CHARS = 500 -P_MAX_PAGE_CHARS = 24000 +MAX_PAGE_CHARS = 24000 P_FETCH_ERROR_PREFIXES = ("HTTP error", "Error fetching", "Refused to fetch") -P_ANSWER_SYSTEM = ( +ANSWER_SYSTEM = ( "Answer the user's request using ONLY the page text provided. Be direct and " "complete in a few sentences; quote exact titles/values from the page. End " "with nothing else.\n" @@ -26,6 +28,54 @@ P_ANSWER_SYSTEM = ( "exactly the single word INSUFFICIENT." ) +# One-hop mode: same contract plus a FOLLOW escape so a link-deep answer costs one more fetch instead of a full browser dispatch. +P_HOP_SYSTEM = ANSWER_SYSTEM + ( + "\nEXCEPTION: if the page text is insufficient but exactly one of the " + "numbered links clearly leads to the page that would contain the answer, " + "reply with exactly 'FOLLOW ' and nothing else." +) +P_FOLLOW_RE = re.compile(r"^FOLLOW\s+(\d+)\s*$", re.I) +P_LINK_RE = re.compile(r"]*?href=[\"'](?!javascript:|#|mailto:)([^\"'>]+)[\"'][^>]*>(.*?)", re.I | re.S) +P_MAX_LINKS = 60 + + +def hop_enabled() -> bool: + return os.environ.get("OSW_FASTREAD_HOP", "1") != "0" + + +def extract_links(html: str, base_url: str) -> list[tuple[str, str]]: + """(anchor text, absolute url) pairs, deduped, capped. Text-less anchors are + useless to the picker so they're dropped.""" + out: list[tuple[str, str]] = [] + seen: set[str] = set() + for href, inner in P_LINK_RE.findall(html or ""): + text = re.sub(r"<[^>]+>", " ", inner) + text = re.sub(r"\s+", " ", text).strip() + if not text: + continue + absolute = urljoin(base_url, href.strip()) + if not absolute.startswith(("http://", "https://")) or absolute in seen: + continue + seen.add(absolute) + out.append((text[:80], absolute)) + if len(out) >= P_MAX_LINKS: + break + return out + + +def format_link_menu(links: list[tuple[str, str]]) -> str: + return "\n".join(f"{i + 1}. {text} -> {url}" for i, (text, url) in enumerate(links)) + + +def parse_follow(answer: str, links: list[tuple[str, str]]) -> tuple[str, str]: + """The (anchor text, url) the aux picked, or ('', ''). Out-of-range picks + are dropped.""" + m = P_FOLLOW_RE.match((answer or "").strip()) + if not m: + return "", "" + idx = int(m.group(1)) - 1 + return links[idx] if 0 <= idx < len(links) else ("", "") + def extract_entry_url(brief: str) -> str: m = P_ENTRY_RE.search(brief or "") @@ -40,6 +90,63 @@ def page_is_thin(text: str) -> bool: return len(body.strip()) < P_MIN_PAGE_CHARS +async def fetch_page_text(url: str, prompt: str) -> str: + from backend.apps.agents.tools.web import WebFetchTool + + parts = await asyncio.wait_for( + WebFetchTool().execute({"url": url, "prompt": prompt}, None), + timeout=12.0, + ) + return "\n".join(p.get("text", "") for p in parts if p.get("type") == "text") + + +P_TAG_STRIP_RES = ( + re.compile(r"<(script|style|noscript)\b.*?", re.I | re.S), + re.compile(r"<[^>]+>"), +) + + +def strip_tags(html: str) -> str: + """Whole-page text incl. bylines/usernames; trafilatura's main-content pass + drops exactly the metadata that answers who/when questions, so hop mode + reads the raw page instead.""" + import html as p_html_mod + + text = html or "" + for rx in P_TAG_STRIP_RES: + text = rx.sub(" ", text) + return re.sub(r"[ \t\r\f\v]+", " ", p_html_mod.unescape(text)).strip() + + +async def fetch_raw(url: str) -> str: + """Raw HTML via the SSRF guard; '' on any miss.""" + try: + from backend.apps.agents.tools.ssrf_guard import safe_fetch + + resp = await asyncio.wait_for( + safe_fetch(url, method="GET", + headers={"User-Agent": "Mozilla/5.0 (Macintosh) AppleWebKit/537.36"}, + timeout=8.0), + timeout=10.0, + ) + return resp.text or "" + except Exception: + return "" + + +async def ask_aux(client, aux_model: str, system: str, content: str) -> str: + from backend.apps.agents.core.aux_llm import safe_resp_text + + resp = await asyncio.wait_for( + client.messages.create( + model=aux_model, max_tokens=500, temperature=0, system=system, + messages=[{"role": "user", "content": content}], + ), + timeout=15.0, + ) + return safe_resp_text(resp).strip() + + async def try_fast_read(prompt: str, brief: str, settings, primary_api: str | None) -> str | None: """Answer text on success; None means fall back to the browser leg.""" entry = extract_entry_url(brief) @@ -47,45 +154,61 @@ async def try_fast_read(prompt: str, brief: str, settings, primary_api: str | No logger.info("[browser-fast-read] no ENTRY url in brief; browser fallback") return None try: - from backend.apps.agents.tools.web import WebFetchTool - + hop = hop_enabled() t0 = time.monotonic() - parts = await asyncio.wait_for( - WebFetchTool().execute({"url": entry, "prompt": prompt}, None), - timeout=12.0, - ) - text = "\n".join(p.get("text", "") for p in parts if p.get("type") == "text") + links: list[tuple[str, str]] = [] + if hop: + raw = await fetch_raw(entry) + text, links = strip_tags(raw), extract_links(raw, entry) + if page_is_thin(text): + text = await fetch_page_text(entry, prompt) + else: + text = await fetch_page_text(entry, prompt) fetch_ms = int((time.monotonic() - t0) * 1000) if page_is_thin(text): logger.info(f"[browser-fast-read] thin/errored read of {entry} ({len(text)}ch in {fetch_ms}ms); browser fallback") return None - logger.info(f"[browser-fast-read] fetched {entry}: {len(text)}ch in {fetch_ms}ms") + logger.info(f"[browser-fast-read] fetched {entry}: {len(text)}ch in {fetch_ms}ms (links={len(links)})") from backend.apps.settings.credentials import get_anthropic_client_for_model from backend.apps.agents.providers.registry import resolve_aux_model - from backend.apps.agents.core.aux_llm import safe_resp_text aux_model, _ = await resolve_aux_model( settings, preferred_tier="haiku", primary_api=primary_api, ) client = get_anthropic_client_for_model(settings, aux_model) t1 = time.monotonic() - resp = await asyncio.wait_for( - client.messages.create( - model=aux_model, - max_tokens=500, - temperature=0, - system=P_ANSWER_SYSTEM, - messages=[{ - "role": "user", - "content": f"Request: {prompt}\n\nPage text from {entry}:\n{text[:P_MAX_PAGE_CHARS]}", - }], - ), - timeout=15.0, - ) - answer = safe_resp_text(resp).strip() + content = f"Request: {prompt}\n\nPage text from {entry}:\n{text[:MAX_PAGE_CHARS]}" + if hop and links: + content += f"\n\nNumbered links found on the page:\n{format_link_menu(links)}" + answer = await ask_aux(client, aux_model, P_HOP_SYSTEM if links else ANSWER_SYSTEM, content) answer_ms = int((time.monotonic() - t1) * 1000) - if not answer or answer.upper().startswith("INSUFFICIENT"): + + hop_anchor, hop_url = parse_follow(answer, links) if hop and links else ("", "") + if hop_url: + t2 = time.monotonic() + hop_text = strip_tags(await fetch_raw(hop_url)) + if page_is_thin(hop_text): + hop_text = await fetch_page_text(hop_url, prompt) + if page_is_thin(hop_text): + logger.info(f"[browser-fast-read] hop to {hop_url} was thin; browser fallback") + return None + answer = await ask_aux( + client, aux_model, ANSWER_SYSTEM, + f"Request: {prompt}\n\n" + f"Context: the navigation in the request is ALREADY DONE. From {entry} " + f"you chose the link '{hop_anchor}' as the one leading to the answer, and " + f"the page text below is that destination. Extract the requested " + f"information from it.\n\n" + f"Page text from {hop_url}:\n{hop_text[:MAX_PAGE_CHARS]}", + ) + logger.info( + f"[browser-fast-read] followed link {hop_url} " + f"(+{int((time.monotonic() - t2) * 1000)}ms hop)" + ) + entry = hop_url + + if not answer or answer.upper().startswith("INSUFFICIENT") or P_FOLLOW_RE.match(answer): logger.info(f"[browser-fast-read] aux found page insufficient ({answer_ms}ms); browser fallback") return None logger.info(f"[browser-fast-read] answered in {answer_ms}ms ({len(answer)}ch, model={aux_model})") diff --git a/backend/apps/agents/browser/browser_login_handoff.py b/backend/apps/agents/browser/browser_login_handoff.py new file mode 100644 index 00000000..a79ba4e7 --- /dev/null +++ b/backend/apps/agents/browser/browser_login_handoff.py @@ -0,0 +1,102 @@ +"""Login-once handoff: when the browser agent lands on a login wall, it pauses for the user to +sign in ONCE in the app's browser card, then continues, and we REMEMBER which sites the user has +authenticated so future runs skip the prompt and only re-ask on a genuine expiry or a different +account. The session itself lives in Electron's persist:openswarm-browser partition (which keeps +it across quits, so "sign in once, never again" is really the partition's doing); this module is +the durable memory of it plus the detection and the wording, keyed by registrable domain. + +Detection reuses the one structural login-wall definition in browser_send_parse, so the pause and +the send-script's decline can never disagree about what a login wall is. +""" + +import datetime +import os +from typing import Dict, List, Optional, Tuple +from urllib.parse import urlparse + +from typeguard import typechecked + +from backend.apps.agents.browser import browser_send_parse +from backend.config.json_store import atomic_write_json, read_json_or_none +from backend.config.paths import SETTINGS_DIR + +P_STORE_PATH = os.path.join(SETTINGS_DIR, "authenticated_domains.json") + + +@typechecked +def registrable_domain(url_or_host: str) -> str: + s = (url_or_host or "").strip() + host = urlparse(s).hostname if "://" in s else s.split("/")[0] + host = (host or "").lower().strip().lstrip(".").split(":")[0] + if host.startswith("www."): + host = host[4:] + return host + + +@typechecked +def p_load() -> Dict[str, Dict[str, str]]: + data = read_json_or_none(P_STORE_PATH) + return data if isinstance(data, dict) else {} + + +@typechecked +def is_authenticated(url_or_host: str) -> bool: + return registrable_domain(url_or_host) in p_load() + + +@typechecked +def authenticated_domains() -> List[str]: + return sorted(p_load().keys()) + + +@typechecked +def login_record(url_or_host: str) -> Optional[Dict[str, str]]: + """The stored {first_seen, last_login} for a site, or None. For a future 'signed-in sites' view.""" + return p_load().get(registrable_domain(url_or_host)) + + +@typechecked +def record_login(url_or_host: str) -> None: + """Remember that the user signed into this site, so future walls read as re-auth not first-run. + Fail-open: a write error just means the next run treats it as a fresh sign-in (harmless).""" + d = registrable_domain(url_or_host) + if not d: + return + store = p_load() + now = datetime.datetime.now(datetime.timezone.utc).isoformat() + prior = store.get(d) or {} + store[d] = {"first_seen": prior.get("first_seen") or now, "last_login": now} + try: + atomic_write_json(P_STORE_PATH, store) + except OSError: + pass + + +@typechecked +def login_wall_domain(current_url: str, state_text: str, allow_soft: bool = False) -> Optional[str]: + """The registrable domain of a login wall the agent is stuck on, or None. One definition of + 'login wall', shared with the send-script's decline gate. + + `allow_soft` additionally accepts a SOFT signed-out page: browsable, no auth form, composer + simply withheld behind a "Sign in" control (bsky/stackoverflow/tiktok). Those never match the + hard wall, so the run used to fail as "couldn't find the compose box" instead of offering the + one thing that fixes it. Off by default because this pause interrupts the user: the caller + turns it on only once the agent is demonstrably stuck, so a stray "Sign up" link on a page + we're actually signed into can't raise a spurious prompt.""" + if browser_send_parse.looks_like_login_wall(current_url or "", state_text or ""): + return registrable_domain(current_url) or None + if allow_soft and browser_send_parse.looks_signed_out(state_text or ""): + return registrable_domain(current_url) or None + return None + + +@typechecked +def prompt_copy(domain: str) -> Tuple[str, str]: + """(problem, instruction) for the pause overlay, worded by whether the user has signed into + this site before (re-auth) or it's a first sign-in.""" + if is_authenticated(domain): + problem = f"Your {domain} sign-in looks signed out, it may have expired or be a different account." + else: + problem = f"{domain} needs you to sign in before I can keep going." + instruction = "Log in to the site in the browser above, then click Done and I'll pick up right where I left off." + return problem, instruction diff --git a/backend/apps/agents/browser/browser_loop.py b/backend/apps/agents/browser/browser_loop.py index 0a536f29..5050ffd7 100644 --- a/backend/apps/agents/browser/browser_loop.py +++ b/backend/apps/agents/browser/browser_loop.py @@ -219,7 +219,7 @@ def stagnation_exhausted(streak: int) -> bool: # State-changing tools: a task that needed to DO something must land one of these. P_PRODUCTIVE_TOOLS = { "BrowserClick", "BrowserClickIndex", "BrowserType", "BrowserNavigate", - "BrowserPressKey", "BrowserScroll", "BrowserBatch", + "BrowserPressKey", "BrowserScroll", "BrowserBatch", "BrowserActVerified", } # Read/extract tools: a look-only task's evidence is that a read returned content. P_READ_TOOLS = { @@ -263,7 +263,7 @@ def recoverable_tool_error(err: str) -> bool: # Actions that DIRTY the page so replay-from-here is no longer equivalent to a clean dispatch. Navigation and reads don't dirty anything (they just get us to the page), so the deferred replay re-check is allowed after only those. P_REPLAY_DIRTYING_TOOLS = { "BrowserType", "BrowserClick", "BrowserClickIndex", - "BrowserPressKey", "BrowserScroll", "BrowserBatch", + "BrowserPressKey", "BrowserScroll", "BrowserBatch", "BrowserActVerified", } @@ -287,6 +287,16 @@ P_ACTION_ASK_RE = re.compile( re.I, ) +P_DELETE_INTENT_RE = re.compile( + r"\b(delete|remove|take ?down|unsend|retract|unpost|discard|trash)\b", re.I) + + +def is_removal_task(task: str) -> bool: + """A delete/remove ask. The send-script must stand down on these: a removal task is also + task_is_send (the classifier keys on the verb), so without this the composer fill would + TYPE the target text and POST it (measured live: delete tasks re-posted the marker).""" + return bool(P_DELETE_INTENT_RE.search(task or "")) + def deliverable_is_informational(summary: str, task: str = "") -> bool: """True if the run's final answer is GATHERED CONTENT (a list/report the model diff --git a/backend/apps/agents/browser/browser_map_reduce_read.py b/backend/apps/agents/browser/browser_map_reduce_read.py new file mode 100644 index 00000000..5ff9f7ba --- /dev/null +++ b/backend/apps/agents/browser/browser_map_reduce_read.py @@ -0,0 +1,205 @@ +""" +Map-reduce READ tier: answer a multi-source public read (a comparison, a +difference, a combine-across-pages) without the big-model loop. When the +single-page fast_read declined because the answer lives across TWO OR MORE +pages, one aux call decomposes the request into independent single-page +lookups, they run CONCURRENTLY (each is a fast_read-class fetch + extract), and +one aux reduce combines them. + +Fail-open everywhere: not multi-source, a thin or insufficient source, or a +reduce that can't answer all return None and the caller falls to the browser +leg, so a partial read can never become a wrong answer. Lives only in the +classifier's READ branch (public pages), so it never taxes an authed read. +""" + +import asyncio +import json +import logging +import os +import re +import time + +from backend.apps.agents.browser import browser_fast_read as fr + +logger = logging.getLogger(__name__) + +P_MAX_SOURCES = 4 + +P_DECOMPOSE_SYSTEM = ( + "Break the user's request into the MINIMUM set of independent factual " + "lookups, each answerable from a SINGLE public web page. Return a JSON array " + "of objects, each {\"q\": a self-contained question, \"url\": a starting URL " + "(a direct page like https://en.wikipedia.org/wiki/NAME, or a search URL " + "like https://www.google.com/search?q=...)}.\n" + "Return 2 or more entries ONLY when the request genuinely needs different " + "pages combined: a comparison, a difference, a sum, a 'both X and Y'. If a " + "single page could answer it, return [].\n" + "Never invent facts; only name the lookups. Output ONLY the JSON array." +) + +P_REDUCE_SYSTEM = ( + "Answer the user's original request using ONLY the sub-answers provided, " + "each gathered from its own page.\n" + "First state each exact value. Then show the SINGLE arithmetic step the " + "request needs (the subtraction, sum, or comparison). Then give the final " + "answer. Your final number MUST equal the result of that step; never state a " + "total or difference that disagrees with your own arithmetic.\n" + "If the sub-answers do not together contain what the request needs, reply " + "with exactly the single word INSUFFICIENT." +) + + +def enabled() -> bool: + """Fail-open additive tier; default on, kill with OSW_MAP_REDUCE_READ=0.""" + return os.environ.get("OSW_MAP_REDUCE_READ", "1") != "0" + + +# The aux reduce got the VALUES right but flipped the arithmetic twice in ~10 live runs ("taller +# by 360.2m" beside its own 113.2 math; "1096-1636=-540, not older" beside "540 years older"), so +# for the two shapes that are pure arithmetic the number is computed HERE and the model never +# does subtraction. Anything unparseable falls open to the aux reduce. +P_DIFF_RE = re.compile(r"\b(difference|older|younger|taller|shorter|higher|lower|farther|further|longer|heavier|lighter|bigger|smaller|faster|slower)\b", re.I) +P_SUM_RE = re.compile(r"\b(combined|total|sum|together|altogether)\b", re.I) +P_VALUE_RE = re.compile(r"VALUE:\s*([0-9]+(?:\.[0-9]+)?)\s*([a-zA-Z%]*)", re.I) +P_VALUE_LINE = ( + "\nEnd with one extra line: VALUE: ." +) + + +P_QUANTITY_RE = re.compile(r"\b(how much|how many|difference|by how)\b", re.I) + + +def op_for(prompt: str) -> str: + """'difference' | 'sum' | '' from the request's own wording; '' = aux reduce as before. + Difference also requires a QUANTITY cue: a bare "which is taller?" wants a name, and a + number-only computed headline would answer the wrong question (caught in audit, not live).""" + low = prompt or "" + if P_SUM_RE.search(low): + return "sum" + if P_DIFF_RE.search(low) and P_QUANTITY_RE.search(low): + return "difference" + return "" + + +def fmt_num(n: float) -> str: + """Human numbers: 35,842,039 not 3.5842e+07; two decimals max on non-integers.""" + return f"{n:,.0f}" if float(n).is_integer() else f"{n:,.2f}" + + +def computed_answer(op: str, plan: list[tuple[str, str]], subs: list) -> str: + """The deterministic answer when every sub-answer carries a parseable VALUE in agreeing + units; '' means fall open to the aux reduce. States both values and the computed number, + and deliberately asserts NO direction prose (that is exactly what the aux got wrong).""" + vals: list[tuple[float, str]] = [] + for s in subs: + m = P_VALUE_RE.search(s or "") + if not m: + return "" + vals.append((float(m.group(1)), m.group(2).lower())) + units = {u for _, u in vals} + if len(units) > 1: + return "" + unit = f" {vals[0][1]}" if vals[0][1] else "" + shown = "\n".join(f"- {q}: {fmt_num(v)}{unit}" for (q, _), (v, _) in zip(plan, vals)) + if op == "difference" and len(vals) == 2: + n = abs(vals[0][0] - vals[1][0]) + return (f"**Answer: {fmt_num(n)}{unit}**\n\n{shown}\n" + f"(computed: |{fmt_num(vals[0][0])} - {fmt_num(vals[1][0])}| = {fmt_num(n)})") + if op == "sum": + n = sum(v for v, _ in vals) + return (f"**Answer: {fmt_num(n)}{unit}**\n\n{shown}\n" + f"(computed: {' + '.join(fmt_num(v) for v, _ in vals)} = {fmt_num(n)})") + return "" + + +def parse_plan(text: str) -> list[tuple[str, str]]: + """(question, url) pairs from the decompose JSON; [] on anything unparseable + or single-source. Bounded to P_MAX_SOURCES so a runaway plan can't fan out.""" + s = (text or "").strip() + i, j = s.find("["), s.rfind("]") + if i < 0 or j <= i: + return [] + try: + arr = json.loads(s[i:j + 1]) + except (json.JSONDecodeError, ValueError): + return [] + out: list[tuple[str, str]] = [] + for it in arr if isinstance(arr, list) else []: + if isinstance(it, dict): + q, url = str(it.get("q") or "").strip(), str(it.get("url") or "").strip() + if q and url.startswith(("http://", "https://")): + out.append((q, url)) + return out[:P_MAX_SOURCES] + + +async def p_fetch_and_extract(client, aux_model: str, q: str, url: str, ask_value: bool) -> str | None: + """One source: fetch the page, aux-extract the answer to q, or None if the + page is thin or insufficient (so the whole map-reduce fails open, never + fabricates a missing piece). ask_value appends the machine-parseable VALUE + line the code-side arithmetic needs.""" + try: + raw = await fr.fetch_raw(url) + text = fr.strip_tags(raw) + if fr.page_is_thin(text): + text = await fr.fetch_page_text(url, q) + if fr.page_is_thin(text): + return None + ans = await fr.ask_aux( + client, aux_model, fr.ANSWER_SYSTEM + (P_VALUE_LINE if ask_value else ""), + f"Request: {q}\n\nPage text from {url}:\n{text[:fr.MAX_PAGE_CHARS]}") + if not ans or ans.upper().startswith("INSUFFICIENT"): + return None + return ans + except Exception: + return None + + +async def try_map_reduce_read(prompt: str, settings, primary_api: str | None) -> str | None: + """Answer text for a multi-source public read, or None (caller falls to the + browser leg). Any missing piece returns None, so it never half-answers.""" + if not enabled(): + return None + t0 = time.monotonic() + try: + from backend.apps.settings.credentials import get_anthropic_client_for_model + from backend.apps.agents.providers.registry import resolve_aux_model + + aux_model, _ = await resolve_aux_model( + settings, preferred_tier="haiku", primary_api=primary_api) + client = get_anthropic_client_for_model(settings, aux_model) + + plan_text = await fr.ask_aux(client, aux_model, P_DECOMPOSE_SYSTEM, f"Request: {prompt[:1200]}") + plan = parse_plan(plan_text) + if len(plan) < 2: + return None + logger.info(f"[browser-mapreduce] {len(plan)} sources: {[u for _, u in plan]}") + + p_op = op_for(prompt) + subs = await asyncio.gather(*[p_fetch_and_extract(client, aux_model, q, u, bool(p_op)) for q, u in plan]) + if any(s is None for s in subs): + logger.info(f"[browser-mapreduce] a source came back thin/insufficient in " + f"{int((time.monotonic() - t0) * 1000)}ms; browser fallback") + return None + + if p_op: + p_coded = computed_answer(p_op, plan, subs) + if p_coded: + logger.info(f"[browser-mapreduce] {p_op} computed in code from {len(plan)} sources " + f"in {int((time.monotonic() - t0) * 1000)}ms") + return f"{p_coded}\n\n(Sources: {', '.join(u for _, u in plan)})" + + joined = "\n\n".join(f"Sub-question: {q}\nAnswer (from {u}): {s}" + for (q, u), s in zip(plan, subs)) + final = await fr.ask_aux(client, aux_model, P_REDUCE_SYSTEM, + f"Original request: {prompt}\n\n{joined}") + if not final or final.upper().startswith("INSUFFICIENT"): + logger.info(f"[browser-mapreduce] reduce insufficient in " + f"{int((time.monotonic() - t0) * 1000)}ms; browser fallback") + return None + logger.info(f"[browser-mapreduce] answered from {len(plan)} sources in " + f"{int((time.monotonic() - t0) * 1000)}ms") + return f"{final}\n\n(Sources: {', '.join(u for _, u in plan)})" + except Exception as e: + logger.info(f"[browser-mapreduce] skipped ({e}); browser fallback") + return None diff --git a/backend/apps/agents/browser/browser_plan_dispatch.py b/backend/apps/agents/browser/browser_plan_dispatch.py new file mode 100644 index 00000000..ba27f0dc --- /dev/null +++ b/backend/apps/agents/browser/browser_plan_dispatch.py @@ -0,0 +1,144 @@ +"""Code-side plan dispatch: the turn-collapser that does NOT depend on the model +adopting a tool (it never does; 0/3 live A/Bs). ONE cheap aux call maps the task + +live page state to a chain of verified steps; run_verified_step executes them in +code (resolve-late, verify-effect, re-aim); the big model then starts with the +mechanical work DONE instead of spending a ~4-6s turn per click. + +Safety mirrors the send-script: the aux may only emit click/fill on elements it +names from the live list, anything irreversible-smelling is refused in code, and +every step must VERIFY or the chain stops and hands off honestly. Fail-open +everywhere: no aux, bad JSON, zero steps = the loop runs exactly as today. +""" + +import asyncio +import json +import logging +import os +import re +import time + +from backend.apps.agents.browser import browser_verified_step +from backend.apps.agents.browser.browser_prestage import BLOCKED_CLICK_RE + +logger = logging.getLogger(__name__) + +P_MAX_STEPS = 6 +P_AUX_TIMEOUT_S = 10.0 +# Cross-page steps land right after a navigation; give the new page a beat before resolving. +P_STEP_SETTLE_S = 1.2 +P_STATE_CAP = 6000 + +P_SYSTEM = ( + "You compile the MECHANICAL prefix of a browser task into steps a dumb executor " + "runs and VERIFIES one at a time. You see the task and the page's interactive " + "elements. Emit ONLY steps in order, as STRICT JSON (no prose): an array of\n" + '{"action":"click"|"fill","target":"",' + '"role":"button"|"link"|"textbox"|"","text":"",' + '"expect":"appeared:"|"gone:"|"url_changed"|"changed"|"",' + '"chosen":true|false}\n' + "Rules: a target is copied verbatim from a listed element name, EXCEPT steps after " + "one that navigates: those may name an element the task implies will appear (e.g. " + "'Message' after opening a profile). Each step is resolved against the live page " + "and verified before the next runs, so a wrong guess stops the chain safely. " + "Expectations: use url_changed for clicks that open a new page, appeared: " + "for clicks that open a dialog or composer. ORDINALS map to rows: 'the 4th " + "story's comments' = copy the name of the 4th row matching that shape; you may " + "and should count. When the task names a person or thing and several rows are " + "similar, PICK the best row using the task's cues and mark that step " + "\"chosen\":true; for messaging a person, a direct/1st-degree connection outranks " + "every other cue (title, company, verified): people message people they know. STOP the chain before " + "anything irreversible (send/submit/post/pay/delete/confirm/apply). NEVER fill a " + "message, comment, or post body: once a composer for one is open, stop, the main " + "agent writes and sends it. If the Current URL shows the page ALREADY is the " + "target's own page, emit []: a click that goes nowhere just fails verification. " + "0-6 steps; [] when nothing is safely mechanical." +) + + +def parse_plan(reply: str) -> list: + """Strict-ish JSON array extraction; anything malformed = [] (fail-open). + A max_tokens-truncated array is salvaged by closing it after the last complete + object: the steps run one at a time with verification, so a shortened plan is + safe, and losing the tail beats losing the whole plan (measured live).""" + text = (reply or "").strip() + m = re.search(r"\[.*\]", text, re.S) + candidate = m.group(0) if m else "" + if not candidate and text.startswith("["): + cut = text.rfind("}") + if cut > 0: + candidate = text[: cut + 1] + "]" + if not candidate: + return [] + try: + raw = json.loads(candidate) + except Exception: + return [] + steps = [] + for r in raw[:P_MAX_STEPS]: + if not isinstance(r, dict): + continue + action = str(r.get("action") or "") + target = str(r.get("target") or "").strip() + if action not in ("click", "fill") or not target: + continue + if BLOCKED_CLICK_RE.search(target): + break # irreversible-smelling: refuse this and everything after it + steps.append(browser_verified_step.VerifiedStep( + kind=action, target=target, role=str(r.get("role") or ""), + text=str(r.get("text") or ""), expect=str(r.get("expect") or ""), + chosen=bool(r.get("chosen")))) + return steps + + +def plan_dispatch_enabled() -> bool: + return os.environ.get("OSW_PLAN_DISPATCH", "0") == "1" + + +async def run_plan_dispatch( + task: str, state_text: str, browser_id: str, tab_id: str, + settings, primary_api, execute_tool, current_url: str = "", +) -> str: + """Returns a handoff note describing verified-executed steps ('' = nothing ran). + Never raises; never acts irreversibly.""" + t0 = time.monotonic() + try: + from backend.apps.settings.credentials import get_anthropic_client_for_model + from backend.apps.agents.providers.registry import resolve_aux_model + from backend.apps.agents.core.aux_llm import safe_resp_text + + aux_model, _ = await resolve_aux_model(settings, preferred_tier="haiku", primary_api=primary_api) + client = get_anthropic_client_for_model(settings, aux_model) + # Assistant prefill "[" makes prose unwritable: the aux was narrating the ambiguity instead of emitting the chosen click (caught live via the empty-plan reply log). + reply = "[" + safe_resp_text(await asyncio.wait_for( + client.messages.create( + model=aux_model, max_tokens=1000, temperature=0, system=P_SYSTEM, + messages=[ + {"role": "user", "content": ( + f"Task: {task[:1200]}\n\nCurrent URL: {current_url[:300]}\n\n" + f"Interactive elements:\n{state_text[:P_STATE_CAP]}")}, + {"role": "assistant", "content": "["}, + ], + ), timeout=P_AUX_TIMEOUT_S)) + steps = parse_plan(reply) + if not steps: + logger.info(f"[plan-dispatch] aux emitted no safe mechanical steps " + f"(state={len(state_text)}ch, reply: {(reply or '')[:160]!r})") + return "" + done: list[str] = [] + for step in steps: + r = await browser_verified_step.run_verified_step( + step, browser_id, tab_id, execute_tool, settle_s=P_STEP_SETTLE_S) + if not r["ok"]: + done.append(f"{step.kind} {step.target!r} FAILED ({r['note']}); stopped there") + break + mark = " [CHOSEN among similar rows: confirm it matches the task before anything irreversible]" if step.chosen else "" + done.append(f"{step.kind} {step.target!r} done+verified{mark}") + note = ( + f"[Plan pre-executed and VERIFIED in code: {'; '.join(done)}. " + "Do NOT redo these; continue from the page's CURRENT state below.]" + ) + logger.info(f"[plan-dispatch] {len(done)} step(s) in {int((time.monotonic() - t0) * 1000)}ms: {'; '.join(done)[:160]}") + return note + except Exception as e: + logger.info(f"[plan-dispatch] skipped ({e})") + return "" diff --git a/backend/apps/agents/browser/browser_prestage.py b/backend/apps/agents/browser/browser_prestage.py new file mode 100644 index 00000000..128ed641 --- /dev/null +++ b/backend/apps/agents/browser/browser_prestage.py @@ -0,0 +1,496 @@ +""" +Navigation pre-stage: before the big model wakes, a cheap aux model drives +NAVIGATE/CLICK-only steps on the live webview until the page is where the main +agent only has to do the final content action (read the answer, type into an +open composer). Deletes the 4-6 cold orientation turns from the big loop; the +big model starts staged instead of exploring at ~3s a thought. + +Safety is code, not prose: the only tools this module can issue are +BrowserNavigate and BrowserClickIndex, and a click whose listed element text +smells irreversible (send/submit/pay/...) is refused in code, ending the +pre-stage so the main loop's full guard stack owns that step. +""" + +import asyncio +import logging +import os +import re +import time +from typing import Awaitable, Callable + +from backend.apps.agents.browser import browser_send_parse, compose_discovery, compose_entry + +logger = logging.getLogger(__name__) + +MAX_STEPS = 4 +STEP_TIMEOUT_S = 8.0 +TOTAL_TIMEOUT_S = 25.0 +# Opener mode reaches one hop deeper (a post/comment surface is often nav -> open item -> reveal box). +OPENER_MAX_STEPS = 6 +OPENER_TOTAL_TIMEOUT_S = 32.0 + +P_STEP_RE = re.compile(r"^\s*(NAVIGATE|CLICK|READY)\b[:\s]*(.*)$", re.I) + +# URL shapes that mean "a list of candidates to pick from" (also drives the agent's candidate scan) +RESULTS_URL_RE = re.compile( + r"[?&](q|query|keywords|search|search_query|find|term)=|/search\b|/results\b", re.I, +) +BLOCKED_CLICK_RE = re.compile( + r"\b(send|submit|post|pay|buy|order|delete|confirm|apply|accept|invite|" + r"connect|purchase|checkout|subscribe|unfollow|sign\s?out|log\s?out)\b", + re.I, +) +# Genuinely irreversible / costly: NEVER a composer-opener, refused in every mode. +P_HARD_BLOCK_RE = re.compile( + r"\b(send|submit|pay|buy|order|delete|confirm|apply|accept|invite|" + r"connect|purchase|checkout|subscribe|unfollow|sign\s?out|log\s?out)\b", + re.I, +) +# Compose-ENTRY words: on a composer-ABSENT page these OPEN a box (X/Threads "Post", +# Reddit "Create Post", "Add a comment", "Reply", "New thread"); the SAME word is the +# submit once a box exists. So allowed only while no composer is in perception. +P_COMPOSE_ENTRY_RE = re.compile(r"\b(post|comment|reply|tweet|write|thread|note|caption)\b", re.I) + + +def opener_mode() -> bool: + """Whether prestage may OPEN a composer (click a person's Message / a 'Reply'/'Post' + surface) instead of only navigating to an already-open one. It's ON when its own flag is + set OR when the send-script is enabled: the send-script can only fire once a composer is + reached, and the opener is what reaches it, so they're a pair (a send-script run that + lands on a search page with no opener just declines and burns the slow model loop, the + exact miss we measured). Safe by construction: the opener never types and refuses any + send/submit/pay word, so a worst-case mis-click opens an empty box, never sends.""" + if os.environ.get("OSW_PRESTAGE_OPENER", "0") != "0": + return True + from backend.apps.agents.browser.browser_send_script import script_enabled + return script_enabled() + + +def click_refused(entry: str, li_text: str) -> bool: + """Whether prestage must refuse this click. Opener-mode-off = the legacy blanket + gate (Phase A byte-identical). Opener-mode-on = structural: hard-irreversible + words refused always; a compose-entry word (post/comment/reply/...) refused ONLY + when a composer textbox is ALREADY in the current perception (then it's the real + submit), allowed when none is present (then the click REVEALS the composer). + Prestage never types, so even a worst-case mis-click submits empty content.""" + if not opener_mode(): + return bool(BLOCKED_CLICK_RE.search(entry)) + if P_HARD_BLOCK_RE.search(entry): + return True + if P_COMPOSE_ENTRY_RE.search(entry): + from backend.apps.agents.browser.browser_send_parse import composer_index_in_state + return bool(composer_index_in_state(li_text or "")) + return False + + +P_SYSTEM_OPENER = ( + "You pre-stage a browser for a main agent. Using ONLY navigation and clicks " + "that OPEN or REVEAL a composer, get the page to where a text box is visible " + "and the main agent only has to type the content and submit.\n" + "OPENING a composer IS your job: click 'Start a post' / 'Create post' / 'New " + "thread' / the compose 'Post' or 'Tweet' button / 'Add a comment' / 'Reply' / " + "a person's 'Message' button so the text box appears.\n" + "The MOMENT a compose text box is visible in the elements, reply READY, the " + "stage is set.\n" + "NEVER submit: do not click Send, Submit, Pay, Buy, Order, Delete, Confirm, " + "Subscribe, or Connect. If the only next step is typing or the final submit, " + "reply READY.\n" + "For a task that messages a PERSON: go to that person (search result, " + "profile), then open their Message surface. For a comment/reply on a thread " + "or video: open the item, then reveal the comment box.\n" + "Reply with exactly ONE line:\n" + "NAVIGATE \n" + "CLICK \n" + "READY \n" + "If unsure, reply READY." +) + +P_SYSTEM = ( + "You pre-stage a browser for a main agent. Using ONLY navigation (opening " + "pages, clicking links or buttons that open/reveal things), get the page to " + "the state where the main agent only has to do the FINAL content action " + "(read the requested info, or type into an already-open composer/form).\n" + "NEVER click anything that sends, submits, posts, pays, buys, deletes, " + "accepts, connects, or subscribes. Opening a composer (e.g. a 'Message' " + "button) is allowed; pressing its Send is not. If the next needed step is " + "typing text or an irreversible click, the stage is set.\n" + "For a task about a specific PERSON or THING (message them, read their " + "details): click through to that person/thing's OWN page first; a " + "search-results list is NOT the staged page. For messaging, then open " + "their Message/compose surface; never detour to a feed or homepage. When " + "several people share the name, a direct/1st-degree connection outranks " + "every other cue (title, company, verified): people ask about people " + "they know.\n" + "Reply with exactly ONE line:\n" + "NAVIGATE \n" + "CLICK \n" + "READY \n" + "If unsure, reply READY." +) + +ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]] + + +def prestage_enabled() -> bool: + return os.environ.get("OSW_PRESTAGE", "1") != "0" + + +def list_entry_for(list_text: str, index: int) -> str: + for line in (list_text or "").splitlines(): + if line.strip().startswith(f"[{index}]"): + return line.strip() + return "" + + +def parse_step(reply: str) -> tuple[str, str]: + m = P_STEP_RE.match((reply or "").strip().splitlines()[0] if reply else "") + if not m: + return "ready", "" + return m.group(1).lower(), m.group(2).strip() + + +def perception_block(li_text: str, gt_text: str, stage_note: str = "") -> str: + parts = [] + if li_text: + parts.append("Interactive elements already on the page:\n" + li_text) + if gt_text: + parts.append("Visible page text (truncated):\n" + gt_text[:2000]) + if not parts: + return "" + return ( + "\n\n[Page already loaded and inspected for you, act directly; " + "no need to screenshot or list elements again unless it changes]\n" + + (f"{stage_note}\n" if stage_note else "") + + "\n\n".join(parts) + ) + + +def stage_note_for(start_url: str, done: list[str], current_url: str, complete: bool) -> str: + """Without this the main model re-verifies the route from scratch (observed: + it navigated straight back to the start page), erasing the staging win. The + note must never overclaim: a partial stage saying 'navigation DONE' sent the + main loop on a 27-turn walkabout (observed live).""" + if not done: + return "" + if complete: + return ( + f"[Pre-staged for you and VERIFIED: starting from {start_url or 'the entry page'}, " + f"already performed: {'; '.join(done)}. You are NOW on {current_url}. The " + "navigation part of the task is DONE, do not go back or re-verify it; " + "perform only the remaining final action(s). Staged runs took 7 solo turns " + "where 2 suffice: if the remaining work is composing, use ONE BrowserBatch to " + "focus the box and type the text, then the Send/Submit click SOLO with expect. " + "Do not re-list first; the elements are listed below.]" + ) + return ( + f"[Partial pre-staging: already performed {'; '.join(done)}. You are NOW on " + f"{current_url}. Continue from HERE (do not restart from the beginning); " + "finish the remaining navigation and the task yourself.]" + ) + + +async def run_prestage( + task: str, + browser_id: str, + tab_id: str, + start_url: str, + settings, + primary_api: str | None, + execute_tool: ToolRunner, + perceive_only: bool = False, + task_is_send: bool = False, +) -> tuple[str, str, list[dict]]: + """(perception_block, current_url, action_records); ('', start_url, []) + means nothing staged and the caller proceeds exactly as before. + perceive_only skips the aux navigation loop and just captures the page: the + caller has a verified click-through tier of its own (plan-dispatch), so the + aux asks here were measured pure overhead (~2s) on that path.""" + t0 = time.monotonic() + recs: list[dict] = [] + try: + from backend.apps.settings.credentials import get_anthropic_client_for_model + from backend.apps.agents.providers.registry import resolve_aux_model + from backend.apps.agents.core.aux_llm import safe_resp_text + + aux_model, _ = await resolve_aux_model(settings, preferred_tier="haiku", primary_api=primary_api) + client = get_anthropic_client_for_model(settings, aux_model) + + async def perceive() -> tuple[str, str, str]: + li, gt = await asyncio.gather( + execute_tool("BrowserListInteractives", {}, browser_id, tab_id), + execute_tool("BrowserGetText", {}, browser_id, tab_id), + return_exceptions=True, + ) + li = li if isinstance(li, dict) else {} + gt = gt if isinstance(gt, dict) else {} + url = str(li.get("url") or gt.get("url") or "") + li_text = str(li.get("text") or "") if "error" not in li else "" + gt_text = str(gt.get("text") or "") if "error" not in gt else "" + return li_text, gt_text, url + + current_url = start_url + li_text, gt_text = "", "" + steps = 0 + done_desc: list[str] = [] + seen_steps: set[tuple[str, str]] = set() + staged_complete = False + + async def open_composer_directly(url: str) -> bool: + """Navigate to the site's own compose URL and confirm a composer actually appeared. + + The confirmation is the whole point. Without it this would be a per-site nav hardcode + that strands the run wherever the URL happens to lead once a site changes it; with it, + a miss costs one navigation and the aux loop below runs exactly as it does today.""" + nonlocal li_text, gt_text, current_url, staged_complete + r = await execute_tool("BrowserNavigate", {"url": url}, browser_id, tab_id) + ok = isinstance(r, dict) and "error" not in r + recs.append({"tool": "BrowserNavigate", "input": {"url": url}, "ok": ok, + "result_summary": f"compose entry for {compose_entry.registrable_host(url)}"[:200], + "elapsed_ms": 0}) + if not ok: + return False + # This is a cold NAVIGATION into a single-page app, not a modal opening on a page that + # is already up, so it gets a longer budget than the opener hop: the app has to boot + # before the composer can exist. Bounded well inside the prestage timeout so a miss + # still leaves room for the aux loop. + p_boxes = 0 + p_prev = "" + p_settled = 0 + for wait_s in (0.8, 1.2, 1.5, 2.0, 2.5, 2.5, 2.5): + await asyncio.sleep(wait_s) + li2, gt2, u2 = await perceive() + if li2: + li_text, gt_text = li2, gt2 + current_url = u2 or url + # Same stop condition as the opener hop: wait while the page is still arriving, + # give up the moment it stops changing. Measured on a loaded machine, x.com's + # compose route reported ZERO textboxes after 8s (nothing had rendered at all, not + # an ambiguous pick), while an idle machine had it in under two. + # TWO identical reads, not one. Gmail's compose window paints To/Cc/Bcc/Subject + # first and holds them steady for a beat while the body field is still arriving, so + # a single stable read declared the page finished and we walked away from a + # composer that was about to exist (measured: "saw 4 textbox(es) but no single + # composer"). + p_settled = p_settled + 1 if li2 and li2 == p_prev else 0 + p_prev = li2 + # A signed-out visit to a compose URL redirects to sign-in, and a login form is + # made of textboxes. Claiming "composer reached" there would tell the rest of the + # run the navigation is done while it sits on an auth wall. + if browser_send_parse.looks_like_login_wall(current_url, li2): + logger.info("[browser-prestage] compose entry landed on a sign-in wall; " + "not staged") + return False + if browser_send_parse.composer_index_in_state(li2): + return True + p_boxes = browser_send_parse.textbox_count(li2) + if p_settled >= 2: + break + # Name the miss. "No composer" covers three different problems with three different + # fixes: the page never mounted one (0 boxes), we were too early (few boxes, still + # hydrating), or several matched and the picker refused as ambiguous. Guessing between + # them is how the last two rounds of timeout tuning made things worse. + logger.info(f"[browser-prestage] compose entry saw {p_boxes} textbox(es) but no single " + f"composer at {current_url[:80]}") + return False + + async def discover_compose_links(page_url: str) -> list[str]: + """The site's own compose links, best first, or nothing. + + Failure is silent on purpose: this runs on whatever page the card happens to be on, so + a page that blocks evaluation or publishes no such link must cost one read and leave the + run exactly as it was.""" + # Read the links of the site the task is ABOUT, not whichever page the card was left + # on. A cold run opens on a blank/search page, so the first attempt at this read + # happened on google.com and correctly found nothing: the site was never visited. + wanted = compose_entry.named_hosts(task, page_url) + if not wanted: + return [] + host = wanted[0] + # The page the user named, not just its host: github's "New issue" lives on the repo, + # and github.com/ publishes no compose link at all. + target = compose_entry.named_page(task, host) + if not (page_url or "").rstrip("/").startswith(target.rstrip("/")): + nav = await execute_tool( + "BrowserNavigate", {"url": target}, browser_id, tab_id) + if not (isinstance(nav, dict) and "error" not in nav): + return [] + # The links live in the app shell, which is not there the instant navigation + # returns. One settle beat, not a ladder: if the shell is slower than this the aux + # loop is the better remaining spend. + await asyncio.sleep(1.5) + try: + raw = await execute_tool( + "BrowserEvaluate", {"expression": compose_discovery.discovery_expression()}, + browser_id, tab_id) + except Exception as exc: + logger.info(f"[browser-prestage] compose discovery could not read the page ({exc})") + return [] + found = compose_discovery.rank_candidates( + compose_discovery.parse_page_read(raw), host) + logger.info(f"[browser-prestage] compose discovery on {host} -> " + f"{found if found else 'no compose link published'}") + return found + + # Composer reachability: when the task creates something top-level on a site that + # publishes its own compose URL, ask for that URL instead of aux-hunting the button. This + # is the 0/20 gap; the fill and receipt behind it were already proven. A hit also skips + # the aux loop, so the cheap path and the reliable path are the same path. + # Where the run BEGAN, before anything moved the card. A card keeps the last URL it was + # left on, so a run can inherit a composer some earlier run opened and look like a win it + # never earned; without this line there is no way to tell those apart after the fact. + logger.info(f"[browser-prestage] start url={(start_url or '(none)')[:120]}") + p_compose_url = "" if perceive_only else ( + compose_entry.compose_entry_for(task, start_url, task_is_send) or "") + if p_compose_url: + if await open_composer_directly(p_compose_url): + staged_complete = True + done_desc.append(f"opened the composer at {p_compose_url}") + logger.info(f"[browser-prestage] compose entry {p_compose_url} reached a composer") + else: + logger.info(f"[browser-prestage] compose entry {p_compose_url} showed no composer; " + f"falling through to the aux loop") + + # No table row for this host. Ask the PAGE where its composer is instead: a site that has + # one links to it ("Start a post", "Ask Question"), and reading that link is what makes this + # work on a host nobody has written down. Measured 0/8 off-table before this tier existed. + if (not staged_complete and not perceive_only and task_is_send + and compose_discovery.enabled() and compose_entry.wants_top_level_compose(task)): + for p_found in await discover_compose_links(current_url or start_url): + if await open_composer_directly(p_found): + staged_complete = True + done_desc.append(f"opened the composer at {p_found}") + logger.info(f"[browser-prestage] discovered compose link {p_found} " + f"reached a composer") + break + logger.info(f"[browser-prestage] discovered compose link {p_found} " + f"showed no composer") + + async def settle(pre_url: str, pre_text: str, pre_li: str) -> bool: + """Wait for the page to actually change after an action, capped. + + Timed by the caller's log line: this polls with a full perceive each round, so it is a + real share of prestage's cost, and separating it from the aux plan is what says whether + the fix is a cheaper decision or a faster wait.""" + # A click returns before the page swaps; perceiving too early reads the OLD page and the aux re-issues the same click (observed 4x loop). Wait for the page to actually change, capped. False = the action verifiably did NOT take. An overlay (message composer) changes the INTERACTIVES but not the URL and often not the first 400 chars of text, so the element list counts as change too. + t_s = time.monotonic() + while time.monotonic() - t_s < 3.0: + await asyncio.sleep(0.35) + li2, gt2, u2 = await perceive() + if ((u2 and u2 != pre_url) or (gt2 and gt2[:400] != pre_text[:400]) + or (li2 and pre_li and li2 != pre_li)): + return True + return False + p_max_steps = 0 if perceive_only else (OPENER_MAX_STEPS if opener_mode() else MAX_STEPS) + p_total_timeout = OPENER_TOTAL_TIMEOUT_S if opener_mode() else TOTAL_TIMEOUT_S + p_system = P_SYSTEM_OPENER if opener_mode() else P_SYSTEM + p_results_overruled = False + while (not staged_complete and steps < p_max_steps + and (time.monotonic() - t0) < p_total_timeout): + # Per-step cost, broken out. Prestage is the largest single phase of a LinkedIn write + # (measured 18.6s of a 50.6s run, more than the send itself), and "steps=2 in 18587ms" + # cannot tell you whether that is the aux deciding, the page settling, or the click. + # Those have completely different fixes, so the log has to separate them. + p_t_step = time.monotonic() + li_text, gt_text, seen_url = await perceive() + p_t_perceive = time.monotonic() - p_t_step + current_url = seen_url or current_url + p_t_aux = time.monotonic() + reply = safe_resp_text(await asyncio.wait_for( + client.messages.create( + model=aux_model, max_tokens=60, temperature=0, system=p_system, + messages=[{"role": "user", "content": ( + f"Task: {task[:1500]}\n\nCurrent URL: {current_url}\n\n" + f"Interactive elements:\n{li_text[:4000]}\n\n" + f"Visible text (truncated):\n{gt_text[:1200]}" + )}], + ), + timeout=STEP_TIMEOUT_S, + )).strip() + p_aux_ms = int((time.monotonic() - p_t_aux) * 1000) + logger.info(f"[browser-prestage] step {steps + 1} plan: perceive={int(p_t_perceive * 1000)}ms " + f"aux={p_aux_ms}ms reply={reply[:40]!r}") + verb, arg = parse_step(reply) + if verb == "ready" or not arg: + # A results LIST is never the staged page for a task about one specific person/thing; the aux accepts it about half the time (measured, 2/4 cold LinkedIn runs) and every downstream tier then declines. Overrule ONCE with a nudge re-ask; a second READY is accepted, some tasks really do target the list. + if RESULTS_URL_RE.search(current_url or "") and not p_results_overruled: + p_results_overruled = True + task = task + ( + "\n\n[You replied READY on a search-results LIST. If the task is about " + "one specific person or thing, CLICK through to its own page first; " + "READY again only if the task really is about this list.]") + continue + staged_complete = True + logger.info(f"[browser-prestage] READY after {steps} step(s): {arg[:80]}") + break + # Any revisit (not just consecutive) is a loop signal: an A/B nav flap slipped past the consecutive-only check. + if (verb, arg) in seen_steps: + logger.info(f"[browser-prestage] repeated step {verb} {arg[:40]!r}; stopping") + break + seen_steps.add((verb, arg)) + if verb == "navigate": + if not arg.startswith(("http://", "https://")): + break + r = await execute_tool("BrowserNavigate", {"url": arg}, browser_id, tab_id) + ok = isinstance(r, dict) and "error" not in r + recs.append({"tool": "BrowserNavigate", "input": {"url": arg}, "ok": ok, + "result_summary": str(r.get("text", r.get("error", "")))[:200] if isinstance(r, dict) else "", + "elapsed_ms": 0}) + logger.info(f"[browser-prestage] step {steps + 1}: nav {arg} ok={ok}") + if not ok: + break + p_t_settle = time.monotonic() + p_settled = await settle(current_url, gt_text, li_text) + logger.info(f"[browser-prestage] step {steps + 1} nav settle={int((time.monotonic() - p_t_settle) * 1000)}ms ok={p_settled}") + if not p_settled: + logger.info(f"[browser-prestage] nav {arg} did not settle; stopping unstaged") + break + done_desc.append(f"navigated to {arg}") + else: + try: + idx = int(re.sub(r"\D", "", arg) or "-1") + except ValueError: + break + entry = list_entry_for(li_text, idx) + if idx < 0 or not entry or click_refused(entry, li_text): + logger.info(f"[browser-prestage] refusing click {idx} ({entry[:80]!r}); handing to main loop") + break + r = await execute_tool("BrowserClickIndex", {"index": idx}, browser_id, tab_id) + ok = isinstance(r, dict) and "error" not in r + recs.append({"tool": "BrowserClickIndex", "input": {"index": idx}, "ok": ok, + "result_summary": entry[:200], "elapsed_ms": 0}) + logger.info(f"[browser-prestage] step {steps + 1}: click [{idx}] {entry[:60]!r} ok={ok}") + if not ok: + break + p_t_settle = time.monotonic() + p_settled = await settle(current_url, gt_text, li_text) + logger.info(f"[browser-prestage] step {steps + 1} click settle={int((time.monotonic() - p_t_settle) * 1000)}ms ok={p_settled}") + if not p_settled: + # The click ran but the page never changed (occluded element, overlay, stale index). Recording it would make the handoff note LIE ("navigation done") and send the main loop on a walkabout; observed live as 27-turn/112s regressions. + logger.info(f"[browser-prestage] click [{idx}] did not settle; stopping unstaged") + break + done_desc.append(f"clicked {entry[:70]}") + steps += 1 + + if steps or not li_text: + li_text, gt_text, seen_url = await perceive() + current_url = seen_url or current_url + # Perceive-only lost the aux asks that ACCIDENTALLY doubled as settle time, so a cold SPA hands back a half-hydrated list (measured: plan-dispatch emitted [] on a thin search page). Wait for substance, bounded. + if perceive_only: + p_sub_t0 = time.monotonic() + while len(li_text or "") < 800 and time.monotonic() - p_sub_t0 < 4.0: + await asyncio.sleep(0.8) + li_text, gt_text, seen_url = await perceive() + current_url = seen_url or current_url + block = perception_block(li_text, gt_text, stage_note_for(start_url, done_desc, current_url, staged_complete)) + for tool_name, text in (("BrowserListInteractives", li_text), ("BrowserGetText", gt_text)): + if text: + recs.append({"tool": tool_name, "input": {}, "ok": True, + "result_summary": text[:200], "elapsed_ms": 0}) + logger.info( + f"[browser-prestage] done: steps={steps}{' (perceive-only)' if perceive_only else ''} " + f"url={current_url[:80]} in {int((time.monotonic() - t0) * 1000)}ms" + ) + return block, current_url, recs + except Exception as e: + logger.info(f"[browser-prestage] skipped ({e})") + return "", start_url, recs diff --git a/backend/apps/agents/browser/browser_read_script.py b/backend/apps/agents/browser/browser_read_script.py new file mode 100644 index 00000000..23d8ff6d --- /dev/null +++ b/backend/apps/agents/browser/browser_read_script.py @@ -0,0 +1,124 @@ +"""READ leg for AUTHED pages: prestage already landed the user's logged-in card on +the target page, so ONE aux call over the live page text can answer a read task and +the big-model loop never starts. The no-browser fast_read can't see behind logins; +this is the same answer-or-INSUFFICIENT contract driven through the real session. +Fail-open everywhere: thin page, decline, error = the loop runs exactly as today. +""" + +import asyncio +import logging +import os +import time +from typing import Awaitable, Callable, Dict, Optional + +from backend.apps.agents.browser.browser_prestage import RESULTS_URL_RE + +logger = logging.getLogger(__name__) + +ToolRunner = Callable[[str, Dict, str, str], Awaitable[Dict]] + +P_MIN_PAGE_CHARS = 500 +P_MAX_PAGE_CHARS = 24000 +P_TEXT_TIMEOUT_S = 8.0 +P_AUX_TIMEOUT_S = 12.0 +# Prestage's click often lands here while the SPA is still hydrating (measured: a +# LinkedIn profile read 184 chars right after the click); wait out the render, bounded. +P_THIN_SETTLE_S = 1.2 +# Crossing the char floor is NOT the same as being finished rendering: a hydrating SPA clears 500 +# chars on nav and footer chrome long before the content lands. Taking that first passing read hands +# the aux a half-drawn page, and it answers confidently from what IS there, so nothing declines and +# the INSUFFICIENT retry below never fires. A confident wrong answer is the one outcome worse than +# just running the loop, so the page has to prove it stopped growing: two reads in a row within this +# much of each other. Costs one extra read plus one short settle on a path that already takes ~7-10s. +P_STABLE_GROWTH = 0.05 +P_STABLE_SETTLE_S = 0.4 +MAX_READS = 4 +# A long-enough-but-still-rendering page reads as INSUFFICIENT (measured: profile +# passed 500 chars with the headline section missing); one settle + re-read + re-ask. +P_INSUFFICIENT_RETRIES = 1 +P_INSUFFICIENT_SETTLE_S = 1.5 + +P_SYSTEM = ( + "Answer the user's request using ONLY the page text provided. Be direct and " + "complete in a few sentences; quote exact titles/values from the page. End " + "with nothing else.\n" + "Reply with exactly the single word INSUFFICIENT only when the requested " + "information would live somewhere this page is not (a different page, behind " + "a click), so the caller should go get it. If THIS page is the right place " + "and it shows a value (even a placeholder) or visibly lacks the field, that " + "IS the answer: report exactly what the page shows. A joke, placeholder, or " + "obviously-fake value is still the answer, quoted, with a note that it looks " + "like a placeholder; never decline because a shown value looks unreal. Never " + "guess at anything the page doesn't show." +) + + +def read_script_enabled() -> bool: + return os.environ.get("OSW_READ_SCRIPT", "1") != "0" + + +def is_answer(reply: str) -> Optional[str]: + """The usable answer text, or None. Declines, empties, and hedge-shaped replies + all fail closed to the loop, so a thin extraction can never become a wrong answer.""" + answer = (reply or "").strip() + if not answer or answer.upper().startswith("INSUFFICIENT"): + return None + return answer + + +async def run_read_script( + aux_client, aux_model, task: str, browser_id: str, tab_id: str, + execute_tool: ToolRunner, current_url: str = "", +) -> Optional[str]: + """The answer to a read task from the staged page, or None (= run the loop). + Never raises; never acts on the page beyond reading it.""" + t0 = time.monotonic() + if aux_client is None or not aux_model: + return None + try: + from backend.apps.agents.core.aux_llm import safe_resp_text + + async def p_page_text() -> tuple: + """Page text, but only once two consecutive reads agree it has stopped growing.""" + prev = -1 + text, url = "", "" + for attempt in range(MAX_READS): + r = await asyncio.wait_for( + execute_tool("BrowserGetText", {}, browser_id, tab_id), timeout=P_TEXT_TIMEOUT_S) + text = str(r.get("text") or "") if isinstance(r, dict) and "error" not in r else "" + url = str(r.get("url") or "") if isinstance(r, dict) else "" + if len(text) >= P_MIN_PAGE_CHARS and 0 <= prev <= len(text) <= prev * (1 + P_STABLE_GROWTH): + return text, url + # Still thin waits longer than merely still-growing: one is a page that has not + # started, the other is one about to finish. + thin = len(text) < P_MIN_PAGE_CHARS + prev = len(text) + await asyncio.sleep(P_THIN_SETTLE_S if thin else P_STABLE_SETTLE_S) + return (text, url) if len(text) >= P_MIN_PAGE_CHARS else ("", "") + + for ask in range(1 + P_INSUFFICIENT_RETRIES): + page, p_live_url = await p_page_text() + if len(page) < P_MIN_PAGE_CHARS: + logger.info(f"[browser-readscript] page too thin ({len(page)} chars); loop runs") + return None + # On a results LIST the miss is structural (the answer lives one click deeper), not hydration; the settle-retry would just re-decline ~3s later. Judged on the LIVE url: the caller's is stale once plan-dispatch has clicked through (that staleness suppressed the retry on the exact page that needed it, measured). + p_retries = 0 if RESULTS_URL_RE.search(p_live_url or current_url or "") else P_INSUFFICIENT_RETRIES + reply = safe_resp_text(await asyncio.wait_for( + aux_client.messages.create( + model=aux_model, max_tokens=500, temperature=0, system=P_SYSTEM, + messages=[{"role": "user", "content": ( + f"Request: {task[:1200]}\n\nPage text:\n{page[:P_MAX_PAGE_CHARS]}")}], + ), timeout=P_AUX_TIMEOUT_S)) + ms = int((time.monotonic() - t0) * 1000) + answer = is_answer(reply) + if answer is not None: + logger.info(f"[browser-readscript] answered from the staged page in {ms}ms (ask {ask + 1})") + return answer + if ask < p_retries: + await asyncio.sleep(P_INSUFFICIENT_SETTLE_S) + logger.info(f"[browser-readscript] insufficient in {int((time.monotonic() - t0) * 1000)}ms; loop runs " + f"(page={len(page)}ch url={p_live_url[:80]!r} reply: {(reply or '')[:160]!r})") + return None + except Exception as e: + logger.info(f"[browser-readscript] skipped ({e})") + return None diff --git a/backend/apps/agents/browser/browser_schema.py b/backend/apps/agents/browser/browser_schema.py index a57ecc43..fc18f8d9 100644 --- a/backend/apps/agents/browser/browser_schema.py +++ b/backend/apps/agents/browser/browser_schema.py @@ -23,12 +23,6 @@ P_MERGE_VERIFY = ( "back 'NOT confirmed' or you forgot to pass one.\n" ) -MODEL_MAP = { - "sonnet": "claude-sonnet-4-6", - "opus": "claude-opus-4-6", - "haiku": "claude-haiku-4-5-20251001", -} - # The change an action should cause, declared by the agent and CONFIRMED after the action runs (success is observed, never assumed). A hit returns fast; a miss tells the agent it may not have worked instead of letting it claim a false success. P_EXPECT_DESC = { "type": "string", @@ -243,7 +237,13 @@ BROWSER_TOOLS_SCHEMA = [ }, { "name": "BrowserNavigate", - "description": "Navigate the browser to a URL.", + "description": ( + "Navigate the browser to a URL. Use a normal page URL a person would see. " + "Do NOT point it at a raw JSON/API endpoint (a /api/... or search-JSON URL " + "like Instagram's web/search/topsearch): that paints an unreadable data wall " + "in the card. To READ a site's own API, use BrowserReplayRoute, which fetches " + "the JSON silently without disturbing the page." + ), "input_schema": { "type": "object", "properties": { @@ -387,10 +387,55 @@ BROWSER_TOOLS_SCHEMA = [ "required": ["index"], }, }, + { + "name": "BrowserActVerified", + "description": ( + "Run a short SEQUENCE of dependent UI steps (2-4) in one call, where each " + "step must take effect before the next: open a menu then pick an item, " + "fill a field then the next one, expand a section then click inside it. " + "Each step names its target ELEMENT BY NAME (resolved fresh against the " + "live page at act time, so a stale index can't bite) and is VERIFIED in " + "code (did the expected change actually happen), with one automatic " + "re-aim on a miss. Steps:\n" + "- { action: 'click', target: '', role?: 'button'|'link'|..., " + "expect?: 'appeared:'|'gone:'|'url_changed'|'changed' }\n" + "- { action: 'fill', target: '', text: '' } " + "(auto-verifies the text committed)\n" + "Execution stops at the first step that can't be verified and you get " + "per-step results plus what went wrong. NEVER put an irreversible action " + "(send/submit/post/pay/delete/confirm) here; those stay SOLO clicks with " + "an `expect` proof, as always." + ), + "input_schema": { + "type": "object", + "properties": { + "steps": { + "type": "array", + "maxItems": 4, + "items": { + "type": "object", + "properties": { + "action": {"type": "string", "enum": ["click", "fill"]}, + "target": {"type": "string"}, + "role": {"type": "string"}, + "text": {"type": "string"}, + "expect": {"type": "string"}, + }, + "required": ["action", "target"], + }, + }, + }, + "required": ["steps"], + }, + }, { "name": "BrowserBatch", "description": ( - "Your standard way to ACT on the page. Every mutation (navigate, " + "Low-level action batch for INDEPENDENT actions (navigate, scroll, press) " + "or when you must act by index/point. For a sequence of DEPENDENT named " + "click/fill steps (open menu then pick item, fill then fill), prefer " + "BrowserActVerified: it self-verifies each step and re-aims in code. " + "Every mutation (navigate, " "click, type, press, scroll) is a sub-action in this array, 1-5 per " "call; an array of one is fine when that is genuinely all you know. " "Each sub-action executes in order with the URL captured before/after. " @@ -562,6 +607,37 @@ BROWSER_TOOLS_SCHEMA = [ "required": ["url"], }, }, + { + "name": "BrowserApiWrite", + "description": ( + "Perform a WRITE on the CURRENT site through the site's OWN API using your " + "already-logged-in session, instead of clicking the UI. This is deterministic " + "and far more reliable: no captcha, no button to miss, no selector to drift, " + "and it hands back the site's REAL receipt (the new post/comment's id and " + "permalink) as proof it landed. Only some sites are supported so far " + "(currently Reddit: comment, reply, post, edit, delete). If the current site " + "has no built-in adapter, you can still do it the GENERAL way: set action='route' " + "with the site's own write endpoint (method + url + body) taken from BrowserListRoutes, " + "and it replays that request with your session. If neither works you get a clean miss, " + "just do the write through the UI instead. This IS a real write: call it ONCE, and the " + "receipt is your confirmation, do not re-check or re-fire it." + ), + "input_schema": { + "type": "object", + "properties": { + "action": {"type": "string", "description": "comment, reply, post, edit, delete (built-in adapter), or route (general: replay a captured write endpoint)."}, + "parent_id": {"type": "string", "description": "comment/reply: fullname of the post/comment you're replying to (e.g. t3_abc, t1_xyz)."}, + "thing_id": {"type": "string", "description": "edit/delete: fullname of your OWN post/comment (e.g. t1_xyz)."}, + "text": {"type": "string", "description": "The body text (comment/reply/post/edit)."}, + "subreddit": {"type": "string", "description": "post: the subreddit name, without the r/ prefix."}, + "title": {"type": "string", "description": "post: the post title."}, + "url": {"type": "string", "description": "post: a link URL; OR route: the write endpoint's full URL from BrowserListRoutes."}, + "method": {"type": "string", "description": "route: the endpoint's HTTP method (POST, PUT, PATCH, DELETE)."}, + "body": {"type": "object", "description": "route: the JSON body to send, matching the endpoint's captured shape, with YOUR content in the text field(s)."}, + }, + "required": ["action"], + }, + }, { "name": "BrowserRepeatFlow", "description": ( @@ -680,6 +756,32 @@ BROWSER_TOOLS_SCHEMA = [ "required": ["problem", "instruction"], }, }, + { + "name": "BrowserDeleteItem", + "description": ( + "Remove ONE item visible on the CURRENT page, named by a distinctive snippet of its " + "own text. Navigate to where the item lives FIRST (your profile, the thread) so it is " + "on screen, then call this. It opens that item's overflow / 'More' menu, clicks " + "Delete/Remove, confirms, and verifies the item is gone. Use this instead of clicking " + "the '...' menu yourself, that menu is small and lazy-rendered and hand-clicking it is " + "unreliable. Only your OWN items expose a Delete option. Returns whether it was " + "verifiably removed; if it says the item isn't on the page, navigate to where it lives " + "and retry." + ), + "input_schema": { + "type": "object", + "properties": { + "target_text": { + "type": "string", + "description": ( + "A distinctive exact snippet of the target item's own text, long enough " + "to match only that one item (a phrase or id from the post/comment)." + ), + }, + }, + "required": ["target_text"], + }, + }, ] # Schema-forced batching: the model ignored every prompt-level batching invitation (0 adoptions across 8 measured runs), so the single-step mutating tools are not offered to it at all; acting means a BrowserBatch array, and the one deliberate solo path is BrowserClickIndex (irreversible step with expect, or a text-box fill). Executors and replay still support everything. @@ -707,6 +809,8 @@ ACTION_MAP = { "BrowserReplayRoute": "replay_route", # Internal replay primitive (skill replay calls it directly; not in the LLM-facing schema). Re-resolves a click target by role+name. "BrowserClickByName": "click_by_name", + # Internal: structural composer finder (send-script fallback; not LLM-facing). Ranks editable regions, marks the winner, optionally fills+verifies in-page. + "BrowserFindComposer": "find_composer", } # --- App agent: driving an OpenSwarm-built app via its native bridge --------- @@ -1119,5 +1223,6 @@ ACTION_TOOLS_REQUIRING_REPORT = { "BrowserClickIndex", # Phase 3 "BrowserClickPoint", # app mode: tap a canvas/game at a screen point "BrowserBatch", # Phase 4 + "BrowserActVerified", # verified-step sequence (mutates state like a batch) "AppInvoke", # app mode: invoking an app action mutates state } diff --git a/backend/apps/agents/browser/browser_send_parse.py b/backend/apps/agents/browser/browser_send_parse.py new file mode 100644 index 00000000..723dbba5 --- /dev/null +++ b/backend/apps/agents/browser/browser_send_parse.py @@ -0,0 +1,188 @@ +"""Pure perception-parsing for the staged send: read the browser's interactives listing + the +user's task and answer the structural questions the send orchestration needs, with no I/O and no +side effects. What quoted payload did the user name? Which listed row is the compose box / the +opener? Is this a login wall or a read-only request the script must decline? One host-agnostic +shape per question, so the same logic generalizes across X/Reddit/LinkedIn/Gmail/Slack/etc. + +Lives BELOW browser_send_script (which orchestrates the fill/click/verify tail): send_script +imports from here, never the reverse. +""" + +import re + +# Double quotes are unambiguous. Single quotes only delimit when the opener is at a word boundary (start/space/colon), so an in-word apostrophe like "chen's" is never mistaken for a payload quote, that mispairing was silently corrupting the canonical "text him '...'" errand. +P_QUOTED_DQ_RE = re.compile(r'"([^"]{4,300})"') +P_QUOTED_SQ_RE = re.compile(r"(?:^|[\s:>])'([^']{4,300})'") +P_COMPOSER_ROW_RE = re.compile(r"\[(\d+)\]\*?<\s*textbox\s+\"([^\"]*)\"", re.I) +# A compose-shaped textbox name, generalized across messaging sites: LinkedIn "Write a +# message", X/Slack "Message", Discord "Message @user", Gmail "Message Body", "Post your +# reply", "What's happening", "Add a comment". Not per-site: one structural shape. +# "text editor" earns its place from a measurement, not a guess: LinkedIn's post box is named +# "Text editor for creating content" and its comment box "Text editor for creating comment", so +# without it the real composer was invisible while the comment box next to it matched on "comment". +# Landing on LinkedIn's own compose surface listed exactly one textbox and we still scored zero. +# Both shapes match now, and telling them apart is surface_mismatch's job, which already does it. +P_COMPOSER_NAME_RE = re.compile( + r"write|messag|compose|reply|comment|post your|post text|what.?s happening|" + r"tweet|caption|say something|start a|new message|body|your (message|note)|" + r"add a comment|write something|text editor|creating content", + re.I, +) + +# Login/auth walls: a logged-out card lands here, and the structural reveal-finder would +# otherwise fill a login field and arm the page's own submit as a "send" (measured live on +# instagram/threads). A real composer never lives on one of these, so decline outright. +P_LOGIN_WALL_URL_RE = re.compile( + r"accounts\.google\.com|/i/flow/login|/accounts/login|/uas/login|/users/sign_in|" + r"/sessions/new|/checkpoint|force_authentication|" + r"/(?:log[_-]?in|sign[_-]?in|signin|logon)(?:[/?#]|$)", + re.I, +) +P_LOGIN_WALL_STATE_RE = re.compile( + r'<\s*textbox\s+"[^"]*(?:password|passwd)|(?:log|sign)\s?in to |' + r"continue with (?:google|apple|facebook)", + re.I, +) + +P_OPENER_ROW_RE = re.compile( + r"\[(\d+)\]\*?<\s*(?:link|button)\s+\"(Message|Reply|Compose|New message|" + r"Direct message|DM|Send message|Write|New chat|Comment|Post)\"", re.I) + +# A verification probe quotes the very payload it's checking for, which is exactly the trap this gate exists for: quoted payload + composer = fire. Caught live (r243): the read-only send-probe delivered a REAL message. Read-only directives decline in code, fail-safe (a false match just means the model path). +P_READONLY_RE = re.compile( + r"read.?only|do\s+not\s+(?:send|type|click|post|submit|change|edit|delete)|" + r"don'?t\s+(?:send|post|submit|change|edit|delete)|" + # "verify/check/tell me/say/confirm WHETHER x is there" is the whole family, not two phrasings + # of it. Measured: "say whether anything containing is still there. Change + # nothing." slipped through and POSTED the quoted text to a real LinkedIn feed, because only + # "verify whether" and "check whether" were listed. Anchor on the question shape. + r"(?:verify|check|confirm|tell\s+me|say|see|find\s+out|look)\s+(?:me\s+)?(?:if|whether)|" + r"is\s+(?:it|there|this|that)\s+(?:still\s+)?(?:there|published|posted|live|present)|" + r"still\s+(?:there|published|posted|live|up)|" + r"change\s+nothing|without\s+(?:sending|posting|changing)|verification", + re.I, +) + + +def looks_like_login_wall(current_url: str, state_text: str) -> bool: + """A login/auth page (by URL) or an auth form in the perception (a password field, a + 'Log in to X' heading, an OAuth 'Continue with ...'). The scripted send declines here: + a real composer never shares a page with these, and filling here types a login field.""" + if current_url and P_LOGIN_WALL_URL_RE.search(current_url): + return True + return bool(state_text and P_LOGIN_WALL_STATE_RE.search(state_text)) + + +# SOFT signed-out: the site serves a browsable page with no auth form and no login URL, it just +# withholds the composer and offers a "Sign in" control (bsky, stackoverflow, tiktok, threads all +# behave this way). The hard-wall gate above sees nothing, so the run used to report "I couldn't +# find the compose box" when the truth was "you are not signed in", which is a different problem +# with a different fix. Only ever consulted AFTER a composer miss, so it cannot affect a success. +P_SIGNIN_AFFORDANCE_RE = re.compile( + r'<\s*(?:link|button)\s+"[^"]*(?:sign[_ -]?in|log[_ -]?in|sign[_ -]?up|create account|join now)', + re.I) +# Anything only a signed-IN page shows. Its presence vetoes the verdict, so a stray "Log in" on an +# authenticated page (a second product's promo) can't make us tell the user to sign in again. +# Deliberately NARROW: an earlier draft also vetoed on "notifications"/"profile"/"inbox", which +# logged-OUT pages advertise all the time, and that silently suppressed the whole detector on the +# exact sites it exists for (measured: bsky with 0 cookies read as signed-in). Only a control that +# is meaningless unless you are already authenticated belongs here. +P_SIGNED_IN_RE = re.compile( + r'(?:sign|log)[_ -]?out\b|your profile|account menu|my account', + re.I) + + +def looks_signed_out(state_text: str) -> bool: + """True when the page offers a way to sign IN and shows nothing only a signed-in user sees.""" + if not state_text: + return False + if P_SIGNED_IN_RE.search(state_text): + return False + return bool(P_SIGNIN_AFFORDANCE_RE.search(state_text)) + + +# Creating a POST and commenting on someone else's are different actions on different content. +# LinkedIn's feed carries a comment box on EVERY post, and the capped interactives listing routinely +# starves the real post modal of its own composer, so the only compose-shaped textbox left in the +# list is a stranger's comment box. Filling that is not a slower path to the same place, it is the +# wrong action on the wrong person's content. Measured in a dry-run sweep: linkedin reached its +# composer 1/4, and two of the three misses targeted 'Text editor for creating comment'. +P_POST_INTENT_RE = re.compile(r"\b(post|tweet|publish|share)\b", re.I) +P_COMMENT_INTENT_RE = re.compile(r"\b(comment|reply|respond)\b", re.I) +P_COMMENT_SURFACE_RE = re.compile(r"\b(comment|reply)\b", re.I) + + +def surface_mismatch(task: str, composer_name: str) -> bool: + """True when the task asks to create a POST but the composer found is a comment/reply box. + + Deliberately one-directional: a task that mentions commenting is left alone, so this can only + ever reject a comment box for a post task, never the reverse. A rejection is cheap (the + structural finder, which does find LinkedIn's real composer, gets its turn instead).""" + t, name = task or "", composer_name or "" + if not P_POST_INTENT_RE.search(t) or P_COMMENT_INTENT_RE.search(t): + return False + return bool(P_COMMENT_SURFACE_RE.search(name)) + + +def is_readonly(text: str) -> bool: + """A read-only directive ('verify whether', 'do not send') that must decline the scripted + send even with a quoted payload in hand. Keeps the regex private to this file.""" + return bool(text and P_READONLY_RE.search(text)) + + +def quoted_payload(task: str) -> str: + """The exact text the user quoted, only when it's unambiguous: exactly one + distinct quoted span in the task. Anything else is the model's judgment call. + Double quotes win outright; single quotes must be word-boundary-delimited so + an apostrophe inside a name can't hijack the match.""" + dq = {m.group(1).strip() for m in P_QUOTED_DQ_RE.finditer(task or "") if m.group(1).strip()} + if dq: + return dq.pop() if len(dq) == 1 else "" + sq = {m.group(1).strip() for m in P_QUOTED_SQ_RE.finditer(task or "") if m.group(1).strip()} + return sq.pop() if len(sq) == 1 else "" + + +def opener_index_in_state(state_text: str): + """(index, name) of the single exact-named composer OPENER, or None. Exact + names only, so an upsell like 'Send InMail' can never match.""" + hits = [(int(m.group(1)), m.group(2)) for m in P_OPENER_ROW_RE.finditer(state_text or "")] + return hits[0] if len(hits) == 1 else None + + +def composer_index_in_state(state_text: str): + """(index, name) of the single compose-shaped textbox, or None. Two + candidates = ambiguous = model's problem.""" + hits = [(int(m.group(1)), m.group(2)) for m in P_COMPOSER_ROW_RE.finditer(state_text or "") + if P_COMPOSER_NAME_RE.search(m.group(2) or "")] + return hits[0] if len(hits) == 1 else None + + +def textbox_count(state_text: str) -> int: + """How many textboxes the perception listed, compose-shaped or not. + + Diagnostic only, and only meaningful next to a failed composer pick: zero means the page never + mounted one, several means the picker refused an ambiguous choice. Those are different bugs.""" + return len(P_COMPOSER_ROW_RE.findall(state_text or "")) + + +def surface_supports_script(current_url: str, state_text: str = "") -> bool: + """STRUCTURAL, not per-site: fire wherever the live perception actually carries a + person-composer (a compose-shaped textbox) OR a single messaging opener to reach + one, on ANY host. This is what generalizes the LinkedIn ~14s send to X/Slack/ + Discord/Instagram/Gmail/etc without per-site URL gates. A page with neither + declines (net-negative to fire where there's no composer). All the downstream + safety gates (quoted payload, fill-seen-committed before the one send, two-sided + receipt) are already site-agnostic, so widening the surface can't loosen safety.""" + if not state_text: + return False + return bool(composer_index_in_state(state_text) or opener_index_in_state(state_text)) + + +def dryrun_report(state_text: str, armed: bool, filled: bool, url: str = "") -> str: + """One grep-stable line for the coverage harness: what the staged perception held + and how far the script got. Only ever emitted in dry-run measurement mode.""" + boxes = len(P_COMPOSER_ROW_RE.findall(state_text or "")) + return (f"[dryrun-report] armed={int(bool(armed))} " + f"composer={int(bool(composer_index_in_state(state_text or '')))} " + f"opener={int(bool(opener_index_in_state(state_text or '')))} " + f"textboxes={boxes} filled={int(bool(filled))} url={(url or '')[:120]}") diff --git a/backend/apps/agents/browser/browser_send_script.py b/backend/apps/agents/browser/browser_send_script.py new file mode 100644 index 00000000..e00532b6 --- /dev/null +++ b/backend/apps/agents/browser/browser_send_script.py @@ -0,0 +1,431 @@ +""" +Staged-send script: when the pre-stage leaves a READY composer (a compose +textbox and a real Send button both visible) and the task names its payload in +quotes, code performs the fill/verify/send/verify tail the model otherwise +spends 4-5 turns (~15s) on. + +Safety is the same bar as the loop's, enforced in code: the payload must be +SEEN committed to the textbox before the one irreversible click, the Send +button is re-resolved from fresh state after the fill (indices shift), and the +composer must be SEEN cleared after. Any ambiguity BEFORE the click aborts to +the untouched model path; ambiguity AFTER the click hands the model a truthful +"clicked, unverified, do NOT re-send" note, never a silent retry. +""" + +import asyncio +import logging +import os +import time +from typing import Awaitable, Callable, Dict + +from backend.apps.agents.browser import ( + browser_delivery_check, browser_fast_path, browser_send_parse, browser_submit_click, + browser_verified_action) + +logger = logging.getLogger(__name__) + + +ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]] + +# The worst case this routine can legitimately take, so the CALLER cannot starve it. Roughly: the +# composer poll (3 backoff waits plus three 6s interactive lists), one structural finder call (which +# carries its own 30s command timeout), then fill-and-commit and submit-and-receipt. +# +# This constant exists because the caller's timeout and this routine's real cost silently drifted +# apart: raising find_composer's command timeout from 15s to 30s made a single finder call able to +# eat the caller's entire 30s budget, so the whole script was killed mid-send and EVERY write fell +# back to the slow model loop. It failed invisibly, because asyncio.TimeoutError stringifies to +# nothing and the log read "outer skip ()". Measured live on LinkedIn: a 190.9s write that never +# posted. Import this instead of writing a number at the call site. +WORST_CASE_BUDGET_S = 75.0 + + +def script_enabled() -> bool: + """Default ON. Every gate below it fails CLOSED to the old model loop, so the worst case is + today's behaviour, never a wrong send: the payload must be unambiguously quoted, the task must + not read as a question, the surface must not be a login wall, a post task will not settle for a + comment box, the fill must be seen committed, and delivery needs a two-sided receipt. + Turning this off also disables the prestage opener and the mid-loop autosend takeover.""" + return os.environ.get("OSW_SEND_SCRIPT", "1") != "0" + + +def autosend_enabled() -> bool: + """The mid-loop post-fill takeover: after the MODEL types the message into a composer, the code + finishes the send (find Send, click, verify receipt) instead of the model burning ~3-4 turns on + a Send button whose index goes stale after the fill. Rides with the send-script family (same + tail + safety), with its own kill switch.""" + return script_enabled() and os.environ.get("OSW_AUTOSEND", "1") != "0" + + +async def complete_send( + payload: str, state_committed: str, browser_id: str, tab_id: str, + execute_tool: ToolRunner, send_index_in_state: Callable[[str, int], object], + composer_index: int = -1, current_url: str = "", +) -> Dict[str, object]: + """Send tail for a composer that ALREADY holds `payload` (visible in state_committed): find the + Send control (ranked index first, else click-by-name over the full DOM), click it once, and + verify the two-sided receipt (the composer cleared the payload). Returns {clicked, sent, log, + note}: `clicked` = the send click landed, `sent` = the clear was verified. Never types, so it + can't fabricate content; a wrong Send match just fails the receipt, never a false claim. Shared + by the dispatch send-script and the mid-loop post-fill takeover.""" + log: list = [] + + async def fresh_list() -> str: + try: + r = await asyncio.wait_for( + execute_tool("BrowserListInteractives", {}, browser_id, tab_id), timeout=6.0) + return str(r.get("text") or "") if isinstance(r, dict) and "error" not in r else "" + except Exception: + return "" + + send_btn = send_index_in_state(state_committed, composer_index) + via = "index" + if send_btn: + r_send = await execute_tool("BrowserClickIndex", {"index": send_btn[0]}, browser_id, tab_id) + send_name = send_btn[1] + else: + # No submit listed below the composer (the capped listing can starve a modal of its own + # button): resolve the submit inside the composer's OWN container and click it with REAL + # input (synthetic clicks are ignored by web-component sites), then last-resort by-name. + r_ev = await execute_tool( + "BrowserEvaluate", + {"expression": browser_submit_click.container_submit_expression(payload)}, browser_id, tab_id) + p_v = browser_submit_click.parse_eval_value(r_ev) + if isinstance(p_v, dict) and p_v.get("ok") and p_v.get("xPct") is not None: + r_send = await execute_tool( + "BrowserClickPoint", + {"xPercent": float(p_v["xPct"]), "yPercent": float(p_v["yPct"])}, browser_id, tab_id) + send_name = str(p_v.get("name") or "submit") + via = "container" + else: + p_why = p_v.get("why") if isinstance(p_v, dict) else "unreadable eval" + logger.info(f"[browser-sendscript] container submit miss ({p_why}); by-name fallback") + r_send = await execute_tool("BrowserClickByName", {"name": "Send", "role": "button"}, browser_id, tab_id) + send_name = "Send (by-name)" + via = "by-name" + clicked = isinstance(r_send, dict) and "error" not in r_send + log.append({"tool": "send click", "input": {"via": via}, + "ok": clicked, "result_summary": f"send click {send_name!r}"[:200], + "elapsed_ms": 0, "clicked_role": "button", "clicked_name": send_name}) + if not clicked: + return {"clicked": False, "sent": False, "log": log, "note": "send click errored; fill committed, NOT sent"} + sent = False + # Name WHY a receipt fails. A withheld receipt costs the whole fast path (measured on LinkedIn: + # the script finished in 9.7s, the receipt missed, and the model then spent 28.6s re-verifying a + # post that HAD landed, 60s total against ~24s when the receipt passes), and "sent_receipt=False" + # alone cannot tell you whether the composer still holds the text or we simply could not read the + # page. Those are different bugs with different fixes. + p_why = "no-poll" + for wait_s in (0.4, 1.0, 1.6): + await asyncio.sleep(wait_s) + state3 = await fresh_list() + if not state3: + p_why = "unreadable-list" + continue + if browser_verified_action.expectation_met(f"cleared:{payload}", state_committed, state3): + sent = True + break + p_why = f"payload-still-in-a-textbox (textbox rows={sum(1 for x in state3.splitlines() if ' dict | None: + """None = stage not script-ready or aborted pre-click (model path, stage + untouched except a possibly committed fill, which the model sees). A dict + means the irreversible click RAN: {'sent': bool_receipt_verified, + 'payload': str, 'log': [...], 'note': str}. payload_source is the RAW user + prompt; the composed task carries the routing brief whose own quoted strings + made every real payload look ambiguous (r242/r243).""" + t0 = time.monotonic() + p_struct = os.environ.get("OSW_COMPOSER_STRUCT") == "1" + + async def fresh_list() -> str: + try: + r = await asyncio.wait_for( + execute_tool("BrowserListInteractives", {}, browser_id, tab_id), timeout=6.0) + return str(r.get("text") or "") if isinstance(r, dict) and "error" not in r else "" + except Exception: + return "" + + # The name-based surface gate can't see an unnamed/non-standard composer; under the + # structural flag, don't early-decline on it, the in-page finder gets a chance below. + if not browser_send_parse.surface_supports_script(current_url, state_text) and not p_struct: + # The composer lazy-renders a beat after prestage snapshotted (X home does this ~half the + # time), so poll a fresh perception before declining, else a late box is a false "no + # composer" and the whole write flakes to the slow model path. + for wait_s in (0.6, 1.0, 1.4): + await asyncio.sleep(wait_s) + fresh = await fresh_list() + if browser_send_parse.surface_supports_script(current_url, fresh): + state_text = fresh + break + else: + logger.info(f"[browser-sendscript] decline: no composer or opener after poll ({current_url[:50]!r})") + return None + # Key read-only on words a HUMAN wrote: the task minus the aux routing brief (the brief wrote + # "do not submit it" for a plain "start a post", falsely read-only-flagging a real send) PLUS + # the raw prompt when threaded through. The task text itself must keep declining regardless: a + # read-only VERIFY probe arrives as the task, and one once delivered a real message (r243). + task_sans_brief = task.split(browser_fast_path.BRIEF_MARKER, 1)[0] + if browser_send_parse.is_readonly(task_sans_brief) or (payload_source and browser_send_parse.is_readonly(payload_source)): + logger.info("[browser-sendscript] decline: read-only directive in user request") + return None + if browser_send_parse.looks_like_login_wall(current_url, state_text): + logger.info(f"[browser-sendscript] decline: login/auth wall ({(current_url or '')[:60]!r})") + return None + payload = browser_send_parse.quoted_payload(payload_source or task) + if not payload: + logger.info("[browser-sendscript] decline: no unambiguous quoted payload") + return None + log: list[dict] = [] + + composer = browser_send_parse.composer_index_in_state(state_text) + if composer and browser_send_parse.surface_mismatch(task_sans_brief, composer[1]): + # Asked to POST, found a COMMENT box: that is someone else's content, not a slower route to + # ours. Drop it and let the tiers below (opener, then the structural finder, which does find + # LinkedIn's real composer) look properly. + logger.info(f"[browser-sendscript] ignoring {composer[1]!r}: a comment box is not where a post goes") + composer = None + if not composer: + # The staged snapshot is prestage's, frozen the instant it clicked Message; the overlay composer lazy-renders a beat later (r263/r269 declined on exactly this, prestage's LAST step was the Message click). Poll a short window so the overlay has time to appear before we fall back to the opener. + for wait_s in (0.6, 1.2, 1.4): + await asyncio.sleep(wait_s) + fresh = await fresh_list() + composer = browser_send_parse.composer_index_in_state(fresh) + if composer: + state_text = fresh + break + p_struct_selector: str = "" + if not composer: + # Reversible-opener hop: prestage often stops on the profile with the "Message" opener visible (its settle raced the overlay). Opening a composer is the allowed opener class; the irreversible bar is unchanged. + opener = browser_send_parse.opener_index_in_state(state_text) + if opener and browser_send_parse.surface_mismatch(task_sans_brief, opener[1]): + # The same post-is-not-a-comment rule the composer already enforces, applied one step + # earlier. Measured on linkedin.com with the task "start a post": the only opener listed + # was 'Comment', so the script opened a stranger's comment box, found no post composer + # inside it, and declined. Opening the wrong surface is not a slower route to the right + # one, and here it also burns the reversible-opener hop we only get once. + logger.info(f"[browser-sendscript] ignoring opener {opener[1]!r}: a comment box is not " + f"where a post goes") + opener = None + if opener: + logger.info(f"[browser-sendscript] firing via opener {opener[1]!r} [{opener[0]}]") + r_open = await execute_tool("BrowserClickIndex", {"index": opener[0]}, browser_id, tab_id) + if not (isinstance(r_open, dict) and "error" not in r_open): + return None + log.append({"tool": "BrowserClickIndex", "input": {"index": opener[0]}, "ok": True, + "result_summary": f"script opened composer via {opener[1]!r}"[:200], "elapsed_ms": 0}) + # Wait for the surface to STOP MOVING, not for a number of seconds. Fixed budgets kept + # being wrong in both directions: 1.8s missed gmail and linkedin entirely, 5.3s still + # missed a cold gmail compose window that existed a beat later, and simply making the + # number bigger taxes every run that was never going to succeed. A mounting surface + # keeps changing the element list; once two consecutive reads are identical, nothing + # more is coming and more waiting is pure cost. + p_prev = "" + p_settled = 0 + for wait_s in (0.6, 1.2, 1.5, 2.0, 2.0, 2.0): + await asyncio.sleep(wait_s) + state_text = await fresh_list() + composer = browser_send_parse.composer_index_in_state(state_text) + if composer: + break + p_settled = p_settled + 1 if state_text and state_text == p_prev else 0 + p_prev = state_text + if p_settled >= 1: + logger.info("[browser-sendscript] opener surface settled with no composer; " + "not waiting out the rest of the budget") + break + # Structural fallback: the AX-name detector missed it (an unnamed contenteditable, a + # non-standard rich editor, or two textboxes it couldn't disambiguate). Ask the page to + # rank its editable regions and fill+read-back the winner IN-PAGE (the only reliable + # commit-check for a React contenteditable, whose text never reaches the AX value). + # Flag-gated so the proven name path stays the default. + if not composer and p_struct: + # OSW_COMPOSER_REVEAL: let the finder take one reversible reveal action (open the + # compose surface: a modal trigger, the first conversation, or a scroll) when the + # composer isn't painted yet. It never commits a send, only opens a surface. + p_reveal = os.environ.get("OSW_COMPOSER_REVEAL") == "1" + # A reveal that OPENS the first list item (a Reddit thread, a TikTok video, a GitHub + # issue) is a full-page NAVIGATION: it kills the finder's own JS context, so that one + # call can't reach the composer that only exists on the destination. When the finder + # reports it fired `open-first` but found nothing, the page is now loading the item; + # give it a beat and run the finder ONCE more on the destination. Bounded to 2 tries so + # a feed-of-feeds can't walk forever. + fc: Dict[str, object] = {} + for attempt in range(2): + fc = await execute_tool("BrowserFindComposer", {"fill": payload, "reveal": p_reveal}, browser_id, tab_id) + if isinstance(fc, dict) and fc.get("found") and fc.get("filled"): + break + revs = fc.get("reveals") if isinstance(fc, dict) else None + navigated = p_reveal and isinstance(revs, list) and "open-first" in revs + if not navigated: + break + logger.info("[browser-sendscript] reveal navigated (open-first); re-perceiving the destination") + await asyncio.sleep(1.5) + dest = await fresh_list() + # open-first can land on a login redirect (a logged-out feed's first item); + # stop before the NEXT fill so we never type into the auth form we just opened. + if browser_send_parse.looks_like_login_wall("", dest): + logger.info("[browser-sendscript] decline: reveal landed on a login/auth wall") + fc = {} + break + if isinstance(fc, dict) and fc.get("found") and fc.get("filled"): + p_struct_selector = str(fc.get("selector") or "") + logger.info(f"[browser-sendscript] structural composer role={fc.get('role')!r} " + f"score={fc.get('score')} nearSubmit={fc.get('nearSubmit')} " + f"reveals={fc.get('reveals')} fillMode={fc.get('fillMode')} filled+verified") + log.append({"tool": "BrowserFindComposer", "input": {"fill": ""}, "ok": True, + "result_summary": f"structural composer {fc.get('role')!r} filled+verified"[:200], "elapsed_ms": 0}) + composer = (-1, str(fc.get("role") or "composer")) + else: + logger.info(f"[browser-sendscript] structural finder: no usable composer ({str(fc)[:120]})") + if not composer: + # Name WHY. A site that withholds the composer because nobody is signed in is a + # different problem from one whose composer we failed to find, and only the first is + # fixable by the user (sign in once). Consulted only here, on the already-failed path. + if browser_send_parse.looks_signed_out(state_text): + logger.info("[browser-sendscript] decline: signed OUT (composer withheld, sign-in offered)") + else: + logger.info("[browser-sendscript] decline: no composer, opener, or structural editable") + return None + # No Send-button precondition: composer sites (LinkedIn) lazy-render Send only AFTER text commits, so it's resolved post-fill; never appearing = clean pre-click abort. + logger.info(f"[browser-sendscript] fill target {composer[1]!r} [{composer[0]}]") + + if p_struct_selector: + # The finder already filled + read-back-verified in-page; nothing to re-fill or re-check. + state2 = await fresh_list() + committed = True + else: + # 1. fill (focused by node, the composer overlay path coordinate clicks miss) + r_fill = await execute_tool("BrowserClickIndex", {"index": composer[0], "text": payload}, browser_id, tab_id) + fill_ok = isinstance(r_fill, dict) and "error" not in r_fill + log.append({"tool": "BrowserClickIndex", "input": {"index": composer[0], "text": payload}, + "ok": fill_ok, "result_summary": f"script fill into {composer[1]!r}"[:200], "elapsed_ms": 0}) + if not fill_ok and browser_submit_click.is_stale_index_error(r_fill): + # The opener click opens a modal that keeps re-rendering after we listed it, so the + # composer node we resolved is already detached by the time the fill lands. Measured + # live on x.com: 'Index 53 is not in the cached element map', on the exact run where + # the script had correctly found opener 'Post' and target 'Post text'. Re-listing is + # what the error itself prescribes, so take it once rather than surrendering a send + # the script had already located. One retry only: a second failure is a different + # problem and the model path is the right answer for it. + # Poll, don't snapshot. A single re-list catches the composer only if the modal happens + # to be settled at that instant; mid-churn it shows zero or two compose-shaped boxes, + # composer_index_in_state returns None (ambiguous), and the retry used to give up + # without a word. Measured: 5 successful retries in one arm, 0 in the next, same code, + # purely on timing. Same poll shape the opener path already uses. + composer_retry = None + for wait_s in (0.0, 0.5, 1.0): + if wait_s: + await asyncio.sleep(wait_s) + state_retry = await fresh_list() + composer_retry = browser_send_parse.composer_index_in_state(state_retry) + if composer_retry: + break + if not composer_retry: + logger.info("[browser-sendscript] composer index went stale and did not re-resolve " + "within 1.5s of polling; handing to model") + if composer_retry: + logger.info(f"[browser-sendscript] stale composer index {composer[0]}; refreshed to " + f"{composer_retry[0]} and retrying the fill once") + composer = composer_retry + r_fill = await execute_tool( + "BrowserClickIndex", {"index": composer[0], "text": payload}, browser_id, tab_id) + fill_ok = isinstance(r_fill, dict) and "error" not in r_fill + log.append({"tool": "BrowserClickIndex", + "input": {"index": composer[0], "text": payload}, "ok": fill_ok, + "result_summary": f"script fill retry into {composer[1]!r}"[:200], + "elapsed_ms": 0}) + if not fill_ok: + # Name the cause. "fill errored" alone cannot tell a stale index from a detached node + # from a site that refuses synthetic input, and those are three different fixes. Same + # lesson as the bare TimeoutError that used to log "outer skip ()". + p_err = r_fill.get("error") if isinstance(r_fill, dict) else type(r_fill).__name__ + logger.info(f"[browser-sendscript] fill errored ({str(p_err)[:160]}); " + f"handing to model untouched") + return None + # 2. verify the fill committed. Send is resolved AFTER, two ways: LinkedIn enables Send only once its JS digests the input (beats later than the text is visible), so the scan waits a little. + state2 = "" + committed = False + for wait_s in (0.4, 0.8, 1.2, 1.6): + await asyncio.sleep(wait_s) + state2 = await fresh_list() + committed = bool(state2 and payload_in_textbox(state2, payload)) + if committed: + break + if not committed: + logger.info("[browser-sendscript] fill not seen committed; aborting pre-click") + return None + # Dry-run probe: prove the script FIRES + fills on a NON-LinkedIn site without ever + # doing the outward send. Everything up to here ran (surface gate passed, composer + # found, fill committed); we stop before the irreversible click and report readiness. + if os.environ.get("OSW_SENDSCRIPT_DRYRUN") == "1": + send_ready = bool(send_index_in_state(state2, composer[0])) + logger.info(f"[browser-sendscript] DRYRUN: WOULD send (fill committed, send_button_listed={send_ready}); not clicking") + return {"sent": False, "payload": payload, "log": log, + "note": "DRYRUN: filled + ready to send, stopped before the irreversible click"} + # 3+4: the irreversible click + two-sided receipt, shared with the mid-loop takeover. A click error hands back to the model (fill committed, not sent); a clicked-but-unverified send returns sent=False so the caller never claims delivery. + r = await complete_send(payload, state2, browser_id, tab_id, execute_tool, send_index_in_state, + composer_index=composer[0], current_url=current_url) + log.extend(r["log"]) + if not r["clicked"]: + logger.info("[browser-sendscript] send click errored; handing to model (fill committed, NOT sent)") + return None + logger.info(f"[browser-sendscript] done sent_receipt={r['sent']} delivered={r.get('delivered')} in {int((time.monotonic() - t0) * 1000)}ms") + return {"sent": bool(r["sent"]), "delivered": r.get("delivered"), + "payload": payload, "log": log, "note": str(r["note"])} diff --git a/backend/apps/agents/browser/browser_session_import.py b/backend/apps/agents/browser/browser_session_import.py new file mode 100644 index 00000000..8a2fdb10 --- /dev/null +++ b/backend/apps/agents/browser/browser_session_import.py @@ -0,0 +1,162 @@ +"""Borrow the sign-in the user already has in their everyday browser, so a browser agent that hits +a login wall can carry on as them instead of stopping to ask them to log in all over again. + +The point is that no password is ever typed, stored, or seen. We copy the SESSION the user's real +Chrome/Arc/Brave/Edge already holds into the app's own browser partition. It is the same mechanism +onboarding uses to read the user's provider chat history, pointed at whatever site the agent is +stuck on instead of at a fixed provider list. + +Four things keep it narrow: + - Off unless the user turned it on (`browser_import_signins`, default False). Reading their real + browser is a decision they make once, explicitly, not one we make for them. + - The domain is never model-chosen. It comes from the URL of the page the agent is already stuck + on, so no amount of prompt injection can name a site to harvest. + - Records only ever travel INTO our own partition. Nothing is read back out. + - Values are never logged. Counts and domains only. + +Coverage is honestly partial: Chromium-family browsers on macOS/Windows, and not Chrome's newer +app-bound (v20) stores. Everything else returns `no_session` and the run falls back to asking the +user to sign in, which is exactly what it did before this existed. + +This is the ONE module in browser/ that knows where the reader lives, so the reader can move house +later without anything else noticing. +""" + +import asyncio +import logging +from typing import Any, Dict, List, Literal +from uuid import uuid4 + +from pydantic import BaseModel, ConfigDict +from typeguard import typechecked + +from backend.apps.agents.browser import browser_login_handoff +from backend.apps.agents.core.ws_manager import ws_manager +from backend.apps.onboarding.usage import browser_cookies +from backend.apps.settings.models import AppSettings + +logger = logging.getLogger(__name__) + +ImportOutcome = Literal["imported", "disabled", "no_session", "bridge_failed"] + +# Google authenticates on the parent SSO domain, so a Gmail/YouTube/Docs session does not live on +# the property's own host. The reader already has a named scope for exactly this, and we reuse it +# rather than sweeping every google entry the user owns. +P_GOOGLE_SUFFIXES = ("google.com", "youtube.com") + +# Chromium counts from 1601-01-01 in microseconds, because of course it does. Electron wants unix +# seconds, and an entry with no expiry is session-scoped, so it would evaporate on the next quit. +P_CHROMIUM_EPOCH_OFFSET_S = 11644473600 + +# Anti-bot clearance tokens are bound to the exact user agent and IP that earned them. Our webview +# keeps an "openswarm/" product token in its UA, so a clearance minted by the user's real Chrome can +# never match ours, and replaying a mismatched one reads as token theft: the edge hands back a fresh +# challenge instead of letting us through, which is WORSE than arriving with no clearance at all. +# Everything else in the jar is the actual session, so we carry that and let the edge re-challenge +# us honestly. +P_FINGERPRINT_BOUND: set = set() + + +class SessionImportResult(BaseModel): + """What happened, in a shape the caller can branch on without parsing prose.""" + + model_config = ConfigDict(validate_assignment=True) + + outcome: ImportOutcome = "no_session" + domain: str = "" + entries_applied: int = 0 + detail: str = "" + + @property + def ok(self) -> bool: + return self.outcome == "imported" + + +@typechecked +def is_enabled(settings: AppSettings) -> bool: + return bool(settings.browser_import_signins) + + +@typechecked +def is_google_property(domain: str) -> bool: + d = (domain or "").lower().lstrip(".") + return any(d == s or d.endswith(f".{s}") for s in P_GOOGLE_SUFFIXES) + + +@typechecked +def read_site_records(domain: str) -> List[Dict[str, Any]]: + """The user's own session records for `domain`. Blocking: touches SQLite and may raise one OS + keychain consent prompt, so callers must keep it off the event loop.""" + try: + if is_google_property(domain): + raw = browser_cookies.read_google_session_records() + else: + raw = browser_cookies.read_provider_cookie_records(domain) + except Exception as exc: + # A browser we cannot read is a fallback, never a crash: the run just asks the user instead. + logger.info(f"[session-import] read failed for {domain}: {type(exc).__name__}") + return [] + return [{**r, "expires": p_unix_expiry(r.get("expires_utc"))} for r in raw + if str(r.get("name") or "").lower() not in P_FINGERPRINT_BOUND] + + +@typechecked +def p_unix_expiry(expires_utc: Any) -> float: + """Chromium's stamp as unix seconds, 0.0 for a session-scoped entry (which Electron then leaves + session-scoped too, so it dies on quit exactly like it would in the source browser).""" + try: + raw = int(expires_utc or 0) + except (TypeError, ValueError): + return 0.0 + return max(0.0, raw / 1_000_000 - P_CHROMIUM_EPOCH_OFFSET_S) if raw > 0 else 0.0 + + +@typechecked +def site_domain(url_or_host: str) -> str: + """Normalise a URL or bare host to the registrable domain the store is keyed by. Delegates so + there is exactly one definition of 'which site is this' across the browser modules.""" + return browser_login_handoff.registrable_domain(url_or_host) + + +@typechecked +def has_importable_session(domain: str) -> bool: + """Whether some browser store holds a session for this domain, WITHOUT decrypting anything and + without touching the keychain. Cheap enough to ask before deciding to interrupt the user.""" + d = site_domain(domain) + if not d: + return False + try: + return browser_cookies.has_store(".google.com" if is_google_property(d) else d) + except Exception: + return False + + +@typechecked +async def import_signin(domain: str, browser_id: str) -> SessionImportResult: + """Copy the user's existing sign-in for `domain` into the app's browser partition. + + Never raises: every failure degrades to a result the caller can fall back from, because that + fallback (ask the user to sign in) is exactly the behaviour that existed before this did. + """ + d = site_domain(domain) + if not d: + return SessionImportResult(outcome="no_session", domain=domain, detail="no domain") + + records = await asyncio.to_thread(read_site_records, d) + if not records: + logger.info(f"[session-import] no readable session for {d}") + return SessionImportResult(outcome="no_session", domain=d, + detail="no session found in your other browsers") + + result = await ws_manager.send_browser_command( + uuid4().hex, "import_session", browser_id, {"domain": d, "cookies": records}) + if not isinstance(result, dict) or result.get("error"): + detail = str(result.get("error") if isinstance(result, dict) else result)[:200] + logger.info(f"[session-import] bridge failed for {d}: {detail}") + return SessionImportResult(outcome="bridge_failed", domain=d, detail=detail) + + count = int(result.get("set") or 0) + if count <= 0: + return SessionImportResult(outcome="no_session", domain=d, detail="nothing applied") + logger.info(f"[session-import] applied {count} entries for {d}") + return SessionImportResult(outcome="imported", domain=d, entries_applied=count) diff --git a/backend/apps/agents/browser/browser_skills.py b/backend/apps/agents/browser/browser_skills.py index 1b23fcce..73cf369d 100644 --- a/backend/apps/agents/browser/browser_skills.py +++ b/backend/apps/agents/browser/browser_skills.py @@ -330,6 +330,23 @@ def replay_settle_target(step: dict) -> str | None: return name if 0 < len(name) <= 60 else None +P_COMPOSER_STEP_RE = re.compile(r"write a message|compose|message body|comment|reply|tweet|post text|type here|editor", re.I) + + +def step_touches_composer(step: dict) -> bool: + """True if this step interacts with the compose box itself (focusing/typing), + as opposed to navigation or the opener click. The send-script owns the composer + (it polls for the lazy overlay), so the marriage replays only the nav+opener and + hands the composer->send tail to the script.""" + tool = step.get("tool", "") + p = step.get("params", {}) or {} + if tool == "BrowserType": + return True + if tool in ("BrowserClickByName", "BrowserClick"): + return bool(P_COMPOSER_STEP_RE.search(str(p.get("name") or p.get("selector") or ""))) + return False + + def first_unsafe_step(steps: list[dict]) -> tuple[int, str]: """Index of the first GENUINELY irreversible step (click Send/Submit/Pay, type into a composer), -1 if none. This is the prefix-replay/batch boundary, so a @@ -781,6 +798,23 @@ def hint_step_adopted(step_key: tuple, action_log: list[dict]) -> bool: return False +def replay_owns_nav(host: str, has_skill: bool, task_is_removal: bool, task_is_send: bool) -> bool: + """Should a learned skill's replay take over navigation, letting the caller skip prestage? + + Yes for a READ: the replayed prefix does the same navigation prestage would aux-drive, faster. + + No for a SEND, and this is the part that was wrong. Prestage is also what hands the send-script + its composer perception; skip it and the entire fill/click/receipt tail is unreachable, so the + model falls back to burning 4-5 turns. Measured live on x.com with a learned skill present: + 5/5 writes went the slow way at 41-146s (median ~57s) with the receipt never speaking, against + 19.4s with the script armed. Prestage on an already-loaded page costs ~2-5s. + + No for a removal either: a delete is a destructive one-shot, not a replayable nav prefix, so a + stale delete-"skill" made of scrolls must never hijack it. + """ + return bool(host and has_skill and not task_is_removal and not task_is_send) + + def mark_replay_succeeded(host: str, task: str) -> None: """A replay ran end to end. Count it and, if the skill was still on probation, PROMOTE it to trusted (the verify gate just passed).""" diff --git a/backend/apps/agents/browser/browser_submit_click.py b/backend/apps/agents/browser/browser_submit_click.py new file mode 100644 index 00000000..e01b95be --- /dev/null +++ b/backend/apps/agents/browser/browser_submit_click.py @@ -0,0 +1,117 @@ +"""Container-scoped submit click for the receipt-gated send path. Exists because the ranked +interactives listing caps at 60 rows and a composer's own submit can fall off it (X's compose +modal: covered feed rows behind the overlay ate the cap, so no "Post" row ever reached the index +picker, measured live 0/2 deliveries). Scope = the dialog/form ancestor of the editable holding +the payload when there is one, else a bounded nearest-scope-first upward walk, so a page-level +opener with the same label can never be chosen. A wrong resolution still fails the send receipt +downstream, never a false delivery claim.""" + +import json +import re +from typing import Any, Dict, Optional + +# BROAD submit vocabulary shared by the index picker (browser_agent) and the JS below, one source +# so the two tiers can never drift apart. +SEND_LABELS = frozenset({ + "send", "send now", "send message", # LinkedIn / Gmail / DMs + "post", "post all", "tweet", "reply", # X / Threads compose + reply + "publish", "comment", "share", # articles / YouTube+FB comments / shares +}) + +# Gmail names its Send button 'Send ‪(⌘Enter)‬': a shortcut suffix wrapped in bidi +# isolates that defeats exact matching. Strip control chars + any parenthesized tail before compare. +P_NAME_NOISE_RE = re.compile(r"[‪‬‎‏⁦-⁩]|\([^)]*\)") + + +def clean_button_name(name: str) -> str: + return P_NAME_NOISE_RE.sub("", name or "").strip().lower() + +# Resolves the submit and returns its viewport center; the caller clicks it through the REAL +# input path (BrowserClickPoint). Synthetic el.click() is ignored by web-component sites +# (shreddit live), and a real click lands on whatever is topmost, so overlays can't be fooled. +P_CONTAINER_SUBMIT_JS = r"""(() => { + const PAYLOAD = %s; + const LABELS = new Set(%s); + const norm = (s) => (s || '').replace(/\s+/g, ' ').trim().toLowerCase(); + const vis = (el) => !!el && el.getClientRects().length > 0; + const enabled = (el) => !el.disabled && el.getAttribute('aria-disabled') !== 'true'; + // Same cleaning as clean_button_name: Gmail's Send is 'Send (⌘Enter)' in bidi isolates. + const clean = (s) => norm((s || '').replace(/[‪‬‎‏⁦-⁩]|\([^)]*\)/g, '')); + const labelOf = (el) => clean(el.getAttribute('aria-label') || el.textContent || ''); + const holds = (el) => ((el.value || el.textContent || '').indexOf(PAYLOAD) !== -1); + // Shadow piercing both ways: reddit's composer AND its submit live in shreddit shadow roots. + const deep = (root, sel, out, depth) => { + if (depth > 10 || out.length > 4000) return out; + let hits; try { hits = root.querySelectorAll(sel); } catch (e) { hits = []; } + for (const el of hits) out.push(el); + let all; try { all = root.querySelectorAll('*'); } catch (e) { return out; } + for (const el of all) { if (el.shadowRoot) deep(el.shadowRoot, sel, out, depth + 1); } + return out; + }; + const up = (el) => el.parentElement || (el.getRootNode() && el.getRootNode().host) || null; + const ed = deep(document, '[contenteditable="true"],textarea,input', [], 0) + .find((e) => vis(e) && holds(e)); + if (!ed) return { ok: false, why: 'no editable holding the payload' }; + const submitIn = (root) => deep(root, 'button,[role="button"]', [], 0) + .find((b) => vis(b) && enabled(b) && LABELS.has(labelOf(b))); + const isScope = (el) => { try { return el.matches('[role="dialog"],[role="alertdialog"],form'); } catch (e) { return false; } }; + let scope = null; + for (let node = ed; node; node = up(node)) { if (isScope(node)) { scope = node; break; } } + let btn = null; + if (scope) { + btn = submitIn(scope); + } else { + // Nearest-scope-first walk: X's inline submit shares an ancestor 20 hops above the Draft.js + // editable while foreign tweets' buttons only enter at 28 (measured live), so 24 finds the + // composer's own submit and stops before any wider scope could. + let node = up(ed); + for (let hop = 0; node && node !== document.body && hop < 24; hop++, node = up(node)) { + btn = submitIn(node); + if (btn) break; + } + } + if (!btn) return { ok: false, why: 'no submit control in the composer container' }; + const r0 = btn.getBoundingClientRect(); + if (r0.top < 0 || r0.bottom > window.innerHeight) btn.scrollIntoView({ block: 'center' }); + const r = btn.getBoundingClientRect(); + return { ok: true, name: labelOf(btn), + xPct: (r.left + r.width / 2) / window.innerWidth * 100, + yPct: (r.top + r.height / 2) / window.innerHeight * 100 }; +})()""" + + +def container_submit_expression(payload: str) -> str: + """The container-scoped submit click for a composer holding `payload` (prefix-matched, same + 24-char truncation the fill verifier uses).""" + return P_CONTAINER_SUBMIT_JS % (json.dumps((payload or "")[:24]), json.dumps(sorted(SEND_LABELS))) + + +def parse_eval_value(res: object) -> Optional[Dict[str, Any]]: + """The dict a BrowserEvaluate returned, or None. Unreadable shapes are None (honest miss).""" + val: object = None + if isinstance(res, dict) and "error" not in res: + val = res.get("value") + if val is None and isinstance(res.get("text"), str): + try: + val = json.loads(res["text"]) + except (json.JSONDecodeError, ValueError): + val = None + return val if isinstance(val, dict) else None + + +# The browser tool's own words when an index no longer resolves. Anchored on the stable half of the +# sentence ("not in the cached element map"), not the whole string, so a reworded tail doesn't +# silently turn the retry off and take the fast write path down with it. +P_STALE_INDEX_RE = re.compile(r"not in the cached element map|refresh the index", re.I) + + +def is_stale_index_error(res: object) -> bool: + """Did this tool result fail because the element index went stale? + + Distinct from every other failure: a stale index means the element was THERE and the page + re-rendered underneath us, so re-listing and retrying is correct. A genuine miss (no such + control, refused input) must not retry, because retrying a real failure is how you double-post. + """ + if not isinstance(res, dict): + return False + return bool(P_STALE_INDEX_RE.search(str(res.get("error") or ""))) diff --git a/backend/apps/agents/browser/browser_trace.py b/backend/apps/agents/browser/browser_trace.py new file mode 100644 index 00000000..35b4c551 --- /dev/null +++ b/backend/apps/agents/browser/browser_trace.py @@ -0,0 +1,160 @@ +"""One auditable record of what the browser actually did, whatever tier did it. + +The user-facing promise is that a browser task is never "just trust me": the chat shows a Browser +Agent bubble you can expand to see the pages visited, what was clicked and typed, and the receipt +that proves a write landed. That promise held only on the sub-agent path, because the panel that +renders it reads from CHILD SESSIONS. The fast path creates no child session and closed its bubble +with a tool_result of literally "done", so on the tier that now handles most tasks the bubble +expanded to nothing at all. + +So the trace stops being a side effect of how the work was routed. Whichever tier ran builds the +same record here, and the bubble shows the same thing every time. + +Pure formatting: no I/O, no side effects, nothing that can fail a run. +""" + +import json +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict +from typeguard import typechecked + +# Enough to see what happened without turning the panel into a log file. A run that exceeds this +# says so rather than silently showing a prefix, because a trace you cannot trust to be complete is +# worse than no trace. +MAX_STEPS = 40 +MAX_ARG_CHARS = 90 + +# Tools whose arguments are the interesting part (where it went, what it typed) versus ones whose +# name already says everything (a screenshot is a screenshot). +P_ARG_KEYS = ("url", "text", "expression", "instruction", "target_text", "index", "name", "key") + + +class BrowserTrace(BaseModel): + """What to show under the bubble. Shaped so the renderer never parses prose.""" + + model_config = ConfigDict(validate_assignment=True) + + tier: str = "" # which path did the work, in plain words + pages: List[str] = [] # URLs actually visited, in order, deduped + steps: List[str] = [] # one line per action, already human-readable + steps_omitted: int = 0 + receipt: str = "" # the proof a write landed, when there was one + note: str = "" # anything the user should know about coverage + + +@typechecked +def p_arg_summary(inp: Any) -> str: + """The part of a tool's input worth showing, short enough to scan.""" + if not isinstance(inp, dict) or not inp: + return "" + for k in P_ARG_KEYS: + v = inp.get(k) + if v not in (None, "", []): + s = str(v).replace("\n", " ").strip() + return s[:MAX_ARG_CHARS] + ("..." if len(s) > MAX_ARG_CHARS else "") + s = json.dumps(inp)[:MAX_ARG_CHARS] + return s + + +@typechecked +def p_step_line(entry: Dict[str, Any]) -> str: + tool = str(entry.get("tool") or "?") + arg = p_arg_summary(entry.get("input")) + ms = entry.get("elapsed_ms") + ok = entry.get("ok") + tail = f" [{int(ms)}ms]" if isinstance(ms, (int, float)) and ms else "" + mark = "" if ok in (None, True) else " (failed)" + return f"{tool}({arg}){tail}{mark}" if arg else f"{tool}{tail}{mark}" + + +@typechecked +def p_pages_from(action_log: List[Dict[str, Any]]) -> List[str]: + """Every URL the run actually landed on, in order, without repeats. This is the spine of the + trace: it answers "where did it go" before "what did it do there".""" + out: List[str] = [] + for e in action_log: + inp = e.get("input") + url = str(inp.get("url") or "") if isinstance(inp, dict) else "" + if url.startswith(("http://", "https://")) and (not out or out[-1] != url): + out.append(url) + return out + + +@typechecked +def build_trace(tier: str, action_logs: List[List[Dict[str, Any]]], + receipt: str = "", note: str = "", entry_url: str = "") -> BrowserTrace: + """Fold every dispatch a run made into one record. Takes a LIST of logs because a fast-path run + can dispatch more than once (a recovery, a send probe) and the user should see all of it, not + just whichever attempt happened to be last. + + `entry_url` matters more than it looks: a cold run creates the card ALREADY pointed at its + target, so no BrowserNavigate is ever issued and harvesting URLs from the log alone leaves the + trace unable to answer "where did it go" at all.""" + merged: List[Dict[str, Any]] = [] + for log in action_logs: + merged.extend(e for e in (log or []) if isinstance(e, dict)) + pages = p_pages_from(merged) + if entry_url.startswith(("http://", "https://")) and entry_url not in pages[:1]: + pages = [entry_url] + pages + shown = merged[-MAX_STEPS:] + return BrowserTrace( + tier=tier, + pages=pages, + steps=[p_step_line(e) for e in shown], + steps_omitted=max(0, len(merged) - len(shown)), + receipt=receipt, + note=note, + ) + + +@typechecked +def trace_payload(trace: BrowserTrace) -> Dict[str, object]: + """The tool_result content the bubble renders. Kept as data rather than a rendered string so the + panel can lay it out, and so a future surface (an export, a report) does not have to re-parse + English.""" + return {"browser_trace": trace.model_dump(mode="json")} + + +@typechecked +def trace_text(trace: BrowserTrace) -> str: + """A plain-text fallback for anywhere that can only show a string.""" + lines: List[str] = [] + if trace.tier: + lines.append(f"Handled by: {trace.tier}") + if trace.pages: + lines.append("Pages: " + " -> ".join(trace.pages[:6])) + if trace.steps_omitted: + lines.append(f"... {trace.steps_omitted} earlier steps omitted ...") + lines.extend(f"{i}. {s}" for i, s in enumerate(trace.steps, trace.steps_omitted + 1)) + if trace.receipt: + lines.append(f"Verified: {trace.receipt}") + if trace.note: + lines.append(trace.note) + return "\n".join(lines) or "No browser actions were recorded." + + +@typechecked +def tier_label(fp_path: str, used_browser: bool) -> str: + """Plain words for the routing string the logs use, because 'read->browser' means nothing to + the person reading their own chat.""" + if not used_browser: + return "read the page directly, no browser needed" + if fp_path.startswith("read"): + return "opened the page in a browser and read it" + return "drove the browser" + + +@typechecked +def receipt_from(result: Optional[Dict[str, Any]]) -> str: + """The two-sided receipt, when the run produced one. This is the line that separates 'it says it + posted' from 'it posted', so it gets its own field rather than being buried in the steps.""" + if not isinstance(result, dict): + return "" + for key in ("receipt", "sent_receipt", "delivery"): + v = result.get(key) + if isinstance(v, str) and v.strip(): + return v.strip()[:300] + if v is True: + return "delivery confirmed on the page" + return "" diff --git a/backend/apps/agents/browser/browser_verified_action.py b/backend/apps/agents/browser/browser_verified_action.py new file mode 100644 index 00000000..276f9694 --- /dev/null +++ b/backend/apps/agents/browser/browser_verified_action.py @@ -0,0 +1,107 @@ +"""Generic, site-agnostic verification: did an action produce the SPECIFIC effect +it was meant to? This is the load-bearing piece that lets a verified-action executor +work on any website without per-site code, the model (or a scripted flow) names a +generic expectation, and this checks it against a cheap before/after page snapshot. + +A snapshot is just the interactives-list text plus the URL, the same things every +site exposes, so nothing here knows about LinkedIn or any particular page. It +generalizes the send-script's proven two-sided receipt ("the composer cleared") from +one hand-tuned flow into one predicate ("cleared:") reusable everywhere. + +Expectations (kind, or "kind:arg"): + url_changed the page navigated + changed the page changed at all (weakest; a fallback) + appeared:X X is present now but wasn't before (a menu/dialog/result opened) + gone:X X was present before but isn't now (an item/row deleted) + filled:X some textbox value now carries X (a fill committed) + cleared:X no textbox value carries X (a composer sent + emptied) +""" + +import re +from typing import List, Optional, Tuple + +# Match payload_in_textbox: long values truncate in the list, so compare on a prefix. +P_VALUE_PREFIX_LEN = 24 +P_TEXTBOX_LINE = "]<*>?< ""...>, the format every list uses. +P_ROW_RE = re.compile(r'\[(\d+)\]\*?<\s*([a-z]+)\s+"([^"]*)"', re.I) +P_NAME_PREFIX_LEN = 40 # long card-blob names mutate their suffix between visits + + +def parse_rows(state_text: str) -> List[Tuple[int, str, str]]: + """(index, role, name) for each interactive row. Site-agnostic: it reads the + universal list shape, not any particular page's elements.""" + return [(int(m.group(1)), m.group(2).lower(), m.group(3)) + for m in P_ROW_RE.finditer(state_text or "")] + + +def resolve_target(state_text: str, name: str, role: str = "") -> Optional[Tuple[int, str, str]]: + """Resolve a semantic target against the LIVE list, late, the moment before acting, + so a stale index can't bite. Strictest UNAMBIGUOUS tier wins: exact (role,name) -> + exact name -> name-prefix. Two matches at a tier = ambiguous = None (hand back + rather than click the wrong thing). Mirrors the renderer's click-by-name tiers.""" + want = (name or "").strip().lower() + if not want: + return None + wrole = (role or "").strip().lower() + rows = parse_rows(state_text) + + def uniq(cands: List[Tuple[int, str, str]]) -> Optional[Tuple[int, str, str]]: + return cands[0] if len(cands) == 1 else None + + hit = uniq([r for r in rows if r[2].strip().lower() == want and (not wrole or r[1] == wrole)]) + if hit: + return hit + hit = uniq([r for r in rows if r[2].strip().lower() == want]) + if hit: + return hit + pre = want[:P_NAME_PREFIX_LEN] + return uniq([r for r in rows if r[2].strip().lower().startswith(pre) and (not wrole or r[1] == wrole)]) + + +def parse_expectation(expect: str) -> Tuple[str, str]: + """(kind, arg) from 'kind' or 'kind:arg'. Unknown kinds are returned as-is and + treated as unmet by expectation_met, so a typo fails safe (verification withheld).""" + raw = (expect or "").strip() + if ":" in raw: + kind, arg = raw.split(":", 1) + return kind.strip().lower(), arg.strip() + return raw.lower(), "" + + +def value_present(state_text: str, sub: str) -> bool: + """True if any listed textbox VALUE carries sub (prefix match, like a committed + fill). Same logic as payload_in_textbox so the send-script stays behavior-identical.""" + probe = (sub or "")[:P_VALUE_PREFIX_LEN] + if not probe: + return False + return any(P_TEXTBOX_LINE in line and probe in line + for line in (state_text or "").splitlines()) + + +def p_contains(state_text: str, sub: str) -> bool: + s = (sub or "").strip().lower() + return bool(s) and s in (state_text or "").lower() + + +def expectation_met( + expect: str, before: str, after: str, + before_url: str = "", after_url: str = "", +) -> bool: + """Did `after` satisfy `expect` given `before`? Pure; unknown expectation = False + (fail safe: verification withheld rather than a false pass).""" + kind, arg = parse_expectation(expect) + if kind == "url_changed": + return bool(after_url) and after_url != before_url + if kind == "changed": + return before != after or (bool(after_url) and after_url != before_url) + if kind == "appeared": + return p_contains(after, arg) and not p_contains(before, arg) + if kind == "gone": + return p_contains(before, arg) and not p_contains(after, arg) + if kind == "filled": + return value_present(after, arg) + if kind == "cleared": + return not value_present(after, arg) + return False diff --git a/backend/apps/agents/browser/browser_verified_step.py b/backend/apps/agents/browser/browser_verified_step.py new file mode 100644 index 00000000..99b4f3f3 --- /dev/null +++ b/backend/apps/agents/browser/browser_verified_step.py @@ -0,0 +1,110 @@ +"""One verified action, the executor's unit of work: resolve the target LATE against +the live page, act, verify the SPECIFIC expected effect, and re-aim on a miss, all in +code, no LLM turn. This generalizes the send-script's proven fill->verify->send->verify +from one LinkedIn flow to any site: the target is a semantic name, the effect is a +generic expectation, and neither knows about any particular page. + +The one safety invariant, same bar as the send-script: an IRREVERSIBLE step (send / +submit / pay) is NEVER re-fired. If it acted but the effect can't be verified, it +returns an honest "acted, unverified, do NOT repeat" note instead of retrying, so a +receipt we couldn't read can never become a double-send. +""" + +import asyncio +import logging +from typing import Awaitable, Callable, Optional, Tuple + +from pydantic import BaseModel, ConfigDict + +from backend.apps.agents.browser import browser_verified_action as va + +logger = logging.getLogger(__name__) + +ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]] + + +class VerifiedStep(BaseModel): + model_config = ConfigDict(validate_assignment=True) + kind: str # "click" | "fill" + target: str # semantic element name to resolve against the live list + role: str = "" # optional role hint ("button", "link", "textbox") to disambiguate + text: str = "" # for a fill + expect: str = "" # generic expectation; defaults to filled: / changed + irreversible: bool = False # send/submit/pay: acted-but-unverified NEVER re-fires + chosen: bool = False # the planner PICKED this among similar rows: flag it for review before anything irreversible + + +async def p_fresh(execute_tool: ToolRunner, browser_id: str, tab_id: str) -> Tuple[str, str]: + try: + r = await asyncio.wait_for( + execute_tool("BrowserListInteractives", {}, browser_id, tab_id), timeout=6.0) + except Exception: + return "", "" + if not isinstance(r, dict) or "error" in r: + return "", "" + return str(r.get("text") or ""), str(r.get("url") or "") + + +async def p_act(step: VerifiedStep, index: Optional[int], + browser_id: str, tab_id: str, execute_tool: ToolRunner) -> dict: + if step.kind == "fill": + return await execute_tool( + "BrowserClickIndex", {"index": index, "text": step.text}, browser_id, tab_id) + if index is not None: + return await execute_tool("BrowserClickIndex", {"index": index}, browser_id, tab_id) + # a click whose index didn't resolve falls to by-name (full-DOM search, past the list cap) + return await execute_tool( + "BrowserClickByName", {"name": step.target, "role": step.role}, browser_id, tab_id) + + +def p_default_expect(step: VerifiedStep) -> str: + if step.expect: + return step.expect + return f"filled:{step.text}" if step.kind == "fill" else "changed" + + +async def run_verified_step( + step: VerifiedStep, browser_id: str, tab_id: str, execute_tool: ToolRunner, + settle_s: float = 0.8, max_reaim: int = 1, +) -> dict: + """{ok, verified, acted, note}. ok == the expected effect was observed. A reversible + step that doesn't verify is re-aimed (re-resolve + re-act) up to max_reaim times; an + irreversible one is never re-fired once it has acted.""" + expect = p_default_expect(step) + note = "" + for attempt in range(max_reaim + 1): + before, before_url = await p_fresh(execute_tool, browser_id, tab_id) + tgt = va.resolve_target(before, step.target, step.role) + index = tgt[0] if tgt else None + if step.kind == "fill" and index is None: + return {"ok": False, "verified": False, "acted": False, + "note": f"could not resolve a field named {step.target!r} to fill"} + r = await p_act(step, index, browser_id, tab_id, execute_tool) + acted = isinstance(r, dict) and "error" not in r + if not acted: + note = f"action errored: {r.get('error') if isinstance(r, dict) else r}" + if step.irreversible: + # an errored irreversible action provably did NOT happen; safe to stop, never retry blindly + return {"ok": False, "verified": False, "acted": False, "note": note} + continue # reversible: re-aim + # Smart-wait: check the expectation early and again at the full settle window; the common fast case exits ~0.5-0.8s sooner than a flat sleep, the slow case keeps its whole window. A settle too small to split keeps the single flat check. + p_waits = [0.4, settle_s - 0.4] if settle_s > 0.4 else [settle_s] + p_met = False + for p_wait in p_waits: + if p_wait: + await asyncio.sleep(p_wait) + after, after_url = await p_fresh(execute_tool, browser_id, tab_id) + if va.expectation_met(expect, before, after, before_url, after_url): + p_met = True + break + if p_met: + logger.info(f"[verified-step] {step.kind} {step.target!r} -> {expect} OK (attempt {attempt + 1})") + return {"ok": True, "verified": True, "acted": True, "note": ""} + if step.irreversible: + # acted, effect unverifiable: the send-script's honesty rule, never a blind repeat + return {"ok": False, "verified": False, "acted": True, + "note": (f"an irreversible {step.target!r} action already RAN but its effect is " + "unverified; verify on the page, do NOT repeat it unless verifiably absent")} + note = f"expected {expect!r} not observed after {step.kind} {step.target!r}" + logger.info(f"[verified-step] {step.kind} {step.target!r} unverified: {note}") + return {"ok": False, "verified": False, "acted": True, "note": note} diff --git a/backend/apps/agents/browser/browser_write_recipes.py b/backend/apps/agents/browser/browser_write_recipes.py new file mode 100644 index 00000000..ca2c8856 --- /dev/null +++ b/backend/apps/agents/browser/browser_write_recipes.py @@ -0,0 +1,232 @@ +"""Learn-on-first-write, replay-on-repeat: the repeated-write half of the API-first tier. + +The first write on a site is unavoidably a DOM drive (a route can only be captured after the +site's own UI fires it, proven live in the V.8 X soak). But the moment a DOM write SUCCEEDS with +a verified receipt, the mutating route the page fired is a complete recipe: method + URL (with +its live queryId) + body, with the user's payload sitting in one JSON leaf. This module persists +that recipe with the payload slot replaced by a sentinel, and replays it with a NEW payload on +the next write to the same site, skipping the DOM entirely. + +SAFETY (documented in SECURITY.md): +- A recipe is learned ONLY from a receipt-verified successful write the user's own task performed, + so its provenance satisfies the captured-route wall (the site's UI genuinely fired it); replay + seeds route_write's captured set from the recipe itself. +- Secret-shaped string leaves in the stored body are redacted at learn time (payload slot + excepted); cookies/headers are never stored (route_write borrows them live per call). +- Same-origin + OSW_ROUTE_WRITE flag + typed fail-open outcomes all still apply at replay. +- Staleness self-heals: a recipe that misses MAX_MISSES times is dropped, and the next + successful DOM write learns a fresh one (queryId rotation just re-learns). +""" + +import json +import logging +import os +import re +import time +from typing import Any, Dict, List, Optional + +from pydantic import BaseModel, ConfigDict +from typeguard import typechecked + +logger = logging.getLogger(__name__) + +SENTINEL = "__OSW_PAYLOAD__" +P_MIN_PAYLOAD_CHARS = 4 +P_MAX_BODY_CHARS = 32768 +MAX_MISSES = 3 +P_MAX_RECIPES_ON_DISK = 200 + +# Same secret heuristics as the electron capture (cdp-routes.js), ported so a token-shaped +# body value can never be persisted; over-redacting is the safe direction. +P_TOKEN_PREFIX = re.compile(r"^(sk-|ghp_|gho_|pk_|xox[bap]-|AIza|eyJ|Bearer )") + + +@typechecked +def looks_secret_value(v: str) -> bool: + if not v: + return False + if P_TOKEN_PREFIX.match(v): + return True + return len(v) >= 20 and bool(re.search(r"[A-Za-z]", v)) and bool(re.search(r"[0-9]", v)) and not re.search(r"\s", v) + + +class WriteRecipe(BaseModel): + """One site's proven write call, payload slot replaced by the sentinel.""" + + model_config = ConfigDict(validate_assignment=True) + + host: str + method: str + url_template: str + url: str + body_template: str + payload_path: str + learned_at: float + wins: int = 0 + misses: int = 0 + + +@typechecked +def p_dir() -> str: + from backend.config.paths import DATA_ROOT + d = os.path.join(DATA_ROOT, "browser_write_recipes") + os.makedirs(d, mode=0o700, exist_ok=True) + return d + + +@typechecked +def p_path(host: str) -> str: + safe = re.sub(r"[^a-z0-9.-]", "_", host.lower()) + return os.path.join(p_dir(), f"{safe}.json") + + +@typechecked +def recipe_for(host: str) -> Optional[WriteRecipe]: + """The persisted recipe for this host, or None. Corrupt files read as None (fail-open).""" + try: + with open(p_path(host)) as f: + return WriteRecipe(**json.load(f)) + except Exception: + return None + + +@typechecked +def save_recipe(recipe: WriteRecipe) -> None: + """Atomic write, browser_skills pattern; cap the directory so it can't grow unbounded.""" + try: + d = p_dir() + entries = sorted(os.listdir(d), key=lambda f: os.path.getmtime(os.path.join(d, f))) + while len(entries) >= P_MAX_RECIPES_ON_DISK: + os.remove(os.path.join(d, entries.pop(0))) + tmp = p_path(recipe.host) + ".tmp" + with open(tmp, "w") as f: + json.dump(recipe.model_dump(mode="json"), f) + os.replace(tmp, p_path(recipe.host)) + except Exception as e: + logger.info(f"[write-recipe] save failed for {recipe.host}: {e}") + + +@typechecked +def drop_recipe(host: str) -> None: + try: + os.remove(p_path(host)) + except OSError: + pass + + +@typechecked +def p_find_payload_leaf(obj: Any, payload: str, path: str = "$") -> Optional[str]: + """JSON path of the leaf whose string value EQUALS the payload (exact, not substring: + a substring hit means the site wrapped it and blind substitution would corrupt).""" + if isinstance(obj, str): + return path if obj == payload else None + if isinstance(obj, dict): + for k, v in obj.items(): + hit = p_find_payload_leaf(v, payload, f"{path}.{k}") + if hit: + return hit + return None + if isinstance(obj, list): + for i, v in enumerate(obj): + hit = p_find_payload_leaf(v, payload, f"{path}[{i}]") + if hit: + return hit + return None + + +@typechecked +def p_transform_leaves(obj: Any, payload: str) -> Any: + """Copy with the payload leaf swapped for the sentinel and secret-shaped strings redacted.""" + if isinstance(obj, str): + if obj == payload: + return SENTINEL + return "" if looks_secret_value(obj) else obj + if isinstance(obj, dict): + return {k: p_transform_leaves(v, payload) for k, v in obj.items()} + if isinstance(obj, list): + return [p_transform_leaves(v, payload) for v in obj] + return obj + + +@typechecked +def learn_recipe(host: str, payload: str, routes: List[Dict[str, Any]]) -> Optional[WriteRecipe]: + """Distill a recipe from the captured mutating routes of a JUST-verified write. Returns the + saved recipe, or None when no route's body carries the payload as an exact string leaf + (then there is nothing provably replayable, so nothing is stored).""" + if len(payload or "") < P_MIN_PAYLOAD_CHARS: + return None + for r in routes: + body = str(r.get("lastBody") or "") + method = str(r.get("method") or "").upper() + if not body or len(body) > P_MAX_BODY_CHARS or method in ("GET", "HEAD"): + continue + try: + parsed = json.loads(body) + except (json.JSONDecodeError, ValueError): + continue + slot = p_find_payload_leaf(parsed, payload) + if not slot: + continue + recipe = WriteRecipe( + host=host, method=method, + url_template=str(r.get("template") or ""), + url=str(r.get("example") or r.get("template") or ""), + body_template=json.dumps(p_transform_leaves(parsed, payload)), + payload_path=slot, learned_at=time.time(), + ) + save_recipe(recipe) + logger.info(f"[write-recipe] learned {host} {method} {recipe.url_template[:80]} slot={slot}") + return recipe + return None + + +@typechecked +def build_body(recipe: WriteRecipe, payload: str) -> Optional[Dict[str, Any]]: + """The recipe body with the NEW payload in the slot; None when the template holds no + sentinel (corrupt or hand-edited = do not replay) or redacted leaves the site requires.""" + if SENTINEL not in recipe.body_template: + return None + try: + parsed = json.loads(recipe.body_template) + except (json.JSONDecodeError, ValueError): + return None + + def p_sub(obj: Any) -> Any: + if isinstance(obj, str): + return payload if obj == SENTINEL else obj + if isinstance(obj, dict): + return {k: p_sub(v) for k, v in obj.items()} + if isinstance(obj, list): + return [p_sub(v) for v in obj] + return obj + + out = p_sub(parsed) + return out if isinstance(out, dict) else None + + +@typechecked +async def replay_recipe(recipe: WriteRecipe, payload: str, origin: str) -> Dict[str, Any]: + """Replay the recipe with a new payload via route_write (same-origin + flag + live-borrowed + cookies all enforced there). The recipe IS the captured provenance: it was learned from a + route the site's UI fired during a receipt-verified write, so it seeds the captured set. + Returns {ok, receipt|error}; a miss bumps the staleness counter and MAX_MISSES drops it.""" + from backend.apps.agents.browser import route_write + body = build_body(recipe, payload) + if body is None: + drop_recipe(recipe.host) + return {"ok": False, "error": "recipe template unusable; dropped"} + captured = [route_write.CapturedRoute(method=recipe.method, template=recipe.url_template)] + import asyncio + out = await asyncio.to_thread( + route_write.replay_write, recipe.method, recipe.url, body, origin, captured) + if out.ok: + recipe.wins += 1 + save_recipe(recipe) + return {"ok": True, "receipt": out.receipt, "latency_ms": out.latency_ms} + recipe.misses += 1 + if recipe.misses >= MAX_MISSES: + drop_recipe(recipe.host) + logger.info(f"[write-recipe] {recipe.host} dropped after {recipe.misses} misses (stale; next DOM win re-learns)") + else: + save_recipe(recipe) + return {"ok": False, "error": out.error} diff --git a/backend/apps/agents/browser/compose_discovery.py b/backend/apps/agents/browser/compose_discovery.py new file mode 100644 index 00000000..f30a78fb --- /dev/null +++ b/backend/apps/agents/browser/compose_discovery.py @@ -0,0 +1,192 @@ +"""Find a site's compose URL by reading the links the site already publishes. + +The compose-URL table in `compose_entry` is worth 92.9% composer reachability on the five hosts it +knows and exactly 0% everywhere else, measured live over 8 logged-in sites. Two attempts to close +that gap by making the aux navigator click better were falsified in the same session (0/8 both +times, at roughly double the wall time), so the lever is not better clicking, it is getting the URL. + +The generalizable source is the page itself. A site that has a composer links to it: "Start a post", +"Ask Question", "Create", "New story". Reading those anchors is precise where guessing paths is not, +because it is the site's own navigation rather than a list of shapes we hope it matches, and it +needs no per-site knowledge. It also degrades honestly: no matching anchor means no candidate, and +the caller runs exactly as it does today. + +The ranking here is deliberately pure, so the part that decides where to send someone's browser is +testable without a browser. The page-reading half is one expression; the judgement half is below it. +""" + +import json +import os +import re +from typing import Dict, List, Optional, Tuple +from urllib.parse import urlparse + +from typeguard import typechecked + +from backend.apps.agents.browser import compose_entry + +# How many links to bring back. Enough to cover a nav bar plus a sidebar; a page with more than +# this has a hundred feed links and the composer is not going to be number 401. +MAX_LINKS = 400 +# How many candidates the caller may actually navigate to. Each miss costs a real page load, and +# past two the aux loop is the cheaper remaining option. +MAX_CANDIDATES = 2 +# Longest a control's label can be before it is prose rather than a button. "Start a post" is 3. +MAX_LABEL_WORDS = 4 + +# Path segments that mean "start something new" across platforms rather than on one host: /submit +# is reddit-shaped, /questions/ask is every StackExchange, /new-story is medium-shaped, /new/text is +# tumblr-shaped, /compose is mail and chat. +# +# A WHOLE segment, never a substring. Substring matching was tried and produced garbage on the +# first live read: "post" matched inside `/explore/top-posts` and "new" inside a permalink +# `/actuallysara/823.../new-photo-of-connor...`, so the tier proposed navigating to a random blog +# post. `post` itself is deliberately absent even as a segment, because permalinks are `/post/` +# on half the web and a false candidate costs a real page load. +P_PATH_HINTS: Tuple[str, ...] = ( + "submit", "compose", "new", "create", "ask", "publish", "write", "share", + "new-story", "new-post", "new-thread", "new-story", +) + +# What the site calls the control. Labels are the stronger signal of the two: a path can be +# incidental ("/new-york-times"), but a link a human reads as "Start a post" is one. +P_LABEL_RE = re.compile( + r"\b(start a post|create a post|new post|create post|write a post|" + r"ask (a )?question|new story|write a story|start writing|new thread|" + r"compose|create new|new message|submit a? ?(post|link|text)?|publish)\b", re.I) + +# Never a composer, and several are actively destructive to wander into mid-run. Checked against +# the whole URL plus the label, because "sign out" hides behind /logout as often as it is written. +P_NEVER_RE = re.compile( + r"\b(log ?out|sign ?out|log ?in|sign ?in|sign ?up|register|settings|preferences|account|" + r"billing|subscribe|upgrade|premium|checkout|cart|delete|privacy|terms|cookie|legal|" + r"about|careers|jobs|advertise|press|help|support|download|api|developer)\b", re.I) + + +@typechecked +def enabled() -> bool: + """OFF by default, unlike the compose-URL table it backs up. + + The mechanism is proven: pointed at StackOverflow it returned `/questions/ask` as its first + candidate, which is exactly right. What is NOT proven is that it helps anyone end to end, and + on the eight-site sweep it did not, for reasons outside itself: three of five create-sites + publish no compose link at all (their composer is a button opening a modal with no route), + StackOverflow was behind a sign-in wall, and GitHub never armed at an upstream gate. Shipping a + tier default-on because its parts work, while its measured contribution is zero, is how latency + accretes. It turns on when a site set exists where it can win and the number says it did.""" + return os.environ.get("OSW_COMPOSE_DISCOVERY", "0") != "0" + + +@typechecked +def discovery_expression() -> str: + """One page read returning every link the site publishes, with the text a human would read. + + Deliberately read-only: it collects hrefs and labels and touches nothing, so it is safe to run + on any page in any state, including one the user is looking at.""" + return ( + "(() => {" + " const out = []; const seen = new Set();" + " for (const a of document.querySelectorAll('a[href]')) {" + " const href = a.href || '';" + " if (!href || seen.has(href)) continue;" + " if (!/^https?:/i.test(href)) continue;" + " seen.add(href);" + " const label = (a.getAttribute('aria-label') || a.innerText || a.title || '')" + " .replace(/\\s+/g, ' ').trim().slice(0, 80);" + " out.push({href: href, label: label});" + f" if (out.length >= {MAX_LINKS}) break;" + " }" + " return {url: location.href, links: out};" + "})()" + ) + + +@typechecked +def p_link_score(href: str, label: str, host: str, current_url: str) -> int: + """How much this link looks like the way in to this site's composer. 0 means never navigate. + + Scored rather than matched so the front-door compose link outranks a deeper one that merely + shares a word, which is what stops "/submit-a-tip" from beating "/submit".""" + parsed = urlparse(href) + found = compose_entry.registrable_host(parsed.netloc) + want = compose_entry.registrable_host(host) + # Off-host links go to a different company; a "share to X" button must not hijack a tumblr post. + if not found or (found != want and not found.endswith("." + want)): + return 0 + path = parsed.path.strip("/").lower() + if not path: + return 0 + if P_NEVER_RE.search(href) or P_NEVER_RE.search(label or ""): + return 0 + # Already here. Re-navigating would remount the page and throw away whatever is on it. + if href.rstrip("/") == (current_url or "").rstrip("/"): + return 0 + score = 0 + # A compose control is labelled like a button, not like an article. Length is what separates + # them, and it separates them on every site at once: StackOverflow offered a QUESTION titled + # "Compose preview different from emulator" as a compose link, because a Q&A site is full of + # titles containing the word. Buttons say "Ask Question" or "Start a post". + if len((label or "").split()) <= MAX_LABEL_WORDS and P_LABEL_RE.search(label or ""): + score += 10 + segments = [s for s in path.split("/") if s] + if any(seg in P_PATH_HINTS for seg in segments): + score += 6 + if not score: + return 0 + # A shallow path is the site's own front door to composing; a deep one is usually a specific + # item that happens to share a word. Never lets a match drop to zero. + score += max(0, 3 - len(segments)) + return score + + +@typechecked +def rank_candidates(payload: Optional[Dict[str, object]], host: str) -> List[str]: + """The URLs worth trying, best first, capped. Empty when the page publishes nothing composer-ish. + + Takes the raw page read so the whole decision is one pure function over data, which is the only + reason this is testable without driving a browser.""" + if not isinstance(payload, dict): + return [] + links = payload.get("links") + if not isinstance(links, list): + return [] + current = str(payload.get("url") or "") + scored: List[Tuple[int, str]] = [] + seen: set = set() + for row in links: + if not isinstance(row, dict): + continue + href = str(row.get("href") or "") + label = str(row.get("label") or "") + if not href or href in seen: + continue + seen.add(href) + score = p_link_score(href, label, host, current) + if score > 0: + scored.append((score, href)) + # Sort by score, then by URL so a tie is deterministic across runs rather than DOM-order luck. + scored.sort(key=lambda pair: (-pair[0], pair[1])) + return [href for _, href in scored[:MAX_CANDIDATES]] + + +@typechecked +def parse_page_read(raw: object) -> Optional[Dict[str, object]]: + """The evaluate result, whatever shape the bridge handed back. + + BrowserEvaluate returns the value directly on some paths and JSON in a text field on others, and + a discovery tier that silently sees nothing looks identical to a site with no compose link.""" + if isinstance(raw, dict) and "links" in raw: + return raw + if isinstance(raw, dict): + for key in ("result", "value", "text"): + inner = raw.get(key) + if isinstance(inner, dict) and "links" in inner: + return inner + if isinstance(inner, str) and inner.strip().startswith("{"): + try: + parsed = json.loads(inner) + except ValueError: + continue + if isinstance(parsed, dict) and "links" in parsed: + return parsed + return None diff --git a/backend/apps/agents/browser/compose_entry.py b/backend/apps/agents/browser/compose_entry.py new file mode 100644 index 00000000..e8984135 --- /dev/null +++ b/backend/apps/agents/browser/compose_entry.py @@ -0,0 +1,239 @@ +"""The site's own URL for opening a composer, used when the task is to create something new. + +Reachability, not fill mechanics, is what a 20-run dry sweep measured as the ceiling on scripted +writes: the send script armed 20/20 and reached a composer 0/20. The fill, the commit check and the +two-sided receipt are all proven; what fails is that prestage's aux navigator lands on the site's +HOME page, and the composer is one more hop that it does not reliably take. Polling harder does not +help, it was tried twice and the reached rate went 40% -> 0%. + +Every site here publishes a URL that opens its own composer. Asking for that URL is deterministic +where hunting for a button is not: no selector to drift, no modal to race, no capped element list to +starve. It is also cheaper, since a hit skips the aux navigation loop entirely. + +Deliberately narrow, because composing in the wrong place is worse than not composing at all: + + - only for a task that creates something TOP-LEVEL. A reply or comment belongs on the thread the + user is looking at, so those keep their own target (the same rule the post-is-not-a-comment + guard enforces on the composer, applied one layer earlier and one layer cheaper). + - only when the task points at the bare site. Any deeper URL on that host is a specific target + the user chose, and it outranks the generic composer every time. + - only as a PROPOSAL. The caller navigates, then checks whether a composer actually appeared; if + it did not, the normal path runs untouched. A site that changes its compose URL degrades to + today's behaviour rather than stranding the run somewhere useless. +""" + +import os +import re +from typing import Dict, List, Optional +from urllib.parse import urlparse + +from typeguard import typechecked + +from backend.apps.agents.browser import browser_fast_path, browser_send_parse + +# Registrable host -> the site's own compose URL. A dynamic-key map keyed by host, matched by +# suffix so www./m./mobile. variants all resolve. Adding a site is one row. +# +# These are the sites' own documented entry points, not scraped links: +# x /compose/post opens the post dialog on a fresh page +# linkedin ?shareActive=true opens the "start a post" modal on the feed +# reddit /submit?type=TEXT the self-post form +# gmail ?compose=new opens a compose window in the mail UI +P_COMPOSE_URLS: Dict[str, str] = { + "x.com": "https://x.com/compose/post", + "twitter.com": "https://x.com/compose/post", + "linkedin.com": "https://www.linkedin.com/feed/?shareActive=true", + "reddit.com": "https://www.reddit.com/submit?type=TEXT", + "mail.google.com": "https://mail.google.com/mail/u/0/#inbox?compose=new", +} + +# Creating something new. "compose"/"draft"/"email" carry gmail, where nobody says "post". +P_CREATE_RE = re.compile( + r"\b(post|posting|tweet|tweeting|publish|publishing|share|sharing|compose|composing|" + r"draft|drafting|write|writing|send|create|creating|start|starting|ask|asking)\b" + # "open a new issue/thread" is a create; bare "open" is navigation ("open the first video"), + # so only the whole phrase counts. Measured: github never reached the tier without this. + r"|\bopen(ing)? a new\b", re.I) +# Answering something that already exists. One of these and the target is the thread, not the site. +P_RESPOND_RE = re.compile(r"\b(reply|replies|replying|comment|commenting|respond|responding|" + r"answer|answering|quote|retweet|dm|message)\b", re.I) +P_URL_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I) +# "go to x.com and post ..." names its site without a scheme, which is how most tasks arrive. +P_BARE_HOST_RE = re.compile(r"(?:^|[\s/@(,])((?:[\w-]+\.)+[a-z]{2,})\b", re.I) + + +@typechecked +def registrable_host(url_or_host: str) -> str: + """The host with any www./m./mobile. prefix removed, lowercased, port dropped. + + Prefix stripping is done with a real prefix check; `lstrip("www.")` would eat any leading w + or dot and quietly turn `w3schools.com` into `3schools.com`.""" + raw = (url_or_host or "").strip() + host = urlparse(raw).netloc if "//" in raw else raw + host = host.lower().split("@")[-1].split(":")[0] + for prefix in ("www.", "m.", "mobile."): + if host.startswith(prefix): + host = host[len(prefix):] + break + return host + + +@typechecked +def p_table_hit(host: str) -> str: + """The compose URL for this host, matching a parent domain too (`old.reddit.com` -> reddit). + + Exact-or-dotted-suffix only: a bare `endswith` would match `notreddit.com` against `reddit.com` + and send a post to a site the user never named.""" + host = registrable_host(host) + if not host: + return "" + if host in P_COMPOSE_URLS: + return P_COMPOSE_URLS[host] + for known, url in P_COMPOSE_URLS.items(): + if host.endswith("." + known): + return url + return "" + + +@typechecked +def p_names_deeper_target(task: str, host: str) -> bool: + """True when the task carries a URL on this host that points somewhere more specific than its + front page. That URL is the user's chosen target and must win over the generic composer.""" + want = registrable_host(host) + for m in P_URL_RE.finditer(human_words(task)): + parsed = urlparse(m.group(0).rstrip(".,;)")) + found = registrable_host(parsed.netloc) + if not found or (found != want and not found.endswith("." + want)): + continue + if parsed.path.strip("/") or parsed.query or parsed.fragment: + return True + return False + + +@typechecked +def human_words(task: str) -> str: + """The task minus the aux-written routing brief. + + A dispatched task is the user's prompt followed by a brief a model wrote about how to route it. + The brief is prose, and it both quotes things of its own and uses answering words, so reading + intent off the whole string reads the model's commentary as the user's request. The send script + hit this first (a brief saying "do not submit it" read-only-flagged a real send). + + Everything reads this, refusals included. Letting the brief veto looked like the safe choice + and was not: briefs routinely spell out a route ("navigate to https://x.com/home"), so the + deeper-target veto fired on the model's own suggestion and silently disabled the tier on two of + four sites. A brief cannot turn a post into a reply either, since the words that would say so + are the user's and are still read here.""" + return (task or "").split(browser_fast_path.BRIEF_MARKER, 1)[0] + + +@typechecked +def wants_top_level_compose(task: str) -> bool: + """True when the task creates something new rather than answering something that exists. + + An intent word alone is not enough: "what is the top post on reddit" is a READ, and `post` there + is a noun. Reading the verb correctly needs a model, so instead this asks for the thing every + real write in this product carries and no read does, the quoted text to write. That is also the + send script's own precondition, so a task this refuses is one the script would decline anyway, + and the aux navigator handles it exactly as it does today.""" + text = human_words(task) + if not browser_send_parse.quoted_payload(text): + return False + return bool(P_CREATE_RE.search(text)) and not P_RESPOND_RE.search(text) + + +@typechecked +def enabled() -> bool: + """On by default. This tier only ever navigates the user's browser to a page that site + publishes for exactly this purpose, and the caller verifies the result before relying on it, so + the failure mode is a wasted page load rather than a wrong action. The switch exists to A/B it + against the aux navigator and to turn it off in the field without a rebuild.""" + return os.environ.get("OSW_COMPOSE_ENTRY", "1") != "0" + + +@typechecked +def named_hosts(task: str, start_url: str) -> List[str]: + """The hosts this task is about, in the order they should be trusted. + + The site the USER NAMED wins, and where the card happens to sit is only the fallback for a task + that names none ("post this" while already on the site). That order is load-bearing for anything + that has to be ON a site before it can read from it: a cold run opens on a blank search page, so + trusting the card first made the first live discovery read google.com and correctly find + nothing. `compose_entry_for` can afford the opposite order because a table hit already proves + relevance; without a table there is nothing to filter a wrong host out. + + Same extraction either way, kept in one place so the two cannot drift into disagreeing about + which site a task is for. The routing brief is excluded, so a model cannot redirect the post.""" + out: List[str] = [] + for candidate in (*[m.group(0) for m in P_URL_RE.finditer(human_words(task))], + *[m.group(1) for m in P_BARE_HOST_RE.finditer(human_words(task))], + start_url): + host = registrable_host(candidate) + if host and "." in host and host not in out: + out.append(host) + return out + + +@typechecked +def named_page(task: str, host: str) -> str: + """The most specific page the user named on this host, else its front page. + + Composing often lives under a section rather than at the root: "open a new issue" on + `github.com/owner/repo` is reachable from the repo and nowhere near `github.com/`, which + publishes no compose link at all. Measured, so the host alone is not enough to go on.""" + want = registrable_host(host) + for m in P_URL_RE.finditer(human_words(task)): + url = m.group(0).rstrip(".,;)") + found = registrable_host(urlparse(url).netloc) + if found and (found == want or found.endswith("." + want)): + return url + return f"https://{want}/" + + +@typechecked +def compose_entry_for(task: str, start_url: str, task_is_send: bool) -> Optional[str]: + """The URL to open to reach this site's composer, or None to leave navigation alone. + + `start_url` is where the card already is; a host named in the task counts too, since a run that + begins on a blank tab still says "go to x.com and post ...". + + `task_is_send` is the caller's already-computed write verdict and is REQUIRED, not defaulted, + because forgetting it is silently destructive: a quote is not proof of a write, and + `find the reddit post that says "..."` reads as a create to any regex short enough to be + readable (`post` is a noun there). Four such phrasings each resolved to reddit's SUBMIT page in + a probe, which would derail a plain read. The verdict the send script itself gates on is the + right authority, so this asks for it rather than growing a second opinion that can drift.""" + if not enabled() or not task_is_send or not wants_top_level_compose(task): + return None + # Which site to open comes from the user's words; a brief naming some other site must not + # redirect the post. + asked = human_words(task) + named = [m.group(0) for m in P_URL_RE.finditer(asked)] + named += [m.group(1) for m in P_BARE_HOST_RE.finditer(asked)] + for candidate in (start_url, *named): + url = p_table_hit(candidate) + if not url: + continue + host = registrable_host(candidate) + if p_names_deeper_target(task, host): + return None + # Already on the compose surface: navigating again would remount it and throw away a + # composer that is right there. + if registrable_host(start_url) == host and p_on_compose_surface(start_url, url): + return None + return url + return None + + +@typechecked +def p_on_compose_surface(current_url: str, compose_url: str) -> bool: + """True when the current URL is already this site's compose surface.""" + cur, target = urlparse(current_url or ""), urlparse(compose_url or "") + if not cur.netloc: + return False + cur_mark = (cur.path.strip("/") + "?" + cur.query + "#" + cur.fragment).lower() + target_mark = (target.path.strip("/") + "?" + target.query + "#" + target.fragment).lower() + for token in ("compose", "submit", "shareactive"): + if token in target_mark and token in cur_mark: + return True + return False diff --git a/backend/apps/agents/browser/route_write.py b/backend/apps/agents/browser/route_write.py new file mode 100644 index 00000000..9fd97df9 --- /dev/null +++ b/backend/apps/agents/browser/route_write.py @@ -0,0 +1,218 @@ +"""General capture-and-replay write tier: replay a write the site's OWN UI issues, via the +borrowed session, WITHOUT a hand-written per-site adapter. This is the site-agnostic path to +write coverage (the "all popular sites" lever): the browser passively captures the internal API +routes the page fires (electron/cdp-routes.js), and this replays a MUTATING one with the agent's +content substituted, using live-borrowed cookies (never persisted) plus any CSRF header the site +derives from a cookie. + +SAFETY (this IS the posture flip away from GET/HEAD-only, so the walls are belt-and-suspenders): +- Same-origin: the target must be the site currently loaded, nothing else. +- Captured-route match: the target must correspond to a mutating route the site's OWN UI actually + fired. The agent can't invent an endpoint; it can only replay one the page genuinely uses. This + is the wall against a prompt-injected page steering the agent to an arbitrary write. +- Flag-gated default OFF (OSW_ROUTE_WRITE=1 to arm). The deterministic per-site adapters (Reddit) + stay always-on; this general tier is opt-in until it's soaked. +- Behind the caller's send-safety guard (solo, verified, receipt-or-honest-miss, never a false + claim of success). +- Secret-safe: cookies are live-borrowed per call, never logged, never persisted; the CSRF header + is derived from a cookie at call time, not stored. +""" + +import json +import logging +import os +import re +import time +import urllib.error +import urllib.request +from typing import Any, Dict, List +from urllib.parse import urlparse + +from pydantic import BaseModel, ConfigDict +from typeguard import typechecked + +from backend.apps.social_shims.session_source import get_session + +logger = logging.getLogger(__name__) + +WRITE_METHODS = frozenset({"POST", "PUT", "PATCH", "DELETE"}) +# CSRF header a site derives from a cookie (so it survives a fresh borrowed session). Small on +# purpose: most cookie-auth internal APIs need nothing extra; this covers the common header case. +P_CSRF_FROM_COOKIE: Dict[str, Dict[str, str]] = { + "x.com": {"header": "x-csrf-token", "cookie": "ct0"}, + "twitter.com": {"header": "x-csrf-token", "cookie": "ct0"}, +} + + +class CapturedRoute(BaseModel): + """One mutating route the site's own UI was seen to fire (from the CDP route capture). The + method + templated path are the identity we match a replay target against; nothing secret + lives here (the capture redacts auth headers and strips body values).""" + + model_config = ConfigDict(validate_assignment=True) + + method: str + template: str + + +class ReplayOutcome(BaseModel): + """Typed result of a route replay. `receipt` is the site's own confirmation pulled from the + response (an id / permalink / url); `ok` is False with a legible `error` on any refusal or + rejection, so the caller falls back to the UI, never a crash and never a false success.""" + + model_config = ConfigDict(validate_assignment=True) + + ok: bool + receipt: str = "" + error: str = "" + status: int = 0 + latency_ms: int = 0 + + +@typechecked +def enabled() -> bool: + """The general route-write tier is opt-in (posture flip); armed only by OSW_ROUTE_WRITE=1.""" + return os.environ.get("OSW_ROUTE_WRITE", "0") == "1" + + +@typechecked +def p_template_path(url: str) -> str: + """Collapse volatile path segments (numeric ids, long hex/uuids) to '{id}', mirroring the + capture side (cdp-routes.js templateUrl) so a concrete replay URL matches the captured + template. Origin + path only; query keys are ignored for the match.""" + try: + u = urlparse(url) + path = re.sub(r"/(\d+|[0-9a-fA-F]{8,}(?:-[0-9a-fA-F]+)*)(?=/|$)", "/{id}", u.path) + return f"{u.scheme}://{u.netloc}{path}" + except Exception: + return url + + +@typechecked +def same_origin(url: str, origin: str) -> bool: + """True when url is on the same origin as the loaded page (scheme+host+port), the first wall.""" + try: + a, b = urlparse(url), urlparse(origin) + return bool(a.scheme and a.netloc) and (a.scheme, a.netloc) == (b.scheme, b.netloc) + except Exception: + return False + + +@typechecked +def route_is_captured(method: str, url: str, captured: List[CapturedRoute]) -> bool: + """True when (method, templated url) matches a mutating route the site's UI actually fired. + The safety wall that stops a prompt-injected page from steering the agent to an invented + endpoint: the agent can only replay a write the page genuinely uses.""" + m = method.upper() + if m not in WRITE_METHODS: + return False + target = p_template_path(url) + return any(r.method.upper() == m and p_template_path(r.template) == target for r in captured) + + +@typechecked +def p_cookie_value(cookie_header: str, name: str) -> str: + """Pull one cookie's value out of a 'k=v; k2=v2' header, for CSRF-from-cookie derivation.""" + for part in (cookie_header or "").split(";"): + k, _, v = part.strip().partition("=") + if k == name: + return v + return "" + + +@typechecked +def derive_csrf_headers(url: str, cookie_header: str) -> Dict[str, str]: + """The CSRF header a site expects, re-derived from the live cookie (e.g. X's x-csrf-token is + its ct0 cookie). Empty for the common cookie-only-auth site, which needs nothing extra.""" + host = (urlparse(url).netloc or "").lower().lstrip(".") + apex = ".".join(host.split(".")[-2:]) if host.count(".") >= 1 else host + rule = P_CSRF_FROM_COOKIE.get(apex) + if not rule: + return {} + val = p_cookie_value(cookie_header, rule["cookie"]) + return {rule["header"]: val} if val else {} + + +@typechecked +def receipt_from_json(obj: Any) -> str: + """The most proof-bearing id/permalink/url anywhere in a response JSON (shallow-first), so the + caller gets a real receipt without knowing each site's response shape.""" + seen: List[Any] = [obj] + for _ in range(400): # bounded walk; a receipt lives near the top of a write response + if not seen: + break + cur = seen.pop(0) + if isinstance(cur, dict): + for key in ("permalink", "url", "id_str", "rest_id", "id", "name"): + v = cur.get(key) + if isinstance(v, (str, int)) and str(v): + return str(v) + seen.extend(cur.values()) + elif isinstance(cur, list): + seen.extend(cur) + return "" + + +@typechecked +def outcome_from_response(status: int, text: str, latency_ms: int) -> ReplayOutcome: + """Map an HTTP response to a typed outcome: 2xx = landed (with a parsed receipt), anything else + = a legible error the caller surfaces so the model does the write via the UI instead.""" + if not (200 <= status < 300): + return ReplayOutcome(ok=False, status=status, latency_ms=latency_ms, + error=f"site returned HTTP {status}: {text[:160]}") + receipt = "" + try: + receipt = receipt_from_json(json.loads(text)) if text.strip() else "" + except (json.JSONDecodeError, ValueError): + receipt = "" + return ReplayOutcome(ok=True, status=status, latency_ms=latency_ms, receipt=receipt or "ok") + + +@typechecked +def issue_request(method: str, url: str, body: Dict[str, Any], headers: Dict[str, str]) -> Any: + """Issue the write from the backend using the borrowed session. JSON body (the shape internal + APIs overwhelmingly use). Returns (status, text). Isolated so tests stub the network.""" + data = json.dumps(body).encode() if body else b"" + req = urllib.request.Request(url, data=data, headers=headers, method=method.upper()) + try: + with urllib.request.urlopen(req, timeout=30.0) as resp: + return resp.status, resp.read().decode("utf-8", "replace") + except urllib.error.HTTPError as e: + return e.code, (e.read().decode("utf-8", "replace") if e.fp else "") + except urllib.error.URLError as e: + raise RuntimeError(f"site unreachable: {getattr(e, 'reason', e)}") + + +@typechecked +def replay_write(method: str, url: str, body: Dict[str, Any], origin: str, + captured: List[CapturedRoute]) -> ReplayOutcome: + """Replay one captured mutating route with the agent's content, via the live-borrowed session. + Every failure (disarmed, off-origin, un-captured, no session, site-reject) is a typed ok=False + so the caller falls back to the UI, never a crash. Secrets are live-borrowed, never logged.""" + if not enabled(): + return ReplayOutcome(ok=False, error="route-write tier disarmed (set OSW_ROUTE_WRITE=1); use the UI") + if not same_origin(url, origin): + return ReplayOutcome(ok=False, error="target is not the current site (same-origin only)") + if not route_is_captured(method, url, captured): + return ReplayOutcome(ok=False, error="no matching write route was captured from this site's UI; use the UI") + domain = (urlparse(origin).netloc or "").lstrip(".") + t0 = time.monotonic() + try: + cookie, ua = get_session(domain) + except Exception as e: + return ReplayOutcome(ok=False, error=f"no borrowable session for {domain}: {str(e)[:120]}") + headers = { + "Cookie": cookie, "User-Agent": ua, "Accept": "application/json", + "Content-Type": "application/json", "Origin": origin, "Referer": origin + "/", + **derive_csrf_headers(url, cookie), + } + try: + status, text = issue_request(method, url, body, headers) + except Exception as e: + # UNEXPECTED: the request itself blew up (network/DNS/TLS), not a same-origin/captured refusal. Fails open to the UI, so log it or the broken route-write tier is invisible. + logger.warning(f"[route-write] {method} {url} request FAILED (fast path broken here): {e}") + return ReplayOutcome(ok=False, error=str(e)[:160], latency_ms=int((time.monotonic() - t0) * 1000)) + out = outcome_from_response(status, text, int((time.monotonic() - t0) * 1000)) + if not out.ok: + logger.info(f"[route-write] {method} {url} rejected by site: HTTP {out.status}") + return out diff --git a/backend/apps/agents/browser/seed_for.py b/backend/apps/agents/browser/seed_for.py index 728e259d..4fa978b9 100644 --- a/backend/apps/agents/browser/seed_for.py +++ b/backend/apps/agents/browser/seed_for.py @@ -1,117 +1,9 @@ -""" -Shipped seed playbooks: a starting strategy memory for popular sites so a fresh -install isn't fully cold on its first task there. These are FALLBACKS, the moment -a user does a real verified run on a site, the reflective distill writes a learned -playbook that supersedes the seed (and refines it). So a wrong seed bullet can only -gently mislead a first run and is self-corrected, exactly the playbook's fail-safe. +"""Loader for the shipped seed playbooks (data lives in seed_playbooks.py). A fresh install isn't +fully cold on a popular site's first task: seed_for returns the site's starting strategy bullets, +which browser_playbook uses as a FALLBACK until a real verified run distills a learned playbook +that supersedes them. Match is canonical-host, with and without a leading 'www.'.""" -Sourced from real observation (a read-only recon pass over the top sites) plus the -stable, documented deep-URL search patterns, NOT guessed mechanics. Host keys are -canonical (no leading 'www.'); the loader strips 'www.' before matching. Kept to -the same per-site shape and caps as a learned playbook. - -Coverage note: only LinkedIn carries full task mechanics (it's the one we fully -exercised). The rest carry the high-value generalizable facts a first run wants: -the deep-URL search shortcut, whether the site is usable logged-out, and where the -primary controls live. Richer per-site mechanics accrue as users actually use them. -""" - -SEED_PLAYBOOKS: dict[str, list[str]] = { - "linkedin.com": [ - "Find people via URL: linkedin.com/search/results/people/?keywords=NAME (one nav beats driving the search UI).", - "Open a person's profile, then click Message to open the compose box for that specific person.", - "A 1:1 thread is titled ' and '; that IS the direct thread, do NOT start a new one.", - "In the composer, type the message then click Send; do NOT press Enter (in the rich composer it only inserts a newline).", - ], - "amazon.com": [ - "Search via URL: amazon.com/s?k=QUERY (spaces become +). Browsing and reading prices/ratings work logged-out.", - "Results are cards with a product link, price, and rating; pull them in one shot with BrowserExtract.", - ], - "ebay.com": [ - "Search via URL: ebay.com/sch/i.html?_nkw=QUERY. Browsing works logged-out.", - ], - "walmart.com": [ - "Search via URL: walmart.com/search?q=QUERY. Browsing works logged-out; it can show a press-and-hold bot check on heavy use.", - ], - "etsy.com": [ - "Search via URL: etsy.com/search?q=QUERY. Browsing works logged-out.", - ], - "target.com": [ - "Search via URL: target.com/s?searchTerm=QUERY. Browsing works logged-out.", - ], - "bestbuy.com": [ - "Search via URL: bestbuy.com/site/searchpage.jsp?st=QUERY. A country-select splash may appear first; pick United States.", - ], - "aliexpress.com": [ - "Browsing works logged-out; use the top search box rather than guessing the URL (the search path changes often).", - ], - "craigslist.org": [ - "Listings are per-city: go to the city subdomain first (e.g. sfbay.craigslist.org), search is local, not global.", - ], - "airbnb.com": [ - "Drive the homepage search (Where / check-in-out / Who) then Search; the results URL params are brittle, don't hand-build them.", - ], - "booking.com": [ - "Search via URL: booking.com/searchresults.html?ss=DESTINATION. Browsing works logged-out.", - ], - "expedia.com": [ - "Drive the homepage search widget (Where to, dates, travelers); its URL is complex, don't hand-build it.", - ], - "yelp.com": [ - "Search via URL: yelp.com/search?find_desc=WHAT&find_loc=WHERE. Browsing works logged-out.", - ], - "google.com": [ - "Web search via URL: google.com/search?q=QUERY. Maps search via URL: google.com/maps/search/PLACE.", - ], - "doordash.com": [ - "It gates on a delivery address up front; set the address before browsing restaurants or you'll see nothing.", - ], - "netflix.com": [ - "Requires sign-in to browse or play. Once logged in, search via URL: netflix.com/search?q=QUERY.", - ], - "spotify.com": [ - "Search via URL: open.spotify.com/search/QUERY. Reading catalog works, but playing full tracks needs a logged-in session.", - ], - "twitch.tv": [ - "Search via URL: twitch.tv/search?term=QUERY. A channel lives at twitch.tv/CHANNELNAME.", - ], - "tiktok.com": [ - "Search via URL: tiktok.com/search?q=QUERY. Heavy anti-bot, expect occasional captcha or a login prompt.", - ], - "pinterest.com": [ - "Search pins via URL: pinterest.com/search/pins/?q=QUERY. Most actions (save, follow) need sign-in.", - ], - "facebook.com": [ - "Requires sign-in. The login wall appears immediately; if you aren't signed in, use RequestHumanIntervention, do not try to log in.", - ], - "instagram.com": [ - "Requires sign-in. The login wall appears immediately; if you aren't signed in, use RequestHumanIntervention, do not try to log in.", - ], - "x.com": [ - "Most actions need sign-in. Once logged in, search via URL: x.com/search?q=QUERY (twitter.com redirects here).", - ], - "quora.com": [ - "Search via URL: quora.com/search?q=QUERY. Reading often triggers a sign-in wall after a bit of scrolling.", - ], - "github.com": [ - "Search via URL: github.com/search?q=QUERY&type=repositories. Public repos, issues, and code are readable logged-out.", - ], - "threads.net": [ - "A login overlay sits over the feed; no composer or search is reachable until signed in.", - ], - "web.whatsapp.com": [ - "Needs a phone-linked session via QR. In automation it often serves a 'use a supported browser' wall, treat as not reliably automatable.", - ], - "web.telegram.org": [ - "Web login is QR-code or passkey; if not already logged in, use RequestHumanIntervention rather than attempting it.", - ], - "trello.com": [ - "The landing page is marketing; the app needs sign-in. If not signed in, use RequestHumanIntervention.", - ], - "figma.com": [ - "The landing page is marketing; the app needs sign-in. If not signed in, use RequestHumanIntervention.", - ], -} +from backend.apps.agents.browser.seed_playbooks import SEED_PLAYBOOKS def seed_for(host: str) -> list[str]: diff --git a/backend/apps/agents/browser/seed_playbooks.py b/backend/apps/agents/browser/seed_playbooks.py new file mode 100644 index 00000000..6f7595a1 --- /dev/null +++ b/backend/apps/agents/browser/seed_playbooks.py @@ -0,0 +1,176 @@ +"""Shipped seed playbooks DATA: starting strategy memory for popular sites so a fresh install +isn't fully cold on its first task there. FALLBACKS only, the moment a user does a real verified +run on a site the reflective distill writes a learned playbook that supersedes the seed, so a +wrong bullet can only gently mislead a first run and is self-corrected (the playbook's fail-safe). + +Facts are the stable, documented deep-URL search patterns + observed login/bot walls, NOT guessed +mechanics; where a URL pattern is unstable the bullet says to drive the search box instead. Host +keys are canonical (no leading 'www.'); the loader (seed_for) strips 'www.' before matching. + +Money sites are deliberately framed READ-ONLY: read balances/activity, NEVER move money, pay, trade, +or transfer, hand any transaction to the user. This mirrors the product's financial-action rule. +""" + +SEED_PLAYBOOKS: dict[str, list[str]] = { + # --- search --- + "google.com": [ + "Web search via URL: google.com/search?q=QUERY. Maps search via URL: google.com/maps/search/PLACE.", + ], + "bing.com": ["Search via URL: bing.com/search?q=QUERY. Works logged-out."], + "duckduckgo.com": ["Search via URL: duckduckgo.com/?q=QUERY. Works logged-out, no login."], + "yahoo.com": ["Search via URL: search.yahoo.com/search?p=QUERY. Works logged-out."], + # --- reference / knowledge --- + "wikipedia.org": [ + "Read via URL: en.wikipedia.org/wiki/TITLE (spaces become _). Search via en.wikipedia.org/w/index.php?search=QUERY. Fully readable logged-out.", + ], + "imdb.com": ["A title lives at imdb.com/title/ttID. Search via URL: imdb.com/find/?q=QUERY. Readable logged-out."], + "quora.com": ["Search via URL: quora.com/search?q=QUERY. Reading often triggers a sign-in wall after a bit of scrolling."], + "goodreads.com": ["Search via URL: goodreads.com/search?q=QUERY. Browsing works logged-out; shelving/rating needs sign-in."], + "stackoverflow.com": [ + "Search via URL: stackoverflow.com/search?q=QUERY. Questions and answers are readable logged-out; voting/answering needs sign-in.", + ], + "github.com": ["Search via URL: github.com/search?q=QUERY&type=repositories. Public repos, issues, and code are readable logged-out."], + "news.ycombinator.com": [ + "The front page is news.ycombinator.com; an item is news.ycombinator.com/item?id=ID. Readable logged-out; commenting/voting needs sign-in.", + ], + "medium.com": ["Articles are readable but many hit a metered paywall after a few reads; search via URL: medium.com/search?q=QUERY."], + # --- news / weather --- + "news.google.com": ["Google News: news.google.com/search?q=QUERY. Works logged-out."], + "weather.com": ["Type a city into the search box (weather.com keys pages on an internal location code, so don't hand-build a URL from a city name); forecasts read fine logged-out."], + "cnn.com": ["Readable logged-out; search via URL: cnn.com/search?q=QUERY."], + "bbc.com": ["Readable logged-out; search via URL: bbc.co.uk/search?q=QUERY."], + "nytimes.com": ["A metered paywall appears after a few articles; use the site's own search box, not a guessed URL."], + # --- video / streaming / music --- + "youtube.com": [ + "Search via URL: youtube.com/results?search_query=QUERY. A video is youtube.com/watch?v=ID; browsing and watching work logged-out.", + "To comment: open the video, click the 'Add a comment...' box, type, then click Comment (needs sign-in).", + ], + "netflix.com": ["Requires sign-in to browse or play. Once logged in, search via URL: netflix.com/search?q=QUERY."], + "hulu.com": ["Requires sign-in to browse or play; if not signed in use RequestHumanIntervention."], + "disneyplus.com": ["Requires sign-in to browse or play; if not signed in use RequestHumanIntervention."], + "max.com": ["Requires sign-in to browse or play; if not signed in use RequestHumanIntervention."], + "twitch.tv": ["Search via URL: twitch.tv/search?term=QUERY. A channel lives at twitch.tv/CHANNELNAME; browsing works logged-out."], + "spotify.com": ["Search via URL: open.spotify.com/search/QUERY. Reading catalog works, but playing full tracks needs a logged-in session."], + "music.apple.com": ["Search via URL: music.apple.com/us/search?term=QUERY. Browsing works; full playback needs a signed-in subscription."], + "soundcloud.com": ["Search via URL: soundcloud.com/search?q=QUERY. Browsing and streaming work logged-out."], + "pandora.com": ["Needs sign-in for most listening; if not signed in use RequestHumanIntervention."], + # --- shopping --- + "amazon.com": [ + "Search via URL: amazon.com/s?k=QUERY (spaces become +). Browsing and reading prices/ratings work logged-out.", + "Results are cards with a product link, price, and rating; pull them in one shot with BrowserExtract.", + ], + "ebay.com": ["Search via URL: ebay.com/sch/i.html?_nkw=QUERY. Browsing works logged-out."], + "walmart.com": ["Search via URL: walmart.com/search?q=QUERY. Browsing works logged-out; it can show a press-and-hold bot check on heavy use."], + "target.com": ["Search via URL: target.com/s?searchTerm=QUERY. Browsing works logged-out."], + "bestbuy.com": ["Search via URL: bestbuy.com/site/searchpage.jsp?st=QUERY. A country-select splash may appear first; pick United States."], + "etsy.com": ["Search via URL: etsy.com/search?q=QUERY. Browsing works logged-out."], + "aliexpress.com": ["Browsing works logged-out; use the top search box rather than guessing the URL (the search path changes often)."], + "temu.com": ["Browsing works logged-out but expect aggressive popups; use the top search box rather than a hand-built URL."], + "shein.com": ["Search via URL: shein.com/pdsearch/QUERY. Browsing works logged-out."], + "costco.com": ["Search via URL: costco.com/CatalogSearch?keyword=QUERY. Some prices and buying need a member sign-in."], + "homedepot.com": ["Search via URL: homedepot.com/s/QUERY. Browsing works logged-out."], + "wayfair.com": ["Search via URL: wayfair.com/keyword.php?keyword=QUERY. Browsing works logged-out."], + "craigslist.org": ["Listings are per-city: go to the city subdomain first (e.g. sfbay.craigslist.org); search is local, not global."], + "instacart.com": ["Gates on a delivery address (and usually sign-in) before showing stores; set the location first."], + # --- food delivery / rides / reservations --- + "doordash.com": ["It gates on a delivery address up front; set the address before browsing restaurants or you'll see nothing."], + "ubereats.com": ["Gates on a delivery address up front; set it before browsing. Ordering needs sign-in."], + "grubhub.com": ["Set a delivery address first (browsing restaurants needs it). Ordering needs sign-in."], + "uber.com": ["The ride app needs sign-in; if not signed in use RequestHumanIntervention, do not attempt to log in."], + "lyft.com": ["The ride app needs sign-in; if not signed in use RequestHumanIntervention, do not attempt to log in."], + "opentable.com": ["Search via URL: opentable.com/s?term=QUERY. Booking a table needs sign-in."], + # --- social / messaging --- + "facebook.com": ["Requires sign-in. The login wall appears immediately; if you aren't signed in, use RequestHumanIntervention, do not try to log in."], + "instagram.com": ["Requires sign-in. The login wall appears immediately; if you aren't signed in, use RequestHumanIntervention, do not try to log in."], + "x.com": [ + "Most actions need sign-in. Once logged in, search via URL: x.com/search?q=QUERY (twitter.com redirects here).", + "To post: click the composer ('What is happening?'), type, then click Post; do NOT press Enter (it inserts a newline). To reply, open the tweet and use its Reply box then the Reply button.", + ], + "reddit.com": [ + "Search via URL: reddit.com/search/?q=QUERY. A subreddit is reddit.com/r/NAME; most browsing works logged-out.", + "Posting/commenting needs sign-in and the composer is bot-gated; prefer the built-in write path (BrowserApiWrite) over driving the UI composer.", + ], + "tiktok.com": ["Search via URL: tiktok.com/search?q=QUERY. Heavy anti-bot, expect occasional captcha or a login prompt."], + "pinterest.com": ["Search pins via URL: pinterest.com/search/pins/?q=QUERY. Most actions (save, follow) need sign-in."], + "linkedin.com": [ + "Find people via URL: linkedin.com/search/results/people/?keywords=NAME (one nav beats driving the search UI).", + "Open a person's profile, then click Message to open the compose box for that specific person.", + "A 1:1 thread is titled ' and '; that IS the direct thread, do NOT start a new one.", + "In the composer, type the message then click Send; do NOT press Enter (in the rich composer it only inserts a newline).", + ], + "threads.net": ["A login overlay sits over the feed; no composer or search is reachable until signed in."], + "messenger.com": ["Uses the Facebook login; if not signed in use RequestHumanIntervention, do not try to log in."], + "snapchat.com": ["Primarily a mobile app; the web is limited and login-walled. Use RequestHumanIntervention on a wall."], + "nextdoor.com": ["The neighborhood feed needs sign-in; if signed in, confirm the neighborhood, then browse posts."], + "web.whatsapp.com": ["Needs a phone-linked session via QR; it often serves a 'use a supported browser' wall, treat as not reliably automatable."], + "web.telegram.org": ["Web login is QR-code or passkey; if not already logged in, use RequestHumanIntervention rather than attempting it."], + # --- email / productivity --- + "mail.google.com": [ + "Needs sign-in. Search via URL: mail.google.com/mail/u/0/#search/QUERY. To send: click Compose, fill To then Subject then the body, then click Send.", + ], + "outlook.com": ["Microsoft email; needs sign-in. If not signed in use RequestHumanIntervention."], + "office.com": ["Microsoft 365 hub; needs sign-in. If not signed in use RequestHumanIntervention."], + "docs.google.com": ["Google Docs/Sheets/Slides; needs sign-in. A doc is docs.google.com/document/d/ID, a sheet docs.google.com/spreadsheets/d/ID; edit once signed in."], + "drive.google.com": ["Google Drive; needs sign-in. Search files via URL: drive.google.com/drive/search?q=QUERY."], + "calendar.google.com": ["Google Calendar; needs sign-in. Read events and create via the '+ Create' button once signed in."], + "dropbox.com": ["The landing page is marketing; files need sign-in. If not signed in use RequestHumanIntervention."], + "notion.so": ["The landing page is marketing; the workspace needs sign-in. If not signed in, use RequestHumanIntervention."], + "trello.com": ["The landing page is marketing; the app needs sign-in. If not signed in, use RequestHumanIntervention."], + "figma.com": ["The landing page is marketing; the app needs sign-in. If not signed in, use RequestHumanIntervention."], + # --- travel / local --- + "airbnb.com": ["Drive the homepage search (Where / check-in-out / Who) then Search; the results URL params are brittle, don't hand-build them."], + "booking.com": ["Search via URL: booking.com/searchresults.html?ss=DESTINATION. Browsing works logged-out."], + "expedia.com": ["Drive the homepage search widget (Where to, dates, travelers); its URL is complex, don't hand-build it."], + "tripadvisor.com": ["Search via URL: tripadvisor.com/Search?q=QUERY. Browsing works logged-out."], + "kayak.com": ["Drive the homepage flight/hotel search; result URLs are brittle, don't hand-build them."], + "hotels.com": ["Drive the homepage search widget; result URLs are complex, don't hand-build them."], + "vrbo.com": ["Drive the homepage search; result URLs are brittle, don't hand-build them."], + "yelp.com": ["Search via URL: yelp.com/search?find_desc=WHAT&find_loc=WHERE. Browsing works logged-out."], + # --- jobs / real estate --- + "indeed.com": ["Job search via URL: indeed.com/jobs?q=WHAT&l=WHERE. Browsing works logged-out; applying needs sign-in and is heavily bot-gated (expect a captcha)."], + "glassdoor.com": ["Heavy sign-in and anti-bot walls appear quickly; treat as often not reliably automatable, use RequestHumanIntervention on a wall."], + "ziprecruiter.com": ["Job search via URL: ziprecruiter.com/jobs-search?search=WHAT&location=WHERE. Applying needs sign-in."], + "zillow.com": ["Search via URL: zillow.com/homes/CITY-STATE_rb/. Heavy anti-bot: a press-and-hold or captcha wall is common on more than a few requests."], + "realtor.com": ["Search via URL: realtor.com/realestateandhomes-search/CITY_STATE. Browsing works logged-out."], + "redfin.com": ["Drive the homepage search box; scripted URL access is heavily anti-bot."], + "apartments.com": ["Search via URL: apartments.com/CITY-STATE/. Browsing works logged-out."], + # --- tickets / events --- + "ticketmaster.com": ["Search via URL: ticketmaster.com/search?q=QUERY. Buying needs sign-in and hits queue/anti-bot walls."], + "stubhub.com": ["Search events from the homepage; buying needs sign-in."], + "eventbrite.com": ["Search via URL: eventbrite.com/d/online/QUERY/. Browsing works logged-out; registering needs sign-in."], + # --- health --- + "webmd.com": ["Readable logged-out; search via URL: webmd.com/search/search_results/default.aspx?query=QUERY."], + "goodrx.com": ["A drug's price page is goodrx.com/DRUG-NAME (a specific drug, e.g. goodrx.com/lipitor), NOT free-text search; for an unknown name use the site search box. Prices readable logged-out."], + "cvs.com": ["General browsing works logged-out; pharmacy and account pages need sign-in and are sensitive."], + "walgreens.com": ["General browsing works logged-out; pharmacy and account pages need sign-in and are sensitive."], + # --- education --- + "quizlet.com": ["Search via URL: quizlet.com/search?query=QUERY. Browsing study sets works logged-out."], + "duolingo.com": ["The lessons app needs sign-in; the landing page is marketing. If not signed in use RequestHumanIntervention."], + "khanacademy.org": ["Browsing lessons works logged-out; search via URL: khanacademy.org/search?page_search_query=QUERY."], + "coursera.org": ["Search via URL: coursera.org/search?query=QUERY. Enrolling and course content need sign-in."], + "chegg.com": ["Most content is paywalled behind sign-in; treat as read-limited without an account."], + # --- AI assistants --- + "chatgpt.com": ["A chat app that needs sign-in; if a task requires it and you're not signed in, use RequestHumanIntervention."], + "claude.ai": ["A chat app that needs sign-in; if a task requires it and you're not signed in, use RequestHumanIntervention."], + "perplexity.ai": ["Answer-search AI; search via URL: perplexity.ai/search?q=QUERY (some features need sign-in)."], + "gemini.google.com": ["A chat app that needs sign-in; if a task requires it and you're not signed in, use RequestHumanIntervention."], + # --- government / utilities --- + "usps.com": ["Track a package via URL: tools.usps.com/go/TrackConfirmAction?tLabels=NUMBER. General info is readable logged-out."], + "irs.gov": ["Forms and info are readable logged-out; any personal account access needs sign-in and is sensitive."], + # --- money (READ-ONLY, never transact) --- + "paypal.com": ["Needs sign-in. READ-ONLY: read balances/activity, but NEVER send money, pay, or transfer, hand any money movement to the user (RequestHumanIntervention)."], + "venmo.com": ["Needs sign-in. READ-ONLY: read activity, but NEVER pay, request, or transfer money, hand any payment to the user."], + "cash.app": ["Needs sign-in. READ-ONLY: read activity, but NEVER send, request, or move money, hand any payment to the user."], + "chase.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/transactions, but NEVER move money, pay a bill, or transfer, hand any transaction to the user."], + "bankofamerica.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/transactions, but NEVER move money, pay a bill, or transfer, hand any transaction to the user."], + "wellsfargo.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/transactions, but NEVER move money, pay a bill, or transfer, hand any transaction to the user."], + "capitalone.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/transactions, but NEVER move money, pay, or transfer, hand any transaction to the user."], + "robinhood.com": ["Needs sign-in and is sensitive. READ-ONLY: read positions/prices, but NEVER place, cancel, or modify a trade, hand any trade to the user."], + "coinbase.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/prices, but NEVER buy, sell, send, or trade crypto, hand any transaction to the user."], + "fidelity.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/positions, but NEVER trade or move money, hand any transaction to the user."], + "creditkarma.com": ["Needs sign-in; read-only credit score/report info, do not apply for anything on the user's behalf."], + # --- dating (fingerprint-walled) --- + "tinder.com": ["Mobile-first with heavy fingerprinting and a login wall; treat as not reliably automatable, use RequestHumanIntervention."], + "bumble.com": ["Mobile-first with heavy fingerprinting and a login wall; treat as not reliably automatable, use RequestHumanIntervention."], + "hinge.co": ["Mobile-first with heavy fingerprinting and a login wall; treat as not reliably automatable, use RequestHumanIntervention."], +} diff --git a/backend/apps/agents/browser/site_write_registry.py b/backend/apps/agents/browser/site_write_registry.py new file mode 100644 index 00000000..b17eb151 --- /dev/null +++ b/backend/apps/agents/browser/site_write_registry.py @@ -0,0 +1,148 @@ +"""The API-first write tier, unified for the browser agent. + +When a write targets a site that has a borrowed-session write adapter, route the write HERE +instead of UI puppeteering: borrow the user's live cookies, call the site's OWN write API, and +return the site's typed receipt (its own id / permalink = proof it landed). This is +deterministic (a typed success/error envelope, no captcha on the API surface, no DOM selector to +drift) and ~50-190x faster than driving the UI (measured Reddit: 271ms vs 13-52s). Adding a site +is one adapter entry; a site with no adapter falls back to the existing UI+model write path. + +Live-validated end to end on Reddit (comment 271ms + reversible delete 246ms, typed receipts). +""" + +import asyncio +import logging +import os +import time +from typing import Any, Callable, Dict, FrozenSet, List, Tuple +from urllib.parse import urlparse + +from pydantic import BaseModel, ConfigDict +from typeguard import typechecked + +from backend.apps.agents.browser import route_write +from backend.apps.reddit_mcp_shim import reddit_writes + +logger = logging.getLogger(__name__) + + +class WriteResult(BaseModel): + """The typed outcome of an API-first write. `receipt` is the site's own id/permalink, the + proof the write landed (a real receipt, not a pixel guess); `ok` is False with a legible + `error` when the site's API rejected it or no session could be borrowed.""" + + model_config = ConfigDict(validate_assignment=True) + + ok: bool + action: str + domain: str + receipt: str = "" + error: str = "" + latency_ms: int = 0 + + +@typechecked +def p_reddit_dispatch(action: str, params: Dict[str, Any]) -> Dict[str, Any]: + """Route a generic write action to the proven reddit_writes function; returns its receipt + dict (raises RedditError on the API's own error envelope, surfaced as ok=False upstream).""" + if action in ("comment", "reply"): + return reddit_writes.comment(str(params["parent_id"]), str(params["text"])) + if action in ("post", "submit"): + return reddit_writes.submit( + str(params["subreddit"]), str(params["title"]), str(params.get("kind", "self")), + str(params.get("text", "")), str(params.get("url", "")), + bool(params.get("nsfw", False)), bool(params.get("spoiler", False)), + bool(params.get("send_replies", True)), + ) + if action == "edit": + return reddit_writes.edit(str(params["thing_id"]), str(params["text"])) + if action == "delete": + return reddit_writes.delete(str(params["thing_id"])) + raise ValueError(f"reddit adapter has no action {action!r}") + + +# domain -> (actions it can do via the site's own API, sync dispatch fn). A dynamic-key registry +# keyed by domain; adding a site is one row. X/others plug in the same shape once their write API +# (GraphQL queryId + ct0) is proven, replacing their current UI-driving shim. +P_ADAPTERS: Dict[str, Tuple[FrozenSet[str], Callable[[str, Dict[str, Any]], Dict[str, Any]]]] = { + "reddit.com": (frozenset({"comment", "reply", "post", "submit", "edit", "delete"}), p_reddit_dispatch), +} + + +@typechecked +def has_api_write(domain: str, action: str) -> bool: + """True when this domain has a deterministic API adapter for this write action, so the agent + should route around the UI puppeteer tier.""" + entry = P_ADAPTERS.get(domain.lower().strip().lstrip(".")) + return bool(entry and action in entry[0]) + + +@typechecked +def receipt_str(receipt: Dict[str, Any]) -> str: + """Flatten a site receipt dict into the single most-proof-bearing string (permalink beats a + bare id) so callers get one legible confirmation without knowing each site's shape.""" + for key in ("permalink", "url", "id"): + v = receipt.get(key) + if v: + return str(v) + return "ok" + + +@typechecked +def p_ensure_session_env() -> None: + """Point session_source at the running backend so the in-process agent can borrow cookies the + same token-gated way the subprocess shims do (module globals are read at import, so patch + them). No-op once set.""" + from backend.apps.social_shims import session_source as ss + port = os.environ.get("OPENSWARM_PORT", "8324") + ss.BACKEND_PORT = port + ss.BRIDGE_URL = f"http://127.0.0.1:{port}/api/browser-session/cookies" + if not ss.AUTH_TOKEN: + try: + from backend.auth import get_auth_token + ss.AUTH_TOKEN = get_auth_token() or "" + except Exception: + pass + + +@typechecked +async def api_route_write(origin: str, method: str, url: str, body: Dict[str, Any], + captured: List[route_write.CapturedRoute]) -> WriteResult: + """The GENERAL tier: replay a captured mutating route the site's own UI fired, for sites with + no hand-written adapter. Wraps route_write's typed outcome into the registry's WriteResult so + callers get one shape. Every refusal/rejection is ok=False, so the agent falls back to the UI.""" + p_ensure_session_env() + d = (urlparse(origin).netloc or origin).lstrip(".") + out = await asyncio.to_thread(route_write.replay_write, method, url, body, origin, captured) + return WriteResult(ok=out.ok, action="route", domain=d, receipt=out.receipt, + error=out.error, latency_ms=out.latency_ms) + + +@typechecked +async def api_write(domain: str, action: str, params: Dict[str, Any]) -> WriteResult: + """Perform a write via the site's own API using the borrowed session. Times it, and turns any + failure (rejected by the site, no session, bad params) into a typed ok=False result rather + than raising, so the agent can fall back to the UI path on a miss without a crash.""" + d = domain.lower().strip().lstrip(".") + entry = P_ADAPTERS.get(d) + if not entry or action not in entry[0]: + return WriteResult(ok=False, action=action, domain=d, + error=f"no API-first adapter for {d}/{action}; use the UI path") + p_ensure_session_env() + dispatch = entry[1] + t0 = time.monotonic() + try: + receipt = await asyncio.to_thread(dispatch, action, params) + return WriteResult(ok=True, action=action, domain=d, + receipt=receipt_str(receipt), + latency_ms=int((time.monotonic() - t0) * 1000)) + except reddit_writes.RedditError as e: + # EXPECTED site-side reject (not logged in, rate-limited, bad params): the model sees it and falls back; info, not an alarm. (Future adapters should raise their own recognizable reject type to land here.) + logger.info(f"[api-write] {d}/{action} rejected by the site: {e}") + return WriteResult(ok=False, action=action, domain=d, error=str(e)[:200], + latency_ms=int((time.monotonic() - t0) * 1000)) + except Exception as e: + # UNEXPECTED: the adapter code itself threw (a bug, or the site changed shape). It fails open to the UI, so without this WARNING a systemically-broken fast path is INVISIBLE (looks like the tier just isn't used). + logger.warning(f"[api-write] {d}/{action} adapter FAILED unexpectedly (fast path broken here): {e}") + return WriteResult(ok=False, action=action, domain=d, error=str(e)[:200], + latency_ms=int((time.monotonic() - t0) * 1000)) diff --git a/backend/apps/agents/core/ws_manager.py b/backend/apps/agents/core/ws_manager.py index 6996e5cc..041732ab 100644 --- a/backend/apps/agents/core/ws_manager.py +++ b/backend/apps/agents/core/ws_manager.py @@ -17,6 +17,8 @@ BROWSER_CMD_TIMEOUTS = { "perform_action": 35.0, # session-borrow shims pack navigate + wait + scrape into ONE command, so it needs more than navigate alone "browser_fetch": 32.0, # offscreen window: load + settle + DOM read on an arbitrary (maybe slow/JS-heavy) page "browser_search": 45.0, # tries up to 3 engines sequentially, each a full load + settle + "find_composer": 30.0, # packs trigger + scroll-ladder + retop + open-first into ONE command; on the 15s default the last two tiers were unreachable and heavy pages died mid-ladder (measured: linkedin timed out 2 of 3 runs). The in-page routine self-caps well under this. + "import_session": 40.0, # applies the borrowed cookies AND lets a hidden window sit through the site's bot challenge so the card inherits the clearance; warmBorrowedSession.js self-caps at 15+2.5+5s, and this must outlast that or the warm is killed mid-challenge and we throw away the whole point of it (same trap as find_composer). } BROWSER_CMD_REBROADCAST_S = 3.0 # A CPU-starved renderer can briefly drop its WS (a missed heartbeat) and the frontend auto-reconnects a beat later; bridge that gap instead of hard-failing a live run into it. Short enough that a genuinely-closed window still fails quickly (and no LLM turns are ever burned waiting); long enough to ride out a reconnect even on a loaded machine. diff --git a/backend/apps/agents/manager/run_browser_fast_path.py b/backend/apps/agents/manager/run_browser_fast_path.py index 2f17d3d6..d2cdc82f 100644 --- a/backend/apps/agents/manager/run_browser_fast_path.py +++ b/backend/apps/agents/manager/run_browser_fast_path.py @@ -40,6 +40,8 @@ async def run_browser_fast_path( # The fast-path skips the orchestrator, so the UI never gets the BrowserAgent tool-call that draws the "Browser Agent" bubble. Emit a synthetic tool_call/ tool_result pair (same shape + mcp__ name the orchestrator uses) so the bubble shows here too. None until we actually dispatch a browser (a pure READ answer has no browser, so no bubble). p_browser_tool = "mcp__openswarm-browser-agent__CreateBrowserAgent" p_bubble_tid: Optional[str] = None + p_action_logs: List[List[Dict[str, object]]] = [] + p_last_result: Dict[str, object] = {} try: from backend.apps.agents.browser.browser_agent import run_browser_agents from backend.apps.agents.browser import browser_fast_path @@ -48,11 +50,17 @@ async def run_browser_fast_path( if verdict == "read": from backend.apps.agents.browser import browser_fast_read from backend.apps.agents.providers.registry import get_api_type + p_read_api = get_api_type(session.model) text = await browser_fast_read.try_fast_read( - prompt, brief, load_settings(), get_api_type(session.model), + prompt, brief, load_settings(), p_read_api, ) or "" if not text: - p_fp_path = "read->browser" + # The single page couldn't answer it; a multi-source read (a difference, a compare) fans out to N concurrent single-page reads + one reduce, still no browser and still fail-open. + from backend.apps.agents.browser import browser_map_reduce_read + text = await browser_map_reduce_read.try_map_reduce_read( + prompt, load_settings(), p_read_api, + ) or "" + p_fp_path = "read-mapreduce" if text else "read->browser" p_entry = browser_fast_path.entry_url_from_brief(brief) if p_entry: @@ -60,16 +68,21 @@ async def run_browser_fast_path( @typechecked async def p_dispatch(task_text: str) -> Dict[str, object]: + # user_prompt rides along RAW: the composed task's routing brief carries its own quoted strings, which made every real send payload look ambiguous to the send-script (r242/r243) results = await run_browser_agents( tasks=[{"task": task_text, "browser_id": selected[0] if selected else "", - "url": "", "entry_url": p_entry}], + "url": "", "entry_url": p_entry, "user_prompt": prompt}], model=session.model, dashboard_id=session.dashboard_id, pre_selected_browser_ids=selected, parent_session_id=session_id, ) r = results[0] if results else {} - return r if isinstance(r, dict) else {"summary": str(r or ""), "action_log": []} + r = r if isinstance(r, dict) else {"summary": str(r or ""), "action_log": []} + # Keep EVERY dispatch's actions, not just the last: a run that needed a recovery or a + # send probe did that work on the user's behalf and the trace has to show it. + p_action_logs.append(list(r.get("action_log") or [])) + return r @typechecked def p_summary(r: Dict[str, object]) -> str: @@ -84,6 +97,7 @@ async def run_browser_fast_path( await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": p_tc.model_dump(mode="json")}) first = await p_dispatch(browser_fast_path.compose_task(prompt, brief)) + p_last_result = first text = p_summary(first) if browser_fast_path.dispatch_failed(first): # Retry only transient failures; a dead dashboard fails the retry identically, so skip it and tell the user instead. @@ -125,8 +139,19 @@ async def run_browser_fast_path( ) # Close the synthetic bubble (always, even if the dispatch threw) so it never hangs as "running"; the bubble pairs this result with its call positionally. if p_bubble_tid: + # The bubble carries the same auditable record the sub-agent path shows. It used to close + # with the literal string "done", so expanding it on this tier revealed nothing. + from backend.apps.agents.browser import browser_trace + p_trace = browser_trace.build_trace( + tier=browser_trace.tier_label(p_fp_path, used_browser=True), + action_logs=p_action_logs, + receipt=browser_trace.receipt_from(p_last_result), + entry_url=p_entry or "", + ) p_tr = Message(role="tool_result", branch_id=session.active_branch_id, - content={"tool_use_id": p_bubble_tid, "tool": p_browser_tool, "text": "done"}) + content={"tool_use_id": p_bubble_tid, "tool": p_browser_tool, + "text": browser_trace.trace_text(p_trace), + **browser_trace.trace_payload(p_trace)}) session.messages.append(p_tr) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": p_tr.model_dump(mode="json")}) diff --git a/backend/apps/agents/providers/registry.py b/backend/apps/agents/providers/registry.py index 6c8881be..ae89e774 100644 --- a/backend/apps/agents/providers/registry.py +++ b/backend/apps/agents/providers/registry.py @@ -144,7 +144,7 @@ BUILTIN_MODELS: dict[str, list[dict[str, Any]]] = { {"value": "gemini-3.1-flash-lite", "label": "Gemini 3.1 Flash Lite", "context_window": 1_000_000, "router_model_id": "gc/gemini-3.1-flash-lite-preview", "api": "gemini-cli", "subscription_only": True, "reasoning": True}, - # gemini-3-pro removed 2026-03-09 and gemini-3-flash removed 2026-07-03: gemini-3-flash-preview aged out upstream (API-key route hangs with no fail-fast; only an Antigravity sub still masked it). 3.5-flash / 3.1-flash-lite cover the slots. + # gemini-3-pro removed 2026-03-09 and gemini-3-flash removed 2026-07-03 (both rows, independently on two branches): gemini-3-flash-preview aged out upstream (API-key lane hangs/429s with no fail-fast, measured 7-21s; only an Antigravity sub masked it). 3.5-flash / 3.1-flash-lite cover the slots; ag/gemini-3-flash lives on as an aux model, not a picker row. # API-key entries: bypass 9Router, call generativelanguage.googleapis.com. # Gemini 3.6 Flash + 3.5 Flash-Lite (both GA 2026-07-21, changelog-verified ids) are API-key # only for the same reason as 3.5 Flash: the pinned 0.3.60 gc/ registry predates them. No 3.5 @@ -287,7 +287,7 @@ def resolve_model_id_for_sdk(short_name: str, settings: AppSettings) -> str: # Gemini lane order: Antigravity OAuth (for the models it serves), then AI Studio apikey, then Gemini CLI. AG bypasses the thoughtSignature validator that breaks multi-step Gemini turns AND supports real reasoning, so a connected AG sub is preferred over the AI Studio key, which otherwise silently shadowed it. The map is AG's allowlist; pro variants 404/400 on AG and are deliberately absent, so they fall through to the key. P_ANTIGRAVITY_MAP = { # gemini-3-pro-preview disabled: AG returns 404 even with active conn. gemini-3.1-pro-preview disabled: AG's `gemini-3.1-pro-high` variant 400s every request with "invalid argument" (the `-high` thinking- budget alias on AG requires a thinking_config the CLI doesn't emit). Falls through to the AI Studio key / gc/ instead. gemini-3-flash-preview key dropped with its registry entry (aged out upstream). - "gemini-3.1-flash-lite-preview": "gemini-3-flash", + "gemini-3.1-flash-lite-preview": "gemini-3-flash", # 3.1-flash-lite has no AG variant, so AG serves it via gemini-3-flash } if entry.get("api") == "gemini-cli": rid = entry.get("router_model_id", "") @@ -426,7 +426,6 @@ COST_PER_1M_TOKENS: dict[tuple[str, str], tuple[float, float]] = { # Google; Gemini CLI subscription path, user pays nothing per token ("Google", "gemini-3.5-flash"): (0.0, 0.0), ("Google", "gemini-3.1-flash-lite"): (0.0, 0.0), - ("Google", "gemini-3-flash"): (0.0, 0.0), ("Google", "gemini-2.5-pro"): (0.0, 0.0), ("Google", "gemini-2.5-flash"): (0.0, 0.0), # OpenRouter-backed (approximate) diff --git a/backend/apps/nine_router/oauth.py b/backend/apps/nine_router/oauth.py index 538bade6..f66aa132 100644 --- a/backend/apps/nine_router/oauth.py +++ b/backend/apps/nine_router/oauth.py @@ -7,6 +7,7 @@ Talks to the already-running 9Router over HTTP; never spawns the subprocess import asyncio import logging import os +from typing import Optional import httpx @@ -211,7 +212,7 @@ async def p_start_codex_callback_listener(timeout: float = 300.0) -> int | None: return bound_port -# Providers whose OAuth flow MUST run in the user's real browser via shell.openExternal, not the in-Electron window.open popup: - gemini-cli, antigravity: Google's Embedded WebView Restrictions policy uses JS-fingerprint detection that no UA spoof defeats. RFC 8252 and Google's own Desktop-app OAuth guidance both prescribe the system browser. - codex: auth.openai.com renders blank in our popup on some machines (newer embed detection + regional checks); system browser surfaces the real error. - claude: email magic-link opens in the user's default browser, which is a different cookie jar from the embedded popup, so the popup can never receive the auth. Forcing the OAuth flow into the system browser keeps everything in one cookie jar. The callback for gemini-cli/antigravity lands on /api/subscriptions/callback and runs the exchange server-side; codex uses its fixed 1455 listener; claude is special-cased in p_callback_uri_for_provider below. +# Providers whose OAuth flow MUST run in the user's real browser via shell.openExternal, not the in-Electron window.open popup: - gemini-cli, antigravity: Google's Embedded WebView Restrictions policy uses JS-fingerprint detection that no UA spoof defeats. RFC 8252 and Google's own Desktop-app OAuth guidance both prescribe the system browser. - codex: auth.openai.com renders blank in our popup on some machines (newer embed detection + regional checks); system browser surfaces the real error. - claude: email magic-link opens in the user's default browser, which is a different cookie jar from the embedded popup, so the popup can never receive the auth. Forcing the OAuth flow into the system browser keeps everything in one cookie jar. The callback for gemini-cli/antigravity lands on /api/subscriptions/callback and runs the exchange server-side; codex uses its fixed 1455 listener; claude is special-cased in callback_uri_for_provider below. P_EXTERNAL_BROWSER_PROVIDERS: set[str] = {"gemini-cli", "antigravity", "codex", "claude"} @@ -219,21 +220,32 @@ def p_should_use_external_browser(provider: str) -> bool: return provider in P_EXTERNAL_BROWSER_PROVIDERS -def p_backend_port() -> int: - """Best-effort lookup of the OpenSwarm backend HTTP port. +def resolve_backend_port(observed: Optional[int] = None) -> int: + """The port this backend is actually reachable on, for building OAuth redirect URIs. - Falls back to 8324 (the default in backend/main.py) if OPENSWARM_PORT - hasn't been set yet. backend/main.py:239 sets this env var at startup - before any request handler runs, so `start_oauth` will always see the - correct value. + OPENSWARM_PORT is authoritative and Electron always passes it (main.js), so packaged builds + take the first branch and behave exactly as before. + + It is NOT always set in dev. main.py only exports it inside its `if __name__ == "__main__"` + block, which never runs under `python -m uvicorn backend.main:app --port N`. The old code then + assumed 8324 and stamped that into the redirect URI while uvicorn served a different port, so + Google bounced the user to a dead port and Claude's callback missed the router rewrite. Codex + kept working throughout, because OpenAI pins its own localhost:1455 listener, which is what + made the failure look like "two providers are broken" instead of "the port is wrong". + + `observed` is the port the caller was actually reached on (from the live request), which is + ground truth on every launch path. Only consulted when the env var is absent. """ - try: - return int(os.environ.get("OPENSWARM_PORT", "8324")) - except (TypeError, ValueError): - return 8324 + raw = os.environ.get("OPENSWARM_PORT") + if raw: + try: + return int(raw) + except (TypeError, ValueError): + pass + return observed or 8324 -def p_callback_uri_for_provider(provider: str) -> str: +def callback_uri_for_provider(provider: str, backend_port: Optional[int] = None) -> str: """Return the redirect URI to pass to 9Router's authorize endpoint. Most providers accept 9Router's built-in callback page at port 20128. @@ -252,15 +264,18 @@ def p_callback_uri_for_provider(provider: str) -> str: if provider == "claude": return f"http://localhost:{NINE_ROUTER_PORT}/callback" if provider in P_EXTERNAL_BROWSER_PROVIDERS: - return f"http://localhost:{p_backend_port()}/api/subscriptions/callback" + return f"http://localhost:{resolve_backend_port(backend_port)}/api/subscriptions/callback" return f"http://localhost:{NINE_ROUTER_PORT}/callback" -async def start_oauth(provider: str) -> dict: +async def start_oauth(provider: str, backend_port: Optional[int] = None) -> dict: """Start OAuth flow for a provider. For device_code providers (github, qwen, kiro): returns {user_code, verification_uri, device_code} For authorization_code providers (claude, codex, gemini-cli): returns {authUrl, codeVerifier, state} + + `backend_port` is the port the connect request arrived on; it only matters when OPENSWARM_PORT + is unset, which is the dev-launch case that used to send Google to a dead port. """ async with httpx.AsyncClient(timeout=15.0, headers=cli_auth_headers()) as client: try: @@ -278,7 +293,7 @@ async def start_oauth(provider: str) -> dict: except Exception: pass - callback_url = p_callback_uri_for_provider(provider) + callback_url = callback_uri_for_provider(provider, backend_port) if provider == "codex": # Codex's redirect must be an OpenAI allow-listed loopback port; bind the first free one (1455 else 1457) and use ITS redirect_uri so authorize + token exchange agree. bound_port = await p_start_codex_callback_listener() diff --git a/backend/apps/onboarding/usage/browser_cookies.py b/backend/apps/onboarding/usage/browser_cookies.py index 7aca0489..f10d8144 100644 --- a/backend/apps/onboarding/usage/browser_cookies.py +++ b/backend/apps/onboarding/usage/browser_cookies.py @@ -64,7 +64,7 @@ p_key_cache: Dict[str, Optional[bytes]] = {} @typechecked -def p_win_dpapi_unprotect(data: bytes) -> Optional[bytes]: +def win_dpapi_unprotect(data: bytes) -> Optional[bytes]: """CryptUnprotectData via crypt32.dll (no pywin32 dependency). None on any failure.""" if sys.platform != "win32": return None @@ -105,7 +105,7 @@ def win_storage_key(browser: str) -> Optional[bytes]: raw = base64.b64decode(enc_b64) if raw[:5] != b"DPAPI": return None - return p_win_dpapi_unprotect(raw[5:]) + return win_dpapi_unprotect(raw[5:]) except Exception: return None @@ -203,6 +203,12 @@ def decrypt_cookie_value(enc: bytes, key: bytes) -> Optional[str]: return None +@typechecked +def has_store(domain: str) -> bool: + """Whether any browser store holds records for `domain`, without decrypting and without touching the keychain. The public shape of the presence check, so callers outside this file never need the store tuple.""" + return p_best_store(domain) is not None + + @typechecked def read_provider_cookies(domain: str) -> Dict[str, str]: """Decrypted cookie jar for `domain`, from whichever browser store actually has the session. At most one keychain touch (that store's browser), cached for the process.""" @@ -239,7 +245,7 @@ def read_provider_cookies(domain: str) -> Dict[str, str]: @typechecked def read_provider_cookie_records(domain: str) -> List[Dict[str, Any]]: - """Full cookie records ({name,value,domain,path,secure,httponly}) for `domain`, so Electron's offscreen browser can re-inject the session faithfully and pass Cloudflare with a real Chrome TLS handshake. Same one-store, one-keychain-touch path as read_provider_cookies.""" + """Full cookie records ({name,value,domain,path,secure,httponly,expires_utc}) for `domain`, so Electron's offscreen browser can re-inject the session faithfully and pass Cloudflare with a real Chrome TLS handshake. Same one-store, one-keychain-touch path as read_provider_cookies. `expires_utc` stays in Chromium's own units (microseconds since 1601, 0 = session cookie); whoever needs unix seconds converts.""" store = p_best_store(domain) if store is None: return [] @@ -254,10 +260,11 @@ def read_provider_cookie_records(domain: str) -> List[Dict[str, Any]]: con = sqlite3.connect(f"file:{tmp}?mode=ro", uri=True) cur = con.cursor() cur.execute( - "SELECT name, encrypted_value, host_key, path, is_secure, is_httponly FROM cookies WHERE host_key LIKE ?", + "SELECT name, encrypted_value, host_key, path, is_secure, is_httponly, expires_utc " + "FROM cookies WHERE host_key LIKE ?", (f"%{domain}",), ) - for name, enc, host_key, path, is_secure, is_httponly in cur.fetchall(): + for name, enc, host_key, path, is_secure, is_httponly, expires_utc in cur.fetchall(): if not enc: continue val = decrypt_cookie_value(bytes(enc), key) @@ -266,6 +273,7 @@ def read_provider_cookie_records(domain: str) -> List[Dict[str, Any]]: records.append({ "name": str(name), "value": val, "domain": str(host_key), "path": str(path) or "/", "secure": bool(is_secure), "httponly": bool(is_httponly), + "expires_utc": int(expires_utc or 0), }) con.close() except Exception: @@ -297,3 +305,13 @@ def read_google_session_records() -> List[Dict[str, Any]]: @typechecked def cookie_header(jar: Dict[str, str]) -> str: return "; ".join(f"{k}={v}" for k, v in jar.items()) + + +@typechecked +def logged_in_providers() -> List[str]: + """Which providers have a readable session, WITHOUT decrypting or touching the keychain: safe for a UI presence check.""" + out: List[str] = [] + for provider, domain in (("codex", "chatgpt.com"), ("claude", "claude.ai"), ("gemini", "gemini.google.com")): + if p_best_store(domain) is not None: + out.append(provider) + return out diff --git a/backend/apps/reddit_mcp_shim/reddit_writes.py b/backend/apps/reddit_mcp_shim/reddit_writes.py index 96b2ac20..89e757ee 100644 --- a/backend/apps/reddit_mcp_shim/reddit_writes.py +++ b/backend/apps/reddit_mcp_shim/reddit_writes.py @@ -4,8 +4,18 @@ Posts, comments, edits, deletes, votes, saves, subscriptions, and DMs, all via the user's own session. Each call goes through the rate limiter's write buckets. """ +import json +import re +from typing import Any, Dict, Optional + from backend.apps.reddit_mcp_shim.reddit_http import RedditError, api +# Reddit's own id shapes echoed back in a write response: a "fullname" (t1_ comment, +# t3_ post, ...) and a comment/post permalink. Used to recover a real receipt when the +# structured envelope is absent (the legacy "jquery" response shape, see p_receipt). +P_FULLNAME_RE = re.compile(r"t[1-6]_[0-9a-z]+", re.I) +P_PERMALINK_RE = re.compile(r"/r/[A-Za-z0-9_]+/comments/[A-Za-z0-9_/\-]+") + def p_check(resp: dict) -> dict: """Raise on Reddit's json.errors envelope; return the inner data otherwise.""" @@ -16,6 +26,32 @@ def p_check(resp: dict) -> dict: return j.get("data", {}) if isinstance(j, dict) else {} +def p_receipt(resp: Any, kind: str, exclude: str = "") -> Dict[str, Optional[str]]: + """The just-created thing's own fullname + permalink, robust to Reddit's TWO write + response shapes. Modern api_type=json returns data.things[0].data (comment) or the + fields at data top-level (submit); the LEGACY web endpoint returns a 'jquery' command + array with neither, so the naive parse came back empty = the receipt='ok' bug. Prefer + the structured field; else scan the echoed response for a fullname of the right kind + (t1 comment / t3 post), never the parent id we're replying to.""" + data = p_check(resp) # raises on Reddit's real error envelope + d: Dict[str, Any] = {} + if isinstance(data, dict): + things = data.get("things") + if things and isinstance(things[0], dict): + d = things[0].get("data", {}) or {} + elif data.get("name") or data.get("id") or data.get("url"): + d = data + if d.get("name") or d.get("permalink") or d.get("url"): + return {"id": d.get("name") or d.get("id"), + "permalink": d.get("permalink") or d.get("url")} + blob = json.dumps(resp, default=str) + ex = (exclude or "").lower() + name = next((m for m in P_FULLNAME_RE.findall(blob) + if m.lower().startswith(kind.lower()) and m.lower() != ex), None) + pm = P_PERMALINK_RE.search(blob) + return {"id": name, "permalink": pm.group(0) if pm else None} + + def p_dir(direction: str) -> int: return {"up": 1, "upvote": 1, "down": -1, "downvote": -1, "clear": 0, "none": 0, "unvote": 0}.get( (direction or "").lower(), 0 @@ -34,22 +70,20 @@ def submit(subreddit: str, title: str, kind: str, text: str, url: str, nsfw: boo "api_type": "json", } form["url" if kind == "link" else "text"] = url if kind == "link" else text - data = p_check(api("POST", "/api/submit", form=form, action="submit")) - return {"id": data.get("name") or data.get("id"), "url": data.get("url")} + r = p_receipt(api("POST", "/api/submit", form=form, action="submit"), kind="t3") + return {"id": r["id"], "url": r["permalink"]} def comment(parent_id: str, text: str) -> dict: - data = p_check(api("POST", "/api/comment", form={"thing_id": parent_id, "text": text, "api_type": "json"}, action="comment")) - things = data.get("things", []) - new = things[0].get("data", {}) if things else {} - return {"id": new.get("name"), "permalink": new.get("permalink")} + resp = api("POST", "/api/comment", form={"thing_id": parent_id, "text": text, "api_type": "json"}, action="comment") + r = p_receipt(resp, kind="t1", exclude=parent_id) + return {"id": r["id"], "permalink": r["permalink"]} def edit(thing_id: str, text: str) -> dict: - data = p_check(api("POST", "/api/editusertext", form={"thing_id": thing_id, "text": text, "api_type": "json"}, action="comment")) - things = data.get("things", []) - new = things[0].get("data", {}) if things else {} - return {"id": new.get("name") or thing_id, "edited": True} + resp = api("POST", "/api/editusertext", form={"thing_id": thing_id, "text": text, "api_type": "json"}, action="comment") + r = p_receipt(resp, kind=(thing_id[:2] or "t1"), exclude="") + return {"id": r["id"] or thing_id, "edited": True} def delete(thing_id: str) -> dict: diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index 384677c5..a504dc18 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -54,6 +54,10 @@ class AppSettings(BaseModel): voice_hold_to_talk: bool = True anthropic_api_key: Optional[str] = None browser_homepage: str = "https://www.google.com" + # Opt-in: let a blocked browser agent borrow the sign-in you already have in your everyday + # browser instead of stopping to ask you to log in again. Default OFF because reading your real + # browser's session is your decision to make once, explicitly, not ours to assume. + browser_import_signins: bool = False openai_api_key: Optional[str] = None google_api_key: Optional[str] = None openrouter_api_key: Optional[str] = None diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index 03dbaff1..cc1cbe36 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -32,6 +32,13 @@ def _isolate_browser_state(monkeypatch): monkeypatch.setenv("OPENSWARM_BROWSER_SKILLS_DIR", skills_dir) monkeypatch.setenv("OPENSWARM_BROWSER_METRICS_DIR", metrics_dir) monkeypatch.setenv("OPENSWARM_BROWSER_PLAYBOOK_DIR", playbook_dir) + # The speed levers are default-ON in prod; pin them off for the suite so mocked loop tests keep exact aux-call/turn expectations (same pattern as OPENSWARM_PERSISTENT_CLIENT). The levers are exercised by their own live gates + targeted tests that set the flag explicitly. + monkeypatch.setenv("OSW_PRESTAGE", "0") + monkeypatch.setenv("OSW_FASTREAD_HOP", "0") + monkeypatch.setenv("OSW_PRELUDE_TRIM", "0") + monkeypatch.setenv("OSW_DEADCARD_EVICT", "0") + monkeypatch.setenv("OSW_RECEIPT_DONE", "0") + monkeypatch.setenv("OSW_SEND_SCRIPT", "0") def _reset(): for mod in ("browser_skills", "browser_playbook"): diff --git a/backend/tests/test_browser_agent_loop.py b/backend/tests/test_browser_agent_loop.py index 181935f1..0ec1e5ac 100644 --- a/backend/tests/test_browser_agent_loop.py +++ b/backend/tests/test_browser_agent_loop.py @@ -30,17 +30,32 @@ class Resp: self.usage = type("U", (), {"input_tokens": 1, "output_tokens": 1})() +class FakeStream: + # mirrors anthropic's messages.stream(): async CM whose get_final_message() returns the turn + def __init__(self, resp): self.resp = resp + async def __aenter__(self): return self + async def __aexit__(self, *a): return False + async def get_final_message(self): return self.resp + + class FakeLLM: def __init__(self, scripted): self.scripted = scripted; self.turn = 0; self.calls = [] self.messages = self - async def create(self, **kw): + def p_next(self, kw): self.calls.append(kw) i = min(self.turn, len(self.scripted) - 1) self.turn += 1 return self.scripted[i] + async def create(self, **kw): + return self.p_next(kw) + + def stream(self, **kw): + # the loop now streams; return an async-CM yielding the scripted turn + return FakeStream(self.p_next(kw)) + class FakeAux: def __init__(self): @@ -63,6 +78,17 @@ def p_rp(goal, mem="Share dialog is a cross-origin iframe; use the index list.") DOC_URL = "https://docs.google.com/document/d/abc/edit" +def p_run_settled(**kw): + """run_browser_agent then drain the backgrounded learning task; the distill + no longer blocks the reply path, so tests asserting its effects must settle it.""" + async def p_go(): + r = await BA.run_browser_agent(**kw) + if BA.learn_tasks: + await asyncio.gather(*list(BA.learn_tasks), return_exceptions=True) + return r + return asyncio.run(p_go()) + + def p_install(monkeypatch, primary, aux): # local imports inside run_browser_agent resolve from these source modules import backend.apps.settings.settings as settings_mod @@ -71,10 +97,11 @@ def p_install(monkeypatch, primary, aux): import backend.apps.agents.agent_manager as am_mod monkeypatch.setattr(settings_mod, "load_settings", lambda: {"fake": True}, raising=True) - monkeypatch.setattr(reg_mod, "find_builtin_model", lambda m: object(), raising=True) + # a dict (not object()) so get_api_type's (entry or {}).get("api") works like the real registry rows + monkeypatch.setattr(reg_mod, "find_builtin_model", lambda m: {"api": "anthropic"}, raising=True) monkeypatch.setattr(reg_mod, "resolve_model_id_for_sdk", lambda m, s: "primary-x", raising=True) - async def p_aux_resolve(s, preferred_tier="haiku"): + async def p_aux_resolve(s, preferred_tier="haiku", primary_api=None): return ("aux-x", None) monkeypatch.setattr(reg_mod, "resolve_aux_model", p_aux_resolve, raising=True) @@ -222,10 +249,15 @@ def test_confirmed_send_ends_the_run_instead_of_stalling(monkeypatch): # the send ran and the run ended FAST (the stall guard stopped it), well before consuming all 8 scripted stall turns assert any(c["action"] == "click_index" and c["params"].get("index") == 99 for c in sent) assert primary.turn <= 4, f"run stalled {primary.turn} turns after a confirmed send" - # structured success + a clean human summary, never the internal tag - assert result.get("done") is True + # A clean human summary, never the internal tag. NOT `done is True`: this run only ever saw the + # click register, and no composer receipt ever arrived, so it has no evidence the message + # landed. Reporting success here is the exact live failure measured on X 2026-07-28 ("your + # message went through and it's showing" while nothing had posted). The stall guard's job is to + # stop the spinning, not to bless the outcome, so what is asserted here is that it ENDED, and + # ended honestly. See test_browser_send_honesty.py. assert "OUTCOME" not in result["summary"] assert result["summary"].strip() + assert result.get("done") is False, "an unverified send must not report success" def test_done_tool_delivers_a_clean_human_summary(monkeypatch): @@ -938,9 +970,9 @@ def test_playbook_distills_on_success_survives_restart_and_seeds_next_run(monkey ]) pbaux = PBAux() p_install(monkeypatch, primary1, pbaux) - asyncio.run(BA.run_browser_agent( + p_run_settled( task="find design engineers", browser_id="b1", model="sonnet", initial_url=DOC_URL, - )) + ) assert pbaux.calls >= 1, "a substantive success must trigger the distill aux call" assert PB.get_playbook("docs.google.com"), "playbook recorded for the host" @@ -995,7 +1027,7 @@ def test_ambient_memory_signals_fire_calmly(monkeypatch): # Run 1: nothing learned yet -> NO recall line, but it learns -> closing line. p_install(monkeypatch, p_run(), PBAux()) monkeypatch.setattr(BA.ws_manager, "send_to_session", p_cap, raising=False) - asyncio.run(BA.run_browser_agent(task="find engineers", browser_id="b1", model="sonnet", initial_url=DOC_URL)) + p_run_settled(task="find engineers", browser_id="b1", model="sonnet", initial_url=DOC_URL) joined1 = " ".join(msgs) assert "Picking up what I learned" not in joined1, "no recall on the first-ever visit" assert "so I'm faster here next time" in joined1, "closing 'learned' line after first success" @@ -1036,8 +1068,8 @@ def test_playbook_not_learned_from_a_ghost_completion(monkeypatch): asyncio.run(BA.run_browser_agent( task="do the thing", browser_id="b1", model="sonnet", initial_url=DOC_URL, )) - # the only aux call allowed here is the stuck-adjudication; the playbook distill must NOT have stored anything for a dishonest run - assert PB.get_playbook("docs.google.com") == [] + # the only aux call allowed here is the stuck-adjudication; the playbook distill must NOT have LEARNED anything for a dishonest run (load = learned-only; get_playbook would also return the shipped seed for this host) + assert PB.load("docs.google.com") == [] def test_batch_replay_runs_a_read_loop_for_all_values(monkeypatch): @@ -1322,12 +1354,13 @@ def test_post_action_state_truncates_long_lists(monkeypatch): from backend.apps.agents.browser import browser_agent as ba calls = [] monkeypatch.setattr(ba.browser_wait, "smart_wait", p_fake_settle(calls)) - long_list = "\n".join(f'[{i}]