mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] merge: bring eric/browser onto eric/dev (30 conflicts resolved, gates green)
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
name: windows-session-import
|
||||
|
||||
# The Windows half of the browser session-import path can only be proven on Windows: it talks to
|
||||
# DPAPI through crypt32, and every mock of that agrees with whatever we already believed. This job
|
||||
# runs the round-trip tests on a real windows runner, so a layout mistake fails here instead of in
|
||||
# a user's onboarding.
|
||||
|
||||
on:
|
||||
push:
|
||||
paths:
|
||||
- 'backend/apps/onboarding/usage/browser_cookies.py'
|
||||
- 'backend/tests/test_browser_cookies_windows_live.py'
|
||||
- '.github/workflows/windows-session-import.yml'
|
||||
workflow_dispatch:
|
||||
|
||||
jobs:
|
||||
verify:
|
||||
runs-on: windows-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: '3.13'
|
||||
|
||||
- name: Install the two things the module actually needs
|
||||
run: pip install cryptography typeguard pytest
|
||||
|
||||
- name: Prove the DPAPI + AES-GCM round trip on real Windows
|
||||
run: python -m pytest backend/tests/test_browser_cookies_windows_live.py -v
|
||||
|
||||
- name: Fail if the suite skipped itself
|
||||
shell: pwsh
|
||||
run: |
|
||||
# A skip reads as green. On a windows runner these tests MUST run, so an all-skipped
|
||||
# result means the platform gate is wrong and the job is proving nothing.
|
||||
$out = python -m pytest backend/tests/test_browser_cookies_windows_live.py -q 2>&1 | Out-String
|
||||
if ($out -match 'skipped') {
|
||||
Write-Error "Windows tests skipped on a Windows runner; the platform gate is broken:`n$out"
|
||||
exit 1
|
||||
}
|
||||
Write-Host "Ran on real Windows, no skips."
|
||||
@@ -4,7 +4,7 @@ import time
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import Any, Dict
|
||||
|
||||
from fastapi import HTTPException
|
||||
from fastapi import HTTPException, Request
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.agents.agent_manager import agent_manager
|
||||
@@ -425,7 +425,7 @@ async def subscriptions_status():
|
||||
|
||||
|
||||
@agents.router.post("/subscriptions/connect")
|
||||
async def subscriptions_connect(body: dict):
|
||||
async def subscriptions_connect(body: dict, request: Request):
|
||||
"""Start OAuth flow for a subscription provider."""
|
||||
from backend.apps.nine_router import is_running, ensure_running, start_oauth
|
||||
provider = body.get("provider", "")
|
||||
@@ -446,7 +446,9 @@ async def subscriptions_connect(body: dict):
|
||||
pass
|
||||
|
||||
try:
|
||||
result = await start_oauth(provider)
|
||||
# The port the user's app actually reached us on beats guessing the default; only consulted
|
||||
# when OPENSWARM_PORT is unset (dev uvicorn launches), never in packaged builds.
|
||||
result = await start_oauth(provider, request.url.port)
|
||||
|
||||
if result.get("flow") == "authorization_code" and result.get("state"):
|
||||
from backend.apps.oauth_state import pending_oauth
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -111,17 +111,24 @@ P_LIVE_IRREVERSIBLE_RE = re.compile(
|
||||
r"confirm|apply|accept|decline|delete|remove|unsend|withdraw|endorse)\b",
|
||||
re.I,
|
||||
)
|
||||
# Composer OPENERS phrased with a send-word: LinkedIn's profile button is literally
|
||||
# named "Send a message to <person>", which opens the compose box (reversible), not
|
||||
# a real Send. A true Send control is short and exact ("Send", "Send now"); these
|
||||
# describe opening a conversation, so they must NOT trip the irreversible boundary.
|
||||
P_SEND_OPENER_RE = re.compile(r"send (a |an |the )?(message|note|inmail|dm) to\b", re.I)
|
||||
|
||||
|
||||
def is_replay_boundary(step: dict) -> bool:
|
||||
"""The genuinely irreversible step where a learned skill's mechanical replay
|
||||
must STOP and hand to the live agent. Same as is_send_step EXCEPT a composer
|
||||
OPENER ('Message'/'DM' click) is reversible and NOT a boundary: the prefix can
|
||||
mechanically open the composer, and only the real Send (and composer typing)
|
||||
crosses to the live model. Uses the same opener-excluded wordlist the live
|
||||
send-guard already trusts, so a recorded Send still stops the prefix."""
|
||||
OPENER ('Message'/'DM' click, incl. 'Send a message to X') is reversible and NOT
|
||||
a boundary: the prefix can mechanically open the composer, and only the real Send
|
||||
(and composer typing) crosses to the live model."""
|
||||
action = step.get("action")
|
||||
if action == "click" and P_LIVE_IRREVERSIBLE_RE.search(str(step.get("name") or "")):
|
||||
name = str(step.get("name") or "")
|
||||
if action == "click" and P_SEND_OPENER_RE.search(name):
|
||||
return False # opener phrasing, not a real send
|
||||
if action == "click" and P_LIVE_IRREVERSIBLE_RE.search(name):
|
||||
return True
|
||||
if action == "type" and P_COMPOSE_SEL_RE.search(str(step.get("selector") or "")):
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,260 @@
|
||||
"""BrowserDeleteItem: a model-invoked tool that removes ONE on-page item the model names by
|
||||
text, deterministically. The model handles getting to the item's page (its strength); this runs
|
||||
the site's own remove flow (open that item's overflow menu -> Delete -> confirm -> verify-gone),
|
||||
which the model fails at by hand (measured live on X: 4 aborts on the tiny caret menu).
|
||||
|
||||
Resolve-in-JS, click-with-real-input: each step RESOLVES the next control's viewport position in
|
||||
a pierced-shadow DOM query, and the click is dispatched through the OS-level input path
|
||||
(BrowserClickPoint). Synthetic el.click() is ignored by web-component sites (shreddit live:
|
||||
the flow reached Delete yet nothing happened; the model's trusted clicks on the same controls
|
||||
worked), and a real click also lands on whatever is topmost, so overlays can't be mis-clicked.
|
||||
|
||||
Safety, in code:
|
||||
- Resolution happens ONLY inside the element that contains the target text, so it can never
|
||||
touch another item.
|
||||
- The site enforces ownership (only your own item exposes Delete), so a target you don't own has
|
||||
no menu entry and the tool reports that, it never forces one.
|
||||
- Success REQUIRES verify-gone (the target text left the page). One destructive confirm click.
|
||||
- Flag-gated (OSW_DELETE_SCRIPT): the tool is hidden from the model until Eric flips it.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from typing import Any, Awaitable, Callable, Dict
|
||||
|
||||
from backend.apps.agents.browser import browser_submit_click
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MIN_TARGET_CHARS = 6
|
||||
|
||||
ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]]
|
||||
|
||||
|
||||
def delete_tool_enabled() -> bool:
|
||||
return os.environ.get("OSW_DELETE_SCRIPT", "0") != "0"
|
||||
|
||||
|
||||
# One resolver, four steps. Each call re-queries the live DOM (pierced), scrolls the control into
|
||||
# view when needed, and returns the control's viewport center as percents for BrowserClickPoint.
|
||||
# 'verify' returns gone-ness instead of a position. Controls render a beat late, so steps poll.
|
||||
P_RESOLVE_JS = r"""(async () => {
|
||||
const STEP = %s;
|
||||
const TARGET = %s;
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
const norm = (s) => (s || '').replace(/\s+/g, ' ').trim();
|
||||
// Web-component sites (shreddit, YouTube) hide the item AND its menus in shadow roots, so every lookup pierces.
|
||||
const deep = (root, sel, out, depth) => {
|
||||
if (depth > 10 || out.length > 4000) return out;
|
||||
let hits; try { hits = root.querySelectorAll(sel); } catch (e) { hits = []; }
|
||||
for (const el of hits) out.push(el);
|
||||
let all; try { all = root.querySelectorAll('*'); } catch (e) { return out; }
|
||||
for (const el of all) { if (el.shadowRoot) deep(el.shadowRoot, sel, out, depth + 1); }
|
||||
return out;
|
||||
};
|
||||
const vis = (el) => !!el && el.getClientRects().length > 0;
|
||||
const center = (el) => {
|
||||
const r0 = el.getBoundingClientRect();
|
||||
if (r0.top < 0 || r0.bottom > window.innerHeight) el.scrollIntoView({ block: 'center' });
|
||||
const r = el.getBoundingClientRect();
|
||||
return { xPct: (r.left + r.width / 2) / window.innerWidth * 100,
|
||||
yPct: (r.top + r.height / 2) / window.innerHeight * 100,
|
||||
label: norm(el.getAttribute('aria-label') || el.textContent || '').slice(0, 40) };
|
||||
};
|
||||
const CONTAINERS = 'article,[role="article"],li,[role="listitem"],tr,[role="row"],shreddit-post,'
|
||||
+ '[data-testid*="tweet"],[data-testid*="post"],[data-testid*="comment"],[data-testid*="Post"],[id^="t3_"],[id^="t1_"]';
|
||||
const MORE = 'button[aria-label*="More" i],button[aria-label*="option" i],'
|
||||
+ '[data-testid="caret"],button[aria-haspopup="menu"],button[aria-haspopup="true"],[aria-label*="menu" i]';
|
||||
const holders = () => deep(document, CONTAINERS, [], 0)
|
||||
.filter((el) => el.textContent && el.textContent.includes(TARGET))
|
||||
.sort((a, b) => (a.textContent || '').length - (b.textContent || '').length); // tightest match first
|
||||
|
||||
if (STEP === 'verify') {
|
||||
// Gone = the text left the page, OR every remaining holder is a deletion TOMBSTONE (reddit
|
||||
// swaps the tile for a "Post deleted" placeholder that keeps the title text). The substance
|
||||
// gate stops a still-loading page (post-refresh) from reading as gone: never a false removed.
|
||||
const TOMB = /post deleted|comment deleted|\[deleted\]|deleted by|removed by/i;
|
||||
for (let attempt = 0; attempt < 7; attempt++) {
|
||||
if (attempt) await sleep(1000);
|
||||
if ((document.body.innerText || '').length < 500) continue;
|
||||
const hs = holders();
|
||||
if (!hs.length || hs.every((h) => TOMB.test(h.textContent || ''))) return { ok: true, stage: 'verify' };
|
||||
}
|
||||
return { ok: false, stage: 'verify' };
|
||||
}
|
||||
if (STEP === 'direct') {
|
||||
// Row-action sites (Gmail) put a literal Delete control ON the item, no kebab menu; click it
|
||||
// straight and let verify arbitrate. Exact cleaned-label match so 'Delete row' etc never fires.
|
||||
const clean = (s) => norm((s || '').replace(/[-]|\([^)]*\)/g, '')).toLowerCase();
|
||||
const DIRECT = new Set(['delete', 'move to trash', 'trash']);
|
||||
if (!holders().length) return { ok: false, stage: 'find', msg: 'target text not on this page' };
|
||||
for (const h of holders()) {
|
||||
const cands = deep(h, 'button,[role="button"],li,span[role]', [], 0)
|
||||
.filter((b) => DIRECT.has(clean(b.getAttribute('aria-label') || b.textContent || '')));
|
||||
const v = cands.find(vis);
|
||||
if (v) { h.scrollIntoView({ block: 'center' }); return { ok: true, stage: 'direct', ...center(v) }; }
|
||||
if (cands.length) {
|
||||
h.scrollIntoView({ block: 'center' });
|
||||
const hr = h.getBoundingClientRect();
|
||||
return { ok: false, stage: 'direct', hoverFirst: true,
|
||||
xPct: (hr.left + hr.width / 2) / window.innerWidth * 100,
|
||||
yPct: (hr.top + Math.min(20, hr.height / 2)) / window.innerHeight * 100,
|
||||
msg: 'direct delete control hidden until hover' };
|
||||
}
|
||||
}
|
||||
return { ok: false, stage: 'direct', optional: true, msg: 'no direct delete control on the item' };
|
||||
}
|
||||
if (STEP === 'more') {
|
||||
if (!holders().length) return { ok: false, stage: 'find', msg: 'target text not on this page' };
|
||||
// A post tile carries OTHER kebabs too (reddit's user-attribution row: 'Open user actions',
|
||||
// measured live opening the wrong menu). Rank: named overflow first, then reddit's unlabeled
|
||||
// haspopup kebab, then generic; user/share/moderation controls never.
|
||||
const rank = (el) => {
|
||||
// Reddit quirks, all measured live: the post kebab is LABELED 'Open user actions' (inside
|
||||
// shreddit-post-overflow-menu, so the host outranks the misleading label), a 0x0 DECOY
|
||||
// lives in mod-content-state-indicators, and Share/mod controls also carry haspopup.
|
||||
const host = (el.getRootNode() && el.getRootNode().host) ? el.getRootNode().host.tagName : '';
|
||||
if (/^MOD-/.test(host)) return 4;
|
||||
const l = norm(el.getAttribute('aria-label') || el.textContent || '').toLowerCase();
|
||||
if (/overflow|more/.test(l) || /OVERFLOW/.test(host)) return 0;
|
||||
if (!l && el.getAttribute('aria-haspopup')) return 1;
|
||||
if (/share|award|vote|join|follow|moderat|approve/.test(l)) return 4;
|
||||
if (/user|profile|author/.test(l)) return 3;
|
||||
return 2;
|
||||
};
|
||||
for (let attempt = 0; attempt < 6; attempt++) {
|
||||
if (attempt) await sleep(500);
|
||||
for (const h of holders()) {
|
||||
const cands = deep(h, MORE, [], 0).filter((c) => rank(c) < 4);
|
||||
const visBest = cands.filter(vis).sort((a, b) => rank(a) - rank(b))[0];
|
||||
if (visBest) { h.scrollIntoView({ block: 'center' }); return { ok: true, stage: 'more', ...center(visBest) }; }
|
||||
// Hover-revealed kebab: exists but zero rects until the tile is hovered with REAL input.
|
||||
// Only when NOTHING visible qualifies; hand the tile position back for a hover + retry.
|
||||
const hid = cands.find((c) => !vis(c) && rank(c) <= 1);
|
||||
if (hid) {
|
||||
h.scrollIntoView({ block: 'center' });
|
||||
const hr = h.getBoundingClientRect();
|
||||
return { ok: false, stage: 'more', hoverFirst: true,
|
||||
xPct: (hr.left + hr.width / 2) / window.innerWidth * 100,
|
||||
yPct: (hr.top + Math.min(40, hr.height / 2)) / window.innerHeight * 100,
|
||||
msg: 'overflow control hidden until hover' };
|
||||
}
|
||||
}
|
||||
}
|
||||
return { ok: false, stage: 'more', msg: 'found the item but no overflow/More control on it' };
|
||||
}
|
||||
if (STEP === 'menuitem') {
|
||||
const DEL = /^\s*(delete|remove)\b/i; // exact-ish: not "delete row" or "remove filter"
|
||||
const MENUS = '[role="menu"],[role="listbox"],faceplate-menu,faceplate-dropdown-menu,[data-testid="Dropdown"]';
|
||||
// Long poll: shreddit's menu content arrives through an async loader, measured up to ~20s
|
||||
// after the kebab click on a cold page; X resolves on the first attempt so the tail is free.
|
||||
for (let attempt = 0; attempt < 12; attempt++) {
|
||||
if (attempt) await sleep(900);
|
||||
// Only entries inside an OPEN menu count; a page-wide 'Delete' from another context must
|
||||
// never be clicked (measured live after the wrong kebab opened).
|
||||
const menus = deep(document, MENUS, [], 0).filter(vis);
|
||||
for (const s of (menus.length ? menus : [document])) {
|
||||
const del = deep(s, '[role="menuitem"],[role="option"],button,a,li', [], 0)
|
||||
.find((m) => DEL.test(norm(m.textContent)) && vis(m));
|
||||
if (del) return { ok: true, stage: 'menuitem', ...center(del) };
|
||||
}
|
||||
}
|
||||
return { ok: false, stage: 'menuitem',
|
||||
msg: 'no Delete/Remove entry in the menu (is this your own item?)',
|
||||
menu: deep(document, '[role="menuitem"],[role="option"]', [], 0)
|
||||
.map((m) => norm(m.textContent)).filter(Boolean).slice(0, 10) };
|
||||
}
|
||||
if (STEP === 'confirm') {
|
||||
const CONF = /^\s*(delete|remove|yes|confirm)\b/i;
|
||||
// Scan EVERY visible dialog candidate then the whole document; taking [0] once grabbed a
|
||||
// random lazy-loader wrapper and looked straight past the real open dialog (measured live).
|
||||
const DLG = '[role="dialog"],[role="alertdialog"],[data-testid="confirmationSheetDialog"],faceplate-dialog';
|
||||
for (let attempt = 0; attempt < 7; attempt++) {
|
||||
if (attempt) await sleep(700);
|
||||
const scopes = [...deep(document, DLG, [], 0).filter(vis), document];
|
||||
for (const dlg of scopes) {
|
||||
const conf = deep(dlg, '[data-testid="confirmationSheetConfirm"]', [], 0).find(vis)
|
||||
|| deep(dlg, 'button', [], 0).find((b) => CONF.test(norm(b.textContent)) && vis(b));
|
||||
if (conf) return { ok: true, stage: 'confirm', fromDialog: dlg !== document, ...center(conf) };
|
||||
}
|
||||
}
|
||||
return { ok: false, stage: 'confirm', optional: true, msg: 'no confirm dialog appeared' };
|
||||
}
|
||||
return { ok: false, stage: 'eval', msg: 'unknown step' };
|
||||
})()"""
|
||||
|
||||
|
||||
def resolve_expression(step: str, target_text: str) -> str:
|
||||
return P_RESOLVE_JS % (json.dumps(step), json.dumps(target_text))
|
||||
|
||||
|
||||
async def run_delete(target_text: str, browser_id: str, tab_id: str,
|
||||
execute_tool: ToolRunner) -> Dict[str, Any]:
|
||||
"""The full remove flow: resolve each control in-page, click it with REAL input, verify gone.
|
||||
Any unreadable resolve is an honest failure at that stage (never a false 'removed')."""
|
||||
|
||||
async def resolve(step: str) -> Dict[str, Any]:
|
||||
res = await execute_tool(
|
||||
"BrowserEvaluate", {"expression": resolve_expression(step, target_text)}, browser_id, tab_id)
|
||||
return browser_submit_click.parse_eval_value(res) or {"ok": False, "stage": "eval",
|
||||
"msg": "the remove flow returned no readable result"}
|
||||
|
||||
async def hover_then_retry(step: str, r: Dict[str, Any]) -> Dict[str, Any]:
|
||||
# Reveal a hover-only control with a real mouse move over the tile, then re-resolve once.
|
||||
logger.info(f"[browser-deletescript] {step} control hidden; hovering the tile to reveal it")
|
||||
await execute_tool("BrowserClickPoint",
|
||||
{"xPercent": float(r["xPct"]), "yPercent": float(r["yPct"]),
|
||||
"hoverOnly": True}, browser_id, tab_id)
|
||||
await asyncio.sleep(0.7)
|
||||
return await resolve(step)
|
||||
|
||||
async def click_step(step: str, r: Dict[str, Any], settle_s: float) -> None:
|
||||
logger.info(f"[browser-deletescript] step={step} label={str(r.get('label') or '')[:40]!r} "
|
||||
f"at {float(r['xPct']):.1f},{float(r['yPct']):.1f}")
|
||||
await execute_tool("BrowserClickPoint",
|
||||
{"xPercent": float(r["xPct"]), "yPercent": float(r["yPct"])}, browser_id, tab_id)
|
||||
await asyncio.sleep(settle_s)
|
||||
|
||||
# Row-action shortcut first (Gmail): a literal Delete control on the item skips the menu walk;
|
||||
# a confirm may still follow (Drive-style), so that step runs either way and stays optional.
|
||||
p_direct = await resolve("direct")
|
||||
if not p_direct.get("ok") and p_direct.get("hoverFirst"):
|
||||
p_direct = await hover_then_retry("direct", p_direct)
|
||||
p_via_direct = False
|
||||
if p_direct.get("ok"):
|
||||
await click_step("direct", p_direct, 1.5)
|
||||
p_via_direct = True
|
||||
steps = (("confirm", 1.8),)
|
||||
elif str(p_direct.get("stage")) == "find":
|
||||
return {"removed": False, "stage": "find", "msg": str(p_direct.get("msg") or "")}
|
||||
else:
|
||||
steps = (("more", 1.0), ("menuitem", 1.0), ("confirm", 1.8))
|
||||
for step, settle_s in steps:
|
||||
r = await resolve(step)
|
||||
if not r.get("ok") and r.get("hoverFirst"):
|
||||
r = await hover_then_retry(step, r)
|
||||
if not r.get("ok"):
|
||||
if step == "confirm" and r.get("optional"):
|
||||
break # some sites delete without a confirm; the verify below is the arbiter
|
||||
return {"removed": False, "stage": str(r.get("stage") or step), "msg": str(r.get("msg") or "")}
|
||||
if step == "confirm" and p_via_direct and not r.get("fromDialog"):
|
||||
break # after a direct delete, only a REAL dialog earns a confirm click
|
||||
await click_step(step, r, settle_s)
|
||||
v = await resolve("verify")
|
||||
if not v.get("ok"):
|
||||
# Some clients keep the dead tile mounted until a reload (shreddit, measured live: the
|
||||
# delete lands server-side while the DOM never flips). Refresh once and re-verify; the
|
||||
# verify's substance gate keeps a half-loaded page from reading as gone.
|
||||
p_loc = browser_submit_click.parse_eval_value(
|
||||
await execute_tool("BrowserEvaluate", {"expression": "({href: location.href})"}, browser_id, tab_id)) or {}
|
||||
p_href = str(p_loc.get("href") or "")
|
||||
if p_href.startswith("http"):
|
||||
logger.info("[browser-deletescript] tile still mounted; refreshing to re-verify")
|
||||
await execute_tool("BrowserNavigate", {"url": p_href}, browser_id, tab_id)
|
||||
await asyncio.sleep(3.0)
|
||||
v = await resolve("verify")
|
||||
removed = bool(v.get("ok"))
|
||||
return {"removed": removed, "stage": "done",
|
||||
"msg": "item removed" if removed else "clicked delete but the item is still on the page"}
|
||||
@@ -0,0 +1,180 @@
|
||||
"""Delivery ground-truth for a write: did the post ACTUALLY land, or did the site clear the
|
||||
composer and silently eat it?
|
||||
|
||||
A cleared composer proves delivery everywhere EXCEPT the ghost-drop hosts (YouTube-class), which
|
||||
accept an automated post, clear the box, maybe render it for a beat, then drop it server-side. On
|
||||
those we re-read the live page to confirm the post PERSISTS before anyone claims success;
|
||||
everywhere else the cleared composer stays the trusted proxy (proven across X/Reddit/LinkedIn/
|
||||
Gmail) and this module is never consulted, so proven sends keep their exact speed.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import re
|
||||
from typing import Awaitable, Callable, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.agents.browser import browser_submit_click
|
||||
|
||||
ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]]
|
||||
|
||||
# Hosts known to accept-then-silently-drop an automated post. A newly-found one is a one-line add.
|
||||
GHOST_DROP_HOSTS = ("youtube.com",)
|
||||
|
||||
# What a site says when it REFUSED the write. Only ever consulted inside a live announcement
|
||||
# region, never against the whole page, so an unrelated "failed" in an article body can't match.
|
||||
P_REJECTION_RE = re.compile(
|
||||
r"something went wrong|went wrong|couldn'?t\s|could not\s|unable to|failed to|"
|
||||
r"\bfailed\b|try again|too many|rate.?limit|limit exceeded|not allowed|"
|
||||
r"blocked|error occurred|wasn'?t (?:sent|posted)|was not (?:sent|posted)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
@typechecked
|
||||
def is_ghost_drop_host(url: str) -> bool:
|
||||
host = (urlparse(url or "").hostname or "").lower().lstrip(".")
|
||||
return any(host == g or host.endswith("." + g) for g in GHOST_DROP_HOSTS)
|
||||
|
||||
|
||||
@typechecked
|
||||
def delivery_probe_expression(payload: str) -> str:
|
||||
"""JS reporting whether a distinctive chunk of `payload` is rendered in the page's VISIBLE
|
||||
text. Run only AFTER the composer cleared, so a hit means the text lives in real page content
|
||||
(the posted item / a confirmation), not the emptied composer."""
|
||||
needle = " ".join((payload or "").split())[:80]
|
||||
return ("(()=>{try{var n=" + json.dumps(needle) + ";"
|
||||
"var t=(document.body&&document.body.innerText)||'';"
|
||||
"return {visible: n.length>0 && t.indexOf(n)!==-1};}"
|
||||
"catch(e){return {visible:false};}})()")
|
||||
|
||||
|
||||
@typechecked
|
||||
async def payload_visible(
|
||||
payload: str, browser_id: str, tab_id: str, execute_tool: ToolRunner
|
||||
) -> Optional[bool]:
|
||||
"""True = seen on the page, False = looked and it is NOT there, None = could not look.
|
||||
|
||||
The third case is not pedantry. Returning False for a probe that timed out or came back
|
||||
unreadable is asserting absence from a failed observation, and that is the same mistake as a
|
||||
receipt claiming delivery it never saw, pointed the other way: it tells the user a post did not
|
||||
land when nobody actually checked. Measured tonight, the identical shape in the test harness
|
||||
scored every unreadable verification as a successful delete and left six posts on a real
|
||||
account while reporting them cleaned.
|
||||
"""
|
||||
try:
|
||||
r = await asyncio.wait_for(execute_tool(
|
||||
"BrowserEvaluate", {"expression": delivery_probe_expression(payload)},
|
||||
browser_id, tab_id), timeout=6.0)
|
||||
except Exception:
|
||||
return None
|
||||
v = browser_submit_click.parse_eval_value(r)
|
||||
if not isinstance(v, dict) or "visible" not in v:
|
||||
return None
|
||||
return bool(v.get("visible"))
|
||||
|
||||
|
||||
@typechecked
|
||||
def rejection_probe_expression() -> str:
|
||||
"""JS returning the text of the page's live ANNOUNCEMENT regions only.
|
||||
|
||||
role="alert" and aria-live are how sites are required to announce a transient result to
|
||||
assistive tech, so error toasts land here on every major site without us naming any of them.
|
||||
Scoped deliberately: reading whole-page text for the word "failed" would match articles,
|
||||
changelogs and half the internet."""
|
||||
return ("(()=>{try{var out=[];"
|
||||
"var sel='[role=alert],[role=alertdialog],[aria-live=assertive],[aria-live=polite]';"
|
||||
"document.querySelectorAll(sel).forEach(function(e){"
|
||||
"var s=(e.innerText||'').trim(); if(s) out.push(s);});"
|
||||
"return {text: out.join(' | ').slice(0,600)};}"
|
||||
"catch(e){return {text:''};}})()")
|
||||
|
||||
|
||||
@typechecked
|
||||
async def send_rejected(browser_id: str, tab_id: str, execute_tool: ToolRunner) -> bool:
|
||||
"""Did the site announce that the write FAILED, right after the composer cleared?
|
||||
|
||||
A cleared composer is the receipt this whole fast path rests on, and the code has long admitted
|
||||
it "cannot tell submitted from dismissed". The realistic way that bites is not a mis-click: it
|
||||
is the site accepting the click, clearing the box, and popping "Something went wrong" or a rate
|
||||
limit. The receipt then reads as success and the agent tells the user it posted.
|
||||
|
||||
This only ever DEMOTES a claim, and only on an explicit failure announcement, so a normal send
|
||||
(no alert region, or a success toast) is untouched and keeps its measured speed. Any read
|
||||
failure returns False, because refusing to claim delivery on the basis of a broken probe would
|
||||
invent failures that did not happen."""
|
||||
try:
|
||||
r = await asyncio.wait_for(execute_tool(
|
||||
"BrowserEvaluate", {"expression": rejection_probe_expression()},
|
||||
browser_id, tab_id), timeout=4.0)
|
||||
except Exception:
|
||||
return False
|
||||
v = browser_submit_click.parse_eval_value(r)
|
||||
if not isinstance(v, dict):
|
||||
return False
|
||||
return bool(P_REJECTION_RE.search(str(v.get("text") or "")))
|
||||
|
||||
|
||||
@typechecked
|
||||
def rejected_send_note(url: str, payload: str) -> str:
|
||||
"""Honest line for a send the SITE said no to. Distinct from the unverified case: here we are
|
||||
not guessing, the page told us, so the user should be told plainly rather than asked to check."""
|
||||
host = urlparse(url or "").hostname or "the site"
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
clip = payload if len(payload) <= 80 else payload[:77] + "..."
|
||||
return (f'I typed "{clip}" and clicked send, but {host} rejected it: the composer cleared and '
|
||||
f'the page showed an error instead of posting. It did NOT go through. I did not retry, '
|
||||
f'since whatever the site refused is likely to be refused again.')
|
||||
|
||||
|
||||
@typechecked
|
||||
async def ghost_delivery_confirmed(
|
||||
payload: str, browser_id: str, tab_id: str, execute_tool: ToolRunner
|
||||
) -> bool:
|
||||
"""For a ghost-drop host: did the post render AND survive the server-side drop window? True
|
||||
only if the payload is visible now and STILL visible a few seconds later. A post that never
|
||||
rendered, or rendered then vanished, returns False, so we never claim a delivery the site ate.
|
||||
Pure page reads (no navigation), invisible to the site."""
|
||||
# `is not True` deliberately: an unknown must NOT confirm. This is the one place where
|
||||
# collapsing unknown into "no" is right, because the caller is deciding whether to CLAIM a
|
||||
# delivery, and withholding an uncertain claim is the safe direction.
|
||||
if await payload_visible(payload, browser_id, tab_id, execute_tool) is not True:
|
||||
return False
|
||||
await asyncio.sleep(3.5)
|
||||
return await payload_visible(payload, browser_id, tab_id, execute_tool) is True
|
||||
|
||||
|
||||
@typechecked
|
||||
def unverified_send_note(url: str, payload: str) -> str:
|
||||
"""Honest line for a send whose click RAN but whose two-sided receipt never arrived.
|
||||
|
||||
Deliberately weaker than unconfirmed_delivery_note: there the composer cleared and the post
|
||||
later vanished, so we know it was submitted. Here we never got the clear at all, so we know
|
||||
strictly less and must claim strictly less. Measured 2026-07-28 on X: the agent reported "your
|
||||
message went through and it's showing in the conversation now" on exactly this evidence and
|
||||
nothing had been posted. Overclaiming here is the worst failure this agent has, because the
|
||||
user stops checking.
|
||||
"""
|
||||
host = urlparse(url or "").hostname or "the site"
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
clip = payload if len(payload) <= 80 else payload[:77] + "..."
|
||||
return (f'I typed "{clip}" and clicked send on {host}, but I could NOT confirm it actually '
|
||||
f'posted: the composer never cleared, which is the signal I rely on. It may or may not '
|
||||
f'have gone through, so please check before relying on it. I did not try again, because '
|
||||
f'a blind retry is how you end up posting twice.')
|
||||
|
||||
|
||||
def unconfirmed_delivery_note(url: str, payload: str) -> str:
|
||||
"""Plain honest fallback line when a ghost-drop send can't be confirmed (the aux-composed
|
||||
version in browser_agent is preferred; this is the never-fails template behind it)."""
|
||||
host = urlparse(url or "").hostname or "the site"
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
clip = payload if len(payload) <= 80 else payload[:77] + "..."
|
||||
return (f'I submitted "{clip}" and the composer cleared, but I could NOT confirm it stayed '
|
||||
f'live: {host} sometimes accepts an automated post and then drops it without an error. '
|
||||
f'Please check your posts to verify it actually went through before relying on it.')
|
||||
@@ -65,7 +65,9 @@ P_CLASSIFIER_SYSTEM = (
|
||||
"If line 1 is READ or ACT, follow it with a short browsing brief:\n"
|
||||
"ENTRY: the best starting URL; use a direct deep/search URL when the site's "
|
||||
"pattern is well known (LinkedIn people search is "
|
||||
"https://www.linkedin.com/search/results/people/?keywords=NAME).\n"
|
||||
"https://www.linkedin.com/search/results/people/?keywords=NAME). Always a normal "
|
||||
"page a person would see, never a raw JSON/API endpoint (e.g. Instagram's "
|
||||
"web/search/topsearch), which renders as an unreadable data wall.\n"
|
||||
"Then 3-6 numbered steps, one short action each.\n"
|
||||
"Copy any text the user wants typed, sent, or posted EXACTLY, character for "
|
||||
"character. Never invent names, values, or wording the user did not give."
|
||||
@@ -115,6 +117,11 @@ def entry_url_from_brief(brief: str) -> str:
|
||||
return m.group(1).rstrip(".,;)") if m else ""
|
||||
|
||||
|
||||
# Opens the advisory-brief section of a composed task; consumers strip everything after it when a
|
||||
# check must apply only to the human's words (the brief once false-flagged a real send read-only).
|
||||
BRIEF_MARKER = "[routing brief"
|
||||
|
||||
|
||||
def compose_task(prompt: str, brief: str) -> str:
|
||||
"""User's words first and authoritative; the brief is advisory routing.
|
||||
Skill replay keys on the parent's user message, so brief variance is safe."""
|
||||
@@ -122,7 +129,7 @@ def compose_task(prompt: str, brief: str) -> str:
|
||||
return prompt
|
||||
return (
|
||||
f"{prompt}\n\n"
|
||||
"[routing brief from a fast pre-pass; follow it unless the live page disagrees]\n"
|
||||
f"{BRIEF_MARKER} from a fast pre-pass; follow it unless the live page disagrees]\n"
|
||||
f"{brief}"
|
||||
)
|
||||
|
||||
@@ -219,6 +226,22 @@ def normalize_for_classifier(prompt: str) -> str:
|
||||
return re.sub(r"\btext(ing|ed|s)?\b", "message", prompt, flags=re.I)
|
||||
|
||||
|
||||
def seed_hints_for_task(prompt: str) -> str:
|
||||
"""Documented facts for sites the task names, fed to the classifier so its ENTRY
|
||||
uses the site's real search-URL pattern instead of the homepage (measured: the aux
|
||||
sent walmart to the homepage while the seed had the exact /search?q= pattern)."""
|
||||
from backend.apps.agents.browser.seed_playbooks import SEED_PLAYBOOKS
|
||||
low = f" {prompt.lower()} "
|
||||
lines: list[str] = []
|
||||
for domain, facts in SEED_PLAYBOOKS.items():
|
||||
name = domain.split(".")[0]
|
||||
if len(name) >= 4 and f" {name}" in low and facts:
|
||||
lines.append(f"{domain}: {facts[0][:180]}")
|
||||
if len(lines) >= 2:
|
||||
break
|
||||
return ("\n\nKnown site facts (use their URL patterns for ENTRY):\n" + "\n".join(lines)) if lines else ""
|
||||
|
||||
|
||||
async def classify_and_brief(prompt: str, settings, primary_api: str | None) -> tuple[str, str]:
|
||||
"""One cheap aux call returns a READ/ACT/NO verdict plus a routing brief
|
||||
(entry URL + step outline), timeboxed; any failure means NO (normal path)."""
|
||||
@@ -227,22 +250,35 @@ async def classify_and_brief(prompt: str, settings, primary_api: str | None) ->
|
||||
from backend.apps.settings.credentials import get_anthropic_client_for_model
|
||||
from backend.apps.agents.providers.registry import resolve_aux_model
|
||||
|
||||
aux_model, _ = await resolve_aux_model(
|
||||
settings, preferred_tier="haiku", primary_api=primary_api,
|
||||
)
|
||||
client = get_anthropic_client_for_model(settings, aux_model)
|
||||
resp = await asyncio.wait_for(
|
||||
client.messages.create(
|
||||
model=aux_model,
|
||||
max_tokens=250,
|
||||
temperature=0,
|
||||
system=P_CLASSIFIER_SYSTEM,
|
||||
messages=[{"role": "user", "content": normalize_for_classifier(prompt[:2000])}],
|
||||
),
|
||||
timeout=8.0,
|
||||
)
|
||||
from backend.apps.agents.core.aux_llm import safe_resp_text
|
||||
verdict, brief = parse_verdict_and_brief(safe_resp_text(resp))
|
||||
|
||||
async def p_ask(api: str | None) -> tuple[str, str, str]:
|
||||
aux_model, _ = await resolve_aux_model(
|
||||
settings, preferred_tier="haiku", primary_api=api,
|
||||
)
|
||||
client = get_anthropic_client_for_model(settings, aux_model)
|
||||
resp = await asyncio.wait_for(
|
||||
client.messages.create(
|
||||
model=aux_model,
|
||||
max_tokens=250,
|
||||
temperature=0,
|
||||
system=P_CLASSIFIER_SYSTEM,
|
||||
messages=[{"role": "user", "content": (
|
||||
normalize_for_classifier(prompt[:2000]) + seed_hints_for_task(prompt))}],
|
||||
),
|
||||
timeout=8.0,
|
||||
)
|
||||
return safe_resp_text(resp), aux_model, ""
|
||||
|
||||
text, aux_model, _ = await p_ask(primary_api)
|
||||
# An EMPTY body is a broken lane, not a verdict. Measured live: cx/gpt-5.4-mini returns ''
|
||||
# for this call, which parsed to "no" and silently switched the whole browser fast path off
|
||||
# for every GPT user, with no error to show for it. Fall back once to the provider-agnostic
|
||||
# cheap tier (same cure as the distill fix) so a mute aux can't disable a working feature.
|
||||
if not text.strip() and primary_api:
|
||||
logger.info(f"[browser-fast-path] classifier empty on {aux_model}; retrying provider-agnostic")
|
||||
text, aux_model, _ = await p_ask(None)
|
||||
verdict, brief = parse_verdict_and_brief(text)
|
||||
logger.info(
|
||||
f"[browser-fast-path] classifier: {verdict.upper()} brief={len(brief)}ch "
|
||||
f"model={aux_model} in {int((time.monotonic() - t0) * 1000)}ms"
|
||||
|
||||
@@ -8,17 +8,19 @@ old path, never a wrong answer from a thin read.
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from urllib.parse import urljoin
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
P_ENTRY_RE = re.compile(r"^ENTRY:\s*(https?://\S+)", re.I | re.M)
|
||||
P_MIN_PAGE_CHARS = 500
|
||||
P_MAX_PAGE_CHARS = 24000
|
||||
MAX_PAGE_CHARS = 24000
|
||||
P_FETCH_ERROR_PREFIXES = ("HTTP error", "Error fetching", "Refused to fetch")
|
||||
|
||||
P_ANSWER_SYSTEM = (
|
||||
ANSWER_SYSTEM = (
|
||||
"Answer the user's request using ONLY the page text provided. Be direct and "
|
||||
"complete in a few sentences; quote exact titles/values from the page. End "
|
||||
"with nothing else.\n"
|
||||
@@ -26,6 +28,54 @@ P_ANSWER_SYSTEM = (
|
||||
"exactly the single word INSUFFICIENT."
|
||||
)
|
||||
|
||||
# One-hop mode: same contract plus a FOLLOW escape so a link-deep answer costs one more fetch instead of a full browser dispatch.
|
||||
P_HOP_SYSTEM = ANSWER_SYSTEM + (
|
||||
"\nEXCEPTION: if the page text is insufficient but exactly one of the "
|
||||
"numbered links clearly leads to the page that would contain the answer, "
|
||||
"reply with exactly 'FOLLOW <number>' and nothing else."
|
||||
)
|
||||
P_FOLLOW_RE = re.compile(r"^FOLLOW\s+(\d+)\s*$", re.I)
|
||||
P_LINK_RE = re.compile(r"<a\b[^>]*?href=[\"'](?!javascript:|#|mailto:)([^\"'>]+)[\"'][^>]*>(.*?)</a>", re.I | re.S)
|
||||
P_MAX_LINKS = 60
|
||||
|
||||
|
||||
def hop_enabled() -> bool:
|
||||
return os.environ.get("OSW_FASTREAD_HOP", "1") != "0"
|
||||
|
||||
|
||||
def extract_links(html: str, base_url: str) -> list[tuple[str, str]]:
|
||||
"""(anchor text, absolute url) pairs, deduped, capped. Text-less anchors are
|
||||
useless to the picker so they're dropped."""
|
||||
out: list[tuple[str, str]] = []
|
||||
seen: set[str] = set()
|
||||
for href, inner in P_LINK_RE.findall(html or ""):
|
||||
text = re.sub(r"<[^>]+>", " ", inner)
|
||||
text = re.sub(r"\s+", " ", text).strip()
|
||||
if not text:
|
||||
continue
|
||||
absolute = urljoin(base_url, href.strip())
|
||||
if not absolute.startswith(("http://", "https://")) or absolute in seen:
|
||||
continue
|
||||
seen.add(absolute)
|
||||
out.append((text[:80], absolute))
|
||||
if len(out) >= P_MAX_LINKS:
|
||||
break
|
||||
return out
|
||||
|
||||
|
||||
def format_link_menu(links: list[tuple[str, str]]) -> str:
|
||||
return "\n".join(f"{i + 1}. {text} -> {url}" for i, (text, url) in enumerate(links))
|
||||
|
||||
|
||||
def parse_follow(answer: str, links: list[tuple[str, str]]) -> tuple[str, str]:
|
||||
"""The (anchor text, url) the aux picked, or ('', ''). Out-of-range picks
|
||||
are dropped."""
|
||||
m = P_FOLLOW_RE.match((answer or "").strip())
|
||||
if not m:
|
||||
return "", ""
|
||||
idx = int(m.group(1)) - 1
|
||||
return links[idx] if 0 <= idx < len(links) else ("", "")
|
||||
|
||||
|
||||
def extract_entry_url(brief: str) -> str:
|
||||
m = P_ENTRY_RE.search(brief or "")
|
||||
@@ -40,6 +90,63 @@ def page_is_thin(text: str) -> bool:
|
||||
return len(body.strip()) < P_MIN_PAGE_CHARS
|
||||
|
||||
|
||||
async def fetch_page_text(url: str, prompt: str) -> str:
|
||||
from backend.apps.agents.tools.web import WebFetchTool
|
||||
|
||||
parts = await asyncio.wait_for(
|
||||
WebFetchTool().execute({"url": url, "prompt": prompt}, None),
|
||||
timeout=12.0,
|
||||
)
|
||||
return "\n".join(p.get("text", "") for p in parts if p.get("type") == "text")
|
||||
|
||||
|
||||
P_TAG_STRIP_RES = (
|
||||
re.compile(r"<(script|style|noscript)\b.*?</\1>", re.I | re.S),
|
||||
re.compile(r"<[^>]+>"),
|
||||
)
|
||||
|
||||
|
||||
def strip_tags(html: str) -> str:
|
||||
"""Whole-page text incl. bylines/usernames; trafilatura's main-content pass
|
||||
drops exactly the metadata that answers who/when questions, so hop mode
|
||||
reads the raw page instead."""
|
||||
import html as p_html_mod
|
||||
|
||||
text = html or ""
|
||||
for rx in P_TAG_STRIP_RES:
|
||||
text = rx.sub(" ", text)
|
||||
return re.sub(r"[ \t\r\f\v]+", " ", p_html_mod.unescape(text)).strip()
|
||||
|
||||
|
||||
async def fetch_raw(url: str) -> str:
|
||||
"""Raw HTML via the SSRF guard; '' on any miss."""
|
||||
try:
|
||||
from backend.apps.agents.tools.ssrf_guard import safe_fetch
|
||||
|
||||
resp = await asyncio.wait_for(
|
||||
safe_fetch(url, method="GET",
|
||||
headers={"User-Agent": "Mozilla/5.0 (Macintosh) AppleWebKit/537.36"},
|
||||
timeout=8.0),
|
||||
timeout=10.0,
|
||||
)
|
||||
return resp.text or ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
async def ask_aux(client, aux_model: str, system: str, content: str) -> str:
|
||||
from backend.apps.agents.core.aux_llm import safe_resp_text
|
||||
|
||||
resp = await asyncio.wait_for(
|
||||
client.messages.create(
|
||||
model=aux_model, max_tokens=500, temperature=0, system=system,
|
||||
messages=[{"role": "user", "content": content}],
|
||||
),
|
||||
timeout=15.0,
|
||||
)
|
||||
return safe_resp_text(resp).strip()
|
||||
|
||||
|
||||
async def try_fast_read(prompt: str, brief: str, settings, primary_api: str | None) -> str | None:
|
||||
"""Answer text on success; None means fall back to the browser leg."""
|
||||
entry = extract_entry_url(brief)
|
||||
@@ -47,45 +154,61 @@ async def try_fast_read(prompt: str, brief: str, settings, primary_api: str | No
|
||||
logger.info("[browser-fast-read] no ENTRY url in brief; browser fallback")
|
||||
return None
|
||||
try:
|
||||
from backend.apps.agents.tools.web import WebFetchTool
|
||||
|
||||
hop = hop_enabled()
|
||||
t0 = time.monotonic()
|
||||
parts = await asyncio.wait_for(
|
||||
WebFetchTool().execute({"url": entry, "prompt": prompt}, None),
|
||||
timeout=12.0,
|
||||
)
|
||||
text = "\n".join(p.get("text", "") for p in parts if p.get("type") == "text")
|
||||
links: list[tuple[str, str]] = []
|
||||
if hop:
|
||||
raw = await fetch_raw(entry)
|
||||
text, links = strip_tags(raw), extract_links(raw, entry)
|
||||
if page_is_thin(text):
|
||||
text = await fetch_page_text(entry, prompt)
|
||||
else:
|
||||
text = await fetch_page_text(entry, prompt)
|
||||
fetch_ms = int((time.monotonic() - t0) * 1000)
|
||||
if page_is_thin(text):
|
||||
logger.info(f"[browser-fast-read] thin/errored read of {entry} ({len(text)}ch in {fetch_ms}ms); browser fallback")
|
||||
return None
|
||||
logger.info(f"[browser-fast-read] fetched {entry}: {len(text)}ch in {fetch_ms}ms")
|
||||
logger.info(f"[browser-fast-read] fetched {entry}: {len(text)}ch in {fetch_ms}ms (links={len(links)})")
|
||||
|
||||
from backend.apps.settings.credentials import get_anthropic_client_for_model
|
||||
from backend.apps.agents.providers.registry import resolve_aux_model
|
||||
from backend.apps.agents.core.aux_llm import safe_resp_text
|
||||
|
||||
aux_model, _ = await resolve_aux_model(
|
||||
settings, preferred_tier="haiku", primary_api=primary_api,
|
||||
)
|
||||
client = get_anthropic_client_for_model(settings, aux_model)
|
||||
t1 = time.monotonic()
|
||||
resp = await asyncio.wait_for(
|
||||
client.messages.create(
|
||||
model=aux_model,
|
||||
max_tokens=500,
|
||||
temperature=0,
|
||||
system=P_ANSWER_SYSTEM,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": f"Request: {prompt}\n\nPage text from {entry}:\n{text[:P_MAX_PAGE_CHARS]}",
|
||||
}],
|
||||
),
|
||||
timeout=15.0,
|
||||
)
|
||||
answer = safe_resp_text(resp).strip()
|
||||
content = f"Request: {prompt}\n\nPage text from {entry}:\n{text[:MAX_PAGE_CHARS]}"
|
||||
if hop and links:
|
||||
content += f"\n\nNumbered links found on the page:\n{format_link_menu(links)}"
|
||||
answer = await ask_aux(client, aux_model, P_HOP_SYSTEM if links else ANSWER_SYSTEM, content)
|
||||
answer_ms = int((time.monotonic() - t1) * 1000)
|
||||
if not answer or answer.upper().startswith("INSUFFICIENT"):
|
||||
|
||||
hop_anchor, hop_url = parse_follow(answer, links) if hop and links else ("", "")
|
||||
if hop_url:
|
||||
t2 = time.monotonic()
|
||||
hop_text = strip_tags(await fetch_raw(hop_url))
|
||||
if page_is_thin(hop_text):
|
||||
hop_text = await fetch_page_text(hop_url, prompt)
|
||||
if page_is_thin(hop_text):
|
||||
logger.info(f"[browser-fast-read] hop to {hop_url} was thin; browser fallback")
|
||||
return None
|
||||
answer = await ask_aux(
|
||||
client, aux_model, ANSWER_SYSTEM,
|
||||
f"Request: {prompt}\n\n"
|
||||
f"Context: the navigation in the request is ALREADY DONE. From {entry} "
|
||||
f"you chose the link '{hop_anchor}' as the one leading to the answer, and "
|
||||
f"the page text below is that destination. Extract the requested "
|
||||
f"information from it.\n\n"
|
||||
f"Page text from {hop_url}:\n{hop_text[:MAX_PAGE_CHARS]}",
|
||||
)
|
||||
logger.info(
|
||||
f"[browser-fast-read] followed link {hop_url} "
|
||||
f"(+{int((time.monotonic() - t2) * 1000)}ms hop)"
|
||||
)
|
||||
entry = hop_url
|
||||
|
||||
if not answer or answer.upper().startswith("INSUFFICIENT") or P_FOLLOW_RE.match(answer):
|
||||
logger.info(f"[browser-fast-read] aux found page insufficient ({answer_ms}ms); browser fallback")
|
||||
return None
|
||||
logger.info(f"[browser-fast-read] answered in {answer_ms}ms ({len(answer)}ch, model={aux_model})")
|
||||
|
||||
@@ -0,0 +1,102 @@
|
||||
"""Login-once handoff: when the browser agent lands on a login wall, it pauses for the user to
|
||||
sign in ONCE in the app's browser card, then continues, and we REMEMBER which sites the user has
|
||||
authenticated so future runs skip the prompt and only re-ask on a genuine expiry or a different
|
||||
account. The session itself lives in Electron's persist:openswarm-browser partition (which keeps
|
||||
it across quits, so "sign in once, never again" is really the partition's doing); this module is
|
||||
the durable memory of it plus the detection and the wording, keyed by registrable domain.
|
||||
|
||||
Detection reuses the one structural login-wall definition in browser_send_parse, so the pause and
|
||||
the send-script's decline can never disagree about what a login wall is.
|
||||
"""
|
||||
|
||||
import datetime
|
||||
import os
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.agents.browser import browser_send_parse
|
||||
from backend.config.json_store import atomic_write_json, read_json_or_none
|
||||
from backend.config.paths import SETTINGS_DIR
|
||||
|
||||
P_STORE_PATH = os.path.join(SETTINGS_DIR, "authenticated_domains.json")
|
||||
|
||||
|
||||
@typechecked
|
||||
def registrable_domain(url_or_host: str) -> str:
|
||||
s = (url_or_host or "").strip()
|
||||
host = urlparse(s).hostname if "://" in s else s.split("/")[0]
|
||||
host = (host or "").lower().strip().lstrip(".").split(":")[0]
|
||||
if host.startswith("www."):
|
||||
host = host[4:]
|
||||
return host
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_load() -> Dict[str, Dict[str, str]]:
|
||||
data = read_json_or_none(P_STORE_PATH)
|
||||
return data if isinstance(data, dict) else {}
|
||||
|
||||
|
||||
@typechecked
|
||||
def is_authenticated(url_or_host: str) -> bool:
|
||||
return registrable_domain(url_or_host) in p_load()
|
||||
|
||||
|
||||
@typechecked
|
||||
def authenticated_domains() -> List[str]:
|
||||
return sorted(p_load().keys())
|
||||
|
||||
|
||||
@typechecked
|
||||
def login_record(url_or_host: str) -> Optional[Dict[str, str]]:
|
||||
"""The stored {first_seen, last_login} for a site, or None. For a future 'signed-in sites' view."""
|
||||
return p_load().get(registrable_domain(url_or_host))
|
||||
|
||||
|
||||
@typechecked
|
||||
def record_login(url_or_host: str) -> None:
|
||||
"""Remember that the user signed into this site, so future walls read as re-auth not first-run.
|
||||
Fail-open: a write error just means the next run treats it as a fresh sign-in (harmless)."""
|
||||
d = registrable_domain(url_or_host)
|
||||
if not d:
|
||||
return
|
||||
store = p_load()
|
||||
now = datetime.datetime.now(datetime.timezone.utc).isoformat()
|
||||
prior = store.get(d) or {}
|
||||
store[d] = {"first_seen": prior.get("first_seen") or now, "last_login": now}
|
||||
try:
|
||||
atomic_write_json(P_STORE_PATH, store)
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
@typechecked
|
||||
def login_wall_domain(current_url: str, state_text: str, allow_soft: bool = False) -> Optional[str]:
|
||||
"""The registrable domain of a login wall the agent is stuck on, or None. One definition of
|
||||
'login wall', shared with the send-script's decline gate.
|
||||
|
||||
`allow_soft` additionally accepts a SOFT signed-out page: browsable, no auth form, composer
|
||||
simply withheld behind a "Sign in" control (bsky/stackoverflow/tiktok). Those never match the
|
||||
hard wall, so the run used to fail as "couldn't find the compose box" instead of offering the
|
||||
one thing that fixes it. Off by default because this pause interrupts the user: the caller
|
||||
turns it on only once the agent is demonstrably stuck, so a stray "Sign up" link on a page
|
||||
we're actually signed into can't raise a spurious prompt."""
|
||||
if browser_send_parse.looks_like_login_wall(current_url or "", state_text or ""):
|
||||
return registrable_domain(current_url) or None
|
||||
if allow_soft and browser_send_parse.looks_signed_out(state_text or ""):
|
||||
return registrable_domain(current_url) or None
|
||||
return None
|
||||
|
||||
|
||||
@typechecked
|
||||
def prompt_copy(domain: str) -> Tuple[str, str]:
|
||||
"""(problem, instruction) for the pause overlay, worded by whether the user has signed into
|
||||
this site before (re-auth) or it's a first sign-in."""
|
||||
if is_authenticated(domain):
|
||||
problem = f"Your {domain} sign-in looks signed out, it may have expired or be a different account."
|
||||
else:
|
||||
problem = f"{domain} needs you to sign in before I can keep going."
|
||||
instruction = "Log in to the site in the browser above, then click Done and I'll pick up right where I left off."
|
||||
return problem, instruction
|
||||
@@ -219,7 +219,7 @@ def stagnation_exhausted(streak: int) -> bool:
|
||||
# State-changing tools: a task that needed to DO something must land one of these.
|
||||
P_PRODUCTIVE_TOOLS = {
|
||||
"BrowserClick", "BrowserClickIndex", "BrowserType", "BrowserNavigate",
|
||||
"BrowserPressKey", "BrowserScroll", "BrowserBatch",
|
||||
"BrowserPressKey", "BrowserScroll", "BrowserBatch", "BrowserActVerified",
|
||||
}
|
||||
# Read/extract tools: a look-only task's evidence is that a read returned content.
|
||||
P_READ_TOOLS = {
|
||||
@@ -263,7 +263,7 @@ def recoverable_tool_error(err: str) -> bool:
|
||||
# Actions that DIRTY the page so replay-from-here is no longer equivalent to a clean dispatch. Navigation and reads don't dirty anything (they just get us to the page), so the deferred replay re-check is allowed after only those.
|
||||
P_REPLAY_DIRTYING_TOOLS = {
|
||||
"BrowserType", "BrowserClick", "BrowserClickIndex",
|
||||
"BrowserPressKey", "BrowserScroll", "BrowserBatch",
|
||||
"BrowserPressKey", "BrowserScroll", "BrowserBatch", "BrowserActVerified",
|
||||
}
|
||||
|
||||
|
||||
@@ -287,6 +287,16 @@ P_ACTION_ASK_RE = re.compile(
|
||||
re.I,
|
||||
)
|
||||
|
||||
P_DELETE_INTENT_RE = re.compile(
|
||||
r"\b(delete|remove|take ?down|unsend|retract|unpost|discard|trash)\b", re.I)
|
||||
|
||||
|
||||
def is_removal_task(task: str) -> bool:
|
||||
"""A delete/remove ask. The send-script must stand down on these: a removal task is also
|
||||
task_is_send (the classifier keys on the verb), so without this the composer fill would
|
||||
TYPE the target text and POST it (measured live: delete tasks re-posted the marker)."""
|
||||
return bool(P_DELETE_INTENT_RE.search(task or ""))
|
||||
|
||||
|
||||
def deliverable_is_informational(summary: str, task: str = "") -> bool:
|
||||
"""True if the run's final answer is GATHERED CONTENT (a list/report the model
|
||||
|
||||
@@ -0,0 +1,205 @@
|
||||
"""
|
||||
Map-reduce READ tier: answer a multi-source public read (a comparison, a
|
||||
difference, a combine-across-pages) without the big-model loop. When the
|
||||
single-page fast_read declined because the answer lives across TWO OR MORE
|
||||
pages, one aux call decomposes the request into independent single-page
|
||||
lookups, they run CONCURRENTLY (each is a fast_read-class fetch + extract), and
|
||||
one aux reduce combines them.
|
||||
|
||||
Fail-open everywhere: not multi-source, a thin or insufficient source, or a
|
||||
reduce that can't answer all return None and the caller falls to the browser
|
||||
leg, so a partial read can never become a wrong answer. Lives only in the
|
||||
classifier's READ branch (public pages), so it never taxes an authed read.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
from backend.apps.agents.browser import browser_fast_read as fr
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
P_MAX_SOURCES = 4
|
||||
|
||||
P_DECOMPOSE_SYSTEM = (
|
||||
"Break the user's request into the MINIMUM set of independent factual "
|
||||
"lookups, each answerable from a SINGLE public web page. Return a JSON array "
|
||||
"of objects, each {\"q\": a self-contained question, \"url\": a starting URL "
|
||||
"(a direct page like https://en.wikipedia.org/wiki/NAME, or a search URL "
|
||||
"like https://www.google.com/search?q=...)}.\n"
|
||||
"Return 2 or more entries ONLY when the request genuinely needs different "
|
||||
"pages combined: a comparison, a difference, a sum, a 'both X and Y'. If a "
|
||||
"single page could answer it, return [].\n"
|
||||
"Never invent facts; only name the lookups. Output ONLY the JSON array."
|
||||
)
|
||||
|
||||
P_REDUCE_SYSTEM = (
|
||||
"Answer the user's original request using ONLY the sub-answers provided, "
|
||||
"each gathered from its own page.\n"
|
||||
"First state each exact value. Then show the SINGLE arithmetic step the "
|
||||
"request needs (the subtraction, sum, or comparison). Then give the final "
|
||||
"answer. Your final number MUST equal the result of that step; never state a "
|
||||
"total or difference that disagrees with your own arithmetic.\n"
|
||||
"If the sub-answers do not together contain what the request needs, reply "
|
||||
"with exactly the single word INSUFFICIENT."
|
||||
)
|
||||
|
||||
|
||||
def enabled() -> bool:
|
||||
"""Fail-open additive tier; default on, kill with OSW_MAP_REDUCE_READ=0."""
|
||||
return os.environ.get("OSW_MAP_REDUCE_READ", "1") != "0"
|
||||
|
||||
|
||||
# The aux reduce got the VALUES right but flipped the arithmetic twice in ~10 live runs ("taller
|
||||
# by 360.2m" beside its own 113.2 math; "1096-1636=-540, not older" beside "540 years older"), so
|
||||
# for the two shapes that are pure arithmetic the number is computed HERE and the model never
|
||||
# does subtraction. Anything unparseable falls open to the aux reduce.
|
||||
P_DIFF_RE = re.compile(r"\b(difference|older|younger|taller|shorter|higher|lower|farther|further|longer|heavier|lighter|bigger|smaller|faster|slower)\b", re.I)
|
||||
P_SUM_RE = re.compile(r"\b(combined|total|sum|together|altogether)\b", re.I)
|
||||
P_VALUE_RE = re.compile(r"VALUE:\s*([0-9]+(?:\.[0-9]+)?)\s*([a-zA-Z%]*)", re.I)
|
||||
P_VALUE_LINE = (
|
||||
"\nEnd with one extra line: VALUE: <the single number the request needs from this page, "
|
||||
"digits only with no thousands separators, followed by its unit if any (m, ft, km, %, ...)>."
|
||||
)
|
||||
|
||||
|
||||
P_QUANTITY_RE = re.compile(r"\b(how much|how many|difference|by how)\b", re.I)
|
||||
|
||||
|
||||
def op_for(prompt: str) -> str:
|
||||
"""'difference' | 'sum' | '' from the request's own wording; '' = aux reduce as before.
|
||||
Difference also requires a QUANTITY cue: a bare "which is taller?" wants a name, and a
|
||||
number-only computed headline would answer the wrong question (caught in audit, not live)."""
|
||||
low = prompt or ""
|
||||
if P_SUM_RE.search(low):
|
||||
return "sum"
|
||||
if P_DIFF_RE.search(low) and P_QUANTITY_RE.search(low):
|
||||
return "difference"
|
||||
return ""
|
||||
|
||||
|
||||
def fmt_num(n: float) -> str:
|
||||
"""Human numbers: 35,842,039 not 3.5842e+07; two decimals max on non-integers."""
|
||||
return f"{n:,.0f}" if float(n).is_integer() else f"{n:,.2f}"
|
||||
|
||||
|
||||
def computed_answer(op: str, plan: list[tuple[str, str]], subs: list) -> str:
|
||||
"""The deterministic answer when every sub-answer carries a parseable VALUE in agreeing
|
||||
units; '' means fall open to the aux reduce. States both values and the computed number,
|
||||
and deliberately asserts NO direction prose (that is exactly what the aux got wrong)."""
|
||||
vals: list[tuple[float, str]] = []
|
||||
for s in subs:
|
||||
m = P_VALUE_RE.search(s or "")
|
||||
if not m:
|
||||
return ""
|
||||
vals.append((float(m.group(1)), m.group(2).lower()))
|
||||
units = {u for _, u in vals}
|
||||
if len(units) > 1:
|
||||
return ""
|
||||
unit = f" {vals[0][1]}" if vals[0][1] else ""
|
||||
shown = "\n".join(f"- {q}: {fmt_num(v)}{unit}" for (q, _), (v, _) in zip(plan, vals))
|
||||
if op == "difference" and len(vals) == 2:
|
||||
n = abs(vals[0][0] - vals[1][0])
|
||||
return (f"**Answer: {fmt_num(n)}{unit}**\n\n{shown}\n"
|
||||
f"(computed: |{fmt_num(vals[0][0])} - {fmt_num(vals[1][0])}| = {fmt_num(n)})")
|
||||
if op == "sum":
|
||||
n = sum(v for v, _ in vals)
|
||||
return (f"**Answer: {fmt_num(n)}{unit}**\n\n{shown}\n"
|
||||
f"(computed: {' + '.join(fmt_num(v) for v, _ in vals)} = {fmt_num(n)})")
|
||||
return ""
|
||||
|
||||
|
||||
def parse_plan(text: str) -> list[tuple[str, str]]:
|
||||
"""(question, url) pairs from the decompose JSON; [] on anything unparseable
|
||||
or single-source. Bounded to P_MAX_SOURCES so a runaway plan can't fan out."""
|
||||
s = (text or "").strip()
|
||||
i, j = s.find("["), s.rfind("]")
|
||||
if i < 0 or j <= i:
|
||||
return []
|
||||
try:
|
||||
arr = json.loads(s[i:j + 1])
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return []
|
||||
out: list[tuple[str, str]] = []
|
||||
for it in arr if isinstance(arr, list) else []:
|
||||
if isinstance(it, dict):
|
||||
q, url = str(it.get("q") or "").strip(), str(it.get("url") or "").strip()
|
||||
if q and url.startswith(("http://", "https://")):
|
||||
out.append((q, url))
|
||||
return out[:P_MAX_SOURCES]
|
||||
|
||||
|
||||
async def p_fetch_and_extract(client, aux_model: str, q: str, url: str, ask_value: bool) -> str | None:
|
||||
"""One source: fetch the page, aux-extract the answer to q, or None if the
|
||||
page is thin or insufficient (so the whole map-reduce fails open, never
|
||||
fabricates a missing piece). ask_value appends the machine-parseable VALUE
|
||||
line the code-side arithmetic needs."""
|
||||
try:
|
||||
raw = await fr.fetch_raw(url)
|
||||
text = fr.strip_tags(raw)
|
||||
if fr.page_is_thin(text):
|
||||
text = await fr.fetch_page_text(url, q)
|
||||
if fr.page_is_thin(text):
|
||||
return None
|
||||
ans = await fr.ask_aux(
|
||||
client, aux_model, fr.ANSWER_SYSTEM + (P_VALUE_LINE if ask_value else ""),
|
||||
f"Request: {q}\n\nPage text from {url}:\n{text[:fr.MAX_PAGE_CHARS]}")
|
||||
if not ans or ans.upper().startswith("INSUFFICIENT"):
|
||||
return None
|
||||
return ans
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
async def try_map_reduce_read(prompt: str, settings, primary_api: str | None) -> str | None:
|
||||
"""Answer text for a multi-source public read, or None (caller falls to the
|
||||
browser leg). Any missing piece returns None, so it never half-answers."""
|
||||
if not enabled():
|
||||
return None
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
from backend.apps.settings.credentials import get_anthropic_client_for_model
|
||||
from backend.apps.agents.providers.registry import resolve_aux_model
|
||||
|
||||
aux_model, _ = await resolve_aux_model(
|
||||
settings, preferred_tier="haiku", primary_api=primary_api)
|
||||
client = get_anthropic_client_for_model(settings, aux_model)
|
||||
|
||||
plan_text = await fr.ask_aux(client, aux_model, P_DECOMPOSE_SYSTEM, f"Request: {prompt[:1200]}")
|
||||
plan = parse_plan(plan_text)
|
||||
if len(plan) < 2:
|
||||
return None
|
||||
logger.info(f"[browser-mapreduce] {len(plan)} sources: {[u for _, u in plan]}")
|
||||
|
||||
p_op = op_for(prompt)
|
||||
subs = await asyncio.gather(*[p_fetch_and_extract(client, aux_model, q, u, bool(p_op)) for q, u in plan])
|
||||
if any(s is None for s in subs):
|
||||
logger.info(f"[browser-mapreduce] a source came back thin/insufficient in "
|
||||
f"{int((time.monotonic() - t0) * 1000)}ms; browser fallback")
|
||||
return None
|
||||
|
||||
if p_op:
|
||||
p_coded = computed_answer(p_op, plan, subs)
|
||||
if p_coded:
|
||||
logger.info(f"[browser-mapreduce] {p_op} computed in code from {len(plan)} sources "
|
||||
f"in {int((time.monotonic() - t0) * 1000)}ms")
|
||||
return f"{p_coded}\n\n(Sources: {', '.join(u for _, u in plan)})"
|
||||
|
||||
joined = "\n\n".join(f"Sub-question: {q}\nAnswer (from {u}): {s}"
|
||||
for (q, u), s in zip(plan, subs))
|
||||
final = await fr.ask_aux(client, aux_model, P_REDUCE_SYSTEM,
|
||||
f"Original request: {prompt}\n\n{joined}")
|
||||
if not final or final.upper().startswith("INSUFFICIENT"):
|
||||
logger.info(f"[browser-mapreduce] reduce insufficient in "
|
||||
f"{int((time.monotonic() - t0) * 1000)}ms; browser fallback")
|
||||
return None
|
||||
logger.info(f"[browser-mapreduce] answered from {len(plan)} sources in "
|
||||
f"{int((time.monotonic() - t0) * 1000)}ms")
|
||||
return f"{final}\n\n(Sources: {', '.join(u for _, u in plan)})"
|
||||
except Exception as e:
|
||||
logger.info(f"[browser-mapreduce] skipped ({e}); browser fallback")
|
||||
return None
|
||||
@@ -0,0 +1,144 @@
|
||||
"""Code-side plan dispatch: the turn-collapser that does NOT depend on the model
|
||||
adopting a tool (it never does; 0/3 live A/Bs). ONE cheap aux call maps the task +
|
||||
live page state to a chain of verified steps; run_verified_step executes them in
|
||||
code (resolve-late, verify-effect, re-aim); the big model then starts with the
|
||||
mechanical work DONE instead of spending a ~4-6s turn per click.
|
||||
|
||||
Safety mirrors the send-script: the aux may only emit click/fill on elements it
|
||||
names from the live list, anything irreversible-smelling is refused in code, and
|
||||
every step must VERIFY or the chain stops and hands off honestly. Fail-open
|
||||
everywhere: no aux, bad JSON, zero steps = the loop runs exactly as today.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
|
||||
from backend.apps.agents.browser import browser_verified_step
|
||||
from backend.apps.agents.browser.browser_prestage import BLOCKED_CLICK_RE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
P_MAX_STEPS = 6
|
||||
P_AUX_TIMEOUT_S = 10.0
|
||||
# Cross-page steps land right after a navigation; give the new page a beat before resolving.
|
||||
P_STEP_SETTLE_S = 1.2
|
||||
P_STATE_CAP = 6000
|
||||
|
||||
P_SYSTEM = (
|
||||
"You compile the MECHANICAL prefix of a browser task into steps a dumb executor "
|
||||
"runs and VERIFIES one at a time. You see the task and the page's interactive "
|
||||
"elements. Emit ONLY steps in order, as STRICT JSON (no prose): an array of\n"
|
||||
'{"action":"click"|"fill","target":"<element name EXACTLY as listed>",'
|
||||
'"role":"button"|"link"|"textbox"|"","text":"<for fill>",'
|
||||
'"expect":"appeared:<text>"|"gone:<text>"|"url_changed"|"changed"|"",'
|
||||
'"chosen":true|false}\n'
|
||||
"Rules: a target is copied verbatim from a listed element name, EXCEPT steps after "
|
||||
"one that navigates: those may name an element the task implies will appear (e.g. "
|
||||
"'Message' after opening a profile). Each step is resolved against the live page "
|
||||
"and verified before the next runs, so a wrong guess stops the chain safely. "
|
||||
"Expectations: use url_changed for clicks that open a new page, appeared:<text> "
|
||||
"for clicks that open a dialog or composer. ORDINALS map to rows: 'the 4th "
|
||||
"story's comments' = copy the name of the 4th row matching that shape; you may "
|
||||
"and should count. When the task names a person or thing and several rows are "
|
||||
"similar, PICK the best row using the task's cues and mark that step "
|
||||
"\"chosen\":true; for messaging a person, a direct/1st-degree connection outranks "
|
||||
"every other cue (title, company, verified): people message people they know. STOP the chain before "
|
||||
"anything irreversible (send/submit/post/pay/delete/confirm/apply). NEVER fill a "
|
||||
"message, comment, or post body: once a composer for one is open, stop, the main "
|
||||
"agent writes and sends it. If the Current URL shows the page ALREADY is the "
|
||||
"target's own page, emit []: a click that goes nowhere just fails verification. "
|
||||
"0-6 steps; [] when nothing is safely mechanical."
|
||||
)
|
||||
|
||||
|
||||
def parse_plan(reply: str) -> list:
|
||||
"""Strict-ish JSON array extraction; anything malformed = [] (fail-open).
|
||||
A max_tokens-truncated array is salvaged by closing it after the last complete
|
||||
object: the steps run one at a time with verification, so a shortened plan is
|
||||
safe, and losing the tail beats losing the whole plan (measured live)."""
|
||||
text = (reply or "").strip()
|
||||
m = re.search(r"\[.*\]", text, re.S)
|
||||
candidate = m.group(0) if m else ""
|
||||
if not candidate and text.startswith("["):
|
||||
cut = text.rfind("}")
|
||||
if cut > 0:
|
||||
candidate = text[: cut + 1] + "]"
|
||||
if not candidate:
|
||||
return []
|
||||
try:
|
||||
raw = json.loads(candidate)
|
||||
except Exception:
|
||||
return []
|
||||
steps = []
|
||||
for r in raw[:P_MAX_STEPS]:
|
||||
if not isinstance(r, dict):
|
||||
continue
|
||||
action = str(r.get("action") or "")
|
||||
target = str(r.get("target") or "").strip()
|
||||
if action not in ("click", "fill") or not target:
|
||||
continue
|
||||
if BLOCKED_CLICK_RE.search(target):
|
||||
break # irreversible-smelling: refuse this and everything after it
|
||||
steps.append(browser_verified_step.VerifiedStep(
|
||||
kind=action, target=target, role=str(r.get("role") or ""),
|
||||
text=str(r.get("text") or ""), expect=str(r.get("expect") or ""),
|
||||
chosen=bool(r.get("chosen"))))
|
||||
return steps
|
||||
|
||||
|
||||
def plan_dispatch_enabled() -> bool:
|
||||
return os.environ.get("OSW_PLAN_DISPATCH", "0") == "1"
|
||||
|
||||
|
||||
async def run_plan_dispatch(
|
||||
task: str, state_text: str, browser_id: str, tab_id: str,
|
||||
settings, primary_api, execute_tool, current_url: str = "",
|
||||
) -> str:
|
||||
"""Returns a handoff note describing verified-executed steps ('' = nothing ran).
|
||||
Never raises; never acts irreversibly."""
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
from backend.apps.settings.credentials import get_anthropic_client_for_model
|
||||
from backend.apps.agents.providers.registry import resolve_aux_model
|
||||
from backend.apps.agents.core.aux_llm import safe_resp_text
|
||||
|
||||
aux_model, _ = await resolve_aux_model(settings, preferred_tier="haiku", primary_api=primary_api)
|
||||
client = get_anthropic_client_for_model(settings, aux_model)
|
||||
# Assistant prefill "[" makes prose unwritable: the aux was narrating the ambiguity instead of emitting the chosen click (caught live via the empty-plan reply log).
|
||||
reply = "[" + safe_resp_text(await asyncio.wait_for(
|
||||
client.messages.create(
|
||||
model=aux_model, max_tokens=1000, temperature=0, system=P_SYSTEM,
|
||||
messages=[
|
||||
{"role": "user", "content": (
|
||||
f"Task: {task[:1200]}\n\nCurrent URL: {current_url[:300]}\n\n"
|
||||
f"Interactive elements:\n{state_text[:P_STATE_CAP]}")},
|
||||
{"role": "assistant", "content": "["},
|
||||
],
|
||||
), timeout=P_AUX_TIMEOUT_S))
|
||||
steps = parse_plan(reply)
|
||||
if not steps:
|
||||
logger.info(f"[plan-dispatch] aux emitted no safe mechanical steps "
|
||||
f"(state={len(state_text)}ch, reply: {(reply or '')[:160]!r})")
|
||||
return ""
|
||||
done: list[str] = []
|
||||
for step in steps:
|
||||
r = await browser_verified_step.run_verified_step(
|
||||
step, browser_id, tab_id, execute_tool, settle_s=P_STEP_SETTLE_S)
|
||||
if not r["ok"]:
|
||||
done.append(f"{step.kind} {step.target!r} FAILED ({r['note']}); stopped there")
|
||||
break
|
||||
mark = " [CHOSEN among similar rows: confirm it matches the task before anything irreversible]" if step.chosen else ""
|
||||
done.append(f"{step.kind} {step.target!r} done+verified{mark}")
|
||||
note = (
|
||||
f"[Plan pre-executed and VERIFIED in code: {'; '.join(done)}. "
|
||||
"Do NOT redo these; continue from the page's CURRENT state below.]"
|
||||
)
|
||||
logger.info(f"[plan-dispatch] {len(done)} step(s) in {int((time.monotonic() - t0) * 1000)}ms: {'; '.join(done)[:160]}")
|
||||
return note
|
||||
except Exception as e:
|
||||
logger.info(f"[plan-dispatch] skipped ({e})")
|
||||
return ""
|
||||
@@ -0,0 +1,496 @@
|
||||
"""
|
||||
Navigation pre-stage: before the big model wakes, a cheap aux model drives
|
||||
NAVIGATE/CLICK-only steps on the live webview until the page is where the main
|
||||
agent only has to do the final content action (read the answer, type into an
|
||||
open composer). Deletes the 4-6 cold orientation turns from the big loop; the
|
||||
big model starts staged instead of exploring at ~3s a thought.
|
||||
|
||||
Safety is code, not prose: the only tools this module can issue are
|
||||
BrowserNavigate and BrowserClickIndex, and a click whose listed element text
|
||||
smells irreversible (send/submit/pay/...) is refused in code, ending the
|
||||
pre-stage so the main loop's full guard stack owns that step.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from typing import Awaitable, Callable
|
||||
|
||||
from backend.apps.agents.browser import browser_send_parse, compose_discovery, compose_entry
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MAX_STEPS = 4
|
||||
STEP_TIMEOUT_S = 8.0
|
||||
TOTAL_TIMEOUT_S = 25.0
|
||||
# Opener mode reaches one hop deeper (a post/comment surface is often nav -> open item -> reveal box).
|
||||
OPENER_MAX_STEPS = 6
|
||||
OPENER_TOTAL_TIMEOUT_S = 32.0
|
||||
|
||||
P_STEP_RE = re.compile(r"^\s*(NAVIGATE|CLICK|READY)\b[:\s]*(.*)$", re.I)
|
||||
|
||||
# URL shapes that mean "a list of candidates to pick from" (also drives the agent's candidate scan)
|
||||
RESULTS_URL_RE = re.compile(
|
||||
r"[?&](q|query|keywords|search|search_query|find|term)=|/search\b|/results\b", re.I,
|
||||
)
|
||||
BLOCKED_CLICK_RE = re.compile(
|
||||
r"\b(send|submit|post|pay|buy|order|delete|confirm|apply|accept|invite|"
|
||||
r"connect|purchase|checkout|subscribe|unfollow|sign\s?out|log\s?out)\b",
|
||||
re.I,
|
||||
)
|
||||
# Genuinely irreversible / costly: NEVER a composer-opener, refused in every mode.
|
||||
P_HARD_BLOCK_RE = re.compile(
|
||||
r"\b(send|submit|pay|buy|order|delete|confirm|apply|accept|invite|"
|
||||
r"connect|purchase|checkout|subscribe|unfollow|sign\s?out|log\s?out)\b",
|
||||
re.I,
|
||||
)
|
||||
# Compose-ENTRY words: on a composer-ABSENT page these OPEN a box (X/Threads "Post",
|
||||
# Reddit "Create Post", "Add a comment", "Reply", "New thread"); the SAME word is the
|
||||
# submit once a box exists. So allowed only while no composer is in perception.
|
||||
P_COMPOSE_ENTRY_RE = re.compile(r"\b(post|comment|reply|tweet|write|thread|note|caption)\b", re.I)
|
||||
|
||||
|
||||
def opener_mode() -> bool:
|
||||
"""Whether prestage may OPEN a composer (click a person's Message / a 'Reply'/'Post'
|
||||
surface) instead of only navigating to an already-open one. It's ON when its own flag is
|
||||
set OR when the send-script is enabled: the send-script can only fire once a composer is
|
||||
reached, and the opener is what reaches it, so they're a pair (a send-script run that
|
||||
lands on a search page with no opener just declines and burns the slow model loop, the
|
||||
exact miss we measured). Safe by construction: the opener never types and refuses any
|
||||
send/submit/pay word, so a worst-case mis-click opens an empty box, never sends."""
|
||||
if os.environ.get("OSW_PRESTAGE_OPENER", "0") != "0":
|
||||
return True
|
||||
from backend.apps.agents.browser.browser_send_script import script_enabled
|
||||
return script_enabled()
|
||||
|
||||
|
||||
def click_refused(entry: str, li_text: str) -> bool:
|
||||
"""Whether prestage must refuse this click. Opener-mode-off = the legacy blanket
|
||||
gate (Phase A byte-identical). Opener-mode-on = structural: hard-irreversible
|
||||
words refused always; a compose-entry word (post/comment/reply/...) refused ONLY
|
||||
when a composer textbox is ALREADY in the current perception (then it's the real
|
||||
submit), allowed when none is present (then the click REVEALS the composer).
|
||||
Prestage never types, so even a worst-case mis-click submits empty content."""
|
||||
if not opener_mode():
|
||||
return bool(BLOCKED_CLICK_RE.search(entry))
|
||||
if P_HARD_BLOCK_RE.search(entry):
|
||||
return True
|
||||
if P_COMPOSE_ENTRY_RE.search(entry):
|
||||
from backend.apps.agents.browser.browser_send_parse import composer_index_in_state
|
||||
return bool(composer_index_in_state(li_text or ""))
|
||||
return False
|
||||
|
||||
|
||||
P_SYSTEM_OPENER = (
|
||||
"You pre-stage a browser for a main agent. Using ONLY navigation and clicks "
|
||||
"that OPEN or REVEAL a composer, get the page to where a text box is visible "
|
||||
"and the main agent only has to type the content and submit.\n"
|
||||
"OPENING a composer IS your job: click 'Start a post' / 'Create post' / 'New "
|
||||
"thread' / the compose 'Post' or 'Tweet' button / 'Add a comment' / 'Reply' / "
|
||||
"a person's 'Message' button so the text box appears.\n"
|
||||
"The MOMENT a compose text box is visible in the elements, reply READY, the "
|
||||
"stage is set.\n"
|
||||
"NEVER submit: do not click Send, Submit, Pay, Buy, Order, Delete, Confirm, "
|
||||
"Subscribe, or Connect. If the only next step is typing or the final submit, "
|
||||
"reply READY.\n"
|
||||
"For a task that messages a PERSON: go to that person (search result, "
|
||||
"profile), then open their Message surface. For a comment/reply on a thread "
|
||||
"or video: open the item, then reveal the comment box.\n"
|
||||
"Reply with exactly ONE line:\n"
|
||||
"NAVIGATE <absolute url>\n"
|
||||
"CLICK <index>\n"
|
||||
"READY <short reason>\n"
|
||||
"If unsure, reply READY."
|
||||
)
|
||||
|
||||
P_SYSTEM = (
|
||||
"You pre-stage a browser for a main agent. Using ONLY navigation (opening "
|
||||
"pages, clicking links or buttons that open/reveal things), get the page to "
|
||||
"the state where the main agent only has to do the FINAL content action "
|
||||
"(read the requested info, or type into an already-open composer/form).\n"
|
||||
"NEVER click anything that sends, submits, posts, pays, buys, deletes, "
|
||||
"accepts, connects, or subscribes. Opening a composer (e.g. a 'Message' "
|
||||
"button) is allowed; pressing its Send is not. If the next needed step is "
|
||||
"typing text or an irreversible click, the stage is set.\n"
|
||||
"For a task about a specific PERSON or THING (message them, read their "
|
||||
"details): click through to that person/thing's OWN page first; a "
|
||||
"search-results list is NOT the staged page. For messaging, then open "
|
||||
"their Message/compose surface; never detour to a feed or homepage. When "
|
||||
"several people share the name, a direct/1st-degree connection outranks "
|
||||
"every other cue (title, company, verified): people ask about people "
|
||||
"they know.\n"
|
||||
"Reply with exactly ONE line:\n"
|
||||
"NAVIGATE <absolute url>\n"
|
||||
"CLICK <index>\n"
|
||||
"READY <short reason>\n"
|
||||
"If unsure, reply READY."
|
||||
)
|
||||
|
||||
ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]]
|
||||
|
||||
|
||||
def prestage_enabled() -> bool:
|
||||
return os.environ.get("OSW_PRESTAGE", "1") != "0"
|
||||
|
||||
|
||||
def list_entry_for(list_text: str, index: int) -> str:
|
||||
for line in (list_text or "").splitlines():
|
||||
if line.strip().startswith(f"[{index}]"):
|
||||
return line.strip()
|
||||
return ""
|
||||
|
||||
|
||||
def parse_step(reply: str) -> tuple[str, str]:
|
||||
m = P_STEP_RE.match((reply or "").strip().splitlines()[0] if reply else "")
|
||||
if not m:
|
||||
return "ready", ""
|
||||
return m.group(1).lower(), m.group(2).strip()
|
||||
|
||||
|
||||
def perception_block(li_text: str, gt_text: str, stage_note: str = "") -> str:
|
||||
parts = []
|
||||
if li_text:
|
||||
parts.append("Interactive elements already on the page:\n" + li_text)
|
||||
if gt_text:
|
||||
parts.append("Visible page text (truncated):\n" + gt_text[:2000])
|
||||
if not parts:
|
||||
return ""
|
||||
return (
|
||||
"\n\n[Page already loaded and inspected for you, act directly; "
|
||||
"no need to screenshot or list elements again unless it changes]\n"
|
||||
+ (f"{stage_note}\n" if stage_note else "")
|
||||
+ "\n\n".join(parts)
|
||||
)
|
||||
|
||||
|
||||
def stage_note_for(start_url: str, done: list[str], current_url: str, complete: bool) -> str:
|
||||
"""Without this the main model re-verifies the route from scratch (observed:
|
||||
it navigated straight back to the start page), erasing the staging win. The
|
||||
note must never overclaim: a partial stage saying 'navigation DONE' sent the
|
||||
main loop on a 27-turn walkabout (observed live)."""
|
||||
if not done:
|
||||
return ""
|
||||
if complete:
|
||||
return (
|
||||
f"[Pre-staged for you and VERIFIED: starting from {start_url or 'the entry page'}, "
|
||||
f"already performed: {'; '.join(done)}. You are NOW on {current_url}. The "
|
||||
"navigation part of the task is DONE, do not go back or re-verify it; "
|
||||
"perform only the remaining final action(s). Staged runs took 7 solo turns "
|
||||
"where 2 suffice: if the remaining work is composing, use ONE BrowserBatch to "
|
||||
"focus the box and type the text, then the Send/Submit click SOLO with expect. "
|
||||
"Do not re-list first; the elements are listed below.]"
|
||||
)
|
||||
return (
|
||||
f"[Partial pre-staging: already performed {'; '.join(done)}. You are NOW on "
|
||||
f"{current_url}. Continue from HERE (do not restart from the beginning); "
|
||||
"finish the remaining navigation and the task yourself.]"
|
||||
)
|
||||
|
||||
|
||||
async def run_prestage(
|
||||
task: str,
|
||||
browser_id: str,
|
||||
tab_id: str,
|
||||
start_url: str,
|
||||
settings,
|
||||
primary_api: str | None,
|
||||
execute_tool: ToolRunner,
|
||||
perceive_only: bool = False,
|
||||
task_is_send: bool = False,
|
||||
) -> tuple[str, str, list[dict]]:
|
||||
"""(perception_block, current_url, action_records); ('', start_url, [])
|
||||
means nothing staged and the caller proceeds exactly as before.
|
||||
perceive_only skips the aux navigation loop and just captures the page: the
|
||||
caller has a verified click-through tier of its own (plan-dispatch), so the
|
||||
aux asks here were measured pure overhead (~2s) on that path."""
|
||||
t0 = time.monotonic()
|
||||
recs: list[dict] = []
|
||||
try:
|
||||
from backend.apps.settings.credentials import get_anthropic_client_for_model
|
||||
from backend.apps.agents.providers.registry import resolve_aux_model
|
||||
from backend.apps.agents.core.aux_llm import safe_resp_text
|
||||
|
||||
aux_model, _ = await resolve_aux_model(settings, preferred_tier="haiku", primary_api=primary_api)
|
||||
client = get_anthropic_client_for_model(settings, aux_model)
|
||||
|
||||
async def perceive() -> tuple[str, str, str]:
|
||||
li, gt = await asyncio.gather(
|
||||
execute_tool("BrowserListInteractives", {}, browser_id, tab_id),
|
||||
execute_tool("BrowserGetText", {}, browser_id, tab_id),
|
||||
return_exceptions=True,
|
||||
)
|
||||
li = li if isinstance(li, dict) else {}
|
||||
gt = gt if isinstance(gt, dict) else {}
|
||||
url = str(li.get("url") or gt.get("url") or "")
|
||||
li_text = str(li.get("text") or "") if "error" not in li else ""
|
||||
gt_text = str(gt.get("text") or "") if "error" not in gt else ""
|
||||
return li_text, gt_text, url
|
||||
|
||||
current_url = start_url
|
||||
li_text, gt_text = "", ""
|
||||
steps = 0
|
||||
done_desc: list[str] = []
|
||||
seen_steps: set[tuple[str, str]] = set()
|
||||
staged_complete = False
|
||||
|
||||
async def open_composer_directly(url: str) -> bool:
|
||||
"""Navigate to the site's own compose URL and confirm a composer actually appeared.
|
||||
|
||||
The confirmation is the whole point. Without it this would be a per-site nav hardcode
|
||||
that strands the run wherever the URL happens to lead once a site changes it; with it,
|
||||
a miss costs one navigation and the aux loop below runs exactly as it does today."""
|
||||
nonlocal li_text, gt_text, current_url, staged_complete
|
||||
r = await execute_tool("BrowserNavigate", {"url": url}, browser_id, tab_id)
|
||||
ok = isinstance(r, dict) and "error" not in r
|
||||
recs.append({"tool": "BrowserNavigate", "input": {"url": url}, "ok": ok,
|
||||
"result_summary": f"compose entry for {compose_entry.registrable_host(url)}"[:200],
|
||||
"elapsed_ms": 0})
|
||||
if not ok:
|
||||
return False
|
||||
# This is a cold NAVIGATION into a single-page app, not a modal opening on a page that
|
||||
# is already up, so it gets a longer budget than the opener hop: the app has to boot
|
||||
# before the composer can exist. Bounded well inside the prestage timeout so a miss
|
||||
# still leaves room for the aux loop.
|
||||
p_boxes = 0
|
||||
p_prev = ""
|
||||
p_settled = 0
|
||||
for wait_s in (0.8, 1.2, 1.5, 2.0, 2.5, 2.5, 2.5):
|
||||
await asyncio.sleep(wait_s)
|
||||
li2, gt2, u2 = await perceive()
|
||||
if li2:
|
||||
li_text, gt_text = li2, gt2
|
||||
current_url = u2 or url
|
||||
# Same stop condition as the opener hop: wait while the page is still arriving,
|
||||
# give up the moment it stops changing. Measured on a loaded machine, x.com's
|
||||
# compose route reported ZERO textboxes after 8s (nothing had rendered at all, not
|
||||
# an ambiguous pick), while an idle machine had it in under two.
|
||||
# TWO identical reads, not one. Gmail's compose window paints To/Cc/Bcc/Subject
|
||||
# first and holds them steady for a beat while the body field is still arriving, so
|
||||
# a single stable read declared the page finished and we walked away from a
|
||||
# composer that was about to exist (measured: "saw 4 textbox(es) but no single
|
||||
# composer").
|
||||
p_settled = p_settled + 1 if li2 and li2 == p_prev else 0
|
||||
p_prev = li2
|
||||
# A signed-out visit to a compose URL redirects to sign-in, and a login form is
|
||||
# made of textboxes. Claiming "composer reached" there would tell the rest of the
|
||||
# run the navigation is done while it sits on an auth wall.
|
||||
if browser_send_parse.looks_like_login_wall(current_url, li2):
|
||||
logger.info("[browser-prestage] compose entry landed on a sign-in wall; "
|
||||
"not staged")
|
||||
return False
|
||||
if browser_send_parse.composer_index_in_state(li2):
|
||||
return True
|
||||
p_boxes = browser_send_parse.textbox_count(li2)
|
||||
if p_settled >= 2:
|
||||
break
|
||||
# Name the miss. "No composer" covers three different problems with three different
|
||||
# fixes: the page never mounted one (0 boxes), we were too early (few boxes, still
|
||||
# hydrating), or several matched and the picker refused as ambiguous. Guessing between
|
||||
# them is how the last two rounds of timeout tuning made things worse.
|
||||
logger.info(f"[browser-prestage] compose entry saw {p_boxes} textbox(es) but no single "
|
||||
f"composer at {current_url[:80]}")
|
||||
return False
|
||||
|
||||
async def discover_compose_links(page_url: str) -> list[str]:
|
||||
"""The site's own compose links, best first, or nothing.
|
||||
|
||||
Failure is silent on purpose: this runs on whatever page the card happens to be on, so
|
||||
a page that blocks evaluation or publishes no such link must cost one read and leave the
|
||||
run exactly as it was."""
|
||||
# Read the links of the site the task is ABOUT, not whichever page the card was left
|
||||
# on. A cold run opens on a blank/search page, so the first attempt at this read
|
||||
# happened on google.com and correctly found nothing: the site was never visited.
|
||||
wanted = compose_entry.named_hosts(task, page_url)
|
||||
if not wanted:
|
||||
return []
|
||||
host = wanted[0]
|
||||
# The page the user named, not just its host: github's "New issue" lives on the repo,
|
||||
# and github.com/ publishes no compose link at all.
|
||||
target = compose_entry.named_page(task, host)
|
||||
if not (page_url or "").rstrip("/").startswith(target.rstrip("/")):
|
||||
nav = await execute_tool(
|
||||
"BrowserNavigate", {"url": target}, browser_id, tab_id)
|
||||
if not (isinstance(nav, dict) and "error" not in nav):
|
||||
return []
|
||||
# The links live in the app shell, which is not there the instant navigation
|
||||
# returns. One settle beat, not a ladder: if the shell is slower than this the aux
|
||||
# loop is the better remaining spend.
|
||||
await asyncio.sleep(1.5)
|
||||
try:
|
||||
raw = await execute_tool(
|
||||
"BrowserEvaluate", {"expression": compose_discovery.discovery_expression()},
|
||||
browser_id, tab_id)
|
||||
except Exception as exc:
|
||||
logger.info(f"[browser-prestage] compose discovery could not read the page ({exc})")
|
||||
return []
|
||||
found = compose_discovery.rank_candidates(
|
||||
compose_discovery.parse_page_read(raw), host)
|
||||
logger.info(f"[browser-prestage] compose discovery on {host} -> "
|
||||
f"{found if found else 'no compose link published'}")
|
||||
return found
|
||||
|
||||
# Composer reachability: when the task creates something top-level on a site that
|
||||
# publishes its own compose URL, ask for that URL instead of aux-hunting the button. This
|
||||
# is the 0/20 gap; the fill and receipt behind it were already proven. A hit also skips
|
||||
# the aux loop, so the cheap path and the reliable path are the same path.
|
||||
# Where the run BEGAN, before anything moved the card. A card keeps the last URL it was
|
||||
# left on, so a run can inherit a composer some earlier run opened and look like a win it
|
||||
# never earned; without this line there is no way to tell those apart after the fact.
|
||||
logger.info(f"[browser-prestage] start url={(start_url or '(none)')[:120]}")
|
||||
p_compose_url = "" if perceive_only else (
|
||||
compose_entry.compose_entry_for(task, start_url, task_is_send) or "")
|
||||
if p_compose_url:
|
||||
if await open_composer_directly(p_compose_url):
|
||||
staged_complete = True
|
||||
done_desc.append(f"opened the composer at {p_compose_url}")
|
||||
logger.info(f"[browser-prestage] compose entry {p_compose_url} reached a composer")
|
||||
else:
|
||||
logger.info(f"[browser-prestage] compose entry {p_compose_url} showed no composer; "
|
||||
f"falling through to the aux loop")
|
||||
|
||||
# No table row for this host. Ask the PAGE where its composer is instead: a site that has
|
||||
# one links to it ("Start a post", "Ask Question"), and reading that link is what makes this
|
||||
# work on a host nobody has written down. Measured 0/8 off-table before this tier existed.
|
||||
if (not staged_complete and not perceive_only and task_is_send
|
||||
and compose_discovery.enabled() and compose_entry.wants_top_level_compose(task)):
|
||||
for p_found in await discover_compose_links(current_url or start_url):
|
||||
if await open_composer_directly(p_found):
|
||||
staged_complete = True
|
||||
done_desc.append(f"opened the composer at {p_found}")
|
||||
logger.info(f"[browser-prestage] discovered compose link {p_found} "
|
||||
f"reached a composer")
|
||||
break
|
||||
logger.info(f"[browser-prestage] discovered compose link {p_found} "
|
||||
f"showed no composer")
|
||||
|
||||
async def settle(pre_url: str, pre_text: str, pre_li: str) -> bool:
|
||||
"""Wait for the page to actually change after an action, capped.
|
||||
|
||||
Timed by the caller's log line: this polls with a full perceive each round, so it is a
|
||||
real share of prestage's cost, and separating it from the aux plan is what says whether
|
||||
the fix is a cheaper decision or a faster wait."""
|
||||
# A click returns before the page swaps; perceiving too early reads the OLD page and the aux re-issues the same click (observed 4x loop). Wait for the page to actually change, capped. False = the action verifiably did NOT take. An overlay (message composer) changes the INTERACTIVES but not the URL and often not the first 400 chars of text, so the element list counts as change too.
|
||||
t_s = time.monotonic()
|
||||
while time.monotonic() - t_s < 3.0:
|
||||
await asyncio.sleep(0.35)
|
||||
li2, gt2, u2 = await perceive()
|
||||
if ((u2 and u2 != pre_url) or (gt2 and gt2[:400] != pre_text[:400])
|
||||
or (li2 and pre_li and li2 != pre_li)):
|
||||
return True
|
||||
return False
|
||||
p_max_steps = 0 if perceive_only else (OPENER_MAX_STEPS if opener_mode() else MAX_STEPS)
|
||||
p_total_timeout = OPENER_TOTAL_TIMEOUT_S if opener_mode() else TOTAL_TIMEOUT_S
|
||||
p_system = P_SYSTEM_OPENER if opener_mode() else P_SYSTEM
|
||||
p_results_overruled = False
|
||||
while (not staged_complete and steps < p_max_steps
|
||||
and (time.monotonic() - t0) < p_total_timeout):
|
||||
# Per-step cost, broken out. Prestage is the largest single phase of a LinkedIn write
|
||||
# (measured 18.6s of a 50.6s run, more than the send itself), and "steps=2 in 18587ms"
|
||||
# cannot tell you whether that is the aux deciding, the page settling, or the click.
|
||||
# Those have completely different fixes, so the log has to separate them.
|
||||
p_t_step = time.monotonic()
|
||||
li_text, gt_text, seen_url = await perceive()
|
||||
p_t_perceive = time.monotonic() - p_t_step
|
||||
current_url = seen_url or current_url
|
||||
p_t_aux = time.monotonic()
|
||||
reply = safe_resp_text(await asyncio.wait_for(
|
||||
client.messages.create(
|
||||
model=aux_model, max_tokens=60, temperature=0, system=p_system,
|
||||
messages=[{"role": "user", "content": (
|
||||
f"Task: {task[:1500]}\n\nCurrent URL: {current_url}\n\n"
|
||||
f"Interactive elements:\n{li_text[:4000]}\n\n"
|
||||
f"Visible text (truncated):\n{gt_text[:1200]}"
|
||||
)}],
|
||||
),
|
||||
timeout=STEP_TIMEOUT_S,
|
||||
)).strip()
|
||||
p_aux_ms = int((time.monotonic() - p_t_aux) * 1000)
|
||||
logger.info(f"[browser-prestage] step {steps + 1} plan: perceive={int(p_t_perceive * 1000)}ms "
|
||||
f"aux={p_aux_ms}ms reply={reply[:40]!r}")
|
||||
verb, arg = parse_step(reply)
|
||||
if verb == "ready" or not arg:
|
||||
# A results LIST is never the staged page for a task about one specific person/thing; the aux accepts it about half the time (measured, 2/4 cold LinkedIn runs) and every downstream tier then declines. Overrule ONCE with a nudge re-ask; a second READY is accepted, some tasks really do target the list.
|
||||
if RESULTS_URL_RE.search(current_url or "") and not p_results_overruled:
|
||||
p_results_overruled = True
|
||||
task = task + (
|
||||
"\n\n[You replied READY on a search-results LIST. If the task is about "
|
||||
"one specific person or thing, CLICK through to its own page first; "
|
||||
"READY again only if the task really is about this list.]")
|
||||
continue
|
||||
staged_complete = True
|
||||
logger.info(f"[browser-prestage] READY after {steps} step(s): {arg[:80]}")
|
||||
break
|
||||
# Any revisit (not just consecutive) is a loop signal: an A/B nav flap slipped past the consecutive-only check.
|
||||
if (verb, arg) in seen_steps:
|
||||
logger.info(f"[browser-prestage] repeated step {verb} {arg[:40]!r}; stopping")
|
||||
break
|
||||
seen_steps.add((verb, arg))
|
||||
if verb == "navigate":
|
||||
if not arg.startswith(("http://", "https://")):
|
||||
break
|
||||
r = await execute_tool("BrowserNavigate", {"url": arg}, browser_id, tab_id)
|
||||
ok = isinstance(r, dict) and "error" not in r
|
||||
recs.append({"tool": "BrowserNavigate", "input": {"url": arg}, "ok": ok,
|
||||
"result_summary": str(r.get("text", r.get("error", "")))[:200] if isinstance(r, dict) else "",
|
||||
"elapsed_ms": 0})
|
||||
logger.info(f"[browser-prestage] step {steps + 1}: nav {arg} ok={ok}")
|
||||
if not ok:
|
||||
break
|
||||
p_t_settle = time.monotonic()
|
||||
p_settled = await settle(current_url, gt_text, li_text)
|
||||
logger.info(f"[browser-prestage] step {steps + 1} nav settle={int((time.monotonic() - p_t_settle) * 1000)}ms ok={p_settled}")
|
||||
if not p_settled:
|
||||
logger.info(f"[browser-prestage] nav {arg} did not settle; stopping unstaged")
|
||||
break
|
||||
done_desc.append(f"navigated to {arg}")
|
||||
else:
|
||||
try:
|
||||
idx = int(re.sub(r"\D", "", arg) or "-1")
|
||||
except ValueError:
|
||||
break
|
||||
entry = list_entry_for(li_text, idx)
|
||||
if idx < 0 or not entry or click_refused(entry, li_text):
|
||||
logger.info(f"[browser-prestage] refusing click {idx} ({entry[:80]!r}); handing to main loop")
|
||||
break
|
||||
r = await execute_tool("BrowserClickIndex", {"index": idx}, browser_id, tab_id)
|
||||
ok = isinstance(r, dict) and "error" not in r
|
||||
recs.append({"tool": "BrowserClickIndex", "input": {"index": idx}, "ok": ok,
|
||||
"result_summary": entry[:200], "elapsed_ms": 0})
|
||||
logger.info(f"[browser-prestage] step {steps + 1}: click [{idx}] {entry[:60]!r} ok={ok}")
|
||||
if not ok:
|
||||
break
|
||||
p_t_settle = time.monotonic()
|
||||
p_settled = await settle(current_url, gt_text, li_text)
|
||||
logger.info(f"[browser-prestage] step {steps + 1} click settle={int((time.monotonic() - p_t_settle) * 1000)}ms ok={p_settled}")
|
||||
if not p_settled:
|
||||
# The click ran but the page never changed (occluded element, overlay, stale index). Recording it would make the handoff note LIE ("navigation done") and send the main loop on a walkabout; observed live as 27-turn/112s regressions.
|
||||
logger.info(f"[browser-prestage] click [{idx}] did not settle; stopping unstaged")
|
||||
break
|
||||
done_desc.append(f"clicked {entry[:70]}")
|
||||
steps += 1
|
||||
|
||||
if steps or not li_text:
|
||||
li_text, gt_text, seen_url = await perceive()
|
||||
current_url = seen_url or current_url
|
||||
# Perceive-only lost the aux asks that ACCIDENTALLY doubled as settle time, so a cold SPA hands back a half-hydrated list (measured: plan-dispatch emitted [] on a thin search page). Wait for substance, bounded.
|
||||
if perceive_only:
|
||||
p_sub_t0 = time.monotonic()
|
||||
while len(li_text or "") < 800 and time.monotonic() - p_sub_t0 < 4.0:
|
||||
await asyncio.sleep(0.8)
|
||||
li_text, gt_text, seen_url = await perceive()
|
||||
current_url = seen_url or current_url
|
||||
block = perception_block(li_text, gt_text, stage_note_for(start_url, done_desc, current_url, staged_complete))
|
||||
for tool_name, text in (("BrowserListInteractives", li_text), ("BrowserGetText", gt_text)):
|
||||
if text:
|
||||
recs.append({"tool": tool_name, "input": {}, "ok": True,
|
||||
"result_summary": text[:200], "elapsed_ms": 0})
|
||||
logger.info(
|
||||
f"[browser-prestage] done: steps={steps}{' (perceive-only)' if perceive_only else ''} "
|
||||
f"url={current_url[:80]} in {int((time.monotonic() - t0) * 1000)}ms"
|
||||
)
|
||||
return block, current_url, recs
|
||||
except Exception as e:
|
||||
logger.info(f"[browser-prestage] skipped ({e})")
|
||||
return "", start_url, recs
|
||||
@@ -0,0 +1,124 @@
|
||||
"""READ leg for AUTHED pages: prestage already landed the user's logged-in card on
|
||||
the target page, so ONE aux call over the live page text can answer a read task and
|
||||
the big-model loop never starts. The no-browser fast_read can't see behind logins;
|
||||
this is the same answer-or-INSUFFICIENT contract driven through the real session.
|
||||
Fail-open everywhere: thin page, decline, error = the loop runs exactly as today.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Awaitable, Callable, Dict, Optional
|
||||
|
||||
from backend.apps.agents.browser.browser_prestage import RESULTS_URL_RE
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ToolRunner = Callable[[str, Dict, str, str], Awaitable[Dict]]
|
||||
|
||||
P_MIN_PAGE_CHARS = 500
|
||||
P_MAX_PAGE_CHARS = 24000
|
||||
P_TEXT_TIMEOUT_S = 8.0
|
||||
P_AUX_TIMEOUT_S = 12.0
|
||||
# Prestage's click often lands here while the SPA is still hydrating (measured: a
|
||||
# LinkedIn profile read 184 chars right after the click); wait out the render, bounded.
|
||||
P_THIN_SETTLE_S = 1.2
|
||||
# Crossing the char floor is NOT the same as being finished rendering: a hydrating SPA clears 500
|
||||
# chars on nav and footer chrome long before the content lands. Taking that first passing read hands
|
||||
# the aux a half-drawn page, and it answers confidently from what IS there, so nothing declines and
|
||||
# the INSUFFICIENT retry below never fires. A confident wrong answer is the one outcome worse than
|
||||
# just running the loop, so the page has to prove it stopped growing: two reads in a row within this
|
||||
# much of each other. Costs one extra read plus one short settle on a path that already takes ~7-10s.
|
||||
P_STABLE_GROWTH = 0.05
|
||||
P_STABLE_SETTLE_S = 0.4
|
||||
MAX_READS = 4
|
||||
# A long-enough-but-still-rendering page reads as INSUFFICIENT (measured: profile
|
||||
# passed 500 chars with the headline section missing); one settle + re-read + re-ask.
|
||||
P_INSUFFICIENT_RETRIES = 1
|
||||
P_INSUFFICIENT_SETTLE_S = 1.5
|
||||
|
||||
P_SYSTEM = (
|
||||
"Answer the user's request using ONLY the page text provided. Be direct and "
|
||||
"complete in a few sentences; quote exact titles/values from the page. End "
|
||||
"with nothing else.\n"
|
||||
"Reply with exactly the single word INSUFFICIENT only when the requested "
|
||||
"information would live somewhere this page is not (a different page, behind "
|
||||
"a click), so the caller should go get it. If THIS page is the right place "
|
||||
"and it shows a value (even a placeholder) or visibly lacks the field, that "
|
||||
"IS the answer: report exactly what the page shows. A joke, placeholder, or "
|
||||
"obviously-fake value is still the answer, quoted, with a note that it looks "
|
||||
"like a placeholder; never decline because a shown value looks unreal. Never "
|
||||
"guess at anything the page doesn't show."
|
||||
)
|
||||
|
||||
|
||||
def read_script_enabled() -> bool:
|
||||
return os.environ.get("OSW_READ_SCRIPT", "1") != "0"
|
||||
|
||||
|
||||
def is_answer(reply: str) -> Optional[str]:
|
||||
"""The usable answer text, or None. Declines, empties, and hedge-shaped replies
|
||||
all fail closed to the loop, so a thin extraction can never become a wrong answer."""
|
||||
answer = (reply or "").strip()
|
||||
if not answer or answer.upper().startswith("INSUFFICIENT"):
|
||||
return None
|
||||
return answer
|
||||
|
||||
|
||||
async def run_read_script(
|
||||
aux_client, aux_model, task: str, browser_id: str, tab_id: str,
|
||||
execute_tool: ToolRunner, current_url: str = "",
|
||||
) -> Optional[str]:
|
||||
"""The answer to a read task from the staged page, or None (= run the loop).
|
||||
Never raises; never acts on the page beyond reading it."""
|
||||
t0 = time.monotonic()
|
||||
if aux_client is None or not aux_model:
|
||||
return None
|
||||
try:
|
||||
from backend.apps.agents.core.aux_llm import safe_resp_text
|
||||
|
||||
async def p_page_text() -> tuple:
|
||||
"""Page text, but only once two consecutive reads agree it has stopped growing."""
|
||||
prev = -1
|
||||
text, url = "", ""
|
||||
for attempt in range(MAX_READS):
|
||||
r = await asyncio.wait_for(
|
||||
execute_tool("BrowserGetText", {}, browser_id, tab_id), timeout=P_TEXT_TIMEOUT_S)
|
||||
text = str(r.get("text") or "") if isinstance(r, dict) and "error" not in r else ""
|
||||
url = str(r.get("url") or "") if isinstance(r, dict) else ""
|
||||
if len(text) >= P_MIN_PAGE_CHARS and 0 <= prev <= len(text) <= prev * (1 + P_STABLE_GROWTH):
|
||||
return text, url
|
||||
# Still thin waits longer than merely still-growing: one is a page that has not
|
||||
# started, the other is one about to finish.
|
||||
thin = len(text) < P_MIN_PAGE_CHARS
|
||||
prev = len(text)
|
||||
await asyncio.sleep(P_THIN_SETTLE_S if thin else P_STABLE_SETTLE_S)
|
||||
return (text, url) if len(text) >= P_MIN_PAGE_CHARS else ("", "")
|
||||
|
||||
for ask in range(1 + P_INSUFFICIENT_RETRIES):
|
||||
page, p_live_url = await p_page_text()
|
||||
if len(page) < P_MIN_PAGE_CHARS:
|
||||
logger.info(f"[browser-readscript] page too thin ({len(page)} chars); loop runs")
|
||||
return None
|
||||
# On a results LIST the miss is structural (the answer lives one click deeper), not hydration; the settle-retry would just re-decline ~3s later. Judged on the LIVE url: the caller's is stale once plan-dispatch has clicked through (that staleness suppressed the retry on the exact page that needed it, measured).
|
||||
p_retries = 0 if RESULTS_URL_RE.search(p_live_url or current_url or "") else P_INSUFFICIENT_RETRIES
|
||||
reply = safe_resp_text(await asyncio.wait_for(
|
||||
aux_client.messages.create(
|
||||
model=aux_model, max_tokens=500, temperature=0, system=P_SYSTEM,
|
||||
messages=[{"role": "user", "content": (
|
||||
f"Request: {task[:1200]}\n\nPage text:\n{page[:P_MAX_PAGE_CHARS]}")}],
|
||||
), timeout=P_AUX_TIMEOUT_S))
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
answer = is_answer(reply)
|
||||
if answer is not None:
|
||||
logger.info(f"[browser-readscript] answered from the staged page in {ms}ms (ask {ask + 1})")
|
||||
return answer
|
||||
if ask < p_retries:
|
||||
await asyncio.sleep(P_INSUFFICIENT_SETTLE_S)
|
||||
logger.info(f"[browser-readscript] insufficient in {int((time.monotonic() - t0) * 1000)}ms; loop runs "
|
||||
f"(page={len(page)}ch url={p_live_url[:80]!r} reply: {(reply or '')[:160]!r})")
|
||||
return None
|
||||
except Exception as e:
|
||||
logger.info(f"[browser-readscript] skipped ({e})")
|
||||
return None
|
||||
@@ -23,12 +23,6 @@ P_MERGE_VERIFY = (
|
||||
"back 'NOT confirmed' or you forgot to pass one.\n"
|
||||
)
|
||||
|
||||
MODEL_MAP = {
|
||||
"sonnet": "claude-sonnet-4-6",
|
||||
"opus": "claude-opus-4-6",
|
||||
"haiku": "claude-haiku-4-5-20251001",
|
||||
}
|
||||
|
||||
# The change an action should cause, declared by the agent and CONFIRMED after the action runs (success is observed, never assumed). A hit returns fast; a miss tells the agent it may not have worked instead of letting it claim a false success.
|
||||
P_EXPECT_DESC = {
|
||||
"type": "string",
|
||||
@@ -243,7 +237,13 @@ BROWSER_TOOLS_SCHEMA = [
|
||||
},
|
||||
{
|
||||
"name": "BrowserNavigate",
|
||||
"description": "Navigate the browser to a URL.",
|
||||
"description": (
|
||||
"Navigate the browser to a URL. Use a normal page URL a person would see. "
|
||||
"Do NOT point it at a raw JSON/API endpoint (a /api/... or search-JSON URL "
|
||||
"like Instagram's web/search/topsearch): that paints an unreadable data wall "
|
||||
"in the card. To READ a site's own API, use BrowserReplayRoute, which fetches "
|
||||
"the JSON silently without disturbing the page."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
@@ -387,10 +387,55 @@ BROWSER_TOOLS_SCHEMA = [
|
||||
"required": ["index"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserActVerified",
|
||||
"description": (
|
||||
"Run a short SEQUENCE of dependent UI steps (2-4) in one call, where each "
|
||||
"step must take effect before the next: open a menu then pick an item, "
|
||||
"fill a field then the next one, expand a section then click inside it. "
|
||||
"Each step names its target ELEMENT BY NAME (resolved fresh against the "
|
||||
"live page at act time, so a stale index can't bite) and is VERIFIED in "
|
||||
"code (did the expected change actually happen), with one automatic "
|
||||
"re-aim on a miss. Steps:\n"
|
||||
"- { action: 'click', target: '<element name>', role?: 'button'|'link'|..., "
|
||||
"expect?: 'appeared:<text>'|'gone:<text>'|'url_changed'|'changed' }\n"
|
||||
"- { action: 'fill', target: '<field name>', text: '<text to type>' } "
|
||||
"(auto-verifies the text committed)\n"
|
||||
"Execution stops at the first step that can't be verified and you get "
|
||||
"per-step results plus what went wrong. NEVER put an irreversible action "
|
||||
"(send/submit/post/pay/delete/confirm) here; those stay SOLO clicks with "
|
||||
"an `expect` proof, as always."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"steps": {
|
||||
"type": "array",
|
||||
"maxItems": 4,
|
||||
"items": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {"type": "string", "enum": ["click", "fill"]},
|
||||
"target": {"type": "string"},
|
||||
"role": {"type": "string"},
|
||||
"text": {"type": "string"},
|
||||
"expect": {"type": "string"},
|
||||
},
|
||||
"required": ["action", "target"],
|
||||
},
|
||||
},
|
||||
},
|
||||
"required": ["steps"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserBatch",
|
||||
"description": (
|
||||
"Your standard way to ACT on the page. Every mutation (navigate, "
|
||||
"Low-level action batch for INDEPENDENT actions (navigate, scroll, press) "
|
||||
"or when you must act by index/point. For a sequence of DEPENDENT named "
|
||||
"click/fill steps (open menu then pick item, fill then fill), prefer "
|
||||
"BrowserActVerified: it self-verifies each step and re-aims in code. "
|
||||
"Every mutation (navigate, "
|
||||
"click, type, press, scroll) is a sub-action in this array, 1-5 per "
|
||||
"call; an array of one is fine when that is genuinely all you know. "
|
||||
"Each sub-action executes in order with the URL captured before/after. "
|
||||
@@ -562,6 +607,37 @@ BROWSER_TOOLS_SCHEMA = [
|
||||
"required": ["url"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserApiWrite",
|
||||
"description": (
|
||||
"Perform a WRITE on the CURRENT site through the site's OWN API using your "
|
||||
"already-logged-in session, instead of clicking the UI. This is deterministic "
|
||||
"and far more reliable: no captcha, no button to miss, no selector to drift, "
|
||||
"and it hands back the site's REAL receipt (the new post/comment's id and "
|
||||
"permalink) as proof it landed. Only some sites are supported so far "
|
||||
"(currently Reddit: comment, reply, post, edit, delete). If the current site "
|
||||
"has no built-in adapter, you can still do it the GENERAL way: set action='route' "
|
||||
"with the site's own write endpoint (method + url + body) taken from BrowserListRoutes, "
|
||||
"and it replays that request with your session. If neither works you get a clean miss, "
|
||||
"just do the write through the UI instead. This IS a real write: call it ONCE, and the "
|
||||
"receipt is your confirmation, do not re-check or re-fire it."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"action": {"type": "string", "description": "comment, reply, post, edit, delete (built-in adapter), or route (general: replay a captured write endpoint)."},
|
||||
"parent_id": {"type": "string", "description": "comment/reply: fullname of the post/comment you're replying to (e.g. t3_abc, t1_xyz)."},
|
||||
"thing_id": {"type": "string", "description": "edit/delete: fullname of your OWN post/comment (e.g. t1_xyz)."},
|
||||
"text": {"type": "string", "description": "The body text (comment/reply/post/edit)."},
|
||||
"subreddit": {"type": "string", "description": "post: the subreddit name, without the r/ prefix."},
|
||||
"title": {"type": "string", "description": "post: the post title."},
|
||||
"url": {"type": "string", "description": "post: a link URL; OR route: the write endpoint's full URL from BrowserListRoutes."},
|
||||
"method": {"type": "string", "description": "route: the endpoint's HTTP method (POST, PUT, PATCH, DELETE)."},
|
||||
"body": {"type": "object", "description": "route: the JSON body to send, matching the endpoint's captured shape, with YOUR content in the text field(s)."},
|
||||
},
|
||||
"required": ["action"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserRepeatFlow",
|
||||
"description": (
|
||||
@@ -680,6 +756,32 @@ BROWSER_TOOLS_SCHEMA = [
|
||||
"required": ["problem", "instruction"],
|
||||
},
|
||||
},
|
||||
{
|
||||
"name": "BrowserDeleteItem",
|
||||
"description": (
|
||||
"Remove ONE item visible on the CURRENT page, named by a distinctive snippet of its "
|
||||
"own text. Navigate to where the item lives FIRST (your profile, the thread) so it is "
|
||||
"on screen, then call this. It opens that item's overflow / 'More' menu, clicks "
|
||||
"Delete/Remove, confirms, and verifies the item is gone. Use this instead of clicking "
|
||||
"the '...' menu yourself, that menu is small and lazy-rendered and hand-clicking it is "
|
||||
"unreliable. Only your OWN items expose a Delete option. Returns whether it was "
|
||||
"verifiably removed; if it says the item isn't on the page, navigate to where it lives "
|
||||
"and retry."
|
||||
),
|
||||
"input_schema": {
|
||||
"type": "object",
|
||||
"properties": {
|
||||
"target_text": {
|
||||
"type": "string",
|
||||
"description": (
|
||||
"A distinctive exact snippet of the target item's own text, long enough "
|
||||
"to match only that one item (a phrase or id from the post/comment)."
|
||||
),
|
||||
},
|
||||
},
|
||||
"required": ["target_text"],
|
||||
},
|
||||
},
|
||||
]
|
||||
|
||||
# Schema-forced batching: the model ignored every prompt-level batching invitation (0 adoptions across 8 measured runs), so the single-step mutating tools are not offered to it at all; acting means a BrowserBatch array, and the one deliberate solo path is BrowserClickIndex (irreversible step with expect, or a text-box fill). Executors and replay still support everything.
|
||||
@@ -707,6 +809,8 @@ ACTION_MAP = {
|
||||
"BrowserReplayRoute": "replay_route",
|
||||
# Internal replay primitive (skill replay calls it directly; not in the LLM-facing schema). Re-resolves a click target by role+name.
|
||||
"BrowserClickByName": "click_by_name",
|
||||
# Internal: structural composer finder (send-script fallback; not LLM-facing). Ranks editable regions, marks the winner, optionally fills+verifies in-page.
|
||||
"BrowserFindComposer": "find_composer",
|
||||
}
|
||||
|
||||
# --- App agent: driving an OpenSwarm-built app via its native bridge ---------
|
||||
@@ -1119,5 +1223,6 @@ ACTION_TOOLS_REQUIRING_REPORT = {
|
||||
"BrowserClickIndex", # Phase 3
|
||||
"BrowserClickPoint", # app mode: tap a canvas/game at a screen point
|
||||
"BrowserBatch", # Phase 4
|
||||
"BrowserActVerified", # verified-step sequence (mutates state like a batch)
|
||||
"AppInvoke", # app mode: invoking an app action mutates state
|
||||
}
|
||||
|
||||
@@ -0,0 +1,188 @@
|
||||
"""Pure perception-parsing for the staged send: read the browser's interactives listing + the
|
||||
user's task and answer the structural questions the send orchestration needs, with no I/O and no
|
||||
side effects. What quoted payload did the user name? Which listed row is the compose box / the
|
||||
opener? Is this a login wall or a read-only request the script must decline? One host-agnostic
|
||||
shape per question, so the same logic generalizes across X/Reddit/LinkedIn/Gmail/Slack/etc.
|
||||
|
||||
Lives BELOW browser_send_script (which orchestrates the fill/click/verify tail): send_script
|
||||
imports from here, never the reverse.
|
||||
"""
|
||||
|
||||
import re
|
||||
|
||||
# Double quotes are unambiguous. Single quotes only delimit when the opener is at a word boundary (start/space/colon), so an in-word apostrophe like "chen's" is never mistaken for a payload quote, that mispairing was silently corrupting the canonical "text him '...'" errand.
|
||||
P_QUOTED_DQ_RE = re.compile(r'"([^"]{4,300})"')
|
||||
P_QUOTED_SQ_RE = re.compile(r"(?:^|[\s:>])'([^']{4,300})'")
|
||||
P_COMPOSER_ROW_RE = re.compile(r"\[(\d+)\]\*?<\s*textbox\s+\"([^\"]*)\"", re.I)
|
||||
# A compose-shaped textbox name, generalized across messaging sites: LinkedIn "Write a
|
||||
# message", X/Slack "Message", Discord "Message @user", Gmail "Message Body", "Post your
|
||||
# reply", "What's happening", "Add a comment". Not per-site: one structural shape.
|
||||
# "text editor" earns its place from a measurement, not a guess: LinkedIn's post box is named
|
||||
# "Text editor for creating content" and its comment box "Text editor for creating comment", so
|
||||
# without it the real composer was invisible while the comment box next to it matched on "comment".
|
||||
# Landing on LinkedIn's own compose surface listed exactly one textbox and we still scored zero.
|
||||
# Both shapes match now, and telling them apart is surface_mismatch's job, which already does it.
|
||||
P_COMPOSER_NAME_RE = re.compile(
|
||||
r"write|messag|compose|reply|comment|post your|post text|what.?s happening|"
|
||||
r"tweet|caption|say something|start a|new message|body|your (message|note)|"
|
||||
r"add a comment|write something|text editor|creating content",
|
||||
re.I,
|
||||
)
|
||||
|
||||
# Login/auth walls: a logged-out card lands here, and the structural reveal-finder would
|
||||
# otherwise fill a login field and arm the page's own submit as a "send" (measured live on
|
||||
# instagram/threads). A real composer never lives on one of these, so decline outright.
|
||||
P_LOGIN_WALL_URL_RE = re.compile(
|
||||
r"accounts\.google\.com|/i/flow/login|/accounts/login|/uas/login|/users/sign_in|"
|
||||
r"/sessions/new|/checkpoint|force_authentication|"
|
||||
r"/(?:log[_-]?in|sign[_-]?in|signin|logon)(?:[/?#]|$)",
|
||||
re.I,
|
||||
)
|
||||
P_LOGIN_WALL_STATE_RE = re.compile(
|
||||
r'<\s*textbox\s+"[^"]*(?:password|passwd)|(?:log|sign)\s?in to |'
|
||||
r"continue with (?:google|apple|facebook)",
|
||||
re.I,
|
||||
)
|
||||
|
||||
P_OPENER_ROW_RE = re.compile(
|
||||
r"\[(\d+)\]\*?<\s*(?:link|button)\s+\"(Message|Reply|Compose|New message|"
|
||||
r"Direct message|DM|Send message|Write|New chat|Comment|Post)\"", re.I)
|
||||
|
||||
# A verification probe quotes the very payload it's checking for, which is exactly the trap this gate exists for: quoted payload + composer = fire. Caught live (r243): the read-only send-probe delivered a REAL message. Read-only directives decline in code, fail-safe (a false match just means the model path).
|
||||
P_READONLY_RE = re.compile(
|
||||
r"read.?only|do\s+not\s+(?:send|type|click|post|submit|change|edit|delete)|"
|
||||
r"don'?t\s+(?:send|post|submit|change|edit|delete)|"
|
||||
# "verify/check/tell me/say/confirm WHETHER x is there" is the whole family, not two phrasings
|
||||
# of it. Measured: "say whether anything containing <quoted text> is still there. Change
|
||||
# nothing." slipped through and POSTED the quoted text to a real LinkedIn feed, because only
|
||||
# "verify whether" and "check whether" were listed. Anchor on the question shape.
|
||||
r"(?:verify|check|confirm|tell\s+me|say|see|find\s+out|look)\s+(?:me\s+)?(?:if|whether)|"
|
||||
r"is\s+(?:it|there|this|that)\s+(?:still\s+)?(?:there|published|posted|live|present)|"
|
||||
r"still\s+(?:there|published|posted|live|up)|"
|
||||
r"change\s+nothing|without\s+(?:sending|posting|changing)|verification",
|
||||
re.I,
|
||||
)
|
||||
|
||||
|
||||
def looks_like_login_wall(current_url: str, state_text: str) -> bool:
|
||||
"""A login/auth page (by URL) or an auth form in the perception (a password field, a
|
||||
'Log in to X' heading, an OAuth 'Continue with ...'). The scripted send declines here:
|
||||
a real composer never shares a page with these, and filling here types a login field."""
|
||||
if current_url and P_LOGIN_WALL_URL_RE.search(current_url):
|
||||
return True
|
||||
return bool(state_text and P_LOGIN_WALL_STATE_RE.search(state_text))
|
||||
|
||||
|
||||
# SOFT signed-out: the site serves a browsable page with no auth form and no login URL, it just
|
||||
# withholds the composer and offers a "Sign in" control (bsky, stackoverflow, tiktok, threads all
|
||||
# behave this way). The hard-wall gate above sees nothing, so the run used to report "I couldn't
|
||||
# find the compose box" when the truth was "you are not signed in", which is a different problem
|
||||
# with a different fix. Only ever consulted AFTER a composer miss, so it cannot affect a success.
|
||||
P_SIGNIN_AFFORDANCE_RE = re.compile(
|
||||
r'<\s*(?:link|button)\s+"[^"]*(?:sign[_ -]?in|log[_ -]?in|sign[_ -]?up|create account|join now)',
|
||||
re.I)
|
||||
# Anything only a signed-IN page shows. Its presence vetoes the verdict, so a stray "Log in" on an
|
||||
# authenticated page (a second product's promo) can't make us tell the user to sign in again.
|
||||
# Deliberately NARROW: an earlier draft also vetoed on "notifications"/"profile"/"inbox", which
|
||||
# logged-OUT pages advertise all the time, and that silently suppressed the whole detector on the
|
||||
# exact sites it exists for (measured: bsky with 0 cookies read as signed-in). Only a control that
|
||||
# is meaningless unless you are already authenticated belongs here.
|
||||
P_SIGNED_IN_RE = re.compile(
|
||||
r'(?:sign|log)[_ -]?out\b|your profile|account menu|my account',
|
||||
re.I)
|
||||
|
||||
|
||||
def looks_signed_out(state_text: str) -> bool:
|
||||
"""True when the page offers a way to sign IN and shows nothing only a signed-in user sees."""
|
||||
if not state_text:
|
||||
return False
|
||||
if P_SIGNED_IN_RE.search(state_text):
|
||||
return False
|
||||
return bool(P_SIGNIN_AFFORDANCE_RE.search(state_text))
|
||||
|
||||
|
||||
# Creating a POST and commenting on someone else's are different actions on different content.
|
||||
# LinkedIn's feed carries a comment box on EVERY post, and the capped interactives listing routinely
|
||||
# starves the real post modal of its own composer, so the only compose-shaped textbox left in the
|
||||
# list is a stranger's comment box. Filling that is not a slower path to the same place, it is the
|
||||
# wrong action on the wrong person's content. Measured in a dry-run sweep: linkedin reached its
|
||||
# composer 1/4, and two of the three misses targeted 'Text editor for creating comment'.
|
||||
P_POST_INTENT_RE = re.compile(r"\b(post|tweet|publish|share)\b", re.I)
|
||||
P_COMMENT_INTENT_RE = re.compile(r"\b(comment|reply|respond)\b", re.I)
|
||||
P_COMMENT_SURFACE_RE = re.compile(r"\b(comment|reply)\b", re.I)
|
||||
|
||||
|
||||
def surface_mismatch(task: str, composer_name: str) -> bool:
|
||||
"""True when the task asks to create a POST but the composer found is a comment/reply box.
|
||||
|
||||
Deliberately one-directional: a task that mentions commenting is left alone, so this can only
|
||||
ever reject a comment box for a post task, never the reverse. A rejection is cheap (the
|
||||
structural finder, which does find LinkedIn's real composer, gets its turn instead)."""
|
||||
t, name = task or "", composer_name or ""
|
||||
if not P_POST_INTENT_RE.search(t) or P_COMMENT_INTENT_RE.search(t):
|
||||
return False
|
||||
return bool(P_COMMENT_SURFACE_RE.search(name))
|
||||
|
||||
|
||||
def is_readonly(text: str) -> bool:
|
||||
"""A read-only directive ('verify whether', 'do not send') that must decline the scripted
|
||||
send even with a quoted payload in hand. Keeps the regex private to this file."""
|
||||
return bool(text and P_READONLY_RE.search(text))
|
||||
|
||||
|
||||
def quoted_payload(task: str) -> str:
|
||||
"""The exact text the user quoted, only when it's unambiguous: exactly one
|
||||
distinct quoted span in the task. Anything else is the model's judgment call.
|
||||
Double quotes win outright; single quotes must be word-boundary-delimited so
|
||||
an apostrophe inside a name can't hijack the match."""
|
||||
dq = {m.group(1).strip() for m in P_QUOTED_DQ_RE.finditer(task or "") if m.group(1).strip()}
|
||||
if dq:
|
||||
return dq.pop() if len(dq) == 1 else ""
|
||||
sq = {m.group(1).strip() for m in P_QUOTED_SQ_RE.finditer(task or "") if m.group(1).strip()}
|
||||
return sq.pop() if len(sq) == 1 else ""
|
||||
|
||||
|
||||
def opener_index_in_state(state_text: str):
|
||||
"""(index, name) of the single exact-named composer OPENER, or None. Exact
|
||||
names only, so an upsell like 'Send InMail' can never match."""
|
||||
hits = [(int(m.group(1)), m.group(2)) for m in P_OPENER_ROW_RE.finditer(state_text or "")]
|
||||
return hits[0] if len(hits) == 1 else None
|
||||
|
||||
|
||||
def composer_index_in_state(state_text: str):
|
||||
"""(index, name) of the single compose-shaped textbox, or None. Two
|
||||
candidates = ambiguous = model's problem."""
|
||||
hits = [(int(m.group(1)), m.group(2)) for m in P_COMPOSER_ROW_RE.finditer(state_text or "")
|
||||
if P_COMPOSER_NAME_RE.search(m.group(2) or "")]
|
||||
return hits[0] if len(hits) == 1 else None
|
||||
|
||||
|
||||
def textbox_count(state_text: str) -> int:
|
||||
"""How many textboxes the perception listed, compose-shaped or not.
|
||||
|
||||
Diagnostic only, and only meaningful next to a failed composer pick: zero means the page never
|
||||
mounted one, several means the picker refused an ambiguous choice. Those are different bugs."""
|
||||
return len(P_COMPOSER_ROW_RE.findall(state_text or ""))
|
||||
|
||||
|
||||
def surface_supports_script(current_url: str, state_text: str = "") -> bool:
|
||||
"""STRUCTURAL, not per-site: fire wherever the live perception actually carries a
|
||||
person-composer (a compose-shaped textbox) OR a single messaging opener to reach
|
||||
one, on ANY host. This is what generalizes the LinkedIn ~14s send to X/Slack/
|
||||
Discord/Instagram/Gmail/etc without per-site URL gates. A page with neither
|
||||
declines (net-negative to fire where there's no composer). All the downstream
|
||||
safety gates (quoted payload, fill-seen-committed before the one send, two-sided
|
||||
receipt) are already site-agnostic, so widening the surface can't loosen safety."""
|
||||
if not state_text:
|
||||
return False
|
||||
return bool(composer_index_in_state(state_text) or opener_index_in_state(state_text))
|
||||
|
||||
|
||||
def dryrun_report(state_text: str, armed: bool, filled: bool, url: str = "") -> str:
|
||||
"""One grep-stable line for the coverage harness: what the staged perception held
|
||||
and how far the script got. Only ever emitted in dry-run measurement mode."""
|
||||
boxes = len(P_COMPOSER_ROW_RE.findall(state_text or ""))
|
||||
return (f"[dryrun-report] armed={int(bool(armed))} "
|
||||
f"composer={int(bool(composer_index_in_state(state_text or '')))} "
|
||||
f"opener={int(bool(opener_index_in_state(state_text or '')))} "
|
||||
f"textboxes={boxes} filled={int(bool(filled))} url={(url or '')[:120]}")
|
||||
@@ -0,0 +1,431 @@
|
||||
"""
|
||||
Staged-send script: when the pre-stage leaves a READY composer (a compose
|
||||
textbox and a real Send button both visible) and the task names its payload in
|
||||
quotes, code performs the fill/verify/send/verify tail the model otherwise
|
||||
spends 4-5 turns (~15s) on.
|
||||
|
||||
Safety is the same bar as the loop's, enforced in code: the payload must be
|
||||
SEEN committed to the textbox before the one irreversible click, the Send
|
||||
button is re-resolved from fresh state after the fill (indices shift), and the
|
||||
composer must be SEEN cleared after. Any ambiguity BEFORE the click aborts to
|
||||
the untouched model path; ambiguity AFTER the click hands the model a truthful
|
||||
"clicked, unverified, do NOT re-send" note, never a silent retry.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Awaitable, Callable, Dict
|
||||
|
||||
from backend.apps.agents.browser import (
|
||||
browser_delivery_check, browser_fast_path, browser_send_parse, browser_submit_click,
|
||||
browser_verified_action)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]]
|
||||
|
||||
# The worst case this routine can legitimately take, so the CALLER cannot starve it. Roughly: the
|
||||
# composer poll (3 backoff waits plus three 6s interactive lists), one structural finder call (which
|
||||
# carries its own 30s command timeout), then fill-and-commit and submit-and-receipt.
|
||||
#
|
||||
# This constant exists because the caller's timeout and this routine's real cost silently drifted
|
||||
# apart: raising find_composer's command timeout from 15s to 30s made a single finder call able to
|
||||
# eat the caller's entire 30s budget, so the whole script was killed mid-send and EVERY write fell
|
||||
# back to the slow model loop. It failed invisibly, because asyncio.TimeoutError stringifies to
|
||||
# nothing and the log read "outer skip ()". Measured live on LinkedIn: a 190.9s write that never
|
||||
# posted. Import this instead of writing a number at the call site.
|
||||
WORST_CASE_BUDGET_S = 75.0
|
||||
|
||||
|
||||
def script_enabled() -> bool:
|
||||
"""Default ON. Every gate below it fails CLOSED to the old model loop, so the worst case is
|
||||
today's behaviour, never a wrong send: the payload must be unambiguously quoted, the task must
|
||||
not read as a question, the surface must not be a login wall, a post task will not settle for a
|
||||
comment box, the fill must be seen committed, and delivery needs a two-sided receipt.
|
||||
Turning this off also disables the prestage opener and the mid-loop autosend takeover."""
|
||||
return os.environ.get("OSW_SEND_SCRIPT", "1") != "0"
|
||||
|
||||
|
||||
def autosend_enabled() -> bool:
|
||||
"""The mid-loop post-fill takeover: after the MODEL types the message into a composer, the code
|
||||
finishes the send (find Send, click, verify receipt) instead of the model burning ~3-4 turns on
|
||||
a Send button whose index goes stale after the fill. Rides with the send-script family (same
|
||||
tail + safety), with its own kill switch."""
|
||||
return script_enabled() and os.environ.get("OSW_AUTOSEND", "1") != "0"
|
||||
|
||||
|
||||
async def complete_send(
|
||||
payload: str, state_committed: str, browser_id: str, tab_id: str,
|
||||
execute_tool: ToolRunner, send_index_in_state: Callable[[str, int], object],
|
||||
composer_index: int = -1, current_url: str = "",
|
||||
) -> Dict[str, object]:
|
||||
"""Send tail for a composer that ALREADY holds `payload` (visible in state_committed): find the
|
||||
Send control (ranked index first, else click-by-name over the full DOM), click it once, and
|
||||
verify the two-sided receipt (the composer cleared the payload). Returns {clicked, sent, log,
|
||||
note}: `clicked` = the send click landed, `sent` = the clear was verified. Never types, so it
|
||||
can't fabricate content; a wrong Send match just fails the receipt, never a false claim. Shared
|
||||
by the dispatch send-script and the mid-loop post-fill takeover."""
|
||||
log: list = []
|
||||
|
||||
async def fresh_list() -> str:
|
||||
try:
|
||||
r = await asyncio.wait_for(
|
||||
execute_tool("BrowserListInteractives", {}, browser_id, tab_id), timeout=6.0)
|
||||
return str(r.get("text") or "") if isinstance(r, dict) and "error" not in r else ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
send_btn = send_index_in_state(state_committed, composer_index)
|
||||
via = "index"
|
||||
if send_btn:
|
||||
r_send = await execute_tool("BrowserClickIndex", {"index": send_btn[0]}, browser_id, tab_id)
|
||||
send_name = send_btn[1]
|
||||
else:
|
||||
# No submit listed below the composer (the capped listing can starve a modal of its own
|
||||
# button): resolve the submit inside the composer's OWN container and click it with REAL
|
||||
# input (synthetic clicks are ignored by web-component sites), then last-resort by-name.
|
||||
r_ev = await execute_tool(
|
||||
"BrowserEvaluate",
|
||||
{"expression": browser_submit_click.container_submit_expression(payload)}, browser_id, tab_id)
|
||||
p_v = browser_submit_click.parse_eval_value(r_ev)
|
||||
if isinstance(p_v, dict) and p_v.get("ok") and p_v.get("xPct") is not None:
|
||||
r_send = await execute_tool(
|
||||
"BrowserClickPoint",
|
||||
{"xPercent": float(p_v["xPct"]), "yPercent": float(p_v["yPct"])}, browser_id, tab_id)
|
||||
send_name = str(p_v.get("name") or "submit")
|
||||
via = "container"
|
||||
else:
|
||||
p_why = p_v.get("why") if isinstance(p_v, dict) else "unreadable eval"
|
||||
logger.info(f"[browser-sendscript] container submit miss ({p_why}); by-name fallback")
|
||||
r_send = await execute_tool("BrowserClickByName", {"name": "Send", "role": "button"}, browser_id, tab_id)
|
||||
send_name = "Send (by-name)"
|
||||
via = "by-name"
|
||||
clicked = isinstance(r_send, dict) and "error" not in r_send
|
||||
log.append({"tool": "send click", "input": {"via": via},
|
||||
"ok": clicked, "result_summary": f"send click {send_name!r}"[:200],
|
||||
"elapsed_ms": 0, "clicked_role": "button", "clicked_name": send_name})
|
||||
if not clicked:
|
||||
return {"clicked": False, "sent": False, "log": log, "note": "send click errored; fill committed, NOT sent"}
|
||||
sent = False
|
||||
# Name WHY a receipt fails. A withheld receipt costs the whole fast path (measured on LinkedIn:
|
||||
# the script finished in 9.7s, the receipt missed, and the model then spent 28.6s re-verifying a
|
||||
# post that HAD landed, 60s total against ~24s when the receipt passes), and "sent_receipt=False"
|
||||
# alone cannot tell you whether the composer still holds the text or we simply could not read the
|
||||
# page. Those are different bugs with different fixes.
|
||||
p_why = "no-poll"
|
||||
for wait_s in (0.4, 1.0, 1.6):
|
||||
await asyncio.sleep(wait_s)
|
||||
state3 = await fresh_list()
|
||||
if not state3:
|
||||
p_why = "unreadable-list"
|
||||
continue
|
||||
if browser_verified_action.expectation_met(f"cleared:{payload}", state_committed, state3):
|
||||
sent = True
|
||||
break
|
||||
p_why = f"payload-still-in-a-textbox (textbox rows={sum(1 for x in state3.splitlines() if '<textbox' in x)})"
|
||||
if not sent:
|
||||
logger.info(f"[browser-sendscript] receipt withheld after {sum((0.4, 1.0, 1.6)):.1f}s of polling: {p_why}")
|
||||
# A cleared composer is proof of delivery everywhere EXCEPT the ghost-drop hosts, which clear
|
||||
# then silently eat the post; there we verify it persisted. delivered stays None (unchecked,
|
||||
# composer-clear trusted) for every other site, so proven sends keep their exact speed.
|
||||
delivered = None
|
||||
rejected = False
|
||||
# The site gets the first word. A cleared composer plus "Something went wrong" is a REFUSAL, and
|
||||
# trusting the clear there is how the agent ends up announcing a post that never existed.
|
||||
if sent and await browser_delivery_check.send_rejected(browser_id, tab_id, execute_tool):
|
||||
rejected, delivered, sent = True, False, False
|
||||
logger.info("[browser-sendscript] composer cleared but the page announced a failure; "
|
||||
"treating as REJECTED, not delivered")
|
||||
elif sent and browser_delivery_check.is_ghost_drop_host(current_url):
|
||||
delivered = await browser_delivery_check.ghost_delivery_confirmed(
|
||||
payload, browser_id, tab_id, execute_tool)
|
||||
elif sent and via == "by-name":
|
||||
# The by-name click is the ONE path where we never actually located the submit: both
|
||||
# structured resolvers failed, so the literal "Send" is a guess, and it can land on some
|
||||
# OTHER widget's Send while this composer closes anyway. Measured live on LinkedIn's feed
|
||||
# composer (whose submit is "Post", not "Send"): sent_receipt=True and nothing posted, on
|
||||
# either the posts or the comments tab. A cleared composer cannot tell submitted from
|
||||
# dismissed, so a guessed click does not get to be proof by itself; it has to show the
|
||||
# payload actually rendered on the page. The two resolved paths are untouched and keep
|
||||
# their measured speed.
|
||||
delivered = await browser_delivery_check.payload_visible(
|
||||
payload, browser_id, tab_id, execute_tool)
|
||||
if delivered is False:
|
||||
logger.info("[browser-sendscript] by-name click cleared the composer but the payload "
|
||||
"never rendered; treating as NOT delivered")
|
||||
elif delivered is None:
|
||||
# We could not look. That is not the same as looking and finding nothing, and saying
|
||||
# "it did not render" here would be inventing a failure out of a broken probe.
|
||||
logger.info("[browser-sendscript] by-name click cleared the composer but the delivery "
|
||||
"probe was unreadable; leaving delivery UNKNOWN")
|
||||
if rejected:
|
||||
# We are not guessing here: the page said no. Saying "unverified" would send the user off to
|
||||
# check something we already know the answer to.
|
||||
note = browser_delivery_check.rejected_send_note(current_url, payload)
|
||||
elif sent:
|
||||
note = ""
|
||||
else:
|
||||
note = ("A Send-class click already RAN for this payload but the composer state is "
|
||||
"unverified: verify on the page whether it delivered; do NOT send again unless "
|
||||
"verifiably absent.")
|
||||
return {"clicked": True, "sent": sent, "delivered": delivered, "log": log, "note": note}
|
||||
|
||||
|
||||
async def run_send_script(
|
||||
task: str,
|
||||
browser_id: str,
|
||||
tab_id: str,
|
||||
state_text: str,
|
||||
execute_tool: ToolRunner,
|
||||
send_index_in_state,
|
||||
payload_in_textbox,
|
||||
payload_source: str = "",
|
||||
current_url: str = "",
|
||||
) -> dict | None:
|
||||
"""None = stage not script-ready or aborted pre-click (model path, stage
|
||||
untouched except a possibly committed fill, which the model sees). A dict
|
||||
means the irreversible click RAN: {'sent': bool_receipt_verified,
|
||||
'payload': str, 'log': [...], 'note': str}. payload_source is the RAW user
|
||||
prompt; the composed task carries the routing brief whose own quoted strings
|
||||
made every real payload look ambiguous (r242/r243)."""
|
||||
t0 = time.monotonic()
|
||||
p_struct = os.environ.get("OSW_COMPOSER_STRUCT") == "1"
|
||||
|
||||
async def fresh_list() -> str:
|
||||
try:
|
||||
r = await asyncio.wait_for(
|
||||
execute_tool("BrowserListInteractives", {}, browser_id, tab_id), timeout=6.0)
|
||||
return str(r.get("text") or "") if isinstance(r, dict) and "error" not in r else ""
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
# The name-based surface gate can't see an unnamed/non-standard composer; under the
|
||||
# structural flag, don't early-decline on it, the in-page finder gets a chance below.
|
||||
if not browser_send_parse.surface_supports_script(current_url, state_text) and not p_struct:
|
||||
# The composer lazy-renders a beat after prestage snapshotted (X home does this ~half the
|
||||
# time), so poll a fresh perception before declining, else a late box is a false "no
|
||||
# composer" and the whole write flakes to the slow model path.
|
||||
for wait_s in (0.6, 1.0, 1.4):
|
||||
await asyncio.sleep(wait_s)
|
||||
fresh = await fresh_list()
|
||||
if browser_send_parse.surface_supports_script(current_url, fresh):
|
||||
state_text = fresh
|
||||
break
|
||||
else:
|
||||
logger.info(f"[browser-sendscript] decline: no composer or opener after poll ({current_url[:50]!r})")
|
||||
return None
|
||||
# Key read-only on words a HUMAN wrote: the task minus the aux routing brief (the brief wrote
|
||||
# "do not submit it" for a plain "start a post", falsely read-only-flagging a real send) PLUS
|
||||
# the raw prompt when threaded through. The task text itself must keep declining regardless: a
|
||||
# read-only VERIFY probe arrives as the task, and one once delivered a real message (r243).
|
||||
task_sans_brief = task.split(browser_fast_path.BRIEF_MARKER, 1)[0]
|
||||
if browser_send_parse.is_readonly(task_sans_brief) or (payload_source and browser_send_parse.is_readonly(payload_source)):
|
||||
logger.info("[browser-sendscript] decline: read-only directive in user request")
|
||||
return None
|
||||
if browser_send_parse.looks_like_login_wall(current_url, state_text):
|
||||
logger.info(f"[browser-sendscript] decline: login/auth wall ({(current_url or '')[:60]!r})")
|
||||
return None
|
||||
payload = browser_send_parse.quoted_payload(payload_source or task)
|
||||
if not payload:
|
||||
logger.info("[browser-sendscript] decline: no unambiguous quoted payload")
|
||||
return None
|
||||
log: list[dict] = []
|
||||
|
||||
composer = browser_send_parse.composer_index_in_state(state_text)
|
||||
if composer and browser_send_parse.surface_mismatch(task_sans_brief, composer[1]):
|
||||
# Asked to POST, found a COMMENT box: that is someone else's content, not a slower route to
|
||||
# ours. Drop it and let the tiers below (opener, then the structural finder, which does find
|
||||
# LinkedIn's real composer) look properly.
|
||||
logger.info(f"[browser-sendscript] ignoring {composer[1]!r}: a comment box is not where a post goes")
|
||||
composer = None
|
||||
if not composer:
|
||||
# The staged snapshot is prestage's, frozen the instant it clicked Message; the overlay composer lazy-renders a beat later (r263/r269 declined on exactly this, prestage's LAST step was the Message click). Poll a short window so the overlay has time to appear before we fall back to the opener.
|
||||
for wait_s in (0.6, 1.2, 1.4):
|
||||
await asyncio.sleep(wait_s)
|
||||
fresh = await fresh_list()
|
||||
composer = browser_send_parse.composer_index_in_state(fresh)
|
||||
if composer:
|
||||
state_text = fresh
|
||||
break
|
||||
p_struct_selector: str = ""
|
||||
if not composer:
|
||||
# Reversible-opener hop: prestage often stops on the profile with the "Message" opener visible (its settle raced the overlay). Opening a composer is the allowed opener class; the irreversible bar is unchanged.
|
||||
opener = browser_send_parse.opener_index_in_state(state_text)
|
||||
if opener and browser_send_parse.surface_mismatch(task_sans_brief, opener[1]):
|
||||
# The same post-is-not-a-comment rule the composer already enforces, applied one step
|
||||
# earlier. Measured on linkedin.com with the task "start a post": the only opener listed
|
||||
# was 'Comment', so the script opened a stranger's comment box, found no post composer
|
||||
# inside it, and declined. Opening the wrong surface is not a slower route to the right
|
||||
# one, and here it also burns the reversible-opener hop we only get once.
|
||||
logger.info(f"[browser-sendscript] ignoring opener {opener[1]!r}: a comment box is not "
|
||||
f"where a post goes")
|
||||
opener = None
|
||||
if opener:
|
||||
logger.info(f"[browser-sendscript] firing via opener {opener[1]!r} [{opener[0]}]")
|
||||
r_open = await execute_tool("BrowserClickIndex", {"index": opener[0]}, browser_id, tab_id)
|
||||
if not (isinstance(r_open, dict) and "error" not in r_open):
|
||||
return None
|
||||
log.append({"tool": "BrowserClickIndex", "input": {"index": opener[0]}, "ok": True,
|
||||
"result_summary": f"script opened composer via {opener[1]!r}"[:200], "elapsed_ms": 0})
|
||||
# Wait for the surface to STOP MOVING, not for a number of seconds. Fixed budgets kept
|
||||
# being wrong in both directions: 1.8s missed gmail and linkedin entirely, 5.3s still
|
||||
# missed a cold gmail compose window that existed a beat later, and simply making the
|
||||
# number bigger taxes every run that was never going to succeed. A mounting surface
|
||||
# keeps changing the element list; once two consecutive reads are identical, nothing
|
||||
# more is coming and more waiting is pure cost.
|
||||
p_prev = ""
|
||||
p_settled = 0
|
||||
for wait_s in (0.6, 1.2, 1.5, 2.0, 2.0, 2.0):
|
||||
await asyncio.sleep(wait_s)
|
||||
state_text = await fresh_list()
|
||||
composer = browser_send_parse.composer_index_in_state(state_text)
|
||||
if composer:
|
||||
break
|
||||
p_settled = p_settled + 1 if state_text and state_text == p_prev else 0
|
||||
p_prev = state_text
|
||||
if p_settled >= 1:
|
||||
logger.info("[browser-sendscript] opener surface settled with no composer; "
|
||||
"not waiting out the rest of the budget")
|
||||
break
|
||||
# Structural fallback: the AX-name detector missed it (an unnamed contenteditable, a
|
||||
# non-standard rich editor, or two textboxes it couldn't disambiguate). Ask the page to
|
||||
# rank its editable regions and fill+read-back the winner IN-PAGE (the only reliable
|
||||
# commit-check for a React contenteditable, whose text never reaches the AX value).
|
||||
# Flag-gated so the proven name path stays the default.
|
||||
if not composer and p_struct:
|
||||
# OSW_COMPOSER_REVEAL: let the finder take one reversible reveal action (open the
|
||||
# compose surface: a modal trigger, the first conversation, or a scroll) when the
|
||||
# composer isn't painted yet. It never commits a send, only opens a surface.
|
||||
p_reveal = os.environ.get("OSW_COMPOSER_REVEAL") == "1"
|
||||
# A reveal that OPENS the first list item (a Reddit thread, a TikTok video, a GitHub
|
||||
# issue) is a full-page NAVIGATION: it kills the finder's own JS context, so that one
|
||||
# call can't reach the composer that only exists on the destination. When the finder
|
||||
# reports it fired `open-first` but found nothing, the page is now loading the item;
|
||||
# give it a beat and run the finder ONCE more on the destination. Bounded to 2 tries so
|
||||
# a feed-of-feeds can't walk forever.
|
||||
fc: Dict[str, object] = {}
|
||||
for attempt in range(2):
|
||||
fc = await execute_tool("BrowserFindComposer", {"fill": payload, "reveal": p_reveal}, browser_id, tab_id)
|
||||
if isinstance(fc, dict) and fc.get("found") and fc.get("filled"):
|
||||
break
|
||||
revs = fc.get("reveals") if isinstance(fc, dict) else None
|
||||
navigated = p_reveal and isinstance(revs, list) and "open-first" in revs
|
||||
if not navigated:
|
||||
break
|
||||
logger.info("[browser-sendscript] reveal navigated (open-first); re-perceiving the destination")
|
||||
await asyncio.sleep(1.5)
|
||||
dest = await fresh_list()
|
||||
# open-first can land on a login redirect (a logged-out feed's first item);
|
||||
# stop before the NEXT fill so we never type into the auth form we just opened.
|
||||
if browser_send_parse.looks_like_login_wall("", dest):
|
||||
logger.info("[browser-sendscript] decline: reveal landed on a login/auth wall")
|
||||
fc = {}
|
||||
break
|
||||
if isinstance(fc, dict) and fc.get("found") and fc.get("filled"):
|
||||
p_struct_selector = str(fc.get("selector") or "")
|
||||
logger.info(f"[browser-sendscript] structural composer role={fc.get('role')!r} "
|
||||
f"score={fc.get('score')} nearSubmit={fc.get('nearSubmit')} "
|
||||
f"reveals={fc.get('reveals')} fillMode={fc.get('fillMode')} filled+verified")
|
||||
log.append({"tool": "BrowserFindComposer", "input": {"fill": "<payload>"}, "ok": True,
|
||||
"result_summary": f"structural composer {fc.get('role')!r} filled+verified"[:200], "elapsed_ms": 0})
|
||||
composer = (-1, str(fc.get("role") or "composer"))
|
||||
else:
|
||||
logger.info(f"[browser-sendscript] structural finder: no usable composer ({str(fc)[:120]})")
|
||||
if not composer:
|
||||
# Name WHY. A site that withholds the composer because nobody is signed in is a
|
||||
# different problem from one whose composer we failed to find, and only the first is
|
||||
# fixable by the user (sign in once). Consulted only here, on the already-failed path.
|
||||
if browser_send_parse.looks_signed_out(state_text):
|
||||
logger.info("[browser-sendscript] decline: signed OUT (composer withheld, sign-in offered)")
|
||||
else:
|
||||
logger.info("[browser-sendscript] decline: no composer, opener, or structural editable")
|
||||
return None
|
||||
# No Send-button precondition: composer sites (LinkedIn) lazy-render Send only AFTER text commits, so it's resolved post-fill; never appearing = clean pre-click abort.
|
||||
logger.info(f"[browser-sendscript] fill target {composer[1]!r} [{composer[0]}]")
|
||||
|
||||
if p_struct_selector:
|
||||
# The finder already filled + read-back-verified in-page; nothing to re-fill or re-check.
|
||||
state2 = await fresh_list()
|
||||
committed = True
|
||||
else:
|
||||
# 1. fill (focused by node, the composer overlay path coordinate clicks miss)
|
||||
r_fill = await execute_tool("BrowserClickIndex", {"index": composer[0], "text": payload}, browser_id, tab_id)
|
||||
fill_ok = isinstance(r_fill, dict) and "error" not in r_fill
|
||||
log.append({"tool": "BrowserClickIndex", "input": {"index": composer[0], "text": payload},
|
||||
"ok": fill_ok, "result_summary": f"script fill into {composer[1]!r}"[:200], "elapsed_ms": 0})
|
||||
if not fill_ok and browser_submit_click.is_stale_index_error(r_fill):
|
||||
# The opener click opens a modal that keeps re-rendering after we listed it, so the
|
||||
# composer node we resolved is already detached by the time the fill lands. Measured
|
||||
# live on x.com: 'Index 53 is not in the cached element map', on the exact run where
|
||||
# the script had correctly found opener 'Post' and target 'Post text'. Re-listing is
|
||||
# what the error itself prescribes, so take it once rather than surrendering a send
|
||||
# the script had already located. One retry only: a second failure is a different
|
||||
# problem and the model path is the right answer for it.
|
||||
# Poll, don't snapshot. A single re-list catches the composer only if the modal happens
|
||||
# to be settled at that instant; mid-churn it shows zero or two compose-shaped boxes,
|
||||
# composer_index_in_state returns None (ambiguous), and the retry used to give up
|
||||
# without a word. Measured: 5 successful retries in one arm, 0 in the next, same code,
|
||||
# purely on timing. Same poll shape the opener path already uses.
|
||||
composer_retry = None
|
||||
for wait_s in (0.0, 0.5, 1.0):
|
||||
if wait_s:
|
||||
await asyncio.sleep(wait_s)
|
||||
state_retry = await fresh_list()
|
||||
composer_retry = browser_send_parse.composer_index_in_state(state_retry)
|
||||
if composer_retry:
|
||||
break
|
||||
if not composer_retry:
|
||||
logger.info("[browser-sendscript] composer index went stale and did not re-resolve "
|
||||
"within 1.5s of polling; handing to model")
|
||||
if composer_retry:
|
||||
logger.info(f"[browser-sendscript] stale composer index {composer[0]}; refreshed to "
|
||||
f"{composer_retry[0]} and retrying the fill once")
|
||||
composer = composer_retry
|
||||
r_fill = await execute_tool(
|
||||
"BrowserClickIndex", {"index": composer[0], "text": payload}, browser_id, tab_id)
|
||||
fill_ok = isinstance(r_fill, dict) and "error" not in r_fill
|
||||
log.append({"tool": "BrowserClickIndex",
|
||||
"input": {"index": composer[0], "text": payload}, "ok": fill_ok,
|
||||
"result_summary": f"script fill retry into {composer[1]!r}"[:200],
|
||||
"elapsed_ms": 0})
|
||||
if not fill_ok:
|
||||
# Name the cause. "fill errored" alone cannot tell a stale index from a detached node
|
||||
# from a site that refuses synthetic input, and those are three different fixes. Same
|
||||
# lesson as the bare TimeoutError that used to log "outer skip ()".
|
||||
p_err = r_fill.get("error") if isinstance(r_fill, dict) else type(r_fill).__name__
|
||||
logger.info(f"[browser-sendscript] fill errored ({str(p_err)[:160]}); "
|
||||
f"handing to model untouched")
|
||||
return None
|
||||
# 2. verify the fill committed. Send is resolved AFTER, two ways: LinkedIn enables Send only once its JS digests the input (beats later than the text is visible), so the scan waits a little.
|
||||
state2 = ""
|
||||
committed = False
|
||||
for wait_s in (0.4, 0.8, 1.2, 1.6):
|
||||
await asyncio.sleep(wait_s)
|
||||
state2 = await fresh_list()
|
||||
committed = bool(state2 and payload_in_textbox(state2, payload))
|
||||
if committed:
|
||||
break
|
||||
if not committed:
|
||||
logger.info("[browser-sendscript] fill not seen committed; aborting pre-click")
|
||||
return None
|
||||
# Dry-run probe: prove the script FIRES + fills on a NON-LinkedIn site without ever
|
||||
# doing the outward send. Everything up to here ran (surface gate passed, composer
|
||||
# found, fill committed); we stop before the irreversible click and report readiness.
|
||||
if os.environ.get("OSW_SENDSCRIPT_DRYRUN") == "1":
|
||||
send_ready = bool(send_index_in_state(state2, composer[0]))
|
||||
logger.info(f"[browser-sendscript] DRYRUN: WOULD send (fill committed, send_button_listed={send_ready}); not clicking")
|
||||
return {"sent": False, "payload": payload, "log": log,
|
||||
"note": "DRYRUN: filled + ready to send, stopped before the irreversible click"}
|
||||
# 3+4: the irreversible click + two-sided receipt, shared with the mid-loop takeover. A click error hands back to the model (fill committed, not sent); a clicked-but-unverified send returns sent=False so the caller never claims delivery.
|
||||
r = await complete_send(payload, state2, browser_id, tab_id, execute_tool, send_index_in_state,
|
||||
composer_index=composer[0], current_url=current_url)
|
||||
log.extend(r["log"])
|
||||
if not r["clicked"]:
|
||||
logger.info("[browser-sendscript] send click errored; handing to model (fill committed, NOT sent)")
|
||||
return None
|
||||
logger.info(f"[browser-sendscript] done sent_receipt={r['sent']} delivered={r.get('delivered')} in {int((time.monotonic() - t0) * 1000)}ms")
|
||||
return {"sent": bool(r["sent"]), "delivered": r.get("delivered"),
|
||||
"payload": payload, "log": log, "note": str(r["note"])}
|
||||
@@ -0,0 +1,162 @@
|
||||
"""Borrow the sign-in the user already has in their everyday browser, so a browser agent that hits
|
||||
a login wall can carry on as them instead of stopping to ask them to log in all over again.
|
||||
|
||||
The point is that no password is ever typed, stored, or seen. We copy the SESSION the user's real
|
||||
Chrome/Arc/Brave/Edge already holds into the app's own browser partition. It is the same mechanism
|
||||
onboarding uses to read the user's provider chat history, pointed at whatever site the agent is
|
||||
stuck on instead of at a fixed provider list.
|
||||
|
||||
Four things keep it narrow:
|
||||
- Off unless the user turned it on (`browser_import_signins`, default False). Reading their real
|
||||
browser is a decision they make once, explicitly, not one we make for them.
|
||||
- The domain is never model-chosen. It comes from the URL of the page the agent is already stuck
|
||||
on, so no amount of prompt injection can name a site to harvest.
|
||||
- Records only ever travel INTO our own partition. Nothing is read back out.
|
||||
- Values are never logged. Counts and domains only.
|
||||
|
||||
Coverage is honestly partial: Chromium-family browsers on macOS/Windows, and not Chrome's newer
|
||||
app-bound (v20) stores. Everything else returns `no_session` and the run falls back to asking the
|
||||
user to sign in, which is exactly what it did before this existed.
|
||||
|
||||
This is the ONE module in browser/ that knows where the reader lives, so the reader can move house
|
||||
later without anything else noticing.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Any, Dict, List, Literal
|
||||
from uuid import uuid4
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.agents.browser import browser_login_handoff
|
||||
from backend.apps.agents.core.ws_manager import ws_manager
|
||||
from backend.apps.onboarding.usage import browser_cookies
|
||||
from backend.apps.settings.models import AppSettings
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ImportOutcome = Literal["imported", "disabled", "no_session", "bridge_failed"]
|
||||
|
||||
# Google authenticates on the parent SSO domain, so a Gmail/YouTube/Docs session does not live on
|
||||
# the property's own host. The reader already has a named scope for exactly this, and we reuse it
|
||||
# rather than sweeping every google entry the user owns.
|
||||
P_GOOGLE_SUFFIXES = ("google.com", "youtube.com")
|
||||
|
||||
# Chromium counts from 1601-01-01 in microseconds, because of course it does. Electron wants unix
|
||||
# seconds, and an entry with no expiry is session-scoped, so it would evaporate on the next quit.
|
||||
P_CHROMIUM_EPOCH_OFFSET_S = 11644473600
|
||||
|
||||
# Anti-bot clearance tokens are bound to the exact user agent and IP that earned them. Our webview
|
||||
# keeps an "openswarm/" product token in its UA, so a clearance minted by the user's real Chrome can
|
||||
# never match ours, and replaying a mismatched one reads as token theft: the edge hands back a fresh
|
||||
# challenge instead of letting us through, which is WORSE than arriving with no clearance at all.
|
||||
# Everything else in the jar is the actual session, so we carry that and let the edge re-challenge
|
||||
# us honestly.
|
||||
P_FINGERPRINT_BOUND: set = set()
|
||||
|
||||
|
||||
class SessionImportResult(BaseModel):
|
||||
"""What happened, in a shape the caller can branch on without parsing prose."""
|
||||
|
||||
model_config = ConfigDict(validate_assignment=True)
|
||||
|
||||
outcome: ImportOutcome = "no_session"
|
||||
domain: str = ""
|
||||
entries_applied: int = 0
|
||||
detail: str = ""
|
||||
|
||||
@property
|
||||
def ok(self) -> bool:
|
||||
return self.outcome == "imported"
|
||||
|
||||
|
||||
@typechecked
|
||||
def is_enabled(settings: AppSettings) -> bool:
|
||||
return bool(settings.browser_import_signins)
|
||||
|
||||
|
||||
@typechecked
|
||||
def is_google_property(domain: str) -> bool:
|
||||
d = (domain or "").lower().lstrip(".")
|
||||
return any(d == s or d.endswith(f".{s}") for s in P_GOOGLE_SUFFIXES)
|
||||
|
||||
|
||||
@typechecked
|
||||
def read_site_records(domain: str) -> List[Dict[str, Any]]:
|
||||
"""The user's own session records for `domain`. Blocking: touches SQLite and may raise one OS
|
||||
keychain consent prompt, so callers must keep it off the event loop."""
|
||||
try:
|
||||
if is_google_property(domain):
|
||||
raw = browser_cookies.read_google_session_records()
|
||||
else:
|
||||
raw = browser_cookies.read_provider_cookie_records(domain)
|
||||
except Exception as exc:
|
||||
# A browser we cannot read is a fallback, never a crash: the run just asks the user instead.
|
||||
logger.info(f"[session-import] read failed for {domain}: {type(exc).__name__}")
|
||||
return []
|
||||
return [{**r, "expires": p_unix_expiry(r.get("expires_utc"))} for r in raw
|
||||
if str(r.get("name") or "").lower() not in P_FINGERPRINT_BOUND]
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_unix_expiry(expires_utc: Any) -> float:
|
||||
"""Chromium's stamp as unix seconds, 0.0 for a session-scoped entry (which Electron then leaves
|
||||
session-scoped too, so it dies on quit exactly like it would in the source browser)."""
|
||||
try:
|
||||
raw = int(expires_utc or 0)
|
||||
except (TypeError, ValueError):
|
||||
return 0.0
|
||||
return max(0.0, raw / 1_000_000 - P_CHROMIUM_EPOCH_OFFSET_S) if raw > 0 else 0.0
|
||||
|
||||
|
||||
@typechecked
|
||||
def site_domain(url_or_host: str) -> str:
|
||||
"""Normalise a URL or bare host to the registrable domain the store is keyed by. Delegates so
|
||||
there is exactly one definition of 'which site is this' across the browser modules."""
|
||||
return browser_login_handoff.registrable_domain(url_or_host)
|
||||
|
||||
|
||||
@typechecked
|
||||
def has_importable_session(domain: str) -> bool:
|
||||
"""Whether some browser store holds a session for this domain, WITHOUT decrypting anything and
|
||||
without touching the keychain. Cheap enough to ask before deciding to interrupt the user."""
|
||||
d = site_domain(domain)
|
||||
if not d:
|
||||
return False
|
||||
try:
|
||||
return browser_cookies.has_store(".google.com" if is_google_property(d) else d)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
@typechecked
|
||||
async def import_signin(domain: str, browser_id: str) -> SessionImportResult:
|
||||
"""Copy the user's existing sign-in for `domain` into the app's browser partition.
|
||||
|
||||
Never raises: every failure degrades to a result the caller can fall back from, because that
|
||||
fallback (ask the user to sign in) is exactly the behaviour that existed before this did.
|
||||
"""
|
||||
d = site_domain(domain)
|
||||
if not d:
|
||||
return SessionImportResult(outcome="no_session", domain=domain, detail="no domain")
|
||||
|
||||
records = await asyncio.to_thread(read_site_records, d)
|
||||
if not records:
|
||||
logger.info(f"[session-import] no readable session for {d}")
|
||||
return SessionImportResult(outcome="no_session", domain=d,
|
||||
detail="no session found in your other browsers")
|
||||
|
||||
result = await ws_manager.send_browser_command(
|
||||
uuid4().hex, "import_session", browser_id, {"domain": d, "cookies": records})
|
||||
if not isinstance(result, dict) or result.get("error"):
|
||||
detail = str(result.get("error") if isinstance(result, dict) else result)[:200]
|
||||
logger.info(f"[session-import] bridge failed for {d}: {detail}")
|
||||
return SessionImportResult(outcome="bridge_failed", domain=d, detail=detail)
|
||||
|
||||
count = int(result.get("set") or 0)
|
||||
if count <= 0:
|
||||
return SessionImportResult(outcome="no_session", domain=d, detail="nothing applied")
|
||||
logger.info(f"[session-import] applied {count} entries for {d}")
|
||||
return SessionImportResult(outcome="imported", domain=d, entries_applied=count)
|
||||
@@ -330,6 +330,23 @@ def replay_settle_target(step: dict) -> str | None:
|
||||
return name if 0 < len(name) <= 60 else None
|
||||
|
||||
|
||||
P_COMPOSER_STEP_RE = re.compile(r"write a message|compose|message body|comment|reply|tweet|post text|type here|editor", re.I)
|
||||
|
||||
|
||||
def step_touches_composer(step: dict) -> bool:
|
||||
"""True if this step interacts with the compose box itself (focusing/typing),
|
||||
as opposed to navigation or the opener click. The send-script owns the composer
|
||||
(it polls for the lazy overlay), so the marriage replays only the nav+opener and
|
||||
hands the composer->send tail to the script."""
|
||||
tool = step.get("tool", "")
|
||||
p = step.get("params", {}) or {}
|
||||
if tool == "BrowserType":
|
||||
return True
|
||||
if tool in ("BrowserClickByName", "BrowserClick"):
|
||||
return bool(P_COMPOSER_STEP_RE.search(str(p.get("name") or p.get("selector") or "")))
|
||||
return False
|
||||
|
||||
|
||||
def first_unsafe_step(steps: list[dict]) -> tuple[int, str]:
|
||||
"""Index of the first GENUINELY irreversible step (click Send/Submit/Pay, type
|
||||
into a composer), -1 if none. This is the prefix-replay/batch boundary, so a
|
||||
@@ -781,6 +798,23 @@ def hint_step_adopted(step_key: tuple, action_log: list[dict]) -> bool:
|
||||
return False
|
||||
|
||||
|
||||
def replay_owns_nav(host: str, has_skill: bool, task_is_removal: bool, task_is_send: bool) -> bool:
|
||||
"""Should a learned skill's replay take over navigation, letting the caller skip prestage?
|
||||
|
||||
Yes for a READ: the replayed prefix does the same navigation prestage would aux-drive, faster.
|
||||
|
||||
No for a SEND, and this is the part that was wrong. Prestage is also what hands the send-script
|
||||
its composer perception; skip it and the entire fill/click/receipt tail is unreachable, so the
|
||||
model falls back to burning 4-5 turns. Measured live on x.com with a learned skill present:
|
||||
5/5 writes went the slow way at 41-146s (median ~57s) with the receipt never speaking, against
|
||||
19.4s with the script armed. Prestage on an already-loaded page costs ~2-5s.
|
||||
|
||||
No for a removal either: a delete is a destructive one-shot, not a replayable nav prefix, so a
|
||||
stale delete-"skill" made of scrolls must never hijack it.
|
||||
"""
|
||||
return bool(host and has_skill and not task_is_removal and not task_is_send)
|
||||
|
||||
|
||||
def mark_replay_succeeded(host: str, task: str) -> None:
|
||||
"""A replay ran end to end. Count it and, if the skill was still on
|
||||
probation, PROMOTE it to trusted (the verify gate just passed)."""
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Container-scoped submit click for the receipt-gated send path. Exists because the ranked
|
||||
interactives listing caps at 60 rows and a composer's own submit can fall off it (X's compose
|
||||
modal: covered feed rows behind the overlay ate the cap, so no "Post" row ever reached the index
|
||||
picker, measured live 0/2 deliveries). Scope = the dialog/form ancestor of the editable holding
|
||||
the payload when there is one, else a bounded nearest-scope-first upward walk, so a page-level
|
||||
opener with the same label can never be chosen. A wrong resolution still fails the send receipt
|
||||
downstream, never a false delivery claim."""
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
# BROAD submit vocabulary shared by the index picker (browser_agent) and the JS below, one source
|
||||
# so the two tiers can never drift apart.
|
||||
SEND_LABELS = frozenset({
|
||||
"send", "send now", "send message", # LinkedIn / Gmail / DMs
|
||||
"post", "post all", "tweet", "reply", # X / Threads compose + reply
|
||||
"publish", "comment", "share", # articles / YouTube+FB comments / shares
|
||||
})
|
||||
|
||||
# Gmail names its Send button 'Send (⌘Enter)': a shortcut suffix wrapped in bidi
|
||||
# isolates that defeats exact matching. Strip control chars + any parenthesized tail before compare.
|
||||
P_NAME_NOISE_RE = re.compile(r"[-]|\([^)]*\)")
|
||||
|
||||
|
||||
def clean_button_name(name: str) -> str:
|
||||
return P_NAME_NOISE_RE.sub("", name or "").strip().lower()
|
||||
|
||||
# Resolves the submit and returns its viewport center; the caller clicks it through the REAL
|
||||
# input path (BrowserClickPoint). Synthetic el.click() is ignored by web-component sites
|
||||
# (shreddit live), and a real click lands on whatever is topmost, so overlays can't be fooled.
|
||||
P_CONTAINER_SUBMIT_JS = r"""(() => {
|
||||
const PAYLOAD = %s;
|
||||
const LABELS = new Set(%s);
|
||||
const norm = (s) => (s || '').replace(/\s+/g, ' ').trim().toLowerCase();
|
||||
const vis = (el) => !!el && el.getClientRects().length > 0;
|
||||
const enabled = (el) => !el.disabled && el.getAttribute('aria-disabled') !== 'true';
|
||||
// Same cleaning as clean_button_name: Gmail's Send is 'Send (⌘Enter)' in bidi isolates.
|
||||
const clean = (s) => norm((s || '').replace(/[-]|\([^)]*\)/g, ''));
|
||||
const labelOf = (el) => clean(el.getAttribute('aria-label') || el.textContent || '');
|
||||
const holds = (el) => ((el.value || el.textContent || '').indexOf(PAYLOAD) !== -1);
|
||||
// Shadow piercing both ways: reddit's composer AND its submit live in shreddit shadow roots.
|
||||
const deep = (root, sel, out, depth) => {
|
||||
if (depth > 10 || out.length > 4000) return out;
|
||||
let hits; try { hits = root.querySelectorAll(sel); } catch (e) { hits = []; }
|
||||
for (const el of hits) out.push(el);
|
||||
let all; try { all = root.querySelectorAll('*'); } catch (e) { return out; }
|
||||
for (const el of all) { if (el.shadowRoot) deep(el.shadowRoot, sel, out, depth + 1); }
|
||||
return out;
|
||||
};
|
||||
const up = (el) => el.parentElement || (el.getRootNode() && el.getRootNode().host) || null;
|
||||
const ed = deep(document, '[contenteditable="true"],textarea,input', [], 0)
|
||||
.find((e) => vis(e) && holds(e));
|
||||
if (!ed) return { ok: false, why: 'no editable holding the payload' };
|
||||
const submitIn = (root) => deep(root, 'button,[role="button"]', [], 0)
|
||||
.find((b) => vis(b) && enabled(b) && LABELS.has(labelOf(b)));
|
||||
const isScope = (el) => { try { return el.matches('[role="dialog"],[role="alertdialog"],form'); } catch (e) { return false; } };
|
||||
let scope = null;
|
||||
for (let node = ed; node; node = up(node)) { if (isScope(node)) { scope = node; break; } }
|
||||
let btn = null;
|
||||
if (scope) {
|
||||
btn = submitIn(scope);
|
||||
} else {
|
||||
// Nearest-scope-first walk: X's inline submit shares an ancestor 20 hops above the Draft.js
|
||||
// editable while foreign tweets' buttons only enter at 28 (measured live), so 24 finds the
|
||||
// composer's own submit and stops before any wider scope could.
|
||||
let node = up(ed);
|
||||
for (let hop = 0; node && node !== document.body && hop < 24; hop++, node = up(node)) {
|
||||
btn = submitIn(node);
|
||||
if (btn) break;
|
||||
}
|
||||
}
|
||||
if (!btn) return { ok: false, why: 'no submit control in the composer container' };
|
||||
const r0 = btn.getBoundingClientRect();
|
||||
if (r0.top < 0 || r0.bottom > window.innerHeight) btn.scrollIntoView({ block: 'center' });
|
||||
const r = btn.getBoundingClientRect();
|
||||
return { ok: true, name: labelOf(btn),
|
||||
xPct: (r.left + r.width / 2) / window.innerWidth * 100,
|
||||
yPct: (r.top + r.height / 2) / window.innerHeight * 100 };
|
||||
})()"""
|
||||
|
||||
|
||||
def container_submit_expression(payload: str) -> str:
|
||||
"""The container-scoped submit click for a composer holding `payload` (prefix-matched, same
|
||||
24-char truncation the fill verifier uses)."""
|
||||
return P_CONTAINER_SUBMIT_JS % (json.dumps((payload or "")[:24]), json.dumps(sorted(SEND_LABELS)))
|
||||
|
||||
|
||||
def parse_eval_value(res: object) -> Optional[Dict[str, Any]]:
|
||||
"""The dict a BrowserEvaluate returned, or None. Unreadable shapes are None (honest miss)."""
|
||||
val: object = None
|
||||
if isinstance(res, dict) and "error" not in res:
|
||||
val = res.get("value")
|
||||
if val is None and isinstance(res.get("text"), str):
|
||||
try:
|
||||
val = json.loads(res["text"])
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
val = None
|
||||
return val if isinstance(val, dict) else None
|
||||
|
||||
|
||||
# The browser tool's own words when an index no longer resolves. Anchored on the stable half of the
|
||||
# sentence ("not in the cached element map"), not the whole string, so a reworded tail doesn't
|
||||
# silently turn the retry off and take the fast write path down with it.
|
||||
P_STALE_INDEX_RE = re.compile(r"not in the cached element map|refresh the index", re.I)
|
||||
|
||||
|
||||
def is_stale_index_error(res: object) -> bool:
|
||||
"""Did this tool result fail because the element index went stale?
|
||||
|
||||
Distinct from every other failure: a stale index means the element was THERE and the page
|
||||
re-rendered underneath us, so re-listing and retrying is correct. A genuine miss (no such
|
||||
control, refused input) must not retry, because retrying a real failure is how you double-post.
|
||||
"""
|
||||
if not isinstance(res, dict):
|
||||
return False
|
||||
return bool(P_STALE_INDEX_RE.search(str(res.get("error") or "")))
|
||||
@@ -0,0 +1,160 @@
|
||||
"""One auditable record of what the browser actually did, whatever tier did it.
|
||||
|
||||
The user-facing promise is that a browser task is never "just trust me": the chat shows a Browser
|
||||
Agent bubble you can expand to see the pages visited, what was clicked and typed, and the receipt
|
||||
that proves a write landed. That promise held only on the sub-agent path, because the panel that
|
||||
renders it reads from CHILD SESSIONS. The fast path creates no child session and closed its bubble
|
||||
with a tool_result of literally "done", so on the tier that now handles most tasks the bubble
|
||||
expanded to nothing at all.
|
||||
|
||||
So the trace stops being a side effect of how the work was routed. Whichever tier ran builds the
|
||||
same record here, and the bubble shows the same thing every time.
|
||||
|
||||
Pure formatting: no I/O, no side effects, nothing that can fail a run.
|
||||
"""
|
||||
|
||||
import json
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from typeguard import typechecked
|
||||
|
||||
# Enough to see what happened without turning the panel into a log file. A run that exceeds this
|
||||
# says so rather than silently showing a prefix, because a trace you cannot trust to be complete is
|
||||
# worse than no trace.
|
||||
MAX_STEPS = 40
|
||||
MAX_ARG_CHARS = 90
|
||||
|
||||
# Tools whose arguments are the interesting part (where it went, what it typed) versus ones whose
|
||||
# name already says everything (a screenshot is a screenshot).
|
||||
P_ARG_KEYS = ("url", "text", "expression", "instruction", "target_text", "index", "name", "key")
|
||||
|
||||
|
||||
class BrowserTrace(BaseModel):
|
||||
"""What to show under the bubble. Shaped so the renderer never parses prose."""
|
||||
|
||||
model_config = ConfigDict(validate_assignment=True)
|
||||
|
||||
tier: str = "" # which path did the work, in plain words
|
||||
pages: List[str] = [] # URLs actually visited, in order, deduped
|
||||
steps: List[str] = [] # one line per action, already human-readable
|
||||
steps_omitted: int = 0
|
||||
receipt: str = "" # the proof a write landed, when there was one
|
||||
note: str = "" # anything the user should know about coverage
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_arg_summary(inp: Any) -> str:
|
||||
"""The part of a tool's input worth showing, short enough to scan."""
|
||||
if not isinstance(inp, dict) or not inp:
|
||||
return ""
|
||||
for k in P_ARG_KEYS:
|
||||
v = inp.get(k)
|
||||
if v not in (None, "", []):
|
||||
s = str(v).replace("\n", " ").strip()
|
||||
return s[:MAX_ARG_CHARS] + ("..." if len(s) > MAX_ARG_CHARS else "")
|
||||
s = json.dumps(inp)[:MAX_ARG_CHARS]
|
||||
return s
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_step_line(entry: Dict[str, Any]) -> str:
|
||||
tool = str(entry.get("tool") or "?")
|
||||
arg = p_arg_summary(entry.get("input"))
|
||||
ms = entry.get("elapsed_ms")
|
||||
ok = entry.get("ok")
|
||||
tail = f" [{int(ms)}ms]" if isinstance(ms, (int, float)) and ms else ""
|
||||
mark = "" if ok in (None, True) else " (failed)"
|
||||
return f"{tool}({arg}){tail}{mark}" if arg else f"{tool}{tail}{mark}"
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_pages_from(action_log: List[Dict[str, Any]]) -> List[str]:
|
||||
"""Every URL the run actually landed on, in order, without repeats. This is the spine of the
|
||||
trace: it answers "where did it go" before "what did it do there"."""
|
||||
out: List[str] = []
|
||||
for e in action_log:
|
||||
inp = e.get("input")
|
||||
url = str(inp.get("url") or "") if isinstance(inp, dict) else ""
|
||||
if url.startswith(("http://", "https://")) and (not out or out[-1] != url):
|
||||
out.append(url)
|
||||
return out
|
||||
|
||||
|
||||
@typechecked
|
||||
def build_trace(tier: str, action_logs: List[List[Dict[str, Any]]],
|
||||
receipt: str = "", note: str = "", entry_url: str = "") -> BrowserTrace:
|
||||
"""Fold every dispatch a run made into one record. Takes a LIST of logs because a fast-path run
|
||||
can dispatch more than once (a recovery, a send probe) and the user should see all of it, not
|
||||
just whichever attempt happened to be last.
|
||||
|
||||
`entry_url` matters more than it looks: a cold run creates the card ALREADY pointed at its
|
||||
target, so no BrowserNavigate is ever issued and harvesting URLs from the log alone leaves the
|
||||
trace unable to answer "where did it go" at all."""
|
||||
merged: List[Dict[str, Any]] = []
|
||||
for log in action_logs:
|
||||
merged.extend(e for e in (log or []) if isinstance(e, dict))
|
||||
pages = p_pages_from(merged)
|
||||
if entry_url.startswith(("http://", "https://")) and entry_url not in pages[:1]:
|
||||
pages = [entry_url] + pages
|
||||
shown = merged[-MAX_STEPS:]
|
||||
return BrowserTrace(
|
||||
tier=tier,
|
||||
pages=pages,
|
||||
steps=[p_step_line(e) for e in shown],
|
||||
steps_omitted=max(0, len(merged) - len(shown)),
|
||||
receipt=receipt,
|
||||
note=note,
|
||||
)
|
||||
|
||||
|
||||
@typechecked
|
||||
def trace_payload(trace: BrowserTrace) -> Dict[str, object]:
|
||||
"""The tool_result content the bubble renders. Kept as data rather than a rendered string so the
|
||||
panel can lay it out, and so a future surface (an export, a report) does not have to re-parse
|
||||
English."""
|
||||
return {"browser_trace": trace.model_dump(mode="json")}
|
||||
|
||||
|
||||
@typechecked
|
||||
def trace_text(trace: BrowserTrace) -> str:
|
||||
"""A plain-text fallback for anywhere that can only show a string."""
|
||||
lines: List[str] = []
|
||||
if trace.tier:
|
||||
lines.append(f"Handled by: {trace.tier}")
|
||||
if trace.pages:
|
||||
lines.append("Pages: " + " -> ".join(trace.pages[:6]))
|
||||
if trace.steps_omitted:
|
||||
lines.append(f"... {trace.steps_omitted} earlier steps omitted ...")
|
||||
lines.extend(f"{i}. {s}" for i, s in enumerate(trace.steps, trace.steps_omitted + 1))
|
||||
if trace.receipt:
|
||||
lines.append(f"Verified: {trace.receipt}")
|
||||
if trace.note:
|
||||
lines.append(trace.note)
|
||||
return "\n".join(lines) or "No browser actions were recorded."
|
||||
|
||||
|
||||
@typechecked
|
||||
def tier_label(fp_path: str, used_browser: bool) -> str:
|
||||
"""Plain words for the routing string the logs use, because 'read->browser' means nothing to
|
||||
the person reading their own chat."""
|
||||
if not used_browser:
|
||||
return "read the page directly, no browser needed"
|
||||
if fp_path.startswith("read"):
|
||||
return "opened the page in a browser and read it"
|
||||
return "drove the browser"
|
||||
|
||||
|
||||
@typechecked
|
||||
def receipt_from(result: Optional[Dict[str, Any]]) -> str:
|
||||
"""The two-sided receipt, when the run produced one. This is the line that separates 'it says it
|
||||
posted' from 'it posted', so it gets its own field rather than being buried in the steps."""
|
||||
if not isinstance(result, dict):
|
||||
return ""
|
||||
for key in ("receipt", "sent_receipt", "delivery"):
|
||||
v = result.get(key)
|
||||
if isinstance(v, str) and v.strip():
|
||||
return v.strip()[:300]
|
||||
if v is True:
|
||||
return "delivery confirmed on the page"
|
||||
return ""
|
||||
@@ -0,0 +1,107 @@
|
||||
"""Generic, site-agnostic verification: did an action produce the SPECIFIC effect
|
||||
it was meant to? This is the load-bearing piece that lets a verified-action executor
|
||||
work on any website without per-site code, the model (or a scripted flow) names a
|
||||
generic expectation, and this checks it against a cheap before/after page snapshot.
|
||||
|
||||
A snapshot is just the interactives-list text plus the URL, the same things every
|
||||
site exposes, so nothing here knows about LinkedIn or any particular page. It
|
||||
generalizes the send-script's proven two-sided receipt ("the composer cleared") from
|
||||
one hand-tuned flow into one predicate ("cleared:<text>") reusable everywhere.
|
||||
|
||||
Expectations (kind, or "kind:arg"):
|
||||
url_changed the page navigated
|
||||
changed the page changed at all (weakest; a fallback)
|
||||
appeared:X X is present now but wasn't before (a menu/dialog/result opened)
|
||||
gone:X X was present before but isn't now (an item/row deleted)
|
||||
filled:X some textbox value now carries X (a fill committed)
|
||||
cleared:X no textbox value carries X (a composer sent + emptied)
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
# Match payload_in_textbox: long values truncate in the list, so compare on a prefix.
|
||||
P_VALUE_PREFIX_LEN = 24
|
||||
P_TEXTBOX_LINE = "<textbox"
|
||||
|
||||
# One interactives row: [<index>]<*>?<<role> "<name>"...>, the format every list uses.
|
||||
P_ROW_RE = re.compile(r'\[(\d+)\]\*?<\s*([a-z]+)\s+"([^"]*)"', re.I)
|
||||
P_NAME_PREFIX_LEN = 40 # long card-blob names mutate their suffix between visits
|
||||
|
||||
|
||||
def parse_rows(state_text: str) -> List[Tuple[int, str, str]]:
|
||||
"""(index, role, name) for each interactive row. Site-agnostic: it reads the
|
||||
universal list shape, not any particular page's elements."""
|
||||
return [(int(m.group(1)), m.group(2).lower(), m.group(3))
|
||||
for m in P_ROW_RE.finditer(state_text or "")]
|
||||
|
||||
|
||||
def resolve_target(state_text: str, name: str, role: str = "") -> Optional[Tuple[int, str, str]]:
|
||||
"""Resolve a semantic target against the LIVE list, late, the moment before acting,
|
||||
so a stale index can't bite. Strictest UNAMBIGUOUS tier wins: exact (role,name) ->
|
||||
exact name -> name-prefix. Two matches at a tier = ambiguous = None (hand back
|
||||
rather than click the wrong thing). Mirrors the renderer's click-by-name tiers."""
|
||||
want = (name or "").strip().lower()
|
||||
if not want:
|
||||
return None
|
||||
wrole = (role or "").strip().lower()
|
||||
rows = parse_rows(state_text)
|
||||
|
||||
def uniq(cands: List[Tuple[int, str, str]]) -> Optional[Tuple[int, str, str]]:
|
||||
return cands[0] if len(cands) == 1 else None
|
||||
|
||||
hit = uniq([r for r in rows if r[2].strip().lower() == want and (not wrole or r[1] == wrole)])
|
||||
if hit:
|
||||
return hit
|
||||
hit = uniq([r for r in rows if r[2].strip().lower() == want])
|
||||
if hit:
|
||||
return hit
|
||||
pre = want[:P_NAME_PREFIX_LEN]
|
||||
return uniq([r for r in rows if r[2].strip().lower().startswith(pre) and (not wrole or r[1] == wrole)])
|
||||
|
||||
|
||||
def parse_expectation(expect: str) -> Tuple[str, str]:
|
||||
"""(kind, arg) from 'kind' or 'kind:arg'. Unknown kinds are returned as-is and
|
||||
treated as unmet by expectation_met, so a typo fails safe (verification withheld)."""
|
||||
raw = (expect or "").strip()
|
||||
if ":" in raw:
|
||||
kind, arg = raw.split(":", 1)
|
||||
return kind.strip().lower(), arg.strip()
|
||||
return raw.lower(), ""
|
||||
|
||||
|
||||
def value_present(state_text: str, sub: str) -> bool:
|
||||
"""True if any listed textbox VALUE carries sub (prefix match, like a committed
|
||||
fill). Same logic as payload_in_textbox so the send-script stays behavior-identical."""
|
||||
probe = (sub or "")[:P_VALUE_PREFIX_LEN]
|
||||
if not probe:
|
||||
return False
|
||||
return any(P_TEXTBOX_LINE in line and probe in line
|
||||
for line in (state_text or "").splitlines())
|
||||
|
||||
|
||||
def p_contains(state_text: str, sub: str) -> bool:
|
||||
s = (sub or "").strip().lower()
|
||||
return bool(s) and s in (state_text or "").lower()
|
||||
|
||||
|
||||
def expectation_met(
|
||||
expect: str, before: str, after: str,
|
||||
before_url: str = "", after_url: str = "",
|
||||
) -> bool:
|
||||
"""Did `after` satisfy `expect` given `before`? Pure; unknown expectation = False
|
||||
(fail safe: verification withheld rather than a false pass)."""
|
||||
kind, arg = parse_expectation(expect)
|
||||
if kind == "url_changed":
|
||||
return bool(after_url) and after_url != before_url
|
||||
if kind == "changed":
|
||||
return before != after or (bool(after_url) and after_url != before_url)
|
||||
if kind == "appeared":
|
||||
return p_contains(after, arg) and not p_contains(before, arg)
|
||||
if kind == "gone":
|
||||
return p_contains(before, arg) and not p_contains(after, arg)
|
||||
if kind == "filled":
|
||||
return value_present(after, arg)
|
||||
if kind == "cleared":
|
||||
return not value_present(after, arg)
|
||||
return False
|
||||
@@ -0,0 +1,110 @@
|
||||
"""One verified action, the executor's unit of work: resolve the target LATE against
|
||||
the live page, act, verify the SPECIFIC expected effect, and re-aim on a miss, all in
|
||||
code, no LLM turn. This generalizes the send-script's proven fill->verify->send->verify
|
||||
from one LinkedIn flow to any site: the target is a semantic name, the effect is a
|
||||
generic expectation, and neither knows about any particular page.
|
||||
|
||||
The one safety invariant, same bar as the send-script: an IRREVERSIBLE step (send /
|
||||
submit / pay) is NEVER re-fired. If it acted but the effect can't be verified, it
|
||||
returns an honest "acted, unverified, do NOT repeat" note instead of retrying, so a
|
||||
receipt we couldn't read can never become a double-send.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
from typing import Awaitable, Callable, Optional, Tuple
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
|
||||
from backend.apps.agents.browser import browser_verified_action as va
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]]
|
||||
|
||||
|
||||
class VerifiedStep(BaseModel):
|
||||
model_config = ConfigDict(validate_assignment=True)
|
||||
kind: str # "click" | "fill"
|
||||
target: str # semantic element name to resolve against the live list
|
||||
role: str = "" # optional role hint ("button", "link", "textbox") to disambiguate
|
||||
text: str = "" # for a fill
|
||||
expect: str = "" # generic expectation; defaults to filled:<text> / changed
|
||||
irreversible: bool = False # send/submit/pay: acted-but-unverified NEVER re-fires
|
||||
chosen: bool = False # the planner PICKED this among similar rows: flag it for review before anything irreversible
|
||||
|
||||
|
||||
async def p_fresh(execute_tool: ToolRunner, browser_id: str, tab_id: str) -> Tuple[str, str]:
|
||||
try:
|
||||
r = await asyncio.wait_for(
|
||||
execute_tool("BrowserListInteractives", {}, browser_id, tab_id), timeout=6.0)
|
||||
except Exception:
|
||||
return "", ""
|
||||
if not isinstance(r, dict) or "error" in r:
|
||||
return "", ""
|
||||
return str(r.get("text") or ""), str(r.get("url") or "")
|
||||
|
||||
|
||||
async def p_act(step: VerifiedStep, index: Optional[int],
|
||||
browser_id: str, tab_id: str, execute_tool: ToolRunner) -> dict:
|
||||
if step.kind == "fill":
|
||||
return await execute_tool(
|
||||
"BrowserClickIndex", {"index": index, "text": step.text}, browser_id, tab_id)
|
||||
if index is not None:
|
||||
return await execute_tool("BrowserClickIndex", {"index": index}, browser_id, tab_id)
|
||||
# a click whose index didn't resolve falls to by-name (full-DOM search, past the list cap)
|
||||
return await execute_tool(
|
||||
"BrowserClickByName", {"name": step.target, "role": step.role}, browser_id, tab_id)
|
||||
|
||||
|
||||
def p_default_expect(step: VerifiedStep) -> str:
|
||||
if step.expect:
|
||||
return step.expect
|
||||
return f"filled:{step.text}" if step.kind == "fill" else "changed"
|
||||
|
||||
|
||||
async def run_verified_step(
|
||||
step: VerifiedStep, browser_id: str, tab_id: str, execute_tool: ToolRunner,
|
||||
settle_s: float = 0.8, max_reaim: int = 1,
|
||||
) -> dict:
|
||||
"""{ok, verified, acted, note}. ok == the expected effect was observed. A reversible
|
||||
step that doesn't verify is re-aimed (re-resolve + re-act) up to max_reaim times; an
|
||||
irreversible one is never re-fired once it has acted."""
|
||||
expect = p_default_expect(step)
|
||||
note = ""
|
||||
for attempt in range(max_reaim + 1):
|
||||
before, before_url = await p_fresh(execute_tool, browser_id, tab_id)
|
||||
tgt = va.resolve_target(before, step.target, step.role)
|
||||
index = tgt[0] if tgt else None
|
||||
if step.kind == "fill" and index is None:
|
||||
return {"ok": False, "verified": False, "acted": False,
|
||||
"note": f"could not resolve a field named {step.target!r} to fill"}
|
||||
r = await p_act(step, index, browser_id, tab_id, execute_tool)
|
||||
acted = isinstance(r, dict) and "error" not in r
|
||||
if not acted:
|
||||
note = f"action errored: {r.get('error') if isinstance(r, dict) else r}"
|
||||
if step.irreversible:
|
||||
# an errored irreversible action provably did NOT happen; safe to stop, never retry blindly
|
||||
return {"ok": False, "verified": False, "acted": False, "note": note}
|
||||
continue # reversible: re-aim
|
||||
# Smart-wait: check the expectation early and again at the full settle window; the common fast case exits ~0.5-0.8s sooner than a flat sleep, the slow case keeps its whole window. A settle too small to split keeps the single flat check.
|
||||
p_waits = [0.4, settle_s - 0.4] if settle_s > 0.4 else [settle_s]
|
||||
p_met = False
|
||||
for p_wait in p_waits:
|
||||
if p_wait:
|
||||
await asyncio.sleep(p_wait)
|
||||
after, after_url = await p_fresh(execute_tool, browser_id, tab_id)
|
||||
if va.expectation_met(expect, before, after, before_url, after_url):
|
||||
p_met = True
|
||||
break
|
||||
if p_met:
|
||||
logger.info(f"[verified-step] {step.kind} {step.target!r} -> {expect} OK (attempt {attempt + 1})")
|
||||
return {"ok": True, "verified": True, "acted": True, "note": ""}
|
||||
if step.irreversible:
|
||||
# acted, effect unverifiable: the send-script's honesty rule, never a blind repeat
|
||||
return {"ok": False, "verified": False, "acted": True,
|
||||
"note": (f"an irreversible {step.target!r} action already RAN but its effect is "
|
||||
"unverified; verify on the page, do NOT repeat it unless verifiably absent")}
|
||||
note = f"expected {expect!r} not observed after {step.kind} {step.target!r}"
|
||||
logger.info(f"[verified-step] {step.kind} {step.target!r} unverified: {note}")
|
||||
return {"ok": False, "verified": False, "acted": True, "note": note}
|
||||
@@ -0,0 +1,232 @@
|
||||
"""Learn-on-first-write, replay-on-repeat: the repeated-write half of the API-first tier.
|
||||
|
||||
The first write on a site is unavoidably a DOM drive (a route can only be captured after the
|
||||
site's own UI fires it, proven live in the V.8 X soak). But the moment a DOM write SUCCEEDS with
|
||||
a verified receipt, the mutating route the page fired is a complete recipe: method + URL (with
|
||||
its live queryId) + body, with the user's payload sitting in one JSON leaf. This module persists
|
||||
that recipe with the payload slot replaced by a sentinel, and replays it with a NEW payload on
|
||||
the next write to the same site, skipping the DOM entirely.
|
||||
|
||||
SAFETY (documented in SECURITY.md):
|
||||
- A recipe is learned ONLY from a receipt-verified successful write the user's own task performed,
|
||||
so its provenance satisfies the captured-route wall (the site's UI genuinely fired it); replay
|
||||
seeds route_write's captured set from the recipe itself.
|
||||
- Secret-shaped string leaves in the stored body are redacted at learn time (payload slot
|
||||
excepted); cookies/headers are never stored (route_write borrows them live per call).
|
||||
- Same-origin + OSW_ROUTE_WRITE flag + typed fail-open outcomes all still apply at replay.
|
||||
- Staleness self-heals: a recipe that misses MAX_MISSES times is dropped, and the next
|
||||
successful DOM write learns a fresh one (queryId rotation just re-learns).
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
from typing import Any, Dict, List, Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from typeguard import typechecked
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
SENTINEL = "__OSW_PAYLOAD__"
|
||||
P_MIN_PAYLOAD_CHARS = 4
|
||||
P_MAX_BODY_CHARS = 32768
|
||||
MAX_MISSES = 3
|
||||
P_MAX_RECIPES_ON_DISK = 200
|
||||
|
||||
# Same secret heuristics as the electron capture (cdp-routes.js), ported so a token-shaped
|
||||
# body value can never be persisted; over-redacting is the safe direction.
|
||||
P_TOKEN_PREFIX = re.compile(r"^(sk-|ghp_|gho_|pk_|xox[bap]-|AIza|eyJ|Bearer )")
|
||||
|
||||
|
||||
@typechecked
|
||||
def looks_secret_value(v: str) -> bool:
|
||||
if not v:
|
||||
return False
|
||||
if P_TOKEN_PREFIX.match(v):
|
||||
return True
|
||||
return len(v) >= 20 and bool(re.search(r"[A-Za-z]", v)) and bool(re.search(r"[0-9]", v)) and not re.search(r"\s", v)
|
||||
|
||||
|
||||
class WriteRecipe(BaseModel):
|
||||
"""One site's proven write call, payload slot replaced by the sentinel."""
|
||||
|
||||
model_config = ConfigDict(validate_assignment=True)
|
||||
|
||||
host: str
|
||||
method: str
|
||||
url_template: str
|
||||
url: str
|
||||
body_template: str
|
||||
payload_path: str
|
||||
learned_at: float
|
||||
wins: int = 0
|
||||
misses: int = 0
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_dir() -> str:
|
||||
from backend.config.paths import DATA_ROOT
|
||||
d = os.path.join(DATA_ROOT, "browser_write_recipes")
|
||||
os.makedirs(d, mode=0o700, exist_ok=True)
|
||||
return d
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_path(host: str) -> str:
|
||||
safe = re.sub(r"[^a-z0-9.-]", "_", host.lower())
|
||||
return os.path.join(p_dir(), f"{safe}.json")
|
||||
|
||||
|
||||
@typechecked
|
||||
def recipe_for(host: str) -> Optional[WriteRecipe]:
|
||||
"""The persisted recipe for this host, or None. Corrupt files read as None (fail-open)."""
|
||||
try:
|
||||
with open(p_path(host)) as f:
|
||||
return WriteRecipe(**json.load(f))
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
@typechecked
|
||||
def save_recipe(recipe: WriteRecipe) -> None:
|
||||
"""Atomic write, browser_skills pattern; cap the directory so it can't grow unbounded."""
|
||||
try:
|
||||
d = p_dir()
|
||||
entries = sorted(os.listdir(d), key=lambda f: os.path.getmtime(os.path.join(d, f)))
|
||||
while len(entries) >= P_MAX_RECIPES_ON_DISK:
|
||||
os.remove(os.path.join(d, entries.pop(0)))
|
||||
tmp = p_path(recipe.host) + ".tmp"
|
||||
with open(tmp, "w") as f:
|
||||
json.dump(recipe.model_dump(mode="json"), f)
|
||||
os.replace(tmp, p_path(recipe.host))
|
||||
except Exception as e:
|
||||
logger.info(f"[write-recipe] save failed for {recipe.host}: {e}")
|
||||
|
||||
|
||||
@typechecked
|
||||
def drop_recipe(host: str) -> None:
|
||||
try:
|
||||
os.remove(p_path(host))
|
||||
except OSError:
|
||||
pass
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_find_payload_leaf(obj: Any, payload: str, path: str = "$") -> Optional[str]:
|
||||
"""JSON path of the leaf whose string value EQUALS the payload (exact, not substring:
|
||||
a substring hit means the site wrapped it and blind substitution would corrupt)."""
|
||||
if isinstance(obj, str):
|
||||
return path if obj == payload else None
|
||||
if isinstance(obj, dict):
|
||||
for k, v in obj.items():
|
||||
hit = p_find_payload_leaf(v, payload, f"{path}.{k}")
|
||||
if hit:
|
||||
return hit
|
||||
return None
|
||||
if isinstance(obj, list):
|
||||
for i, v in enumerate(obj):
|
||||
hit = p_find_payload_leaf(v, payload, f"{path}[{i}]")
|
||||
if hit:
|
||||
return hit
|
||||
return None
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_transform_leaves(obj: Any, payload: str) -> Any:
|
||||
"""Copy with the payload leaf swapped for the sentinel and secret-shaped strings redacted."""
|
||||
if isinstance(obj, str):
|
||||
if obj == payload:
|
||||
return SENTINEL
|
||||
return "<redacted>" if looks_secret_value(obj) else obj
|
||||
if isinstance(obj, dict):
|
||||
return {k: p_transform_leaves(v, payload) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [p_transform_leaves(v, payload) for v in obj]
|
||||
return obj
|
||||
|
||||
|
||||
@typechecked
|
||||
def learn_recipe(host: str, payload: str, routes: List[Dict[str, Any]]) -> Optional[WriteRecipe]:
|
||||
"""Distill a recipe from the captured mutating routes of a JUST-verified write. Returns the
|
||||
saved recipe, or None when no route's body carries the payload as an exact string leaf
|
||||
(then there is nothing provably replayable, so nothing is stored)."""
|
||||
if len(payload or "") < P_MIN_PAYLOAD_CHARS:
|
||||
return None
|
||||
for r in routes:
|
||||
body = str(r.get("lastBody") or "")
|
||||
method = str(r.get("method") or "").upper()
|
||||
if not body or len(body) > P_MAX_BODY_CHARS or method in ("GET", "HEAD"):
|
||||
continue
|
||||
try:
|
||||
parsed = json.loads(body)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
continue
|
||||
slot = p_find_payload_leaf(parsed, payload)
|
||||
if not slot:
|
||||
continue
|
||||
recipe = WriteRecipe(
|
||||
host=host, method=method,
|
||||
url_template=str(r.get("template") or ""),
|
||||
url=str(r.get("example") or r.get("template") or ""),
|
||||
body_template=json.dumps(p_transform_leaves(parsed, payload)),
|
||||
payload_path=slot, learned_at=time.time(),
|
||||
)
|
||||
save_recipe(recipe)
|
||||
logger.info(f"[write-recipe] learned {host} {method} {recipe.url_template[:80]} slot={slot}")
|
||||
return recipe
|
||||
return None
|
||||
|
||||
|
||||
@typechecked
|
||||
def build_body(recipe: WriteRecipe, payload: str) -> Optional[Dict[str, Any]]:
|
||||
"""The recipe body with the NEW payload in the slot; None when the template holds no
|
||||
sentinel (corrupt or hand-edited = do not replay) or redacted leaves the site requires."""
|
||||
if SENTINEL not in recipe.body_template:
|
||||
return None
|
||||
try:
|
||||
parsed = json.loads(recipe.body_template)
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
return None
|
||||
|
||||
def p_sub(obj: Any) -> Any:
|
||||
if isinstance(obj, str):
|
||||
return payload if obj == SENTINEL else obj
|
||||
if isinstance(obj, dict):
|
||||
return {k: p_sub(v) for k, v in obj.items()}
|
||||
if isinstance(obj, list):
|
||||
return [p_sub(v) for v in obj]
|
||||
return obj
|
||||
|
||||
out = p_sub(parsed)
|
||||
return out if isinstance(out, dict) else None
|
||||
|
||||
|
||||
@typechecked
|
||||
async def replay_recipe(recipe: WriteRecipe, payload: str, origin: str) -> Dict[str, Any]:
|
||||
"""Replay the recipe with a new payload via route_write (same-origin + flag + live-borrowed
|
||||
cookies all enforced there). The recipe IS the captured provenance: it was learned from a
|
||||
route the site's UI fired during a receipt-verified write, so it seeds the captured set.
|
||||
Returns {ok, receipt|error}; a miss bumps the staleness counter and MAX_MISSES drops it."""
|
||||
from backend.apps.agents.browser import route_write
|
||||
body = build_body(recipe, payload)
|
||||
if body is None:
|
||||
drop_recipe(recipe.host)
|
||||
return {"ok": False, "error": "recipe template unusable; dropped"}
|
||||
captured = [route_write.CapturedRoute(method=recipe.method, template=recipe.url_template)]
|
||||
import asyncio
|
||||
out = await asyncio.to_thread(
|
||||
route_write.replay_write, recipe.method, recipe.url, body, origin, captured)
|
||||
if out.ok:
|
||||
recipe.wins += 1
|
||||
save_recipe(recipe)
|
||||
return {"ok": True, "receipt": out.receipt, "latency_ms": out.latency_ms}
|
||||
recipe.misses += 1
|
||||
if recipe.misses >= MAX_MISSES:
|
||||
drop_recipe(recipe.host)
|
||||
logger.info(f"[write-recipe] {recipe.host} dropped after {recipe.misses} misses (stale; next DOM win re-learns)")
|
||||
else:
|
||||
save_recipe(recipe)
|
||||
return {"ok": False, "error": out.error}
|
||||
@@ -0,0 +1,192 @@
|
||||
"""Find a site's compose URL by reading the links the site already publishes.
|
||||
|
||||
The compose-URL table in `compose_entry` is worth 92.9% composer reachability on the five hosts it
|
||||
knows and exactly 0% everywhere else, measured live over 8 logged-in sites. Two attempts to close
|
||||
that gap by making the aux navigator click better were falsified in the same session (0/8 both
|
||||
times, at roughly double the wall time), so the lever is not better clicking, it is getting the URL.
|
||||
|
||||
The generalizable source is the page itself. A site that has a composer links to it: "Start a post",
|
||||
"Ask Question", "Create", "New story". Reading those anchors is precise where guessing paths is not,
|
||||
because it is the site's own navigation rather than a list of shapes we hope it matches, and it
|
||||
needs no per-site knowledge. It also degrades honestly: no matching anchor means no candidate, and
|
||||
the caller runs exactly as it does today.
|
||||
|
||||
The ranking here is deliberately pure, so the part that decides where to send someone's browser is
|
||||
testable without a browser. The page-reading half is one expression; the judgement half is below it.
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, List, Optional, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.agents.browser import compose_entry
|
||||
|
||||
# How many links to bring back. Enough to cover a nav bar plus a sidebar; a page with more than
|
||||
# this has a hundred feed links and the composer is not going to be number 401.
|
||||
MAX_LINKS = 400
|
||||
# How many candidates the caller may actually navigate to. Each miss costs a real page load, and
|
||||
# past two the aux loop is the cheaper remaining option.
|
||||
MAX_CANDIDATES = 2
|
||||
# Longest a control's label can be before it is prose rather than a button. "Start a post" is 3.
|
||||
MAX_LABEL_WORDS = 4
|
||||
|
||||
# Path segments that mean "start something new" across platforms rather than on one host: /submit
|
||||
# is reddit-shaped, /questions/ask is every StackExchange, /new-story is medium-shaped, /new/text is
|
||||
# tumblr-shaped, /compose is mail and chat.
|
||||
#
|
||||
# A WHOLE segment, never a substring. Substring matching was tried and produced garbage on the
|
||||
# first live read: "post" matched inside `/explore/top-posts` and "new" inside a permalink
|
||||
# `/actuallysara/823.../new-photo-of-connor...`, so the tier proposed navigating to a random blog
|
||||
# post. `post` itself is deliberately absent even as a segment, because permalinks are `/post/<id>`
|
||||
# on half the web and a false candidate costs a real page load.
|
||||
P_PATH_HINTS: Tuple[str, ...] = (
|
||||
"submit", "compose", "new", "create", "ask", "publish", "write", "share",
|
||||
"new-story", "new-post", "new-thread", "new-story",
|
||||
)
|
||||
|
||||
# What the site calls the control. Labels are the stronger signal of the two: a path can be
|
||||
# incidental ("/new-york-times"), but a link a human reads as "Start a post" is one.
|
||||
P_LABEL_RE = re.compile(
|
||||
r"\b(start a post|create a post|new post|create post|write a post|"
|
||||
r"ask (a )?question|new story|write a story|start writing|new thread|"
|
||||
r"compose|create new|new message|submit a? ?(post|link|text)?|publish)\b", re.I)
|
||||
|
||||
# Never a composer, and several are actively destructive to wander into mid-run. Checked against
|
||||
# the whole URL plus the label, because "sign out" hides behind /logout as often as it is written.
|
||||
P_NEVER_RE = re.compile(
|
||||
r"\b(log ?out|sign ?out|log ?in|sign ?in|sign ?up|register|settings|preferences|account|"
|
||||
r"billing|subscribe|upgrade|premium|checkout|cart|delete|privacy|terms|cookie|legal|"
|
||||
r"about|careers|jobs|advertise|press|help|support|download|api|developer)\b", re.I)
|
||||
|
||||
|
||||
@typechecked
|
||||
def enabled() -> bool:
|
||||
"""OFF by default, unlike the compose-URL table it backs up.
|
||||
|
||||
The mechanism is proven: pointed at StackOverflow it returned `/questions/ask` as its first
|
||||
candidate, which is exactly right. What is NOT proven is that it helps anyone end to end, and
|
||||
on the eight-site sweep it did not, for reasons outside itself: three of five create-sites
|
||||
publish no compose link at all (their composer is a button opening a modal with no route),
|
||||
StackOverflow was behind a sign-in wall, and GitHub never armed at an upstream gate. Shipping a
|
||||
tier default-on because its parts work, while its measured contribution is zero, is how latency
|
||||
accretes. It turns on when a site set exists where it can win and the number says it did."""
|
||||
return os.environ.get("OSW_COMPOSE_DISCOVERY", "0") != "0"
|
||||
|
||||
|
||||
@typechecked
|
||||
def discovery_expression() -> str:
|
||||
"""One page read returning every link the site publishes, with the text a human would read.
|
||||
|
||||
Deliberately read-only: it collects hrefs and labels and touches nothing, so it is safe to run
|
||||
on any page in any state, including one the user is looking at."""
|
||||
return (
|
||||
"(() => {"
|
||||
" const out = []; const seen = new Set();"
|
||||
" for (const a of document.querySelectorAll('a[href]')) {"
|
||||
" const href = a.href || '';"
|
||||
" if (!href || seen.has(href)) continue;"
|
||||
" if (!/^https?:/i.test(href)) continue;"
|
||||
" seen.add(href);"
|
||||
" const label = (a.getAttribute('aria-label') || a.innerText || a.title || '')"
|
||||
" .replace(/\\s+/g, ' ').trim().slice(0, 80);"
|
||||
" out.push({href: href, label: label});"
|
||||
f" if (out.length >= {MAX_LINKS}) break;"
|
||||
" }"
|
||||
" return {url: location.href, links: out};"
|
||||
"})()"
|
||||
)
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_link_score(href: str, label: str, host: str, current_url: str) -> int:
|
||||
"""How much this link looks like the way in to this site's composer. 0 means never navigate.
|
||||
|
||||
Scored rather than matched so the front-door compose link outranks a deeper one that merely
|
||||
shares a word, which is what stops "/submit-a-tip" from beating "/submit"."""
|
||||
parsed = urlparse(href)
|
||||
found = compose_entry.registrable_host(parsed.netloc)
|
||||
want = compose_entry.registrable_host(host)
|
||||
# Off-host links go to a different company; a "share to X" button must not hijack a tumblr post.
|
||||
if not found or (found != want and not found.endswith("." + want)):
|
||||
return 0
|
||||
path = parsed.path.strip("/").lower()
|
||||
if not path:
|
||||
return 0
|
||||
if P_NEVER_RE.search(href) or P_NEVER_RE.search(label or ""):
|
||||
return 0
|
||||
# Already here. Re-navigating would remount the page and throw away whatever is on it.
|
||||
if href.rstrip("/") == (current_url or "").rstrip("/"):
|
||||
return 0
|
||||
score = 0
|
||||
# A compose control is labelled like a button, not like an article. Length is what separates
|
||||
# them, and it separates them on every site at once: StackOverflow offered a QUESTION titled
|
||||
# "Compose preview different from emulator" as a compose link, because a Q&A site is full of
|
||||
# titles containing the word. Buttons say "Ask Question" or "Start a post".
|
||||
if len((label or "").split()) <= MAX_LABEL_WORDS and P_LABEL_RE.search(label or ""):
|
||||
score += 10
|
||||
segments = [s for s in path.split("/") if s]
|
||||
if any(seg in P_PATH_HINTS for seg in segments):
|
||||
score += 6
|
||||
if not score:
|
||||
return 0
|
||||
# A shallow path is the site's own front door to composing; a deep one is usually a specific
|
||||
# item that happens to share a word. Never lets a match drop to zero.
|
||||
score += max(0, 3 - len(segments))
|
||||
return score
|
||||
|
||||
|
||||
@typechecked
|
||||
def rank_candidates(payload: Optional[Dict[str, object]], host: str) -> List[str]:
|
||||
"""The URLs worth trying, best first, capped. Empty when the page publishes nothing composer-ish.
|
||||
|
||||
Takes the raw page read so the whole decision is one pure function over data, which is the only
|
||||
reason this is testable without driving a browser."""
|
||||
if not isinstance(payload, dict):
|
||||
return []
|
||||
links = payload.get("links")
|
||||
if not isinstance(links, list):
|
||||
return []
|
||||
current = str(payload.get("url") or "")
|
||||
scored: List[Tuple[int, str]] = []
|
||||
seen: set = set()
|
||||
for row in links:
|
||||
if not isinstance(row, dict):
|
||||
continue
|
||||
href = str(row.get("href") or "")
|
||||
label = str(row.get("label") or "")
|
||||
if not href or href in seen:
|
||||
continue
|
||||
seen.add(href)
|
||||
score = p_link_score(href, label, host, current)
|
||||
if score > 0:
|
||||
scored.append((score, href))
|
||||
# Sort by score, then by URL so a tie is deterministic across runs rather than DOM-order luck.
|
||||
scored.sort(key=lambda pair: (-pair[0], pair[1]))
|
||||
return [href for _, href in scored[:MAX_CANDIDATES]]
|
||||
|
||||
|
||||
@typechecked
|
||||
def parse_page_read(raw: object) -> Optional[Dict[str, object]]:
|
||||
"""The evaluate result, whatever shape the bridge handed back.
|
||||
|
||||
BrowserEvaluate returns the value directly on some paths and JSON in a text field on others, and
|
||||
a discovery tier that silently sees nothing looks identical to a site with no compose link."""
|
||||
if isinstance(raw, dict) and "links" in raw:
|
||||
return raw
|
||||
if isinstance(raw, dict):
|
||||
for key in ("result", "value", "text"):
|
||||
inner = raw.get(key)
|
||||
if isinstance(inner, dict) and "links" in inner:
|
||||
return inner
|
||||
if isinstance(inner, str) and inner.strip().startswith("{"):
|
||||
try:
|
||||
parsed = json.loads(inner)
|
||||
except ValueError:
|
||||
continue
|
||||
if isinstance(parsed, dict) and "links" in parsed:
|
||||
return parsed
|
||||
return None
|
||||
@@ -0,0 +1,239 @@
|
||||
"""The site's own URL for opening a composer, used when the task is to create something new.
|
||||
|
||||
Reachability, not fill mechanics, is what a 20-run dry sweep measured as the ceiling on scripted
|
||||
writes: the send script armed 20/20 and reached a composer 0/20. The fill, the commit check and the
|
||||
two-sided receipt are all proven; what fails is that prestage's aux navigator lands on the site's
|
||||
HOME page, and the composer is one more hop that it does not reliably take. Polling harder does not
|
||||
help, it was tried twice and the reached rate went 40% -> 0%.
|
||||
|
||||
Every site here publishes a URL that opens its own composer. Asking for that URL is deterministic
|
||||
where hunting for a button is not: no selector to drift, no modal to race, no capped element list to
|
||||
starve. It is also cheaper, since a hit skips the aux navigation loop entirely.
|
||||
|
||||
Deliberately narrow, because composing in the wrong place is worse than not composing at all:
|
||||
|
||||
- only for a task that creates something TOP-LEVEL. A reply or comment belongs on the thread the
|
||||
user is looking at, so those keep their own target (the same rule the post-is-not-a-comment
|
||||
guard enforces on the composer, applied one layer earlier and one layer cheaper).
|
||||
- only when the task points at the bare site. Any deeper URL on that host is a specific target
|
||||
the user chose, and it outranks the generic composer every time.
|
||||
- only as a PROPOSAL. The caller navigates, then checks whether a composer actually appeared; if
|
||||
it did not, the normal path runs untouched. A site that changes its compose URL degrades to
|
||||
today's behaviour rather than stranding the run somewhere useless.
|
||||
"""
|
||||
|
||||
import os
|
||||
import re
|
||||
from typing import Dict, List, Optional
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.agents.browser import browser_fast_path, browser_send_parse
|
||||
|
||||
# Registrable host -> the site's own compose URL. A dynamic-key map keyed by host, matched by
|
||||
# suffix so www./m./mobile. variants all resolve. Adding a site is one row.
|
||||
#
|
||||
# These are the sites' own documented entry points, not scraped links:
|
||||
# x /compose/post opens the post dialog on a fresh page
|
||||
# linkedin ?shareActive=true opens the "start a post" modal on the feed
|
||||
# reddit /submit?type=TEXT the self-post form
|
||||
# gmail ?compose=new opens a compose window in the mail UI
|
||||
P_COMPOSE_URLS: Dict[str, str] = {
|
||||
"x.com": "https://x.com/compose/post",
|
||||
"twitter.com": "https://x.com/compose/post",
|
||||
"linkedin.com": "https://www.linkedin.com/feed/?shareActive=true",
|
||||
"reddit.com": "https://www.reddit.com/submit?type=TEXT",
|
||||
"mail.google.com": "https://mail.google.com/mail/u/0/#inbox?compose=new",
|
||||
}
|
||||
|
||||
# Creating something new. "compose"/"draft"/"email" carry gmail, where nobody says "post".
|
||||
P_CREATE_RE = re.compile(
|
||||
r"\b(post|posting|tweet|tweeting|publish|publishing|share|sharing|compose|composing|"
|
||||
r"draft|drafting|write|writing|send|create|creating|start|starting|ask|asking)\b"
|
||||
# "open a new issue/thread" is a create; bare "open" is navigation ("open the first video"),
|
||||
# so only the whole phrase counts. Measured: github never reached the tier without this.
|
||||
r"|\bopen(ing)? a new\b", re.I)
|
||||
# Answering something that already exists. One of these and the target is the thread, not the site.
|
||||
P_RESPOND_RE = re.compile(r"\b(reply|replies|replying|comment|commenting|respond|responding|"
|
||||
r"answer|answering|quote|retweet|dm|message)\b", re.I)
|
||||
P_URL_RE = re.compile(r"https?://[^\s\"'<>)\]]+", re.I)
|
||||
# "go to x.com and post ..." names its site without a scheme, which is how most tasks arrive.
|
||||
P_BARE_HOST_RE = re.compile(r"(?:^|[\s/@(,])((?:[\w-]+\.)+[a-z]{2,})\b", re.I)
|
||||
|
||||
|
||||
@typechecked
|
||||
def registrable_host(url_or_host: str) -> str:
|
||||
"""The host with any www./m./mobile. prefix removed, lowercased, port dropped.
|
||||
|
||||
Prefix stripping is done with a real prefix check; `lstrip("www.")` would eat any leading w
|
||||
or dot and quietly turn `w3schools.com` into `3schools.com`."""
|
||||
raw = (url_or_host or "").strip()
|
||||
host = urlparse(raw).netloc if "//" in raw else raw
|
||||
host = host.lower().split("@")[-1].split(":")[0]
|
||||
for prefix in ("www.", "m.", "mobile."):
|
||||
if host.startswith(prefix):
|
||||
host = host[len(prefix):]
|
||||
break
|
||||
return host
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_table_hit(host: str) -> str:
|
||||
"""The compose URL for this host, matching a parent domain too (`old.reddit.com` -> reddit).
|
||||
|
||||
Exact-or-dotted-suffix only: a bare `endswith` would match `notreddit.com` against `reddit.com`
|
||||
and send a post to a site the user never named."""
|
||||
host = registrable_host(host)
|
||||
if not host:
|
||||
return ""
|
||||
if host in P_COMPOSE_URLS:
|
||||
return P_COMPOSE_URLS[host]
|
||||
for known, url in P_COMPOSE_URLS.items():
|
||||
if host.endswith("." + known):
|
||||
return url
|
||||
return ""
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_names_deeper_target(task: str, host: str) -> bool:
|
||||
"""True when the task carries a URL on this host that points somewhere more specific than its
|
||||
front page. That URL is the user's chosen target and must win over the generic composer."""
|
||||
want = registrable_host(host)
|
||||
for m in P_URL_RE.finditer(human_words(task)):
|
||||
parsed = urlparse(m.group(0).rstrip(".,;)"))
|
||||
found = registrable_host(parsed.netloc)
|
||||
if not found or (found != want and not found.endswith("." + want)):
|
||||
continue
|
||||
if parsed.path.strip("/") or parsed.query or parsed.fragment:
|
||||
return True
|
||||
return False
|
||||
|
||||
|
||||
@typechecked
|
||||
def human_words(task: str) -> str:
|
||||
"""The task minus the aux-written routing brief.
|
||||
|
||||
A dispatched task is the user's prompt followed by a brief a model wrote about how to route it.
|
||||
The brief is prose, and it both quotes things of its own and uses answering words, so reading
|
||||
intent off the whole string reads the model's commentary as the user's request. The send script
|
||||
hit this first (a brief saying "do not submit it" read-only-flagged a real send).
|
||||
|
||||
Everything reads this, refusals included. Letting the brief veto looked like the safe choice
|
||||
and was not: briefs routinely spell out a route ("navigate to https://x.com/home"), so the
|
||||
deeper-target veto fired on the model's own suggestion and silently disabled the tier on two of
|
||||
four sites. A brief cannot turn a post into a reply either, since the words that would say so
|
||||
are the user's and are still read here."""
|
||||
return (task or "").split(browser_fast_path.BRIEF_MARKER, 1)[0]
|
||||
|
||||
|
||||
@typechecked
|
||||
def wants_top_level_compose(task: str) -> bool:
|
||||
"""True when the task creates something new rather than answering something that exists.
|
||||
|
||||
An intent word alone is not enough: "what is the top post on reddit" is a READ, and `post` there
|
||||
is a noun. Reading the verb correctly needs a model, so instead this asks for the thing every
|
||||
real write in this product carries and no read does, the quoted text to write. That is also the
|
||||
send script's own precondition, so a task this refuses is one the script would decline anyway,
|
||||
and the aux navigator handles it exactly as it does today."""
|
||||
text = human_words(task)
|
||||
if not browser_send_parse.quoted_payload(text):
|
||||
return False
|
||||
return bool(P_CREATE_RE.search(text)) and not P_RESPOND_RE.search(text)
|
||||
|
||||
|
||||
@typechecked
|
||||
def enabled() -> bool:
|
||||
"""On by default. This tier only ever navigates the user's browser to a page that site
|
||||
publishes for exactly this purpose, and the caller verifies the result before relying on it, so
|
||||
the failure mode is a wasted page load rather than a wrong action. The switch exists to A/B it
|
||||
against the aux navigator and to turn it off in the field without a rebuild."""
|
||||
return os.environ.get("OSW_COMPOSE_ENTRY", "1") != "0"
|
||||
|
||||
|
||||
@typechecked
|
||||
def named_hosts(task: str, start_url: str) -> List[str]:
|
||||
"""The hosts this task is about, in the order they should be trusted.
|
||||
|
||||
The site the USER NAMED wins, and where the card happens to sit is only the fallback for a task
|
||||
that names none ("post this" while already on the site). That order is load-bearing for anything
|
||||
that has to be ON a site before it can read from it: a cold run opens on a blank search page, so
|
||||
trusting the card first made the first live discovery read google.com and correctly find
|
||||
nothing. `compose_entry_for` can afford the opposite order because a table hit already proves
|
||||
relevance; without a table there is nothing to filter a wrong host out.
|
||||
|
||||
Same extraction either way, kept in one place so the two cannot drift into disagreeing about
|
||||
which site a task is for. The routing brief is excluded, so a model cannot redirect the post."""
|
||||
out: List[str] = []
|
||||
for candidate in (*[m.group(0) for m in P_URL_RE.finditer(human_words(task))],
|
||||
*[m.group(1) for m in P_BARE_HOST_RE.finditer(human_words(task))],
|
||||
start_url):
|
||||
host = registrable_host(candidate)
|
||||
if host and "." in host and host not in out:
|
||||
out.append(host)
|
||||
return out
|
||||
|
||||
|
||||
@typechecked
|
||||
def named_page(task: str, host: str) -> str:
|
||||
"""The most specific page the user named on this host, else its front page.
|
||||
|
||||
Composing often lives under a section rather than at the root: "open a new issue" on
|
||||
`github.com/owner/repo` is reachable from the repo and nowhere near `github.com/`, which
|
||||
publishes no compose link at all. Measured, so the host alone is not enough to go on."""
|
||||
want = registrable_host(host)
|
||||
for m in P_URL_RE.finditer(human_words(task)):
|
||||
url = m.group(0).rstrip(".,;)")
|
||||
found = registrable_host(urlparse(url).netloc)
|
||||
if found and (found == want or found.endswith("." + want)):
|
||||
return url
|
||||
return f"https://{want}/"
|
||||
|
||||
|
||||
@typechecked
|
||||
def compose_entry_for(task: str, start_url: str, task_is_send: bool) -> Optional[str]:
|
||||
"""The URL to open to reach this site's composer, or None to leave navigation alone.
|
||||
|
||||
`start_url` is where the card already is; a host named in the task counts too, since a run that
|
||||
begins on a blank tab still says "go to x.com and post ...".
|
||||
|
||||
`task_is_send` is the caller's already-computed write verdict and is REQUIRED, not defaulted,
|
||||
because forgetting it is silently destructive: a quote is not proof of a write, and
|
||||
`find the reddit post that says "..."` reads as a create to any regex short enough to be
|
||||
readable (`post` is a noun there). Four such phrasings each resolved to reddit's SUBMIT page in
|
||||
a probe, which would derail a plain read. The verdict the send script itself gates on is the
|
||||
right authority, so this asks for it rather than growing a second opinion that can drift."""
|
||||
if not enabled() or not task_is_send or not wants_top_level_compose(task):
|
||||
return None
|
||||
# Which site to open comes from the user's words; a brief naming some other site must not
|
||||
# redirect the post.
|
||||
asked = human_words(task)
|
||||
named = [m.group(0) for m in P_URL_RE.finditer(asked)]
|
||||
named += [m.group(1) for m in P_BARE_HOST_RE.finditer(asked)]
|
||||
for candidate in (start_url, *named):
|
||||
url = p_table_hit(candidate)
|
||||
if not url:
|
||||
continue
|
||||
host = registrable_host(candidate)
|
||||
if p_names_deeper_target(task, host):
|
||||
return None
|
||||
# Already on the compose surface: navigating again would remount it and throw away a
|
||||
# composer that is right there.
|
||||
if registrable_host(start_url) == host and p_on_compose_surface(start_url, url):
|
||||
return None
|
||||
return url
|
||||
return None
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_on_compose_surface(current_url: str, compose_url: str) -> bool:
|
||||
"""True when the current URL is already this site's compose surface."""
|
||||
cur, target = urlparse(current_url or ""), urlparse(compose_url or "")
|
||||
if not cur.netloc:
|
||||
return False
|
||||
cur_mark = (cur.path.strip("/") + "?" + cur.query + "#" + cur.fragment).lower()
|
||||
target_mark = (target.path.strip("/") + "?" + target.query + "#" + target.fragment).lower()
|
||||
for token in ("compose", "submit", "shareactive"):
|
||||
if token in target_mark and token in cur_mark:
|
||||
return True
|
||||
return False
|
||||
@@ -0,0 +1,218 @@
|
||||
"""General capture-and-replay write tier: replay a write the site's OWN UI issues, via the
|
||||
borrowed session, WITHOUT a hand-written per-site adapter. This is the site-agnostic path to
|
||||
write coverage (the "all popular sites" lever): the browser passively captures the internal API
|
||||
routes the page fires (electron/cdp-routes.js), and this replays a MUTATING one with the agent's
|
||||
content substituted, using live-borrowed cookies (never persisted) plus any CSRF header the site
|
||||
derives from a cookie.
|
||||
|
||||
SAFETY (this IS the posture flip away from GET/HEAD-only, so the walls are belt-and-suspenders):
|
||||
- Same-origin: the target must be the site currently loaded, nothing else.
|
||||
- Captured-route match: the target must correspond to a mutating route the site's OWN UI actually
|
||||
fired. The agent can't invent an endpoint; it can only replay one the page genuinely uses. This
|
||||
is the wall against a prompt-injected page steering the agent to an arbitrary write.
|
||||
- Flag-gated default OFF (OSW_ROUTE_WRITE=1 to arm). The deterministic per-site adapters (Reddit)
|
||||
stay always-on; this general tier is opt-in until it's soaked.
|
||||
- Behind the caller's send-safety guard (solo, verified, receipt-or-honest-miss, never a false
|
||||
claim of success).
|
||||
- Secret-safe: cookies are live-borrowed per call, never logged, never persisted; the CSRF header
|
||||
is derived from a cookie at call time, not stored.
|
||||
"""
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import time
|
||||
import urllib.error
|
||||
import urllib.request
|
||||
from typing import Any, Dict, List
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.social_shims.session_source import get_session
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
WRITE_METHODS = frozenset({"POST", "PUT", "PATCH", "DELETE"})
|
||||
# CSRF header a site derives from a cookie (so it survives a fresh borrowed session). Small on
|
||||
# purpose: most cookie-auth internal APIs need nothing extra; this covers the common header case.
|
||||
P_CSRF_FROM_COOKIE: Dict[str, Dict[str, str]] = {
|
||||
"x.com": {"header": "x-csrf-token", "cookie": "ct0"},
|
||||
"twitter.com": {"header": "x-csrf-token", "cookie": "ct0"},
|
||||
}
|
||||
|
||||
|
||||
class CapturedRoute(BaseModel):
|
||||
"""One mutating route the site's own UI was seen to fire (from the CDP route capture). The
|
||||
method + templated path are the identity we match a replay target against; nothing secret
|
||||
lives here (the capture redacts auth headers and strips body values)."""
|
||||
|
||||
model_config = ConfigDict(validate_assignment=True)
|
||||
|
||||
method: str
|
||||
template: str
|
||||
|
||||
|
||||
class ReplayOutcome(BaseModel):
|
||||
"""Typed result of a route replay. `receipt` is the site's own confirmation pulled from the
|
||||
response (an id / permalink / url); `ok` is False with a legible `error` on any refusal or
|
||||
rejection, so the caller falls back to the UI, never a crash and never a false success."""
|
||||
|
||||
model_config = ConfigDict(validate_assignment=True)
|
||||
|
||||
ok: bool
|
||||
receipt: str = ""
|
||||
error: str = ""
|
||||
status: int = 0
|
||||
latency_ms: int = 0
|
||||
|
||||
|
||||
@typechecked
|
||||
def enabled() -> bool:
|
||||
"""The general route-write tier is opt-in (posture flip); armed only by OSW_ROUTE_WRITE=1."""
|
||||
return os.environ.get("OSW_ROUTE_WRITE", "0") == "1"
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_template_path(url: str) -> str:
|
||||
"""Collapse volatile path segments (numeric ids, long hex/uuids) to '{id}', mirroring the
|
||||
capture side (cdp-routes.js templateUrl) so a concrete replay URL matches the captured
|
||||
template. Origin + path only; query keys are ignored for the match."""
|
||||
try:
|
||||
u = urlparse(url)
|
||||
path = re.sub(r"/(\d+|[0-9a-fA-F]{8,}(?:-[0-9a-fA-F]+)*)(?=/|$)", "/{id}", u.path)
|
||||
return f"{u.scheme}://{u.netloc}{path}"
|
||||
except Exception:
|
||||
return url
|
||||
|
||||
|
||||
@typechecked
|
||||
def same_origin(url: str, origin: str) -> bool:
|
||||
"""True when url is on the same origin as the loaded page (scheme+host+port), the first wall."""
|
||||
try:
|
||||
a, b = urlparse(url), urlparse(origin)
|
||||
return bool(a.scheme and a.netloc) and (a.scheme, a.netloc) == (b.scheme, b.netloc)
|
||||
except Exception:
|
||||
return False
|
||||
|
||||
|
||||
@typechecked
|
||||
def route_is_captured(method: str, url: str, captured: List[CapturedRoute]) -> bool:
|
||||
"""True when (method, templated url) matches a mutating route the site's UI actually fired.
|
||||
The safety wall that stops a prompt-injected page from steering the agent to an invented
|
||||
endpoint: the agent can only replay a write the page genuinely uses."""
|
||||
m = method.upper()
|
||||
if m not in WRITE_METHODS:
|
||||
return False
|
||||
target = p_template_path(url)
|
||||
return any(r.method.upper() == m and p_template_path(r.template) == target for r in captured)
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_cookie_value(cookie_header: str, name: str) -> str:
|
||||
"""Pull one cookie's value out of a 'k=v; k2=v2' header, for CSRF-from-cookie derivation."""
|
||||
for part in (cookie_header or "").split(";"):
|
||||
k, _, v = part.strip().partition("=")
|
||||
if k == name:
|
||||
return v
|
||||
return ""
|
||||
|
||||
|
||||
@typechecked
|
||||
def derive_csrf_headers(url: str, cookie_header: str) -> Dict[str, str]:
|
||||
"""The CSRF header a site expects, re-derived from the live cookie (e.g. X's x-csrf-token is
|
||||
its ct0 cookie). Empty for the common cookie-only-auth site, which needs nothing extra."""
|
||||
host = (urlparse(url).netloc or "").lower().lstrip(".")
|
||||
apex = ".".join(host.split(".")[-2:]) if host.count(".") >= 1 else host
|
||||
rule = P_CSRF_FROM_COOKIE.get(apex)
|
||||
if not rule:
|
||||
return {}
|
||||
val = p_cookie_value(cookie_header, rule["cookie"])
|
||||
return {rule["header"]: val} if val else {}
|
||||
|
||||
|
||||
@typechecked
|
||||
def receipt_from_json(obj: Any) -> str:
|
||||
"""The most proof-bearing id/permalink/url anywhere in a response JSON (shallow-first), so the
|
||||
caller gets a real receipt without knowing each site's response shape."""
|
||||
seen: List[Any] = [obj]
|
||||
for _ in range(400): # bounded walk; a receipt lives near the top of a write response
|
||||
if not seen:
|
||||
break
|
||||
cur = seen.pop(0)
|
||||
if isinstance(cur, dict):
|
||||
for key in ("permalink", "url", "id_str", "rest_id", "id", "name"):
|
||||
v = cur.get(key)
|
||||
if isinstance(v, (str, int)) and str(v):
|
||||
return str(v)
|
||||
seen.extend(cur.values())
|
||||
elif isinstance(cur, list):
|
||||
seen.extend(cur)
|
||||
return ""
|
||||
|
||||
|
||||
@typechecked
|
||||
def outcome_from_response(status: int, text: str, latency_ms: int) -> ReplayOutcome:
|
||||
"""Map an HTTP response to a typed outcome: 2xx = landed (with a parsed receipt), anything else
|
||||
= a legible error the caller surfaces so the model does the write via the UI instead."""
|
||||
if not (200 <= status < 300):
|
||||
return ReplayOutcome(ok=False, status=status, latency_ms=latency_ms,
|
||||
error=f"site returned HTTP {status}: {text[:160]}")
|
||||
receipt = ""
|
||||
try:
|
||||
receipt = receipt_from_json(json.loads(text)) if text.strip() else ""
|
||||
except (json.JSONDecodeError, ValueError):
|
||||
receipt = ""
|
||||
return ReplayOutcome(ok=True, status=status, latency_ms=latency_ms, receipt=receipt or "ok")
|
||||
|
||||
|
||||
@typechecked
|
||||
def issue_request(method: str, url: str, body: Dict[str, Any], headers: Dict[str, str]) -> Any:
|
||||
"""Issue the write from the backend using the borrowed session. JSON body (the shape internal
|
||||
APIs overwhelmingly use). Returns (status, text). Isolated so tests stub the network."""
|
||||
data = json.dumps(body).encode() if body else b""
|
||||
req = urllib.request.Request(url, data=data, headers=headers, method=method.upper())
|
||||
try:
|
||||
with urllib.request.urlopen(req, timeout=30.0) as resp:
|
||||
return resp.status, resp.read().decode("utf-8", "replace")
|
||||
except urllib.error.HTTPError as e:
|
||||
return e.code, (e.read().decode("utf-8", "replace") if e.fp else "")
|
||||
except urllib.error.URLError as e:
|
||||
raise RuntimeError(f"site unreachable: {getattr(e, 'reason', e)}")
|
||||
|
||||
|
||||
@typechecked
|
||||
def replay_write(method: str, url: str, body: Dict[str, Any], origin: str,
|
||||
captured: List[CapturedRoute]) -> ReplayOutcome:
|
||||
"""Replay one captured mutating route with the agent's content, via the live-borrowed session.
|
||||
Every failure (disarmed, off-origin, un-captured, no session, site-reject) is a typed ok=False
|
||||
so the caller falls back to the UI, never a crash. Secrets are live-borrowed, never logged."""
|
||||
if not enabled():
|
||||
return ReplayOutcome(ok=False, error="route-write tier disarmed (set OSW_ROUTE_WRITE=1); use the UI")
|
||||
if not same_origin(url, origin):
|
||||
return ReplayOutcome(ok=False, error="target is not the current site (same-origin only)")
|
||||
if not route_is_captured(method, url, captured):
|
||||
return ReplayOutcome(ok=False, error="no matching write route was captured from this site's UI; use the UI")
|
||||
domain = (urlparse(origin).netloc or "").lstrip(".")
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
cookie, ua = get_session(domain)
|
||||
except Exception as e:
|
||||
return ReplayOutcome(ok=False, error=f"no borrowable session for {domain}: {str(e)[:120]}")
|
||||
headers = {
|
||||
"Cookie": cookie, "User-Agent": ua, "Accept": "application/json",
|
||||
"Content-Type": "application/json", "Origin": origin, "Referer": origin + "/",
|
||||
**derive_csrf_headers(url, cookie),
|
||||
}
|
||||
try:
|
||||
status, text = issue_request(method, url, body, headers)
|
||||
except Exception as e:
|
||||
# UNEXPECTED: the request itself blew up (network/DNS/TLS), not a same-origin/captured refusal. Fails open to the UI, so log it or the broken route-write tier is invisible.
|
||||
logger.warning(f"[route-write] {method} {url} request FAILED (fast path broken here): {e}")
|
||||
return ReplayOutcome(ok=False, error=str(e)[:160], latency_ms=int((time.monotonic() - t0) * 1000))
|
||||
out = outcome_from_response(status, text, int((time.monotonic() - t0) * 1000))
|
||||
if not out.ok:
|
||||
logger.info(f"[route-write] {method} {url} rejected by site: HTTP {out.status}")
|
||||
return out
|
||||
@@ -1,117 +1,9 @@
|
||||
"""
|
||||
Shipped seed playbooks: a starting strategy memory for popular sites so a fresh
|
||||
install isn't fully cold on its first task there. These are FALLBACKS, the moment
|
||||
a user does a real verified run on a site, the reflective distill writes a learned
|
||||
playbook that supersedes the seed (and refines it). So a wrong seed bullet can only
|
||||
gently mislead a first run and is self-corrected, exactly the playbook's fail-safe.
|
||||
"""Loader for the shipped seed playbooks (data lives in seed_playbooks.py). A fresh install isn't
|
||||
fully cold on a popular site's first task: seed_for returns the site's starting strategy bullets,
|
||||
which browser_playbook uses as a FALLBACK until a real verified run distills a learned playbook
|
||||
that supersedes them. Match is canonical-host, with and without a leading 'www.'."""
|
||||
|
||||
Sourced from real observation (a read-only recon pass over the top sites) plus the
|
||||
stable, documented deep-URL search patterns, NOT guessed mechanics. Host keys are
|
||||
canonical (no leading 'www.'); the loader strips 'www.' before matching. Kept to
|
||||
the same per-site shape and caps as a learned playbook.
|
||||
|
||||
Coverage note: only LinkedIn carries full task mechanics (it's the one we fully
|
||||
exercised). The rest carry the high-value generalizable facts a first run wants:
|
||||
the deep-URL search shortcut, whether the site is usable logged-out, and where the
|
||||
primary controls live. Richer per-site mechanics accrue as users actually use them.
|
||||
"""
|
||||
|
||||
SEED_PLAYBOOKS: dict[str, list[str]] = {
|
||||
"linkedin.com": [
|
||||
"Find people via URL: linkedin.com/search/results/people/?keywords=NAME (one nav beats driving the search UI).",
|
||||
"Open a person's profile, then click Message to open the compose box for that specific person.",
|
||||
"A 1:1 thread is titled '<Other Person> and <You>'; that IS the direct thread, do NOT start a new one.",
|
||||
"In the composer, type the message then click Send; do NOT press Enter (in the rich composer it only inserts a newline).",
|
||||
],
|
||||
"amazon.com": [
|
||||
"Search via URL: amazon.com/s?k=QUERY (spaces become +). Browsing and reading prices/ratings work logged-out.",
|
||||
"Results are cards with a product link, price, and rating; pull them in one shot with BrowserExtract.",
|
||||
],
|
||||
"ebay.com": [
|
||||
"Search via URL: ebay.com/sch/i.html?_nkw=QUERY. Browsing works logged-out.",
|
||||
],
|
||||
"walmart.com": [
|
||||
"Search via URL: walmart.com/search?q=QUERY. Browsing works logged-out; it can show a press-and-hold bot check on heavy use.",
|
||||
],
|
||||
"etsy.com": [
|
||||
"Search via URL: etsy.com/search?q=QUERY. Browsing works logged-out.",
|
||||
],
|
||||
"target.com": [
|
||||
"Search via URL: target.com/s?searchTerm=QUERY. Browsing works logged-out.",
|
||||
],
|
||||
"bestbuy.com": [
|
||||
"Search via URL: bestbuy.com/site/searchpage.jsp?st=QUERY. A country-select splash may appear first; pick United States.",
|
||||
],
|
||||
"aliexpress.com": [
|
||||
"Browsing works logged-out; use the top search box rather than guessing the URL (the search path changes often).",
|
||||
],
|
||||
"craigslist.org": [
|
||||
"Listings are per-city: go to the city subdomain first (e.g. sfbay.craigslist.org), search is local, not global.",
|
||||
],
|
||||
"airbnb.com": [
|
||||
"Drive the homepage search (Where / check-in-out / Who) then Search; the results URL params are brittle, don't hand-build them.",
|
||||
],
|
||||
"booking.com": [
|
||||
"Search via URL: booking.com/searchresults.html?ss=DESTINATION. Browsing works logged-out.",
|
||||
],
|
||||
"expedia.com": [
|
||||
"Drive the homepage search widget (Where to, dates, travelers); its URL is complex, don't hand-build it.",
|
||||
],
|
||||
"yelp.com": [
|
||||
"Search via URL: yelp.com/search?find_desc=WHAT&find_loc=WHERE. Browsing works logged-out.",
|
||||
],
|
||||
"google.com": [
|
||||
"Web search via URL: google.com/search?q=QUERY. Maps search via URL: google.com/maps/search/PLACE.",
|
||||
],
|
||||
"doordash.com": [
|
||||
"It gates on a delivery address up front; set the address before browsing restaurants or you'll see nothing.",
|
||||
],
|
||||
"netflix.com": [
|
||||
"Requires sign-in to browse or play. Once logged in, search via URL: netflix.com/search?q=QUERY.",
|
||||
],
|
||||
"spotify.com": [
|
||||
"Search via URL: open.spotify.com/search/QUERY. Reading catalog works, but playing full tracks needs a logged-in session.",
|
||||
],
|
||||
"twitch.tv": [
|
||||
"Search via URL: twitch.tv/search?term=QUERY. A channel lives at twitch.tv/CHANNELNAME.",
|
||||
],
|
||||
"tiktok.com": [
|
||||
"Search via URL: tiktok.com/search?q=QUERY. Heavy anti-bot, expect occasional captcha or a login prompt.",
|
||||
],
|
||||
"pinterest.com": [
|
||||
"Search pins via URL: pinterest.com/search/pins/?q=QUERY. Most actions (save, follow) need sign-in.",
|
||||
],
|
||||
"facebook.com": [
|
||||
"Requires sign-in. The login wall appears immediately; if you aren't signed in, use RequestHumanIntervention, do not try to log in.",
|
||||
],
|
||||
"instagram.com": [
|
||||
"Requires sign-in. The login wall appears immediately; if you aren't signed in, use RequestHumanIntervention, do not try to log in.",
|
||||
],
|
||||
"x.com": [
|
||||
"Most actions need sign-in. Once logged in, search via URL: x.com/search?q=QUERY (twitter.com redirects here).",
|
||||
],
|
||||
"quora.com": [
|
||||
"Search via URL: quora.com/search?q=QUERY. Reading often triggers a sign-in wall after a bit of scrolling.",
|
||||
],
|
||||
"github.com": [
|
||||
"Search via URL: github.com/search?q=QUERY&type=repositories. Public repos, issues, and code are readable logged-out.",
|
||||
],
|
||||
"threads.net": [
|
||||
"A login overlay sits over the feed; no composer or search is reachable until signed in.",
|
||||
],
|
||||
"web.whatsapp.com": [
|
||||
"Needs a phone-linked session via QR. In automation it often serves a 'use a supported browser' wall, treat as not reliably automatable.",
|
||||
],
|
||||
"web.telegram.org": [
|
||||
"Web login is QR-code or passkey; if not already logged in, use RequestHumanIntervention rather than attempting it.",
|
||||
],
|
||||
"trello.com": [
|
||||
"The landing page is marketing; the app needs sign-in. If not signed in, use RequestHumanIntervention.",
|
||||
],
|
||||
"figma.com": [
|
||||
"The landing page is marketing; the app needs sign-in. If not signed in, use RequestHumanIntervention.",
|
||||
],
|
||||
}
|
||||
from backend.apps.agents.browser.seed_playbooks import SEED_PLAYBOOKS
|
||||
|
||||
|
||||
def seed_for(host: str) -> list[str]:
|
||||
|
||||
@@ -0,0 +1,176 @@
|
||||
"""Shipped seed playbooks DATA: starting strategy memory for popular sites so a fresh install
|
||||
isn't fully cold on its first task there. FALLBACKS only, the moment a user does a real verified
|
||||
run on a site the reflective distill writes a learned playbook that supersedes the seed, so a
|
||||
wrong bullet can only gently mislead a first run and is self-corrected (the playbook's fail-safe).
|
||||
|
||||
Facts are the stable, documented deep-URL search patterns + observed login/bot walls, NOT guessed
|
||||
mechanics; where a URL pattern is unstable the bullet says to drive the search box instead. Host
|
||||
keys are canonical (no leading 'www.'); the loader (seed_for) strips 'www.' before matching.
|
||||
|
||||
Money sites are deliberately framed READ-ONLY: read balances/activity, NEVER move money, pay, trade,
|
||||
or transfer, hand any transaction to the user. This mirrors the product's financial-action rule.
|
||||
"""
|
||||
|
||||
SEED_PLAYBOOKS: dict[str, list[str]] = {
|
||||
# --- search ---
|
||||
"google.com": [
|
||||
"Web search via URL: google.com/search?q=QUERY. Maps search via URL: google.com/maps/search/PLACE.",
|
||||
],
|
||||
"bing.com": ["Search via URL: bing.com/search?q=QUERY. Works logged-out."],
|
||||
"duckduckgo.com": ["Search via URL: duckduckgo.com/?q=QUERY. Works logged-out, no login."],
|
||||
"yahoo.com": ["Search via URL: search.yahoo.com/search?p=QUERY. Works logged-out."],
|
||||
# --- reference / knowledge ---
|
||||
"wikipedia.org": [
|
||||
"Read via URL: en.wikipedia.org/wiki/TITLE (spaces become _). Search via en.wikipedia.org/w/index.php?search=QUERY. Fully readable logged-out.",
|
||||
],
|
||||
"imdb.com": ["A title lives at imdb.com/title/ttID. Search via URL: imdb.com/find/?q=QUERY. Readable logged-out."],
|
||||
"quora.com": ["Search via URL: quora.com/search?q=QUERY. Reading often triggers a sign-in wall after a bit of scrolling."],
|
||||
"goodreads.com": ["Search via URL: goodreads.com/search?q=QUERY. Browsing works logged-out; shelving/rating needs sign-in."],
|
||||
"stackoverflow.com": [
|
||||
"Search via URL: stackoverflow.com/search?q=QUERY. Questions and answers are readable logged-out; voting/answering needs sign-in.",
|
||||
],
|
||||
"github.com": ["Search via URL: github.com/search?q=QUERY&type=repositories. Public repos, issues, and code are readable logged-out."],
|
||||
"news.ycombinator.com": [
|
||||
"The front page is news.ycombinator.com; an item is news.ycombinator.com/item?id=ID. Readable logged-out; commenting/voting needs sign-in.",
|
||||
],
|
||||
"medium.com": ["Articles are readable but many hit a metered paywall after a few reads; search via URL: medium.com/search?q=QUERY."],
|
||||
# --- news / weather ---
|
||||
"news.google.com": ["Google News: news.google.com/search?q=QUERY. Works logged-out."],
|
||||
"weather.com": ["Type a city into the search box (weather.com keys pages on an internal location code, so don't hand-build a URL from a city name); forecasts read fine logged-out."],
|
||||
"cnn.com": ["Readable logged-out; search via URL: cnn.com/search?q=QUERY."],
|
||||
"bbc.com": ["Readable logged-out; search via URL: bbc.co.uk/search?q=QUERY."],
|
||||
"nytimes.com": ["A metered paywall appears after a few articles; use the site's own search box, not a guessed URL."],
|
||||
# --- video / streaming / music ---
|
||||
"youtube.com": [
|
||||
"Search via URL: youtube.com/results?search_query=QUERY. A video is youtube.com/watch?v=ID; browsing and watching work logged-out.",
|
||||
"To comment: open the video, click the 'Add a comment...' box, type, then click Comment (needs sign-in).",
|
||||
],
|
||||
"netflix.com": ["Requires sign-in to browse or play. Once logged in, search via URL: netflix.com/search?q=QUERY."],
|
||||
"hulu.com": ["Requires sign-in to browse or play; if not signed in use RequestHumanIntervention."],
|
||||
"disneyplus.com": ["Requires sign-in to browse or play; if not signed in use RequestHumanIntervention."],
|
||||
"max.com": ["Requires sign-in to browse or play; if not signed in use RequestHumanIntervention."],
|
||||
"twitch.tv": ["Search via URL: twitch.tv/search?term=QUERY. A channel lives at twitch.tv/CHANNELNAME; browsing works logged-out."],
|
||||
"spotify.com": ["Search via URL: open.spotify.com/search/QUERY. Reading catalog works, but playing full tracks needs a logged-in session."],
|
||||
"music.apple.com": ["Search via URL: music.apple.com/us/search?term=QUERY. Browsing works; full playback needs a signed-in subscription."],
|
||||
"soundcloud.com": ["Search via URL: soundcloud.com/search?q=QUERY. Browsing and streaming work logged-out."],
|
||||
"pandora.com": ["Needs sign-in for most listening; if not signed in use RequestHumanIntervention."],
|
||||
# --- shopping ---
|
||||
"amazon.com": [
|
||||
"Search via URL: amazon.com/s?k=QUERY (spaces become +). Browsing and reading prices/ratings work logged-out.",
|
||||
"Results are cards with a product link, price, and rating; pull them in one shot with BrowserExtract.",
|
||||
],
|
||||
"ebay.com": ["Search via URL: ebay.com/sch/i.html?_nkw=QUERY. Browsing works logged-out."],
|
||||
"walmart.com": ["Search via URL: walmart.com/search?q=QUERY. Browsing works logged-out; it can show a press-and-hold bot check on heavy use."],
|
||||
"target.com": ["Search via URL: target.com/s?searchTerm=QUERY. Browsing works logged-out."],
|
||||
"bestbuy.com": ["Search via URL: bestbuy.com/site/searchpage.jsp?st=QUERY. A country-select splash may appear first; pick United States."],
|
||||
"etsy.com": ["Search via URL: etsy.com/search?q=QUERY. Browsing works logged-out."],
|
||||
"aliexpress.com": ["Browsing works logged-out; use the top search box rather than guessing the URL (the search path changes often)."],
|
||||
"temu.com": ["Browsing works logged-out but expect aggressive popups; use the top search box rather than a hand-built URL."],
|
||||
"shein.com": ["Search via URL: shein.com/pdsearch/QUERY. Browsing works logged-out."],
|
||||
"costco.com": ["Search via URL: costco.com/CatalogSearch?keyword=QUERY. Some prices and buying need a member sign-in."],
|
||||
"homedepot.com": ["Search via URL: homedepot.com/s/QUERY. Browsing works logged-out."],
|
||||
"wayfair.com": ["Search via URL: wayfair.com/keyword.php?keyword=QUERY. Browsing works logged-out."],
|
||||
"craigslist.org": ["Listings are per-city: go to the city subdomain first (e.g. sfbay.craigslist.org); search is local, not global."],
|
||||
"instacart.com": ["Gates on a delivery address (and usually sign-in) before showing stores; set the location first."],
|
||||
# --- food delivery / rides / reservations ---
|
||||
"doordash.com": ["It gates on a delivery address up front; set the address before browsing restaurants or you'll see nothing."],
|
||||
"ubereats.com": ["Gates on a delivery address up front; set it before browsing. Ordering needs sign-in."],
|
||||
"grubhub.com": ["Set a delivery address first (browsing restaurants needs it). Ordering needs sign-in."],
|
||||
"uber.com": ["The ride app needs sign-in; if not signed in use RequestHumanIntervention, do not attempt to log in."],
|
||||
"lyft.com": ["The ride app needs sign-in; if not signed in use RequestHumanIntervention, do not attempt to log in."],
|
||||
"opentable.com": ["Search via URL: opentable.com/s?term=QUERY. Booking a table needs sign-in."],
|
||||
# --- social / messaging ---
|
||||
"facebook.com": ["Requires sign-in. The login wall appears immediately; if you aren't signed in, use RequestHumanIntervention, do not try to log in."],
|
||||
"instagram.com": ["Requires sign-in. The login wall appears immediately; if you aren't signed in, use RequestHumanIntervention, do not try to log in."],
|
||||
"x.com": [
|
||||
"Most actions need sign-in. Once logged in, search via URL: x.com/search?q=QUERY (twitter.com redirects here).",
|
||||
"To post: click the composer ('What is happening?'), type, then click Post; do NOT press Enter (it inserts a newline). To reply, open the tweet and use its Reply box then the Reply button.",
|
||||
],
|
||||
"reddit.com": [
|
||||
"Search via URL: reddit.com/search/?q=QUERY. A subreddit is reddit.com/r/NAME; most browsing works logged-out.",
|
||||
"Posting/commenting needs sign-in and the composer is bot-gated; prefer the built-in write path (BrowserApiWrite) over driving the UI composer.",
|
||||
],
|
||||
"tiktok.com": ["Search via URL: tiktok.com/search?q=QUERY. Heavy anti-bot, expect occasional captcha or a login prompt."],
|
||||
"pinterest.com": ["Search pins via URL: pinterest.com/search/pins/?q=QUERY. Most actions (save, follow) need sign-in."],
|
||||
"linkedin.com": [
|
||||
"Find people via URL: linkedin.com/search/results/people/?keywords=NAME (one nav beats driving the search UI).",
|
||||
"Open a person's profile, then click Message to open the compose box for that specific person.",
|
||||
"A 1:1 thread is titled '<Other Person> and <You>'; that IS the direct thread, do NOT start a new one.",
|
||||
"In the composer, type the message then click Send; do NOT press Enter (in the rich composer it only inserts a newline).",
|
||||
],
|
||||
"threads.net": ["A login overlay sits over the feed; no composer or search is reachable until signed in."],
|
||||
"messenger.com": ["Uses the Facebook login; if not signed in use RequestHumanIntervention, do not try to log in."],
|
||||
"snapchat.com": ["Primarily a mobile app; the web is limited and login-walled. Use RequestHumanIntervention on a wall."],
|
||||
"nextdoor.com": ["The neighborhood feed needs sign-in; if signed in, confirm the neighborhood, then browse posts."],
|
||||
"web.whatsapp.com": ["Needs a phone-linked session via QR; it often serves a 'use a supported browser' wall, treat as not reliably automatable."],
|
||||
"web.telegram.org": ["Web login is QR-code or passkey; if not already logged in, use RequestHumanIntervention rather than attempting it."],
|
||||
# --- email / productivity ---
|
||||
"mail.google.com": [
|
||||
"Needs sign-in. Search via URL: mail.google.com/mail/u/0/#search/QUERY. To send: click Compose, fill To then Subject then the body, then click Send.",
|
||||
],
|
||||
"outlook.com": ["Microsoft email; needs sign-in. If not signed in use RequestHumanIntervention."],
|
||||
"office.com": ["Microsoft 365 hub; needs sign-in. If not signed in use RequestHumanIntervention."],
|
||||
"docs.google.com": ["Google Docs/Sheets/Slides; needs sign-in. A doc is docs.google.com/document/d/ID, a sheet docs.google.com/spreadsheets/d/ID; edit once signed in."],
|
||||
"drive.google.com": ["Google Drive; needs sign-in. Search files via URL: drive.google.com/drive/search?q=QUERY."],
|
||||
"calendar.google.com": ["Google Calendar; needs sign-in. Read events and create via the '+ Create' button once signed in."],
|
||||
"dropbox.com": ["The landing page is marketing; files need sign-in. If not signed in use RequestHumanIntervention."],
|
||||
"notion.so": ["The landing page is marketing; the workspace needs sign-in. If not signed in, use RequestHumanIntervention."],
|
||||
"trello.com": ["The landing page is marketing; the app needs sign-in. If not signed in, use RequestHumanIntervention."],
|
||||
"figma.com": ["The landing page is marketing; the app needs sign-in. If not signed in, use RequestHumanIntervention."],
|
||||
# --- travel / local ---
|
||||
"airbnb.com": ["Drive the homepage search (Where / check-in-out / Who) then Search; the results URL params are brittle, don't hand-build them."],
|
||||
"booking.com": ["Search via URL: booking.com/searchresults.html?ss=DESTINATION. Browsing works logged-out."],
|
||||
"expedia.com": ["Drive the homepage search widget (Where to, dates, travelers); its URL is complex, don't hand-build it."],
|
||||
"tripadvisor.com": ["Search via URL: tripadvisor.com/Search?q=QUERY. Browsing works logged-out."],
|
||||
"kayak.com": ["Drive the homepage flight/hotel search; result URLs are brittle, don't hand-build them."],
|
||||
"hotels.com": ["Drive the homepage search widget; result URLs are complex, don't hand-build them."],
|
||||
"vrbo.com": ["Drive the homepage search; result URLs are brittle, don't hand-build them."],
|
||||
"yelp.com": ["Search via URL: yelp.com/search?find_desc=WHAT&find_loc=WHERE. Browsing works logged-out."],
|
||||
# --- jobs / real estate ---
|
||||
"indeed.com": ["Job search via URL: indeed.com/jobs?q=WHAT&l=WHERE. Browsing works logged-out; applying needs sign-in and is heavily bot-gated (expect a captcha)."],
|
||||
"glassdoor.com": ["Heavy sign-in and anti-bot walls appear quickly; treat as often not reliably automatable, use RequestHumanIntervention on a wall."],
|
||||
"ziprecruiter.com": ["Job search via URL: ziprecruiter.com/jobs-search?search=WHAT&location=WHERE. Applying needs sign-in."],
|
||||
"zillow.com": ["Search via URL: zillow.com/homes/CITY-STATE_rb/. Heavy anti-bot: a press-and-hold or captcha wall is common on more than a few requests."],
|
||||
"realtor.com": ["Search via URL: realtor.com/realestateandhomes-search/CITY_STATE. Browsing works logged-out."],
|
||||
"redfin.com": ["Drive the homepage search box; scripted URL access is heavily anti-bot."],
|
||||
"apartments.com": ["Search via URL: apartments.com/CITY-STATE/. Browsing works logged-out."],
|
||||
# --- tickets / events ---
|
||||
"ticketmaster.com": ["Search via URL: ticketmaster.com/search?q=QUERY. Buying needs sign-in and hits queue/anti-bot walls."],
|
||||
"stubhub.com": ["Search events from the homepage; buying needs sign-in."],
|
||||
"eventbrite.com": ["Search via URL: eventbrite.com/d/online/QUERY/. Browsing works logged-out; registering needs sign-in."],
|
||||
# --- health ---
|
||||
"webmd.com": ["Readable logged-out; search via URL: webmd.com/search/search_results/default.aspx?query=QUERY."],
|
||||
"goodrx.com": ["A drug's price page is goodrx.com/DRUG-NAME (a specific drug, e.g. goodrx.com/lipitor), NOT free-text search; for an unknown name use the site search box. Prices readable logged-out."],
|
||||
"cvs.com": ["General browsing works logged-out; pharmacy and account pages need sign-in and are sensitive."],
|
||||
"walgreens.com": ["General browsing works logged-out; pharmacy and account pages need sign-in and are sensitive."],
|
||||
# --- education ---
|
||||
"quizlet.com": ["Search via URL: quizlet.com/search?query=QUERY. Browsing study sets works logged-out."],
|
||||
"duolingo.com": ["The lessons app needs sign-in; the landing page is marketing. If not signed in use RequestHumanIntervention."],
|
||||
"khanacademy.org": ["Browsing lessons works logged-out; search via URL: khanacademy.org/search?page_search_query=QUERY."],
|
||||
"coursera.org": ["Search via URL: coursera.org/search?query=QUERY. Enrolling and course content need sign-in."],
|
||||
"chegg.com": ["Most content is paywalled behind sign-in; treat as read-limited without an account."],
|
||||
# --- AI assistants ---
|
||||
"chatgpt.com": ["A chat app that needs sign-in; if a task requires it and you're not signed in, use RequestHumanIntervention."],
|
||||
"claude.ai": ["A chat app that needs sign-in; if a task requires it and you're not signed in, use RequestHumanIntervention."],
|
||||
"perplexity.ai": ["Answer-search AI; search via URL: perplexity.ai/search?q=QUERY (some features need sign-in)."],
|
||||
"gemini.google.com": ["A chat app that needs sign-in; if a task requires it and you're not signed in, use RequestHumanIntervention."],
|
||||
# --- government / utilities ---
|
||||
"usps.com": ["Track a package via URL: tools.usps.com/go/TrackConfirmAction?tLabels=NUMBER. General info is readable logged-out."],
|
||||
"irs.gov": ["Forms and info are readable logged-out; any personal account access needs sign-in and is sensitive."],
|
||||
# --- money (READ-ONLY, never transact) ---
|
||||
"paypal.com": ["Needs sign-in. READ-ONLY: read balances/activity, but NEVER send money, pay, or transfer, hand any money movement to the user (RequestHumanIntervention)."],
|
||||
"venmo.com": ["Needs sign-in. READ-ONLY: read activity, but NEVER pay, request, or transfer money, hand any payment to the user."],
|
||||
"cash.app": ["Needs sign-in. READ-ONLY: read activity, but NEVER send, request, or move money, hand any payment to the user."],
|
||||
"chase.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/transactions, but NEVER move money, pay a bill, or transfer, hand any transaction to the user."],
|
||||
"bankofamerica.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/transactions, but NEVER move money, pay a bill, or transfer, hand any transaction to the user."],
|
||||
"wellsfargo.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/transactions, but NEVER move money, pay a bill, or transfer, hand any transaction to the user."],
|
||||
"capitalone.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/transactions, but NEVER move money, pay, or transfer, hand any transaction to the user."],
|
||||
"robinhood.com": ["Needs sign-in and is sensitive. READ-ONLY: read positions/prices, but NEVER place, cancel, or modify a trade, hand any trade to the user."],
|
||||
"coinbase.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/prices, but NEVER buy, sell, send, or trade crypto, hand any transaction to the user."],
|
||||
"fidelity.com": ["Needs sign-in and is sensitive. READ-ONLY: read balances/positions, but NEVER trade or move money, hand any transaction to the user."],
|
||||
"creditkarma.com": ["Needs sign-in; read-only credit score/report info, do not apply for anything on the user's behalf."],
|
||||
# --- dating (fingerprint-walled) ---
|
||||
"tinder.com": ["Mobile-first with heavy fingerprinting and a login wall; treat as not reliably automatable, use RequestHumanIntervention."],
|
||||
"bumble.com": ["Mobile-first with heavy fingerprinting and a login wall; treat as not reliably automatable, use RequestHumanIntervention."],
|
||||
"hinge.co": ["Mobile-first with heavy fingerprinting and a login wall; treat as not reliably automatable, use RequestHumanIntervention."],
|
||||
}
|
||||
@@ -0,0 +1,148 @@
|
||||
"""The API-first write tier, unified for the browser agent.
|
||||
|
||||
When a write targets a site that has a borrowed-session write adapter, route the write HERE
|
||||
instead of UI puppeteering: borrow the user's live cookies, call the site's OWN write API, and
|
||||
return the site's typed receipt (its own id / permalink = proof it landed). This is
|
||||
deterministic (a typed success/error envelope, no captcha on the API surface, no DOM selector to
|
||||
drift) and ~50-190x faster than driving the UI (measured Reddit: 271ms vs 13-52s). Adding a site
|
||||
is one adapter entry; a site with no adapter falls back to the existing UI+model write path.
|
||||
|
||||
Live-validated end to end on Reddit (comment 271ms + reversible delete 246ms, typed receipts).
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import time
|
||||
from typing import Any, Callable, Dict, FrozenSet, List, Tuple
|
||||
from urllib.parse import urlparse
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.agents.browser import route_write
|
||||
from backend.apps.reddit_mcp_shim import reddit_writes
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class WriteResult(BaseModel):
|
||||
"""The typed outcome of an API-first write. `receipt` is the site's own id/permalink, the
|
||||
proof the write landed (a real receipt, not a pixel guess); `ok` is False with a legible
|
||||
`error` when the site's API rejected it or no session could be borrowed."""
|
||||
|
||||
model_config = ConfigDict(validate_assignment=True)
|
||||
|
||||
ok: bool
|
||||
action: str
|
||||
domain: str
|
||||
receipt: str = ""
|
||||
error: str = ""
|
||||
latency_ms: int = 0
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_reddit_dispatch(action: str, params: Dict[str, Any]) -> Dict[str, Any]:
|
||||
"""Route a generic write action to the proven reddit_writes function; returns its receipt
|
||||
dict (raises RedditError on the API's own error envelope, surfaced as ok=False upstream)."""
|
||||
if action in ("comment", "reply"):
|
||||
return reddit_writes.comment(str(params["parent_id"]), str(params["text"]))
|
||||
if action in ("post", "submit"):
|
||||
return reddit_writes.submit(
|
||||
str(params["subreddit"]), str(params["title"]), str(params.get("kind", "self")),
|
||||
str(params.get("text", "")), str(params.get("url", "")),
|
||||
bool(params.get("nsfw", False)), bool(params.get("spoiler", False)),
|
||||
bool(params.get("send_replies", True)),
|
||||
)
|
||||
if action == "edit":
|
||||
return reddit_writes.edit(str(params["thing_id"]), str(params["text"]))
|
||||
if action == "delete":
|
||||
return reddit_writes.delete(str(params["thing_id"]))
|
||||
raise ValueError(f"reddit adapter has no action {action!r}")
|
||||
|
||||
|
||||
# domain -> (actions it can do via the site's own API, sync dispatch fn). A dynamic-key registry
|
||||
# keyed by domain; adding a site is one row. X/others plug in the same shape once their write API
|
||||
# (GraphQL queryId + ct0) is proven, replacing their current UI-driving shim.
|
||||
P_ADAPTERS: Dict[str, Tuple[FrozenSet[str], Callable[[str, Dict[str, Any]], Dict[str, Any]]]] = {
|
||||
"reddit.com": (frozenset({"comment", "reply", "post", "submit", "edit", "delete"}), p_reddit_dispatch),
|
||||
}
|
||||
|
||||
|
||||
@typechecked
|
||||
def has_api_write(domain: str, action: str) -> bool:
|
||||
"""True when this domain has a deterministic API adapter for this write action, so the agent
|
||||
should route around the UI puppeteer tier."""
|
||||
entry = P_ADAPTERS.get(domain.lower().strip().lstrip("."))
|
||||
return bool(entry and action in entry[0])
|
||||
|
||||
|
||||
@typechecked
|
||||
def receipt_str(receipt: Dict[str, Any]) -> str:
|
||||
"""Flatten a site receipt dict into the single most-proof-bearing string (permalink beats a
|
||||
bare id) so callers get one legible confirmation without knowing each site's shape."""
|
||||
for key in ("permalink", "url", "id"):
|
||||
v = receipt.get(key)
|
||||
if v:
|
||||
return str(v)
|
||||
return "ok"
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_ensure_session_env() -> None:
|
||||
"""Point session_source at the running backend so the in-process agent can borrow cookies the
|
||||
same token-gated way the subprocess shims do (module globals are read at import, so patch
|
||||
them). No-op once set."""
|
||||
from backend.apps.social_shims import session_source as ss
|
||||
port = os.environ.get("OPENSWARM_PORT", "8324")
|
||||
ss.BACKEND_PORT = port
|
||||
ss.BRIDGE_URL = f"http://127.0.0.1:{port}/api/browser-session/cookies"
|
||||
if not ss.AUTH_TOKEN:
|
||||
try:
|
||||
from backend.auth import get_auth_token
|
||||
ss.AUTH_TOKEN = get_auth_token() or ""
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
@typechecked
|
||||
async def api_route_write(origin: str, method: str, url: str, body: Dict[str, Any],
|
||||
captured: List[route_write.CapturedRoute]) -> WriteResult:
|
||||
"""The GENERAL tier: replay a captured mutating route the site's own UI fired, for sites with
|
||||
no hand-written adapter. Wraps route_write's typed outcome into the registry's WriteResult so
|
||||
callers get one shape. Every refusal/rejection is ok=False, so the agent falls back to the UI."""
|
||||
p_ensure_session_env()
|
||||
d = (urlparse(origin).netloc or origin).lstrip(".")
|
||||
out = await asyncio.to_thread(route_write.replay_write, method, url, body, origin, captured)
|
||||
return WriteResult(ok=out.ok, action="route", domain=d, receipt=out.receipt,
|
||||
error=out.error, latency_ms=out.latency_ms)
|
||||
|
||||
|
||||
@typechecked
|
||||
async def api_write(domain: str, action: str, params: Dict[str, Any]) -> WriteResult:
|
||||
"""Perform a write via the site's own API using the borrowed session. Times it, and turns any
|
||||
failure (rejected by the site, no session, bad params) into a typed ok=False result rather
|
||||
than raising, so the agent can fall back to the UI path on a miss without a crash."""
|
||||
d = domain.lower().strip().lstrip(".")
|
||||
entry = P_ADAPTERS.get(d)
|
||||
if not entry or action not in entry[0]:
|
||||
return WriteResult(ok=False, action=action, domain=d,
|
||||
error=f"no API-first adapter for {d}/{action}; use the UI path")
|
||||
p_ensure_session_env()
|
||||
dispatch = entry[1]
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
receipt = await asyncio.to_thread(dispatch, action, params)
|
||||
return WriteResult(ok=True, action=action, domain=d,
|
||||
receipt=receipt_str(receipt),
|
||||
latency_ms=int((time.monotonic() - t0) * 1000))
|
||||
except reddit_writes.RedditError as e:
|
||||
# EXPECTED site-side reject (not logged in, rate-limited, bad params): the model sees it and falls back; info, not an alarm. (Future adapters should raise their own recognizable reject type to land here.)
|
||||
logger.info(f"[api-write] {d}/{action} rejected by the site: {e}")
|
||||
return WriteResult(ok=False, action=action, domain=d, error=str(e)[:200],
|
||||
latency_ms=int((time.monotonic() - t0) * 1000))
|
||||
except Exception as e:
|
||||
# UNEXPECTED: the adapter code itself threw (a bug, or the site changed shape). It fails open to the UI, so without this WARNING a systemically-broken fast path is INVISIBLE (looks like the tier just isn't used).
|
||||
logger.warning(f"[api-write] {d}/{action} adapter FAILED unexpectedly (fast path broken here): {e}")
|
||||
return WriteResult(ok=False, action=action, domain=d, error=str(e)[:200],
|
||||
latency_ms=int((time.monotonic() - t0) * 1000))
|
||||
@@ -17,6 +17,8 @@ BROWSER_CMD_TIMEOUTS = {
|
||||
"perform_action": 35.0, # session-borrow shims pack navigate + wait + scrape into ONE command, so it needs more than navigate alone
|
||||
"browser_fetch": 32.0, # offscreen window: load + settle + DOM read on an arbitrary (maybe slow/JS-heavy) page
|
||||
"browser_search": 45.0, # tries up to 3 engines sequentially, each a full load + settle
|
||||
"find_composer": 30.0, # packs trigger + scroll-ladder + retop + open-first into ONE command; on the 15s default the last two tiers were unreachable and heavy pages died mid-ladder (measured: linkedin timed out 2 of 3 runs). The in-page routine self-caps well under this.
|
||||
"import_session": 40.0, # applies the borrowed cookies AND lets a hidden window sit through the site's bot challenge so the card inherits the clearance; warmBorrowedSession.js self-caps at 15+2.5+5s, and this must outlast that or the warm is killed mid-challenge and we throw away the whole point of it (same trap as find_composer).
|
||||
}
|
||||
BROWSER_CMD_REBROADCAST_S = 3.0
|
||||
# A CPU-starved renderer can briefly drop its WS (a missed heartbeat) and the frontend auto-reconnects a beat later; bridge that gap instead of hard-failing a live run into it. Short enough that a genuinely-closed window still fails quickly (and no LLM turns are ever burned waiting); long enough to ride out a reconnect even on a loaded machine.
|
||||
|
||||
@@ -40,6 +40,8 @@ async def run_browser_fast_path(
|
||||
# The fast-path skips the orchestrator, so the UI never gets the BrowserAgent tool-call that draws the "Browser Agent" bubble. Emit a synthetic tool_call/ tool_result pair (same shape + mcp__ name the orchestrator uses) so the bubble shows here too. None until we actually dispatch a browser (a pure READ answer has no browser, so no bubble).
|
||||
p_browser_tool = "mcp__openswarm-browser-agent__CreateBrowserAgent"
|
||||
p_bubble_tid: Optional[str] = None
|
||||
p_action_logs: List[List[Dict[str, object]]] = []
|
||||
p_last_result: Dict[str, object] = {}
|
||||
try:
|
||||
from backend.apps.agents.browser.browser_agent import run_browser_agents
|
||||
from backend.apps.agents.browser import browser_fast_path
|
||||
@@ -48,11 +50,17 @@ async def run_browser_fast_path(
|
||||
if verdict == "read":
|
||||
from backend.apps.agents.browser import browser_fast_read
|
||||
from backend.apps.agents.providers.registry import get_api_type
|
||||
p_read_api = get_api_type(session.model)
|
||||
text = await browser_fast_read.try_fast_read(
|
||||
prompt, brief, load_settings(), get_api_type(session.model),
|
||||
prompt, brief, load_settings(), p_read_api,
|
||||
) or ""
|
||||
if not text:
|
||||
p_fp_path = "read->browser"
|
||||
# The single page couldn't answer it; a multi-source read (a difference, a compare) fans out to N concurrent single-page reads + one reduce, still no browser and still fail-open.
|
||||
from backend.apps.agents.browser import browser_map_reduce_read
|
||||
text = await browser_map_reduce_read.try_map_reduce_read(
|
||||
prompt, load_settings(), p_read_api,
|
||||
) or ""
|
||||
p_fp_path = "read-mapreduce" if text else "read->browser"
|
||||
|
||||
p_entry = browser_fast_path.entry_url_from_brief(brief)
|
||||
if p_entry:
|
||||
@@ -60,16 +68,21 @@ async def run_browser_fast_path(
|
||||
|
||||
@typechecked
|
||||
async def p_dispatch(task_text: str) -> Dict[str, object]:
|
||||
# user_prompt rides along RAW: the composed task's routing brief carries its own quoted strings, which made every real send payload look ambiguous to the send-script (r242/r243)
|
||||
results = await run_browser_agents(
|
||||
tasks=[{"task": task_text, "browser_id": selected[0] if selected else "",
|
||||
"url": "", "entry_url": p_entry}],
|
||||
"url": "", "entry_url": p_entry, "user_prompt": prompt}],
|
||||
model=session.model,
|
||||
dashboard_id=session.dashboard_id,
|
||||
pre_selected_browser_ids=selected,
|
||||
parent_session_id=session_id,
|
||||
)
|
||||
r = results[0] if results else {}
|
||||
return r if isinstance(r, dict) else {"summary": str(r or ""), "action_log": []}
|
||||
r = r if isinstance(r, dict) else {"summary": str(r or ""), "action_log": []}
|
||||
# Keep EVERY dispatch's actions, not just the last: a run that needed a recovery or a
|
||||
# send probe did that work on the user's behalf and the trace has to show it.
|
||||
p_action_logs.append(list(r.get("action_log") or []))
|
||||
return r
|
||||
|
||||
@typechecked
|
||||
def p_summary(r: Dict[str, object]) -> str:
|
||||
@@ -84,6 +97,7 @@ async def run_browser_fast_path(
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id, "message": p_tc.model_dump(mode="json")})
|
||||
first = await p_dispatch(browser_fast_path.compose_task(prompt, brief))
|
||||
p_last_result = first
|
||||
text = p_summary(first)
|
||||
if browser_fast_path.dispatch_failed(first):
|
||||
# Retry only transient failures; a dead dashboard fails the retry identically, so skip it and tell the user instead.
|
||||
@@ -125,8 +139,19 @@ async def run_browser_fast_path(
|
||||
)
|
||||
# Close the synthetic bubble (always, even if the dispatch threw) so it never hangs as "running"; the bubble pairs this result with its call positionally.
|
||||
if p_bubble_tid:
|
||||
# The bubble carries the same auditable record the sub-agent path shows. It used to close
|
||||
# with the literal string "done", so expanding it on this tier revealed nothing.
|
||||
from backend.apps.agents.browser import browser_trace
|
||||
p_trace = browser_trace.build_trace(
|
||||
tier=browser_trace.tier_label(p_fp_path, used_browser=True),
|
||||
action_logs=p_action_logs,
|
||||
receipt=browser_trace.receipt_from(p_last_result),
|
||||
entry_url=p_entry or "",
|
||||
)
|
||||
p_tr = Message(role="tool_result", branch_id=session.active_branch_id,
|
||||
content={"tool_use_id": p_bubble_tid, "tool": p_browser_tool, "text": "done"})
|
||||
content={"tool_use_id": p_bubble_tid, "tool": p_browser_tool,
|
||||
"text": browser_trace.trace_text(p_trace),
|
||||
**browser_trace.trace_payload(p_trace)})
|
||||
session.messages.append(p_tr)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id, "message": p_tr.model_dump(mode="json")})
|
||||
|
||||
@@ -144,7 +144,7 @@ BUILTIN_MODELS: dict[str, list[dict[str, Any]]] = {
|
||||
{"value": "gemini-3.1-flash-lite", "label": "Gemini 3.1 Flash Lite",
|
||||
"context_window": 1_000_000, "router_model_id": "gc/gemini-3.1-flash-lite-preview",
|
||||
"api": "gemini-cli", "subscription_only": True, "reasoning": True},
|
||||
# gemini-3-pro removed 2026-03-09 and gemini-3-flash removed 2026-07-03: gemini-3-flash-preview aged out upstream (API-key route hangs with no fail-fast; only an Antigravity sub still masked it). 3.5-flash / 3.1-flash-lite cover the slots.
|
||||
# gemini-3-pro removed 2026-03-09 and gemini-3-flash removed 2026-07-03 (both rows, independently on two branches): gemini-3-flash-preview aged out upstream (API-key lane hangs/429s with no fail-fast, measured 7-21s; only an Antigravity sub masked it). 3.5-flash / 3.1-flash-lite cover the slots; ag/gemini-3-flash lives on as an aux model, not a picker row.
|
||||
# API-key entries: bypass 9Router, call generativelanguage.googleapis.com.
|
||||
# Gemini 3.6 Flash + 3.5 Flash-Lite (both GA 2026-07-21, changelog-verified ids) are API-key
|
||||
# only for the same reason as 3.5 Flash: the pinned 0.3.60 gc/ registry predates them. No 3.5
|
||||
@@ -287,7 +287,7 @@ def resolve_model_id_for_sdk(short_name: str, settings: AppSettings) -> str:
|
||||
# Gemini lane order: Antigravity OAuth (for the models it serves), then AI Studio apikey, then Gemini CLI. AG bypasses the thoughtSignature validator that breaks multi-step Gemini turns AND supports real reasoning, so a connected AG sub is preferred over the AI Studio key, which otherwise silently shadowed it. The map is AG's allowlist; pro variants 404/400 on AG and are deliberately absent, so they fall through to the key.
|
||||
P_ANTIGRAVITY_MAP = {
|
||||
# gemini-3-pro-preview disabled: AG returns 404 even with active conn. gemini-3.1-pro-preview disabled: AG's `gemini-3.1-pro-high` variant 400s every request with "invalid argument" (the `-high` thinking- budget alias on AG requires a thinking_config the CLI doesn't emit). Falls through to the AI Studio key / gc/ instead. gemini-3-flash-preview key dropped with its registry entry (aged out upstream).
|
||||
"gemini-3.1-flash-lite-preview": "gemini-3-flash",
|
||||
"gemini-3.1-flash-lite-preview": "gemini-3-flash", # 3.1-flash-lite has no AG variant, so AG serves it via gemini-3-flash
|
||||
}
|
||||
if entry.get("api") == "gemini-cli":
|
||||
rid = entry.get("router_model_id", "")
|
||||
@@ -426,7 +426,6 @@ COST_PER_1M_TOKENS: dict[tuple[str, str], tuple[float, float]] = {
|
||||
# Google; Gemini CLI subscription path, user pays nothing per token
|
||||
("Google", "gemini-3.5-flash"): (0.0, 0.0),
|
||||
("Google", "gemini-3.1-flash-lite"): (0.0, 0.0),
|
||||
("Google", "gemini-3-flash"): (0.0, 0.0),
|
||||
("Google", "gemini-2.5-pro"): (0.0, 0.0),
|
||||
("Google", "gemini-2.5-flash"): (0.0, 0.0),
|
||||
# OpenRouter-backed (approximate)
|
||||
|
||||
@@ -7,6 +7,7 @@ Talks to the already-running 9Router over HTTP; never spawns the subprocess
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
from typing import Optional
|
||||
|
||||
import httpx
|
||||
|
||||
@@ -211,7 +212,7 @@ async def p_start_codex_callback_listener(timeout: float = 300.0) -> int | None:
|
||||
return bound_port
|
||||
|
||||
|
||||
# Providers whose OAuth flow MUST run in the user's real browser via shell.openExternal, not the in-Electron window.open popup: - gemini-cli, antigravity: Google's Embedded WebView Restrictions policy uses JS-fingerprint detection that no UA spoof defeats. RFC 8252 and Google's own Desktop-app OAuth guidance both prescribe the system browser. - codex: auth.openai.com renders blank in our popup on some machines (newer embed detection + regional checks); system browser surfaces the real error. - claude: email magic-link opens in the user's default browser, which is a different cookie jar from the embedded popup, so the popup can never receive the auth. Forcing the OAuth flow into the system browser keeps everything in one cookie jar. The callback for gemini-cli/antigravity lands on /api/subscriptions/callback and runs the exchange server-side; codex uses its fixed 1455 listener; claude is special-cased in p_callback_uri_for_provider below.
|
||||
# Providers whose OAuth flow MUST run in the user's real browser via shell.openExternal, not the in-Electron window.open popup: - gemini-cli, antigravity: Google's Embedded WebView Restrictions policy uses JS-fingerprint detection that no UA spoof defeats. RFC 8252 and Google's own Desktop-app OAuth guidance both prescribe the system browser. - codex: auth.openai.com renders blank in our popup on some machines (newer embed detection + regional checks); system browser surfaces the real error. - claude: email magic-link opens in the user's default browser, which is a different cookie jar from the embedded popup, so the popup can never receive the auth. Forcing the OAuth flow into the system browser keeps everything in one cookie jar. The callback for gemini-cli/antigravity lands on /api/subscriptions/callback and runs the exchange server-side; codex uses its fixed 1455 listener; claude is special-cased in callback_uri_for_provider below.
|
||||
P_EXTERNAL_BROWSER_PROVIDERS: set[str] = {"gemini-cli", "antigravity", "codex", "claude"}
|
||||
|
||||
|
||||
@@ -219,21 +220,32 @@ def p_should_use_external_browser(provider: str) -> bool:
|
||||
return provider in P_EXTERNAL_BROWSER_PROVIDERS
|
||||
|
||||
|
||||
def p_backend_port() -> int:
|
||||
"""Best-effort lookup of the OpenSwarm backend HTTP port.
|
||||
def resolve_backend_port(observed: Optional[int] = None) -> int:
|
||||
"""The port this backend is actually reachable on, for building OAuth redirect URIs.
|
||||
|
||||
Falls back to 8324 (the default in backend/main.py) if OPENSWARM_PORT
|
||||
hasn't been set yet. backend/main.py:239 sets this env var at startup
|
||||
before any request handler runs, so `start_oauth` will always see the
|
||||
correct value.
|
||||
OPENSWARM_PORT is authoritative and Electron always passes it (main.js), so packaged builds
|
||||
take the first branch and behave exactly as before.
|
||||
|
||||
It is NOT always set in dev. main.py only exports it inside its `if __name__ == "__main__"`
|
||||
block, which never runs under `python -m uvicorn backend.main:app --port N`. The old code then
|
||||
assumed 8324 and stamped that into the redirect URI while uvicorn served a different port, so
|
||||
Google bounced the user to a dead port and Claude's callback missed the router rewrite. Codex
|
||||
kept working throughout, because OpenAI pins its own localhost:1455 listener, which is what
|
||||
made the failure look like "two providers are broken" instead of "the port is wrong".
|
||||
|
||||
`observed` is the port the caller was actually reached on (from the live request), which is
|
||||
ground truth on every launch path. Only consulted when the env var is absent.
|
||||
"""
|
||||
try:
|
||||
return int(os.environ.get("OPENSWARM_PORT", "8324"))
|
||||
except (TypeError, ValueError):
|
||||
return 8324
|
||||
raw = os.environ.get("OPENSWARM_PORT")
|
||||
if raw:
|
||||
try:
|
||||
return int(raw)
|
||||
except (TypeError, ValueError):
|
||||
pass
|
||||
return observed or 8324
|
||||
|
||||
|
||||
def p_callback_uri_for_provider(provider: str) -> str:
|
||||
def callback_uri_for_provider(provider: str, backend_port: Optional[int] = None) -> str:
|
||||
"""Return the redirect URI to pass to 9Router's authorize endpoint.
|
||||
|
||||
Most providers accept 9Router's built-in callback page at port 20128.
|
||||
@@ -252,15 +264,18 @@ def p_callback_uri_for_provider(provider: str) -> str:
|
||||
if provider == "claude":
|
||||
return f"http://localhost:{NINE_ROUTER_PORT}/callback"
|
||||
if provider in P_EXTERNAL_BROWSER_PROVIDERS:
|
||||
return f"http://localhost:{p_backend_port()}/api/subscriptions/callback"
|
||||
return f"http://localhost:{resolve_backend_port(backend_port)}/api/subscriptions/callback"
|
||||
return f"http://localhost:{NINE_ROUTER_PORT}/callback"
|
||||
|
||||
|
||||
async def start_oauth(provider: str) -> dict:
|
||||
async def start_oauth(provider: str, backend_port: Optional[int] = None) -> dict:
|
||||
"""Start OAuth flow for a provider.
|
||||
|
||||
For device_code providers (github, qwen, kiro): returns {user_code, verification_uri, device_code}
|
||||
For authorization_code providers (claude, codex, gemini-cli): returns {authUrl, codeVerifier, state}
|
||||
|
||||
`backend_port` is the port the connect request arrived on; it only matters when OPENSWARM_PORT
|
||||
is unset, which is the dev-launch case that used to send Google to a dead port.
|
||||
"""
|
||||
async with httpx.AsyncClient(timeout=15.0, headers=cli_auth_headers()) as client:
|
||||
try:
|
||||
@@ -278,7 +293,7 @@ async def start_oauth(provider: str) -> dict:
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
callback_url = p_callback_uri_for_provider(provider)
|
||||
callback_url = callback_uri_for_provider(provider, backend_port)
|
||||
if provider == "codex":
|
||||
# Codex's redirect must be an OpenAI allow-listed loopback port; bind the first free one (1455 else 1457) and use ITS redirect_uri so authorize + token exchange agree.
|
||||
bound_port = await p_start_codex_callback_listener()
|
||||
|
||||
@@ -64,7 +64,7 @@ p_key_cache: Dict[str, Optional[bytes]] = {}
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_win_dpapi_unprotect(data: bytes) -> Optional[bytes]:
|
||||
def win_dpapi_unprotect(data: bytes) -> Optional[bytes]:
|
||||
"""CryptUnprotectData via crypt32.dll (no pywin32 dependency). None on any failure."""
|
||||
if sys.platform != "win32":
|
||||
return None
|
||||
@@ -105,7 +105,7 @@ def win_storage_key(browser: str) -> Optional[bytes]:
|
||||
raw = base64.b64decode(enc_b64)
|
||||
if raw[:5] != b"DPAPI":
|
||||
return None
|
||||
return p_win_dpapi_unprotect(raw[5:])
|
||||
return win_dpapi_unprotect(raw[5:])
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
@@ -203,6 +203,12 @@ def decrypt_cookie_value(enc: bytes, key: bytes) -> Optional[str]:
|
||||
return None
|
||||
|
||||
|
||||
@typechecked
|
||||
def has_store(domain: str) -> bool:
|
||||
"""Whether any browser store holds records for `domain`, without decrypting and without touching the keychain. The public shape of the presence check, so callers outside this file never need the store tuple."""
|
||||
return p_best_store(domain) is not None
|
||||
|
||||
|
||||
@typechecked
|
||||
def read_provider_cookies(domain: str) -> Dict[str, str]:
|
||||
"""Decrypted cookie jar for `domain`, from whichever browser store actually has the session. At most one keychain touch (that store's browser), cached for the process."""
|
||||
@@ -239,7 +245,7 @@ def read_provider_cookies(domain: str) -> Dict[str, str]:
|
||||
|
||||
@typechecked
|
||||
def read_provider_cookie_records(domain: str) -> List[Dict[str, Any]]:
|
||||
"""Full cookie records ({name,value,domain,path,secure,httponly}) for `domain`, so Electron's offscreen browser can re-inject the session faithfully and pass Cloudflare with a real Chrome TLS handshake. Same one-store, one-keychain-touch path as read_provider_cookies."""
|
||||
"""Full cookie records ({name,value,domain,path,secure,httponly,expires_utc}) for `domain`, so Electron's offscreen browser can re-inject the session faithfully and pass Cloudflare with a real Chrome TLS handshake. Same one-store, one-keychain-touch path as read_provider_cookies. `expires_utc` stays in Chromium's own units (microseconds since 1601, 0 = session cookie); whoever needs unix seconds converts."""
|
||||
store = p_best_store(domain)
|
||||
if store is None:
|
||||
return []
|
||||
@@ -254,10 +260,11 @@ def read_provider_cookie_records(domain: str) -> List[Dict[str, Any]]:
|
||||
con = sqlite3.connect(f"file:{tmp}?mode=ro", uri=True)
|
||||
cur = con.cursor()
|
||||
cur.execute(
|
||||
"SELECT name, encrypted_value, host_key, path, is_secure, is_httponly FROM cookies WHERE host_key LIKE ?",
|
||||
"SELECT name, encrypted_value, host_key, path, is_secure, is_httponly, expires_utc "
|
||||
"FROM cookies WHERE host_key LIKE ?",
|
||||
(f"%{domain}",),
|
||||
)
|
||||
for name, enc, host_key, path, is_secure, is_httponly in cur.fetchall():
|
||||
for name, enc, host_key, path, is_secure, is_httponly, expires_utc in cur.fetchall():
|
||||
if not enc:
|
||||
continue
|
||||
val = decrypt_cookie_value(bytes(enc), key)
|
||||
@@ -266,6 +273,7 @@ def read_provider_cookie_records(domain: str) -> List[Dict[str, Any]]:
|
||||
records.append({
|
||||
"name": str(name), "value": val, "domain": str(host_key),
|
||||
"path": str(path) or "/", "secure": bool(is_secure), "httponly": bool(is_httponly),
|
||||
"expires_utc": int(expires_utc or 0),
|
||||
})
|
||||
con.close()
|
||||
except Exception:
|
||||
@@ -297,3 +305,13 @@ def read_google_session_records() -> List[Dict[str, Any]]:
|
||||
@typechecked
|
||||
def cookie_header(jar: Dict[str, str]) -> str:
|
||||
return "; ".join(f"{k}={v}" for k, v in jar.items())
|
||||
|
||||
|
||||
@typechecked
|
||||
def logged_in_providers() -> List[str]:
|
||||
"""Which providers have a readable session, WITHOUT decrypting or touching the keychain: safe for a UI presence check."""
|
||||
out: List[str] = []
|
||||
for provider, domain in (("codex", "chatgpt.com"), ("claude", "claude.ai"), ("gemini", "gemini.google.com")):
|
||||
if p_best_store(domain) is not None:
|
||||
out.append(provider)
|
||||
return out
|
||||
|
||||
@@ -4,8 +4,18 @@ Posts, comments, edits, deletes, votes, saves, subscriptions, and DMs, all via
|
||||
the user's own session. Each call goes through the rate limiter's write buckets.
|
||||
"""
|
||||
|
||||
import json
|
||||
import re
|
||||
from typing import Any, Dict, Optional
|
||||
|
||||
from backend.apps.reddit_mcp_shim.reddit_http import RedditError, api
|
||||
|
||||
# Reddit's own id shapes echoed back in a write response: a "fullname" (t1_ comment,
|
||||
# t3_ post, ...) and a comment/post permalink. Used to recover a real receipt when the
|
||||
# structured envelope is absent (the legacy "jquery" response shape, see p_receipt).
|
||||
P_FULLNAME_RE = re.compile(r"t[1-6]_[0-9a-z]+", re.I)
|
||||
P_PERMALINK_RE = re.compile(r"/r/[A-Za-z0-9_]+/comments/[A-Za-z0-9_/\-]+")
|
||||
|
||||
|
||||
def p_check(resp: dict) -> dict:
|
||||
"""Raise on Reddit's json.errors envelope; return the inner data otherwise."""
|
||||
@@ -16,6 +26,32 @@ def p_check(resp: dict) -> dict:
|
||||
return j.get("data", {}) if isinstance(j, dict) else {}
|
||||
|
||||
|
||||
def p_receipt(resp: Any, kind: str, exclude: str = "") -> Dict[str, Optional[str]]:
|
||||
"""The just-created thing's own fullname + permalink, robust to Reddit's TWO write
|
||||
response shapes. Modern api_type=json returns data.things[0].data (comment) or the
|
||||
fields at data top-level (submit); the LEGACY web endpoint returns a 'jquery' command
|
||||
array with neither, so the naive parse came back empty = the receipt='ok' bug. Prefer
|
||||
the structured field; else scan the echoed response for a fullname of the right kind
|
||||
(t1 comment / t3 post), never the parent id we're replying to."""
|
||||
data = p_check(resp) # raises on Reddit's real error envelope
|
||||
d: Dict[str, Any] = {}
|
||||
if isinstance(data, dict):
|
||||
things = data.get("things")
|
||||
if things and isinstance(things[0], dict):
|
||||
d = things[0].get("data", {}) or {}
|
||||
elif data.get("name") or data.get("id") or data.get("url"):
|
||||
d = data
|
||||
if d.get("name") or d.get("permalink") or d.get("url"):
|
||||
return {"id": d.get("name") or d.get("id"),
|
||||
"permalink": d.get("permalink") or d.get("url")}
|
||||
blob = json.dumps(resp, default=str)
|
||||
ex = (exclude or "").lower()
|
||||
name = next((m for m in P_FULLNAME_RE.findall(blob)
|
||||
if m.lower().startswith(kind.lower()) and m.lower() != ex), None)
|
||||
pm = P_PERMALINK_RE.search(blob)
|
||||
return {"id": name, "permalink": pm.group(0) if pm else None}
|
||||
|
||||
|
||||
def p_dir(direction: str) -> int:
|
||||
return {"up": 1, "upvote": 1, "down": -1, "downvote": -1, "clear": 0, "none": 0, "unvote": 0}.get(
|
||||
(direction or "").lower(), 0
|
||||
@@ -34,22 +70,20 @@ def submit(subreddit: str, title: str, kind: str, text: str, url: str, nsfw: boo
|
||||
"api_type": "json",
|
||||
}
|
||||
form["url" if kind == "link" else "text"] = url if kind == "link" else text
|
||||
data = p_check(api("POST", "/api/submit", form=form, action="submit"))
|
||||
return {"id": data.get("name") or data.get("id"), "url": data.get("url")}
|
||||
r = p_receipt(api("POST", "/api/submit", form=form, action="submit"), kind="t3")
|
||||
return {"id": r["id"], "url": r["permalink"]}
|
||||
|
||||
|
||||
def comment(parent_id: str, text: str) -> dict:
|
||||
data = p_check(api("POST", "/api/comment", form={"thing_id": parent_id, "text": text, "api_type": "json"}, action="comment"))
|
||||
things = data.get("things", [])
|
||||
new = things[0].get("data", {}) if things else {}
|
||||
return {"id": new.get("name"), "permalink": new.get("permalink")}
|
||||
resp = api("POST", "/api/comment", form={"thing_id": parent_id, "text": text, "api_type": "json"}, action="comment")
|
||||
r = p_receipt(resp, kind="t1", exclude=parent_id)
|
||||
return {"id": r["id"], "permalink": r["permalink"]}
|
||||
|
||||
|
||||
def edit(thing_id: str, text: str) -> dict:
|
||||
data = p_check(api("POST", "/api/editusertext", form={"thing_id": thing_id, "text": text, "api_type": "json"}, action="comment"))
|
||||
things = data.get("things", [])
|
||||
new = things[0].get("data", {}) if things else {}
|
||||
return {"id": new.get("name") or thing_id, "edited": True}
|
||||
resp = api("POST", "/api/editusertext", form={"thing_id": thing_id, "text": text, "api_type": "json"}, action="comment")
|
||||
r = p_receipt(resp, kind=(thing_id[:2] or "t1"), exclude="")
|
||||
return {"id": r["id"] or thing_id, "edited": True}
|
||||
|
||||
|
||||
def delete(thing_id: str) -> dict:
|
||||
|
||||
@@ -54,6 +54,10 @@ class AppSettings(BaseModel):
|
||||
voice_hold_to_talk: bool = True
|
||||
anthropic_api_key: Optional[str] = None
|
||||
browser_homepage: str = "https://www.google.com"
|
||||
# Opt-in: let a blocked browser agent borrow the sign-in you already have in your everyday
|
||||
# browser instead of stopping to ask you to log in again. Default OFF because reading your real
|
||||
# browser's session is your decision to make once, explicitly, not ours to assume.
|
||||
browser_import_signins: bool = False
|
||||
openai_api_key: Optional[str] = None
|
||||
google_api_key: Optional[str] = None
|
||||
openrouter_api_key: Optional[str] = None
|
||||
|
||||
@@ -32,6 +32,13 @@ def _isolate_browser_state(monkeypatch):
|
||||
monkeypatch.setenv("OPENSWARM_BROWSER_SKILLS_DIR", skills_dir)
|
||||
monkeypatch.setenv("OPENSWARM_BROWSER_METRICS_DIR", metrics_dir)
|
||||
monkeypatch.setenv("OPENSWARM_BROWSER_PLAYBOOK_DIR", playbook_dir)
|
||||
# The speed levers are default-ON in prod; pin them off for the suite so mocked loop tests keep exact aux-call/turn expectations (same pattern as OPENSWARM_PERSISTENT_CLIENT). The levers are exercised by their own live gates + targeted tests that set the flag explicitly.
|
||||
monkeypatch.setenv("OSW_PRESTAGE", "0")
|
||||
monkeypatch.setenv("OSW_FASTREAD_HOP", "0")
|
||||
monkeypatch.setenv("OSW_PRELUDE_TRIM", "0")
|
||||
monkeypatch.setenv("OSW_DEADCARD_EVICT", "0")
|
||||
monkeypatch.setenv("OSW_RECEIPT_DONE", "0")
|
||||
monkeypatch.setenv("OSW_SEND_SCRIPT", "0")
|
||||
|
||||
def _reset():
|
||||
for mod in ("browser_skills", "browser_playbook"):
|
||||
|
||||
@@ -30,17 +30,32 @@ class Resp:
|
||||
self.usage = type("U", (), {"input_tokens": 1, "output_tokens": 1})()
|
||||
|
||||
|
||||
class FakeStream:
|
||||
# mirrors anthropic's messages.stream(): async CM whose get_final_message() returns the turn
|
||||
def __init__(self, resp): self.resp = resp
|
||||
async def __aenter__(self): return self
|
||||
async def __aexit__(self, *a): return False
|
||||
async def get_final_message(self): return self.resp
|
||||
|
||||
|
||||
class FakeLLM:
|
||||
def __init__(self, scripted):
|
||||
self.scripted = scripted; self.turn = 0; self.calls = []
|
||||
self.messages = self
|
||||
|
||||
async def create(self, **kw):
|
||||
def p_next(self, kw):
|
||||
self.calls.append(kw)
|
||||
i = min(self.turn, len(self.scripted) - 1)
|
||||
self.turn += 1
|
||||
return self.scripted[i]
|
||||
|
||||
async def create(self, **kw):
|
||||
return self.p_next(kw)
|
||||
|
||||
def stream(self, **kw):
|
||||
# the loop now streams; return an async-CM yielding the scripted turn
|
||||
return FakeStream(self.p_next(kw))
|
||||
|
||||
|
||||
class FakeAux:
|
||||
def __init__(self):
|
||||
@@ -63,6 +78,17 @@ def p_rp(goal, mem="Share dialog is a cross-origin iframe; use the index list.")
|
||||
DOC_URL = "https://docs.google.com/document/d/abc/edit"
|
||||
|
||||
|
||||
def p_run_settled(**kw):
|
||||
"""run_browser_agent then drain the backgrounded learning task; the distill
|
||||
no longer blocks the reply path, so tests asserting its effects must settle it."""
|
||||
async def p_go():
|
||||
r = await BA.run_browser_agent(**kw)
|
||||
if BA.learn_tasks:
|
||||
await asyncio.gather(*list(BA.learn_tasks), return_exceptions=True)
|
||||
return r
|
||||
return asyncio.run(p_go())
|
||||
|
||||
|
||||
def p_install(monkeypatch, primary, aux):
|
||||
# local imports inside run_browser_agent resolve from these source modules
|
||||
import backend.apps.settings.settings as settings_mod
|
||||
@@ -71,10 +97,11 @@ def p_install(monkeypatch, primary, aux):
|
||||
import backend.apps.agents.agent_manager as am_mod
|
||||
|
||||
monkeypatch.setattr(settings_mod, "load_settings", lambda: {"fake": True}, raising=True)
|
||||
monkeypatch.setattr(reg_mod, "find_builtin_model", lambda m: object(), raising=True)
|
||||
# a dict (not object()) so get_api_type's (entry or {}).get("api") works like the real registry rows
|
||||
monkeypatch.setattr(reg_mod, "find_builtin_model", lambda m: {"api": "anthropic"}, raising=True)
|
||||
monkeypatch.setattr(reg_mod, "resolve_model_id_for_sdk", lambda m, s: "primary-x", raising=True)
|
||||
|
||||
async def p_aux_resolve(s, preferred_tier="haiku"):
|
||||
async def p_aux_resolve(s, preferred_tier="haiku", primary_api=None):
|
||||
return ("aux-x", None)
|
||||
monkeypatch.setattr(reg_mod, "resolve_aux_model", p_aux_resolve, raising=True)
|
||||
|
||||
@@ -222,10 +249,15 @@ def test_confirmed_send_ends_the_run_instead_of_stalling(monkeypatch):
|
||||
# the send ran and the run ended FAST (the stall guard stopped it), well before consuming all 8 scripted stall turns
|
||||
assert any(c["action"] == "click_index" and c["params"].get("index") == 99 for c in sent)
|
||||
assert primary.turn <= 4, f"run stalled {primary.turn} turns after a confirmed send"
|
||||
# structured success + a clean human summary, never the internal tag
|
||||
assert result.get("done") is True
|
||||
# A clean human summary, never the internal tag. NOT `done is True`: this run only ever saw the
|
||||
# click register, and no composer receipt ever arrived, so it has no evidence the message
|
||||
# landed. Reporting success here is the exact live failure measured on X 2026-07-28 ("your
|
||||
# message went through and it's showing" while nothing had posted). The stall guard's job is to
|
||||
# stop the spinning, not to bless the outcome, so what is asserted here is that it ENDED, and
|
||||
# ended honestly. See test_browser_send_honesty.py.
|
||||
assert "OUTCOME" not in result["summary"]
|
||||
assert result["summary"].strip()
|
||||
assert result.get("done") is False, "an unverified send must not report success"
|
||||
|
||||
|
||||
def test_done_tool_delivers_a_clean_human_summary(monkeypatch):
|
||||
@@ -938,9 +970,9 @@ def test_playbook_distills_on_success_survives_restart_and_seeds_next_run(monkey
|
||||
])
|
||||
pbaux = PBAux()
|
||||
p_install(monkeypatch, primary1, pbaux)
|
||||
asyncio.run(BA.run_browser_agent(
|
||||
p_run_settled(
|
||||
task="find design engineers", browser_id="b1", model="sonnet", initial_url=DOC_URL,
|
||||
))
|
||||
)
|
||||
assert pbaux.calls >= 1, "a substantive success must trigger the distill aux call"
|
||||
assert PB.get_playbook("docs.google.com"), "playbook recorded for the host"
|
||||
|
||||
@@ -995,7 +1027,7 @@ def test_ambient_memory_signals_fire_calmly(monkeypatch):
|
||||
# Run 1: nothing learned yet -> NO recall line, but it learns -> closing line.
|
||||
p_install(monkeypatch, p_run(), PBAux())
|
||||
monkeypatch.setattr(BA.ws_manager, "send_to_session", p_cap, raising=False)
|
||||
asyncio.run(BA.run_browser_agent(task="find engineers", browser_id="b1", model="sonnet", initial_url=DOC_URL))
|
||||
p_run_settled(task="find engineers", browser_id="b1", model="sonnet", initial_url=DOC_URL)
|
||||
joined1 = " ".join(msgs)
|
||||
assert "Picking up what I learned" not in joined1, "no recall on the first-ever visit"
|
||||
assert "so I'm faster here next time" in joined1, "closing 'learned' line after first success"
|
||||
@@ -1036,8 +1068,8 @@ def test_playbook_not_learned_from_a_ghost_completion(monkeypatch):
|
||||
asyncio.run(BA.run_browser_agent(
|
||||
task="do the thing", browser_id="b1", model="sonnet", initial_url=DOC_URL,
|
||||
))
|
||||
# the only aux call allowed here is the stuck-adjudication; the playbook distill must NOT have stored anything for a dishonest run
|
||||
assert PB.get_playbook("docs.google.com") == []
|
||||
# the only aux call allowed here is the stuck-adjudication; the playbook distill must NOT have LEARNED anything for a dishonest run (load = learned-only; get_playbook would also return the shipped seed for this host)
|
||||
assert PB.load("docs.google.com") == []
|
||||
|
||||
|
||||
def test_batch_replay_runs_a_read_loop_for_all_values(monkeypatch):
|
||||
@@ -1322,12 +1354,13 @@ def test_post_action_state_truncates_long_lists(monkeypatch):
|
||||
from backend.apps.agents.browser import browser_agent as ba
|
||||
calls = []
|
||||
monkeypatch.setattr(ba.browser_wait, "smart_wait", p_fake_settle(calls))
|
||||
long_list = "\n".join(f'[{i}]<button "b{i}">' for i in range(60))
|
||||
# cap is 60 (matches the frontend list cap); truncation only kicks in past that
|
||||
long_list = "\n".join(f'[{i}]<button "b{i}">' for i in range(80))
|
||||
out = asyncio.run(ba.post_action_state(
|
||||
"BrowserType", {"selector": "#q", "text": "hi"}, {"text": "Typed"},
|
||||
"b1", "", p_fake_exec(calls, long_list), "",
|
||||
))
|
||||
assert "(+25 more rows" in out and '[34]<button "b34">' in out and '[35]' not in out
|
||||
assert "(+20 more rows" in out and '[59]<button "b59">' in out and '[60]' not in out
|
||||
|
||||
|
||||
def test_post_action_state_hung_settle_attaches_nothing(monkeypatch):
|
||||
@@ -1486,6 +1519,32 @@ def test_composer_fill_detection():
|
||||
assert not is_composer_fill("BrowserScroll", {})
|
||||
|
||||
|
||||
def test_compose_send_confirmation_model_voice_with_safe_fallback():
|
||||
# The done line is model-written (aux), but validated: a clean sentence is used as-is; tool-ish
|
||||
# / JSON / URL output is rejected so the caller falls back to a template (never leaks machinery).
|
||||
import asyncio
|
||||
from backend.apps.agents.browser.browser_agent import compose_send_confirmation
|
||||
|
||||
class Blk2:
|
||||
def __init__(self, text): self.type = "text"; self.text = text
|
||||
class Resp2:
|
||||
def __init__(self, text): self.content = [Blk2(text)]
|
||||
class Aux:
|
||||
def __init__(self, text): self.txt = text; self.messages = self
|
||||
async def create(self, **kw): return Resp2(self.txt)
|
||||
|
||||
def run(a): return asyncio.run(a)
|
||||
# clean natural sentence -> used verbatim
|
||||
assert run(compose_send_confirmation(Aux("Done, I messaged Tyler and said hi."), "m", "say hi", "hi")) \
|
||||
== "Done, I messaged Tyler and said hi."
|
||||
# tool-ish / JSON / url -> rejected (empty) so caller templates
|
||||
assert run(compose_send_confirmation(Aux("Try BrowserClickIndex then list."), "m", "t", "hi")) == ""
|
||||
assert run(compose_send_confirmation(Aux('{"done": true}'), "m", "t", "hi")) == ""
|
||||
# no aux / no payload -> empty (fail-open)
|
||||
assert run(compose_send_confirmation(None, "m", "t", "hi")) == ""
|
||||
assert run(compose_send_confirmation(Aux("Done!"), "m", "t", "")) == ""
|
||||
|
||||
|
||||
def test_send_index_handoff_points_only_at_a_real_send_button():
|
||||
# after a composer fill we hand the model the Send button's index so it clicks it directly instead of hunting; must never mistake an upsell/profile link for it
|
||||
from backend.apps.agents.browser.browser_agent import send_index_in_state
|
||||
@@ -1496,6 +1555,39 @@ def test_send_index_handoff_points_only_at_a_real_send_button():
|
||||
assert send_index_in_state("") is None
|
||||
|
||||
|
||||
def test_send_submit_matcher_broad_but_hint_matcher_tight():
|
||||
# SCOPING: the send-script's submit finder must know Post/Reply/Tweet/etc so the fast path
|
||||
# COMPLETES on the giants; the ALWAYS-ON model hint must STAY tight so it never mislabels a
|
||||
# stray feed 'Reply'/'Share'/'Comment' button as the Send button after an unrelated fill.
|
||||
from backend.apps.agents.browser.browser_agent import send_index_in_state, send_submit_index_in_state
|
||||
# broad (send-script) finds the popular composers' submit buttons
|
||||
assert send_submit_index_in_state('[3]<textbox "Post your reply">\n[8]<button "Reply">') == (8, "Reply")
|
||||
assert send_submit_index_in_state('[2]<textbox "What is happening?">\n[9]<button "Post">') == (9, "Post")
|
||||
assert send_submit_index_in_state('[4]<button "Tweet">') == (4, "Tweet")
|
||||
# exact + button-only keeps its own safety
|
||||
assert send_submit_index_in_state('[5]<button "Post a job">') is None
|
||||
assert send_submit_index_in_state('[6]<menuitem "Share">') is None
|
||||
# TIGHT hint matcher: Send family only, and NOT the common feed buttons (the regression guard)
|
||||
assert send_index_in_state('[44]<button "Send">') == (44, "Send")
|
||||
assert send_index_in_state('[8]<button "Reply">') is None
|
||||
assert send_index_in_state('[9]<button "Post">') is None
|
||||
assert send_index_in_state('[7]<button "Comment">') is None
|
||||
assert send_index_in_state('[3]<button "Share">') is None
|
||||
|
||||
|
||||
def test_send_submit_scoped_below_composer():
|
||||
# X ships its sidebar compose OPENER as button "Post" ABOVE the composer; picking it posts
|
||||
# nothing (live 0/2). after_index scopes the scan to buttons BELOW the filled composer.
|
||||
from backend.apps.agents.browser.browser_agent import send_submit_index_in_state
|
||||
x_home = '[25]<button "Post">\n[35]<textbox "Post text" value="check two">\n[58]<button "Post">'
|
||||
assert send_submit_index_in_state(x_home, 35) == (58, "Post")
|
||||
# no submit below the composer = None (falls to by-name, never the opener above)
|
||||
opener_only = '[25]<button "Post">\n[35]<textbox "Post text" value="check two">'
|
||||
assert send_submit_index_in_state(opener_only, 35) is None
|
||||
# unscoped callers keep the old first-match behavior
|
||||
assert send_submit_index_in_state(x_home) == (25, "Post")
|
||||
|
||||
|
||||
def test_strip_lone_surrogates():
|
||||
from backend.apps.agents.browser.browser_agent import strip_lone_surrogates, format_tool_result
|
||||
# an orphan UTF-16 surrogate (half an emoji from the webview) is what crashes the turn at .encode('utf-8'); it must be swapped, not carried through
|
||||
@@ -1524,7 +1616,7 @@ def test_transient_429_retries_then_succeeds(monkeypatch):
|
||||
super().__init__(scripted)
|
||||
self.failures = failures
|
||||
|
||||
async def create(self, **kw):
|
||||
def stream(self, **kw):
|
||||
if self.failures > 0:
|
||||
self.failures -= 1
|
||||
self.calls.append(kw)
|
||||
@@ -1532,7 +1624,7 @@ def test_transient_429_retries_then_succeeds(monkeypatch):
|
||||
"Error code: 429 - {'type': 'error', 'error': {'type': 'free_pool_busy', "
|
||||
"'message': \"OpenSwarm's free pool is busy right now.\"}}"
|
||||
)
|
||||
return await super().create(**kw)
|
||||
return super().stream(**kw)
|
||||
|
||||
primary = FlakyLLM([Resp([Blk("text", "All done.")], stop_reason="end_turn")], failures=2)
|
||||
aux = FakeAux()
|
||||
@@ -1546,7 +1638,7 @@ def test_free_trial_exhausted_gets_friendly_summary(monkeypatch):
|
||||
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
|
||||
|
||||
class DeadLLM(FakeLLM):
|
||||
async def create(self, **kw):
|
||||
def stream(self, **kw):
|
||||
raise Exception(
|
||||
"Error code: 402 - {'type': 'error', 'error': {'type': 'free_trial_exhausted', "
|
||||
"'message': \"You've used all your free runs.\"}}"
|
||||
@@ -1566,7 +1658,7 @@ def test_capacity_budget_exhausted_gets_friendly_summary(monkeypatch):
|
||||
monkeypatch.setattr(EC, "CAPACITY_BACKOFFS", [0], raising=True)
|
||||
|
||||
class Busy429LLM(FakeLLM):
|
||||
async def create(self, **kw):
|
||||
def stream(self, **kw):
|
||||
raise Exception(
|
||||
"Error code: 429 - {'type': 'error', 'error': {'type': 'free_pool_busy', "
|
||||
"'message': \"OpenSwarm's free pool is busy right now.\"}}"
|
||||
@@ -1578,3 +1670,212 @@ def test_capacity_budget_exhausted_gets_friendly_summary(monkeypatch):
|
||||
result = asyncio.run(BA.run_browser_agent(task="check the page", browser_id="b1", model="sonnet"))
|
||||
assert "at capacity right now" in result["summary"]
|
||||
assert "free_pool_busy" not in result["summary"]
|
||||
def test_loop_tier_pin_flag_overrides_model_failsafe(monkeypatch):
|
||||
# V7: OPENSWARM_BROWSER_LOOP_TIER pins the loop to a fast-capable tier (provider-agnostic),
|
||||
# default off inherits the parent model, and a resolver failure falls back to the inherited id.
|
||||
import backend.apps.settings.credentials as cred_mod
|
||||
import backend.apps.agents.providers.registry as reg_mod
|
||||
BH.BROWSER_HISTORY.clear()
|
||||
primary = FakeLLM([Resp([Blk("text", "done")], stop_reason="end_turn")] * 5)
|
||||
p_install(monkeypatch, primary, FakeAux())
|
||||
|
||||
async def p_tier_resolve(s, preferred_tier="haiku", primary_api=None):
|
||||
return (f"pinned-{preferred_tier}", None)
|
||||
monkeypatch.setattr(reg_mod, "resolve_aux_model", p_tier_resolve, raising=True)
|
||||
# same client whatever the id, so the loop runs and we can read which model it used
|
||||
monkeypatch.setattr(cred_mod, "get_anthropic_client_for_model", lambda s, m: primary, raising=True)
|
||||
|
||||
# default OFF: inherit the parent's resolved id
|
||||
monkeypatch.delenv("OPENSWARM_BROWSER_LOOP_TIER", raising=False)
|
||||
asyncio.run(BA.run_browser_agent(task="t", browser_id="b1", model="opus", initial_url=None))
|
||||
assert primary.calls[-1]["model"] == "primary-x"
|
||||
|
||||
# flag ON: pin to the fast-capable tier
|
||||
primary.turn = 0
|
||||
monkeypatch.setenv("OPENSWARM_BROWSER_LOOP_TIER", "sonnet")
|
||||
asyncio.run(BA.run_browser_agent(task="t", browser_id="b2", model="opus", initial_url=None))
|
||||
assert primary.calls[-1]["model"] == "pinned-sonnet"
|
||||
|
||||
# fail-safe: resolver raises -> keep the inherited id, never break the run
|
||||
async def p_boom(s, preferred_tier="haiku", primary_api=None):
|
||||
raise RuntimeError("no provider")
|
||||
monkeypatch.setattr(reg_mod, "resolve_aux_model", p_boom, raising=True)
|
||||
primary.turn = 0
|
||||
asyncio.run(BA.run_browser_agent(task="t", browser_id="b3", model="opus", initial_url=None))
|
||||
assert primary.calls[-1]["model"] == "primary-x"
|
||||
|
||||
|
||||
def test_act_verified_refuses_irreversible_and_runs_reversible(monkeypatch):
|
||||
# BrowserActVerified: an irreversible-smelling target is REFUSED in code (the
|
||||
# solo-send rule holds), and a reversible step actually executes through the
|
||||
# verified path (resolve-late -> click_index) with an honest per-step verdict.
|
||||
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
|
||||
primary = FakeLLM([
|
||||
Resp([p_rp("send it via the plan tool"),
|
||||
p_tu("BrowserActVerified", steps=[{"action": "click", "target": "Send message"}])]),
|
||||
Resp([p_rp("ok, do a reversible step"),
|
||||
p_tu("BrowserActVerified", steps=[{"action": "click", "target": "Search", "role": "button"}])]),
|
||||
Resp([Blk("text", "done exploring")], stop_reason="end_turn"),
|
||||
])
|
||||
aux = FakeAux()
|
||||
sent = p_install(monkeypatch, primary, aux)
|
||||
|
||||
asyncio.run(BA.run_browser_agent(task="use the search", browser_id="b1", model="sonnet"))
|
||||
|
||||
all_msgs = json.dumps([c["messages"] for c in primary.calls])
|
||||
# 1) the irreversible target never executed; the model got the refusal + guidance
|
||||
assert "REFUSED" in all_msgs and "SOLO click" in all_msgs
|
||||
# 2) the reversible step resolved "Search" against the live list and clicked index 1
|
||||
assert any(c["action"] == "click_index" and c["params"].get("index") == 1 for c in sent)
|
||||
# 3) honest verdict fed back (static fake page = no observable change; never a fake OK)
|
||||
assert "FAILED" in all_msgs or "OK (verified)" in all_msgs
|
||||
|
||||
|
||||
def test_warm_send_prefix_replay_marries_send_script_zero_llm_turns(monkeypatch):
|
||||
# THE WARM-WRITE PATH (B): a learned send-gated skill replays its navigation
|
||||
# prefix mechanically, then hands the post-prefix state to the verified
|
||||
# send-script tail (fill -> verify -> send -> receipt). The model is NEVER
|
||||
# called: a warm write is replay + code, end to end.
|
||||
monkeypatch.setenv("OSW_SEND_SCRIPT", "1")
|
||||
monkeypatch.setenv("OSW_REPLAY_SENDTAIL", "1")
|
||||
import backend.apps.agents.browser.browser_skills as SK
|
||||
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear(); SK.SKILLS.clear()
|
||||
|
||||
TASK = "go to tyler chen's linkedin and text him '[test] warm hi w1'"
|
||||
HOST = "www.linkedin.com"
|
||||
THREAD = "https://www.linkedin.com/messaging/thread/2-abc/"
|
||||
sig = SK.compute_sig(TASK)
|
||||
SK.SKILLS[f"{HOST}|{sig}"] = {
|
||||
"host": HOST, "task_sig": sig, "recorded_at": 0, "replays": 0,
|
||||
"persisted": False, "rev": 1, "state": SK.PROBATION, "fails": 0,
|
||||
"composed_of": [],
|
||||
"steps": [
|
||||
{"tool": "BrowserNavigate", "params": {"url": THREAD}},
|
||||
{"tool": "BrowserClickByName", "params": {"name": "Send"}}, # send-gated tail
|
||||
],
|
||||
}
|
||||
|
||||
primary = FakeLLM([Resp([Blk("text", "should never be called")], stop_reason="end_turn")])
|
||||
aux = FakeAux()
|
||||
sent = p_install(monkeypatch, primary, aux)
|
||||
|
||||
COMPOSER = '[2]<textbox "Write a message">'
|
||||
COMMITTED = '[2]<textbox "Write a message" value="[test] warm hi w1">\n[14]<button "Send">'
|
||||
CLEARED = '[2]<textbox "Write a message">\n[9]<button "Attach">'
|
||||
seq = {"n": 0}
|
||||
states = [COMPOSER, COMMITTED, CLEARED, CLEARED]
|
||||
|
||||
async def p_cmd(request_id, action, browser_id, params, tab_id=""):
|
||||
sent.append({"action": action, "params": params})
|
||||
if action == "list_interactives":
|
||||
s = states[min(seq["n"], len(states) - 1)]; seq["n"] += 1
|
||||
return {"text": s, "url": THREAD}
|
||||
if action == "navigate":
|
||||
return {"text": "Navigated", "url": THREAD}
|
||||
if action == "click_index":
|
||||
return {"text": "Clicked", "url": THREAD, "clickedRole": "button", "clickedName": "Send"}
|
||||
if action == "evaluate":
|
||||
return {"text": json.dumps({"ready": True, "quiet": 9999, "elems": 100, "found": True}), "url": THREAD}
|
||||
return {"text": "ok", "url": THREAD}
|
||||
monkeypatch.setattr(BA.ws_manager, "send_browser_command", p_cmd, raising=False)
|
||||
|
||||
result = asyncio.run(BA.run_browser_agent(
|
||||
task=TASK, browser_id="b1", model="sonnet", initial_url=THREAD,
|
||||
))
|
||||
# prefix replayed (navigate dispatched), script filled + sent, receipt passed
|
||||
assert any(c["action"] == "navigate" for c in sent)
|
||||
assert any(c["action"] == "click_index" and c["params"].get("text") for c in sent), "script fill ran"
|
||||
assert result.get("done") is True
|
||||
assert "sent" in str(result.get("summary", "")).lower()
|
||||
# the whole warm write took ZERO model turns
|
||||
assert primary.calls == [], f"model was called {len(primary.calls)}x; warm write should be replay+code only"
|
||||
|
||||
|
||||
def test_autosend_finishes_the_send_after_the_model_fills(monkeypatch):
|
||||
# B (mid-loop takeover): on an UN-quoted send ("say hi"), the model opens the composer and TYPES
|
||||
# the message; the code then finishes the send (find Send + click + two-sided receipt), so the
|
||||
# model never spends a turn hunting the stale-indexed Send button. Uses what the model typed.
|
||||
monkeypatch.setenv("OSW_SEND_SCRIPT", "1") # autosend rides with the send-script family
|
||||
BH.BROWSER_HISTORY.clear(); BH.DOMAIN_NOTES.clear()
|
||||
URL = "https://www.linkedin.com/messaging/thread/2-abc/"
|
||||
COMMITTED = '[2]<textbox "Write a message" value="hi">\n[14]<button "Send">'
|
||||
CLEARED = '[2]<textbox "Write a message">\n[9]<button "Attach">'
|
||||
st = {"filled": False, "sent": False}
|
||||
|
||||
async def p_cmd(request_id, action, browser_id, params, tab_id=""):
|
||||
sent.append({"action": action, "params": params})
|
||||
if action == "click_index":
|
||||
if params.get("text"):
|
||||
st["filled"] = True
|
||||
return {"text": "Clicked", "url": URL}
|
||||
st["sent"] = True
|
||||
return {"text": "Clicked", "url": URL, "clickedRole": "button", "clickedName": "Send"}
|
||||
if action == "click_by_name":
|
||||
st["sent"] = True
|
||||
return {"text": "Clicked", "url": URL, "clickedRole": "button", "clickedName": "Send"}
|
||||
if action == "list_interactives":
|
||||
return {"text": CLEARED if st["sent"] else (COMMITTED if st["filled"] else COMMITTED), "url": URL}
|
||||
if action == "evaluate":
|
||||
return {"text": json.dumps({"ready": True, "quiet": 9999, "elems": 100, "found": True}), "url": URL}
|
||||
return {"text": "ok", "url": URL}
|
||||
|
||||
primary = FakeLLM([
|
||||
Resp([p_rp("type the message"), p_tu("BrowserClickIndex", index=2, text="hi", expect="hi")]),
|
||||
Resp([Blk("text", "fallback, should not be reached")], stop_reason="end_turn"),
|
||||
])
|
||||
aux = FakeAux()
|
||||
sent = p_install(monkeypatch, primary, aux)
|
||||
monkeypatch.setattr(BA.ws_manager, "send_browser_command", p_cmd, raising=False)
|
||||
|
||||
result = asyncio.run(BA.run_browser_agent(
|
||||
task="say hi to tyler chen on linkedin", browser_id="b1", model="sonnet", initial_url=URL))
|
||||
# the CODE clicked Send (index 14), which the model never scripted (it only filled index 2)
|
||||
assert any(c["action"] == "click_index" and c["params"].get("index") == 14 for c in sent), "code did the send"
|
||||
assert result.get("done") is True
|
||||
assert "sent" in str(result.get("summary", "")).lower()
|
||||
# the model was called ONCE (the fill turn); autosend ended the run, no second send turn
|
||||
assert len(primary.calls) == 1, f"model called {len(primary.calls)}x; the send should cost zero model turns"
|
||||
|
||||
|
||||
def test_login_wall_pauses_and_remembers_the_site(monkeypatch, tmp_path):
|
||||
"""Landing on a login wall auto-fires the RequestHumanIntervention pause with sign-in wording,
|
||||
and once the user resolves it (Done), the domain is remembered so future runs skip re-prompting."""
|
||||
from backend.apps.agents.browser import browser_login_handoff as H
|
||||
monkeypatch.setattr(H, "P_STORE_PATH", str(tmp_path / "auth.json"))
|
||||
|
||||
approvals = []
|
||||
|
||||
async def p_fake_approval(session, tool_name, tool_input):
|
||||
approvals.append((tool_name, tool_input))
|
||||
return {"behavior": "allow"}
|
||||
monkeypatch.setattr(BA, "p_request_browser_approval", p_fake_approval)
|
||||
|
||||
primary = FakeLLM([
|
||||
Resp([p_rp("open the login page"), p_tu("BrowserNavigate", url="https://acme.example/login")]),
|
||||
Resp([p_tu("Done", message="all set")]),
|
||||
])
|
||||
p_install(monkeypatch, primary, FakeAux())
|
||||
p_run_settled(task="log into acme and open my dashboard", browser_id="b1", model="sonnet")
|
||||
|
||||
assert any(t == "RequestHumanIntervention" and "sign in" in ti["problem"].lower()
|
||||
for t, ti in approvals), approvals
|
||||
assert H.is_authenticated("acme.example")
|
||||
|
||||
|
||||
def test_login_wall_skip_does_not_remember(monkeypatch, tmp_path):
|
||||
"""Skipping the sign-in (deny) leaves the site UNremembered and lets the run continue."""
|
||||
from backend.apps.agents.browser import browser_login_handoff as H
|
||||
monkeypatch.setattr(H, "P_STORE_PATH", str(tmp_path / "auth.json"))
|
||||
|
||||
async def p_deny(session, tool_name, tool_input):
|
||||
return {"behavior": "deny", "message": "Skipped by user"}
|
||||
monkeypatch.setattr(BA, "p_request_browser_approval", p_deny)
|
||||
|
||||
primary = FakeLLM([
|
||||
Resp([p_rp("open the login page"), p_tu("BrowserNavigate", url="https://acme.example/login")]),
|
||||
Resp([p_tu("Done", message="ok")]),
|
||||
])
|
||||
p_install(monkeypatch, primary, FakeAux())
|
||||
p_run_settled(task="log into acme", browser_id="b1", model="sonnet")
|
||||
|
||||
assert not H.is_authenticated("acme.example")
|
||||
|
||||
@@ -0,0 +1,43 @@
|
||||
"""The fast-path classifier must not let a mute provider disable the browser fast path.
|
||||
|
||||
Measured live 2026-07-26: on the codex lane the aux resolves to cx/gpt-5.4-mini, which returns an
|
||||
EMPTY body for this call. Empty parsed to verdict 'no', which reads identically to "this is not a
|
||||
browser task", so the entire browser fast path silently switched off for every GPT user with no
|
||||
error to show for it (gpt-5.4 filled 0/3; after the empty-body fallback, 2/2). These tests pin both
|
||||
halves of the contract: the parser may call empty 'no', and the caller must not accept that as a
|
||||
verdict when a provider was pinned.
|
||||
"""
|
||||
from backend.apps.agents.browser import browser_fast_path as fp
|
||||
|
||||
CLAUDE_REPLY = "ACT\n\nENTRY: https://x.com/home\n\n1. Navigate to X.\n2. Click the composer."
|
||||
|
||||
|
||||
def test_empty_body_parses_as_no() -> None:
|
||||
"""The parser is allowed to say 'no' on empty; the BUG was the caller treating that as a real
|
||||
verdict instead of a mute lane."""
|
||||
assert fp.parse_verdict_and_brief("") == ("no", "")
|
||||
|
||||
|
||||
def test_whitespace_only_body_is_treated_as_empty() -> None:
|
||||
"""A lane answering with only whitespace is exactly as mute as one answering ''."""
|
||||
assert fp.parse_verdict_and_brief(" \n\t \n") == ("no", "")
|
||||
|
||||
|
||||
def test_real_verdict_still_parses() -> None:
|
||||
verdict, brief = fp.parse_verdict_and_brief(CLAUDE_REPLY)
|
||||
assert verdict == "act"
|
||||
assert "ENTRY: https://x.com/home" in brief
|
||||
|
||||
|
||||
def test_read_verdict_still_parses() -> None:
|
||||
verdict, _ = fp.parse_verdict_and_brief("READ\n\nENTRY: https://example.com")
|
||||
assert verdict == "read"
|
||||
|
||||
|
||||
def test_caller_retries_provider_agnostic_on_a_mute_lane() -> None:
|
||||
"""The fix itself: classify_and_brief must re-ask WITHOUT the provider pin when the pinned lane
|
||||
returns nothing. Pinned by source so the retry cannot be quietly deleted."""
|
||||
import inspect
|
||||
src = inspect.getsource(fp.classify_and_brief)
|
||||
assert "if not text.strip() and primary_api:" in src, "empty-body fallback missing"
|
||||
assert "p_ask(None)" in src, "fallback must drop the provider pin"
|
||||
@@ -0,0 +1,138 @@
|
||||
"""The Windows cookie path, proven as far as a Mac can prove it.
|
||||
|
||||
browser_cookies.py carries a Windows branch (DPAPI key unwrap, then AES-256-GCM) that has only ever
|
||||
run on macOS hardware. Its own docstring admits it is written-to-spec and unverified, which is
|
||||
honest but leaves the riskiest part, the crypto layout, resting on a reading of the Chromium source.
|
||||
|
||||
That part does not actually need Windows. AES-GCM is AES-GCM, and Chromium's on-disk shape is a
|
||||
fixed byte layout: b"v10" + 12-byte nonce + ciphertext||tag. So this builds a real blob in exactly
|
||||
that layout and shows the branch recovers the plaintext, which narrows the genuinely unverified
|
||||
surface down to two things a Mac cannot fake: CryptUnprotectData, and where the files live.
|
||||
|
||||
What still needs a real Windows machine, stated plainly so nobody reads these greens as more than
|
||||
they are: DPAPI itself, the LOCALAPPDATA store paths, and app-bound (v20) profiles, which are out
|
||||
of reach on every platform by design.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.apps.onboarding.usage import browser_cookies as bc
|
||||
|
||||
KEY = bytes(range(32)) # AES-256
|
||||
NONCE = b"\x00" * 12
|
||||
|
||||
|
||||
def p_win_blob(plaintext: bytes, key: bytes = KEY, prefix: bytes = b"") -> bytes:
|
||||
"""A cookie value encrypted the way Windows Chromium writes it."""
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
return b"v10" + NONCE + AESGCM(key).encrypt(NONCE, prefix + plaintext, None)
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def on_windows(monkeypatch):
|
||||
monkeypatch.setattr(bc, "IS_WIN", True)
|
||||
|
||||
|
||||
# --- the crypto layout -----------------------------------------------------------------------
|
||||
|
||||
def test_a_real_windows_blob_decrypts(on_windows):
|
||||
"""The whole point: the byte offsets in the GCM branch are right."""
|
||||
assert bc.decrypt_cookie_value(p_win_blob(b"sessionid=abc123"), KEY) == "sessionid=abc123"
|
||||
|
||||
|
||||
def test_the_32_byte_domain_hash_prefix_is_stripped(on_windows):
|
||||
"""Newer Chromium prepends a 32-byte domain hash. Miss it and every cookie value is silently
|
||||
corrupted at the front, which would send a mangled session cookie rather than fail loudly."""
|
||||
assert bc.decrypt_cookie_value(p_win_blob(b"tok=xyz", prefix=b"\xa5" * 32), KEY) == "tok=xyz"
|
||||
|
||||
|
||||
def test_a_wrong_key_returns_none_instead_of_raising(on_windows):
|
||||
"""A DPAPI key from the wrong profile must degrade, not crash the import."""
|
||||
assert bc.decrypt_cookie_value(p_win_blob(b"secret"), bytes(32)) is None
|
||||
|
||||
|
||||
def test_app_bound_v20_is_refused_on_windows_too(on_windows):
|
||||
"""v20 is app-bound encryption; it is out of reach and must never be half-decoded."""
|
||||
assert bc.decrypt_cookie_value(b"v20" + NONCE + b"whatever", KEY) is None
|
||||
|
||||
|
||||
def test_truncated_ciphertext_degrades(on_windows):
|
||||
assert bc.decrypt_cookie_value(b"v10" + NONCE, KEY) is None
|
||||
|
||||
|
||||
def test_the_mac_branch_does_not_try_gcm():
|
||||
"""Same bytes, no Windows flag: the CBC branch must not accidentally accept a GCM blob."""
|
||||
assert bc.decrypt_cookie_value(p_win_blob(b"sessionid=abc123"), KEY) is None
|
||||
|
||||
|
||||
# --- the key unwrap, minus DPAPI itself ------------------------------------------------------
|
||||
|
||||
def test_local_state_key_is_base64_decoded_and_the_dpapi_magic_stripped(monkeypatch, tmp_path):
|
||||
"""Everything up to the CryptUnprotectData call is plain parsing and can be checked here: the
|
||||
key must arrive base64-decoded with the 5-byte b'DPAPI' prefix removed."""
|
||||
(tmp_path / "Local State").write_text(json.dumps(
|
||||
{"os_crypt": {"encrypted_key": base64.b64encode(b"DPAPI" + b"\x11" * 32).decode()}}))
|
||||
monkeypatch.setitem(bc.CHROMIUM_ROOTS, "Chrome", str(tmp_path))
|
||||
|
||||
seen = []
|
||||
|
||||
def fake_unprotect(blob: bytes) -> bytes:
|
||||
seen.append(blob)
|
||||
return KEY
|
||||
|
||||
monkeypatch.setattr(bc, "win_dpapi_unprotect", fake_unprotect)
|
||||
|
||||
assert bc.win_storage_key("Chrome") == KEY
|
||||
assert seen == [b"\x11" * 32], "the DPAPI magic must be stripped before unwrapping"
|
||||
|
||||
|
||||
def test_dpapi_failure_yields_no_key_rather_than_an_exception(monkeypatch, tmp_path):
|
||||
"""Wrong user account, roamed profile, corrupted blob: all of it must fail open."""
|
||||
(tmp_path / "Local State").write_text(json.dumps(
|
||||
{"os_crypt": {"encrypted_key": base64.b64encode(b"DPAPI" + b"\x11" * 32).decode()}}))
|
||||
monkeypatch.setitem(bc.CHROMIUM_ROOTS, "Chrome", str(tmp_path))
|
||||
monkeypatch.setattr(bc, "win_dpapi_unprotect", lambda d: None)
|
||||
|
||||
assert bc.win_storage_key("Chrome") is None
|
||||
|
||||
|
||||
def test_a_non_dpapi_local_state_is_refused(monkeypatch, tmp_path):
|
||||
"""Chrome on macOS writes a key with no DPAPI prefix; handing that to CryptUnprotectData is
|
||||
nonsense, so the magic check is what keeps the platforms from crossing."""
|
||||
(tmp_path / "Local State").write_text(json.dumps(
|
||||
{"os_crypt": {"encrypted_key": base64.b64encode(b"v10" + b"\x11" * 32).decode()}}))
|
||||
monkeypatch.setitem(bc.CHROMIUM_ROOTS, "Chrome", str(tmp_path))
|
||||
monkeypatch.setattr(bc, "win_dpapi_unprotect", lambda d: KEY)
|
||||
|
||||
assert bc.win_storage_key("Chrome") is None
|
||||
|
||||
|
||||
def test_a_missing_local_state_degrades(monkeypatch, tmp_path):
|
||||
monkeypatch.setitem(bc.CHROMIUM_ROOTS, "Chrome", str(tmp_path / "nope"))
|
||||
assert bc.win_storage_key("Chrome") is None
|
||||
|
||||
|
||||
def test_an_unknown_browser_has_no_key():
|
||||
assert bc.win_storage_key("Netscape") is None
|
||||
|
||||
|
||||
def test_windows_stores_are_rooted_in_localappdata():
|
||||
"""Not a behaviour test, a documentation lock: if someone re-points the Windows roots at the
|
||||
macOS Application Support layout, every Windows user silently gets the scan floor instead."""
|
||||
src = bc.__file__
|
||||
with open(src) as f:
|
||||
text = f.read()
|
||||
assert "LOCALAPPDATA" in text
|
||||
assert 'os.path.expanduser("~/AppData/Local")' in text, "the LOCALAPPDATA fallback went missing"
|
||||
|
||||
|
||||
def test_the_module_still_imports_on_this_platform():
|
||||
"""ctypes/wintypes only exist on Windows; the import must stay inside the function so the whole
|
||||
onboarding package does not explode on macOS."""
|
||||
assert callable(bc.win_dpapi_unprotect)
|
||||
assert bc.win_dpapi_unprotect(b"not really a blob") is None
|
||||
assert os.path.basename(bc.__file__) == "browser_cookies.py"
|
||||
@@ -0,0 +1,125 @@
|
||||
"""The Windows cookie path, exercised against the real DPAPI on a real Windows machine.
|
||||
|
||||
`test_browser_cookies_windows.py` covers the same code with the OS calls mocked, which proves the
|
||||
branching and nothing about the layout. Every assumption that could actually be wrong is an
|
||||
agreement with Windows itself:
|
||||
|
||||
- CryptUnprotectData through ctypes/crypt32 (struct layout, the LocalFree, the out-blob copy)
|
||||
- the "DPAPI" prefix on the base64 key in Local State
|
||||
- AES-256-GCM framing: nonce at [3:15], tag bundled at the tail
|
||||
- LOCALAPPDATA resolution for the four Chromium roots
|
||||
|
||||
A mock can agree with a wrong belief forever, so these skip everywhere except Windows and are run
|
||||
by the windows-verify CI job. They build a Local State and a cookie blob with the OS's own
|
||||
CryptProtectData, then read them back with the shipping code: if our idea of the layout is wrong
|
||||
the round trip fails, which is exactly the signal that cannot be faked from a Mac.
|
||||
"""
|
||||
import base64
|
||||
import json
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.apps.onboarding.usage import browser_cookies as bc
|
||||
|
||||
pytestmark = pytest.mark.skipif(not bc.IS_WIN, reason="real DPAPI; Windows only")
|
||||
|
||||
|
||||
def p_dpapi_protect(data: bytes) -> bytes:
|
||||
"""CryptProtectData, the exact mirror of the shipping unprotect. Deliberately written against
|
||||
the same ctypes surface so a struct-layout mistake shows up as a failed round trip."""
|
||||
import ctypes
|
||||
from ctypes import wintypes
|
||||
|
||||
class DATA_BLOB(ctypes.Structure):
|
||||
_fields_ = [("cbData", wintypes.DWORD), ("pbData", ctypes.POINTER(ctypes.c_char))]
|
||||
|
||||
buf = ctypes.create_string_buffer(data, len(data))
|
||||
blob_in = DATA_BLOB(len(data), ctypes.cast(buf, ctypes.POINTER(ctypes.c_char)))
|
||||
blob_out = DATA_BLOB()
|
||||
ok = ctypes.windll.crypt32.CryptProtectData(
|
||||
ctypes.byref(blob_in), None, None, None, None, 0, ctypes.byref(blob_out))
|
||||
assert ok, "CryptProtectData failed; the test harness itself is broken"
|
||||
n = int(blob_out.cbData)
|
||||
out = ctypes.create_string_buffer(n)
|
||||
ctypes.memmove(out, blob_out.pbData, n)
|
||||
ctypes.windll.kernel32.LocalFree(blob_out.pbData)
|
||||
return out.raw
|
||||
|
||||
|
||||
def test_dpapi_round_trips_through_the_shipping_unprotect():
|
||||
secret = os.urandom(32)
|
||||
assert bc.win_dpapi_unprotect(p_dpapi_protect(secret)) == secret
|
||||
|
||||
|
||||
def test_unprotecting_garbage_returns_none_rather_than_raising():
|
||||
"""A corrupt or foreign-user blob must degrade to "no key", never take the process down."""
|
||||
assert bc.win_dpapi_unprotect(b"not a dpapi blob") is None
|
||||
|
||||
|
||||
def test_storage_key_is_read_from_a_real_local_state(tmp_path, monkeypatch):
|
||||
"""Local State holds base64("DPAPI" + protected key). This pins the prefix strip and the
|
||||
base64 handling against a file Windows itself produced."""
|
||||
key = os.urandom(32)
|
||||
root = tmp_path / "User Data"
|
||||
root.mkdir()
|
||||
(root / "Local State").write_text(json.dumps(
|
||||
{"os_crypt": {"encrypted_key": base64.b64encode(b"DPAPI" + p_dpapi_protect(key)).decode()}}),
|
||||
encoding="utf-8")
|
||||
monkeypatch.setitem(bc.CHROMIUM_ROOTS, "Chrome", str(root))
|
||||
assert bc.win_storage_key("Chrome") == key
|
||||
|
||||
|
||||
def test_a_local_state_without_the_dpapi_prefix_is_refused(tmp_path, monkeypatch):
|
||||
root = tmp_path / "User Data"
|
||||
root.mkdir()
|
||||
(root / "Local State").write_text(json.dumps(
|
||||
{"os_crypt": {"encrypted_key": base64.b64encode(b"NOPE" + os.urandom(16)).decode()}}),
|
||||
encoding="utf-8")
|
||||
monkeypatch.setitem(bc.CHROMIUM_ROOTS, "Chrome", str(root))
|
||||
assert bc.win_storage_key("Chrome") is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("version", [b"v10", b"v11"])
|
||||
def test_a_v10_cookie_decrypts_with_the_windows_gcm_framing(version):
|
||||
"""The framing claim: nonce at [3:15], ciphertext+tag from [15:]."""
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
key, nonce = os.urandom(32), os.urandom(12)
|
||||
blob = version + nonce + AESGCM(key).encrypt(nonce, b"session-token-xyz", None)
|
||||
assert bc.decrypt_cookie_value(blob, key) == "session-token-xyz"
|
||||
|
||||
|
||||
def test_the_32_byte_domain_hash_prefix_is_stripped():
|
||||
"""Newer Chromium prepends a 32-byte domain hash inside the plaintext."""
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
key, nonce = os.urandom(32), os.urandom(12)
|
||||
plain = os.urandom(32) + b"session-token-xyz"
|
||||
blob = b"v10" + nonce + AESGCM(key).encrypt(nonce, plain, None)
|
||||
assert bc.decrypt_cookie_value(blob, key) == "session-token-xyz"
|
||||
|
||||
|
||||
def test_v20_is_reported_unreachable_rather_than_guessed():
|
||||
"""App-bound encryption cannot be read without the browser. Saying so is the honest answer;
|
||||
returning a wrong string here would silently corrupt a borrowed session."""
|
||||
assert bc.decrypt_cookie_value(b"v20" + os.urandom(40), os.urandom(32)) is None
|
||||
|
||||
|
||||
def test_the_wrong_key_fails_closed():
|
||||
from cryptography.hazmat.primitives.ciphers.aead import AESGCM
|
||||
|
||||
nonce = os.urandom(12)
|
||||
blob = b"v10" + nonce + AESGCM(os.urandom(32)).encrypt(nonce, b"session-token-xyz", None)
|
||||
assert bc.decrypt_cookie_value(blob, os.urandom(32)) is None
|
||||
|
||||
|
||||
def test_every_chromium_root_resolves_under_localappdata():
|
||||
"""Path resolution is the other half a Mac cannot check: these must sit under the real
|
||||
LOCALAPPDATA, not a POSIX-shaped guess."""
|
||||
local = os.environ.get("LOCALAPPDATA", "")
|
||||
assert local, "LOCALAPPDATA is unset; the roots below would silently fall back"
|
||||
for name in ("Chrome", "Brave", "Edge"):
|
||||
root = bc.CHROMIUM_ROOTS[name]
|
||||
assert root.startswith(local), f"{name} root {root!r} is not under {local!r}"
|
||||
assert "\\" in root, f"{name} root {root!r} is not a Windows path"
|
||||
@@ -0,0 +1,155 @@
|
||||
"""BrowserDeleteItem tool logic (the resolve JS is proven live; this pins the pure pieces and
|
||||
the resolve->trusted-click->verify orchestration with a mocked tool runner)."""
|
||||
import json
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_delete_script as d
|
||||
|
||||
|
||||
def test_flag_default_off(monkeypatch):
|
||||
monkeypatch.delenv("OSW_DELETE_SCRIPT", raising=False)
|
||||
assert d.delete_tool_enabled() is False
|
||||
monkeypatch.setenv("OSW_DELETE_SCRIPT", "1")
|
||||
assert d.delete_tool_enabled() is True
|
||||
|
||||
|
||||
def test_expression_embeds_target_json_safe():
|
||||
# a target with quotes must not break out of the JS string literal
|
||||
tricky = 'he said "hi"; alert(1)'
|
||||
expr = d.resolve_expression("more", tricky)
|
||||
assert json.dumps(tricky) in expr # embedded as a JSON literal
|
||||
assert 'const TARGET = ' + json.dumps(tricky) in expr
|
||||
assert "const STEP = \"more\"" in expr
|
||||
|
||||
|
||||
def make_exec(step_results, click_ok=True, verify_after_refresh=None):
|
||||
"""Tool-runner mock: BrowserEvaluate returns the scripted resolve result for the step it was
|
||||
called with (read out of the expression); BrowserClickPoint records real-input clicks; the
|
||||
location probe and BrowserNavigate model the refresh-reverify pass."""
|
||||
calls = {"clicks": [], "steps": [], "navs": []}
|
||||
|
||||
async def execute(tool, params, bid, tid):
|
||||
if tool == "BrowserEvaluate":
|
||||
expr = params["expression"]
|
||||
if "location.href" in expr:
|
||||
return {"value": {"href": "https://site.test/profile"}}
|
||||
step = expr.split('const STEP = "', 1)[1].split('"', 1)[0]
|
||||
calls["steps"].append(step)
|
||||
if step == "verify" and calls["navs"] and verify_after_refresh is not None:
|
||||
return {"value": verify_after_refresh}
|
||||
if step not in step_results: # menu-flow fixtures: no direct control on the item
|
||||
return {"value": {"ok": False, "stage": step, "optional": True, "msg": "none"}}
|
||||
return {"value": step_results[step]}
|
||||
if tool == "BrowserClickPoint":
|
||||
calls["clicks"].append((params["xPercent"], params["yPercent"]))
|
||||
return {"ok": True} if click_ok else {"error": "no webview"}
|
||||
if tool == "BrowserNavigate":
|
||||
calls["navs"].append(params["url"])
|
||||
return {"ok": True}
|
||||
return {"ok": True}
|
||||
return execute, calls
|
||||
|
||||
|
||||
POS = {"xPct": 50.0, "yPct": 40.0}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_full_flow_clicks_each_stage_with_real_input():
|
||||
ex, calls = make_exec({
|
||||
"more": {"ok": True, "stage": "more", **POS},
|
||||
"menuitem": {"ok": True, "stage": "menuitem", **POS},
|
||||
"confirm": {"ok": True, "stage": "confirm", **POS},
|
||||
"verify": {"ok": True, "stage": "verify"},
|
||||
})
|
||||
r = await d.run_delete("coffee notes abc123", "b1", "", ex)
|
||||
assert r == {"removed": True, "stage": "done", "msg": "item removed"}
|
||||
assert len(calls["clicks"]) == 3 # more, menuitem, confirm: all real-input clicks
|
||||
assert calls["steps"][-1] == "verify"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_verify_still_present_is_honest():
|
||||
# still-present survives the refresh re-verify too: honest not-removed, one refresh attempted
|
||||
ex, calls = make_exec({
|
||||
"more": {"ok": True, "stage": "more", **POS},
|
||||
"menuitem": {"ok": True, "stage": "menuitem", **POS},
|
||||
"confirm": {"ok": True, "stage": "confirm", **POS},
|
||||
"verify": {"ok": False, "stage": "verify"},
|
||||
})
|
||||
r = await d.run_delete("coffee notes abc123", "b1", "", ex)
|
||||
assert r["removed"] is False and r["stage"] == "done"
|
||||
assert calls["navs"] == ["https://site.test/profile"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stale_tile_flips_removed_after_refresh():
|
||||
"""Shreddit keeps the dead tile mounted until a reload; the refresh re-verify turns a real
|
||||
server-side delete into removed=True instead of an honest-but-wrong still-present."""
|
||||
ex, calls = make_exec({
|
||||
"more": {"ok": True, "stage": "more", **POS},
|
||||
"menuitem": {"ok": True, "stage": "menuitem", **POS},
|
||||
"confirm": {"ok": True, "stage": "confirm", **POS},
|
||||
"verify": {"ok": False, "stage": "verify"},
|
||||
}, verify_after_refresh={"ok": True, "stage": "verify"})
|
||||
r = await d.run_delete("coffee notes abc123", "b1", "", ex)
|
||||
assert r["removed"] is True
|
||||
assert len(calls["navs"]) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_confirm_is_optional_verify_decides():
|
||||
ex, calls = make_exec({
|
||||
"more": {"ok": True, "stage": "more", **POS},
|
||||
"menuitem": {"ok": True, "stage": "menuitem", **POS},
|
||||
"confirm": {"ok": False, "stage": "confirm", "optional": True, "msg": "no confirm dialog appeared"},
|
||||
"verify": {"ok": True, "stage": "verify"},
|
||||
})
|
||||
r = await d.run_delete("coffee notes abc123", "b1", "", ex)
|
||||
assert r["removed"] is True
|
||||
assert len(calls["clicks"]) == 2 # confirm never clicked
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_delete_skips_menu_walk():
|
||||
"""Row-action sites (Gmail): a literal Delete control on the item is clicked straight; the
|
||||
menu ladder never runs and a page-wide non-dialog 'Delete' earns NO bonus confirm click."""
|
||||
ex, calls = make_exec({
|
||||
"direct": {"ok": True, "stage": "direct", "label": "delete", **POS},
|
||||
"confirm": {"ok": True, "stage": "confirm", "fromDialog": False, **POS},
|
||||
"verify": {"ok": True, "stage": "verify"},
|
||||
})
|
||||
r = await d.run_delete("note to self abc123", "b1", "", ex)
|
||||
assert r["removed"] is True
|
||||
assert len(calls["clicks"]) == 1
|
||||
assert "more" not in calls["steps"] and "menuitem" not in calls["steps"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_direct_delete_confirms_only_in_dialog():
|
||||
ex, calls = make_exec({
|
||||
"direct": {"ok": True, "stage": "direct", "label": "move to trash", **POS},
|
||||
"confirm": {"ok": True, "stage": "confirm", "fromDialog": True, **POS},
|
||||
"verify": {"ok": True, "stage": "verify"},
|
||||
})
|
||||
r = await d.run_delete("note to self abc123", "b1", "", ex)
|
||||
assert r["removed"] is True
|
||||
assert len(calls["clicks"]) == 2 # direct + real-dialog confirm
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_stage_failure_reports_stage_and_never_removed():
|
||||
ex, calls = make_exec({
|
||||
"more": {"ok": False, "stage": "find", "msg": "target text not on this page"},
|
||||
})
|
||||
r = await d.run_delete("coffee notes abc123", "b1", "", ex)
|
||||
assert r == {"removed": False, "stage": "find", "msg": "target text not on this page"}
|
||||
assert not calls["clicks"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_unreadable_resolve_is_honest_failure():
|
||||
async def execute(tool, params, bid, tid):
|
||||
return {"error": "eval blew up"}
|
||||
r = await d.run_delete("coffee notes abc123", "b1", "", execute)
|
||||
assert r["removed"] is False and r["stage"] == "eval"
|
||||
@@ -0,0 +1,87 @@
|
||||
"""Delivery ground-truth for writes: ghost-drop host detection + the persistence probe that
|
||||
keeps a cleared-composer from being reported as a real delivery on sites that silently eat posts.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_delivery_check as dc
|
||||
|
||||
|
||||
def test_ghost_drop_host_matches_youtube_only():
|
||||
assert dc.is_ghost_drop_host("https://www.youtube.com/watch?v=abc")
|
||||
assert dc.is_ghost_drop_host("https://youtube.com/watch")
|
||||
assert dc.is_ghost_drop_host("https://m.youtube.com/watch")
|
||||
# every proven-good write host stays OFF the ghost path (zero added latency there)
|
||||
assert not dc.is_ghost_drop_host("https://x.com/home")
|
||||
assert not dc.is_ghost_drop_host("https://www.reddit.com/r/test")
|
||||
assert not dc.is_ghost_drop_host("https://mail.google.com/mail")
|
||||
assert not dc.is_ghost_drop_host("https://www.linkedin.com/feed/")
|
||||
assert not dc.is_ghost_drop_host("")
|
||||
# a lookalike domain must not match the bare-endswith by accident
|
||||
assert not dc.is_ghost_drop_host("https://notyoutube.com.evil.test/")
|
||||
|
||||
|
||||
def test_probe_expression_escapes_and_truncates():
|
||||
expr = dc.delivery_probe_expression('hi "there"\nsecond line ' + "z" * 200)
|
||||
assert expr.startswith("(()=>{")
|
||||
assert '\\"there\\"' in expr # the quote is JSON-escaped, not raw
|
||||
assert "z" * 80 not in expr # needle capped at 80 chars
|
||||
empty = dc.delivery_probe_expression("")
|
||||
assert 'n.length>0' in empty # an empty payload can never falsely "match"
|
||||
|
||||
|
||||
def make_exec(visibility_sequence):
|
||||
"""execute_tool stub: each BrowserEvaluate returns the next scripted visibility."""
|
||||
seq = list(visibility_sequence)
|
||||
calls = {"n": 0}
|
||||
|
||||
async def execute_tool(tool, params, browser_id, tab_id):
|
||||
assert tool == "BrowserEvaluate"
|
||||
v = seq[calls["n"]] if calls["n"] < len(seq) else False
|
||||
calls["n"] += 1
|
||||
return {"value": {"visible": bool(v)}}
|
||||
|
||||
return execute_tool, calls
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def no_real_sleep(monkeypatch):
|
||||
async def instant(_):
|
||||
return None
|
||||
monkeypatch.setattr(dc.asyncio, "sleep", instant)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delivery_confirmed_when_post_persists():
|
||||
ex, calls = make_exec([True, True]) # visible now, still visible after the drop window
|
||||
assert await dc.ghost_delivery_confirmed("payload", "b", "t", ex) is True
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delivery_false_when_never_rendered():
|
||||
ex, calls = make_exec([False]) # never rendered -> no second probe, honest False
|
||||
assert await dc.ghost_delivery_confirmed("payload", "b", "t", ex) is False
|
||||
assert calls["n"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_delivery_false_when_rendered_then_dropped():
|
||||
ex, calls = make_exec([True, False]) # optimistic render, then silently gone (the YouTube class)
|
||||
assert await dc.ghost_delivery_confirmed("payload", "b", "t", ex) is False
|
||||
assert calls["n"] == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ghost_confirm_false_when_probe_tool_errors():
|
||||
async def boom(tool, params, browser_id, tab_id):
|
||||
return {"error": "boom"}
|
||||
# a broken/erroring page read must fail closed to "not confirmed", never a false delivery
|
||||
assert await dc.ghost_delivery_confirmed("x", "b", "t", boom) is False
|
||||
|
||||
|
||||
def test_unconfirmed_note_is_honest_and_names_the_host():
|
||||
note = dc.unconfirmed_delivery_note("https://www.youtube.com/watch", "hello world")
|
||||
assert "hello world" in note
|
||||
assert "youtube.com" in note and "www." not in note
|
||||
assert "could NOT confirm" in note
|
||||
assert "check your posts" in note.lower()
|
||||
@@ -0,0 +1,105 @@
|
||||
"""A submit we never located must prove the payload rendered before it counts as delivered.
|
||||
|
||||
Measured live on LinkedIn's feed composer 2026-07-28: both structured resolvers missed, so the tail
|
||||
fell back to clicking a button literally named "Send". LinkedIn's feed composer submits with "Post",
|
||||
so that click landed on some OTHER widget's Send, returned no error, and the composer cleared
|
||||
anyway. Receipt passed. `sent_receipt=True`. Nothing was posted, on either the posts or the
|
||||
comments tab.
|
||||
|
||||
That is the dangerous direction: a cleared composer cannot tell "submitted" from "dismissed", and
|
||||
every honesty guarantee downstream is built on that receipt. So the guessed path, and only the
|
||||
guessed path, has to show the text actually rendered on the page. The two resolved paths keep their
|
||||
measured speed and are untouched.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_send_script as ss
|
||||
|
||||
PAYLOAD = "hello from my automation"
|
||||
COMMITTED = f'[2]<textbox "Write a message" value="{PAYLOAD}">'
|
||||
CLEARED = '[2]<textbox "Write a message">\n[9]<button "Attach">'
|
||||
|
||||
|
||||
def make_exec(*, submit_listed: bool, container_ok: bool, visible_after: bool, cleared: bool = True):
|
||||
"""A composer whose submit is or isn't resolvable, and a page that does or doesn't end up
|
||||
showing the payload. Records which route the tail took."""
|
||||
calls = {"clicks": [], "lists": 0, "evals": []}
|
||||
|
||||
async def execute(tool, params, bid, tid):
|
||||
if tool == "BrowserListInteractives":
|
||||
calls["lists"] += 1
|
||||
first = calls["lists"] == 1
|
||||
state = COMMITTED if (first or not cleared) else CLEARED
|
||||
if submit_listed:
|
||||
state += '\n[14]<button "Send">'
|
||||
return {"text": state}
|
||||
if tool == "BrowserEvaluate":
|
||||
expr = str(params.get("expression") or "")
|
||||
calls["evals"].append(expr)
|
||||
if "visible" in expr: # the delivery probe
|
||||
return {"text": f'{{"visible": {str(visible_after).lower()}}}'}
|
||||
if container_ok: # the container submit resolver
|
||||
return {"text": '{"ok": true, "xPct": 50.0, "yPct": 50.0, "name": "Post"}'}
|
||||
return {"text": '{"ok": false, "why": "no submit control in the composer container"}'}
|
||||
calls["clicks"].append((tool, params))
|
||||
return {"ok": True}
|
||||
return execute, calls
|
||||
|
||||
|
||||
def send_index(state, composer_index):
|
||||
for line in (state or "").splitlines():
|
||||
if '<button "Send">' in line:
|
||||
return (14, "Send")
|
||||
return None
|
||||
|
||||
|
||||
async def run_tail(**kw):
|
||||
ex, calls = make_exec(**kw)
|
||||
r = await ss.complete_send(PAYLOAD, COMMITTED, "b1", "t1", ex, send_index,
|
||||
composer_index=2, current_url="https://www.linkedin.com/feed/")
|
||||
return r, calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_guessed_send_that_never_rendered_is_not_delivered():
|
||||
"""The exact live failure. Composer cleared, so the old code called it sent; the payload is
|
||||
nowhere on the page, so it wasn't."""
|
||||
r, calls = await run_tail(submit_listed=False, container_ok=False, visible_after=False)
|
||||
assert any(t == "BrowserClickByName" for t, _ in calls["clicks"]), "this must take the guessed path"
|
||||
assert r["delivered"] is False, "a guessed click with no rendered payload must not claim delivery"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_guessed_send_that_did_render_is_delivered():
|
||||
"""The guard must not punish the guessed path when it genuinely worked, or the by-name fallback
|
||||
(load-bearing for LinkedIn message threads, where the button really is 'Send') becomes useless."""
|
||||
r, calls = await run_tail(submit_listed=False, container_ok=False, visible_after=True)
|
||||
assert any(t == "BrowserClickByName" for t, _ in calls["clicks"])
|
||||
assert r["delivered"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_resolved_submit_is_not_slowed_down_by_the_new_probe():
|
||||
"""Speed guard. The ranked-index path is the proven one (X, ~23s end to end); it must not start
|
||||
paying for a delivery probe it never needed."""
|
||||
r, calls = await run_tail(submit_listed=True, container_ok=True, visible_after=False)
|
||||
assert r["sent"] is True
|
||||
assert r["delivered"] is None, "a resolved submit keeps trusting the receipt, as measured"
|
||||
assert not any("visible" in e for e in calls["evals"]), "no delivery probe on the resolved path"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_container_resolved_submit_also_skips_the_probe():
|
||||
"""Container resolution locates the real control inside the composer, so it is not a guess."""
|
||||
r, calls = await run_tail(submit_listed=False, container_ok=True, visible_after=False)
|
||||
assert r["sent"] is True
|
||||
assert r["delivered"] is None
|
||||
assert not any("visible" in e for e in calls["evals"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_composer_that_never_cleared_is_still_not_sent():
|
||||
"""The probe is an EXTRA hurdle for guessed clicks, never a way around the receipt: if the
|
||||
composer still holds the text, nothing was sent no matter what else is on the page."""
|
||||
r, _ = await run_tail(submit_listed=False, container_ok=False, visible_after=True, cleared=False)
|
||||
assert r["sent"] is False
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Login-once handoff: registrable-domain keying, login-wall detection reuse, and the durable
|
||||
authenticated-domains memory that keeps future runs from re-prompting."""
|
||||
import os
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_login_handoff as h
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def temp_store(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(h, "P_STORE_PATH", os.path.join(str(tmp_path), "authenticated_domains.json"))
|
||||
|
||||
|
||||
def test_registrable_domain_normalizes():
|
||||
assert h.registrable_domain("https://www.x.com/i/flow/login") == "x.com"
|
||||
assert h.registrable_domain("https://mail.google.com/mail/u/0") == "mail.google.com"
|
||||
assert h.registrable_domain("reddit.com") == "reddit.com"
|
||||
assert h.registrable_domain("https://X.COM:443/home") == "x.com"
|
||||
assert h.registrable_domain("") == ""
|
||||
|
||||
|
||||
def test_login_wall_domain_reuses_the_one_detector():
|
||||
# a login URL is a wall
|
||||
assert h.login_wall_domain("https://x.com/i/flow/login", "") == "x.com"
|
||||
# a password field in the perception is a wall even off a login URL
|
||||
assert h.login_wall_domain("https://acme.example/app", '[3]<textbox "Password">') == "acme.example"
|
||||
# a normal page is not a wall
|
||||
assert h.login_wall_domain("https://x.com/home", '[1]<button "Post">') is None
|
||||
assert h.login_wall_domain("", "") is None
|
||||
|
||||
|
||||
def test_record_then_authenticated():
|
||||
assert h.is_authenticated("x.com") is False
|
||||
h.record_login("https://x.com/i/flow/login")
|
||||
assert h.is_authenticated("x.com") is True
|
||||
assert h.is_authenticated("https://www.x.com/anything") is True
|
||||
assert h.authenticated_domains() == ["x.com"]
|
||||
|
||||
|
||||
def test_first_seen_preserved_last_login_advances():
|
||||
h.record_login("reddit.com")
|
||||
first = h.login_record("reddit.com")
|
||||
h.record_login("reddit.com")
|
||||
second = h.login_record("reddit.com")
|
||||
assert second["first_seen"] == first["first_seen"]
|
||||
assert second["last_login"] >= first["last_login"]
|
||||
|
||||
|
||||
def test_prompt_copy_wording_flips_on_history():
|
||||
assert "needs you to sign in" in h.prompt_copy("reddit.com")[0]
|
||||
h.record_login("reddit.com")
|
||||
assert "expired or be a different account" in h.prompt_copy("reddit.com")[0]
|
||||
# instruction is always the same actionable line
|
||||
assert "click Done" in h.prompt_copy("reddit.com")[1]
|
||||
|
||||
|
||||
def test_record_login_is_fail_open(monkeypatch):
|
||||
# an unwritable path must not raise; the run just treats it as a fresh sign-in next time
|
||||
monkeypatch.setattr(h, "P_STORE_PATH", "/nonexistent-dir-xyz/authenticated_domains.json")
|
||||
h.record_login("x.com") # no exception
|
||||
assert h.is_authenticated("x.com") is False
|
||||
|
||||
|
||||
def test_blank_domain_never_recorded():
|
||||
h.record_login("")
|
||||
assert h.authenticated_domains() == []
|
||||
@@ -0,0 +1,48 @@
|
||||
"""Code-side plan dispatch: the aux's JSON plan is parsed fail-open and the
|
||||
irreversible wall holds in the PARSER (before anything could ever run)."""
|
||||
from backend.apps.agents.browser import browser_plan_dispatch as pd
|
||||
|
||||
|
||||
def test_parse_valid_plan():
|
||||
steps = pd.parse_plan('[{"action":"click","target":"Options","role":"button"},'
|
||||
'{"action":"fill","target":"Search","text":"cats"}]')
|
||||
assert [s.kind for s in steps] == ["click", "fill"]
|
||||
assert steps[1].text == "cats"
|
||||
|
||||
|
||||
def test_parse_stops_at_irreversible_and_drops_the_rest():
|
||||
steps = pd.parse_plan('[{"action":"click","target":"Options"},'
|
||||
'{"action":"click","target":"Send message"},'
|
||||
'{"action":"click","target":"Home"}]')
|
||||
assert [s.target for s in steps] == ["Options"] # Send refused, Home never reached
|
||||
|
||||
|
||||
def test_parse_malformed_and_junk_fail_open():
|
||||
assert pd.parse_plan("I think you should click Options") == []
|
||||
assert pd.parse_plan('{"action":"click"}') == []
|
||||
assert pd.parse_plan("[]") == []
|
||||
assert pd.parse_plan('[{"action":"hover","target":"X"},{"action":"click","target":""}]') == []
|
||||
|
||||
|
||||
def test_parse_caps_at_six_steps():
|
||||
plan = "[" + ",".join('{"action":"click","target":"B%d"}' % i for i in range(9)) + "]"
|
||||
assert len(pd.parse_plan(plan)) == 6
|
||||
|
||||
|
||||
def test_parse_carries_the_chosen_flag():
|
||||
# The planner may PICK among similar rows (the disambiguation turn-collapser); the
|
||||
# flag must survive parsing so the handoff note can mark the pick for review.
|
||||
steps = pd.parse_plan(
|
||||
'[{"action":"click","target":"Tyler Chen · 1st · Irvine, CA","chosen":true},'
|
||||
'{"action":"click","target":"Message","role":"button"}]')
|
||||
assert [s.chosen for s in steps] == [True, False]
|
||||
|
||||
|
||||
def test_chosen_cannot_smuggle_an_irreversible_click():
|
||||
# A chosen pick relaxes WHICH row gets clicked, never WHAT may be clicked: the
|
||||
# irreversible wall still breaks the chain even on a chosen step.
|
||||
steps = pd.parse_plan(
|
||||
'[{"action":"click","target":"Options","chosen":true},'
|
||||
'{"action":"click","target":"Send now","chosen":true},'
|
||||
'{"action":"click","target":"Home"}]')
|
||||
assert [s.target for s in steps] == ["Options"]
|
||||
@@ -186,3 +186,31 @@ def test_seed_playbook_fallback_and_supersede():
|
||||
pb.persist("github.com", ["learned: use the org filter"])
|
||||
pb.CACHE.clear()
|
||||
assert pb.get_playbook("github.com") == ["learned: use the org filter"]
|
||||
|
||||
|
||||
def test_broadened_seed_coverage_and_write_mechanics():
|
||||
from backend.apps.agents.browser.seed_playbooks import SEED_PLAYBOOKS
|
||||
from backend.apps.agents.browser.seed_for import seed_for
|
||||
# a broad mainstream cross-section resolves to non-empty guidance (search, shopping, food,
|
||||
# streaming, productivity, travel, health, education, AI, gov, social)
|
||||
for host in ("bing.com", "duckduckgo.com", "youtube.com", "netflix.com", "hulu.com",
|
||||
"temu.com", "costco.com", "ubereats.com", "uber.com", "opentable.com",
|
||||
"docs.google.com", "drive.google.com", "calendar.google.com", "outlook.com",
|
||||
"tripadvisor.com", "realtor.com", "ticketmaster.com", "webmd.com", "goodrx.com",
|
||||
"quizlet.com", "khanacademy.org", "chatgpt.com", "perplexity.ai", "usps.com",
|
||||
"messenger.com", "nextdoor.com", "weather.com", "wikipedia.org"):
|
||||
assert seed_for(host), f"{host} should have a seed"
|
||||
assert seed_for("www." + host) == seed_for(host) # www-strip still matches
|
||||
# the popular WRITE targets carry a first-run write mechanic
|
||||
assert any("Post" in b for b in seed_for("x.com"))
|
||||
assert any("Comment" in b for b in seed_for("youtube.com"))
|
||||
assert any("Send" in b for b in seed_for("mail.google.com"))
|
||||
assert any("BrowserApiWrite" in b for b in seed_for("reddit.com"))
|
||||
# SAFETY: every money site is framed READ-ONLY with an explicit NEVER-transact instruction
|
||||
for money in ("paypal.com", "venmo.com", "cash.app", "chase.com", "bankofamerica.com",
|
||||
"robinhood.com", "coinbase.com", "fidelity.com"):
|
||||
bullets = " ".join(seed_for(money))
|
||||
assert "READ-ONLY" in bullets and "NEVER" in bullets, f"{money} must be read-only framed"
|
||||
# coverage grew well past the original set, and no duplicate host key silently dropped a site
|
||||
assert len(SEED_PLAYBOOKS) >= 90
|
||||
assert len(SEED_PLAYBOOKS) == len(set(SEED_PLAYBOOKS))
|
||||
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Opener-mode block rule (the coverage treatment). The safety-critical property:
|
||||
a compose-entry word (post/comment/reply/...) is allowed to REVEAL a composer only
|
||||
while none is present, and is REFUSED the instant a composer textbox exists (then the
|
||||
same word is the real submit). Hard-irreversible words are refused in every mode, and
|
||||
opener-mode-off is byte-identical to the legacy blanket gate."""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_prestage as pre
|
||||
from backend.apps.agents.browser import browser_send_script as script
|
||||
|
||||
COMPOSER_PRESENT = '[2]<textbox "Write a message">\n[14]<button "Post">'
|
||||
NO_COMPOSER = '[1]<link "Home">\n[9]<button "Create post">\n[10]<button "Start a post">'
|
||||
|
||||
|
||||
def test_opener_mode_off_is_legacy_blanket_gate(monkeypatch):
|
||||
monkeypatch.delenv("OSW_PRESTAGE_OPENER", raising=False)
|
||||
# legacy blanket gate refuses its own word set regardless of composer state (and
|
||||
# "Reply" was never in it, so legacy lets it through: exactly the gap the treatment closes)
|
||||
assert pre.click_refused('[9]<button "Create post">', NO_COMPOSER) is True
|
||||
assert pre.click_refused('[9]<button "Submit">', NO_COMPOSER) is True
|
||||
assert pre.click_refused('[9]<link "Jobs">', NO_COMPOSER) is False
|
||||
|
||||
|
||||
def test_opener_allows_compose_entry_when_no_composer(monkeypatch):
|
||||
monkeypatch.setenv("OSW_PRESTAGE_OPENER", "1")
|
||||
# composer absent => these OPEN a box, so allowed
|
||||
assert pre.click_refused('[9]<button "Create post">', NO_COMPOSER) is False
|
||||
assert pre.click_refused('[10]<button "Start a post">', NO_COMPOSER) is False
|
||||
assert pre.click_refused('[3]<button "Add a comment">', NO_COMPOSER) is False
|
||||
assert pre.click_refused('[5]<button "Reply">', NO_COMPOSER) is False
|
||||
assert pre.click_refused('[7]<button "Post">', NO_COMPOSER) is False
|
||||
|
||||
|
||||
def test_opener_refuses_submit_once_composer_present(monkeypatch):
|
||||
"""THE safety invariant: the same 'Post' word that opened a box is the SUBMIT once
|
||||
a composer textbox is in perception, so it must be refused there."""
|
||||
monkeypatch.setenv("OSW_PRESTAGE_OPENER", "1")
|
||||
assert pre.click_refused('[14]<button "Post">', COMPOSER_PRESENT) is True
|
||||
assert pre.click_refused('[14]<button "Reply">', COMPOSER_PRESENT) is True
|
||||
assert pre.click_refused('[14]<button "Comment">', COMPOSER_PRESENT) is True
|
||||
|
||||
|
||||
def test_opener_hard_blocks_irreversible_always(monkeypatch):
|
||||
"""Pay/Buy/Delete/Send/Submit/Subscribe are NEVER composer-openers: refused even
|
||||
on a composer-absent page, both modes."""
|
||||
monkeypatch.setenv("OSW_PRESTAGE_OPENER", "1")
|
||||
for word in ("Send", "Submit", "Pay", "Buy now", "Delete", "Subscribe", "Confirm", "Connect"):
|
||||
assert pre.click_refused(f'[9]<button "{word}">', NO_COMPOSER) is True, word
|
||||
|
||||
|
||||
def test_opener_allows_plain_navigation(monkeypatch):
|
||||
monkeypatch.setenv("OSW_PRESTAGE_OPENER", "1")
|
||||
assert pre.click_refused('[1]<link "Notifications">', NO_COMPOSER) is False
|
||||
assert pre.click_refused('[2]<button "Open first post">', NO_COMPOSER) is False
|
||||
|
||||
|
||||
def test_deeper_reach_only_in_opener_mode(monkeypatch):
|
||||
monkeypatch.setenv("OSW_SEND_SCRIPT", "0") # isolate the opener flag from its default-on family
|
||||
monkeypatch.setenv("OSW_PRESTAGE_OPENER", "1")
|
||||
assert pre.opener_mode() is True
|
||||
monkeypatch.delenv("OSW_PRESTAGE_OPENER", raising=False)
|
||||
assert pre.opener_mode() is False
|
||||
assert pre.OPENER_MAX_STEPS > pre.MAX_STEPS
|
||||
|
||||
|
||||
def test_send_script_family_ships_on(monkeypatch):
|
||||
# Pins the ship decision, because the whole family hangs off this one default and the suite
|
||||
# itself pins it OFF in conftest for determinism (so a green suite proves nothing about it).
|
||||
monkeypatch.delenv("OSW_SEND_SCRIPT", raising=False)
|
||||
monkeypatch.delenv("OSW_AUTOSEND", raising=False)
|
||||
monkeypatch.delenv("OSW_PRESTAGE_OPENER", raising=False)
|
||||
assert script.script_enabled() is True
|
||||
assert script.autosend_enabled() is True
|
||||
assert pre.opener_mode() is True
|
||||
|
||||
|
||||
def test_send_script_enables_opener_mode(monkeypatch):
|
||||
# The coupling: the send-script needs a composer to fire and the opener is what reaches one,
|
||||
# so enabling the send-script turns the opener on even without its own flag (else prestage
|
||||
# lands on a search page, the send-script declines, and the slow model loop runs).
|
||||
monkeypatch.delenv("OSW_PRESTAGE_OPENER", raising=False)
|
||||
monkeypatch.setenv("OSW_SEND_SCRIPT", "1")
|
||||
assert pre.opener_mode() is True
|
||||
monkeypatch.setenv("OSW_SEND_SCRIPT", "0")
|
||||
assert pre.opener_mode() is False
|
||||
@@ -0,0 +1,81 @@
|
||||
"""The browser agent must never name a provider's model.
|
||||
|
||||
A user driving the browser on ChatGPT or Gemini has no Claude lane at all, so a single hardcoded
|
||||
model id is not a slow path, it is a dead one. The rule from CLAUDE.md is that aux calls ask the
|
||||
registry for a TIER ("haiku", "sonnet") and the registry returns whatever the user actually
|
||||
connected. Tier names are fine; model ids are not.
|
||||
|
||||
This caught real debt: browser_agent imported a MODEL_MAP of three claude-* ids that nothing used,
|
||||
left behind after the resolve_aux_model migration. Dead, but it made the whole package read as
|
||||
Claude-only to anyone auditing it, and a dead map is exactly the thing someone later wires back up.
|
||||
|
||||
Comments are deliberately exempt. A comment recording that cx/gpt-5.4-mini returned an empty body
|
||||
is evidence worth keeping; it cannot route a request.
|
||||
"""
|
||||
import io
|
||||
import pathlib
|
||||
import re
|
||||
import tokenize
|
||||
|
||||
# Concrete, dated model ids. Not "gemini.google.com" (a host) and not "haiku" (a tier).
|
||||
P_MODEL_ID_RE = re.compile(
|
||||
r"claude-(?:sonnet|opus|haiku)-[\w.-]+|gpt-5[\w.-]*|gemini-[0-9][\w.-]*")
|
||||
|
||||
P_BROWSER_DIR = pathlib.Path(__file__).resolve().parents[1] / "apps" / "agents" / "browser"
|
||||
|
||||
|
||||
def p_model_ids_in_code(path: pathlib.Path):
|
||||
"""Every concrete model id appearing in a STRING literal, with its line number."""
|
||||
hits = []
|
||||
with path.open() as f:
|
||||
for tok in tokenize.generate_tokens(f.readline):
|
||||
if tok.type != tokenize.STRING:
|
||||
continue
|
||||
for m in P_MODEL_ID_RE.finditer(tok.string):
|
||||
hits.append((tok.start[0], m.group(0)))
|
||||
return hits
|
||||
|
||||
|
||||
def test_no_browser_module_names_a_providers_model():
|
||||
offenders = []
|
||||
for path in sorted(P_BROWSER_DIR.glob("*.py")):
|
||||
for line, model in p_model_ids_in_code(path):
|
||||
offenders.append(f"{path.name}:{line} -> {model}")
|
||||
assert not offenders, (
|
||||
"browser modules must ask resolve_aux_model for a tier, never name a model:\n "
|
||||
+ "\n ".join(offenders))
|
||||
|
||||
|
||||
def test_the_guard_would_actually_catch_a_regression():
|
||||
"""A test that can never fail is not a guard. Prove the detector fires on the exact shape that
|
||||
was just removed, so a future MODEL_MAP cannot slip back in under a green suite."""
|
||||
source = 'MODEL_MAP = {"sonnet": "claude-sonnet-4-6", "haiku": "claude-haiku-4-5-20251001"}\n'
|
||||
found = []
|
||||
for tok in tokenize.generate_tokens(io.StringIO(source).readline):
|
||||
if tok.type == tokenize.STRING:
|
||||
found += [m.group(0) for m in P_MODEL_ID_RE.finditer(tok.string)]
|
||||
assert found == ["claude-sonnet-4-6", "claude-haiku-4-5-20251001"]
|
||||
|
||||
|
||||
def test_tier_names_and_hostnames_are_not_flagged():
|
||||
"""The guard must leave the legitimate vocabulary alone, or it will just get deleted."""
|
||||
source = 'x = "haiku"\ny = "sonnet"\nz = "gemini.google.com"\nw = "chat.openai.com"\n'
|
||||
found = []
|
||||
for tok in tokenize.generate_tokens(io.StringIO(source).readline):
|
||||
if tok.type == tokenize.STRING:
|
||||
found += [m.group(0) for m in P_MODEL_ID_RE.finditer(tok.string)]
|
||||
assert not found
|
||||
|
||||
|
||||
def test_every_aux_call_asks_for_a_tier():
|
||||
"""resolve_aux_model's preferred_tier must be a tier literal. If a model id ever gets passed
|
||||
here the call still works on Claude and silently dies for everyone else, which is precisely
|
||||
the failure that is invisible on the developer's own machine."""
|
||||
call_re = re.compile(r"preferred_tier\s*=\s*\"([^\"]+)\"")
|
||||
bad = []
|
||||
for path in sorted(P_BROWSER_DIR.glob("*.py")):
|
||||
for i, line in enumerate(path.read_text().splitlines(), 1):
|
||||
for m in call_re.finditer(line):
|
||||
if m.group(1) not in ("haiku", "sonnet", "opus"):
|
||||
bad.append(f"{path.name}:{i} -> {m.group(1)}")
|
||||
assert not bad, f"preferred_tier must be a tier, not a model: {bad}"
|
||||
@@ -0,0 +1,134 @@
|
||||
"""Read-script (authed-page extraction turn-collapser): answer-or-INSUFFICIENT
|
||||
contract, fail-open on thin pages / declines / errors, and the flag gate."""
|
||||
import asyncio
|
||||
|
||||
from backend.apps.agents.browser import browser_read_script as rs
|
||||
|
||||
|
||||
class Blk:
|
||||
def __init__(self, text): self.type = "text"; self.text = text
|
||||
|
||||
|
||||
class Resp:
|
||||
def __init__(self, text): self.content = [Blk(text)]
|
||||
|
||||
|
||||
class Aux:
|
||||
def __init__(self, text):
|
||||
self.txt = text; self.messages = self; self.calls = 0; self.last_page = ""
|
||||
|
||||
async def create(self, **kw):
|
||||
self.calls += 1
|
||||
# Keep what the aux was actually shown: which page reached it IS the hydration contract.
|
||||
self.last_page = str(kw.get("messages", [{}])[0].get("content", ""))
|
||||
return Resp(self.txt)
|
||||
|
||||
|
||||
def tool_returning(text):
|
||||
async def run_tool(name, params, browser_id, tab_id):
|
||||
assert name == "BrowserGetText"
|
||||
return {"text": text}
|
||||
return run_tool
|
||||
|
||||
|
||||
PAGE = "Tyler Chen\nHe/Him · 1st\nSomething Here\nIrvine, California\nEntrepreneurs First\n" + ("filler " * 200)
|
||||
|
||||
|
||||
def tool_returning_sequence(pages):
|
||||
"""A page that changes between reads, like an SPA finishing its render."""
|
||||
seen = []
|
||||
|
||||
async def run_tool(name, params, browser_id, tab_id):
|
||||
assert name == "BrowserGetText"
|
||||
seen.append(len(seen))
|
||||
return {"text": pages[min(len(seen) - 1, len(pages) - 1)]}
|
||||
run_tool.seen = seen
|
||||
return run_tool
|
||||
|
||||
|
||||
# Chrome that clears the 500-char floor while the actual content is still missing. This is the
|
||||
# shape that made the bug invisible: it is long enough to look like a real page.
|
||||
CHROME_ONLY = ("Home Feed My Network Jobs Messaging Notifications Me Work "
|
||||
"Skip to main content Keyboard shortcuts Close jump menu " * 12)
|
||||
HYDRATED = CHROME_ONLY + "\nTyler Chen\nSomething Here\nEntrepreneurs First\n" + ("filler " * 200)
|
||||
|
||||
|
||||
def test_waits_for_the_page_to_stop_growing(monkeypatch):
|
||||
"""The false-clean: a hydrating SPA crosses the char floor on nav and footer chrome long before
|
||||
the content lands. Answering from that first passing read produces a CONFIDENT WRONG answer,
|
||||
because the aux reports what it can see and nothing declines, so the INSUFFICIENT retry never
|
||||
fires. Two reads have to agree the page stopped growing before the aux sees anything."""
|
||||
monkeypatch.setattr(rs, "P_THIN_SETTLE_S", 0)
|
||||
monkeypatch.setattr(rs, "P_STABLE_SETTLE_S", 0)
|
||||
aux = Aux("His title is \"Something Here\".")
|
||||
tool = tool_returning_sequence([CHROME_ONLY, HYDRATED, HYDRATED])
|
||||
out = asyncio.run(rs.run_read_script(aux, "m", "find tyler chen's title", "b1", "t1", tool))
|
||||
|
||||
assert out == "His title is \"Something Here\"."
|
||||
assert len(tool.seen) >= 3, "must re-read until two reads agree, not answer off the first"
|
||||
sent = aux.last_page
|
||||
assert "Tyler Chen" in sent, "the aux must be handed the HYDRATED page, not the chrome-only one"
|
||||
|
||||
|
||||
def test_a_settled_page_still_answers_without_extra_waiting(monkeypatch):
|
||||
"""The guard must not turn every read into a slow read: a page that is already done answers as
|
||||
soon as two reads agree, which is immediately."""
|
||||
monkeypatch.setattr(rs, "P_THIN_SETTLE_S", 0)
|
||||
monkeypatch.setattr(rs, "P_STABLE_SETTLE_S", 0)
|
||||
aux = Aux("answer")
|
||||
tool = tool_returning_sequence([HYDRATED, HYDRATED])
|
||||
assert asyncio.run(rs.run_read_script(aux, "m", "q", "b1", "t1", tool)) == "answer"
|
||||
assert len(tool.seen) == 2, "a settled page costs exactly one confirming re-read"
|
||||
|
||||
|
||||
def test_a_page_that_never_settles_still_answers_from_the_last_read(monkeypatch):
|
||||
"""Something that keeps streaming forever (a live feed) must not fail closed to the loop just
|
||||
for being busy; after the read budget we use the fullest page we got."""
|
||||
monkeypatch.setattr(rs, "P_THIN_SETTLE_S", 0)
|
||||
monkeypatch.setattr(rs, "P_STABLE_SETTLE_S", 0)
|
||||
grows = [HYDRATED + ("more " * 200 * i) for i in range(1, 8)]
|
||||
aux = Aux("answer")
|
||||
tool = tool_returning_sequence(grows)
|
||||
assert asyncio.run(rs.run_read_script(aux, "m", "q", "b1", "t1", tool)) == "answer"
|
||||
assert len(tool.seen) == rs.MAX_READS
|
||||
|
||||
|
||||
def test_flag_gate(monkeypatch):
|
||||
monkeypatch.delenv("OSW_READ_SCRIPT", raising=False)
|
||||
assert rs.read_script_enabled() is True
|
||||
monkeypatch.setenv("OSW_READ_SCRIPT", "0")
|
||||
assert rs.read_script_enabled() is False
|
||||
monkeypatch.setenv("OSW_READ_SCRIPT", "1")
|
||||
assert rs.read_script_enabled() is True
|
||||
|
||||
|
||||
def test_answers_from_the_staged_page():
|
||||
aux = Aux('His title is "Something Here" at Entrepreneurs First.')
|
||||
out = asyncio.run(rs.run_read_script(
|
||||
aux, "m", "find tyler chen's title", "b1", "t1", tool_returning(PAGE)))
|
||||
assert out == 'His title is "Something Here" at Entrepreneurs First.'
|
||||
assert aux.calls == 1
|
||||
|
||||
|
||||
def test_insufficient_falls_open_to_the_loop():
|
||||
out = asyncio.run(rs.run_read_script(
|
||||
Aux("INSUFFICIENT"), "m", "find his email", "b1", "t1", tool_returning(PAGE)))
|
||||
assert out is None
|
||||
|
||||
|
||||
def test_thin_page_skips_the_aux_call_entirely():
|
||||
aux = Aux("should never be consulted")
|
||||
out = asyncio.run(rs.run_read_script(
|
||||
aux, "m", "find tyler", "b1", "t1", tool_returning("Loading...")))
|
||||
assert out is None
|
||||
assert aux.calls == 0
|
||||
|
||||
|
||||
def test_no_aux_client_and_tool_error_both_fail_open():
|
||||
assert asyncio.run(rs.run_read_script(
|
||||
None, "", "t", "b1", "t1", tool_returning(PAGE))) is None
|
||||
|
||||
async def broken_tool(name, params, browser_id, tab_id):
|
||||
return {"error": "card is gone"}
|
||||
assert asyncio.run(rs.run_read_script(
|
||||
Aux("answer"), "m", "t", "b1", "t1", broken_tool)) is None
|
||||
@@ -0,0 +1,148 @@
|
||||
"""A question about a post must never become another post.
|
||||
|
||||
A verification prompt quotes the very text it is asking about, so it looks identical to a send task:
|
||||
quoted payload plus a composer equals fire. Measured live on a real LinkedIn account 2026-07-28:
|
||||
|
||||
"say whether anything containing "<text>" is still there. Change nothing."
|
||||
|
||||
posted <text> to the feed. Only "verify whether" and "check whether" were in the guard, so the
|
||||
question shape was matched by two exact phrasings rather than by what it actually is.
|
||||
|
||||
The guard fails SAFE in the other direction: a false match only means the send goes through the
|
||||
model path instead of the script, which costs turns, never a wrong action.
|
||||
"""
|
||||
from backend.apps.agents.browser import browser_send_parse as sp
|
||||
|
||||
# Every one of these is somebody asking a QUESTION about content, with the content quoted.
|
||||
QUESTIONS = [
|
||||
'say whether anything containing "hello there" is still there. Change nothing.',
|
||||
'check whether "hello there" is still on my profile',
|
||||
'verify whether the post "hello there" went through',
|
||||
'tell me if "hello there" is still published',
|
||||
'confirm whether "hello there" posted',
|
||||
'is "hello there" still there?',
|
||||
'is the post "hello there" still live',
|
||||
'find out if "hello there" is still up',
|
||||
'look for "hello there" without posting anything',
|
||||
'this is read-only, do not post: is "hello there" there',
|
||||
'do not change anything, just say if "hello there" is present',
|
||||
]
|
||||
|
||||
# Real send intents that must STILL fire the script; over-widening the guard would quietly disable
|
||||
# the whole fast write path, which is the failure mode on the other side.
|
||||
SENDS = [
|
||||
'post this, exactly: "hello there"',
|
||||
'tweet "hello there"',
|
||||
'message Tyler and say "hello there"',
|
||||
'reply to that thread with "hello there"',
|
||||
'comment "hello there" on the first post',
|
||||
'send "hello there" to my brother on whatsapp',
|
||||
'put "hello there" in a new linkedin post',
|
||||
]
|
||||
|
||||
|
||||
def test_every_question_shape_declines():
|
||||
missed = [q for q in QUESTIONS if not sp.is_readonly(q)]
|
||||
assert not missed, f"these questions would be treated as sends: {missed}"
|
||||
|
||||
|
||||
def test_the_exact_prompt_that_posted_for_real_is_caught():
|
||||
"""The literal string that put a test post on a real LinkedIn feed."""
|
||||
assert sp.is_readonly(
|
||||
'Go to linkedin.com and say whether anything containing "x" is still there. Change nothing.')
|
||||
|
||||
|
||||
def test_real_sends_still_fire():
|
||||
blocked = [s for s in SENDS if sp.is_readonly(s)]
|
||||
assert not blocked, f"the guard swallowed real send intents: {blocked}"
|
||||
|
||||
|
||||
def test_the_guard_is_case_and_spacing_insensitive():
|
||||
assert sp.is_readonly('IS "hello" STILL THERE?')
|
||||
assert sp.is_readonly("Tell me if it posted")
|
||||
|
||||
|
||||
def test_empty_and_junk_are_not_readonly():
|
||||
"""An empty task is not a question; treating it as read-only would silently disable the script
|
||||
on a malformed input rather than letting the normal gates decide."""
|
||||
assert not sp.is_readonly("")
|
||||
assert not sp.is_readonly(" ")
|
||||
|
||||
|
||||
# --- surface targeting: a post is not a comment ---------------------------------------------
|
||||
|
||||
def test_a_post_task_rejects_a_comment_box():
|
||||
"""Measured: on LinkedIn's feed the capped listing starved the post modal of its own composer,
|
||||
so the only compose-shaped textbox left was a stranger's comment box. Filling it is the wrong
|
||||
action on the wrong content, not a slower route to the right one."""
|
||||
assert sp.surface_mismatch('post this, exactly: "hi"', "Text editor for creating comment")
|
||||
assert sp.surface_mismatch("start a post saying hi", "Add a comment")
|
||||
assert sp.surface_mismatch("tweet hello", "Post your reply")
|
||||
|
||||
|
||||
def test_a_comment_task_keeps_its_comment_box():
|
||||
"""One-directional by design: asking to comment must still land in a comment box."""
|
||||
assert not sp.surface_mismatch("comment on the first post saying hi", "Text editor for creating comment")
|
||||
assert not sp.surface_mismatch("reply to that thread with hi", "Add a comment")
|
||||
assert not sp.surface_mismatch("respond to his post", "Post your reply")
|
||||
|
||||
|
||||
def test_a_post_task_keeps_a_real_post_composer():
|
||||
assert not sp.surface_mismatch('post this, exactly: "hi"', "Post text")
|
||||
assert not sp.surface_mismatch("start a post", "Share your thoughts")
|
||||
assert not sp.surface_mismatch("tweet hello", "What is happening?")
|
||||
|
||||
|
||||
def test_a_task_with_no_post_intent_is_left_alone():
|
||||
"""Messaging a person is neither posting nor commenting; the guard must not touch it."""
|
||||
assert not sp.surface_mismatch("text tyler hello", "Write a message")
|
||||
assert not sp.surface_mismatch("", "Add a comment")
|
||||
|
||||
|
||||
# --- the opener is a surface too --------------------------------------------------------------
|
||||
|
||||
def test_a_post_task_rejects_a_comment_OPENER():
|
||||
"""Measured in the dry-run coverage sweep: on linkedin.com with "start a post", the only opener
|
||||
listed was 'Comment'. The script opened a stranger's comment box, found no post composer inside
|
||||
it, and declined. The composer already had this guard; the opener did not, so the wrong surface
|
||||
got opened one step earlier and burned the single reversible-opener hop."""
|
||||
assert sp.surface_mismatch("start a post saying hi", "Comment")
|
||||
assert sp.surface_mismatch('post this, exactly: "hi"', "Reply")
|
||||
|
||||
|
||||
def test_a_comment_task_keeps_its_comment_opener():
|
||||
"""One-directional, same as the composer rule: asking to comment must still open a comment box."""
|
||||
assert not sp.surface_mismatch("comment on the first post saying hi", "Comment")
|
||||
assert not sp.surface_mismatch("reply to that thread with hi", "Reply")
|
||||
|
||||
|
||||
def test_a_post_task_keeps_a_real_post_opener():
|
||||
for opener in ("Post", "Compose", "New message", "Message"):
|
||||
assert not sp.surface_mismatch('post this, exactly: "hi"', opener), opener
|
||||
|
||||
|
||||
# --- LinkedIn names its editors, and the names are the only thing telling them apart -----------
|
||||
|
||||
def p_row(name: str) -> str:
|
||||
return f'[12]*<textbox "{name}" value="">'
|
||||
|
||||
|
||||
def test_linkedins_post_editor_is_recognised_as_a_composer():
|
||||
"""Measured: landing on LinkedIn's own compose surface listed exactly ONE textbox, named
|
||||
"Text editor for creating content", and the composer picker scored zero because no pattern
|
||||
matched it. The whole site failed 3/3 on a page that was showing the right box."""
|
||||
from backend.apps.agents.browser import browser_send_parse as sp
|
||||
assert sp.composer_index_in_state(p_row("Text editor for creating content")) is not None
|
||||
|
||||
|
||||
def test_linkedins_comment_editor_is_still_refused_for_a_post_task():
|
||||
"""The names differ by one word, so widening the picker must not swallow the comment box.
|
||||
It matches now, and surface_mismatch is what rejects it: recognising a box and choosing it are
|
||||
different jobs, and only the second one is allowed to be wrong here."""
|
||||
from backend.apps.agents.browser import browser_send_parse as sp
|
||||
row = p_row("Text editor for creating comment")
|
||||
assert sp.composer_index_in_state(row) is not None
|
||||
assert sp.surface_mismatch('start a post saying "hello there"',
|
||||
"Text editor for creating comment") is True
|
||||
assert sp.surface_mismatch('start a post saying "hello there"',
|
||||
"Text editor for creating content") is False
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Structural composer finder + reveal-and-find reach + cross-nav retry, verified without a
|
||||
live webview: a mock executor scripts BrowserFindComposer results so the send-script's
|
||||
structural fallback path (find -> reveal -> re-perceive-on-nav -> fill) is exercised end to
|
||||
end, including the flag gating and the safety that reveal only forwards under the flag."""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_send_script as ss
|
||||
from backend.apps.agents.browser.browser_agent import send_submit_index_in_state, payload_in_textbox
|
||||
|
||||
TASK = "go to tyler chen's linkedin hes in entrepreneurs first and text him '[test] hello world r9-os'"
|
||||
NAMELESS = '[1]<link "Home">\n[2]<button "Search">' # no name-matched composer, no opener
|
||||
|
||||
|
||||
def make_struct_exec(fc_result, list_states=None):
|
||||
"""execute_tool mock for the structural-finder path: list calls return no-composer
|
||||
states; a BrowserFindComposer call returns the scripted structural result."""
|
||||
calls = {"clicks": [], "find": 0}
|
||||
states = list(list_states or [NAMELESS] * 6)
|
||||
i = {"n": 0}
|
||||
|
||||
# fc_result may be a single dict (returned every call) or a list of dicts (returned in
|
||||
# sequence, last one repeating) to script the cross-nav retry.
|
||||
fc_seq = fc_result if isinstance(fc_result, list) else [fc_result]
|
||||
|
||||
async def execute(tool, params, bid, tid):
|
||||
if tool == "BrowserListInteractives":
|
||||
s = states[min(i["n"], len(states) - 1)]
|
||||
i["n"] += 1
|
||||
return {"text": s}
|
||||
if tool == "BrowserFindComposer":
|
||||
r = fc_seq[min(calls["find"], len(fc_seq) - 1)]
|
||||
calls["find"] += 1
|
||||
calls["clicks"].append((tool, params))
|
||||
return r
|
||||
calls["clicks"].append((tool, params))
|
||||
return {"ok": True}
|
||||
return execute, calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structural_finder_fills_when_name_detector_misses(monkeypatch):
|
||||
"""The generalization: no AX-named composer and no opener, but the in-page finder
|
||||
ranks an editable region, fills + reads it back, and the script reports filled-ready
|
||||
(dry-run stops before the send). This is what unblocks Reddit-style contenteditables."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
|
||||
ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
|
||||
"selector": '[data-osw-composer="1"]', "score": 6.2, "nearSubmit": True})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK,
|
||||
current_url="https://www.reddit.com/r/test/comments/x/")
|
||||
assert r is not None and r["sent"] is False # filled, stopped before the send
|
||||
assert calls["find"] == 1
|
||||
assert any(c[0] == "BrowserFindComposer" for c in calls["clicks"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structural_finder_declines_when_no_editable(monkeypatch):
|
||||
"""The finder honestly finds nothing usable (a page with only a search box) -> decline
|
||||
to the model path, never a false fire."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
ex, calls = make_struct_exec({"found": False})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
assert r is None
|
||||
assert calls["find"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_structural_off_by_default_never_calls_finder(monkeypatch):
|
||||
"""Flag off = the proven name path only; a name-less perception declines and the finder
|
||||
is never invoked (the structural path can't perturb the default)."""
|
||||
monkeypatch.delenv("OSW_COMPOSER_STRUCT", raising=False)
|
||||
ex, calls = make_struct_exec({"found": True, "filled": True, "selector": "x", "role": "textarea"})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
assert r is None
|
||||
assert calls["find"] == 0
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reveal_flag_passed_to_finder_when_enabled(monkeypatch):
|
||||
"""OSW_COMPOSER_REVEAL=1 lets the finder take a reversible reveal action: the send-script
|
||||
must forward reveal=True to BrowserFindComposer (the composer isn't painted yet)."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
|
||||
monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
|
||||
ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable",
|
||||
"selector": '[data-osw-composer="1"]', "score": 6.0,
|
||||
"nearSubmit": True, "reveals": ["trigger"]})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK,
|
||||
current_url="https://www.linkedin.com/feed/")
|
||||
assert r is not None and r["sent"] is False
|
||||
find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
|
||||
assert find_call[1].get("reveal") is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_nav_retry_after_open_first_navigates(monkeypatch):
|
||||
"""open-first reveal navigates to the item page (killing the finder's context) so the first
|
||||
call finds nothing; the send-script re-perceives and calls the finder a SECOND time on the
|
||||
destination, where the composer now fills. Proves the cross-nav retry."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
|
||||
monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1")
|
||||
seq = [
|
||||
{"found": False, "reveals": ["trigger:noop", "open-first", "scroll"]}, # navigated away
|
||||
{"found": True, "filled": True, "role": "contenteditable",
|
||||
"selector": '[data-osw-composer="1"]', "score": 6.0, "reveals": ["scroll"]},
|
||||
]
|
||||
ex, calls = make_struct_exec(seq)
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK,
|
||||
current_url="https://www.reddit.com/r/test")
|
||||
assert r is not None and r["sent"] is False
|
||||
assert calls["find"] == 2 # retried on the destination
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_cross_nav_no_retry_when_reveal_did_not_navigate(monkeypatch):
|
||||
"""If the finder did NOT fire open-first (a same-doc trigger/scroll that just failed), there
|
||||
is no destination to re-perceive, so it declines after ONE call, no wasted retry."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1")
|
||||
ex, calls = make_struct_exec({"found": False, "reveals": ["trigger:noop", "open-first:noop", "scroll"]})
|
||||
r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
assert r is None
|
||||
assert calls["find"] == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reveal_flag_off_by_default(monkeypatch):
|
||||
"""Struct on but reveal unset: the finder is still called, but reveal=False, so it only
|
||||
scans what's painted and never clicks a trigger (the safe default)."""
|
||||
monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1")
|
||||
monkeypatch.delenv("OSW_COMPOSER_REVEAL", raising=False)
|
||||
ex, calls = make_struct_exec({"found": False})
|
||||
await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url="https://example.com/")
|
||||
find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer")
|
||||
assert find_call[1].get("reveal") is False
|
||||
@@ -0,0 +1,110 @@
|
||||
"""A send may only claim success on evidence the SYSTEM holds, never on the model's say-so.
|
||||
|
||||
Measured live on X, 2026-07-28, at the defaults being considered for the flip: the agent replied
|
||||
"All set, your message went through and it's showing in the conversation now" and a read-only check
|
||||
of the real profile proved nothing had been posted. The log showed exactly why:
|
||||
|
||||
[browser-sendscript] fill target 'Post text' [52]
|
||||
[browser-sendscript] done sent_receipt=False delivered=None
|
||||
[browser-autosend] post-fill send click ran, receipt unverified; model verifies
|
||||
|
||||
`send_confirmed` was carrying two unrelated facts: "the click ran, so never fire another" and "it
|
||||
landed, so we may say so". The stall backstop read the first and printed a sentence that only the
|
||||
second could justify. This is the worst failure this agent has, because a user who is told it posted
|
||||
stops checking.
|
||||
"""
|
||||
import inspect
|
||||
import re
|
||||
|
||||
from backend.apps.agents.browser import browser_agent as ba
|
||||
from backend.apps.agents.browser import browser_delivery_check as dc
|
||||
|
||||
P_SRC = inspect.getsource(ba.run_browser_agent) if hasattr(ba, "run_browser_agent") else \
|
||||
open(ba.__file__, encoding="utf-8").read()
|
||||
CONFIDENT = "All set, your message went through"
|
||||
|
||||
|
||||
def test_the_hardcoded_confident_sentence_no_longer_exists_anywhere():
|
||||
"""The literal claim that shipped the lie is gone, and nothing may reintroduce it. Completion
|
||||
wording is composed by the model from what actually happened; a stock success sentence sitting
|
||||
in the source is a claim that gets made whether or not it is true."""
|
||||
assert CONFIDENT not in P_SRC, (
|
||||
"the stock success sentence is back in the source; completions must be composed from "
|
||||
"evidence, not printed from a template")
|
||||
|
||||
|
||||
def test_the_stall_backstop_branches_on_evidence():
|
||||
"""Where the lie was emitted. That backstop fires on a spinning run, so it decides the user's
|
||||
final sentence without the model ever getting to speak; it must therefore read the evidence
|
||||
flag, not the resend guard."""
|
||||
idx = P_SRC.index("done_success = delivery_verified")
|
||||
window = P_SRC[idx:idx + 900]
|
||||
assert "compose_unverified_send" in window, \
|
||||
"the unverified branch must compose an honest line, not fall straight to a template"
|
||||
assert "unverified_send_note" in window, \
|
||||
"and it needs a never-fails honest fallback for when the aux is unavailable"
|
||||
|
||||
|
||||
def test_the_two_facts_are_not_the_same_variable():
|
||||
"""send_confirmed exists to stop a SECOND send. Reusing it as permission to claim success is
|
||||
what made the wrong state representable, so they must stay distinct."""
|
||||
assert "delivery_verified = False" in P_SRC, "the evidence flag is gone"
|
||||
assert re.search(r"done_success = delivery_verified", P_SRC), \
|
||||
"the run's success must be a function of evidence, not of the model's Done argument"
|
||||
|
||||
|
||||
def test_the_caller_cannot_starve_the_send_script():
|
||||
"""INVARIANT, and the third time this exact trap has bitten (find_composer, import_session, now
|
||||
this). The send-script's cost and its caller's timeout drifted apart when find_composer went
|
||||
15s -> 30s: one finder call could eat the caller's whole 30s budget, so the script was killed
|
||||
mid-send and EVERY write silently fell back to the slow model loop. It was invisible because
|
||||
asyncio.TimeoutError stringifies to nothing, so the log read "outer skip ()".
|
||||
|
||||
Measured live on LinkedIn: a 190.9s write that never posted."""
|
||||
from backend.apps.agents.browser import browser_send_script as ss
|
||||
from backend.apps.agents.core.ws_manager import BROWSER_CMD_TIMEOUTS
|
||||
|
||||
assert "timeout=browser_send_script.WORST_CASE_BUDGET_S" in P_SRC, (
|
||||
"the caller hardcodes its own timeout again; it must import the script's stated worst case "
|
||||
"so the two cannot drift")
|
||||
# A single finder call must not be able to consume the whole budget.
|
||||
assert ss.WORST_CASE_BUDGET_S > BROWSER_CMD_TIMEOUTS.get("find_composer", 15.0) * 2, \
|
||||
"the budget must leave room for fill and submit after the finder, not just the finder"
|
||||
|
||||
|
||||
def test_a_starved_send_is_logged_by_exception_class():
|
||||
"""A bare TimeoutError has an empty message, so a starved send read exactly like a page we
|
||||
deliberately declined to touch. The class name is what makes those two distinguishable."""
|
||||
assert "type(p_se).__name__" in P_SRC
|
||||
|
||||
|
||||
def test_an_unverified_send_gets_an_honest_line_that_does_not_claim_delivery():
|
||||
note = dc.unverified_send_note("https://x.com/home", "hello from my automation")
|
||||
low = note.lower()
|
||||
assert "could not confirm" in low or "not confirm" in low
|
||||
assert "went through and it's showing" not in low
|
||||
assert "hello from my automation" in note, "the user needs to know WHICH message is in doubt"
|
||||
|
||||
|
||||
def test_the_honest_line_tells_the_user_to_check_and_says_it_did_not_retry():
|
||||
"""Two things the user needs and cannot get anywhere else: that they must verify by hand, and
|
||||
that we did NOT blindly retry, because a silent retry is how you post twice."""
|
||||
note = dc.unverified_send_note("https://www.reddit.com/", "x" * 200).lower()
|
||||
assert "check" in note
|
||||
assert "twice" in note or "again" in note
|
||||
|
||||
|
||||
def test_the_unverified_line_claims_strictly_less_than_the_ghost_drop_line():
|
||||
"""These are different evidence states and must not collapse into one message: the ghost-drop
|
||||
note knows the composer cleared (so it WAS submitted); the unverified note knows only that a
|
||||
click ran. Saying 'the composer cleared' when it did not is a small lie inside an honest one."""
|
||||
unverified = dc.unverified_send_note("https://x.com/", "payload")
|
||||
ghost = dc.unconfirmed_delivery_note("https://x.com/", "payload")
|
||||
assert "composer cleared" in ghost
|
||||
assert "composer never cleared" in unverified
|
||||
assert unverified != ghost
|
||||
|
||||
|
||||
def test_the_host_is_named_so_the_user_knows_where_to_look():
|
||||
assert "x.com" in dc.unverified_send_note("https://www.x.com/compose", "p")
|
||||
assert "the site" in dc.unverified_send_note("", "p")
|
||||
@@ -0,0 +1,288 @@
|
||||
"""Send-script mechanism, verified without a live webview: real LinkedIn-shaped
|
||||
interactives fixtures driven through a mock executor exercise the exact path the
|
||||
live rig would (opener -> composer -> fill -> commit-check -> send -> clear-check),
|
||||
plus every abort/honesty branch. The wall-clock a live run measures is not here;
|
||||
the correctness the live run would prove is."""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_send_script as ss
|
||||
from backend.apps.agents.browser import browser_send_parse as sp
|
||||
from backend.apps.agents.browser.browser_agent import send_submit_index_in_state, payload_in_textbox
|
||||
|
||||
PROFILE = '[22]*<link "Tyler Chen Premium 1st">\n[50]*<link "Message">\n[51]<button "Follow">'
|
||||
COMPOSER_EMPTY = '[2]<textbox "Write a message">\n[9]<button "Attach">'
|
||||
COMPOSER_FILLED = '[2]<textbox "Write a message" value="[test] hello world r9-os">\n[14]<button "Send">'
|
||||
COMPOSER_SENT = '[2]<textbox "Write a message">\n[9]<button "Attach">' # cleared, Send gone
|
||||
# The profile-overlay committed-fill (ground truth from probe r-dump): payload IS
|
||||
# in the box but the Send button ranks OUT of the capped numbered list, so the
|
||||
# script must fall back to click-by-name (how the model itself sends there).
|
||||
COMPOSER_FILLED_NO_SEND = '[1]<textbox "I\'m looking for…">\n[24]<textbox "Write a message" value="[test] hello world r9-os">'
|
||||
|
||||
TASK = "go to tyler chen's linkedin hes in entrepreneurs first and text him '[test] hello world r9-os'"
|
||||
|
||||
|
||||
def make_exec(list_script):
|
||||
"""execute_tool mock: each BrowserListInteractives call returns the next
|
||||
scripted state; clicks/fills succeed and are recorded."""
|
||||
calls = {"list": 0, "clicks": []}
|
||||
states = list(list_script)
|
||||
|
||||
async def execute(tool, params, bid, tid):
|
||||
if tool == "BrowserListInteractives":
|
||||
i = min(calls["list"], len(states) - 1)
|
||||
calls["list"] += 1
|
||||
return {"text": states[i]}
|
||||
calls["clicks"].append((tool, params))
|
||||
return {"ok": True}
|
||||
return execute, calls
|
||||
|
||||
|
||||
# A full-page messaging composer: the only surface the script fires on (the live
|
||||
# A/B proved firing on the profile /in/ overlay is net-negative). Tests that exercise
|
||||
# the fill/send mechanism pass this; the surface-gate test passes the overlay URL.
|
||||
THREAD_URL = "https://www.linkedin.com/messaging/thread/2-abc/"
|
||||
PROFILE_URL = "https://www.linkedin.com/in/tylerchen1200/"
|
||||
|
||||
|
||||
async def run(task, state0, list_script, url=THREAD_URL):
|
||||
ex, calls = make_exec(list_script)
|
||||
r = await ss.run_send_script(task, "b1", "", state0, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, current_url=url)
|
||||
return r, calls
|
||||
|
||||
|
||||
FEED_URL = "https://www.linkedin.com/feed/"
|
||||
NO_COMPOSER = '[1]<link "Home">\n[2]<button "Search">\n[3]<link "Jobs">'
|
||||
X_COMPOSER = '[3]<textbox "Post your reply">\n[8]<button "Reply">' # X, not LinkedIn
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_surface_gate_declines_when_no_composer_in_perception():
|
||||
"""STRUCTURAL gate: a page whose perception has no compose-shaped textbox and no
|
||||
messaging opener declines UNTOUCHED, regardless of URL, so the script never fires
|
||||
where a fill would land nowhere useful. Stays composer-less through the poll."""
|
||||
ex, calls = make_exec([NO_COMPOSER, NO_COMPOSER, NO_COMPOSER, NO_COMPOSER])
|
||||
r = await ss.run_send_script(TASK, "b1", "", NO_COMPOSER, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url=FEED_URL)
|
||||
assert r is None
|
||||
assert not calls["clicks"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_surface_gate_polls_for_a_late_rendering_composer():
|
||||
"""The flake fix: the composer isn't in the FIRST perception (prestage snapshotted early,
|
||||
X home does this ~half the time) but lazy-renders a beat later. The gate polls a fresh
|
||||
perception before declining, so the write doesn't falsely fall to the slow model path."""
|
||||
# initial perception (passed in) has no composer; the fresh poll finds X's composer, then
|
||||
# the fill-committed and cleared states follow.
|
||||
X_FILLED = '[3]<textbox "Post your reply" value="[test] hello world r9-os">\n[8]<button "Reply">'
|
||||
X_SENT = '[3]<textbox "Post your reply">\n[1]<link "Home">'
|
||||
ex, calls = make_exec([X_COMPOSER, X_FILLED, X_SENT])
|
||||
r = await ss.run_send_script(TASK, "b1", "", NO_COMPOSER, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url=FEED_URL)
|
||||
assert r is not None and r["sent"] is True # it fired instead of a false decline
|
||||
assert calls["clicks"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_surface_gate_fires_on_a_NON_linkedin_composer():
|
||||
"""The whole generalization: the same send-script fires on ANY site whose perception
|
||||
carries a composer (here X's 'Post your reply'), no per-site URL gate, AND completes
|
||||
by clicking X's real submit button ('Reply') BY INDEX. Before the submit-vocabulary
|
||||
was generalized this only 'passed' because the mock succeeds on the by-name 'Send'
|
||||
fallback that doesn't exist on real X, so assert the real index path is taken."""
|
||||
X_FILLED = '[3]<textbox "Post your reply" value="[test] hello world r9-os">\n[8]<button "Reply">'
|
||||
X_SENT = '[3]<textbox "Post your reply">\n[1]<link "Home">'
|
||||
ex, calls = make_exec([X_FILLED, X_FILLED, X_SENT])
|
||||
r = await ss.run_send_script(TASK, "b1", "", X_COMPOSER, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK,
|
||||
current_url="https://x.com/messages/123")
|
||||
assert r is not None and r["sent"] is True
|
||||
# the send went via BrowserClickIndex on X's real Reply submit (8), not the by-name 'Send' crutch
|
||||
assert ("BrowserClickIndex", {"index": 8}) in calls["clicks"]
|
||||
assert not any(t == "BrowserClickByName" for t, _ in calls["clicks"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_surface_gate_allows_profile_overlay():
|
||||
"""The profile /in/ overlay is winnable via click-by-name (ground truth: its Send
|
||||
ranks out of the list but is a real button), so it's back in scope, not declined."""
|
||||
ex, calls = make_exec([COMPOSER_FILLED_NO_SEND, COMPOSER_FILLED_NO_SEND, COMPOSER_SENT])
|
||||
r = await ss.run_send_script(TASK, "b1", "", COMPOSER_EMPTY, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url=PROFILE_URL)
|
||||
assert r is not None and r["sent"] is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_send_via_click_by_name_when_send_absent_from_ranked_list():
|
||||
"""Ground truth (probe r-dump): the committed-fill state has the payload in the
|
||||
box but NO Send in the capped numbered list. The script falls back to
|
||||
click-by-name (the model's own send path there) and the receipt still passes."""
|
||||
ex, calls = make_exec([COMPOSER_FILLED_NO_SEND, COMPOSER_FILLED_NO_SEND, COMPOSER_SENT])
|
||||
r = await ss.run_send_script(TASK, "b1", "", COMPOSER_EMPTY, ex, send_submit_index_in_state,
|
||||
payload_in_textbox, payload_source=TASK, current_url=THREAD_URL)
|
||||
assert r is not None and r["sent"] is True
|
||||
# the send went through click-by-name, not an index click
|
||||
byname = [c for c in calls["clicks"] if c[0] == "BrowserClickByName"]
|
||||
assert byname and byname[0][1] == {"name": "Send", "role": "button"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_opener_hop_full_success():
|
||||
"""On a messaging THREAD-LIST page (full-page surface): script opens the
|
||||
conversation composer, fills, sees it commit, finds the late Send, clicks,
|
||||
sees it clear -> receipt passes."""
|
||||
# poll reads stay no-composer (PROFILE x3) -> opener path -> click Message -> composer appears
|
||||
r, calls = await run(TASK, PROFILE, [PROFILE, PROFILE, PROFILE, COMPOSER_EMPTY, COMPOSER_FILLED, COMPOSER_SENT])
|
||||
assert r is not None and r["sent"] is True
|
||||
assert r["payload"] == "[test] hello world r9-os"
|
||||
# opener click (50), fill into composer (2 w/ text), solo send click (14)
|
||||
idxs = [c[1].get("index") for c in calls["clicks"]]
|
||||
assert 50 in idxs and 2 in idxs and 14 in idxs
|
||||
fill = next(c for c in calls["clicks"] if c[1].get("text"))
|
||||
assert fill[1]["text"] == "[test] hello world r9-os"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_composer_already_open_skips_opener():
|
||||
"""Prestage left the composer open: no opener click, straight to fill+send."""
|
||||
r, calls = await run(TASK, COMPOSER_EMPTY, [COMPOSER_FILLED, COMPOSER_FILLED, COMPOSER_SENT])
|
||||
assert r is not None and r["sent"] is True
|
||||
assert 50 not in [c[1].get("index") for c in calls["clicks"]] # never clicked an opener
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort_when_no_payload():
|
||||
"""No quoted payload = the model's judgment call, never the script's."""
|
||||
r, _ = await run("open tyler chen's linkedin and message him something nice", PROFILE,
|
||||
[COMPOSER_EMPTY])
|
||||
assert r is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_abort_when_fill_not_seen_committed():
|
||||
"""Fill click ran but the textbox never shows the payload -> abort PRE-click,
|
||||
the irreversible send never fires (no false 'sent')."""
|
||||
r, calls = await run(TASK, COMPOSER_EMPTY, [COMPOSER_EMPTY, COMPOSER_EMPTY, COMPOSER_EMPTY])
|
||||
assert r is None
|
||||
# a Send-class click (index 14) must NEVER have been issued
|
||||
assert all(c[1].get("index") != 14 for c in calls["clicks"])
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_post_click_unverified_yields_honest_note_not_resend():
|
||||
"""Send clicked but the composer never verifiably clears -> the run returns
|
||||
sent=False with a do-not-resend note, never a silent retry."""
|
||||
r, calls = await run(TASK, COMPOSER_EMPTY, [COMPOSER_FILLED, COMPOSER_FILLED,
|
||||
COMPOSER_FILLED, COMPOSER_FILLED, COMPOSER_FILLED])
|
||||
assert r is not None and r["sent"] is False
|
||||
assert "do NOT send again" in r["note"] or "not send again" in r["note"].lower()
|
||||
# exactly one send-class click was issued (no blind re-fire)
|
||||
assert sum(1 for c in calls["clicks"] if c[1].get("index") == 14) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ambiguous_opener_aborts():
|
||||
"""Two 'Message' openers = ambiguous = hands to the model, no guess."""
|
||||
two = '[50]*<link "Message">\n[70]*<link "Message">'
|
||||
r, _ = await run(TASK, two, [COMPOSER_EMPTY])
|
||||
assert r is None
|
||||
|
||||
|
||||
COMPOSED = (
|
||||
f"{TASK}\n\n"
|
||||
"[routing brief from a fast pre-pass; follow it unless the live page disagrees]\n"
|
||||
'ENTRY: https://www.linkedin.com/search/results/people/\nSTEPS: click the "Tyler Chen" result, '
|
||||
'then the "Message" button, type the text, click "Send"'
|
||||
)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_composed_task_brief_quotes_fire_via_payload_source():
|
||||
"""The routing brief's own quoted strings made the payload ambiguous on every
|
||||
real dispatch (r242/r243 declined live); the raw user prompt rides separately."""
|
||||
ex, calls = make_exec([COMPOSER_FILLED, COMPOSER_FILLED, COMPOSER_SENT])
|
||||
r = await ss.run_send_script(COMPOSED, "b1", "", COMPOSER_EMPTY, ex,
|
||||
send_submit_index_in_state, payload_in_textbox,
|
||||
payload_source=TASK, current_url=THREAD_URL)
|
||||
assert r is not None and r["sent"] is True
|
||||
assert r["payload"] == "[test] hello world r9-os"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_brief_readonly_wording_does_not_disarm():
|
||||
"""The aux brief wrote 'do not submit it' for a plain send once, false-flagging it read-only;
|
||||
the brief section is stripped before the read-only check, so the send still fires."""
|
||||
composed = (
|
||||
f"{TASK}\n\n"
|
||||
"[routing brief from a fast pre-pass; follow it unless the live page disagrees]\n"
|
||||
"STEPS: type the text into the composer, do not submit anything else, read-only elsewhere"
|
||||
)
|
||||
ex, calls = make_exec([COMPOSER_FILLED, COMPOSER_FILLED, COMPOSER_SENT])
|
||||
r = await ss.run_send_script(composed, "b1", "", COMPOSER_EMPTY, ex,
|
||||
send_submit_index_in_state, payload_in_textbox,
|
||||
payload_source=TASK, current_url=THREAD_URL)
|
||||
assert r is not None and r["sent"] is True
|
||||
|
||||
|
||||
def test_dryrun_report_encodes_the_gate_funnel():
|
||||
"""The coverage harness greps this one line for gate attribution: a staged composer
|
||||
reports composer=1, a bare page reports zeros, and armed/filled ride the booleans."""
|
||||
r = sp.dryrun_report(COMPOSER_EMPTY, armed=True, filled=True, url=THREAD_URL)
|
||||
assert "armed=1" in r and "composer=1" in r and "filled=1" in r and "textboxes=1" in r
|
||||
r0 = sp.dryrun_report(NO_COMPOSER, armed=False, filled=False)
|
||||
assert "armed=0" in r0 and "composer=0" in r0 and "opener=0" in r0 and "textboxes=0" in r0
|
||||
|
||||
|
||||
def test_dryrun_report_counts_unmatched_textboxes():
|
||||
"""The X 'Post text' class: a textbox present but name-unmatched must be visible in
|
||||
the report (textboxes>0, composer=0), or the funnel can't tell R from N failures."""
|
||||
state = '[4]<textbox "Some Novel Label">\n[7]<button "Go">'
|
||||
r = sp.dryrun_report(state, armed=True, filled=False)
|
||||
assert "composer=0" in r and "textboxes=1" in r
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_readonly_probe_never_fires():
|
||||
"""The send-probe quotes the very payload it checks for; without this wall the
|
||||
script DELIVERED a real message from a read-only probe (r243 live)."""
|
||||
probe = (
|
||||
"READ-ONLY verification, do NOT send, type, click any send/submit control. "
|
||||
'Check the thread for this exact text:\n"[test] hello world r9-os"\n'
|
||||
"End with OUTCOME: PAYLOAD-FOUND or PAYLOAD-NOT-FOUND."
|
||||
)
|
||||
ex, calls = make_exec([COMPOSER_FILLED, COMPOSER_FILLED, COMPOSER_SENT])
|
||||
r = await ss.run_send_script(probe, "b1", "", COMPOSER_EMPTY, ex,
|
||||
send_submit_index_in_state, payload_in_textbox,
|
||||
payload_source=TASK, current_url=THREAD_URL)
|
||||
assert r is None
|
||||
assert not calls["clicks"]
|
||||
|
||||
|
||||
def test_looks_like_login_wall_hits_and_false_positives():
|
||||
# login/auth URLs (the live instagram/threads mis-fire was one of these)
|
||||
assert sp.looks_like_login_wall("https://www.instagram.com/accounts/login/?force_authentication", "")
|
||||
assert sp.looks_like_login_wall("https://accounts.google.com/v3/signin/identifier", "")
|
||||
assert sp.looks_like_login_wall("https://www.reddit.com/login/", "")
|
||||
assert sp.looks_like_login_wall("https://x.com/i/flow/login", "")
|
||||
# auth-form perception signals with an innocuous url
|
||||
assert sp.looks_like_login_wall("https://site.com/x", '[3]<textbox "Password">')
|
||||
assert sp.looks_like_login_wall("https://site.com/x", "Log in to X to continue")
|
||||
# false positives: a real composer page, a blog path, gmail inbox, a /author/ path
|
||||
assert not sp.looks_like_login_wall("https://x.com/home", X_COMPOSER)
|
||||
assert not sp.looks_like_login_wall("https://example.com/blog/login-tips", "")
|
||||
assert not sp.looks_like_login_wall("https://mail.google.com/mail/u/0/#inbox?compose=new", "")
|
||||
assert not sp.looks_like_login_wall("https://site.com/author/jane", "")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_login_wall_url_declines_before_any_fill():
|
||||
"""A login URL declines even when the perception carries a composer and the task quotes a
|
||||
payload: a real send surface never shares a page with a login wall, and filling here types
|
||||
into the auth form (the live instagram/threads mis-fire under the reveal finder)."""
|
||||
ex, calls = make_exec([X_COMPOSER])
|
||||
r = await ss.run_send_script('post this exactly: "hello from the test x9"', "b1", "", X_COMPOSER, ex,
|
||||
send_submit_index_in_state, payload_in_textbox,
|
||||
current_url="https://x.com/i/flow/login")
|
||||
assert r is None
|
||||
assert not calls["clicks"]
|
||||
@@ -0,0 +1,278 @@
|
||||
"""Borrowing the user's existing sign-in instead of interrupting them for a password.
|
||||
|
||||
This module reads the user's real browser, so the tests are mostly about what it must REFUSE to do.
|
||||
Nothing here touches a real store or a keychain: the reader is stubbed at every call site.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_session_import as si
|
||||
from backend.apps.settings.models import AppSettings
|
||||
|
||||
P_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
P_MAIN_JS = os.path.join(P_REPO_ROOT, "electron", "main.js")
|
||||
|
||||
RECORDS = [{"name": "sid", "value": "opaque", "domain": ".x.com", "path": "/",
|
||||
"secure": True, "httponly": True, "expires": 1900000000.0}]
|
||||
|
||||
|
||||
def test_opt_in_is_off_by_default():
|
||||
"""Reading someone's real browser is their call to make explicitly. If this ever defaults True,
|
||||
an upgrade would silently start reading stores the user never agreed to expose."""
|
||||
assert si.is_enabled(AppSettings()) is False
|
||||
|
||||
|
||||
def test_opt_in_flips():
|
||||
s = AppSettings()
|
||||
s.browser_import_signins = True
|
||||
assert si.is_enabled(s) is True
|
||||
|
||||
|
||||
def test_google_properties_route_to_the_sso_scope():
|
||||
"""A Gmail/YouTube session lives on the parent SSO domain, not the property's own host, and the
|
||||
reader has a NAMED scope for it. Getting this wrong means either no session at all or a general
|
||||
sweep of every google entry the user owns."""
|
||||
for d in ("mail.google.com", "google.com", "docs.google.com", "youtube.com", "www.youtube.com"):
|
||||
assert si.is_google_property(d), d
|
||||
for d in ("reddit.com", "x.com", "notgoogle.com", "google.com.evil.net", ""):
|
||||
assert not si.is_google_property(d), d
|
||||
|
||||
|
||||
def test_domain_normalisation_matches_the_handoff():
|
||||
"""One definition of 'which site is this', shared with the login handoff, or the two can
|
||||
disagree about which domain we just borrowed for."""
|
||||
assert si.site_domain("https://www.reddit.com/submit?x=1") == "reddit.com"
|
||||
assert si.site_domain("x.com") == "x.com"
|
||||
assert si.site_domain("") == ""
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_session_never_wakes_the_bridge(monkeypatch):
|
||||
"""Nothing to import means nothing to send. Calling the renderer with an empty payload would
|
||||
burn a round trip and log a bogus failure."""
|
||||
called = []
|
||||
monkeypatch.setattr(si, "read_site_records", lambda d: [])
|
||||
monkeypatch.setattr(si.ws_manager, "send_browser_command",
|
||||
lambda *a, **k: called.append(a) or {})
|
||||
result = await si.import_signin("x.com", "b1")
|
||||
assert result.outcome == "no_session"
|
||||
assert result.ok is False
|
||||
assert called == []
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_empty_domain_reads_nothing(monkeypatch):
|
||||
"""A blank URL must not turn into a wildcard read."""
|
||||
monkeypatch.setattr(si, "read_site_records",
|
||||
lambda d: pytest.fail("must not read for an empty domain"))
|
||||
assert (await si.import_signin("", "b1")).outcome == "no_session"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_successful_import_reports_what_landed(monkeypatch):
|
||||
async def fake_send(rid, action, browser_id, params, **kw):
|
||||
assert action == "import_session"
|
||||
assert params["domain"] == "x.com"
|
||||
assert params["cookies"] == RECORDS
|
||||
return {"ok": True, "set": 1, "total": 1}
|
||||
|
||||
monkeypatch.setattr(si, "read_site_records", lambda d: list(RECORDS))
|
||||
monkeypatch.setattr(si.ws_manager, "send_browser_command", fake_send)
|
||||
result = await si.import_signin("https://x.com/compose/post", "b1")
|
||||
assert result.outcome == "imported"
|
||||
assert result.ok is True
|
||||
assert result.entries_applied == 1
|
||||
assert result.domain == "x.com"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_bridge_error_is_a_result_not_an_exception(monkeypatch):
|
||||
"""Every failure has to degrade into something the caller can fall back from, because the
|
||||
fallback (ask the user to sign in) is the behaviour that existed before this did."""
|
||||
async def fake_send(*a, **k):
|
||||
return {"error": "No dashboard is connected."}
|
||||
|
||||
monkeypatch.setattr(si, "read_site_records", lambda d: list(RECORDS))
|
||||
monkeypatch.setattr(si.ws_manager, "send_browser_command", fake_send)
|
||||
result = await si.import_signin("x.com", "b1")
|
||||
assert result.outcome == "bridge_failed"
|
||||
assert result.ok is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_applied_nothing_is_not_success(monkeypatch):
|
||||
"""The bridge answering 'ok' while applying zero entries must NOT read as signed in, or the run
|
||||
skips the pause and then fails on a page it still cannot use."""
|
||||
async def fake_send(*a, **k):
|
||||
return {"ok": True, "set": 0, "total": 4}
|
||||
|
||||
monkeypatch.setattr(si, "read_site_records", lambda d: list(RECORDS))
|
||||
monkeypatch.setattr(si.ws_manager, "send_browser_command", fake_send)
|
||||
assert (await si.import_signin("x.com", "b1")).ok is False
|
||||
|
||||
|
||||
def test_expiry_is_translated_out_of_chromium_time(monkeypatch):
|
||||
"""Chromium counts microseconds from 1601; Electron wants unix seconds. Get this wrong and every
|
||||
borrowed entry is either already expired or session-scoped, so the sign-in dies on the next quit
|
||||
and the user quietly stops believing the feature works."""
|
||||
monkeypatch.setattr(si.browser_cookies, "read_provider_cookie_records",
|
||||
lambda d: [{"name": "sid", "value": "opaque", "expires_utc": 13400000000000000},
|
||||
{"name": "tmp", "value": "opaque", "expires_utc": 0}])
|
||||
out = si.read_site_records("x.com")
|
||||
assert out[0]["expires"] == pytest.approx(1755526400.0)
|
||||
assert out[1]["expires"] == 0.0, "a session entry must stay session-scoped, not become 1601"
|
||||
|
||||
|
||||
def test_the_whole_jar_travels_including_clearance_tokens(monkeypatch):
|
||||
"""A real browser sends everything it has, so we do too.
|
||||
|
||||
An earlier version held back the anti-bot clearance tokens (cf_clearance and friends) on the
|
||||
theory that ours could never match the user agent they were minted for. That was measured live
|
||||
on medium with the UA swap CONFIRMED firing in the Electron log, and it changed nothing in
|
||||
either direction, so the filter was carrying a story rather than its weight. Holding a cookie
|
||||
back is a claim about the site's auth that we could not support."""
|
||||
monkeypatch.setattr(si.browser_cookies, "read_provider_cookie_records", lambda d: [
|
||||
{"name": "sid", "value": "opaque", "expires_utc": 0},
|
||||
{"name": "uid", "value": "opaque", "expires_utc": 0},
|
||||
{"name": "cf_clearance", "value": "opaque", "expires_utc": 0},
|
||||
{"name": "__cf_bm", "value": "opaque", "expires_utc": 0},
|
||||
])
|
||||
assert sorted(r["name"] for r in si.read_site_records("medium.com")) == [
|
||||
"__cf_bm", "cf_clearance", "sid", "uid"]
|
||||
|
||||
|
||||
def test_google_reads_go_through_the_named_sso_scope(monkeypatch):
|
||||
"""A Gmail borrow must use the reader's named SSO set, never a general sweep of the user's
|
||||
google entries."""
|
||||
monkeypatch.setattr(si.browser_cookies, "read_google_session_records",
|
||||
lambda: [{"name": "SID", "value": "opaque", "expires_utc": 0}])
|
||||
monkeypatch.setattr(si.browser_cookies, "read_provider_cookie_records",
|
||||
lambda d: pytest.fail("google must not go through the generic read"))
|
||||
assert [r["name"] for r in si.read_site_records("mail.google.com")] == ["SID"]
|
||||
|
||||
|
||||
def test_unreadable_browser_degrades_instead_of_crashing(monkeypatch):
|
||||
"""A locked keychain, a v20 app-bound store, a browser that isn't installed: all of it is a
|
||||
fallback, never an exception that kills the run."""
|
||||
def boom(d):
|
||||
raise RuntimeError("read denied")
|
||||
|
||||
monkeypatch.setattr(si.browser_cookies, "read_provider_cookie_records", boom)
|
||||
assert si.read_site_records("x.com") == []
|
||||
|
||||
monkeypatch.setattr(si.browser_cookies, "p_best_store", boom)
|
||||
assert si.has_importable_session("x.com") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_broken_borrow_can_never_break_the_run(monkeypatch):
|
||||
"""The class this seals: borrowing is a convenience bolted onto the critical path, so ANY
|
||||
failure inside it must cost at most the pause we were going to show anyway. Caught for real by
|
||||
the suite, where a loose settings double made the helper raise and killed the whole browser run
|
||||
before it could even reach the sign-in prompt."""
|
||||
from backend.apps.agents.browser import browser_agent
|
||||
|
||||
def boom(*a, **k):
|
||||
raise TypeError("settings double is not the real thing")
|
||||
|
||||
monkeypatch.setattr(browser_agent.browser_session_import, "is_enabled", boom)
|
||||
assert await browser_agent.try_borrow_signin("acme.example", "b1", "", "") is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_borrow_happens_at_the_door_not_only_at_the_wall(monkeypatch):
|
||||
"""Borrowing only at a detected wall was too late: a task the model answers in one turn calls
|
||||
Done, which breaks the loop BEFORE the handoff runs, so short tasks never got the session at
|
||||
all. Navigating must carry it."""
|
||||
from backend.apps.agents.browser import browser_agent
|
||||
|
||||
seen = []
|
||||
browser_agent.signin_borrowed.discard("x.com")
|
||||
monkeypatch.setattr(browser_agent.browser_session_import, "is_enabled", lambda s: True)
|
||||
monkeypatch.setattr(browser_agent.browser_session_import, "has_importable_session", lambda d: True)
|
||||
|
||||
async def fake_import(domain, browser_id):
|
||||
seen.append(domain)
|
||||
return si.SessionImportResult(outcome="imported", domain=domain, entries_applied=3)
|
||||
|
||||
monkeypatch.setattr(browser_agent.browser_session_import, "import_signin", fake_import)
|
||||
await browser_agent.borrow_signin_before_nav("https://x.com/compose/post", "b1")
|
||||
assert seen == ["x.com"], "navigating to a site must borrow its sign-in first"
|
||||
|
||||
# Second navigate to the same site must not re-read the user's browser.
|
||||
await browser_agent.borrow_signin_before_nav("https://x.com/home", "b1")
|
||||
assert seen == ["x.com"], "a borrowed site must not be re-imported on every navigate"
|
||||
browser_agent.signin_borrowed.discard("x.com")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_pre_nav_borrow_respects_the_opt_in(monkeypatch):
|
||||
"""The door is the busiest path in the whole agent, so the gate has to hold there too."""
|
||||
from backend.apps.agents.browser import browser_agent
|
||||
|
||||
browser_agent.signin_borrowed.discard("x.com")
|
||||
monkeypatch.setattr(browser_agent.browser_session_import, "is_enabled", lambda s: False)
|
||||
monkeypatch.setattr(browser_agent.browser_session_import, "has_importable_session",
|
||||
lambda d: pytest.fail("must not probe the user's browser while opted out"))
|
||||
await browser_agent.borrow_signin_before_nav("https://x.com/home", "b1")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_wall_handoff_asks_a_human_once_the_door_borrow_did_not_take(monkeypatch):
|
||||
"""If we already borrowed at the door and are STILL at a wall, the session did not work.
|
||||
Re-importing identical values would change nothing, so this case belongs to the human, and
|
||||
silently returning True here would skip the prompt and strand the run."""
|
||||
from backend.apps.agents.browser import browser_agent
|
||||
|
||||
browser_agent.signin_borrowed.add("acme.example")
|
||||
monkeypatch.setattr(browser_agent.browser_session_import, "is_enabled", lambda s: True)
|
||||
monkeypatch.setattr(browser_agent.browser_session_import, "import_signin",
|
||||
lambda d, b: pytest.fail("must not re-import the same values"))
|
||||
try:
|
||||
assert await browser_agent.try_borrow_signin("acme.example", "b1", "", "") is False
|
||||
finally:
|
||||
browser_agent.signin_borrowed.discard("acme.example")
|
||||
|
||||
|
||||
def test_import_timeout_outlasts_the_hidden_window_warm():
|
||||
"""INVARIANT, and the second time this exact trap has bitten (see find_composer): the warm sits
|
||||
through the site's bot challenge inside the import command, so the command's timeout has to
|
||||
outlast the warm's own budget. Set them past each other and the window is killed mid-challenge,
|
||||
which throws away the entire reason the warm exists while still looking like a clean import."""
|
||||
from backend.apps.agents.core.ws_manager import BROWSER_CMD_TIMEOUTS, BROWSER_CMD_TIMEOUT_DEFAULT
|
||||
|
||||
with open(os.path.join(P_REPO_ROOT, "electron", "warmBorrowedSession.js"), encoding="utf-8") as fh:
|
||||
src = fh.read()
|
||||
budget_ms = sum(int(m) for m in re.findall(
|
||||
r"^const (?:LOAD_TIMEOUT_MS|SETTLE_MS|DESTROY_GRACE_MS) = (\d+);", src, re.M))
|
||||
assert budget_ms > 0, "warm budget constants not found; did warmBorrowedSession change shape?"
|
||||
timeout_s = BROWSER_CMD_TIMEOUTS.get("import_session", BROWSER_CMD_TIMEOUT_DEFAULT)
|
||||
assert timeout_s * 1000 > budget_ms, (
|
||||
f"import_session timeout {timeout_s}s must outlast the warm budget {budget_ms}ms")
|
||||
|
||||
|
||||
def test_agent_checks_the_opt_in_before_reading_anything():
|
||||
"""INVARIANT: the borrow helper must consult the setting FIRST. Pinned by source because the
|
||||
ordering is the whole consent story, and an innocent-looking reorder would start reading the
|
||||
user's browser before asking whether they wanted that."""
|
||||
import inspect
|
||||
|
||||
from backend.apps.agents.browser import browser_agent
|
||||
|
||||
src = inspect.getsource(browser_agent.try_borrow_signin)
|
||||
gate = src.index("is_enabled")
|
||||
assert gate < src.index("has_importable_session"), "opt-in must be checked before probing"
|
||||
assert gate < src.index("import_signin"), "opt-in must be checked before importing"
|
||||
|
||||
|
||||
def test_partition_write_confines_entries_to_the_requested_domain():
|
||||
"""INVARIANT on the Electron side: importing one site must never plant another site's session
|
||||
in the partition. Pinned by source since main.js needs a live Electron to execute."""
|
||||
with open(P_MAIN_JS, encoding="utf-8") as fh:
|
||||
src = fh.read()
|
||||
body = src[src.index("async function writePartitionCookies"):]
|
||||
body = body[:body.index("ipcMain.handle('set-partition-cookies'")]
|
||||
assert re.search(r"if \(host !== d && !host\.endsWith\(`\.\$\{d\}`\)\) continue;", body), \
|
||||
"the per-entry domain confinement guard is gone from writePartitionCookies"
|
||||
@@ -0,0 +1,95 @@
|
||||
"""Soft signed-out detection, plus the two invariants behind the 2026-07-26 bug hunt.
|
||||
|
||||
Each test here corresponds to a bug that actually escaped to a live run, so the point is to make
|
||||
that whole CLASS unwritable rather than to re-check a line:
|
||||
|
||||
1. Sites that browse fine while withholding the composer (bsky/stackoverflow/tiktok/threads) hit
|
||||
no login URL and show no password field, so the hard-wall gate saw nothing and the run
|
||||
reported "couldn't find the compose box" when the truth was "you are not signed in".
|
||||
2. The composer finder budgeted ~24s of reveal work into a 15s command timeout, so heavy pages
|
||||
were killed mid-ladder and threw away everything, and the last two tiers were unreachable.
|
||||
"""
|
||||
import os
|
||||
import re
|
||||
|
||||
from backend.apps.agents.browser import browser_send_parse as sp
|
||||
from backend.apps.agents.browser import browser_login_handoff as lh
|
||||
from backend.apps.agents.core.ws_manager import BROWSER_CMD_TIMEOUTS, BROWSER_CMD_TIMEOUT_DEFAULT
|
||||
|
||||
P_REPO_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
P_HANDLER_TS = os.path.join(P_REPO_ROOT, "frontend", "src", "shared", "browserCommandHandler.ts")
|
||||
|
||||
# Page-shaped perceptions, in the interactives format the agent actually sees.
|
||||
BSKY_SIGNED_OUT = '[1]<link "Sign in">\n[2]<button "Create account">\n[3]<heading "Discover">'
|
||||
# The live shape that broke the first draft: a LOGGED-OUT page still advertising "Notifications".
|
||||
# Vetoing on that word suppressed the detector on exactly the sites it exists for (bsky, 0 cookies,
|
||||
# read as signed-in). Only controls meaningless-unless-authenticated may veto.
|
||||
BSKY_SIGNED_OUT_WITH_NAV = ('[1]<link "Notifications">\n[2]<button "Sign in">\n'
|
||||
'[3]<button "Create account">')
|
||||
HN_SIGNED_OUT = '[1]<textbox "title">\n[2]<link "login">\n[3]<button "submit">'
|
||||
SO_SIGNED_OUT = '[4]<link "Log in">\n[5]<link "Sign up">\n[6]<heading "Questions">'
|
||||
X_SIGNED_IN = '[1]<button "Profile">\n[2]<textbox "What is happening?">\n[9]<button "Post">'
|
||||
# A signed-IN page that still advertises a sign-up somewhere: the veto must win, or we would tell
|
||||
# an authenticated user to log in again.
|
||||
SIGNED_IN_WITH_PROMO = '[1]<link "Sign up">\n[2]<button "Sign out">\n[3]<textbox "Post text">'
|
||||
LINKEDIN_FEED = '[1]<button "Start a post">\n[2]<link "Notifications">\n[3]<link "My Network">'
|
||||
|
||||
|
||||
def test_soft_signed_out_pages_are_detected():
|
||||
assert sp.looks_signed_out(BSKY_SIGNED_OUT)
|
||||
assert sp.looks_signed_out(SO_SIGNED_OUT)
|
||||
assert sp.looks_signed_out(HN_SIGNED_OUT)
|
||||
|
||||
|
||||
def test_logged_out_page_advertising_notifications_still_reads_signed_out():
|
||||
"""Regression for the veto that was too broad: a signed-OUT page may still show Notifications,
|
||||
Profile or Inbox links, so those must never veto. Only 'sign out'-class controls may."""
|
||||
assert sp.looks_signed_out(BSKY_SIGNED_OUT_WITH_NAV)
|
||||
|
||||
|
||||
def test_signed_in_pages_are_never_called_signed_out():
|
||||
for state in (X_SIGNED_IN, SIGNED_IN_WITH_PROMO, LINKEDIN_FEED, ""):
|
||||
assert not sp.looks_signed_out(state), state[:40]
|
||||
|
||||
|
||||
def test_signed_in_marker_vetoes_a_sign_in_link():
|
||||
"""The whole point of the veto: 'Sign up' present AND 'Sign out' present means signed IN."""
|
||||
assert not sp.looks_signed_out(SIGNED_IN_WITH_PROMO)
|
||||
|
||||
|
||||
def test_soft_detection_is_separate_from_the_hard_wall():
|
||||
"""A soft page must NOT read as a hard login wall: the hard gate declines the send outright,
|
||||
and mislabelling here would change behaviour on pages we can still act on."""
|
||||
assert not sp.looks_like_login_wall("https://bsky.app/", BSKY_SIGNED_OUT)
|
||||
assert sp.looks_signed_out(BSKY_SIGNED_OUT)
|
||||
|
||||
|
||||
def test_handoff_ignores_soft_pages_unless_explicitly_allowed():
|
||||
"""allow_soft is off by default because the pause interrupts the user; the caller turns it on
|
||||
only once the agent is demonstrably stuck."""
|
||||
assert lh.login_wall_domain("https://bsky.app/", BSKY_SIGNED_OUT) is None
|
||||
assert lh.login_wall_domain("https://bsky.app/", BSKY_SIGNED_OUT, allow_soft=True) == "bsky.app"
|
||||
|
||||
|
||||
def test_handoff_still_catches_hard_walls_without_soft():
|
||||
assert lh.login_wall_domain("https://www.instagram.com/accounts/login/", "") == "instagram.com"
|
||||
|
||||
|
||||
def test_find_composer_timeout_exceeds_its_own_in_page_budget():
|
||||
"""INVARIANT that seals the linkedin bug: the reveal ladder self-caps at a deadline, and the
|
||||
command carrying it must outlast that deadline. If someone lowers the timeout (or raises the
|
||||
in-page budget) past each other, heavy pages silently die mid-ladder again."""
|
||||
budget_ms = p_in_page_budget_ms()
|
||||
timeout_s = BROWSER_CMD_TIMEOUTS.get("find_composer", BROWSER_CMD_TIMEOUT_DEFAULT)
|
||||
assert timeout_s * 1000 > budget_ms, (
|
||||
f"find_composer timeout {timeout_s}s must exceed the in-page deadline {budget_ms}ms")
|
||||
|
||||
|
||||
def p_in_page_budget_ms() -> int:
|
||||
"""The deadline the in-page finder gives itself, read from the renderer source so the two sides
|
||||
of the invariant can never drift apart silently."""
|
||||
with open(P_HANDLER_TS, encoding="utf-8") as fh:
|
||||
src = fh.read()
|
||||
m = re.search(r"const DEADLINE = Date\.now\(\) \+ (\d+)", src)
|
||||
assert m, "in-page finder deadline not found; did the reveal ladder change?"
|
||||
return int(m.group(1))
|
||||
@@ -717,3 +717,24 @@ def test_replay_settle_target_for_click_by_name():
|
||||
{"tool": "BrowserClickByName", "params": {"name": "x" * 80}}) is None
|
||||
assert sk.replay_settle_target(
|
||||
{"tool": "BrowserClickByName", "params": {"name": ""}}) is None
|
||||
|
||||
|
||||
def test_send_opener_is_not_a_replay_boundary():
|
||||
# The LinkedIn profile opener is literally "Send a message to Tyler Chen":
|
||||
# it OPENS the composer (reversible), so it must NOT trip the send boundary
|
||||
# (v902 bug: it did, killing the prefix replay).
|
||||
from backend.apps.agents.browser import browser_batch_replay as BR
|
||||
assert BR.is_replay_boundary({"action": "click", "name": "Send a message to Tyler Chen"}) is False
|
||||
assert BR.is_replay_boundary({"action": "click", "name": "Send a note to Ada"}) is False
|
||||
# a REAL send still stops the prefix
|
||||
assert BR.is_replay_boundary({"action": "click", "name": "Send"}) is True
|
||||
assert BR.is_replay_boundary({"action": "click", "name": "Send now"}) is True
|
||||
|
||||
|
||||
def test_step_touches_composer():
|
||||
from backend.apps.agents.browser import browser_skills as SK
|
||||
assert SK.step_touches_composer({"tool": "BrowserType", "params": {"text": "hi"}}) is True
|
||||
assert SK.step_touches_composer({"tool": "BrowserClickByName", "params": {"name": "Write a message…"}}) is True
|
||||
# nav + opener are NOT composer steps (they stay in the marriage prefix)
|
||||
assert SK.step_touches_composer({"tool": "BrowserNavigate", "params": {"url": "https://x"}}) is False
|
||||
assert SK.step_touches_composer({"tool": "BrowserClickByName", "params": {"name": "Send a message to Tyler Chen"}}) is False
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
"""Container-scoped submit tier: the middle rung between the below-composer index pick and the
|
||||
by-name 'Send' last resort, plus the expression builder's escaping. The JS itself is proven live
|
||||
(X modal resolves tweetButton, inline resolves tweetButtonInline at hop 20); these tests pin the
|
||||
tier ORDER and the fail-safe parse."""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_send_script as ss
|
||||
from backend.apps.agents.browser import browser_submit_click as sc
|
||||
from backend.apps.agents.browser.browser_agent import send_submit_index_in_state
|
||||
|
||||
# Committed fill whose submit ranks OUT of the capped listing (the shape that starves the picker).
|
||||
COMPOSER_FILLED_NO_SEND = '[1]<textbox "I\'m looking for…">\n[24]<textbox "Write a message" value="[test] hello world r9-os">'
|
||||
COMPOSER_SENT = '[2]<textbox "Write a message">\n[9]<button "Attach">'
|
||||
|
||||
|
||||
def make_exec(eval_result):
|
||||
calls = {"clicks": []}
|
||||
|
||||
async def execute(tool, params, bid, tid):
|
||||
if tool == "BrowserListInteractives":
|
||||
return {"text": COMPOSER_SENT}
|
||||
calls["clicks"].append((tool, params))
|
||||
if tool == "BrowserEvaluate":
|
||||
return eval_result
|
||||
return {"ok": True}
|
||||
return execute, calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_container_submit_tier_between_index_and_by_name():
|
||||
"""X's compose modal: covered feed rows behind the overlay eat the 60-row cap, so the modal's
|
||||
own Post never reaches the picker (live 0/2 deliveries). The container tier resolves the
|
||||
submit inside the composer's own container and clicks it with REAL input; by-name stays last."""
|
||||
execute, calls = make_exec({"value": {"ok": True, "name": "post", "xPct": 61.0, "yPct": 33.0}})
|
||||
r = await ss.complete_send("[test] hello world r9-os", COMPOSER_FILLED_NO_SEND,
|
||||
"b1", "", execute, send_submit_index_in_state, composer_index=24)
|
||||
assert r["clicked"] is True and r["sent"] is True
|
||||
tools = [c[0] for c in calls["clicks"]]
|
||||
assert "BrowserEvaluate" in tools and "BrowserClickByName" not in tools
|
||||
point = [c for c in calls["clicks"] if c[0] == "BrowserClickPoint"]
|
||||
assert point and point[0][1] == {"xPercent": 61.0, "yPercent": 33.0}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_container_submit_miss_falls_to_by_name():
|
||||
execute, calls = make_exec({"value": {"ok": False, "why": "no submit control in the composer container"}})
|
||||
r = await ss.complete_send("[test] hello world r9-os", COMPOSER_FILLED_NO_SEND,
|
||||
"b1", "", execute, send_submit_index_in_state, composer_index=24)
|
||||
assert r["clicked"] is True
|
||||
tools = [c[0] for c in calls["clicks"]]
|
||||
assert tools.index("BrowserEvaluate") < tools.index("BrowserClickByName")
|
||||
|
||||
|
||||
def test_container_submit_expression_escapes_and_truncates():
|
||||
expr = sc.container_submit_expression('he said "hi" ' + "x" * 60)
|
||||
assert '\\"hi\\"' in expr # payload lands as a JS string literal, quotes escaped
|
||||
assert "x" * 30 not in expr # truncated to the 24-char prefix the fill verifier uses
|
||||
assert '"post"' in expr and '"send"' in expr
|
||||
|
||||
|
||||
def test_clean_button_name_strips_shortcut_suffix_and_bidi():
|
||||
# Gmail's real Send accessible name: 'Send (⌘Enter)'
|
||||
assert sc.clean_button_name("Send (⌘Enter)") == "send"
|
||||
assert sc.clean_button_name(" Post ") == "post"
|
||||
assert sc.clean_button_name("Reply (Ctrl-Enter)") == "reply"
|
||||
assert sc.clean_button_name("") == ""
|
||||
|
||||
|
||||
def test_parse_eval_value_reads_value_text_and_garbage():
|
||||
assert sc.parse_eval_value({"value": {"ok": True}}) == {"ok": True}
|
||||
assert sc.parse_eval_value({"text": '{"ok": false, "why": "x"}'}) == {"ok": False, "why": "x"}
|
||||
assert sc.parse_eval_value({"text": "not json"}) is None
|
||||
assert sc.parse_eval_value({"error": "boom"}) is None
|
||||
assert sc.parse_eval_value("weird") is None
|
||||
@@ -0,0 +1,127 @@
|
||||
"""The browser trace: every tier owes the user the same auditable record.
|
||||
|
||||
The bug this exists for: the expandable Browser Agent panel renders from CHILD SESSIONS, which only
|
||||
the sub-agent path creates. The fast path closed its bubble with a tool_result of literally "done",
|
||||
so on the tier that handles most tasks there was nothing to expand. "Trust me, I did it" is exactly
|
||||
what a browser agent must never say.
|
||||
"""
|
||||
from backend.apps.agents.browser import browser_trace as bt
|
||||
|
||||
NAV = {"tool": "BrowserNavigate", "input": {"url": "https://x.com/compose/post"},
|
||||
"elapsed_ms": 820, "ok": True}
|
||||
TYPE = {"tool": "BrowserType", "input": {"text": "hello from my automation"}, "elapsed_ms": 140, "ok": True}
|
||||
CLICK = {"tool": "BrowserClickByName", "input": {"name": "Post"}, "elapsed_ms": 310, "ok": True}
|
||||
|
||||
|
||||
def test_the_trace_says_where_it_went_and_what_it_did():
|
||||
t = bt.build_trace("drove the browser", [[NAV, TYPE, CLICK]])
|
||||
assert t.pages == ["https://x.com/compose/post"]
|
||||
assert len(t.steps) == 3
|
||||
text = bt.trace_text(t)
|
||||
assert "x.com/compose/post" in text
|
||||
assert "hello from my automation" in text, "what was typed is the whole point of an audit"
|
||||
assert "Post" in text
|
||||
|
||||
|
||||
def test_the_receipt_is_its_own_field_not_buried_in_the_steps():
|
||||
"""The receipt is what separates 'it says it posted' from 'it posted', so it must be
|
||||
structurally distinguishable, not a line the user has to spot among forty."""
|
||||
t = bt.build_trace("drove the browser", [[NAV]], receipt="composer cleared, post is on your profile")
|
||||
assert t.receipt.startswith("composer cleared")
|
||||
assert "Verified: composer cleared" in bt.trace_text(t)
|
||||
|
||||
|
||||
def test_every_dispatch_shows_up_not_just_the_last():
|
||||
"""A fast-path run can dispatch more than once (a recovery, a send probe). That work happened on
|
||||
the user's behalf, so hiding all but the final attempt would misrepresent what was done."""
|
||||
t = bt.build_trace("drove the browser", [[NAV], [TYPE, CLICK]])
|
||||
assert len(t.steps) == 3
|
||||
|
||||
|
||||
def test_a_long_run_says_what_it_omitted_instead_of_silently_truncating():
|
||||
"""A trace the user cannot tell is partial is worse than no trace, because they would read it as
|
||||
the whole story."""
|
||||
t = bt.build_trace("drove the browser", [[NAV] * (bt.MAX_STEPS + 12)])
|
||||
assert len(t.steps) == bt.MAX_STEPS
|
||||
assert t.steps_omitted == 12
|
||||
assert "12 earlier steps omitted" in bt.trace_text(t)
|
||||
|
||||
|
||||
def test_pages_read_as_a_journey_not_a_log():
|
||||
"""Consecutive repeats collapse (a reload is not a new place) but a genuine return does not, so
|
||||
the list reads as where it went, in order. Revisiting after going elsewhere is real movement and
|
||||
must survive."""
|
||||
home = {"tool": "BrowserNavigate", "input": {"url": "https://x.com/home"}, "ok": True}
|
||||
t = bt.build_trace("x", [[NAV, TYPE, NAV, home, home]])
|
||||
assert t.pages == ["https://x.com/compose/post", "https://x.com/home"]
|
||||
|
||||
back = bt.build_trace("x", [[NAV, home, NAV]])
|
||||
assert back.pages == ["https://x.com/compose/post", "https://x.com/home", "https://x.com/compose/post"]
|
||||
|
||||
|
||||
def test_the_landing_page_shows_even_when_nothing_navigated():
|
||||
"""Measured live: a cold run creates the card ALREADY pointed at its target, so no
|
||||
BrowserNavigate is ever issued and a log-only trace could not say where the agent went. The
|
||||
first thing anyone wants from an audit is the destination, so the entry URL is carried in."""
|
||||
reads = [{"tool": "BrowserGetText", "input": {}}, {"tool": "BrowserListInteractives", "input": {}}]
|
||||
t = bt.build_trace("drove the browser", [reads], entry_url="https://claude.ai/")
|
||||
assert t.pages == ["https://claude.ai/"]
|
||||
assert "claude.ai" in bt.trace_text(t)
|
||||
|
||||
|
||||
def test_the_landing_page_is_not_duplicated_when_it_did_navigate():
|
||||
t = bt.build_trace("x", [[NAV]], entry_url="https://x.com/compose/post")
|
||||
assert t.pages == ["https://x.com/compose/post"]
|
||||
|
||||
|
||||
def test_a_junk_entry_url_is_ignored_rather_than_shown():
|
||||
t = bt.build_trace("x", [[NAV]], entry_url="not a url")
|
||||
assert t.pages == ["https://x.com/compose/post"]
|
||||
|
||||
|
||||
def test_a_failed_step_is_marked_not_hidden():
|
||||
"""A run that limped to its answer must not read as a clean one."""
|
||||
bad = {"tool": "BrowserClickByName", "input": {"name": "Post"}, "ok": False}
|
||||
assert "(failed)" in bt.trace_text(bt.build_trace("x", [[bad]]))
|
||||
|
||||
|
||||
def test_no_actions_is_stated_plainly_rather_than_rendering_blank():
|
||||
"""The old failure mode was an empty panel, which reads as a broken UI rather than as a run that
|
||||
genuinely did nothing in a browser."""
|
||||
assert bt.trace_text(bt.build_trace("", [], "")) == "No browser actions were recorded."
|
||||
assert bt.trace_text(bt.build_trace("", [[]], "")) == "No browser actions were recorded."
|
||||
|
||||
|
||||
def test_tier_is_described_in_words_a_user_understands():
|
||||
"""'read->browser' is a routing string from a log line, not something anyone should be shown."""
|
||||
assert bt.tier_label("read", used_browser=False) == "read the page directly, no browser needed"
|
||||
assert "browser" in bt.tier_label("read->browser", used_browser=True)
|
||||
assert "->" not in bt.tier_label("read->browser", used_browser=True)
|
||||
|
||||
|
||||
def test_the_payload_is_data_so_the_panel_never_parses_prose():
|
||||
t = bt.build_trace("drove the browser", [[NAV, TYPE]], receipt="delivered")
|
||||
payload = bt.trace_payload(t)["browser_trace"]
|
||||
assert payload["pages"] == ["https://x.com/compose/post"]
|
||||
assert payload["receipt"] == "delivered"
|
||||
assert isinstance(payload["steps"], list)
|
||||
|
||||
|
||||
def test_malformed_entries_never_break_the_trace():
|
||||
"""action_log comes from a live run; a half-written entry must degrade to a readable line rather
|
||||
than take down the record of everything that DID happen."""
|
||||
junk = [{}, {"tool": None}, {"tool": "X", "input": "not-a-dict"}, {"input": {"url": 5}}]
|
||||
text = bt.trace_text(bt.build_trace("x", [junk]))
|
||||
assert text and "Traceback" not in text
|
||||
|
||||
|
||||
def test_the_fast_path_actually_emits_a_trace():
|
||||
"""INVARIANT: the whole point is that the tier which handles most tasks stops closing its bubble
|
||||
with the string "done". Pinned by source because the emission sits inside a long async flow."""
|
||||
import inspect
|
||||
|
||||
from backend.apps.agents.manager import run_browser_fast_path as fp
|
||||
|
||||
src = inspect.getsource(fp.run_browser_fast_path)
|
||||
assert '"text": "done"' not in src, 'the placeholder result is back; the bubble expands to nothing again'
|
||||
assert "browser_trace.trace_payload" in src, "the bubble must carry the structured trace"
|
||||
@@ -0,0 +1,72 @@
|
||||
"""The generic, site-agnostic verification core of the verified-action executor:
|
||||
does an action produce the SPECIFIC expected effect, checked against a before/after
|
||||
snapshot with zero per-site code. Pinned here, and pinned to match the send-script's
|
||||
proven receipt so wiring it in was behavior-preserving."""
|
||||
from backend.apps.agents.browser import browser_verified_action as va
|
||||
from backend.apps.agents.browser.browser_agent import payload_in_textbox
|
||||
|
||||
EMPTY = '[2]<textbox "Write a message">\n[9]<button "Attach">'
|
||||
FILLED = '[2]<textbox "Write a message" value="[test] hello world r9-os">\n[14]<button "Send">'
|
||||
SENT = '[2]<textbox "Write a message">\n[9]<button "Attach">'
|
||||
PAYLOAD = "[test] hello world r9-os"
|
||||
|
||||
|
||||
def test_url_changed_and_changed():
|
||||
assert va.expectation_met("url_changed", "s", "s", "u1", "u2")
|
||||
assert not va.expectation_met("url_changed", "s", "s", "u1", "u1")
|
||||
assert va.expectation_met("changed", "a", "b")
|
||||
assert not va.expectation_met("changed", "a", "a")
|
||||
|
||||
|
||||
def test_appeared_and_gone():
|
||||
assert va.expectation_met("appeared:Send", EMPTY, FILLED) # Send button showed up
|
||||
assert not va.expectation_met("appeared:Send", FILLED, FILLED)
|
||||
assert va.expectation_met("gone:Send", FILLED, SENT) # Send button vanished after send
|
||||
assert not va.expectation_met("gone:Send", EMPTY, EMPTY)
|
||||
|
||||
|
||||
def test_filled_and_cleared_match_the_send_receipt():
|
||||
# filled == the fill committed; cleared == the composer emptied (the send receipt)
|
||||
assert va.expectation_met("filled:" + PAYLOAD, EMPTY, FILLED)
|
||||
assert va.expectation_met("cleared:" + PAYLOAD, FILLED, SENT)
|
||||
assert not va.expectation_met("cleared:" + PAYLOAD, EMPTY, FILLED) # still in the box
|
||||
|
||||
|
||||
def test_generic_verifier_agrees_with_the_proven_inline_check():
|
||||
# The send-script's receipt was `not payload_in_textbox(state, payload)`; the
|
||||
# generic `cleared:` predicate must give the identical verdict on every state.
|
||||
for state in (EMPTY, FILLED, SENT):
|
||||
old = not payload_in_textbox(state, PAYLOAD)
|
||||
new = va.expectation_met("cleared:" + PAYLOAD, FILLED, state)
|
||||
assert old == new, f"mismatch on state={state!r}"
|
||||
|
||||
|
||||
def test_unknown_expectation_fails_safe():
|
||||
assert not va.expectation_met("teleported:X", EMPTY, FILLED) # typo/unknown = not met
|
||||
|
||||
|
||||
PROFILE = '[22]*<link "Tyler Chen Premium 1st">\n[50]*<link "Message">\n[51]<button "Follow">\n[52]<link "Message a friend">'
|
||||
|
||||
|
||||
def test_resolve_exact_name_over_partial():
|
||||
# exact "Message" link wins over the partial "Message a friend"
|
||||
hit = va.resolve_target(PROFILE, "Message", "link")
|
||||
assert hit == (50, "link", "Message")
|
||||
|
||||
|
||||
def test_resolve_role_disambiguates():
|
||||
two = '[1]<link "Send">\n[2]<button "Send">'
|
||||
assert va.resolve_target(two, "Send", "button") == (2, "button", "Send")
|
||||
# no role given + two exact matches = ambiguous = None (never guess)
|
||||
assert va.resolve_target(two, "Send") is None
|
||||
|
||||
|
||||
def test_resolve_prefix_when_suffix_mutates():
|
||||
row = '[7]<link "Ada Lovelace Premium 1st Mathematician and writer at Analytical">'
|
||||
assert va.resolve_target(row, "Ada Lovelace Premium 1st Mathematician and writer at Analytical Engine Co") == (7, "link", 'Ada Lovelace Premium 1st Mathematician and writer at Analytical')
|
||||
|
||||
|
||||
def test_resolve_absent_or_empty_is_none():
|
||||
assert va.resolve_target(PROFILE, "Checkout") is None
|
||||
assert va.resolve_target(PROFILE, "") is None
|
||||
assert va.resolve_target("", "Message") is None
|
||||
@@ -0,0 +1,108 @@
|
||||
"""The verified-step loop: resolve-late -> act -> verify-effect -> re-aim, in code.
|
||||
Pins the two properties the executor stands on: a reversible miss re-aims without an
|
||||
LLM turn, and an irreversible action NEVER re-fires once it has acted (the send-script's
|
||||
honesty rule, generalized)."""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_verified_step as vs
|
||||
|
||||
MENU_CLOSED = '[5]<button "Options">\n[9]<link "Home">'
|
||||
MENU_OPEN = '[5]<button "Options">\n[6]<menuitem "Delete draft">\n[9]<link "Home">'
|
||||
BOX_EMPTY = '[2]<textbox "Write a message">'
|
||||
BOX_FILLED = '[2]<textbox "Write a message" value="hello there friend">'
|
||||
BOX_SENT = '[2]<textbox "Write a message">\n[9]<button "Attach">'
|
||||
|
||||
|
||||
def make_exec(states, fail_actions=0):
|
||||
"""List calls pop states in order (last repeats); actions succeed after
|
||||
fail_actions initial failures; everything is recorded."""
|
||||
calls = {"lists": 0, "acts": [], "fails_left": fail_actions}
|
||||
seq = list(states)
|
||||
|
||||
async def execute(tool, params, bid, tid):
|
||||
if tool == "BrowserListInteractives":
|
||||
i = min(calls["lists"], len(seq) - 1)
|
||||
calls["lists"] += 1
|
||||
return {"text": seq[i], "url": "https://site.test/page"}
|
||||
calls["acts"].append((tool, params))
|
||||
if calls["fails_left"] > 0:
|
||||
calls["fails_left"] -= 1
|
||||
return {"error": "click failed"}
|
||||
return {"ok": True}
|
||||
return execute, calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_verifies_specific_effect():
|
||||
"""Click 'Options' expecting the menu to appear; before lacks it, after has it."""
|
||||
ex, calls = make_exec([MENU_CLOSED, MENU_OPEN])
|
||||
step = vs.VerifiedStep(kind="click", target="Options", role="button",
|
||||
expect="appeared:Delete draft")
|
||||
r = await vs.run_verified_step(step, "b1", "", ex, settle_s=0)
|
||||
assert r == {"ok": True, "verified": True, "acted": True, "note": ""}
|
||||
assert calls["acts"][0][1]["index"] == 5 # resolved late against the live list
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_reversible_miss_reaims_in_code():
|
||||
"""First click produces no effect (stale page); the loop re-resolves and re-acts
|
||||
WITHOUT an LLM turn, and the second attempt verifies."""
|
||||
ex, calls = make_exec([MENU_CLOSED, MENU_CLOSED, MENU_CLOSED, MENU_OPEN])
|
||||
step = vs.VerifiedStep(kind="click", target="Options", expect="appeared:Delete draft")
|
||||
r = await vs.run_verified_step(step, "b1", "", ex, settle_s=0, max_reaim=1)
|
||||
assert r["ok"] is True
|
||||
assert len(calls["acts"]) == 2 # acted twice: the re-aim, not a model turn
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_irreversible_never_refires_when_unverified():
|
||||
"""A send-class step acts once, the effect can't be verified -> honest note,
|
||||
exactly ONE action ever dispatched."""
|
||||
ex, calls = make_exec([BOX_FILLED, BOX_FILLED, BOX_FILLED])
|
||||
step = vs.VerifiedStep(kind="click", target="Send", role="button",
|
||||
expect="cleared:hello there friend", irreversible=True)
|
||||
r = await vs.run_verified_step(step, "b1", "", ex, settle_s=0, max_reaim=3)
|
||||
assert r["ok"] is False and r["acted"] is True
|
||||
assert "do NOT repeat" in r["note"]
|
||||
assert len(calls["acts"]) == 1 # the invariant: one irreversible dispatch, ever
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_irreversible_errored_action_stops_clean():
|
||||
"""An irreversible action that ERRORS provably never ran; stop without retry."""
|
||||
ex, calls = make_exec([BOX_FILLED], fail_actions=1)
|
||||
step = vs.VerifiedStep(kind="click", target="Send", irreversible=True)
|
||||
r = await vs.run_verified_step(step, "b1", "", ex, settle_s=0)
|
||||
assert r == {"ok": False, "verified": False, "acted": False, "note": "action errored: click failed"}
|
||||
assert len(calls["acts"]) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fill_defaults_to_filled_expectation():
|
||||
ex, calls = make_exec([BOX_EMPTY, BOX_FILLED])
|
||||
step = vs.VerifiedStep(kind="fill", target="Write a message", role="textbox",
|
||||
text="hello there friend")
|
||||
r = await vs.run_verified_step(step, "b1", "", ex, settle_s=0)
|
||||
assert r["ok"] is True
|
||||
assert calls["acts"][0][1] == {"index": 2, "text": "hello there friend"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_click_falls_to_by_name_when_index_unresolved():
|
||||
"""Target absent from the capped list (the overlay-Send lesson): the act goes
|
||||
through click-by-name's full-DOM search instead of failing."""
|
||||
ex, calls = make_exec([BOX_FILLED, BOX_SENT])
|
||||
step = vs.VerifiedStep(kind="click", target="Send", role="button",
|
||||
expect="cleared:hello there friend", irreversible=True)
|
||||
r = await vs.run_verified_step(step, "b1", "", ex, settle_s=0)
|
||||
assert r["ok"] is True
|
||||
assert calls["acts"][0] == ("BrowserClickByName", {"name": "Send", "role": "button"})
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_fill_with_unresolvable_field_hands_back():
|
||||
ex, calls = make_exec([MENU_CLOSED])
|
||||
step = vs.VerifiedStep(kind="fill", target="Write a message", text="hi")
|
||||
r = await vs.run_verified_step(step, "b1", "", ex, settle_s=0)
|
||||
assert r["acted"] is False and "could not resolve" in r["note"]
|
||||
assert not calls["acts"]
|
||||
@@ -0,0 +1,97 @@
|
||||
"""Unit tests for the learn-on-first-write / replay-on-repeat recipe module.
|
||||
Network mocked at route_write.issue_request; disk redirected to tmp_path."""
|
||||
|
||||
import json
|
||||
from typing import Any, Dict
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_write_recipes as wr
|
||||
from backend.apps.agents.browser import route_write
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def recipes_tmp(tmp_path, monkeypatch):
|
||||
monkeypatch.setattr(wr, "p_dir", lambda: str(tmp_path))
|
||||
|
||||
|
||||
P_X_BODY = {
|
||||
"variables": {"tweet_text": "hello from the test", "dark_request": False,
|
||||
"media": {"media_entities": [], "possibly_sensitive": False}},
|
||||
"features": {"tweetypie_unmention_optimization_enabled": True, "longform_notetweets_consumption_enabled": True},
|
||||
"queryId": "AbCdEf123456",
|
||||
}
|
||||
|
||||
|
||||
def p_routes(body: Dict[str, Any]) -> list:
|
||||
return [
|
||||
{"method": "GET", "template": "https://x.com/i/api/graphql/{id}/HomeTimeline", "example": "", "lastBody": ""},
|
||||
{"method": "POST", "template": "https://x.com/i/api/graphql/{id}/CreateTweet",
|
||||
"example": "https://x.com/i/api/graphql/AbCdEf123456/CreateTweet", "lastBody": json.dumps(body)},
|
||||
]
|
||||
|
||||
|
||||
def test_learn_finds_payload_slot_and_saves():
|
||||
r = wr.learn_recipe("x.com", "hello from the test", p_routes(P_X_BODY))
|
||||
assert r is not None
|
||||
assert r.payload_path == "$.variables.tweet_text"
|
||||
assert wr.SENTINEL in r.body_template
|
||||
assert "hello from the test" not in r.body_template
|
||||
assert wr.recipe_for("x.com") is not None
|
||||
|
||||
|
||||
def test_learn_refuses_substring_and_short_payloads():
|
||||
body = {"variables": {"tweet_text": "prefix hello from the test suffix"}}
|
||||
assert wr.learn_recipe("x.com", "hello from the test", p_routes(body)) is None
|
||||
assert wr.learn_recipe("x.com", "hi", p_routes(P_X_BODY)) is None
|
||||
|
||||
|
||||
def test_learn_redacts_secret_leaves_but_keeps_structure():
|
||||
body = {"variables": {"tweet_text": "hello from the test"}, "csrfish": "a1B2c3D4e5F6g7H8i9J0kk"}
|
||||
r = wr.learn_recipe("x.com", "hello from the test", p_routes(body))
|
||||
parsed = json.loads(r.body_template)
|
||||
assert parsed["csrfish"] == "<redacted>"
|
||||
assert parsed["variables"]["tweet_text"] == wr.SENTINEL
|
||||
|
||||
|
||||
def test_build_body_substitutes_new_payload_only():
|
||||
r = wr.learn_recipe("x.com", "hello from the test", p_routes(P_X_BODY))
|
||||
body = wr.build_body(r, "a brand new tweet")
|
||||
assert body["variables"]["tweet_text"] == "a brand new tweet"
|
||||
assert body["queryId"] == "AbCdEf123456"
|
||||
r.body_template = json.dumps({"variables": {"tweet_text": "no sentinel here"}})
|
||||
assert wr.build_body(r, "x") is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replay_ok_bumps_wins_and_returns_receipt(monkeypatch):
|
||||
monkeypatch.setenv("OSW_ROUTE_WRITE", "1")
|
||||
r = wr.learn_recipe("x.com", "hello from the test", p_routes(P_X_BODY))
|
||||
monkeypatch.setattr(route_write, "get_session", lambda d: ("auth=1; ct0=abc", "UA"))
|
||||
monkeypatch.setattr(route_write, "issue_request",
|
||||
lambda m, u, b, h: (200, json.dumps({"data": {"rest_id": "999"}})))
|
||||
out = await wr.replay_recipe(r, "new text", "https://x.com")
|
||||
assert out["ok"] is True and out["receipt"] == "999"
|
||||
assert wr.recipe_for("x.com").wins == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replay_miss_bumps_and_drops_after_cap(monkeypatch):
|
||||
monkeypatch.setenv("OSW_ROUTE_WRITE", "1")
|
||||
wr.learn_recipe("x.com", "hello from the test", p_routes(P_X_BODY))
|
||||
monkeypatch.setattr(route_write, "get_session", lambda d: ("auth=1", "UA"))
|
||||
monkeypatch.setattr(route_write, "issue_request", lambda m, u, b, h: (404, "gone"))
|
||||
for i in range(wr.MAX_MISSES):
|
||||
r = wr.recipe_for("x.com")
|
||||
assert r is not None, f"recipe gone before miss {i + 1}"
|
||||
out = await wr.replay_recipe(r, "t", "https://x.com")
|
||||
assert out["ok"] is False
|
||||
assert wr.recipe_for("x.com") is None # stale recipe self-evicted
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_replay_respects_flag_off(monkeypatch):
|
||||
monkeypatch.delenv("OSW_ROUTE_WRITE", raising=False)
|
||||
r = wr.learn_recipe("x.com", "hello from the test", p_routes(P_X_BODY))
|
||||
out = await wr.replay_recipe(r, "t", "https://x.com")
|
||||
assert out["ok"] is False and "disarmed" in out["error"]
|
||||
@@ -0,0 +1,178 @@
|
||||
"""Ranking the compose links a site publishes.
|
||||
|
||||
This is the half of the tier that decides where to point someone's browser, so it is a pure
|
||||
function over a page read and every rule below is a live failure it has to keep out: an off-host
|
||||
share button, a sign-out link, the page we are already on, and a deep link that merely shares a
|
||||
word with a compose path.
|
||||
"""
|
||||
from backend.apps.agents.browser import compose_discovery as cd
|
||||
|
||||
|
||||
def read(links, url="https://www.tumblr.com/dashboard"):
|
||||
return {"url": url, "links": [{"href": h, "label": l} for h, l in links]}
|
||||
|
||||
|
||||
def test_finds_the_link_the_site_calls_its_composer():
|
||||
got = cd.rank_candidates(read([
|
||||
("https://www.tumblr.com/explore", "Explore"),
|
||||
("https://www.tumblr.com/new/text", "Start a post"),
|
||||
]), "tumblr.com")
|
||||
assert got[0] == "https://www.tumblr.com/new/text"
|
||||
|
||||
|
||||
def test_a_label_outranks_a_path_that_merely_shares_a_word():
|
||||
""""Ask Question" is the front door; /questions/12345/new-answers only looks like one."""
|
||||
got = cd.rank_candidates(read([
|
||||
("https://stackoverflow.com/questions/12345/new-answers", "Recent answers"),
|
||||
("https://stackoverflow.com/questions/ask", "Ask Question"),
|
||||
], url="https://stackoverflow.com/"), "stackoverflow.com")
|
||||
assert got[0] == "https://stackoverflow.com/questions/ask"
|
||||
|
||||
|
||||
def test_a_share_button_never_hijacks_the_post():
|
||||
"""Every site carries share links to other networks; composing on tumblr must not open X."""
|
||||
got = cd.rank_candidates(read([
|
||||
("https://x.com/intent/tweet?text=hi", "Share on X"),
|
||||
("https://www.reddit.com/submit?url=x", "Share to Reddit"),
|
||||
]), "tumblr.com")
|
||||
assert got == []
|
||||
|
||||
|
||||
def test_destructive_and_account_links_are_never_candidates():
|
||||
for href, label in (
|
||||
("https://www.tumblr.com/logout", "Log out"),
|
||||
("https://www.tumblr.com/settings/account", "Settings"),
|
||||
("https://www.tumblr.com/purchase/premium", "Subscribe"),
|
||||
("https://www.tumblr.com/login", "Log in"),
|
||||
):
|
||||
assert cd.rank_candidates(read([(href, label)]), "tumblr.com") == [], href
|
||||
|
||||
|
||||
def test_the_page_we_are_already_on_is_not_a_candidate():
|
||||
"""Re-navigating remounts the page and throws away a composer that may be right there."""
|
||||
here = "https://www.tumblr.com/new/text"
|
||||
assert cd.rank_candidates(read([(here, "Start a post")], url=here), "tumblr.com") == []
|
||||
|
||||
|
||||
def test_a_page_with_no_compose_link_yields_nothing():
|
||||
got = cd.rank_candidates(read([
|
||||
("https://www.tumblr.com/explore/trending", "Trending"),
|
||||
("https://www.tumblr.com/tagged/cats", "cats"),
|
||||
]), "tumblr.com")
|
||||
assert got == []
|
||||
|
||||
|
||||
def test_subdomains_of_the_host_still_count():
|
||||
got = cd.rank_candidates(read([
|
||||
("https://medium.com/new-story", "Write a story"),
|
||||
], url="https://www.medium.com/"), "medium.com")
|
||||
assert got == ["https://medium.com/new-story"]
|
||||
|
||||
|
||||
def test_candidates_are_capped_so_a_miss_cannot_cost_the_whole_budget():
|
||||
many = [(f"https://www.tumblr.com/new/{i}", "Start a post") for i in range(10)]
|
||||
assert len(cd.rank_candidates(read(many), "tumblr.com")) <= cd.MAX_CANDIDATES
|
||||
|
||||
|
||||
def test_ties_are_deterministic():
|
||||
"""DOM order is not stable across loads; two runs on one site must probe the same URL."""
|
||||
a = [("https://www.tumblr.com/new/text", "Start a post"),
|
||||
("https://www.tumblr.com/new/link", "Start a post")]
|
||||
assert cd.rank_candidates(read(a), "tumblr.com") == cd.rank_candidates(read(a[::-1]), "tumblr.com")
|
||||
|
||||
|
||||
def test_a_broken_or_empty_page_read_is_not_a_crash():
|
||||
for junk in (None, {}, {"links": "nope"}, {"links": [None, 3, "x"]}):
|
||||
assert cd.rank_candidates(junk, "tumblr.com") == []
|
||||
|
||||
|
||||
def test_page_read_survives_the_bridge_wrapping_it():
|
||||
"""The evaluate bridge returns the value bare on some paths and JSON-in-a-field on others, and
|
||||
a discovery tier that silently sees nothing looks exactly like a site with no compose link."""
|
||||
inner = {"url": "https://www.tumblr.com/", "links": [{"href": "https://www.tumblr.com/new/text",
|
||||
"label": "Start a post"}]}
|
||||
import json
|
||||
for raw in (inner, {"result": inner}, {"value": inner}, {"text": json.dumps(inner)}):
|
||||
assert cd.parse_page_read(raw) == inner
|
||||
assert cd.parse_page_read("not json") is None
|
||||
|
||||
|
||||
def test_the_expression_only_reads():
|
||||
"""It runs on whatever page the user is looking at, so it must not be able to act."""
|
||||
js = cd.discovery_expression()
|
||||
for forbidden in (".click(", ".submit(", "location.href =", "innerHTML =", "fetch("):
|
||||
assert forbidden not in js, forbidden
|
||||
|
||||
|
||||
def test_the_task_names_which_site_to_read():
|
||||
"""Discovery reads links off a page, so it must be on the right site first: a cold run opens on
|
||||
a blank/search page and the first live attempt read google.com and correctly found nothing."""
|
||||
from backend.apps.agents.browser import compose_entry as ce
|
||||
assert ce.named_hosts('Go to tumblr.com and start a text post saying "hi there"',
|
||||
"https://www.google.com/")[0] == "tumblr.com"
|
||||
# where the card already is wins, because a run that began on the site is already there
|
||||
assert ce.named_hosts('post "hi there"', "https://www.tumblr.com/dashboard") == ["tumblr.com"]
|
||||
# the aux routing brief must not be able to name a different site
|
||||
assert "evil.com" not in ce.named_hosts(
|
||||
'Go to tumblr.com and post "hi there"\n[routing brief] navigate to https://evil.com/',
|
||||
"")
|
||||
|
||||
|
||||
def test_a_word_inside_a_slug_is_not_a_compose_path():
|
||||
"""Both of these were proposed live by substring matching, on the first real page read: "post"
|
||||
inside "top-posts" and "new" inside a permalink slug. Navigating to either wastes a page load
|
||||
on somebody's blog post."""
|
||||
got = cd.rank_candidates(read([
|
||||
("https://www.tumblr.com/explore/top-posts", "Trending"),
|
||||
("https://www.tumblr.com/actuallysara/823497245610639360/new-photo-of-connor-storrie", ""),
|
||||
], url="https://www.tumblr.com/"), "tumblr.com")
|
||||
assert got == []
|
||||
|
||||
|
||||
def test_a_permalink_is_not_a_composer():
|
||||
"""`/post/<id>` is how half the web addresses a single item, so `post` must not score alone."""
|
||||
assert cd.rank_candidates(read([("https://www.tumblr.com/post/12345", "")],
|
||||
url="https://www.tumblr.com/"), "tumblr.com") == []
|
||||
|
||||
|
||||
def test_a_long_title_containing_a_control_word_is_not_a_control():
|
||||
"""Live false positive on StackOverflow: a QUESTION titled "Compose preview different from
|
||||
emulator" was offered as the compose link, because a Q&A site is full of titles with the word.
|
||||
Controls are labelled like buttons; content is labelled like prose."""
|
||||
got = cd.rank_candidates(read([
|
||||
("https://stackoverflow.com/questions/79988032/compose-preview-different-from-emulator",
|
||||
"Compose preview different from emulator"),
|
||||
("https://stackoverflow.com/questions/ask", "Ask Question"),
|
||||
], url="https://stackoverflow.com/"), "stackoverflow.com")
|
||||
assert got == ["https://stackoverflow.com/questions/ask"]
|
||||
|
||||
|
||||
def test_open_a_new_x_is_a_create_but_bare_open_is_navigation():
|
||||
from backend.apps.agents.browser import compose_entry as ce
|
||||
assert ce.wants_top_level_compose(
|
||||
'Go to github.com and open a new issue whose body is exactly: "hello there"') is True
|
||||
# bare "open" is how every read-then-act task begins, and those are replies, not top-level posts
|
||||
assert ce.wants_top_level_compose(
|
||||
'Go to youtube.com, open the first video, and write a comment saying: "hello there"') is False
|
||||
|
||||
|
||||
def test_discovery_reads_the_page_the_user_named_not_just_the_host():
|
||||
"""Live: github.com/ publishes no compose link, so reading the host root found nothing while
|
||||
the repo the user actually named carries "New issue"."""
|
||||
from backend.apps.agents.browser import compose_entry as ce
|
||||
task = 'Go to https://github.com/openswarm-ai/openswarm and open a new issue saying "hi there"'
|
||||
assert ce.named_page(task, "github.com") == "https://github.com/openswarm-ai/openswarm"
|
||||
# no page named: the front page is the only thing to go on
|
||||
assert ce.named_page('Go to tumblr.com and post "hi there"', "tumblr.com") == "https://tumblr.com/"
|
||||
# a URL on some OTHER host must not become the page we read
|
||||
assert ce.named_page('post "hi" about https://nytimes.com/x on tumblr.com',
|
||||
"tumblr.com") == "https://tumblr.com/"
|
||||
|
||||
|
||||
def test_the_tier_ships_off_until_it_is_shown_to_help(monkeypatch):
|
||||
"""Its parts are proven (it returned StackOverflow's /questions/ask first, correctly) but its
|
||||
measured end-to-end contribution is zero, so it stays behind a switch."""
|
||||
monkeypatch.delenv("OSW_COMPOSE_DISCOVERY", raising=False)
|
||||
assert cd.enabled() is False
|
||||
monkeypatch.setenv("OSW_COMPOSE_DISCOVERY", "1")
|
||||
assert cd.enabled() is True
|
||||
@@ -0,0 +1,218 @@
|
||||
"""The compose-entry table may only ever fire on a top-level create aimed at the bare site.
|
||||
|
||||
This tier navigates the user's real browser somewhere it chose, so its guards are the safety
|
||||
surface, not the table. Composing in the wrong place is worse than not composing: a hijacked reply
|
||||
posts a stranger's answer to the whole feed, and a hijacked permalink abandons the target the user
|
||||
explicitly named. Both are silent, and neither is undone by the two-sided receipt downstream, which
|
||||
proves only that SOMETHING was posted.
|
||||
|
||||
So the interesting cases here are all refusals.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import compose_entry as ce
|
||||
|
||||
|
||||
def entry(task, start, is_send=True):
|
||||
"""Call the tier the way the browser agent does. `is_send` is its already-computed write
|
||||
verdict; almost every case here is a write, so it defaults on and the read cases pass it
|
||||
explicitly."""
|
||||
return ce.compose_entry_for(task, start, is_send)
|
||||
|
||||
|
||||
|
||||
# --- it fires where it should ------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("start,task,want_host", [
|
||||
("https://x.com/home", 'post this tweet: "hello"', "x.com"),
|
||||
("https://www.linkedin.com/feed/", 'start a post saying "hello"', "linkedin.com"),
|
||||
("https://www.reddit.com/", 'start a text post whose body is "hello"', "reddit.com"),
|
||||
("https://mail.google.com/mail/u/0/", 'compose an email saying "hello"', "mail.google.com"),
|
||||
])
|
||||
def test_a_top_level_create_gets_the_sites_compose_url(start, task, want_host):
|
||||
got = entry(task, start, True)
|
||||
assert got and ce.registrable_host(got) == want_host
|
||||
|
||||
|
||||
def test_the_host_can_come_from_the_task_when_the_tab_is_blank():
|
||||
"""A run that starts on about:blank still says where it is going."""
|
||||
got = entry('go to x.com and post "hello there"', "about:blank", True)
|
||||
assert got == "https://x.com/compose/post"
|
||||
|
||||
|
||||
def test_subdomains_resolve_to_the_parent_sites_composer():
|
||||
assert entry('post "hello there"', "https://old.reddit.com/r/test") is not None
|
||||
|
||||
|
||||
def test_www_and_mobile_prefixes_are_the_same_site():
|
||||
for host in ("www.x.com", "m.x.com", "mobile.x.com", "x.com"):
|
||||
assert ce.registrable_host(host) == "x.com"
|
||||
|
||||
|
||||
# --- it refuses where it must ------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("task", [
|
||||
'reply to this tweet with "hello there"',
|
||||
'comment "nice work" on the top post',
|
||||
'respond to her message saying "hello there"',
|
||||
'quote tweet it with "hello there"',
|
||||
'send a DM saying "hello there"',
|
||||
])
|
||||
def test_answering_something_keeps_its_own_target(task):
|
||||
"""A reply belongs on the thread in front of the user. Hijacking it to the site composer
|
||||
publishes a private answer to everyone, which is the worst failure this file guards."""
|
||||
assert entry(task, "https://x.com/home") is None
|
||||
|
||||
|
||||
def test_the_refusals_above_are_caused_by_the_respond_word():
|
||||
"""Positive control. Every refusal test would also pass if the tier were simply broken and
|
||||
always returned None, so prove the same sentence DOES fire once the answering word is gone."""
|
||||
assert entry('reply with "hello there"', "https://x.com/home") is None
|
||||
assert entry('post "hello there"', "https://x.com/home") is not None
|
||||
|
||||
|
||||
def test_a_permalink_in_the_task_outranks_the_generic_composer():
|
||||
task = 'post "hello there" on https://x.com/someone/status/12345'
|
||||
assert entry(task, "https://x.com/home") is None
|
||||
|
||||
|
||||
def test_a_bare_host_url_in_the_task_is_not_a_deeper_target():
|
||||
assert entry('go to https://x.com/ and post "hello there"', "about:blank") is not None
|
||||
|
||||
|
||||
def test_a_query_or_fragment_also_counts_as_a_chosen_target():
|
||||
for url in ("https://www.reddit.com/?feed=home", "https://www.reddit.com/#top"):
|
||||
assert entry(f'post "hello there" at {url}', "https://www.reddit.com/") is None
|
||||
|
||||
|
||||
def test_a_read_task_never_navigates():
|
||||
"""Two independent reasons, and the tier needs only one of them."""
|
||||
assert entry("what is the top post on reddit", "https://www.reddit.com/") is None
|
||||
assert entry("what is the top post on reddit", "https://www.reddit.com/", False) is None
|
||||
|
||||
|
||||
@pytest.mark.parametrize("task", [
|
||||
'find the reddit post that says "hello there"',
|
||||
'check if my tweet "hello there" got any likes',
|
||||
'what does the post "hello there" say',
|
||||
'summarize the linkedin post about "quarterly results"',
|
||||
])
|
||||
def test_a_quoted_READ_never_opens_a_composer(task):
|
||||
"""The bug this argument exists for. Each of these quotes something and contains a word that
|
||||
reads as a create ("post" as a NOUN, "tweet" as a noun), and each one resolved to reddit's
|
||||
SUBMIT page before the caller's verdict was required. Navigating a read to a compose form
|
||||
derails the task and leaves the user staring at a half-open post box."""
|
||||
assert entry(task, "https://www.reddit.com/", False) is None
|
||||
|
||||
|
||||
def test_an_unknown_site_is_left_alone():
|
||||
assert entry('post "hello there"', "https://example.com/") is None
|
||||
|
||||
|
||||
def test_a_lookalike_domain_is_not_the_real_one():
|
||||
"""`endswith("reddit.com")` matches `notreddit.com`. That would navigate a post intended for a
|
||||
site the user named to a completely different one."""
|
||||
for host in ("notreddit.com", "fake-x.com", "linkedin.com.evil.test"):
|
||||
assert entry('post "hello there"', f"https://{host}/") is None
|
||||
|
||||
|
||||
def test_already_on_the_compose_surface_does_not_remount_it():
|
||||
"""Re-navigating throws away a composer that is already open, and on a modal that means the
|
||||
user's half-typed state too."""
|
||||
assert entry('post "hello there"', "https://x.com/compose/post") is None
|
||||
assert entry('start a post "hello there"',
|
||||
"https://www.linkedin.com/feed/?shareActive=true") is None
|
||||
|
||||
|
||||
# --- the aux routing brief is commentary, not the request ---------------------------------------
|
||||
|
||||
def p_dispatched(prompt: str, brief: str) -> str:
|
||||
"""Exactly what a dispatched browser task looks like: the user's words, then a brief a model
|
||||
wrote about routing them."""
|
||||
from backend.apps.agents.browser import browser_fast_path
|
||||
return browser_fast_path.compose_task(prompt, brief)
|
||||
|
||||
|
||||
def test_a_brief_that_quotes_something_does_not_disarm_the_tier():
|
||||
"""The live miss. The brief adds a second quoted span, `quoted_payload` calls the task
|
||||
ambiguous and returns "", and the tier silently declined on every real run while its unit tests
|
||||
(which passed the bare prompt) stayed green."""
|
||||
task = p_dispatched('Go to x.com and post this tweet, exactly: "coverage probe 1"',
|
||||
'Open the composer and type the text. Click "Post" to publish.')
|
||||
assert entry(task, "") == "https://x.com/compose/post"
|
||||
|
||||
|
||||
def test_a_brief_using_answering_words_does_not_disarm_the_tier():
|
||||
task = p_dispatched('post "hello there" on x.com',
|
||||
'If a dialog appears, respond to it and reply to any prompt.')
|
||||
assert entry(task, "") is not None
|
||||
|
||||
|
||||
def test_a_brief_naming_another_site_cannot_redirect_the_post():
|
||||
"""Which site to open is the user's call. A brief is a model's guess and must not move a post
|
||||
to a different service."""
|
||||
task = p_dispatched('post "hello there" on x.com', 'You may find this on reddit.com instead.')
|
||||
assert entry(task, "") == "https://x.com/compose/post"
|
||||
|
||||
|
||||
def test_only_a_permalink_the_USER_named_vetoes():
|
||||
"""Measured live: briefs spell out a route ("navigate to https://x.com/home"), and letting that
|
||||
veto silently disabled the tier on x and linkedin while reddit worked. A brief cannot turn a
|
||||
post into a reply, because the words that would say so are the user's and are still read."""
|
||||
brief_only = p_dispatched('post "hello there" on x.com',
|
||||
'The target appears to be https://x.com/someone/status/12345')
|
||||
assert entry(brief_only, "") == "https://x.com/compose/post"
|
||||
user_named = p_dispatched('post "hello there" on https://x.com/someone/status/12345',
|
||||
'Open the composer.')
|
||||
assert entry(user_named, "") is None
|
||||
|
||||
|
||||
# --- host parsing can't be sloppy ---------------------------------------------------------------
|
||||
|
||||
def test_prefix_stripping_uses_a_real_prefix_check():
|
||||
"""`lstrip("www.")` eats any leading w or dot; it turns w3schools into 3schools."""
|
||||
assert ce.registrable_host("w3schools.com") == "w3schools.com"
|
||||
assert ce.registrable_host("wow.com") == "wow.com"
|
||||
|
||||
|
||||
def test_ports_and_credentials_are_dropped():
|
||||
assert ce.registrable_host("https://user@x.com:443/home") == "x.com"
|
||||
|
||||
|
||||
def test_garbage_never_raises():
|
||||
for bad in ("", " ", "not a url", "://", "https://"):
|
||||
assert isinstance(ce.registrable_host(bad), str)
|
||||
assert entry('post "hello there"', bad) is None
|
||||
|
||||
|
||||
def test_natural_create_phrasings_are_recognised():
|
||||
"""Measured live: 4 of 5 top-level-create tasks never reached the tier because the verb was
|
||||
missing from the create vocabulary, so the gate was what got measured rather than the tier."""
|
||||
for phrasing in (
|
||||
'Go to threads.net and start a new thread saying exactly: "hello there"',
|
||||
'Go to medium.com and start writing a new story whose body is exactly: "hello there"',
|
||||
'Go to stackoverflow.com and start asking a question whose body is exactly: "hello there"',
|
||||
'Go to reddit.com and create a text post whose body is exactly: "hello there"',
|
||||
):
|
||||
assert ce.wants_top_level_compose(phrasing) is True, phrasing
|
||||
|
||||
|
||||
def test_replies_stay_out_of_the_top_level_tier():
|
||||
"""Widening the create verbs must not swallow responses: a comment box lives on the item the
|
||||
user is looking at, never at a site-wide compose URL."""
|
||||
for phrasing in (
|
||||
'Go to youtube.com, open the first video, and write a comment saying exactly: "hello there"',
|
||||
'Go to quora.com and start writing an answer that says exactly: "hello there"',
|
||||
'Go to instagram.com, open the first post, and write a comment saying exactly: "hello there"',
|
||||
):
|
||||
assert ce.wants_top_level_compose(phrasing) is False, phrasing
|
||||
|
||||
|
||||
def test_a_read_is_still_never_a_create():
|
||||
"""The words got broader, so the guard that keeps a quoted READ off a submit page matters more."""
|
||||
for phrasing in (
|
||||
'find the reddit post that says "hello there"',
|
||||
'search x.com for the tweet that starts with "hello there"',
|
||||
'what does the post saying "hello there" contain',
|
||||
):
|
||||
assert ce.compose_entry_for(phrasing, "https://reddit.com/", False) is None, phrasing
|
||||
@@ -0,0 +1,96 @@
|
||||
"""The OAuth redirect URI must name the port this backend is actually reachable on.
|
||||
|
||||
Measured live 2026-07-28. Google bounced a real connect attempt to
|
||||
http://localhost:8324/api/subscriptions/callback -> ERR_CONNECTION_REFUSED, while the backend was
|
||||
serving 8326. Claude failed at the same moment for its own reason and Codex kept working, so the
|
||||
symptom presented as "two providers are broken" rather than "the port is wrong", which is the
|
||||
expensive kind of wrong.
|
||||
|
||||
Root cause: main.py exports OPENSWARM_PORT inside `if __name__ == "__main__"`. That block does not
|
||||
run under `python -m uvicorn backend.main:app --port N`, so the env var was absent and the helper
|
||||
fell back to the 8324 literal while uvicorn served something else. Packaged builds were never
|
||||
affected (electron/main.js passes OPENSWARM_PORT explicitly), which is exactly why it survived:
|
||||
it is invisible on the default port and invisible in prod.
|
||||
|
||||
The fix keeps the env var authoritative and uses the live request's port only as the fallback, so
|
||||
prod behaviour is unchanged and the dev path stops guessing.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from backend.apps.nine_router import oauth
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def no_ambient_port(monkeypatch):
|
||||
"""The suite must not inherit a real OPENSWARM_PORT from the developer's shell."""
|
||||
monkeypatch.delenv("OPENSWARM_PORT", raising=False)
|
||||
|
||||
|
||||
# --- the port helper -------------------------------------------------------------------------
|
||||
|
||||
def test_the_env_var_wins_when_set(monkeypatch):
|
||||
"""Packaged builds always set it; that path must not change."""
|
||||
monkeypatch.setenv("OPENSWARM_PORT", "8324")
|
||||
assert oauth.resolve_backend_port(observed=9999) == 8324
|
||||
|
||||
|
||||
def test_the_observed_port_is_used_when_the_env_var_is_missing():
|
||||
"""The regression: uvicorn on 8326 with no env var used to answer 8324."""
|
||||
assert oauth.resolve_backend_port(observed=8326) == 8326
|
||||
|
||||
|
||||
def test_it_still_falls_back_when_nothing_is_known():
|
||||
assert oauth.resolve_backend_port() == 8324
|
||||
|
||||
|
||||
def test_a_garbage_env_var_does_not_crash_the_connect_flow(monkeypatch):
|
||||
"""A malformed value must degrade to what we can observe, not raise mid-OAuth."""
|
||||
monkeypatch.setenv("OPENSWARM_PORT", "not-a-port")
|
||||
assert oauth.resolve_backend_port(observed=8326) == 8326
|
||||
|
||||
|
||||
def test_an_empty_env_var_is_treated_as_unset(monkeypatch):
|
||||
monkeypatch.setenv("OPENSWARM_PORT", "")
|
||||
assert oauth.resolve_backend_port(observed=8326) == 8326
|
||||
|
||||
|
||||
# --- the redirect URI itself -----------------------------------------------------------------
|
||||
|
||||
def test_google_gets_the_live_port_not_the_default():
|
||||
"""The exact failure: gemini-cli's callback runs through our own backend endpoint."""
|
||||
uri = oauth.callback_uri_for_provider("gemini-cli", 8326)
|
||||
assert uri == "http://localhost:8326/api/subscriptions/callback"
|
||||
|
||||
|
||||
def test_google_on_the_default_port_is_unchanged(monkeypatch):
|
||||
monkeypatch.setenv("OPENSWARM_PORT", "8324")
|
||||
assert oauth.callback_uri_for_provider("gemini-cli", 8326) == (
|
||||
"http://localhost:8324/api/subscriptions/callback")
|
||||
|
||||
|
||||
def test_codex_keeps_its_pinned_listener_port():
|
||||
"""OpenAI's client is bound to a fixed URI, which is why Codex kept connecting while the other
|
||||
two failed. It must never pick up the backend port."""
|
||||
uri = oauth.callback_uri_for_provider("codex", 8326)
|
||||
assert uri == "http://localhost:1455/auth/callback", (
|
||||
"OpenAI's OAuth client is registered against this exact URI; changing it breaks every "
|
||||
"ChatGPT connect")
|
||||
assert "8326" not in uri
|
||||
|
||||
|
||||
def test_claude_still_routes_through_the_router_callback():
|
||||
"""Anthropic only whitelists the router's callback; the backend port is not ours to substitute."""
|
||||
uri = oauth.callback_uri_for_provider("claude", 8326)
|
||||
assert str(oauth.NINE_ROUTER_PORT) in uri
|
||||
assert "8326" not in uri
|
||||
|
||||
|
||||
def test_no_provider_silently_hardcodes_the_default_port():
|
||||
"""Sweep every provider the flow knows about: on a non-default port, nothing may still say 8324
|
||||
unless it is deliberately router- or listener-pinned."""
|
||||
pinned = {"claude", "codex"}
|
||||
for provider in ("gemini-cli", "antigravity", "github", "qwen", "kiro"):
|
||||
if provider in pinned:
|
||||
continue
|
||||
uri = oauth.callback_uri_for_provider(provider, 8326)
|
||||
assert "8324" not in uri, f"{provider} stamped the default port into {uri}"
|
||||
@@ -0,0 +1,114 @@
|
||||
"""A check that could not observe must say "unknown", never "no".
|
||||
|
||||
This bug class cost most of a night. The identical shape appeared in three places, and in every one
|
||||
it manufactured a confident false answer:
|
||||
|
||||
1. the send receipt a cleared composer was read as delivered even when the site had refused
|
||||
2. the campaign verifier a verify turn that could not reach the profile scored the post "not landed"
|
||||
3. the cleanup accounting a recheck that could not read the page scored the post "deleted"
|
||||
|
||||
(3) is the one that did real damage: it reported a clean account while six test posts sat live on
|
||||
it, twice. All three are the same mistake, absence of evidence recorded as evidence of absence, and
|
||||
patching them one at a time is treating symptoms.
|
||||
|
||||
The rule, and what this file pins:
|
||||
|
||||
- An observation returns True / False / None, where None means the observation did not happen.
|
||||
- Collapsing None is allowed in exactly one direction: when deciding whether to CLAIM something,
|
||||
unknown must behave as "do not claim". Withholding an uncertain claim is safe; asserting a
|
||||
negative from a failed look is not.
|
||||
"""
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_delivery_check as dc
|
||||
|
||||
|
||||
def p_eval(value):
|
||||
async def run(tool, args, browser_id, tab_id):
|
||||
return {"value": value}
|
||||
return run
|
||||
|
||||
|
||||
async def p_boom(tool, args, browser_id, tab_id):
|
||||
raise RuntimeError("probe died")
|
||||
|
||||
|
||||
async def p_hang(tool, args, browser_id, tab_id):
|
||||
await asyncio.sleep(30)
|
||||
return {}
|
||||
|
||||
|
||||
# --- payload_visible is a three-valued observation -------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_seen_is_true():
|
||||
assert await dc.payload_visible("hi", "b", "t", p_eval({"visible": True})) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_looked_and_absent_is_false():
|
||||
"""The genuine negative: the probe ran and the text is not on the page."""
|
||||
assert await dc.payload_visible("hi", "b", "t", p_eval({"visible": False})) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_failed_probe_is_unknown_not_absent():
|
||||
"""The regression. Returning False here tells the user a post did not land when nobody looked."""
|
||||
assert await dc.payload_visible("hi", "b", "t", p_boom) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_hung_probe_is_unknown():
|
||||
assert await dc.payload_visible("hi", "b", "t", p_hang) is None
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_an_unreadable_shape_is_unknown():
|
||||
"""A dict without the key is not a negative answer; it is a broken answer."""
|
||||
assert await dc.payload_visible("hi", "b", "t", p_eval({"nope": 1})) is None
|
||||
assert await dc.payload_visible("hi", "b", "t", p_eval("garbage")) is None
|
||||
|
||||
|
||||
# --- the one legitimate collapse: do not CLAIM on unknown ------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ghost_confirmation_refuses_to_claim_on_unknown():
|
||||
"""Deciding whether to assert a delivery: unknown must behave as "do not assert"."""
|
||||
assert await dc.ghost_delivery_confirmed("hi", "b", "t", p_boom) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ghost_confirmation_still_confirms_a_real_survival():
|
||||
assert await dc.ghost_delivery_confirmed("hi", "b", "t", p_eval({"visible": True})) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_ghost_confirmation_returns_a_hard_bool():
|
||||
"""Its contract is binary by design (claim / do not claim); leaking a None here would make an
|
||||
unknown look like a confirmed delivery to any `if` further up."""
|
||||
for probe in (p_boom, p_eval({"visible": True}), p_eval({"visible": False})):
|
||||
got = await dc.ghost_delivery_confirmed("hi", "b", "t", probe)
|
||||
assert got is True or got is False
|
||||
|
||||
|
||||
# --- the rejection probe collapses the other way, and that is also correct --------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_broken_rejection_probe_does_not_invent_a_refusal():
|
||||
"""Mirror image: send_rejected decides whether to assert a FAILURE, so unknown must behave as
|
||||
"do not assert" there too. Same rule, opposite polarity, because the claim is inverted."""
|
||||
assert await dc.send_rejected("b", "t", p_boom) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_send_script_distinguishes_absent_from_unlooked():
|
||||
"""The caller must branch on `is False` / `is None`, not truthiness. `if not delivered` treats
|
||||
an unknown exactly like a proven absence, which is the bug this whole file exists for."""
|
||||
src = dc.__file__.replace("browser_delivery_check.py", "browser_send_script.py")
|
||||
with open(src) as f:
|
||||
text = f.read()
|
||||
assert "delivered is False" in text
|
||||
assert "delivered is None" in text
|
||||
assert "if not delivered:" not in text, "truthiness collapses unknown into absent"
|
||||
@@ -64,7 +64,7 @@ def test_win_storage_key_parses_local_state_and_unwraps(monkeypatch, tmp_path):
|
||||
seen["passed"] = data
|
||||
return b"unwrapped-aes-key"
|
||||
|
||||
monkeypatch.setattr(browser_cookies, "p_win_dpapi_unprotect", fake_unprotect)
|
||||
monkeypatch.setattr(browser_cookies, "win_dpapi_unprotect", fake_unprotect)
|
||||
key = browser_cookies.win_storage_key("Chrome")
|
||||
assert key == b"unwrapped-aes-key"
|
||||
assert seen["passed"] == b"wrapped-key-bytes" # the 5-byte "DPAPI" prefix was stripped
|
||||
|
||||
@@ -0,0 +1,37 @@
|
||||
"""When to stop waiting for a compose surface that was just opened.
|
||||
|
||||
Split out of test_browser_send_script.py, which hit the 300-line cap. The rule under test is its
|
||||
own idea and deserves its own file: a fixed wait was wrong in both directions (1.8s missed gmail
|
||||
and linkedin; 5.3s still missed a cold gmail compose window; raising it further taxes every run
|
||||
that was never going to succeed), so the stop condition is page stability instead of a clock.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from backend.tests.test_browser_send_script import (
|
||||
COMPOSER_EMPTY, COMPOSER_FILLED, COMPOSER_SENT, PROFILE, TASK, run,
|
||||
)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_opener_wait_stops_once_the_surface_settles():
|
||||
"""A budget spent waiting on a page that has stopped changing is pure cost.
|
||||
|
||||
Fixed budgets were wrong in both directions: 1.8s missed gmail and linkedin, 5.3s still missed
|
||||
a cold gmail compose window, and just raising the number taxes every run that was never going
|
||||
to work. The stop condition is now page stability, so an opener that leads nowhere gives up as
|
||||
soon as two reads match instead of sleeping out the rest."""
|
||||
# Opener clicked, then the same composer-less page twice: identical reads = nothing coming.
|
||||
r, calls = await run(TASK, PROFILE, [PROFILE, PROFILE, PROFILE, PROFILE, PROFILE, PROFILE])
|
||||
assert r is None
|
||||
# 6 sleeps were budgeted; settling must cut the reads well short of consuming them all.
|
||||
assert calls["list"] < 9, f"kept polling a settled page: {calls['list']} reads"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_opener_wait_still_catches_a_late_composer():
|
||||
"""The other direction: a surface that is still mounting must not be abandoned early, which is
|
||||
the failure that made gmail intermittent."""
|
||||
# Changing reads (so never 'settled'), composer only on the 4th look.
|
||||
late = [PROFILE, PROFILE, PROFILE, PROFILE + "\n[99]<button \"x\">",
|
||||
COMPOSER_EMPTY, COMPOSER_FILLED, COMPOSER_SENT]
|
||||
r, calls = await run(TASK, PROFILE, late)
|
||||
assert r is not None and r["sent"] is True
|
||||
@@ -102,6 +102,36 @@ def test_comment_parses_new_thing():
|
||||
assert json.loads(p_text(out))["id"] == "t1_new"
|
||||
|
||||
|
||||
def test_comment_recovers_receipt_from_legacy_jquery_shape():
|
||||
# The classic web endpoint sometimes answers /api/comment with a "jquery" command
|
||||
# array (no data.things), which the old parse read as empty -> receipt "ok". The new
|
||||
# thing's fullname + permalink are still echoed, so p_receipt must recover them and
|
||||
# NOT return the parent id we replied to.
|
||||
resp = {"jquery": [
|
||||
[0, 1, "attr", "find"],
|
||||
[1, 2, "call", ["#thing_t3_parent"]],
|
||||
[2, 3, "call", ["t1_new1", "/r/x/comments/parent/_/t1_new1/"]],
|
||||
], "success": True}
|
||||
with patch.object(reddit_writes, "api", return_value=resp):
|
||||
out = handle_tool_call("reddit_comment", {"parent_id": "t3_parent", "text": "nice"})
|
||||
data = json.loads(p_text(out))
|
||||
assert data["id"] == "t1_new1"
|
||||
assert data["permalink"] == "/r/x/comments/parent/_/t1_new1/"
|
||||
|
||||
|
||||
def test_comment_reply_scan_skips_the_parent_comment():
|
||||
# Replying to a t1_ comment: the jquery response echoes BOTH the parent t1_ and the
|
||||
# new t1_. Parent appears first (its DOM node is the insert target); exclude it.
|
||||
resp = {"jquery": [
|
||||
[0, 1, "call", ["#thing_t1_parent"]],
|
||||
[1, 2, "call", ["t1_parent"]],
|
||||
[2, 3, "call", ["t1_child"]],
|
||||
]}
|
||||
with patch.object(reddit_writes, "api", return_value=resp):
|
||||
out = handle_tool_call("reddit_comment", {"parent_id": "t1_parent", "text": "reply"})
|
||||
assert json.loads(p_text(out))["id"] == "t1_child"
|
||||
|
||||
|
||||
def test_submit_surfaces_reddit_errors():
|
||||
envelope = {"json": {"errors": [["SUBREDDIT_NOEXIST", "that subreddit doesn't exist", "sr"]], "data": {}}}
|
||||
with patch.object(reddit_writes, "api", return_value=envelope):
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
"""Unit tests for the general capture-replay write engine (route_write): the safety walls
|
||||
(disarmed / off-origin / un-captured all refuse), CSRF-from-cookie derivation, the generic
|
||||
receipt parse, and the fail-open contract (every failure is a typed ok=False, never a crash,
|
||||
never a false success). Network is stubbed; the live cross-site round-trip is owed on a healthy
|
||||
rig (this bench's renderer command path is wedged, same as all browser live-tests)."""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import route_write as rw
|
||||
|
||||
|
||||
def p_arm(monkeypatch):
|
||||
monkeypatch.setenv("OSW_ROUTE_WRITE", "1")
|
||||
|
||||
|
||||
def p_reddit_route():
|
||||
return [rw.CapturedRoute(method="POST", template="https://www.reddit.com/api/comment")]
|
||||
|
||||
|
||||
# --- safety walls -----------------------------------------------------------
|
||||
def test_disarmed_by_default_refuses(monkeypatch):
|
||||
monkeypatch.delenv("OSW_ROUTE_WRITE", raising=False)
|
||||
out = rw.replay_write("POST", "https://www.reddit.com/api/comment", {"text": "hi"},
|
||||
"https://www.reddit.com", p_reddit_route())
|
||||
assert out.ok is False and "disarmed" in out.error
|
||||
|
||||
|
||||
def test_off_origin_target_refused(monkeypatch):
|
||||
p_arm(monkeypatch)
|
||||
out = rw.replay_write("POST", "https://evil.com/api/comment", {"text": "hi"},
|
||||
"https://www.reddit.com", p_reddit_route())
|
||||
assert out.ok is False and "same-origin" in out.error
|
||||
|
||||
|
||||
def test_uncaptured_route_refused(monkeypatch):
|
||||
p_arm(monkeypatch)
|
||||
# same origin, but the site's UI never fired /api/delete_account -> the agent can't invent it
|
||||
out = rw.replay_write("POST", "https://www.reddit.com/api/delete_account", {},
|
||||
"https://www.reddit.com", p_reddit_route())
|
||||
assert out.ok is False and "captured" in out.error
|
||||
|
||||
|
||||
def test_get_is_not_a_write_route(monkeypatch):
|
||||
p_arm(monkeypatch)
|
||||
assert rw.route_is_captured("GET", "https://www.reddit.com/api/comment", p_reddit_route()) is False
|
||||
|
||||
|
||||
def test_template_match_ignores_volatile_ids(monkeypatch):
|
||||
# A volatile id that IS a full path segment collapses to {id} on both sides (same regex as the
|
||||
# capture), so a concrete replay URL matches the captured template but a different path doesn't.
|
||||
routes = [rw.CapturedRoute(method="DELETE", template="https://api.site.com/orders/{id}/cancel")]
|
||||
assert rw.route_is_captured("DELETE", "https://api.site.com/orders/4821/cancel", routes) is True
|
||||
assert rw.route_is_captured("DELETE", "https://api.site.com/refunds/4821/cancel", routes) is False
|
||||
|
||||
|
||||
# --- CSRF-from-cookie derivation --------------------------------------------
|
||||
def test_csrf_header_derived_from_cookie():
|
||||
h = rw.derive_csrf_headers("https://x.com/i/api/graphql/CreateTweet", "ct0=abc123; auth_token=z")
|
||||
assert h == {"x-csrf-token": "abc123"}
|
||||
|
||||
|
||||
def test_no_csrf_for_plain_cookie_auth_site():
|
||||
assert rw.derive_csrf_headers("https://www.reddit.com/api/comment", "reddit_session=z") == {}
|
||||
|
||||
|
||||
def test_csrf_absent_when_cookie_missing():
|
||||
assert rw.derive_csrf_headers("https://x.com/foo", "auth_token=z") == {}
|
||||
|
||||
|
||||
# --- receipt parse ----------------------------------------------------------
|
||||
def test_receipt_prefers_permalink_then_ids():
|
||||
assert rw.receipt_from_json({"json": {"data": {"permalink": "/r/x/c/1", "id": "t1_9"}}}) == "/r/x/c/1"
|
||||
assert rw.receipt_from_json({"data": {"create_tweet": {"tweet_results": {"rest_id": "1899"}}}}) == "1899"
|
||||
assert rw.receipt_from_json({"nothing": True}) == ""
|
||||
|
||||
|
||||
def test_outcome_2xx_is_ok_with_receipt():
|
||||
out = rw.outcome_from_response(200, '{"id_str": "1899"}', 42)
|
||||
assert out.ok is True and out.receipt == "1899" and out.status == 200
|
||||
|
||||
|
||||
def test_outcome_2xx_non_json_is_ok_generic_receipt():
|
||||
out = rw.outcome_from_response(201, "created", 5)
|
||||
assert out.ok is True and out.receipt == "ok"
|
||||
|
||||
|
||||
def test_outcome_4xx_is_error():
|
||||
out = rw.outcome_from_response(403, "forbidden csrf", 9)
|
||||
assert out.ok is False and "403" in out.error
|
||||
|
||||
|
||||
# --- end-to-end with the network + session stubbed --------------------------
|
||||
def test_full_replay_success(monkeypatch):
|
||||
p_arm(monkeypatch)
|
||||
monkeypatch.setattr(rw, "get_session", lambda d: ("ct0=tok; sess=z", "UA/1.0"))
|
||||
seen = {}
|
||||
|
||||
def fake_issue(method, url, body, headers):
|
||||
seen["method"], seen["url"], seen["body"], seen["headers"] = method, url, body, headers
|
||||
return 200, '{"json": {"data": {"things": [{"data": {"name": "t1_new", "permalink": "/r/x/c/a/_/t1_new"}}]}}}'
|
||||
|
||||
monkeypatch.setattr(rw, "issue_request", fake_issue)
|
||||
out = rw.replay_write("POST", "https://www.reddit.com/api/comment", {"text": "nice", "thing_id": "t3_a"},
|
||||
"https://www.reddit.com", p_reddit_route())
|
||||
assert out.ok is True and out.receipt == "/r/x/c/a/_/t1_new"
|
||||
assert seen["method"] == "POST" and "Cookie" in seen["headers"]
|
||||
assert "ct0=tok" in seen["headers"]["Cookie"] # live-borrowed session, not persisted
|
||||
|
||||
|
||||
def test_full_replay_no_session_is_typed_miss(monkeypatch):
|
||||
p_arm(monkeypatch)
|
||||
def boom(domain):
|
||||
raise RuntimeError("Not logged in")
|
||||
monkeypatch.setattr(rw, "get_session", boom)
|
||||
out = rw.replay_write("POST", "https://www.reddit.com/api/comment", {"text": "hi"},
|
||||
"https://www.reddit.com", p_reddit_route())
|
||||
assert out.ok is False and "no borrowable session" in out.error
|
||||
|
||||
|
||||
def test_full_replay_site_reject_is_typed_error(monkeypatch):
|
||||
p_arm(monkeypatch)
|
||||
monkeypatch.setattr(rw, "get_session", lambda d: ("sess=z", "UA/1.0"))
|
||||
monkeypatch.setattr(rw, "issue_request", lambda *a: (429, "rate limited"))
|
||||
out = rw.replay_write("POST", "https://www.reddit.com/api/comment", {"text": "hi"},
|
||||
"https://www.reddit.com", p_reddit_route())
|
||||
assert out.ok is False and "429" in out.error
|
||||
@@ -0,0 +1,140 @@
|
||||
"""A cleared composer is not proof when the site just said no.
|
||||
|
||||
The whole fast write path rests on one signal: the payload left the composer. browser_send_script
|
||||
has always carried the admission that this "cannot tell submitted from dismissed", guarded only for
|
||||
guessed clicks and ghost-drop hosts. The realistic way it bites is neither: the site accepts the
|
||||
click, clears the box, and pops "Something went wrong" or a rate limit. The receipt then reads as a
|
||||
clean success and the agent tells the user it posted.
|
||||
|
||||
The guard reads only the page's live announcement regions (role=alert, aria-live), which is the
|
||||
accessibility contract sites already follow, so it needs no per-site knowledge. It can only ever
|
||||
DEMOTE a claim, never manufacture one, and it fails open: a broken probe returns "not rejected"
|
||||
rather than inventing a failure that did not happen.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_delivery_check as dc
|
||||
|
||||
REJECTIONS = [
|
||||
"Something went wrong. Try again.",
|
||||
"Couldn't post your reply",
|
||||
"Could not send message",
|
||||
"Unable to post right now",
|
||||
"Failed to send",
|
||||
"You've reached your daily limit. Try again later.",
|
||||
"Too many requests, please slow down",
|
||||
"Rate limit exceeded",
|
||||
"Your post wasn't sent",
|
||||
"An error occurred",
|
||||
]
|
||||
|
||||
# A live region is also how sites announce SUCCESS and ordinary chatter. Matching these would turn
|
||||
# every good send into a scary "it was rejected", which is a worse lie than the one being fixed.
|
||||
NOT_REJECTIONS = [
|
||||
"Your post was sent.",
|
||||
"Posted",
|
||||
"Message sent",
|
||||
"Draft saved",
|
||||
"1 new notification",
|
||||
"Copied to clipboard",
|
||||
"",
|
||||
" ",
|
||||
]
|
||||
|
||||
|
||||
def p_probe(text):
|
||||
"""A fake BrowserEvaluate returning whatever the announcement regions supposedly held.
|
||||
|
||||
Shape matters: parse_eval_value reads {"value": ...} at the TOP level. An earlier version of
|
||||
this fixture nested it one level deeper, which made every negative case pass for the wrong
|
||||
reason (unreadable result, not correct matching) while every positive case failed loudly."""
|
||||
async def run(tool, args, browser_id, tab_id):
|
||||
assert tool == "BrowserEvaluate"
|
||||
return {"value": {"text": text}}
|
||||
return run
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("text", REJECTIONS)
|
||||
async def test_a_refusal_is_detected(text):
|
||||
assert await dc.send_rejected("b", "t", p_probe(text)) is True
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("text", NOT_REJECTIONS)
|
||||
async def test_success_and_chatter_are_left_alone(text):
|
||||
assert await dc.send_rejected("b", "t", p_probe(text)) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_broken_probe_does_not_invent_a_failure():
|
||||
"""Fail OPEN. Claiming rejection because we could not read the page would be the same class of
|
||||
lie in the other direction."""
|
||||
async def boom(tool, args, browser_id, tab_id):
|
||||
raise RuntimeError("evaluate died")
|
||||
assert await dc.send_rejected("b", "t", boom) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_hung_probe_does_not_block_the_send_path():
|
||||
"""The probe sits on the irreversible path; it must time out rather than wedge the turn."""
|
||||
import asyncio
|
||||
|
||||
async def hang(tool, args, browser_id, tab_id):
|
||||
await asyncio.sleep(30)
|
||||
return {}
|
||||
assert await dc.send_rejected("b", "t", hang) is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_junk_shaped_result_is_not_a_rejection():
|
||||
async def junk(tool, args, browser_id, tab_id):
|
||||
return {"value": "not a dict"}
|
||||
assert await dc.send_rejected("b", "t", junk) is False
|
||||
|
||||
|
||||
# --- the probe itself ------------------------------------------------------------------------
|
||||
|
||||
def test_the_probe_reads_only_announcement_regions():
|
||||
"""Scope is the entire point. Whole-page text contains the word 'failed' on a huge fraction of
|
||||
the internet, and matching that would demote correct sends at random."""
|
||||
js = dc.rejection_probe_expression()
|
||||
assert "role=alert" in js
|
||||
assert "aria-live" in js
|
||||
assert "document.body" not in js, "the probe must not fall back to whole-page text"
|
||||
|
||||
|
||||
def test_the_probe_bounds_what_it_returns():
|
||||
"""An unbounded innerText from a chatty live region would bloat every send's payload."""
|
||||
assert "slice(0,600)" in dc.rejection_probe_expression()
|
||||
|
||||
|
||||
# --- what the user is told -------------------------------------------------------------------
|
||||
|
||||
def test_the_rejection_note_states_plainly_that_it_did_not_send():
|
||||
note = dc.rejected_send_note("https://x.com/home", "hello there")
|
||||
assert "did NOT go through" in note
|
||||
assert "x.com" in note and "www." not in note
|
||||
assert "hello there" in note
|
||||
|
||||
|
||||
def test_the_rejection_note_says_it_will_not_retry():
|
||||
"""A blind retry on a refusal is how you get rate-limited harder, or post twice if the refusal
|
||||
was cosmetic."""
|
||||
assert "did not retry" in dc.rejected_send_note("https://x.com", "hi").lower()
|
||||
|
||||
|
||||
def test_a_long_payload_is_clipped_in_the_note():
|
||||
note = dc.rejected_send_note("https://x.com", "y" * 500)
|
||||
assert "..." in note
|
||||
assert len(note) < 400
|
||||
|
||||
|
||||
def test_the_rejection_note_is_distinct_from_the_unverified_one():
|
||||
"""Different evidence, different claim. Collapsing them would either overclaim or send the user
|
||||
to go check something we already know."""
|
||||
rejected = dc.rejected_send_note("https://x.com", "hi")
|
||||
unverified = dc.unverified_send_note("https://x.com", "hi")
|
||||
assert rejected != unverified
|
||||
assert "could NOT confirm" in unverified
|
||||
assert "could NOT confirm" not in rejected
|
||||
@@ -0,0 +1,143 @@
|
||||
"""Unit tests for the API-first write registry: routing, receipt extraction, and the fail-safe
|
||||
(any adapter failure becomes a typed ok=False so the agent can fall back to the UI path, never a
|
||||
crash). Network is mocked; the live end-to-end proof is in PROTOCOL_apifirst.md.
|
||||
|
||||
Plus the agent tool-wiring layer (run_api_write in browser_agent): domain resolution from the
|
||||
current URL, a truthful receipt on success (and send_confirmed via ok), and a MISS surfaced as an
|
||||
`error` so the model falls back to the UI, never a false claim."""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.browser import browser_agent as BA
|
||||
from backend.apps.agents.browser import site_write_registry as reg
|
||||
|
||||
|
||||
def test_has_api_write_knows_reddit_and_rejects_unknown():
|
||||
assert reg.has_api_write("reddit.com", "comment") is True
|
||||
assert reg.has_api_write("REDDIT.COM", "delete") is True # case + normalization
|
||||
assert reg.has_api_write("reddit.com", "wire_money") is False # unknown action
|
||||
assert reg.has_api_write("example.com", "comment") is False # no adapter
|
||||
|
||||
|
||||
def test_receipt_prefers_permalink_then_url_then_id():
|
||||
assert reg.receipt_str({"permalink": "/r/x/c/1", "id": "t1_9"}) == "/r/x/c/1"
|
||||
assert reg.receipt_str({"url": "https://x/p", "id": "t3_9"}) == "https://x/p"
|
||||
assert reg.receipt_str({"id": "t1_9"}) == "t1_9"
|
||||
assert reg.receipt_str({}) == "ok"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_write_routes_and_returns_typed_receipt(monkeypatch):
|
||||
monkeypatch.setattr(reg, "p_ensure_session_env", lambda: None)
|
||||
monkeypatch.setattr(reg.reddit_writes, "comment",
|
||||
lambda parent_id, text: {"id": "t1_abc", "permalink": "/r/test/comments/x/_/t1_abc"})
|
||||
r = await reg.api_write("reddit.com", "comment", {"parent_id": "t3_x", "text": "hi"})
|
||||
assert r.ok is True
|
||||
assert r.receipt == "/r/test/comments/x/_/t1_abc"
|
||||
assert r.action == "comment" and r.domain == "reddit.com"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_write_unknown_domain_is_typed_miss_not_crash(monkeypatch):
|
||||
r = await reg.api_write("nosuchsite.com", "comment", {"text": "hi"})
|
||||
assert r.ok is False
|
||||
assert "no API-first adapter" in r.error
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_write_adapter_failure_is_caught_as_typed_error(monkeypatch):
|
||||
monkeypatch.setattr(reg, "p_ensure_session_env", lambda: None)
|
||||
def boom(parent_id, text):
|
||||
raise reg.reddit_writes.RedditError("RATELIMIT: try later")
|
||||
monkeypatch.setattr(reg.reddit_writes, "comment", boom)
|
||||
r = await reg.api_write("reddit.com", "comment", {"parent_id": "t3_x", "text": "hi"})
|
||||
assert r.ok is False
|
||||
assert "RATELIMIT" in r.error # site's own error surfaced, no crash
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_api_write_missing_required_param_is_typed_error(monkeypatch):
|
||||
monkeypatch.setattr(reg, "p_ensure_session_env", lambda: None)
|
||||
monkeypatch.setattr(reg.reddit_writes, "comment", lambda parent_id, text: {"id": "t1_x"})
|
||||
r = await reg.api_write("reddit.com", "comment", {"text": "no parent id"}) # missing parent_id
|
||||
assert r.ok is False and r.error # KeyError -> typed miss, not a crash
|
||||
|
||||
|
||||
# --- agent tool-wiring layer (run_api_write) ------------------------------
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_resolves_domain_from_url_and_returns_receipt(monkeypatch):
|
||||
async def fake(domain, action, params):
|
||||
assert domain == "reddit.com" and action == "comment"
|
||||
return reg.WriteResult(ok=True, action=action, domain=domain,
|
||||
receipt="/r/x/comments/a/_/t1_z", latency_ms=271)
|
||||
monkeypatch.setattr(reg, "api_write", fake)
|
||||
out = await BA.run_api_write(
|
||||
{"action": "comment", "parent_id": "t3_a", "text": "hi"},
|
||||
"https://www.reddit.com/r/x/comments/a/title/")
|
||||
assert out.get("ok") is True
|
||||
assert "/r/x/comments/a/_/t1_z" in out["text"] and "error" not in out
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_miss_is_an_error_so_model_falls_back_to_ui(monkeypatch):
|
||||
async def fake(domain, action, params):
|
||||
return reg.WriteResult(ok=False, action=action, domain=domain,
|
||||
error="no API-first adapter for example.com/comment; use the UI path")
|
||||
monkeypatch.setattr(reg, "api_write", fake)
|
||||
out = await BA.run_api_write({"action": "comment", "text": "hi"}, "https://example.com/thread")
|
||||
assert "error" in out and "ok" not in out # a miss reads as an error -> UI fallback
|
||||
assert "UI" in out["error"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_no_url_yet_is_an_error_not_a_crash():
|
||||
out = await BA.run_api_write({"action": "comment", "text": "hi"}, "")
|
||||
assert "error" in out and "site" in out["error"].lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_missing_action_is_an_error():
|
||||
out = await BA.run_api_write({"text": "hi"}, "https://www.reddit.com/r/x/")
|
||||
assert "error" in out and "action" in out["error"].lower()
|
||||
|
||||
|
||||
# --- general capture-replay tier (action='route') ---------------------------
|
||||
@pytest.mark.asyncio
|
||||
async def test_registry_route_write_wraps_replay_outcome(monkeypatch):
|
||||
from backend.apps.agents.browser import route_write as rw
|
||||
monkeypatch.setattr(reg, "p_ensure_session_env", lambda: None)
|
||||
monkeypatch.setattr(rw, "replay_write",
|
||||
lambda m, u, b, o, c: rw.ReplayOutcome(ok=True, receipt="t1_z", latency_ms=88))
|
||||
res = await reg.api_route_write("https://www.reddit.com", "POST",
|
||||
"https://www.reddit.com/api/comment", {"text": "hi"}, [])
|
||||
assert res.ok is True and res.receipt == "t1_z"
|
||||
assert res.domain == "www.reddit.com" and res.action == "route"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_route_fetches_captured_and_replays(monkeypatch):
|
||||
# The tool fetches the site's captured write routes (safety wall) then replays. Both boundaries
|
||||
# (the renderer list + the replay) are stubbed; this proves the wiring shape end to end.
|
||||
async def fake_exec(tool, params, bid, tid):
|
||||
assert tool == "BrowserListRoutes" and params == {"writes": True}
|
||||
return {"routes": [{"method": "POST", "template": "https://www.reddit.com/api/comment"}]}
|
||||
|
||||
async def fake_route_write(origin, method, url, body, captured):
|
||||
assert origin == "https://www.reddit.com" and method == "POST"
|
||||
assert len(captured) == 1 and captured[0].template.endswith("/api/comment")
|
||||
return reg.WriteResult(ok=True, action="route", domain="www.reddit.com",
|
||||
receipt="/r/x/c/a", latency_ms=120)
|
||||
|
||||
monkeypatch.setattr(BA, "execute_browser_tool", fake_exec)
|
||||
monkeypatch.setattr(reg, "api_route_write", fake_route_write)
|
||||
out = await BA.run_api_write(
|
||||
{"action": "route", "method": "POST", "url": "https://www.reddit.com/api/comment",
|
||||
"body": {"thing_id": "t3_a", "text": "hi"}},
|
||||
"https://www.reddit.com/r/x/comments/a/", "b1", "t1")
|
||||
assert out.get("ok") is True and "/r/x/c/a" in out["text"]
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_tool_route_needs_a_url():
|
||||
out = await BA.run_api_write({"action": "route", "method": "POST"},
|
||||
"https://www.reddit.com/r/x/", "b1", "t1")
|
||||
assert "error" in out and "url" in out["error"].lower()
|
||||
@@ -0,0 +1,67 @@
|
||||
"""A learned skill must not disarm the fast write path.
|
||||
|
||||
Measured live on x.com 2026-07-28. A skill had been learned for the host, so every write took the
|
||||
"skill exists ... skipping prestage (replay owns the nav)" branch. Prestage is also what hands the
|
||||
send-script its composer perception, so with it skipped the whole fill/click/receipt tail was
|
||||
unreachable and the model fell back to 4-5 turns:
|
||||
|
||||
with a skill (replay owns nav) 5/5 writes, 41-146s, median ~57s, receipt never spoke
|
||||
with the send-script armed 19.4s, receipt correct
|
||||
|
||||
The skip was a real optimisation for READS, where the replayed prefix genuinely replaces prestage's
|
||||
navigation. It just was never true for sends. Prestage on an already-loaded page measured 1.9-5.0s,
|
||||
so a send trades a few seconds to save tens.
|
||||
|
||||
This is the second time this exact interaction bit: the removal case was already carved out because
|
||||
a stale delete-"skill" of scrolls could hijack a destructive one-shot. Same shape, so it is worth a
|
||||
pure predicate with a test rather than a condition buried in a 2000-line function.
|
||||
"""
|
||||
from backend.apps.agents.browser import browser_skills as bs
|
||||
|
||||
|
||||
def test_a_read_with_a_learned_skill_lets_replay_own_the_nav():
|
||||
"""The optimisation this branch exists for must survive."""
|
||||
assert bs.replay_owns_nav("x.com", has_skill=True, task_is_removal=False, task_is_send=False)
|
||||
|
||||
|
||||
def test_a_send_keeps_its_prestage_even_when_a_skill_exists():
|
||||
"""The regression: skipping here costs the send-script its composer perception."""
|
||||
assert not bs.replay_owns_nav("x.com", has_skill=True, task_is_removal=False, task_is_send=True)
|
||||
|
||||
|
||||
def test_a_removal_never_replays_a_skill():
|
||||
"""A delete is a destructive one-shot, not a replayable nav prefix."""
|
||||
assert not bs.replay_owns_nav("x.com", has_skill=True, task_is_removal=True, task_is_send=False)
|
||||
|
||||
|
||||
def test_no_skill_means_nothing_to_replay():
|
||||
assert not bs.replay_owns_nav("x.com", has_skill=False, task_is_removal=False, task_is_send=False)
|
||||
|
||||
|
||||
def test_no_host_means_nothing_to_replay():
|
||||
"""host_of returns "" for a task with no resolvable URL; that must not read as a skill hit."""
|
||||
assert not bs.replay_owns_nav("", has_skill=True, task_is_removal=False, task_is_send=False)
|
||||
|
||||
|
||||
def test_a_send_that_is_also_a_removal_still_stands_down():
|
||||
"""is_removal_task and task_is_send both fire on "delete the post that says X" (the classifier
|
||||
keys on the verb). Either one alone must be enough to keep replay out."""
|
||||
assert not bs.replay_owns_nav("x.com", has_skill=True, task_is_removal=True, task_is_send=True)
|
||||
|
||||
|
||||
def test_the_predicate_returns_a_real_bool():
|
||||
"""It feeds an `if`; a truthy string or None would still work by accident and then stop working
|
||||
the moment someone logs or serialises it."""
|
||||
for send in (True, False):
|
||||
got = bs.replay_owns_nav("x.com", True, False, send)
|
||||
assert got is True or got is False
|
||||
|
||||
|
||||
def test_the_agent_uses_the_predicate_rather_than_reinventing_the_condition():
|
||||
"""The whole point of extracting it. If someone re-inlines the check, these tests would keep
|
||||
passing while the live path regressed, which is exactly how this bug survived the first time."""
|
||||
src = (bs.__file__).replace("browser_skills.py", "browser_agent.py")
|
||||
with open(src) as f:
|
||||
text = f.read()
|
||||
assert "browser_skills.replay_owns_nav(" in text
|
||||
assert "p_skip_prestage_for_skill = bool(" not in text, "the condition was re-inlined"
|
||||
@@ -0,0 +1,90 @@
|
||||
"""A stale element index must be retried; every other failure must not.
|
||||
|
||||
Measured live on x.com 2026-07-28, with the send-script correctly armed and having already found
|
||||
opener 'Post' and target 'Post text':
|
||||
|
||||
[browser-sendscript] fill errored (Index 53 is not in the cached element map.
|
||||
Call BrowserListInteractives first to refresh the index, then try again.)
|
||||
|
||||
The compose modal keeps re-rendering after we list it, so the composer node is detached by the time
|
||||
the fill lands. The script then surrendered a send it had already located and handed the whole task
|
||||
back to the model, which is most of the gap between a ~19s scripted write and a ~56s modelled one.
|
||||
|
||||
The discrimination is the safety-critical part. A stale index means the control WAS there and the
|
||||
page moved underneath us, so re-listing and retrying is correct and cannot duplicate anything (the
|
||||
fill has not committed). A genuine miss (no such control, refused input, a dead card) must never
|
||||
retry, because retrying a real failure on a write path is how you post twice.
|
||||
"""
|
||||
from backend.apps.agents.browser import browser_submit_click as sc
|
||||
|
||||
STALE = [
|
||||
{"error": "Index 53 is not in the cached element map. Call BrowserListInteractives first to "
|
||||
"refresh the index, then try again."},
|
||||
{"error": "Index 7 is not in the cached element map."},
|
||||
{"error": "Stale index; refresh the index and retry"},
|
||||
{"error": "NOT IN THE CACHED ELEMENT MAP"},
|
||||
]
|
||||
|
||||
# Real failures. Retrying any of these is either useless or dangerous.
|
||||
NOT_STALE = [
|
||||
{"error": "No element found matching that selector"},
|
||||
{"error": "Element is not visible"},
|
||||
{"error": "Navigation timed out"},
|
||||
{"error": "Target closed"},
|
||||
{"error": "browser card is gone"},
|
||||
{"error": ""},
|
||||
{},
|
||||
{"text": "ok"},
|
||||
]
|
||||
|
||||
|
||||
def test_a_stale_index_is_recognised():
|
||||
missed = [r for r in STALE if not sc.is_stale_index_error(r)]
|
||||
assert not missed, f"these stale-index errors would not be retried: {missed}"
|
||||
|
||||
|
||||
def test_the_exact_string_measured_live_is_recognised():
|
||||
assert sc.is_stale_index_error({"error": (
|
||||
"Index 53 is not in the cached element map. Call BrowserListInteractives first to refresh "
|
||||
"the index, then try again.")})
|
||||
|
||||
|
||||
def test_real_failures_are_never_retried():
|
||||
"""The dangerous direction: a retry on a genuine failure is how a write path double-posts."""
|
||||
wrong = [r for r in NOT_STALE if sc.is_stale_index_error(r)]
|
||||
assert not wrong, f"these would be wrongly retried: {wrong}"
|
||||
|
||||
|
||||
def test_non_dict_results_are_not_stale():
|
||||
for junk in (None, "boom", 42, [], object()):
|
||||
assert sc.is_stale_index_error(junk) is False
|
||||
|
||||
|
||||
def test_a_success_is_not_a_stale_error():
|
||||
assert not sc.is_stale_index_error({"value": {"ok": True}})
|
||||
|
||||
|
||||
def test_the_matcher_is_anchored_on_the_stable_half_of_the_sentence():
|
||||
"""Anchoring on the whole sentence would let a reworded tail silently disable the retry, taking
|
||||
the fast write path down with it and looking like a performance regression, not a bug."""
|
||||
assert sc.is_stale_index_error({"error": "index 12 is not in the cached element map (v2)"})
|
||||
assert sc.is_stale_index_error({"error": "please refresh the index"})
|
||||
|
||||
|
||||
def test_the_send_script_actually_calls_it():
|
||||
"""A predicate nothing consults is decoration. Guards against the retry being dropped while
|
||||
these tests keep passing."""
|
||||
src = sc.__file__.replace("browser_submit_click.py", "browser_send_script.py")
|
||||
with open(src) as f:
|
||||
text = f.read()
|
||||
assert "is_stale_index_error(" in text
|
||||
assert "retrying the fill once" in text
|
||||
|
||||
|
||||
def test_the_retry_is_bounded_to_one_attempt():
|
||||
"""Two fills is a refresh; a loop is a way to hammer a site that is refusing input."""
|
||||
src = sc.__file__.replace("browser_submit_click.py", "browser_send_script.py")
|
||||
with open(src) as f:
|
||||
body = f.read()
|
||||
seg = body[body.index("stale composer index") - 1200: body.index("stale composer index") + 1200]
|
||||
assert "for " not in seg.split("stale composer index")[1][:600], "the retry must not be a loop"
|
||||
@@ -555,7 +555,7 @@ def test_resolve_sdk_gemini_prefers_antigravity_over_api_key():
|
||||
s = AppSettings()
|
||||
s.google_api_key = "ai-studio-key"
|
||||
with patch.object(registry, "p_antigravity_connected", return_value=True):
|
||||
# flash-lite IS AG-serveable -> AG wins over the key (probe retargeted after gemini-3-flash was removed)
|
||||
# flash-lite IS AG-serveable (via ag/gemini-3-flash) -> AG wins over the key (probe retargeted after gemini-3-flash was removed on both branches)
|
||||
assert registry.resolve_model_id_for_sdk("gemini-3.1-flash-lite", s) == "ag/gemini-3-flash"
|
||||
with patch.object(registry, "p_antigravity_connected", return_value=False):
|
||||
# AG not connected -> key
|
||||
@@ -671,13 +671,14 @@ def test_banned_models_not_offered():
|
||||
left 2026-07-02 after its ban lifted.)"""
|
||||
from backend.apps.agents.providers.registry import BUILTIN_MODELS
|
||||
all_values = {m["value"] for models in BUILTIN_MODELS.values() for m in models}
|
||||
for dead in ("gemini-3.1-pro", "gemini-3.1-pro-api"):
|
||||
for dead in ("gemini-3.1-pro", "gemini-3.1-pro-api", "gemini-3-flash", "gemini-3-flash-api"):
|
||||
assert dead not in all_values, f"{dead} is back in the picker"
|
||||
assert "gpt-5.5-api" in all_values
|
||||
assert "gpt-5.5" in all_values # cx lane restored 2026-07-26 (live-probed)
|
||||
# No '3.1 pro' label survives in any provider group either.
|
||||
all_labels = " | ".join(m["label"].lower() for models in BUILTIN_MODELS.values() for m in models)
|
||||
assert "3.1 pro" not in all_labels
|
||||
assert "gemini 3 flash" not in all_labels
|
||||
|
||||
|
||||
# =========================================================================== Group E, 9Router-streamed 401 detection =========================================================================== 9Router sometimes returns upstream auth failures AS the assistant's reply text, not as an exception. We detect the pattern in the stream handler to substitute a friendly bubble.
|
||||
|
||||
@@ -0,0 +1,165 @@
|
||||
// The user-agent half of borrowing a sign-in out of the user's real Chrome.
|
||||
//
|
||||
// Borrowing the session and presenting as the browser that earned it are ONE decision. Our browser
|
||||
// cards deliberately advertise an "openswarm/<ver>" product token, because Google's sign-in rejects
|
||||
// a bare Chrome UA as not-genuine-Chrome. But the anti-bot layer in front of a borrowed site checks
|
||||
// the session against the UA it was minted for, so that same token reads as "this is not the
|
||||
// browser that logged in" and the session is refused. Measured live: medium (7 entries) and
|
||||
// instagram (10 entries) both imported cleanly and both still reported signed-out.
|
||||
//
|
||||
// main.js needs a real Electron to load, so the two pure functions are lifted out of the source and
|
||||
// exercised directly. That keeps the test honest about WHICH code it covers: if either function is
|
||||
// renamed or reshaped, the extraction fails loudly rather than silently testing a stale copy.
|
||||
const test = require('node:test');
|
||||
const assert = require('node:assert');
|
||||
const fs = require('node:fs');
|
||||
const os = require('node:os');
|
||||
const path = require('node:path');
|
||||
|
||||
const SRC = fs.readFileSync(path.join(__dirname, 'main.js'), 'utf8');
|
||||
|
||||
function lift(name) {
|
||||
const m = SRC.match(new RegExp(`function ${name}\\([\\s\\S]*?\\n\\}`));
|
||||
assert.ok(m, `${name} not found in main.js; did the borrowed-session UA path change shape?`);
|
||||
return m[0];
|
||||
}
|
||||
|
||||
// eslint-disable-next-line no-eval
|
||||
eval(`${lift('bareChromeUserAgent')}\nvar p_borrowedSessionDomains = new Set();\n${lift('hostHasBorrowedSession')}`);
|
||||
|
||||
const APP_UA = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
|
||||
+ '(KHTML, like Gecko) openswarm/1.5.8 Chrome/148.0.7778.218 Electron/42.3.3 Safari/537.36';
|
||||
const REAL_CHROME = /^Mozilla\/5\.0 \(Macintosh; Intel Mac OS X 10_15_7\) AppleWebKit\/537\.36 \(KHTML, like Gecko\) Chrome\/[\d.]+ Safari\/537\.36$/;
|
||||
|
||||
test('a borrowed site sees a UA indistinguishable from real Chrome', () => {
|
||||
const bare = bareChromeUserAgent(APP_UA);
|
||||
assert.match(bare, REAL_CHROME);
|
||||
assert.ok(!/openswarm/i.test(bare), 'the product token is the whole tell; it must be gone');
|
||||
assert.ok(!/Electron/i.test(bare), 'the Electron token must be gone too');
|
||||
});
|
||||
|
||||
test('the Chrome version is preserved, not invented', () => {
|
||||
// sec-ch-ua headers carry the real version. Substituting a different one here would make the UA
|
||||
// and the client hints disagree, which is a louder tell than the token we just removed.
|
||||
assert.ok(bareChromeUserAgent(APP_UA).includes('Chrome/148.0.7778.218'));
|
||||
});
|
||||
|
||||
test('a UA with no product token is left exactly alone', () => {
|
||||
const already = 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 '
|
||||
+ '(KHTML, like Gecko) Chrome/148.0.7778.218 Safari/537.36';
|
||||
assert.strictEqual(bareChromeUserAgent(already), already);
|
||||
});
|
||||
|
||||
test('empty and malformed input degrade to a string, never a throw', () => {
|
||||
for (const bad of [undefined, null, '', 123, {}]) {
|
||||
assert.strictEqual(typeof bareChromeUserAgent(bad), 'string');
|
||||
}
|
||||
});
|
||||
|
||||
test('only borrowed sites are rewritten', () => {
|
||||
p_borrowedSessionDomains.clear();
|
||||
assert.strictEqual(hostHasBorrowedSession('https://medium.com/'), false,
|
||||
'nothing borrowed yet means nothing is rewritten');
|
||||
p_borrowedSessionDomains.add('medium.com');
|
||||
assert.strictEqual(hostHasBorrowedSession('https://medium.com/me'), true);
|
||||
assert.strictEqual(hostHasBorrowedSession('https://cdn.medium.com/x.js'), true,
|
||||
'subdomains of a borrowed site carry the same session');
|
||||
});
|
||||
|
||||
test('a lookalike domain is never treated as borrowed', () => {
|
||||
// Suffix matching done wrong is how "notmedium.com" or "medium.com.evil.net" would inherit the
|
||||
// borrowed identity. Only the exact host or a real dot-separated subdomain may match.
|
||||
p_borrowedSessionDomains.clear();
|
||||
p_borrowedSessionDomains.add('medium.com');
|
||||
assert.strictEqual(hostHasBorrowedSession('https://notmedium.com/'), false);
|
||||
assert.strictEqual(hostHasBorrowedSession('https://medium.com.evil.net/'), false);
|
||||
assert.strictEqual(hostHasBorrowedSession('https://google.com/'), false,
|
||||
'Google must keep the product token: its own sign-in is what the token exists to satisfy');
|
||||
});
|
||||
|
||||
test('a malformed URL is not a borrowed site', () => {
|
||||
p_borrowedSessionDomains.clear();
|
||||
p_borrowedSessionDomains.add('medium.com');
|
||||
assert.strictEqual(hostHasBorrowedSession('not a url'), false);
|
||||
assert.strictEqual(hostHasBorrowedSession(''), false);
|
||||
});
|
||||
|
||||
test('the borrowed-domain list survives a restart', () => {
|
||||
// The cookies live in a PERSISTENT partition, so they outlive a quit. This list has to as well:
|
||||
// the backend memoizes "already borrowed" and skips re-importing, so when Electron came back with
|
||||
// an empty set the site kept its session while silently no longer being told we were plain
|
||||
// Chrome. That desync produced a false negative during the 2026-07-27 measurements.
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'borrowed-'));
|
||||
const scope = {
|
||||
p_borrowedSessionDomains: new Set(),
|
||||
app: { getPath: () => dir },
|
||||
path,
|
||||
fs,
|
||||
};
|
||||
const body = `${lift('p_borrowedDomainsPath')}\n${lift('loadBorrowedDomains')}\n${lift('saveBorrowedDomains')}\n`;
|
||||
const run = new Function(
|
||||
'p_borrowedSessionDomains', 'app', 'path', 'fs',
|
||||
`${body}; return { loadBorrowedDomains, saveBorrowedDomains, set: p_borrowedSessionDomains };`,
|
||||
);
|
||||
|
||||
const first = run(scope.p_borrowedSessionDomains, scope.app, path, fs);
|
||||
first.set.add('medium.com');
|
||||
first.set.add('claude.ai');
|
||||
first.saveBorrowedDomains();
|
||||
|
||||
// A fresh process: new empty set, same on-disk file.
|
||||
const second = run(new Set(), scope.app, path, fs);
|
||||
second.loadBorrowedDomains();
|
||||
assert.deepStrictEqual([...second.set].sort(), ['claude.ai', 'medium.com']);
|
||||
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('a missing or corrupt list is not a crash', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'borrowed-bad-'));
|
||||
const body = `${lift('p_borrowedDomainsPath')}\n${lift('loadBorrowedDomains')}\n${lift('saveBorrowedDomains')}\n`;
|
||||
const run = new Function(
|
||||
'p_borrowedSessionDomains', 'app', 'path', 'fs',
|
||||
`${body}; return { loadBorrowedDomains, set: p_borrowedSessionDomains };`,
|
||||
);
|
||||
const app = { getPath: () => dir };
|
||||
|
||||
const fresh = run(new Set(), app, path, fs);
|
||||
fresh.loadBorrowedDomains(); // no file at all
|
||||
assert.strictEqual(fresh.set.size, 0);
|
||||
|
||||
fs.writeFileSync(path.join(dir, 'borrowed-session-domains.json'), '{not json');
|
||||
const corrupt = run(new Set(), app, path, fs);
|
||||
corrupt.loadBorrowedDomains(); // garbage on disk
|
||||
assert.strictEqual(corrupt.set.size, 0);
|
||||
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
});
|
||||
|
||||
test('clearing browsing data drops the borrowed claims with it', () => {
|
||||
// Cookies are gone, so continuing to present as the browser that earned them is a lie about a
|
||||
// session that no longer exists.
|
||||
const handler = SRC.slice(SRC.indexOf("ipcMain.handle('browser:clear-data'"));
|
||||
const body = handler.slice(0, handler.indexOf('});'));
|
||||
assert.match(body, /p_borrowedSessionDomains\.clear\(\)/);
|
||||
assert.match(body, /saveBorrowedDomains\(\)/);
|
||||
});
|
||||
|
||||
test('the list is restored before any card can load a site', () => {
|
||||
const configureAt = SRC.indexOf('configureBrowsingSession(session.fromPartition(BROWSER_PARTITION)');
|
||||
const loadAt = SRC.indexOf('loadBorrowedDomains();');
|
||||
assert.ok(configureAt > 0 && loadAt > configureAt,
|
||||
'loadBorrowedDomains must run during startup, right after the partition is configured');
|
||||
});
|
||||
|
||||
test('importing a session is what registers the domain', () => {
|
||||
// The two halves must stay wired together: cookies applied without the matching UA get refused,
|
||||
// which is exactly the bug this whole path exists to fix.
|
||||
assert.match(SRC, /p_borrowedSessionDomains\.add\(d\);\s*\n\s*saveBorrowedDomains\(\);/,
|
||||
'writePartitionCookies must register the domain it borrowed for AND persist it');
|
||||
});
|
||||
|
||||
test('the request header is actually rewritten for borrowed sites', () => {
|
||||
assert.match(SRC, /borrowed && lk === 'user-agent'/,
|
||||
'the onBeforeSendHeaders hook must swap the UA on borrowed sites');
|
||||
});
|
||||
+29
-1
@@ -95,9 +95,32 @@ function routeKey(method, template) {
|
||||
return String(method || 'GET').toUpperCase() + ' ' + template;
|
||||
}
|
||||
|
||||
// The mutating body WITH values, but secret-shaped string leaves redacted (belt; the backend
|
||||
// recipe learner redacts again). Only JSON, capped: a write recipe needs the real body (the user's
|
||||
// payload sits in one leaf), which bodyShape (types only) can't provide. Non-JSON / oversized = null.
|
||||
const MAX_BODY_CHARS = 65536;
|
||||
|
||||
function redactBodyValues(postData) {
|
||||
if (!postData || postData.length > MAX_BODY_CHARS) return null;
|
||||
try {
|
||||
const walk = (v) =>
|
||||
typeof v === 'string'
|
||||
? (looksSecretValue(v) ? '<redacted>' : v)
|
||||
: Array.isArray(v)
|
||||
? v.map(walk)
|
||||
: v && typeof v === 'object'
|
||||
? Object.fromEntries(Object.keys(v).map((k) => [k, walk(v[k])]))
|
||||
: v;
|
||||
return JSON.stringify(walk(JSON.parse(postData)));
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function makeRouteEntry(request, resourceType) {
|
||||
const method = String(request.method || 'GET').toUpperCase();
|
||||
const template = templateUrl(request.url);
|
||||
const safe = isSafeMethod(method);
|
||||
return {
|
||||
method,
|
||||
template,
|
||||
@@ -105,7 +128,9 @@ function makeRouteEntry(request, resourceType) {
|
||||
resourceType,
|
||||
headers: redactHeaders(request.headers),
|
||||
bodyShape: bodyShape(request.postData),
|
||||
safe: isSafeMethod(method),
|
||||
// Only mutating routes carry a replayable body; a GET's body (if any) is never a write recipe.
|
||||
lastBody: safe ? null : redactBodyValues(request.postData),
|
||||
safe,
|
||||
hits: 1,
|
||||
lastSeen: Date.now(),
|
||||
};
|
||||
@@ -123,6 +148,9 @@ function recordRoute(routesMap, request, resourceType, now = Date.now()) {
|
||||
if (existing) {
|
||||
existing.hits += 1;
|
||||
existing.lastSeen = now;
|
||||
// Refresh the body so a later replay learns from the FRESHEST call (queryId/token rotation
|
||||
// lives in the URL/headers, but a stale body could hold an old nonce); keep the latest.
|
||||
if (entry.lastBody) existing.lastBody = entry.lastBody;
|
||||
} else {
|
||||
routesMap.set(key, entry);
|
||||
if (routesMap.size > MAX_ROUTES_PER_WC) {
|
||||
|
||||
@@ -136,3 +136,30 @@ test('makeRouteEntry carries a redacted example url', () => {
|
||||
assert.ok(e.example.includes('redacted'));
|
||||
assert.ok(!e.example.includes('secretAbc123Long'));
|
||||
});
|
||||
|
||||
test('makeRouteEntry: mutating route keeps a body, GET does not, secrets redacted', () => {
|
||||
const post = R.makeRouteEntry({
|
||||
method: 'POST',
|
||||
url: 'https://x.com/i/api/graphql/AbC123/CreateTweet',
|
||||
headers: { 'x-csrf-token': 'ct0secret' },
|
||||
postData: JSON.stringify({ variables: { tweet_text: 'hello world' }, authToken: 'aB3xK9mQ2pL7wR4tY8nZ' }),
|
||||
}, 'Fetch');
|
||||
assert.equal(post.safe, false);
|
||||
const body = JSON.parse(post.lastBody);
|
||||
assert.equal(body.variables.tweet_text, 'hello world'); // payload survives
|
||||
assert.equal(body.authToken, '<redacted>'); // secret leaf redacted
|
||||
assert.equal(post.headers['x-csrf-token'], '<redacted>'); // header redacted
|
||||
|
||||
const get = R.makeRouteEntry({ method: 'GET', url: 'https://x.com/i/api/graphql/AbC123/Home', headers: {}, postData: '' }, 'XHR');
|
||||
assert.equal(get.lastBody, null); // no body on a safe route
|
||||
});
|
||||
|
||||
test('recordRoute: repeat write refreshes lastBody to the freshest call', () => {
|
||||
const m = new Map();
|
||||
const mk = (text) => ({ method: 'POST', url: 'https://x.com/i/api/graphql/AbC/CreateTweet', headers: {}, postData: JSON.stringify({ variables: { tweet_text: text } }) });
|
||||
R.recordRoute(m, mk('first'), 'Fetch');
|
||||
R.recordRoute(m, mk('second'), 'Fetch');
|
||||
const entry = [...m.values()][0];
|
||||
assert.equal(entry.hits, 2);
|
||||
assert.equal(JSON.parse(entry.lastBody).variables.tweet_text, 'second');
|
||||
});
|
||||
|
||||
+159
-1
@@ -5,6 +5,65 @@ const { injectText } = require('./voice/textInjector');
|
||||
// Browser cards live in their own persistent partition so cookies/localStorage/IndexedDB survive reload + quit (Discord etc. stay logged in) and site data stays isolated from the app's defaultSession. The "clear browsing data" wipe nukes only this partition. MUST match BROWSER_PARTITION in frontend BrowserCard.tsx.
|
||||
const BROWSER_PARTITION = 'persist:openswarm-browser';
|
||||
|
||||
// Sites whose sign-in we borrowed out of the user's real Chrome. Populated when a session is
|
||||
// imported, and it changes exactly one thing: what user agent we present to that site.
|
||||
//
|
||||
// Our normal browser-card UA deliberately carries an "openswarm/<ver>" product token, because
|
||||
// Google's sign-in rejects a BARE Chrome UA as not-genuine-Chrome (see BrowserCard.tsx). But a
|
||||
// borrowed session was minted by real Chrome, and the anti-bot layer in front of these sites checks
|
||||
// the session against the UA that earned it, so that same token reads as "this is not the browser
|
||||
// that logged in" and the session is refused. On a borrowed site only, we drop the token and
|
||||
// present the same Chrome version bare, which is exactly what the onboarding harvest window does
|
||||
// (hiddenBrowser.js) and how it gets through Cloudflare with borrowed cookies. Google keeps the
|
||||
// token because we never borrow for it: its own sign-in is the thing the token exists to satisfy.
|
||||
const p_borrowedSessionDomains = new Set();
|
||||
const p_uaSwapLogged = new Set();
|
||||
const { warmBorrowedSession } = require('./warmBorrowedSession');
|
||||
|
||||
// The borrowed cookies live in a PERSISTENT partition, so they outlive a quit; this list has to as
|
||||
// well or the two halves drift apart. They did: the backend memoizes "already borrowed for this
|
||||
// domain" and skips re-importing, so after an Electron restart the site kept its session but
|
||||
// silently stopped being told we were plain Chrome. That desync produced a false negative in the
|
||||
// 2026-07-27 measurements and would have been invisible in normal use. Domain names only, never
|
||||
// cookie values.
|
||||
function p_borrowedDomainsPath() {
|
||||
return path.join(app.getPath('userData'), 'borrowed-session-domains.json');
|
||||
}
|
||||
|
||||
function loadBorrowedDomains() {
|
||||
try {
|
||||
const raw = JSON.parse(fs.readFileSync(p_borrowedDomainsPath(), 'utf8'));
|
||||
if (Array.isArray(raw)) raw.filter((d) => typeof d === 'string').forEach((d) => p_borrowedSessionDomains.add(d));
|
||||
} catch {
|
||||
// No file yet, or unreadable: an empty list just means the first navigate re-borrows.
|
||||
}
|
||||
}
|
||||
|
||||
function saveBorrowedDomains() {
|
||||
try {
|
||||
fs.writeFileSync(p_borrowedDomainsPath(), JSON.stringify([...p_borrowedSessionDomains]));
|
||||
} catch {
|
||||
// Losing this only costs one redundant re-import next launch; never worth failing an import over.
|
||||
}
|
||||
}
|
||||
|
||||
function bareChromeUserAgent(ua) {
|
||||
return String(ua || '').replace(/\s*(?:openswarm|Electron)\/\S+/gi, '').replace(/\s{2,}/g, ' ').trim();
|
||||
}
|
||||
|
||||
function hostHasBorrowedSession(url) {
|
||||
if (!p_borrowedSessionDomains.size) return false;
|
||||
try {
|
||||
const host = new URL(url).hostname.toLowerCase();
|
||||
for (const d of p_borrowedSessionDomains) {
|
||||
if (host === d || host.endsWith(`.${d}`)) return true;
|
||||
}
|
||||
} catch {
|
||||
// A malformed URL simply isn't a borrowed site.
|
||||
}
|
||||
return false;
|
||||
}
|
||||
|
||||
// E2E flag: when OPENSWARM_E2E=1, append a Chromium command-line switch the
|
||||
// renderer reads at startup to set window.__OPENSWARM_E2E__ = true BEFORE any
|
||||
// page script parses, so the production-build store-on-window gate fires
|
||||
@@ -1902,6 +1961,9 @@ app.whenReady().then(async () => {
|
||||
};
|
||||
configureBrowsingSession(session.defaultSession, { allowFullscreen: false });
|
||||
configureBrowsingSession(session.fromPartition(BROWSER_PARTITION), { allowFullscreen: true });
|
||||
// Restore which sites we borrowed a sign-in for BEFORE any card can load one, so the very first
|
||||
// request of the launch already presents as the browser that earned the session.
|
||||
loadBorrowedDomains();
|
||||
|
||||
// PASSKEY SPIKE: when a site offers several discoverable passkeys, Electron fires this so we pick one; without a handler the WebAuthn flow stalls. For the spike just take the first; a real impl would surface a picker. macOS-only event (no-op elsewhere).
|
||||
for (const ses of [session.defaultSession, session.fromPartition(BROWSER_PARTITION)]) {
|
||||
@@ -1924,10 +1986,20 @@ app.whenReady().then(async () => {
|
||||
{ urls: ['http://*/*', 'https://*/*'] },
|
||||
(details, callback) => {
|
||||
const headers = { ...(details.requestHeaders || {}) };
|
||||
const borrowed = hostHasBorrowedSession(details.url);
|
||||
for (const k of Object.keys(headers)) {
|
||||
const lk = k.toLowerCase();
|
||||
if (lk === 'sec-ch-ua' || lk === 'sec-ch-ua-full-version-list') {
|
||||
headers[k] = addGoogleChromeBrand(headers[k]);
|
||||
} else if (borrowed && lk === 'user-agent') {
|
||||
const swapped = bareChromeUserAgent(headers[k]);
|
||||
// Once per site: proof the swap actually fired, so "borrowed but still signed out" can be
|
||||
// read as the site refusing us rather than as this code silently never running.
|
||||
if (swapped !== headers[k] && !p_uaSwapLogged.has(details.url.split('/')[2])) {
|
||||
p_uaSwapLogged.add(details.url.split('/')[2]);
|
||||
console.log(`[borrowed-ua] ${details.url.split('/')[2]} -> ${swapped}`);
|
||||
}
|
||||
headers[k] = swapped;
|
||||
}
|
||||
}
|
||||
callback({ requestHeaders: headers });
|
||||
@@ -2500,6 +2572,25 @@ app.on('web-contents-created', (_event, contents) => {
|
||||
`).catch(() => {});
|
||||
});
|
||||
|
||||
// On a site whose sign-in we borrowed we send a bare Chrome UA header (see
|
||||
// p_borrowedSessionDomains), so navigator.userAgent has to say the same thing. A page that
|
||||
// reads one UA in JS while the request carried another is a louder automation tell than the
|
||||
// product token we removed, and plenty of anti-bot scripts compare exactly those two.
|
||||
contents.on('dom-ready', () => {
|
||||
let borrowed = false;
|
||||
try { borrowed = hostHasBorrowedSession(contents.getURL()); } catch { borrowed = false; }
|
||||
if (!borrowed) return;
|
||||
const bare = bareChromeUserAgent(contents.getUserAgent());
|
||||
contents.executeJavaScript(`
|
||||
(function(){
|
||||
try {
|
||||
if (navigator.userAgent === ${JSON.stringify(bare)}) return;
|
||||
Object.defineProperty(navigator, 'userAgent', { get: function(){ return ${JSON.stringify(bare)}; }, configurable: true });
|
||||
} catch (e) {}
|
||||
})();
|
||||
`).catch(() => {});
|
||||
});
|
||||
|
||||
// Real headed Chrome exposes window.chrome.app/csi/loadTimes; an Electron webview's window.chrome is empty ({}), the single most-checked headless/automation tell (PerimeterX/DataDome et al). Stub the same shape real Chrome reports (app = object, csi + loadTimes = functions, NO runtime, matching a non-extension page). Also restore the base 'en' language Electron drops. Page-world (contextIsolation hides the preload), measured to flip every bot.sannysoft row to its Chrome value.
|
||||
contents.on('dom-ready', () => {
|
||||
contents.executeJavaScript(`
|
||||
@@ -2982,6 +3073,10 @@ ipcMain.handle('browser:clear-data', async () => {
|
||||
const ses = session.fromPartition(BROWSER_PARTITION);
|
||||
await ses.clearStorageData();
|
||||
await ses.clearCache();
|
||||
// The borrowed sessions just went with it, so stop claiming we are the browser that earned them.
|
||||
p_borrowedSessionDomains.clear();
|
||||
p_uaSwapLogged.clear();
|
||||
saveBorrowedDomains();
|
||||
return { ok: true };
|
||||
});
|
||||
|
||||
@@ -3039,6 +3134,60 @@ ipcMain.on('browser-capsule-take', (event, origin) => {
|
||||
pendingSessionCapsules.delete(event.sender.id);
|
||||
event.returnValue = entry.capsule;
|
||||
});
|
||||
// Populate the browser-card partition with the user's OWN existing sign-in for a site, so an agent
|
||||
// stuck at a login wall can carry on as them without anybody typing a password. This is the exact
|
||||
// opposite direction from the read above: cookies only go INTO our own partition, never out, so it
|
||||
// is not a disclosure surface. The backend gates it behind an explicit opt-in setting and always
|
||||
// derives the domain from the page the agent is already stuck on, never from model text.
|
||||
async function writePartitionCookies(domain, cookies) {
|
||||
const d = String(domain || '').toLowerCase().trim().replace(/^\./, '');
|
||||
if (!/^[a-z0-9-]+(\.[a-z0-9-]+)+$/.test(d)) return { ok: false, set: 0, error: `bad domain: ${d || '(empty)'}` };
|
||||
const list = Array.isArray(cookies) ? cookies : [];
|
||||
const ses = session.fromPartition(BROWSER_PARTITION);
|
||||
let set = 0;
|
||||
for (const c of list) {
|
||||
if (!c || !c.name) continue;
|
||||
const rawHost = String(c.domain || d);
|
||||
const host = rawHost.replace(/^\./, '');
|
||||
// Every cookie has to belong to the domain we were asked for, so importing one site can never
|
||||
// plant another site's session in the partition.
|
||||
if (host !== d && !host.endsWith(`.${d}`)) continue;
|
||||
const path = String(c.path || '/') || '/';
|
||||
try {
|
||||
await ses.cookies.set({
|
||||
url: `https://${host}${path.startsWith('/') ? path : `/${path}`}`,
|
||||
name: String(c.name),
|
||||
value: String(c.value == null ? '' : c.value),
|
||||
// A leading dot is Chromium's marker for a domain-wide cookie; without it the cookie is
|
||||
// host-only and passing `domain` at all would silently widen it.
|
||||
domain: rawHost.startsWith('.') ? rawHost : undefined,
|
||||
path,
|
||||
secure: !!c.secure,
|
||||
httpOnly: !!c.httponly,
|
||||
expirationDate: Number(c.expires) > 0 ? Number(c.expires) : undefined,
|
||||
});
|
||||
set += 1;
|
||||
} catch (err) {
|
||||
// One malformed cookie must not sink the whole sign-in.
|
||||
}
|
||||
}
|
||||
// Borrowing the session and presenting as the browser that earned it are one decision, not two:
|
||||
// apply the cookies without the matching UA and the site refuses them.
|
||||
if (set > 0) {
|
||||
p_borrowedSessionDomains.add(d);
|
||||
saveBorrowedDomains();
|
||||
}
|
||||
// Let a plain hidden window take the site's challenge before the card does. It shares this
|
||||
// partition, so whatever clearance it earns is already waiting when the card loads.
|
||||
let warmed = false;
|
||||
if (set > 0) {
|
||||
const ua = bareChromeUserAgent(session.fromPartition(BROWSER_PARTITION).getUserAgent());
|
||||
warmed = await warmBorrowedSession(BROWSER_PARTITION, `https://${d}/`, ua);
|
||||
console.log(`[borrowed-warm] ${d} warmed=${warmed}`);
|
||||
}
|
||||
return { ok: set > 0, set, total: list.length, warmed };
|
||||
}
|
||||
ipcMain.handle('set-partition-cookies', (_e, domain, cookies) => writePartitionCookies(domain, cookies));
|
||||
|
||||
// The renderer relays cookie reads for the session-borrow bridge, but macOS throttles it when the
|
||||
// window is backgrounded, so those reads intermittently time out. Main never throttles: hold our own
|
||||
@@ -3427,14 +3576,23 @@ async function ensureDebuggerAttached(wc) {
|
||||
// Bounded + fail-open: a wedged pipe must never block the card from closing.
|
||||
async function detachCdpCleanly(wc) {
|
||||
if (!wc || wc.isDestroyed()) return;
|
||||
cdpTearingDown.add(wc.id);
|
||||
let attached = false;
|
||||
try { attached = wc.debugger.isAttached(); } catch (_) { return; }
|
||||
// Mark tearing-down only once we know there IS a session to tear down. The flag is a one-way
|
||||
// latch that permanently bars re-attach, so setting it before this check meant a card that had
|
||||
// never used CDP got latched by any teardown call and could never be perceived again: a live,
|
||||
// healthy card with permanently dead perception, which reads exactly like a wedged webview.
|
||||
if (!attached) return;
|
||||
cdpTearingDown.add(wc.id);
|
||||
const drain = (method, params) =>
|
||||
raceCdp(wc.debugger.sendCommand(method, params || {}), 1200, method).catch(() => {});
|
||||
// Recheck between every await. Each drain can take 1.2s, and this runs while the guest is being
|
||||
// torn down, so the webContents these commands target can die mid-sequence; both crashes we
|
||||
// have logs for end in a CDP detach racing a guest teardown.
|
||||
await drain('Target.setAutoAttach', { autoAttach: false, waitForDebuggerOnStart: false, flatten: true });
|
||||
if (wc.isDestroyed() || wc.isCrashed()) return;
|
||||
await drain('Network.disable', {});
|
||||
if (wc.isDestroyed() || wc.isCrashed()) return;
|
||||
try { wc.debugger.detach(); } catch (_) { /* already detached / gone */ }
|
||||
}
|
||||
|
||||
|
||||
@@ -93,6 +93,8 @@ contextBridge.exposeInMainWorld('openswarm', {
|
||||
harvestUsage: (provider) => ipcRenderer.invoke('harvest-usage', provider),
|
||||
// Suspend/resume state capsule: stages a resumed webview's sessionStorage snapshot in main (keyed by webContents id, short TTL) so the guest preload can sync-take it at document-start. Fire-and-forget; main validates the sender.
|
||||
setSessionCapsule: (wcId, capsule) => ipcRenderer.send('browser-capsule-set', wcId, capsule),
|
||||
// Loads the user's own existing sign-in for a site INTO the browser partition so a blocked agent can continue as them. Writes only, never reads back; main re-checks every cookie belongs to the domain asked for.
|
||||
setPartitionCookies: (domain, cookies) => ipcRenderer.invoke('set-partition-cookies', domain, cookies),
|
||||
sendCdpCommand: (wcId, method, params, sessionId) => ipcRenderer.invoke('send-cdp-command', wcId, method, params, sessionId),
|
||||
cdpDetachClean: (wcId) => ipcRenderer.invoke('cdp-detach-clean', wcId),
|
||||
cdpCacheSet: (wcId, indexMap) => ipcRenderer.invoke('cdp-cache-set', wcId, indexMap),
|
||||
|
||||
@@ -0,0 +1,71 @@
|
||||
// Warm a borrowed sign-in in a hidden window before the visible card loads the site.
|
||||
//
|
||||
// Measured 2026-07-27: transplanting the user's own Chrome session into the browser partition works
|
||||
// (claude.ai accepted it and showed the real account name and plan), but sites behind a serious
|
||||
// anti-bot edge refuse it in a browser CARD: chatgpt.com, medium.com and instagram.com all had the
|
||||
// cookies applied and the user agent matched and still reported signed-out. The decisive clue is
|
||||
// that onboarding's harvest beats Cloudflare on chatgpt.com with those SAME cookies, and the only
|
||||
// thing it does differently is load them in a plain hidden BrowserWindow instead of a <webview>
|
||||
// guest. A guest carries a preload and the automation tells that come with being embedded; a hidden
|
||||
// window is just a browser.
|
||||
//
|
||||
// So we let the context that passes do the handshake. The hidden window shares the card's partition,
|
||||
// which means any clearance it earns lands in the same cookie jar the card is about to use. The
|
||||
// card then arrives already cleared instead of being challenged on its first request.
|
||||
//
|
||||
// Main-process only (an offscreen BrowserWindow is not a renderer webview). Always destroys its
|
||||
// window in a finally, so a failure can never leak one, and never throws: a warm that does not work
|
||||
// just leaves the card exactly as it would have been.
|
||||
const { BrowserWindow } = require('electron');
|
||||
|
||||
const LOAD_TIMEOUT_MS = 15000;
|
||||
// Anti-bot edges run a JS challenge after the document lands; the clearance cookie is only written
|
||||
// once that finishes, so returning at load time would throw away the entire point of doing this.
|
||||
const SETTLE_MS = 2500;
|
||||
const DESTROY_GRACE_MS = 5000;
|
||||
|
||||
async function warmBorrowedSession(partition, url, userAgent) {
|
||||
let win = null;
|
||||
try {
|
||||
win = new BrowserWindow({
|
||||
show: false,
|
||||
width: 1280,
|
||||
height: 900,
|
||||
webPreferences: {
|
||||
partition,
|
||||
sandbox: true,
|
||||
contextIsolation: true,
|
||||
nodeIntegration: false,
|
||||
backgroundThrottling: false,
|
||||
},
|
||||
});
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
|
||||
const killer = setTimeout(() => {
|
||||
try { if (win && !win.isDestroyed()) win.destroy(); } catch { /* already gone */ }
|
||||
}, LOAD_TIMEOUT_MS + SETTLE_MS + DESTROY_GRACE_MS);
|
||||
|
||||
try {
|
||||
// Passed to loadURL, not just setUserAgent: the popup-UA spoofer in main.js rewrites any
|
||||
// contents of type 'window' during construction, and the per-load option is what wins.
|
||||
if (userAgent) {
|
||||
try { win.webContents.setUserAgent(userAgent); } catch { /* the load option still carries it */ }
|
||||
}
|
||||
const opts = userAgent ? { userAgent } : undefined;
|
||||
// loadURL rejects when any sub-resource aborts even though the main frame is fine, so a
|
||||
// rejection here is noise; what matters is that the challenge got time to run.
|
||||
const load = win.loadURL(url, opts).catch(() => {});
|
||||
await Promise.race([load, new Promise((r) => setTimeout(r, LOAD_TIMEOUT_MS))]);
|
||||
await new Promise((r) => setTimeout(r, SETTLE_MS));
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
} finally {
|
||||
clearTimeout(killer);
|
||||
try { if (win && !win.isDestroyed()) win.destroy(); } catch { /* already gone */ }
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = { warmBorrowedSession, LOAD_TIMEOUT_MS, SETTLE_MS, DESTROY_GRACE_MS };
|
||||
@@ -66,7 +66,7 @@ import { registerCapsuleForRestore } from '@/shared/browserStateCapsule';
|
||||
import BrowserFindBar from './BrowserFindBar';
|
||||
import { openCardContextMenu } from '../desktop/CardContextMenu';
|
||||
import { useBrowserActivity } from '@/shared/useBrowserActivity';
|
||||
import { getActionLabel } from '@/shared/browserCommandHandler';
|
||||
import { getActionLabel, readDataDocument, recoverCardOffDataWall } from '@/shared/browserCommandHandler';
|
||||
import { resolveInput, isGoogleSearch } from '@/shared/resolveUrl';
|
||||
import BrowserAgentOverlay from './BrowserAgentOverlay';
|
||||
import { useOverlayScrollPassthrough } from '../hooks/interaction/useOverlayScrollPassthrough';
|
||||
@@ -400,7 +400,14 @@ const BrowserCard: React.FC<Props> = ({
|
||||
if (isWindows) markWindowsWebviewSurvived();
|
||||
// Registered BEFORE loadURL so the guest preload can sync-take it at document-start: a resumed tab gets its sessionStorage back Chrome-style instead of a logged-out reload. No-op when no capsule exists.
|
||||
registerCapsuleForRestore(wv, tabId);
|
||||
wv.loadURL(targetUrl).catch(() => {});
|
||||
wv.loadURL(targetUrl)
|
||||
.then(async () => {
|
||||
// If this card's own entry URL is a raw JSON/API endpoint, it paints an unreadable data
|
||||
// wall; get it onto a real page. (The agent-navigate path is handled in handleNavigate;
|
||||
// this covers the initial load, which never goes through the command handler.)
|
||||
if (await readDataDocument(wv)) recoverCardOffDataWall(wv, targetUrl);
|
||||
})
|
||||
.catch(() => {});
|
||||
try {
|
||||
(wv as any).setVisualZoomLevelLimits?.(1, 1);
|
||||
(wv as any).setZoomFactor?.(1);
|
||||
|
||||
@@ -132,7 +132,7 @@ const SettingsBody: React.FC<SettingsBodyProps> = ({ active, requestedTab, onReq
|
||||
</Box>
|
||||
) : activeTab === 'privacy' ? (
|
||||
<Box sx={{ pt: 0.5, pb: 2, animation: 'fadeIn 0.2s ease', '@keyframes fadeIn': { from: { opacity: 0 }, to: { opacity: 1 } } }}>
|
||||
<DataPrivacySection styles={styles} />
|
||||
<DataPrivacySection form={form} setForm={setForm} styles={styles} />
|
||||
</Box>
|
||||
) : activeTab === 'advanced' ? (
|
||||
<Box sx={{ pt: 0.5, pb: 2, animation: 'fadeIn 0.2s ease', '@keyframes fadeIn': { from: { opacity: 0 }, to: { opacity: 1 } } }}>
|
||||
|
||||
@@ -3,7 +3,9 @@ import Box from '@mui/material/Box';
|
||||
import Typography from '@mui/material/Typography';
|
||||
import Button from '@mui/material/Button';
|
||||
import Dialog from '@mui/material/Dialog';
|
||||
import Switch from '@mui/material/Switch';
|
||||
import TextField from '@mui/material/TextField';
|
||||
import type { AppSettings } from '@/shared/state/settingsSlice';
|
||||
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
|
||||
import { API_BASE } from '@/shared/config';
|
||||
import type { SettingsStyles } from '../settingsStyles';
|
||||
@@ -11,7 +13,11 @@ import type { SettingsStyles } from '../settingsStyles';
|
||||
const ERASE_WORD = 'ERASE';
|
||||
|
||||
// The iOS Reset menu, two actions only: "Reset All Settings" (preferences back to defaults, your stuff + sign-in stay) and "Erase All Content and Settings" (factory wipe + relaunch). Flat rows, not a boxed "danger zone": red lives only on the destructive label, and the real friction is the typed-confirm in the dialog.
|
||||
const DataPrivacySection: React.FC<{ styles: SettingsStyles }> = ({ styles }) => {
|
||||
const DataPrivacySection: React.FC<{
|
||||
form: AppSettings;
|
||||
setForm: React.Dispatch<React.SetStateAction<AppSettings>>;
|
||||
styles: SettingsStyles;
|
||||
}> = ({ form, setForm, styles }) => {
|
||||
const c = useClaudeTokens();
|
||||
const { labelSx, descSx, inlineRowSx, inlineRowLastSx } = styles;
|
||||
|
||||
@@ -113,6 +119,21 @@ const DataPrivacySection: React.FC<{ styles: SettingsStyles }> = ({ styles }) =>
|
||||
return (
|
||||
<Box>
|
||||
|
||||
<Box sx={inlineRowSx}>
|
||||
<Box sx={{ mr: 3 }}>
|
||||
<Typography sx={labelSx}>Use my sign-ins from my other browser</Typography>
|
||||
<Typography sx={descSx}>When an agent hits a site you're not signed into here, borrow the sign-in you already have in Chrome, Arc, Brave, or Edge instead of stopping to ask you. Reads only the site it's stuck on, and never asks for a password. Off by default.</Typography>
|
||||
</Box>
|
||||
<Switch
|
||||
checked={form.browser_import_signins}
|
||||
onChange={(e) => setForm({ ...form, browser_import_signins: e.target.checked })}
|
||||
sx={{
|
||||
'& .MuiSwitch-switchBase.Mui-checked': { color: c.accent.primary },
|
||||
'& .MuiSwitch-switchBase.Mui-checked + .MuiSwitch-track': { bgcolor: c.accent.primary },
|
||||
}}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
<Box sx={inlineRowSx}>
|
||||
<Box sx={{ mr: 3 }}>
|
||||
<Typography sx={labelSx}>Reset all settings</Typography>
|
||||
|
||||
@@ -1,14 +1,17 @@
|
||||
import { getWebview, findWebviewByDomain, hasDomReady, markDomReady, isPendingLoad, wakePendingLoad, clearPendingLoad, type BrowserWebview } from './browserRegistry';
|
||||
import { shouldSelfHealClick } from './selfHealClick';
|
||||
import { FP_EXPR, clickEffect } from './clickEffect';
|
||||
import { store } from './state/store';
|
||||
import { resumeBrowserCard } from './state/dashboardLayoutSlice';
|
||||
import { dashboardWs } from './ws/WebSocketManager';
|
||||
import { resolveInput } from './resolveUrl';
|
||||
import { rankAndCapInteractives, type RankItem } from './interactiveRanking';
|
||||
import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettle';
|
||||
import { unwrapCdpEval } from './cdpEval';
|
||||
|
||||
let initialized = false;
|
||||
|
||||
export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'click_point' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name';
|
||||
export type BrowserAction = 'screenshot' | 'get_text' | 'get_console' | 'navigate' | 'click' | 'type' | 'evaluate' | 'get_elements' | 'scroll' | 'wait' | 'press_key' | 'list_interactives' | 'click_index' | 'click_point' | 'batch' | 'detect_webmcp' | 'list_routes' | 'replay_route' | 'click_by_name' | 'find_composer';
|
||||
|
||||
export interface BrowserActivity {
|
||||
action: BrowserAction;
|
||||
@@ -193,7 +196,40 @@ async function countSafeRoutes(wv: BrowserWebview): Promise<number> {
|
||||
const STUCK_EVAL_GRACE_MS = 2500;
|
||||
const STUCK_EVAL_LIMIT_MS = 9000;
|
||||
|
||||
// Run `code` in the guest page. In Electron we prefer CDP Runtime.evaluate: it runs in the
|
||||
// browser process, so it is NOT suspended while the page is still loading, the way
|
||||
// webContents.executeJavaScript is (that suspend, on a page whose trackers never let it "stop
|
||||
// loading", is the 15s command wedge). When the CDP bridge isn't there (dev Chrome, or a forced
|
||||
// A/B via window.__OSW_CDP_EVAL__ = false) we fall back to the executeJavaScript path unchanged,
|
||||
// so behavior never regresses where CDP can't run. Both paths keep the same contract: return the
|
||||
// value, throw on a page-side error, mark dom-ready on success.
|
||||
async function evalInPage(wv: BrowserWebview, code: string): Promise<any> {
|
||||
const cdpBridge = (window as any).openswarm?.sendCdpCommand;
|
||||
if (cdpBridge && (window as any).__OSW_CDP_EVAL__ !== false) {
|
||||
let cdp: any;
|
||||
try {
|
||||
cdp = await sendCdp(wv, 'Runtime.evaluate',
|
||||
{ expression: code, returnByValue: true, awaitPromise: true });
|
||||
} catch {
|
||||
// CDP INFRA failure (the debugger can't attach because DevTools or a remote-debugging
|
||||
// port already holds this webContents, or the bridge errored). Never worse than today:
|
||||
// fall through to the executeJavaScript path. A real PAGE exception is NOT an infra
|
||||
// failure, it rides exceptionDetails below, so it still surfaces as a throw.
|
||||
cdp = undefined;
|
||||
}
|
||||
if (cdp !== undefined) {
|
||||
const value = unwrapCdpEval(cdp); // throws on a real page-side exception
|
||||
markDomReady(wv);
|
||||
return value;
|
||||
}
|
||||
}
|
||||
return await evalViaExecuteJs(wv, code);
|
||||
}
|
||||
|
||||
// The original webContents.executeJavaScript path, kept as the dev-Chrome / bridge-absent
|
||||
// fallback: it suspends until the page stops loading, so a grace/limit race cancels stragglers
|
||||
// with wv.stop() once the document is ready and flushes the queue.
|
||||
async function evalViaExecuteJs(wv: BrowserWebview, code: string): Promise<any> {
|
||||
const run = wv.executeJavaScript(code).then((v) => {
|
||||
markDomReady(wv);
|
||||
return { done: true as const, value: v };
|
||||
@@ -276,10 +312,75 @@ async function handleNavigate(wv: BrowserWebview, params: Record<string, any>):
|
||||
} finally {
|
||||
removeReady();
|
||||
}
|
||||
// A navigate that lands on a raw JSON/API document (Instagram's topsearch, any /api/... GET)
|
||||
// paints an unreadable wall in the card and reads as a crash to the user. Hand the data to the
|
||||
// agent as the result instead, and quietly get the card off the wall, so a person never sees
|
||||
// raw JSON where a page should be.
|
||||
const data = await readDataDocument(wv);
|
||||
if (data) {
|
||||
recoverCardOffDataWall(wv, url);
|
||||
return {
|
||||
text: `Fetched ${data.contentType} data from ${url} (this URL is a raw API endpoint, not a page):\n${data.body}`,
|
||||
url: wv.getURL(),
|
||||
data_document: true,
|
||||
};
|
||||
}
|
||||
// Route-count is sampled on the next READ (handleGetText), not here: at navigate-return the SPA's XHRs haven't fired yet, so this would always be ~0.
|
||||
return { text: `Navigated to ${url}`, url };
|
||||
}
|
||||
|
||||
// Chromium renders any JSON (or JSON-shaped text) response with its built-in viewer, so a navigate
|
||||
// to an API endpoint leaves the card showing a wall of raw data. contentType is the reliable tell;
|
||||
// re-fetch same-origin (the just-loaded GET, idempotent) to hand the agent the exact bytes.
|
||||
// Exported so BrowserCard can reuse the exact same detection for the card's own initial load.
|
||||
export async function readDataDocument(wv: BrowserWebview): Promise<{ contentType: string; body: string } | null> {
|
||||
const code = `(async () => {
|
||||
const ct = String(document.contentType || '').toLowerCase();
|
||||
const isJson = ct.startsWith('application/json');
|
||||
const maybePlain = ct.startsWith('text/plain');
|
||||
if (!isJson && !maybePlain) return null;
|
||||
try {
|
||||
const r = await fetch(location.href, { credentials: 'include', signal: AbortSignal.timeout(2500) });
|
||||
const body = await r.text();
|
||||
if (maybePlain && !isJson) { try { JSON.parse(body.trim()); } catch (e) { return null; } }
|
||||
return { contentType: ct, body: body.slice(0, 15000) };
|
||||
} catch (e) { return null; }
|
||||
})()`;
|
||||
try {
|
||||
const res = await evalInPage(wv, code);
|
||||
if (res && !res.error && typeof res.body === 'string') return res;
|
||||
return null;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
// Get the card off a raw-data wall: step back to the real page the agent came from, or, if it opened
|
||||
// straight onto the data URL, fall back to the site homepage. Fire-and-forget, the agent already has
|
||||
// the data; this is purely so a human sees a page instead of JSON. Exported for BrowserCard's own-load path.
|
||||
export function recoverCardOffDataWall(wv: BrowserWebview, dataUrl: string): void {
|
||||
let origin = '';
|
||||
try { origin = new URL(dataUrl).origin; } catch { /* keep '' */ }
|
||||
try {
|
||||
if (wv.canGoBack()) {
|
||||
wv.goBack();
|
||||
// A card that opened STRAIGHT onto the data URL has only the webview's blank initial entry
|
||||
// behind it, so goBack lands on about:blank. Detect that and fall back to the site homepage
|
||||
// so the card shows a real page rather than a blank one.
|
||||
window.setTimeout(() => {
|
||||
try {
|
||||
const u = wv.getURL();
|
||||
if ((!u || u === 'about:blank') && origin) void wv.loadURL(origin).catch(() => {});
|
||||
} catch { /* leave as-is */ }
|
||||
}, 400);
|
||||
return;
|
||||
}
|
||||
if (origin) void wv.loadURL(origin).catch(() => {});
|
||||
} catch {
|
||||
/* leave the card as-is if recovery fails; the data still reached the agent */
|
||||
}
|
||||
}
|
||||
|
||||
async function handleClick(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
||||
const selector = params.selector as string;
|
||||
if (!selector) return { error: 'selector parameter is required' };
|
||||
@@ -315,6 +416,9 @@ async function handleType(wv: BrowserWebview, params: Record<string, any>): Prom
|
||||
if (text == null) return { error: 'text parameter is required' };
|
||||
const safeSelector = JSON.stringify(selector);
|
||||
const safeText = JSON.stringify(text);
|
||||
// Try the cheap in-page fill first, and read it back: an editor that rejects synthetic
|
||||
// execCommand insertText (Reddit's Lexical, strict React contenteditables) leaves the box
|
||||
// empty, and we only learn that by checking the value, not by assuming the call worked.
|
||||
const code = `(async ()=>{
|
||||
const el = document.querySelector(${safeSelector});
|
||||
if (!el) return { error: 'Element not found: ' + ${safeSelector} };
|
||||
@@ -328,14 +432,263 @@ async function handleType(wv: BrowserWebview, params: Record<string, any>): Prom
|
||||
bubbles: true, cancelable: true, inputType: 'insertText', data: ${safeText},
|
||||
}));
|
||||
el.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
return {
|
||||
text: 'Typed into: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''),
|
||||
};
|
||||
const now = (el.value != null ? el.value : (el.textContent || ''));
|
||||
return { text: 'Typed into: ' + el.tagName.toLowerCase() + (el.id ? '#' + el.id : ''),
|
||||
committed: now.includes(${safeText}) };
|
||||
})()`;
|
||||
const result = await evalInPage(wv, code);
|
||||
// Real-keystroke fallback when the synthetic fill did not commit (same lever the finder uses).
|
||||
if (result && !result.error && result.committed === false) {
|
||||
const ok = await keystrokeFill(wv, selector, text);
|
||||
result.text = ok ? `Typed into ${selector} via keystrokes` : `Type may not have committed into ${selector}`;
|
||||
result.committed = ok;
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Find the page's composer STRUCTURALLY, not by accessible name: the biggest editable
|
||||
// region (contenteditable / textarea / text input) that is a real writing surface, not a
|
||||
// search box. Ranks by editable-richness + proximity to a Send/Post/Reply control + size,
|
||||
// so it picks the comment box out of a page that also has a search field. Marks the winner
|
||||
// with data-osw-composer so a follow-up type/click can target it by a stable selector.
|
||||
// With {fill}, it also types + reads back in the SAME in-page context (the only reliable
|
||||
// commit-check for a React-controlled contenteditable, whose value never reaches the AX tree).
|
||||
// With {reveal:true}, when no composer is painted yet it takes ONE reversible reveal action
|
||||
// (click a compose-trigger, open the first list item, or scroll) and rescans, up to a small
|
||||
// bound. Reveal actions are same-document only: SPA route changes (pushState) keep this JS
|
||||
// context alive, so X/LinkedIn/Reddit/YouTube stay drivable in one call; a full navigation
|
||||
// just cuts the await short and the backend re-perceives. Reveal NEVER clicks an irreversible
|
||||
// control (send/submit/pay/delete...): it only opens a surface, it never commits one.
|
||||
async function handleFindComposer(wv: BrowserWebview, params: Record<string, any>): Promise<Record<string, any>> {
|
||||
const fill = params.fill != null ? String(params.fill) : null;
|
||||
const safeFill = JSON.stringify(fill);
|
||||
const reveal = params.reveal === true ? 'true' : 'false';
|
||||
const code = `(async () => {
|
||||
const SUBMIT = /\\b(send|post|reply|comment|tweet|publish|share|message)\\b/i;
|
||||
const SEARCH = /search|find\\b|filter|query|lookup|explore|jump to/i;
|
||||
// Reversible compose openers: reveal a writing surface, never commit one.
|
||||
const OPENER = /\\b(start a post|start a thread|create( a)?( new)? (post|thread)|new post|new thread|add a comment|write a comment|leave a comment|post a comment|write a review|create a review|new message|new chat|send a message|compose|reply|comment|tweet|create|write)\\b/i;
|
||||
// Never clickable by reveal, even if the label also looks like an opener.
|
||||
const HARDBLOCK = /\\b(send|submit|pay|buy|purchase|checkout|order|delete|remove|unfollow|unsubscribe|log ?out|sign ?out|report|block|confirm|deactivate|save)\\b/i;
|
||||
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
||||
const vis = (el) => {
|
||||
const r = el.getBoundingClientRect(); const s = getComputedStyle(el);
|
||||
return r.width >= 80 && r.height >= 16 && s.visibility !== 'hidden'
|
||||
&& s.display !== 'none' && s.opacity !== '0';
|
||||
};
|
||||
// Pierce shadow DOM: Reddit (shreddit), Telegram, WhatsApp, Slack build their composer
|
||||
// inside web-component shadow roots that a light-DOM querySelectorAll can't see. Collect
|
||||
// only the SELECTOR matches (a few hundred at most), never every node: a heavy SPA like X
|
||||
// has 10k+ nodes and the composer sits late in document order, so an all-node walk with a
|
||||
// node cap truncates before ever reaching it (the regression that hid X's own composer).
|
||||
const deepMatch = (root, sel, out, depth) => {
|
||||
if (depth > 8 || out.length > 400) return out;
|
||||
let hits; try { hits = root.querySelectorAll(sel); } catch (e) { hits = []; }
|
||||
for (const el of hits) out.push(el);
|
||||
let all; try { all = root.querySelectorAll('*'); } catch (e) { return out; }
|
||||
for (const el of all) { if (el.shadowRoot) deepMatch(el.shadowRoot, sel, out, depth + 1); }
|
||||
return out;
|
||||
};
|
||||
const EDIT = 'textarea, [contenteditable="true"], [role="textbox"], input[type="text"]';
|
||||
const labelOf = (el) => ((el.getAttribute('aria-label')||'') + ' ' + (el.getAttribute('placeholder')||'')
|
||||
+ ' ' + (el.getAttribute('data-placeholder')||'') + ' ' + (el.getAttribute('name')||'')).trim();
|
||||
|
||||
const findBest = () => {
|
||||
const all = deepMatch(document, EDIT, [], 0);
|
||||
let best = null, bestScore = 0, bestNear = false;
|
||||
for (const el of all) {
|
||||
if (!vis(el) || el.readOnly || el.disabled) continue;
|
||||
const label = labelOf(el);
|
||||
if (el.type === 'search' || SEARCH.test(label)) continue;
|
||||
const rich = el.tagName === 'TEXTAREA' || el.isContentEditable || el.getAttribute('role') === 'textbox';
|
||||
const area = el.closest('form, [role="dialog"], [role="group"], section, main, [role="main"]') || document.body;
|
||||
let near = false;
|
||||
for (const b of area.querySelectorAll('button, [role="button"], input[type="submit"]')) {
|
||||
const bt = ((b.textContent||'') + ' ' + (b.getAttribute('aria-label')||'')).trim();
|
||||
if (bt.length < 40 && SUBMIT.test(bt)) { near = true; break; }
|
||||
}
|
||||
if (!rich && !near) continue; // a bare form input near nothing is not a composer
|
||||
const r = el.getBoundingClientRect();
|
||||
const score = (el.isContentEditable ? 2 : 0) + (el.tagName === 'TEXTAREA' ? 2 : 0)
|
||||
+ (near ? 3 : 0) + Math.min((r.width * r.height) / 40000, 3) + (SUBMIT.test(label) ? 1 : 0);
|
||||
if (score > bestScore) { bestScore = score; best = el; bestNear = near; }
|
||||
}
|
||||
return (best && bestScore >= 2) ? { el: best, score: bestScore, near: bestNear } : null;
|
||||
};
|
||||
|
||||
const pollFind = async (ms) => {
|
||||
const t0 = Date.now(); let h = findBest();
|
||||
while (!h && Date.now() - t0 < ms) { await sleep(140); h = findBest(); }
|
||||
return h;
|
||||
};
|
||||
|
||||
// The reveal actions, tried in yield order. Each returns true if it clicked/scrolled.
|
||||
// Match id-based placeholders too (YouTube's #placeholder-area "Add a comment...") not just
|
||||
// class-based ones, so a lazy comment box becomes a clickable trigger once it scrolls in.
|
||||
const TRIGGER_SEL = 'button, [role="button"], a[href], summary, [tabindex], [id*="placeholder" i], [class*="placeholder" i]';
|
||||
// inViewOnly confines the pick to what is CURRENTLY on screen: during the scroll phase a
|
||||
// header "Create"/"Reply" (scrolled off the top, so r.top<0) would otherwise outscore the
|
||||
// comment box that just scrolled into view, and clicking it opens/navigates the wrong thing.
|
||||
const clickTrigger = (inViewOnly) => {
|
||||
const all = deepMatch(document, TRIGGER_SEL, [], 0);
|
||||
let best = null, bestScore = -1;
|
||||
for (const el of all) {
|
||||
if (!vis(el)) continue;
|
||||
const r = el.getBoundingClientRect();
|
||||
if (inViewOnly && (r.bottom < 0 || r.top > window.innerHeight)) continue;
|
||||
const txt = ((el.textContent||'') + ' ' + (el.getAttribute('aria-label')||'') + ' ' + (el.getAttribute('placeholder')||'')).trim();
|
||||
if (txt.length > 60 || !OPENER.test(txt) || HARDBLOCK.test(txt)) continue;
|
||||
// Short-labelled wins; higher-on-page when scanning the whole page, viewport-center when scrolling.
|
||||
const posScore = inViewOnly
|
||||
? Math.max(0, 300 - Math.abs(r.top - window.innerHeight / 2)) / 100
|
||||
: Math.max(0, 600 - r.top) / 100;
|
||||
const score = (60 - txt.length) + posScore;
|
||||
if (score > bestScore) { bestScore = score; best = el; }
|
||||
}
|
||||
if (!best) return false;
|
||||
best.scrollIntoView({ block: 'center', behavior: 'instant' });
|
||||
best.click();
|
||||
return true;
|
||||
};
|
||||
const openFirstItem = () => {
|
||||
// Chat/message lists (X DM [data-testid=conversation], WhatsApp/Telegram [role=grid]
|
||||
// rows, conversation links) plus generic feeds. Pierce shadow DOM so web-component chat
|
||||
// lists are reachable. Take the first visible list-like item and open it.
|
||||
const ITEM_SEL = '[data-testid="conversation"], [role="listitem"], [role="row"], [role="article"], article, li a[href], a[role="link"], a[href*="/messages/"], a[href*="/chat/"]';
|
||||
const items = deepMatch(document, ITEM_SEL, [], 0);
|
||||
for (const it of items) {
|
||||
if (!vis(it)) continue;
|
||||
const r = it.getBoundingClientRect();
|
||||
if (r.top < 40 || r.top > window.innerHeight || r.height > window.innerHeight * 0.6) continue; // skip headers / offscreen / the whole pane
|
||||
const target = it.matches('a[href], [role="link"]') ? it : (it.querySelector('a[href], [role="link"], [role="button"]') || it);
|
||||
target.scrollIntoView({ block: 'center', behavior: 'instant' });
|
||||
target.click();
|
||||
return true;
|
||||
}
|
||||
return false;
|
||||
};
|
||||
let hit = findBest();
|
||||
const acts = [];
|
||||
if (!hit && ${reveal}) {
|
||||
// Self-imposed budget. The tiers below sum to ~24s of worst case but the command that
|
||||
// carries them is killed at its own timeout, which threw away ALL the work and returned
|
||||
// nothing (measured: linkedin died mid-scroll 2 of 3 runs, so retop/open-first could never
|
||||
// run). Each tier now only STARTS if the budget can still pay for it, and polls are clipped
|
||||
// to what is left, so the routine always returns its own best answer instead of being shot.
|
||||
const DEADLINE = Date.now() + 21000;
|
||||
const left = () => DEADLINE - Date.now();
|
||||
const budget = (want) => Math.max(0, Math.min(want, left()));
|
||||
// 1. A compose opener visible up top (Gmail "Compose", LinkedIn "Start a post"). Patient
|
||||
// poll: LinkedIn code-splits its share modal, and under a heavy session the editor
|
||||
// chunk lands past 2.5s (measured: the 2.5s poll missed ~half the time; poll exits the
|
||||
// moment the editable appears, so the patience costs nothing on the happy path).
|
||||
try { if (clickTrigger(false)) { acts.push('trigger'); hit = await pollFind(budget(6000)); } } catch (e) { /* keep going */ }
|
||||
// 2. Progressive scroll for a below-fold / lazy composer: YouTube comments hydrate on
|
||||
// scroll and start as a placeholder that only becomes editable once clicked, so scroll
|
||||
// a step, re-scan, and re-click the trigger on whatever just entered the viewport, up to
|
||||
// a small bound, stopping at page bottom. This is what a human does to reach comments.
|
||||
if (!hit) {
|
||||
const sc = document.scrollingElement || document.documentElement;
|
||||
let scrolled = false;
|
||||
// Reserve time for the retop + open-first tiers below, so the ladder can't eat the budget.
|
||||
for (let step = 0; step < 6 && !hit && left() > 7000; step++) {
|
||||
const before = sc.scrollTop;
|
||||
sc.scrollBy(0, Math.round(window.innerHeight * 0.9));
|
||||
scrolled = true;
|
||||
await sleep(500);
|
||||
hit = findBest();
|
||||
if (!hit) { try { if (clickTrigger(true)) hit = await pollFind(budget(1400)); } catch (e) { /* keep going */ } }
|
||||
if (sc.scrollTop === before) break;
|
||||
}
|
||||
if (scrolled) acts.push('scroll');
|
||||
}
|
||||
// 3. Back to the top opener: the scroll ladder ends at page bottom with the top compose
|
||||
// entry off-screen; a modal that opened slowly (or needed a second click) is only
|
||||
// winnable by returning and retrying once.
|
||||
if (!hit && left() > 2500) {
|
||||
try {
|
||||
(document.scrollingElement || document.documentElement).scrollTo(0, 0);
|
||||
await sleep(400);
|
||||
if (clickTrigger(false)) { acts.push('retop'); hit = await pollFind(budget(4000)); }
|
||||
} catch (e) { /* keep going */ }
|
||||
}
|
||||
// 4. Last resort: open the first list item (X DMs / chat lists). Navigational, so it runs
|
||||
// only after trigger+scroll fail, which stops it from yanking YouTube to another video.
|
||||
if (!hit && left() > 1200) { let did = false; try { did = openFirstItem(); } catch (e) { did = false; } if (did) { acts.push('open-first'); hit = await pollFind(budget(2000)); } }
|
||||
}
|
||||
if (!hit) return { found: false, reveals: acts };
|
||||
|
||||
const fillEl = (el) => {
|
||||
el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus();
|
||||
if (el.select) el.select();
|
||||
document.execCommand('selectAll', false); document.execCommand('delete', false);
|
||||
document.execCommand('insertText', false, ${safeFill});
|
||||
el.dispatchEvent(new InputEvent('input', { bubbles: true, cancelable: true, inputType: 'insertText', data: ${safeFill} }));
|
||||
el.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
const now = (el.value != null ? el.value : (el.textContent || ''));
|
||||
return now.includes(${safeFill});
|
||||
};
|
||||
|
||||
let best = hit.el;
|
||||
best.setAttribute('data-osw-composer', '1');
|
||||
let filled = false;
|
||||
if (${safeFill} != null) {
|
||||
filled = fillEl(best);
|
||||
// Activation-click fallback: YouTube (and many comment widgets) render a placeholder
|
||||
// that only spawns the real contenteditable once clicked. If the fill didn't commit,
|
||||
// click the found element, let the real editor mount, rescan, and fill THAT.
|
||||
if (!filled) {
|
||||
try { best.click(); } catch (e) { /* click may be intercepted */ }
|
||||
const re = await pollFind(1500);
|
||||
if (re) {
|
||||
best.removeAttribute('data-osw-composer');
|
||||
best = re.el; hit = re; best.setAttribute('data-osw-composer', '1');
|
||||
filled = fillEl(best);
|
||||
acts.push('activate');
|
||||
}
|
||||
}
|
||||
}
|
||||
return { found: true, selector: '[data-osw-composer="1"]', tag: best.tagName.toLowerCase(),
|
||||
role: best.isContentEditable ? 'contenteditable' : (best.getAttribute('role') || best.tagName.toLowerCase()),
|
||||
score: Math.round(hit.score * 10) / 10, nearSubmit: hit.near, filled, reveals: acts };
|
||||
})()`;
|
||||
const result = await evalInPage(wv, code);
|
||||
// Real-keystroke fill fallback: some editors (Reddit's Lexical, strict React contenteditables)
|
||||
// manage their own state through beforeinput and ignore execCommand insertText, so the in-page
|
||||
// fill reads back empty. Retype the SAME text as OS-level key events (isTrusted, the way a human
|
||||
// types), which those editors DO honor, then re-read to confirm. Only fires when the cheap
|
||||
// in-page fill already failed, so the fast path is untouched.
|
||||
if (fill != null && result && result.found && !result.filled && result.selector) {
|
||||
const ok = await keystrokeFill(wv, String(result.selector), fill);
|
||||
result.filled = ok;
|
||||
result.fillMode = ok ? 'keystroke' : 'keystroke-failed';
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Type `text` into the element at `selector` as real OS-level key events (Electron
|
||||
// sendInputEvent 'char'), for editors that reject synthetic execCommand insertText. Focuses +
|
||||
// clears in-page first, types each char natively, then reads the value back in-page to verify.
|
||||
async function keystrokeFill(wv: BrowserWebview, selector: string, text: string): Promise<boolean> {
|
||||
const safeSel = JSON.stringify(selector);
|
||||
await evalInPage(wv, `(() => { const el = document.querySelector(${safeSel});
|
||||
if (!el) return false; el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus();
|
||||
if (el.select) el.select(); document.execCommand('selectAll', false); document.execCommand('delete', false);
|
||||
return true; })()`);
|
||||
for (const ch of text) {
|
||||
wv.sendInputEvent({ type: 'char', keyCode: ch });
|
||||
}
|
||||
// Read back from the marked element OR the active element: editors like Reddit's swap the
|
||||
// node on activation, so the original selector can go stale even though the keystrokes landed
|
||||
// in whatever now holds focus. Checking both survives that re-render.
|
||||
const readBack = `(() => {
|
||||
const check = (el) => { if (!el) return false;
|
||||
el.dispatchEvent(new Event('input', { bubbles: true })); el.dispatchEvent(new Event('change', { bubbles: true }));
|
||||
const now = (el.value != null ? el.value : (el.textContent || '')); return now.includes(${JSON.stringify(text)}); };
|
||||
return check(document.querySelector(${safeSel})) || check(document.activeElement); })()`;
|
||||
return Boolean(await evalInPage(wv, readBack));
|
||||
}
|
||||
|
||||
// Electron sendInputEvent expects names like 'Up', 'Enter', 'Space', not 'ArrowUp'/' '/'Esc'.
|
||||
const KEY_NAME_MAP: Record<string, string> = {
|
||||
ArrowUp: 'Up',
|
||||
@@ -405,6 +758,8 @@ async function handlePressKey(wv: BrowserWebview, params: Record<string, any>):
|
||||
}
|
||||
// Legacy focus-dependent fallback (exotic keys or CDP unavailable): keeps every key that worked before working.
|
||||
const keyCode = KEY_NAME_MAP[rawKey] || rawKey;
|
||||
await evalInPage(wv, 'document.body && document.body.focus && document.body.focus(); true');
|
||||
// Native OS-level key events have isTrusted=true, so hostile sites' keyboard handlers respect them.
|
||||
wv.sendInputEvent({ type: 'keyDown', keyCode });
|
||||
wv.sendInputEvent({ type: 'char', keyCode });
|
||||
wv.sendInputEvent({ type: 'keyUp', keyCode });
|
||||
@@ -436,16 +791,21 @@ async function handleClickPoint(wv: BrowserWebview, params: Record<string, any>)
|
||||
} catch { /* use the element box as a fallback */ }
|
||||
const x = (cx / 100) * vw;
|
||||
const y = (cy / 100) * vh;
|
||||
// hoverOnly: real mouse move, no press. Hover-revealed controls (reddit's post kebab only
|
||||
// materializes on tile hover) need this; a click there would navigate into the post.
|
||||
const hoverOnly = params.hoverOnly === true;
|
||||
try {
|
||||
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y });
|
||||
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: 1 });
|
||||
if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs));
|
||||
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: 1 });
|
||||
if (!hoverOnly) {
|
||||
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: 1 });
|
||||
if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs));
|
||||
await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: 1 });
|
||||
}
|
||||
} catch (err: any) {
|
||||
return { error: `Click point failed: ${err?.message || String(err)}` };
|
||||
}
|
||||
return {
|
||||
text: `Clicked at (${Math.round(x)}, ${Math.round(y)})${holdMs ? ` held ${holdMs}ms` : ''}.`,
|
||||
text: `${hoverOnly ? 'Hovered' : 'Clicked'} at (${Math.round(x)}, ${Math.round(y)})${holdMs && !hoverOnly ? ` held ${holdMs}ms` : ''}.`,
|
||||
clickX: cx, clickY: cy, url: wv.getURL(),
|
||||
};
|
||||
}
|
||||
@@ -794,7 +1154,9 @@ async function clickBackendNode(
|
||||
const ly = (content[1] + content[5]) / 2;
|
||||
|
||||
// Hit-test before dispatching: a sticky banner or header twin can cover the element's center, and a blind coordinate click lands on the overlay instead (the "Reactivate Premium" misfire). If covered, click the chosen node itself.
|
||||
// This is also the ground-truth "did my hand land where I aimed" signal: when the element at the click point is NOT the intended node, a naive coordinate click hits the WRONG element (the failure a page-change metric can't see, because the wrong element also changes the page). We surface what was actually hit so the wrong-target RATE is measurable.
|
||||
let covered = false;
|
||||
let hitDesc = '';
|
||||
let targetObjectId: string | undefined;
|
||||
try {
|
||||
const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
|
||||
@@ -803,13 +1165,20 @@ async function clickBackendNode(
|
||||
const rel = await sendCdp(wv, 'Runtime.callFunctionOn', {
|
||||
objectId: targetObjectId,
|
||||
functionDeclaration:
|
||||
'function(x, y) { const r = this.getRootNode(); const h = (r.elementFromPoint ? r : document).elementFromPoint(x, y); return h ? (this === h || this.contains(h)) : true; }',
|
||||
'function(x, y) { const r = this.getRootNode(); const h = (r.elementFromPoint ? r : document).elementFromPoint(x, y);'
|
||||
+ ' const landed = h ? (this === h || this.contains(h) || h.contains(this)) : true;'
|
||||
+ ' const d = h ? (h.tagName.toLowerCase() + (h.getAttribute("aria-label") ? "[" + h.getAttribute("aria-label").slice(0,30) + "]" : (h.textContent ? "[" + h.textContent.trim().slice(0,30) + "]" : ""))) : "none";'
|
||||
+ ' return { landed: landed, hit: d }; }',
|
||||
arguments: [{ value: lx }, { value: ly }],
|
||||
returnByValue: true,
|
||||
}, sessionId);
|
||||
covered = rel?.result?.value === false;
|
||||
const hv = rel?.result?.value as { landed?: boolean; hit?: string } | undefined;
|
||||
covered = hv?.landed === false;
|
||||
hitDesc = hv?.hit || '';
|
||||
}
|
||||
} catch { /* hit-test is best-effort; fall through to the coordinate click */ }
|
||||
// Attach the aim signal to every click result (near-zero cost, already computed): landed=false means the intended element was not under the click point (occluded / stale box / moved), i.e. a wrong-target click.
|
||||
const p_aim = { clickLanded: !covered, clickHit: hitDesc };
|
||||
|
||||
let rx = lx, ry = ly;
|
||||
if (sessionId) {
|
||||
@@ -828,7 +1197,7 @@ async function clickBackendNode(
|
||||
functionDeclaration:
|
||||
'function() { if (this.click) { this.click(); } else { this.dispatchEvent(new MouseEvent("click", { bubbles: true, cancelable: true })); } }',
|
||||
}, sessionId);
|
||||
return { text: `Clicked ${label} via its element (another element covers its screen position).`, ...ripple };
|
||||
return { text: `Clicked ${label} via its element (another element covers its screen position).`, ...ripple, ...p_aim };
|
||||
} catch (err: any) {
|
||||
return { error: `${label} is covered by another element and could not be clicked (${err?.message || String(err)}). Scroll, or pick a different element.` };
|
||||
}
|
||||
@@ -843,9 +1212,28 @@ async function clickBackendNode(
|
||||
return {
|
||||
text: `Clicked ${label} at (${Math.round(rx)}, ${Math.round(ry)})`,
|
||||
...ripple,
|
||||
...p_aim,
|
||||
};
|
||||
}
|
||||
|
||||
// Wrap a click so we can measure whether it actually did anything, covering EVERY
|
||||
// exit of clickBackendNode (coordinate, covered-element JS click, ...), which is why
|
||||
// it lives out here and not in the one coordinate branch. Metric only, flag-gated.
|
||||
async function measureClickEffect(
|
||||
wv: BrowserWebview, run: () => Promise<Record<string, any>>,
|
||||
): Promise<Record<string, any>> {
|
||||
let before = '';
|
||||
try { before = String(await wv.executeJavaScript(FP_EXPR)); } catch { /* unreadable */ }
|
||||
const result = await run();
|
||||
if (!result.error) {
|
||||
await new Promise((r) => setTimeout(r, 400));
|
||||
let after = '';
|
||||
try { after = String(await wv.executeJavaScript(FP_EXPR)); } catch { /* unreadable */ }
|
||||
result.clickEffect = clickEffect(before, after);
|
||||
}
|
||||
return result;
|
||||
}
|
||||
|
||||
// Drop list rows the user literally cannot click: zero-size nodes and ones whose center hits a DIFFERENT element (modal backdrop, sticky header, cookie banner). Ground truth via elementFromPoint, the same predicate the click path trusts. Offscreen-but-scrollable elements are kept; the page-wide list is deliberately wider than the viewport. Chunked with a hard budget so a heavy page degrades to an unfiltered list, never a stall.
|
||||
const _OCCLUSION_BUDGET_MS = 1500;
|
||||
const _OCCLUSION_CHUNK = 10;
|
||||
@@ -904,7 +1292,7 @@ async function handleListInteractives(wv: BrowserWebview, params: Record<string,
|
||||
|
||||
// Dedupe twins, rank what a human acts on first (and the current goal highest), cap the long tail.
|
||||
const goal = typeof params?.goal === 'string' ? params.goal : '';
|
||||
const { shown: ranked, truncated } = rankAndCapInteractives(candidates, { goal });
|
||||
const { shown: ranked, truncated } = rankAndCapInteractives(candidates, { goal, docOrder: params?.docOrder !== false });
|
||||
const { kept: shown, dropped: covered } = await dropCoveredElements(wv, ranked);
|
||||
|
||||
// The previous look's cache feeds two things: * markers for brand-new elements, and STABLE indices so the same element keeps the same number across looks (the model can act on a remembered index without re-reading the whole list, browser-use's stable-hash trick on our node ids).
|
||||
@@ -1022,8 +1410,24 @@ async function handleClickIndex(wv: BrowserWebview, params: Record<string, any>)
|
||||
};
|
||||
}
|
||||
|
||||
const result = await clickBackendNode(wv, backendNodeId, sessionId, `index ${idx}`,
|
||||
{ role, text: typeof params.text === 'string' ? params.text : undefined });
|
||||
const wantsText = typeof params.text === 'string' && params.text.length > 0;
|
||||
const doClick = () => clickBackendNode(wv, backendNodeId as number, sessionId, `index ${idx}`,
|
||||
{ role, text: wantsText ? params.text : undefined });
|
||||
// Effect metric on plain clicks only (a fill verifies via readback); covers every clickBackendNode exit.
|
||||
const result = (params.effectProbe === true && !wantsText)
|
||||
? await measureClickEffect(wv, doClick)
|
||||
: await doClick();
|
||||
// Self-healing escalation: a cached index goes stale the instant the page mutates, which is HALF of all runs' tool-errors. An explicit clickBackendNode error PROVES the click never landed (so re-trying the same target can't double-act), so before we hand a ~3s re-strategize turn back to the model, resolve the SAME element fresh from the full DOM by its name+role, the exact rung the model would have climbed to itself. Plain clicks only (a text-fill has its own readback path); gated so an A/B can turn it off.
|
||||
if (shouldSelfHealClick(!!result.error, wantsText, name, params.selfheal)) {
|
||||
const healed = await handleClickByName(wv, { name: name as string, role: role || '' });
|
||||
if (!healed.error) {
|
||||
console.log(`[selfheal] index ${idx} stale -> recovered via name "${String(name).slice(0, 40)}"`);
|
||||
healed.clickedRole = role || '';
|
||||
healed.clickedName = name || '';
|
||||
healed.selfHealed = 'by-name';
|
||||
return healed;
|
||||
}
|
||||
}
|
||||
// Surface what was clicked so the agent loop can record a stable, replayable click-by-name step (indices are ephemeral; names aren't).
|
||||
if (!result.error) {
|
||||
result.clickedRole = role || '';
|
||||
@@ -1106,6 +1510,11 @@ async function handleBatch(wv: BrowserWebview, params: Record<string, any>): Pro
|
||||
}
|
||||
|
||||
const urlBefore = wv.getURL();
|
||||
// Carry the self-heal + click-effect toggles into click sub-actions (most clicks are batched, so gating only the top-level click_index misses them).
|
||||
if (subType === 'click_index') {
|
||||
if (params.selfheal !== undefined && subParams.selfheal === undefined) subParams.selfheal = params.selfheal;
|
||||
if (params.effectProbe && subParams.effectProbe === undefined) subParams.effectProbe = true;
|
||||
}
|
||||
let subResult: Record<string, any>;
|
||||
try {
|
||||
subResult = await BATCH_DISPATCH[subType](wv, subParams);
|
||||
@@ -1360,8 +1769,12 @@ async function handleDetectWebMCP(wv: BrowserWebview): Promise<Record<string, an
|
||||
}
|
||||
}
|
||||
|
||||
// Tier 2: the safe GET routes captured for the current site, so the agent can fetch data directly instead of re-scraping the UI. Only same-origin GET/HEAD routes are listed; those are all that replay_route will run.
|
||||
async function handleListRoutes(wv: BrowserWebview): Promise<Record<string, any>> {
|
||||
// Tier 2: the API routes captured for the current site, so the agent can act directly instead of
|
||||
// re-scraping/clicking the UI. Default lists the safe GET/HEAD routes (all replay_route will run).
|
||||
// With { writes: true } it lists the MUTATING routes (POST/PUT/PATCH/DELETE) the site's own UI
|
||||
// fired, for BrowserApiWrite's general 'route' path; the write itself is same-origin + captured +
|
||||
// session-borrowed + flag-gated in the backend, this only SURFACES the endpoint shape.
|
||||
async function handleListRoutes(wv: BrowserWebview, params?: Record<string, any>): Promise<Record<string, any>> {
|
||||
const bridge = (window as any).openswarm?.cdpRoutesGet as
|
||||
| ((id: number, origin?: string) => Promise<any[]>) | undefined;
|
||||
if (!bridge) return { error: 'Route capture not available, restart the app.' };
|
||||
@@ -1369,6 +1782,21 @@ async function handleListRoutes(wv: BrowserWebview): Promise<Record<string, any>
|
||||
try { origin = new URL(wv.getURL()).origin; } catch {}
|
||||
let routes: any[] = [];
|
||||
try { routes = (await bridge(wv.getWebContentsId(), origin)) || []; } catch {}
|
||||
|
||||
if (params?.writes) {
|
||||
const writes = routes.filter((r) => r && r.safe === false);
|
||||
if (!writes.length) {
|
||||
return { text: 'No write (POST/PUT/PATCH/DELETE) API routes captured for this site yet. Do the write once through the UI so it gets recorded, then the route path can replay it.', url: wv.getURL() };
|
||||
}
|
||||
const wlines = writes.slice(0, 40).map((r) => `${r.method} ${r.template} body-shape: ${JSON.stringify(r.bodyShape)} (seen ${r.hits}x)`);
|
||||
return {
|
||||
text: `Write endpoints this site's UI uses (for BrowserApiWrite action='route'). Pass the `
|
||||
+ `method + url + a body matching the shape, with your content in the text field:\n${wlines.join('\n')}`,
|
||||
routes: writes.slice(0, 40),
|
||||
url: wv.getURL(),
|
||||
};
|
||||
}
|
||||
|
||||
const safe = routes.filter((r) => r && r.safe);
|
||||
if (!safe.length) {
|
||||
return { text: 'No replayable (GET) API routes captured for this site yet. Use the page first so they get recorded, then try again.', url: wv.getURL() };
|
||||
@@ -1510,6 +1938,22 @@ async function handleSessionCookies(params: Record<string, any>): Promise<Record
|
||||
}
|
||||
}
|
||||
|
||||
// Load the user's own existing sign-in for a site into the browser partition, so an agent stuck at
|
||||
// a login wall can carry on as them instead of interrupting to ask for a password. Like the cookie
|
||||
// bridge above this needs no webview: it writes straight to the main-process cookie store.
|
||||
async function handleImportSession(params: Record<string, any>): Promise<Record<string, any>> {
|
||||
const bridge = (window as any).openswarm?.setPartitionCookies as
|
||||
| ((domain: string, cookies: Record<string, any>[]) => Promise<{ ok: boolean; set: number; error?: string }>)
|
||||
| undefined;
|
||||
if (!bridge) return { ok: false, set: 0, error: 'Session import unavailable (desktop app only)' };
|
||||
const cookies = Array.isArray(params.cookies) ? params.cookies : [];
|
||||
try {
|
||||
return await bridge(String(params.domain || ''), cookies);
|
||||
} catch (err: any) {
|
||||
return { ok: false, set: 0, error: `Session import failed: ${err?.message || String(err)}` };
|
||||
}
|
||||
}
|
||||
|
||||
// Drive a session-borrow site's own already-open card: resolve the webview by its live domain
|
||||
// (no browser_id, like the cookie bridge), then run a small navigate/evaluate step sequence.
|
||||
// The shims use this for writes on sites that sign every HTTP request (TikTok).
|
||||
@@ -1560,6 +2004,11 @@ async function runBrowserCommand(
|
||||
dashboardWs.send('browser:result', { request_id, ...result });
|
||||
return;
|
||||
}
|
||||
if (action === 'import_session') {
|
||||
const result = await handleImportSession(params);
|
||||
dashboardWs.send('browser:result', { request_id, ...result });
|
||||
return;
|
||||
}
|
||||
const wv = await awaitWebview(browser_id, tab_id || undefined, action);
|
||||
if (!wv) {
|
||||
dashboardWs.send('browser:result', {
|
||||
@@ -1600,6 +2049,9 @@ async function runBrowserCommand(
|
||||
case 'type':
|
||||
result = await handleType(wv, params);
|
||||
break;
|
||||
case 'find_composer':
|
||||
result = await handleFindComposer(wv, params);
|
||||
break;
|
||||
case 'evaluate':
|
||||
result = await handleEvaluate(wv, params);
|
||||
break;
|
||||
@@ -1645,7 +2097,7 @@ async function runBrowserCommand(
|
||||
result = await handleDetectWebMCP(wv);
|
||||
break;
|
||||
case 'list_routes':
|
||||
result = await handleListRoutes(wv);
|
||||
result = await handleListRoutes(wv, params);
|
||||
break;
|
||||
case 'click_by_name':
|
||||
result = await handleClickByName(wv, params);
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// Run: node --test frontend/src/shared/cdpEval.test.ts
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { unwrapCdpEval } from './cdpEval.ts';
|
||||
|
||||
test('returns the serialized value on success', () => {
|
||||
assert.equal(unwrapCdpEval({ result: { value: 'hello', type: 'string' } }), 'hello');
|
||||
});
|
||||
|
||||
test('returns an object value untouched (returnByValue serialized)', () => {
|
||||
const v = unwrapCdpEval({ result: { value: { found: true, filled: false } } }) as any;
|
||||
assert.equal(v.found, true);
|
||||
assert.equal(v.filled, false);
|
||||
});
|
||||
|
||||
test('undefined result value comes back as undefined, not a throw', () => {
|
||||
assert.equal(unwrapCdpEval({ result: { type: 'undefined' } }), undefined);
|
||||
});
|
||||
|
||||
test('a page-side throw surfaces as an Error with the exception description', () => {
|
||||
assert.throws(
|
||||
() => unwrapCdpEval({ exceptionDetails: { exception: { description: 'ReferenceError: x is not defined' } } }),
|
||||
/x is not defined/,
|
||||
);
|
||||
});
|
||||
|
||||
test('falls back to exceptionDetails.text when no exception description', () => {
|
||||
assert.throws(
|
||||
() => unwrapCdpEval({ exceptionDetails: { text: 'Uncaught' } }),
|
||||
/Uncaught/,
|
||||
);
|
||||
});
|
||||
|
||||
test('exceptionDetails wins even if a result is also present', () => {
|
||||
assert.throws(
|
||||
() => unwrapCdpEval({ result: { value: 'partial' }, exceptionDetails: { text: 'boom' } }),
|
||||
/boom/,
|
||||
);
|
||||
});
|
||||
@@ -0,0 +1,21 @@
|
||||
// Turn a CDP `Runtime.evaluate` result into the value, the way webContents.executeJavaScript
|
||||
// hands it back: return the serialized value, and throw when the page code itself threw (that
|
||||
// arrives as `exceptionDetails`, not as an infra error). Kept pure + separate so it's unit
|
||||
// testable without the whole browser-command module and its Electron globals.
|
||||
|
||||
export interface CdpEvalResult {
|
||||
result?: { value?: unknown; type?: string };
|
||||
exceptionDetails?: {
|
||||
text?: string;
|
||||
exception?: { description?: string; value?: unknown };
|
||||
};
|
||||
}
|
||||
|
||||
export function unwrapCdpEval(cdp: CdpEvalResult): unknown {
|
||||
if (cdp && cdp.exceptionDetails) {
|
||||
const ex = cdp.exceptionDetails;
|
||||
const msg = (ex.exception && (ex.exception.description || ex.exception.value)) || ex.text || 'eval error in page';
|
||||
throw new Error(String(msg));
|
||||
}
|
||||
return cdp && cdp.result ? cdp.result.value : undefined;
|
||||
}
|
||||
@@ -0,0 +1,20 @@
|
||||
// Run: node --test frontend/src/shared/clickEffect.test.ts
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { clickEffect } from './clickEffect.ts';
|
||||
|
||||
test('a click that changed the page fingerprint = changed', () => {
|
||||
assert.equal(clickEffect('u1|1200|BUTTON|0', 'u1|1214|BUTTON|0'), 'changed'); // menu opened (+14 nodes)
|
||||
assert.equal(clickEffect('u1|1200|BUTTON|0', 'u2|1200|BUTTON|0'), 'changed'); // navigated
|
||||
assert.equal(clickEffect('u1|1200|DIVfalse|0', 'u1|1200|DIVtrue|0'), 'changed'); // aria-expanded toggled
|
||||
assert.equal(clickEffect('u1|1200|BODY|0', 'u1|1200|BODY|380'), 'changed'); // scrolled
|
||||
});
|
||||
|
||||
test('a click that changed NOTHING = none (the invisible wrong/dead-element failure)', () => {
|
||||
assert.equal(clickEffect('u1|1200|BUTTON|0', 'u1|1200|BUTTON|0'), 'none');
|
||||
});
|
||||
|
||||
test('an unreadable fingerprint (empty) is not counted as a real no-effect', () => {
|
||||
assert.equal(clickEffect('', 'u1|1200|BUTTON|0'), 'none');
|
||||
assert.equal(clickEffect('u1|1200|BUTTON|0', ''), 'none');
|
||||
});
|
||||
@@ -0,0 +1,13 @@
|
||||
// A cheap page fingerprint taken before and after a click, so we can MEASURE the
|
||||
// invisible failure the tool-error counter misses: a click that "succeeds" (dispatches
|
||||
// fine) but lands on the wrong element or a dead one, so nothing on the page changes.
|
||||
// A real click almost always moves at least one of: the URL, the element count (menu
|
||||
// opened / row added), the focused element, or the scroll position.
|
||||
export const FP_EXPR =
|
||||
"location.href + '|' + document.getElementsByTagName('*').length + '|' + "
|
||||
+ "(document.activeElement ? document.activeElement.tagName + (document.activeElement.getAttribute('aria-expanded')||'') + (document.activeElement.getAttribute('aria-checked')||'') : '') + '|' + "
|
||||
+ "Math.round(window.scrollY)";
|
||||
|
||||
export function clickEffect(before: string, after: string): 'changed' | 'none' {
|
||||
return before && after && before !== after ? 'changed' : 'none';
|
||||
}
|
||||
@@ -37,14 +37,26 @@ test('non-consecutive same name is preserved (real list items)', () => {
|
||||
assert.equal(carts.length, 3);
|
||||
});
|
||||
|
||||
test('ranks by role priority: input > button/link > toggle > option', () => {
|
||||
test('display order is DOCUMENT order, not rank order (so ordinals can be counted)', () => {
|
||||
const { shown } = rankAndCapInteractives([
|
||||
mk('option', 'opt', 1),
|
||||
mk('checkbox', 'agree', 2),
|
||||
mk('button', 'Go', 3),
|
||||
mk('textbox', 'email', 4),
|
||||
]);
|
||||
assert.deepEqual(shown.map((x) => x.role), ['textbox', 'button', 'checkbox', 'option']);
|
||||
// all survive the cap; they render top-to-bottom as they appear on the page,
|
||||
// NOT re-sorted by role (which would scramble "the 4th thing")
|
||||
assert.deepEqual(shown.map((x) => x.backendNodeId), [1, 2, 3, 4]);
|
||||
});
|
||||
|
||||
test('rank still decides cap SURVIVAL: a high-priority input buried in options is kept', () => {
|
||||
const items = [
|
||||
...Array.from({ length: 40 }, (_, i) => mk('option', `opt${i}`, i)),
|
||||
mk('textbox', 'email', 900),
|
||||
...Array.from({ length: 40 }, (_, i) => mk('option', `optb${i}`, 100 + i)),
|
||||
];
|
||||
const { shown } = rankAndCapInteractives(items, { cap: 30 });
|
||||
assert.ok(shown.some((x) => x.backendNodeId === 900), 'the input survives the cap by rank');
|
||||
});
|
||||
|
||||
test('preserves document order within the same priority tier', () => {
|
||||
@@ -71,31 +83,23 @@ test('cap of 0 means no cap', () => {
|
||||
assert.equal(truncated, 0);
|
||||
});
|
||||
|
||||
test('goal-matched elements float to the top, above role priority', () => {
|
||||
const { shown } = rankAndCapInteractives([
|
||||
mk('textbox', 'Search', 1),
|
||||
mk('link', 'Account settings', 2),
|
||||
mk('button', 'Save', 3),
|
||||
], { goal: 'open the settings page' });
|
||||
// "Account settings" matches "settings" and jumps ahead of the textbox
|
||||
assert.equal(shown[0].backendNodeId, 2);
|
||||
});
|
||||
|
||||
test('goal match survives the cap even when buried deep', () => {
|
||||
test('goal-matched element survives the cap and displays IN PLACE (document order)', () => {
|
||||
const items = Array.from({ length: 100 }, (_, i) => mk('link', `Item ${i}`, i));
|
||||
items.push(mk('button', 'Checkout now', 999));
|
||||
items.push(mk('button', 'Checkout now', 999)); // last on the page
|
||||
const { shown } = rankAndCapInteractives(items, { cap: 30, goal: 'click checkout' });
|
||||
// it survives the cap (rank kept it)...
|
||||
assert.ok(shown.some((x) => x.backendNodeId === 999), 'checkout should be retained');
|
||||
assert.equal(shown[0].backendNodeId, 999);
|
||||
// ...and renders at its real position (last), not floated to [0]
|
||||
assert.equal(shown[shown.length - 1].backendNodeId, 999);
|
||||
});
|
||||
|
||||
test('no goal leaves pure role-priority ordering', () => {
|
||||
test('display order stays document order with no goal', () => {
|
||||
const { shown } = rankAndCapInteractives([
|
||||
mk('option', 'opt', 1),
|
||||
mk('button', 'Go', 2),
|
||||
mk('textbox', 'email', 3),
|
||||
]);
|
||||
assert.deepEqual(shown.map((x) => x.role), ['textbox', 'button', 'option']);
|
||||
assert.deepEqual(shown.map((x) => x.backendNodeId), [1, 2, 3]);
|
||||
});
|
||||
|
||||
test('goalKeywords strips stopwords, action verbs, and short tokens', () => {
|
||||
@@ -110,11 +114,22 @@ test('empty input yields empty result', () => {
|
||||
assert.equal(truncated, 0);
|
||||
});
|
||||
|
||||
test('unknown role falls into the middle tier, not dropped', () => {
|
||||
test('unknown role is not dropped (survives the cap), displayed in document order', () => {
|
||||
const { shown } = rankAndCapInteractives([
|
||||
mk('option', 'opt', 1),
|
||||
mk('weirdrole', 'mystery', 2),
|
||||
mk('textbox', 'field', 3),
|
||||
]);
|
||||
assert.deepEqual(shown.map((x) => x.role), ['textbox', 'weirdrole', 'option']);
|
||||
assert.deepEqual(shown.map((x) => x.backendNodeId), [1, 2, 3]);
|
||||
assert.ok(shown.some((x) => x.role === 'weirdrole'));
|
||||
});
|
||||
|
||||
test('docOrder:false keeps legacy rank-order display (the A/B off-arm)', () => {
|
||||
const { shown } = rankAndCapInteractives([
|
||||
mk('option', 'opt', 1),
|
||||
mk('button', 'Go', 2),
|
||||
mk('textbox', 'email', 3),
|
||||
], { docOrder: false });
|
||||
// rank order: textbox(0) < button(1) < option(3)
|
||||
assert.deepEqual(shown.map((x) => x.backendNodeId), [3, 2, 1]);
|
||||
});
|
||||
|
||||
@@ -48,6 +48,9 @@ export interface RankOptions {
|
||||
cap?: number;
|
||||
// The agent's current goal; elements whose name matches it float to the top so the thing the model is actually looking for survives the cap.
|
||||
goal?: string;
|
||||
// Display the surviving items in document order (default). false = legacy rank-order
|
||||
// display, kept only so the A/B can measure the document-order win against it.
|
||||
docOrder?: boolean;
|
||||
}
|
||||
|
||||
// Words too generic to be useful signal, including the browser-action verbs and UI nouns that would otherwise match half the page ("click the button").
|
||||
@@ -77,17 +80,23 @@ export function rankAndCapInteractives(
|
||||
const cap = opts.cap ?? DEFAULT_INTERACTIVE_CAP;
|
||||
const keywords = opts.goal ? goalKeywords(opts.goal) : [];
|
||||
const deduped = dedupeConsecutive(items);
|
||||
// Sort: goal-matched first, then role priority, tiebroken on original document order so the result is deterministic regardless of engine sort.
|
||||
const ranked = deduped
|
||||
.map((it, i) => ({ it, i, m: matchesGoal(it.name, keywords) ? 0 : 1 }))
|
||||
.sort((a, b) => {
|
||||
if (a.m !== b.m) return a.m - b.m;
|
||||
const pa = rolePriority(a.it.role);
|
||||
const pb = rolePriority(b.it.role);
|
||||
if (pa !== pb) return pa - pb;
|
||||
return a.i - b.i;
|
||||
})
|
||||
.map((x) => x.it);
|
||||
const shown = cap > 0 ? ranked.slice(0, cap) : ranked;
|
||||
const scored = deduped.map((it, i) => ({ it, i, m: matchesGoal(it.name, keywords) ? 0 : 1 }));
|
||||
// Rank picks WHICH items survive the cap (goal-matched first, then role priority),
|
||||
// so the thing the model wants is never truncated away.
|
||||
const ranked = [...scored].sort((a, b) => {
|
||||
if (a.m !== b.m) return a.m - b.m;
|
||||
const pa = rolePriority(a.it.role);
|
||||
const pb = rolePriority(b.it.role);
|
||||
if (pa !== pb) return pa - pb;
|
||||
return a.i - b.i;
|
||||
});
|
||||
const selected = cap > 0 ? ranked.slice(0, cap) : ranked;
|
||||
// But the DISPLAY order is document order (by original index), so the numbered
|
||||
// list reads top-to-bottom the way the page looks. A rank-sorted list scrambled
|
||||
// ordinals ("the 4th story" landed at [48], out of order), forcing the model to
|
||||
// burn a turn reading page text just to recover position; document order lets it
|
||||
// count directly. The high-signal-subset win (from the cap) is untouched.
|
||||
const displayed = opts.docOrder === false ? selected : selected.slice().sort((a, b) => a.i - b.i);
|
||||
const shown = displayed.map((x) => x.it);
|
||||
return { shown, truncated: Math.max(0, ranked.length - shown.length) };
|
||||
}
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
// Run: node --test frontend/src/shared/selfHealClick.test.ts
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { shouldSelfHealClick } from './selfHealClick.ts';
|
||||
|
||||
test('escalates a plain named click that errored', () => {
|
||||
assert.equal(shouldSelfHealClick(true, false, 'Send', true), true);
|
||||
});
|
||||
|
||||
test('never escalates a click that SUCCEEDED (no double-act)', () => {
|
||||
assert.equal(shouldSelfHealClick(false, false, 'Send', true), false);
|
||||
});
|
||||
|
||||
test('never escalates a text-fill (fills verify themselves)', () => {
|
||||
assert.equal(shouldSelfHealClick(true, true, 'Write a message', true), false);
|
||||
});
|
||||
|
||||
test('cannot escalate without a name to re-resolve by', () => {
|
||||
assert.equal(shouldSelfHealClick(true, false, undefined, true), false);
|
||||
assert.equal(shouldSelfHealClick(true, false, '', true), false);
|
||||
});
|
||||
|
||||
test('A/B off-arm (selfheal === false) disables it; undefined/absent stays on', () => {
|
||||
assert.equal(shouldSelfHealClick(true, false, 'Send', false), false);
|
||||
assert.equal(shouldSelfHealClick(true, false, 'Send', undefined), true);
|
||||
});
|
||||
@@ -0,0 +1,9 @@
|
||||
// Escalate a failed cached-index click to a fresh by-name resolution only when it's
|
||||
// safe and possible: the click explicitly ERRORED (so it provably never landed, no
|
||||
// double-act risk), it wasn't a text-fill (fills verify themselves), we know the
|
||||
// element's name to re-find it, and the A/B toggle is on. Pure so the invariant is testable.
|
||||
export function shouldSelfHealClick(
|
||||
errored: boolean, wantsText: boolean, name: string | undefined, selfheal: unknown,
|
||||
): boolean {
|
||||
return errored && !wantsText && !!name && selfheal !== false;
|
||||
}
|
||||
@@ -52,6 +52,7 @@ export interface AppSettings {
|
||||
openrouter_api_key?: string | null;
|
||||
custom_providers?: CustomProvider[];
|
||||
browser_homepage: string;
|
||||
browser_import_signins: boolean;
|
||||
auto_select_mode_on_new_agent: boolean;
|
||||
expand_new_chats_in_dashboard: boolean;
|
||||
auto_reveal_sub_agents: boolean;
|
||||
@@ -169,6 +170,7 @@ export const DEFAULT_SETTINGS: AppSettings = {
|
||||
dictation_shortcut: null,
|
||||
anthropic_api_key: null,
|
||||
browser_homepage: 'https://duckduckgo.com',
|
||||
browser_import_signins: false,
|
||||
auto_select_mode_on_new_agent: false,
|
||||
expand_new_chats_in_dashboard: true,
|
||||
auto_reveal_sub_agents: true,
|
||||
|
||||
@@ -110,16 +110,24 @@ class WebSocketManager {
|
||||
// Frame-aligned message coalescer. Buffers incoming WS messages from all WebSocketManager instances and flushes them in ONE batched React render per animation frame. Without this, N concurrent agents each cause their own renders on every WS message, dozens of full app re-renders per second, fanning out to every useSelector. With it: max one render per frame regardless of message volume.
|
||||
private static _messageQueue: Array<{ mgr: WebSocketManager; msg: WSEvent }> = [];
|
||||
private static _flushScheduled = false;
|
||||
// rAF never fires when the window paints no frames (minimized, on another Space, occluded). Without a fallback, WS-delivered dashboard mutations (a spawned browser card, an evict) buffer forever and the UI silently desyncs, agent browser cards just never mount. A timer flushes the queue when rAF won't; whichever fires first wins and cancels the other, so a visible window is unchanged (rAF beats a 250ms timer every frame).
|
||||
private static _flushTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
private static _enqueueMessage(mgr: WebSocketManager, msg: WSEvent) {
|
||||
WebSocketManager._messageQueue.push({ mgr, msg });
|
||||
if (WebSocketManager._flushScheduled) return;
|
||||
WebSocketManager._flushScheduled = true;
|
||||
requestAnimationFrame(WebSocketManager._flushMessages);
|
||||
WebSocketManager._flushTimer = setTimeout(WebSocketManager._flushMessages, 250);
|
||||
}
|
||||
|
||||
private static _flushMessages = () => {
|
||||
if (!WebSocketManager._flushScheduled) return; // the other trigger already drained this batch
|
||||
WebSocketManager._flushScheduled = false;
|
||||
if (WebSocketManager._flushTimer !== null) {
|
||||
clearTimeout(WebSocketManager._flushTimer);
|
||||
WebSocketManager._flushTimer = null;
|
||||
}
|
||||
if (WebSocketManager._messageQueue.length === 0) return;
|
||||
const batch = WebSocketManager._messageQueue;
|
||||
WebSocketManager._messageQueue = [];
|
||||
|
||||
@@ -22,7 +22,7 @@
|
||||
"max-folder-items-exceptions": "Exact-path allow for folders intentionally over the cap. The rule trips at >7 (7 items is fine, the 8th tips it), so only genuinely 8+ folders are listed. backend/ and backend/apps are FastAPI feature-package registries (each child is an app mounted in main.py); agents/ aggregates agent subsystems; agents/manager/ is the agent_manager god-object decomposition (cohesive AgentManager mixins + standalone run helpers + the streaming/permissions/prompt/session subtrees), conventionally flat like agents/ and core/ since its standalone helpers are heterogeneous and don't group cleanly; agents/manager/streaming and agents/manager/session are flat peer collections of one-module-per-concern handlers; core/, tools_lib/, tests/ are conventionally flat. Frontend: app/pages is the page registry, AgentChat/ChatInput/Settings-sections/Onboarding are organizational parents, and shared/state (Redux slices) plus hooks/steps/mcp-cards/Views are flat peer collections. scripts/, electron/, linter/checks/ are flat tool dirs. These replaced blanket .lintignore-max-folder-items sentinels (backend, frontend, scripts, electron, linter/checks) so the rule still catches NEW unplanned bloat everywhere else. Kept as whole-subtree sentinels on purpose: debugger/ (self-contained injected sub-tool with its own Vite GUI), webapp_template (Vite scaffold payload), and vendored mcp-bundles. 2026-07 desktop-shell additions: Dashboard canvas/cards/desktop + hooks/interaction + hooks/lifecycle, AgentChat bubbles/tool-ui, and shared/styles are flat peer collections (one component or hook per concern) that crossed 7 as the redesign surface grew. frontend/src/toolui carries a whole-subtree .lintignore: vendored tool-ui component library (pierre), same treatment as mcp-bundles.",
|
||||
"import-cycles": "Flags RUNTIME circular imports only (SCC>1). Skips type-only imports (import type / export type) and dynamic import() since neither runs at module init, which is why the idiomatic Redux store<->hooks type cycle is not flagged. Frontend alias resolution comes from import-cycle-aliases. Zero cycles today; the check keeps it that way.",
|
||||
"ruff + pyright": "Ported from Haik's linter (haik/feat/ingest). ruff is narrowed to F401/F811/F841 (unused imports/redefs/locals) and intentionally DROPS Haik's ARG001/ARG002 (unused args): our SDK-callback signatures require unused params (can_use_tool/pre_tool_hook take a `context` they don't use) and we ban the `_unused` prefix, so ARG is noise here. pyright runs Haik's existence-only config (typeCheckingMode off) with reportAttributeAccessIssue ENABLED: the AgentManager behavior classes now inherit a typing-only AgentManagerProtocol base (manager/AgentManagerProtocol.py) that declares the composed __init__ state + cross-class methods, so the checker sees self.sessions etc. from inside a mixin. pyright caught real bugs: a dangling `_conns` ref + TWO broken lazy imports (`_load_all`/`_load` from outputs.py, renamed to load_all/load in workspace_io but the import sites weren't updated \u2014 App Builder workspace seeding/name-sync was silently failing in a try/except). The one grandfathered SURFACE file (handle_assistant_message) is the SDK-optional try/except-import boundary (TextBlock=object fallback defeats isinstance narrowing). Both grandfather pre-existing debt by file; the refactor surface is clean. Requires `ruff` + `pyright` on PATH (added to requirements-dev.txt); pyright's config expects the venv at backend/.venv.",
|
||||
"no-underscore-names + p-private": "Convention checks ported verbatim from Haik's linter (haik/feat/ingest): no-underscore-names bans leading-underscore names (a dead-code-tooling blind spot; use p_ for private), p-private enforces that p_-prefixed names are accessed only inside their owning file/class (cross-file/class use means the name should be public). Backend Python only. The exception lists grandfather pre-existing debt that landed with the workflows/analytics forward-ports (eric's 'don't mass-migrate untouched files' rule); the agent_manager refactor surface is clean. NOTE: Haik's full linter (his branch also adds pyright + ruff and runs a different enabled set) should eventually supersede this; these two were lifted to enforce the p_ conventions on eric/dev now. browser_cookies.py is excepted for `_fields_` only: a ctypes.Structure protocol name required by the ctypes metaclass, not our naming."
|
||||
"no-underscore-names + p-private": "Convention checks ported verbatim from Haik's linter (haik/feat/ingest): no-underscore-names bans leading-underscore names (a dead-code-tooling blind spot; use p_ for private), p-private enforces that p_-prefixed names are accessed only inside their owning file/class (cross-file/class use means the name should be public). Backend Python only. The exception lists grandfather pre-existing debt that landed with the workflows/analytics forward-ports (eric's 'don't mass-migrate untouched files' rule); the agent_manager refactor surface is clean. NOTE: Haik's full linter (his branch also adds pyright + ruff and runs a different enabled set) should eventually supersede this; these two were lifted to enforce the p_ conventions on eric/dev now. browser_cookies.py and its Windows round-trip test are excepted for `_fields_` only: a ctypes.Structure protocol name required by the ctypes metaclass, not our naming."
|
||||
},
|
||||
"rules": {
|
||||
"max-file-lines": 300,
|
||||
@@ -201,6 +201,7 @@
|
||||
"classes": [],
|
||||
"no-underscore-names": [
|
||||
"backend/apps/onboarding/usage/browser_cookies.py",
|
||||
"backend/tests/test_browser_cookies_windows_live.py",
|
||||
"backend/apps/agents/schedule_mcp_server.py",
|
||||
"backend/apps/workflows/audit.py",
|
||||
"backend/apps/workflows/escalation.py",
|
||||
|
||||
@@ -0,0 +1,197 @@
|
||||
"""Drift canary for the browser write path.
|
||||
|
||||
WHY THIS EXISTS: on 2026-07-21 X renamed its compose control and our sends silently went to 0/2
|
||||
WITH false success claims. No code of ours changed. A passing test suite is a snapshot; sites move
|
||||
underneath it, so the only way a coverage claim stays true is to re-prove it against the live site
|
||||
on a schedule and shout the day it breaks.
|
||||
|
||||
WHAT IT DOES: per site, one full round trip on the user's own account, then cleans up after itself:
|
||||
post a unique marker -> confirm the receipt -> delete it -> confirm it is gone
|
||||
A site "passes" only if the post was receipt-verified AND the cleanup verified gone. Anything else
|
||||
is drift, reported with the stage it died at.
|
||||
|
||||
SAFETY:
|
||||
- Never runs by itself. No cron, no import side effects; a human or a CI job invokes it.
|
||||
- --dry (default) posts NOTHING: it exercises discovery only, so it is safe anywhere.
|
||||
- --live is required to actually post, and every post is deleted in the same run.
|
||||
- Markers are random and carry no removal words ("delete"/"remove" in a payload trips the
|
||||
removal classifier and makes the send path stand down, which cost us a confusing hour once).
|
||||
- Exits 1 on drift so a scheduler can alert on it.
|
||||
|
||||
USAGE
|
||||
python scripts/browser_canary.py # discovery only, posts nothing
|
||||
python scripts/browser_canary.py --live # real round trip, self-cleaning
|
||||
python scripts/browser_canary.py --live --sites x,reddit
|
||||
(backend must be running; set OSW_CANARY_BASE if it is not on :8326)
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import secrets
|
||||
import sys
|
||||
import time
|
||||
import urllib.request
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
BASE = os.environ.get("OSW_CANARY_BASE", "http://127.0.0.1:8326") + "/api/agents"
|
||||
LOG = os.environ.get("OSW_CANARY_LOG", "/tmp/osw_backend_mr.log")
|
||||
MODEL = os.environ.get("OSW_CANARY_MODEL", "opus-4-8")
|
||||
|
||||
# Per site: how to post it, how to delete it. Kept deliberately small: these are the surfaces we
|
||||
# CLAIM to support, so the canary's job is to keep that claim honest, not to explore new ones.
|
||||
SITES: Dict[str, Dict[str, str]] = {
|
||||
"x": {
|
||||
"probe": 'Go to x.com. Do NOT type or post anything. Is the tweet compose box present on the page? Answer with exactly one word: YES or NO.',
|
||||
"post": 'Go to x.com and post this tweet, exactly: "{m}"',
|
||||
"delete": 'Go to x.com/{handle} and delete the post that says "{m}"',
|
||||
"handle_env": "OSW_CANARY_X_HANDLE",
|
||||
},
|
||||
"linkedin": {
|
||||
"probe": 'Go to linkedin.com. Do NOT type or post anything. Is the "Start a post" compose control present on the feed? Answer with exactly one word: YES or NO.',
|
||||
"post": 'Go to linkedin.com and create a post with exactly this text: "{m}"',
|
||||
"delete": 'Go to my LinkedIn activity and delete the post that says "{m}"',
|
||||
},
|
||||
"reddit": {
|
||||
"probe": 'Go to reddit.com/r/test/submit. Do NOT type or submit anything. Is the post title/body compose form present? Answer with exactly one word: YES or NO.',
|
||||
"post": 'Go to reddit.com and create a text post in r/test titled "{m}" with body "canary check". Submit it.',
|
||||
"delete": 'Go to reddit.com and delete my post titled "{m}"',
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def req(method: str, url: str, body: Optional[dict] = None) -> dict:
|
||||
tok = open(os.path.join(ROOT, "backend/data/auth.token")).read().strip()
|
||||
data = json.dumps(body).encode() if body is not None else None
|
||||
r = urllib.request.Request(url, data=data, method=method,
|
||||
headers={"Content-Type": "application/json",
|
||||
"Authorization": "Bearer " + tok})
|
||||
with urllib.request.urlopen(r, timeout=260) as resp:
|
||||
return json.loads(resp.read().decode() or "{}")
|
||||
|
||||
|
||||
def log_lines() -> int:
|
||||
try:
|
||||
return sum(1 for _ in open(LOG, errors="ignore"))
|
||||
except OSError:
|
||||
return 0
|
||||
|
||||
|
||||
def run_task(prompt: str, name: str, budget: int = 180) -> Dict[str, object]:
|
||||
"""Dispatch one browser task; return {said, status, wall, log} for the slice it produced."""
|
||||
mark = log_lines()
|
||||
dash = req("GET", BASE.replace("/agents", "") + "/dashboards/list")
|
||||
dashboards = dash if isinstance(dash, list) else dash.get("dashboards", [])
|
||||
if not dashboards:
|
||||
return {"said": "", "status": "error", "wall": 0.0, "log": "", "err": "no dashboard"}
|
||||
sid = req("POST", f"{BASE}/launch", {"mode": "agent", "model": MODEL, "provider": "anthropic",
|
||||
"dashboard_id": dashboards[0]["id"], "name": name})["session"]["id"]
|
||||
t0 = time.time()
|
||||
try:
|
||||
req("POST", f"{BASE}/sessions/{sid}/message", {"prompt": prompt, "mode": "agent", "model": MODEL})
|
||||
except Exception:
|
||||
pass
|
||||
status = ""
|
||||
said = ""
|
||||
while time.time() - t0 < budget:
|
||||
try:
|
||||
s = req("GET", f"{BASE}/sessions/{sid}")
|
||||
status = str(s.get("status") or "")
|
||||
if status in ("completed", "error", "stopped"):
|
||||
msgs = [m for m in s.get("messages", []) if m.get("role") == "assistant"]
|
||||
if msgs:
|
||||
c = msgs[-1].get("content")
|
||||
said = c if isinstance(c, str) else str(c)
|
||||
break
|
||||
except Exception:
|
||||
pass
|
||||
time.sleep(1.5)
|
||||
time.sleep(1.0)
|
||||
try:
|
||||
slice_ = "".join(open(LOG, errors="ignore").readlines()[mark:])
|
||||
except OSError:
|
||||
slice_ = ""
|
||||
return {"said": said, "status": status, "wall": round(time.time() - t0, 1), "log": slice_}
|
||||
|
||||
|
||||
def check_site(site: str, cfg: Dict[str, str], live: bool) -> Dict[str, object]:
|
||||
marker = "canary" + secrets.token_hex(4) # no removal words, unique per run
|
||||
res: Dict[str, object] = {"site": site, "marker": marker, "live": live}
|
||||
|
||||
if not live:
|
||||
# Discovery-only, safe BY CONSTRUCTION rather than by configuration: we ask the agent to
|
||||
# REPORT whether the compose surface is reachable, never to write. An earlier draft set
|
||||
# OSW_SENDSCRIPT_DRYRUN in this process, which does nothing to the backend, so a "dry" run
|
||||
# against a normal backend would have posted for real. Never trust a flag you don't own.
|
||||
r = run_task(cfg["probe"].format(handle=os.environ.get(cfg.get("handle_env", ""), "")),
|
||||
f"canary-probe-{site}")
|
||||
said = str(r["said"])
|
||||
found = bool(re.search(r"\bYES\b", said)) and not re.search(r"\bNO\b", said)
|
||||
if "sent_receipt=True" in str(r["log"]): # must never happen on this path
|
||||
res.update(stage="SAFETY", ok=False,
|
||||
detail="ABORT: a discovery probe performed a real send; check the task text")
|
||||
return res
|
||||
res.update(stage="discovery", ok=found, wall=r["wall"],
|
||||
detail=("compose surface reachable" if found else f"NOT reachable: {said[:110]}"))
|
||||
return res
|
||||
|
||||
# 1. POST, and require the two-sided receipt (composer cleared), not the model's word for it.
|
||||
handle = os.environ.get(cfg.get("handle_env", ""), "") if cfg.get("handle_env") else ""
|
||||
r = run_task(cfg["post"].format(m=marker, handle=handle), f"canary-post-{site}")
|
||||
log = str(r["log"])
|
||||
delivered = "done sent_receipt=True" in log or "DELIVERY CONFIRMED" in log
|
||||
res["post_wall"] = r["wall"]
|
||||
if not delivered:
|
||||
res.update(stage="post", ok=False,
|
||||
detail=f"no receipt (status={r['status']}): {str(r['said'])[:120]}")
|
||||
return res
|
||||
|
||||
# 2. DELETE, and require the in-page verify-gone, so cleanup can't be claimed falsely.
|
||||
d = run_task(cfg["delete"].format(m=marker, handle=handle), f"canary-clean-{site}")
|
||||
dlog = str(d["log"])
|
||||
removed = "removed=True" in dlog
|
||||
res["delete_wall"] = d["wall"]
|
||||
if not removed:
|
||||
res.update(stage="cleanup", ok=False,
|
||||
detail=f"POSTED BUT NOT CLEANED, marker {marker} may be live: {str(d['said'])[:100]}")
|
||||
return res
|
||||
res.update(stage="done", ok=True, detail="posted, receipt-verified, deleted, verified gone")
|
||||
return res
|
||||
|
||||
|
||||
def main() -> int:
|
||||
ap = argparse.ArgumentParser(description=__doc__)
|
||||
ap.add_argument("--live", action="store_true",
|
||||
help="actually post (and delete) on the real accounts; default is discovery-only")
|
||||
ap.add_argument("--sites", default="", help="comma list (default: all)")
|
||||
args = ap.parse_args()
|
||||
|
||||
want: List[str] = [s.strip() for s in args.sites.split(",") if s.strip()] or list(SITES)
|
||||
unknown = [s for s in want if s not in SITES]
|
||||
if unknown:
|
||||
print(f"unknown site(s): {', '.join(unknown)}; known: {', '.join(SITES)}")
|
||||
return 2
|
||||
|
||||
print(f"browser drift canary mode={'LIVE (posts+deletes)' if args.live else 'discovery-only'} "
|
||||
f"sites={','.join(want)} model={MODEL}")
|
||||
rows = []
|
||||
for s in want:
|
||||
r = check_site(s, SITES[s], args.live)
|
||||
rows.append(r)
|
||||
flag = "PASS" if r.get("ok") else "DRIFT"
|
||||
print(f" [{flag}] {s:10} stage={r.get('stage','?'):10} {r.get('detail','')}", flush=True)
|
||||
|
||||
bad = [r for r in rows if not r.get("ok")]
|
||||
stranded = [r for r in bad if r.get("stage") == "cleanup"]
|
||||
print(f"\n{len(rows) - len(bad)}/{len(rows)} sites healthy")
|
||||
if stranded:
|
||||
print("!! MANUAL CLEANUP NEEDED: " + ", ".join(f"{r['site']}:{r['marker']}" for r in stranded))
|
||||
if bad:
|
||||
print("DRIFT DETECTED on: " + ", ".join(str(r["site"]) for r in bad))
|
||||
return 1 if bad else 0
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
sys.exit(main())
|
||||
Reference in New Issue
Block a user