From 113e8d220c09153f0902dd0e758ac42510db4b07 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Tue, 21 Jul 2026 23:38:21 -0700 Subject: [PATCH] [eric] browser: delete+submit act via trusted input; reddit delete fast path proven live --- backend/apps/agents/browser/browser_agent.py | 12 +- .../agents/browser/browser_delete_script.py | 224 +++++++++++++----- .../agents/browser/browser_send_script.py | 12 +- .../agents/browser/browser_submit_click.py | 37 ++- backend/tests/test_browser_delete_script.py | 116 +++++++-- backend/tests/test_browser_submit_click.py | 8 +- frontend/src/shared/browserCommandHandler.ts | 13 +- 7 files changed, 322 insertions(+), 100 deletions(-) diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index e9b92440..f79316e1 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -264,10 +264,7 @@ async def execute_browser_tool( p_target = str((tool_input or {}).get("target_text") or "") if len(p_target) < browser_delete_script.MIN_TARGET_CHARS: return {"error": "target_text too short; give a longer distinctive snippet of the item's own text"} - p_res = await ws_manager.send_browser_command( - uuid4().hex, "evaluate", browser_id, - {"expression": browser_delete_script.delete_item_expression(p_target)}, tab_id=tab_id) - p_parsed = browser_delete_script.parse_delete_result(p_res) + p_parsed = await browser_delete_script.run_delete(p_target, browser_id, tab_id, execute_browser_tool) logger.info(f"[browser-deleteitem] target={p_target[:40]!r} removed={p_parsed['removed']} stage={p_parsed['stage']}") if p_parsed["removed"]: return {"text": f'Removed the item containing "{p_target[:60]}" (verified gone).', "removed": True} @@ -326,6 +323,8 @@ async def execute_browser_tool( result = await ws_manager.send_browser_command( request_id, action, browser_id, params, tab_id=tab_id, ) + if os.environ.get("OSW_DEBUG_LIST") == "1" and action == "list_interactives" and isinstance(result, dict): + logger.info(f"[debug-list] {str(result.get('text') or '')[:2400]}") # Click telemetry lives at the top level for a solo click and inside `results[]` for a batched one; scan both. p_click_parts = [result] if isinstance(result, dict) else [] if isinstance(result, dict) and isinstance(result.get("results"), list): @@ -1510,10 +1509,7 @@ async def run_browser_agent( p_del_target = browser_send_script.quoted_payload(user_prompt or task) if p_del_target and len(p_del_target) >= browser_delete_script.MIN_TARGET_CHARS: try: - p_del_res = await ws_manager.send_browser_command( - uuid4().hex, "evaluate", browser_id, - {"expression": browser_delete_script.delete_item_expression(p_del_target)}, tab_id=tab_id) - p_del = browser_delete_script.parse_delete_result(p_del_res) + p_del = await browser_delete_script.run_delete(p_del_target, browser_id, tab_id, execute_browser_tool) except Exception as p_de: logger.info(f"[browser-deletedispatch] outer skip ({p_de})") p_del = {"removed": False, "stage": "eval", "msg": str(p_de)} diff --git a/backend/apps/agents/browser/browser_delete_script.py b/backend/apps/agents/browser/browser_delete_script.py index 82c3cec7..26711a71 100644 --- a/backend/apps/agents/browser/browser_delete_script.py +++ b/backend/apps/agents/browser/browser_delete_script.py @@ -3,35 +3,47 @@ text, deterministically. The model handles getting to the item's page (its stren the site's own remove flow (open that item's overflow menu -> Delete -> confirm -> verify-gone), which the model fails at by hand (measured live on X: 4 aborts on the tiny caret menu). -It is a tool, not a pre-navigation tier: the earlier tier fired before the model reached the -item and always declined. As a tool the model calls it AFTER navigating, so the item and its -(late-rendering) caret are present. Translated to a single BrowserEvaluate in execute_browser_tool -(the App-bridge pattern), so no frontend handler is needed. +Resolve-in-JS, click-with-real-input: each step RESOLVES the next control's viewport position in +a pierced-shadow DOM query, and the click is dispatched through the OS-level input path +(BrowserClickPoint). Synthetic el.click() is ignored by web-component sites (shreddit live: +the flow reached Delete yet nothing happened; the model's trusted clicks on the same controls +worked), and a real click also lands on whatever is topmost, so overlays can't be mis-clicked. Safety, in code: -- Acts ONLY inside the element that contains the target text, so it can never touch another item. +- Resolution happens ONLY inside the element that contains the target text, so it can never + touch another item. - The site enforces ownership (only your own item exposes Delete), so a target you don't own has no menu entry and the tool reports that, it never forces one. - Success REQUIRES verify-gone (the target text left the page). One destructive confirm click. - Flag-gated (OSW_DELETE_SCRIPT): the tool is hidden from the model until Eric flips it. """ +import asyncio import json +import logging import os -from typing import Any, Dict +from typing import Any, Awaitable, Callable, Dict + +from backend.apps.agents.browser import browser_submit_click + +logger = logging.getLogger(__name__) MIN_TARGET_CHARS = 6 +ToolRunner = Callable[[str, dict, str, str], Awaitable[dict]] + def delete_tool_enabled() -> bool: return os.environ.get("OSW_DELETE_SCRIPT", "0") != "0" -# The in-page removal flow, scoped to the item holding the target text. Returns a JSON-able -# {stage, ok, msg} that parse_delete_result reads. The caret renders a beat late, so it polls. -P_DELETE_JS = r"""(async () => { - const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); +# One resolver, four steps. Each call re-queries the live DOM (pierced), scrolls the control into +# view when needed, and returns the control's viewport center as percents for BrowserClickPoint. +# 'verify' returns gone-ness instead of a position. Controls render a beat late, so steps poll. +P_RESOLVE_JS = r"""(async () => { + const STEP = %s; const TARGET = %s; + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); const norm = (s) => (s || '').replace(/\s+/g, ' ').trim(); // Web-component sites (shreddit, YouTube) hide the item AND its menus in shadow roots, so every lookup pierces. const deep = (root, sel, out, depth) => { @@ -42,59 +54,163 @@ P_DELETE_JS = r"""(async () => { for (const el of all) { if (el.shadowRoot) deep(el.shadowRoot, sel, out, depth + 1); } return out; }; - const vis = (el) => el && el.offsetParent !== null && el.getClientRects().length > 0; + const vis = (el) => !!el && el.getClientRects().length > 0; + const center = (el) => { + const r0 = el.getBoundingClientRect(); + if (r0.top < 0 || r0.bottom > window.innerHeight) el.scrollIntoView({ block: 'center' }); + const r = el.getBoundingClientRect(); + return { xPct: (r.left + r.width / 2) / window.innerWidth * 100, + yPct: (r.top + r.height / 2) / window.innerHeight * 100, + label: norm(el.getAttribute('aria-label') || el.textContent || '').slice(0, 40) }; + }; const CONTAINERS = 'article,[role="article"],li,[role="listitem"],shreddit-post,' + '[data-testid*="tweet"],[data-testid*="post"],[data-testid*="comment"],[data-testid*="Post"],[id^="t3_"],[id^="t1_"]'; const MORE = 'button[aria-label*="More" i],button[aria-label*="option" i],' - + '[data-testid="caret"],button[aria-haspopup="menu"],[aria-label*="menu" i]'; + + '[data-testid="caret"],button[aria-haspopup="menu"],button[aria-haspopup="true"],[aria-label*="menu" i]'; const holders = () => deep(document, CONTAINERS, [], 0) .filter((el) => el.textContent && el.textContent.includes(TARGET)) .sort((a, b) => (a.textContent || '').length - (b.textContent || '').length); // tightest match first - if (!holders().length) return { stage: 'find', ok: false, msg: 'target text not on this page' }; - // The caret/overflow renders a beat late and may sit in a shadow root, so poll + pierce. Take the - // first holding container that owns an overflow control, so the control belongs to THIS item. - let item = null, more = null; - for (let attempt = 0; attempt < 6 && !more; attempt++) { - if (attempt) await sleep(500); - for (const h of holders()) { const m = deep(h, MORE, [], 0).find(vis); if (m) { item = h; more = m; break; } } + + if (STEP === 'verify') { + // Gone = the text left the page, OR every remaining holder is a deletion TOMBSTONE (reddit + // swaps the tile for a "Post deleted" placeholder that keeps the title text). The substance + // gate stops a still-loading page (post-refresh) from reading as gone: never a false removed. + const TOMB = /post deleted|comment deleted|\[deleted\]|deleted by|removed by/i; + for (let attempt = 0; attempt < 7; attempt++) { + if (attempt) await sleep(1000); + if ((document.body.innerText || '').length < 500) continue; + const hs = holders(); + if (!hs.length || hs.every((h) => TOMB.test(h.textContent || ''))) return { ok: true, stage: 'verify' }; + } + return { ok: false, stage: 'verify' }; } - if (!more) return { stage: 'more', ok: false, msg: 'found the item but no overflow/More control on it' }; - more.click(); - await sleep(1000); - const DEL = /^\s*(delete|remove)\b/i; // exact-ish: not "delete row" or "remove filter" - const del = deep(document, '[role="menuitem"],[role="option"],button,a,li', [], 0) - .find((m) => DEL.test(norm(m.innerText)) && vis(m)); - if (!del) return { stage: 'menuitem', ok: false, - msg: 'no Delete/Remove entry in the menu (is this your own item?)', - menu: deep(document, '[role="menuitem"],[role="option"]', [], 0).map((m) => norm(m.innerText)).filter(Boolean).slice(0, 10) }; - del.click(); - await sleep(1000); - // Confirm dialog (pierce shadow DOM; Reddit's confirm is a faceplate/shreddit web component). - const dlg = deep(document, '[role="dialog"],[role="alertdialog"],[data-testid="confirmationSheetDialog"],faceplate-dialog,shreddit-async-loader', [], 0)[0] || document; - const CONF = /^\s*(delete|remove|yes|confirm)\b/i; - const conf = deep(dlg, '[data-testid="confirmationSheetConfirm"]', [], 0)[0] - || deep(dlg, 'button', [], 0).find((b) => CONF.test(norm(b.innerText)) && vis(b)); - if (conf) { conf.click(); await sleep(1800); } - const still = holders().length > 0; - return { stage: 'done', ok: !still, msg: still ? 'clicked delete but the item is still on the page' : 'item removed' }; + if (STEP === 'more') { + if (!holders().length) return { ok: false, stage: 'find', msg: 'target text not on this page' }; + // A post tile carries OTHER kebabs too (reddit's user-attribution row: 'Open user actions', + // measured live opening the wrong menu). Rank: named overflow first, then reddit's unlabeled + // haspopup kebab, then generic; user/share/moderation controls never. + const rank = (el) => { + // Reddit quirks, all measured live: the post kebab is LABELED 'Open user actions' (inside + // shreddit-post-overflow-menu, so the host outranks the misleading label), a 0x0 DECOY + // lives in mod-content-state-indicators, and Share/mod controls also carry haspopup. + const host = (el.getRootNode() && el.getRootNode().host) ? el.getRootNode().host.tagName : ''; + if (/^MOD-/.test(host)) return 4; + const l = norm(el.getAttribute('aria-label') || el.textContent || '').toLowerCase(); + if (/overflow|more/.test(l) || /OVERFLOW/.test(host)) return 0; + if (!l && el.getAttribute('aria-haspopup')) return 1; + if (/share|award|vote|join|follow|moderat|approve/.test(l)) return 4; + if (/user|profile|author/.test(l)) return 3; + return 2; + }; + for (let attempt = 0; attempt < 6; attempt++) { + if (attempt) await sleep(500); + for (const h of holders()) { + const cands = deep(h, MORE, [], 0).filter((c) => rank(c) < 4); + const visBest = cands.filter(vis).sort((a, b) => rank(a) - rank(b))[0]; + if (visBest) { h.scrollIntoView({ block: 'center' }); return { ok: true, stage: 'more', ...center(visBest) }; } + // Hover-revealed kebab: exists but zero rects until the tile is hovered with REAL input. + // Only when NOTHING visible qualifies; hand the tile position back for a hover + retry. + const hid = cands.find((c) => !vis(c) && rank(c) <= 1); + if (hid) { + h.scrollIntoView({ block: 'center' }); + const hr = h.getBoundingClientRect(); + return { ok: false, stage: 'more', hoverFirst: true, + xPct: (hr.left + hr.width / 2) / window.innerWidth * 100, + yPct: (hr.top + Math.min(40, hr.height / 2)) / window.innerHeight * 100, + msg: 'overflow control hidden until hover' }; + } + } + } + return { ok: false, stage: 'more', msg: 'found the item but no overflow/More control on it' }; + } + if (STEP === 'menuitem') { + const DEL = /^\s*(delete|remove)\b/i; // exact-ish: not "delete row" or "remove filter" + const MENUS = '[role="menu"],[role="listbox"],faceplate-menu,faceplate-dropdown-menu,[data-testid="Dropdown"]'; + // Long poll: shreddit's menu content arrives through an async loader, measured up to ~20s + // after the kebab click on a cold page; X resolves on the first attempt so the tail is free. + for (let attempt = 0; attempt < 12; attempt++) { + if (attempt) await sleep(900); + // Only entries inside an OPEN menu count; a page-wide 'Delete' from another context must + // never be clicked (measured live after the wrong kebab opened). + const menus = deep(document, MENUS, [], 0).filter(vis); + for (const s of (menus.length ? menus : [document])) { + const del = deep(s, '[role="menuitem"],[role="option"],button,a,li', [], 0) + .find((m) => DEL.test(norm(m.textContent)) && vis(m)); + if (del) return { ok: true, stage: 'menuitem', ...center(del) }; + } + } + return { ok: false, stage: 'menuitem', + msg: 'no Delete/Remove entry in the menu (is this your own item?)', + menu: deep(document, '[role="menuitem"],[role="option"]', [], 0) + .map((m) => norm(m.textContent)).filter(Boolean).slice(0, 10) }; + } + if (STEP === 'confirm') { + const CONF = /^\s*(delete|remove|yes|confirm)\b/i; + // Scan EVERY visible dialog candidate then the whole document; taking [0] once grabbed a + // random lazy-loader wrapper and looked straight past the real open dialog (measured live). + const DLG = '[role="dialog"],[role="alertdialog"],[data-testid="confirmationSheetDialog"],faceplate-dialog'; + for (let attempt = 0; attempt < 7; attempt++) { + if (attempt) await sleep(700); + const scopes = [...deep(document, DLG, [], 0).filter(vis), document]; + for (const dlg of scopes) { + const conf = deep(dlg, '[data-testid="confirmationSheetConfirm"]', [], 0).find(vis) + || deep(dlg, 'button', [], 0).find((b) => CONF.test(norm(b.textContent)) && vis(b)); + if (conf) return { ok: true, stage: 'confirm', ...center(conf) }; + } + } + return { ok: false, stage: 'confirm', optional: true, msg: 'no confirm dialog appeared' }; + } + return { ok: false, stage: 'eval', msg: 'unknown step' }; })()""" -def delete_item_expression(target_text: str) -> str: - return P_DELETE_JS % json.dumps(target_text) +def resolve_expression(step: str, target_text: str) -> str: + return P_RESOLVE_JS % (json.dumps(step), json.dumps(target_text)) -def parse_delete_result(res: object) -> Dict[str, Any]: - """Turn the BrowserEvaluate result into {removed: bool, stage, msg}. A shape we can't read - is an honest failure (never a false 'removed').""" - val: object = None - if isinstance(res, dict) and "error" not in res: - val = res.get("value") - if val is None and isinstance(res.get("text"), str): - try: - val = json.loads(res["text"]) - except (json.JSONDecodeError, ValueError): - val = None - if not isinstance(val, dict): - return {"removed": False, "stage": "eval", "msg": "the remove flow returned no readable result"} - return {"removed": bool(val.get("ok")), "stage": str(val.get("stage") or ""), "msg": str(val.get("msg") or "")} +async def run_delete(target_text: str, browser_id: str, tab_id: str, + execute_tool: ToolRunner) -> Dict[str, Any]: + """The full remove flow: resolve each control in-page, click it with REAL input, verify gone. + Any unreadable resolve is an honest failure at that stage (never a false 'removed').""" + + async def resolve(step: str) -> Dict[str, Any]: + res = await execute_tool( + "BrowserEvaluate", {"expression": resolve_expression(step, target_text)}, browser_id, tab_id) + return browser_submit_click.parse_eval_value(res) or {"ok": False, "stage": "eval", + "msg": "the remove flow returned no readable result"} + + for step, settle_s in (("more", 1.0), ("menuitem", 1.0), ("confirm", 1.8)): + r = await resolve(step) + if not r.get("ok") and step == "more" and r.get("hoverFirst"): + # Reveal a hover-only kebab with a real mouse move over the tile, then re-resolve once. + logger.info("[browser-deletescript] kebab hidden; hovering the tile to reveal it") + await execute_tool("BrowserClickPoint", + {"xPercent": float(r["xPct"]), "yPercent": float(r["yPct"]), + "hoverOnly": True}, browser_id, tab_id) + await asyncio.sleep(0.7) + r = await resolve(step) + if not r.get("ok"): + if step == "confirm" and r.get("optional"): + break # some sites delete without a confirm; the verify below is the arbiter + return {"removed": False, "stage": str(r.get("stage") or step), "msg": str(r.get("msg") or "")} + logger.info(f"[browser-deletescript] step={step} label={str(r.get('label') or '')[:40]!r} " + f"at {float(r['xPct']):.1f},{float(r['yPct']):.1f}") + await execute_tool("BrowserClickPoint", + {"xPercent": float(r["xPct"]), "yPercent": float(r["yPct"])}, browser_id, tab_id) + await asyncio.sleep(settle_s) + v = await resolve("verify") + if not v.get("ok"): + # Some clients keep the dead tile mounted until a reload (shreddit, measured live: the + # delete lands server-side while the DOM never flips). Refresh once and re-verify; the + # verify's substance gate keeps a half-loaded page from reading as gone. + p_loc = browser_submit_click.parse_eval_value( + await execute_tool("BrowserEvaluate", {"expression": "({href: location.href})"}, browser_id, tab_id)) or {} + p_href = str(p_loc.get("href") or "") + if p_href.startswith("http"): + logger.info("[browser-deletescript] tile still mounted; refreshing to re-verify") + await execute_tool("BrowserNavigate", {"url": p_href}, browser_id, tab_id) + await asyncio.sleep(3.0) + v = await resolve("verify") + removed = bool(v.get("ok")) + return {"removed": removed, "stage": "done", + "msg": "item removed" if removed else "clicked delete but the item is still on the page"} diff --git a/backend/apps/agents/browser/browser_send_script.py b/backend/apps/agents/browser/browser_send_script.py index 68663efd..65a1e3aa 100644 --- a/backend/apps/agents/browser/browser_send_script.py +++ b/backend/apps/agents/browser/browser_send_script.py @@ -107,12 +107,16 @@ async def complete_send( send_name = send_btn[1] else: # No submit listed below the composer (the capped listing can starve a modal of its own - # button): click the submit inside the composer's OWN container, then last-resort by-name. - r_send = await execute_tool( + # button): resolve the submit inside the composer's OWN container and click it with REAL + # input (synthetic clicks are ignored by web-component sites), then last-resort by-name. + r_ev = await execute_tool( "BrowserEvaluate", {"expression": browser_submit_click.container_submit_expression(payload)}, browser_id, tab_id) - p_v = browser_submit_click.parse_eval_value(r_send) - if isinstance(p_v, dict) and p_v.get("ok"): + p_v = browser_submit_click.parse_eval_value(r_ev) + if isinstance(p_v, dict) and p_v.get("ok") and p_v.get("xPct") is not None: + r_send = await execute_tool( + "BrowserClickPoint", + {"xPercent": float(p_v["xPct"]), "yPercent": float(p_v["yPct"])}, browser_id, tab_id) send_name = str(p_v.get("name") or "submit") via = "container" else: diff --git a/backend/apps/agents/browser/browser_submit_click.py b/backend/apps/agents/browser/browser_submit_click.py index dae0ee9d..c23a2153 100644 --- a/backend/apps/agents/browser/browser_submit_click.py +++ b/backend/apps/agents/browser/browser_submit_click.py @@ -17,20 +17,35 @@ SEND_LABELS = frozenset({ "publish", "comment", "share", # articles / YouTube+FB comments / shares }) +# Resolves the submit and returns its viewport center; the caller clicks it through the REAL +# input path (BrowserClickPoint). Synthetic el.click() is ignored by web-component sites +# (shreddit live), and a real click lands on whatever is topmost, so overlays can't be fooled. P_CONTAINER_SUBMIT_JS = r"""(() => { const PAYLOAD = %s; const LABELS = new Set(%s); const norm = (s) => (s || '').replace(/\s+/g, ' ').trim().toLowerCase(); - const vis = (el) => !!el && el.getClientRects().length > 0 && el.offsetParent !== null; + const vis = (el) => !!el && el.getClientRects().length > 0; const enabled = (el) => !el.disabled && el.getAttribute('aria-disabled') !== 'true'; - const labelOf = (el) => norm(el.getAttribute('aria-label') || el.innerText || ''); + const labelOf = (el) => norm(el.getAttribute('aria-label') || el.textContent || ''); const holds = (el) => ((el.value || el.textContent || '').indexOf(PAYLOAD) !== -1); - const ed = [...document.querySelectorAll('[contenteditable="true"],textarea,input')] + // Shadow piercing both ways: reddit's composer AND its submit live in shreddit shadow roots. + const deep = (root, sel, out, depth) => { + if (depth > 10 || out.length > 4000) return out; + let hits; try { hits = root.querySelectorAll(sel); } catch (e) { hits = []; } + for (const el of hits) out.push(el); + let all; try { all = root.querySelectorAll('*'); } catch (e) { return out; } + for (const el of all) { if (el.shadowRoot) deep(el.shadowRoot, sel, out, depth + 1); } + return out; + }; + const up = (el) => el.parentElement || (el.getRootNode() && el.getRootNode().host) || null; + const ed = deep(document, '[contenteditable="true"],textarea,input', [], 0) .find((e) => vis(e) && holds(e)); if (!ed) return { ok: false, why: 'no editable holding the payload' }; - const submitIn = (root) => [...root.querySelectorAll('button,[role="button"]')] + const submitIn = (root) => deep(root, 'button,[role="button"]', [], 0) .find((b) => vis(b) && enabled(b) && LABELS.has(labelOf(b))); - const scope = ed.closest('[role="dialog"],[role="alertdialog"],form'); + const isScope = (el) => { try { return el.matches('[role="dialog"],[role="alertdialog"],form'); } catch (e) { return false; } }; + let scope = null; + for (let node = ed; node; node = up(node)) { if (isScope(node)) { scope = node; break; } } let btn = null; if (scope) { btn = submitIn(scope); @@ -38,15 +53,19 @@ P_CONTAINER_SUBMIT_JS = r"""(() => { // Nearest-scope-first walk: X's inline submit shares an ancestor 20 hops above the Draft.js // editable while foreign tweets' buttons only enter at 28 (measured live), so 24 finds the // composer's own submit and stops before any wider scope could. - let node = ed.parentElement; - for (let hop = 0; node && node !== document.body && hop < 24; hop++, node = node.parentElement) { + let node = up(ed); + for (let hop = 0; node && node !== document.body && hop < 24; hop++, node = up(node)) { btn = submitIn(node); if (btn) break; } } if (!btn) return { ok: false, why: 'no submit control in the composer container' }; - btn.click(); - return { ok: true, name: labelOf(btn) }; + const r0 = btn.getBoundingClientRect(); + if (r0.top < 0 || r0.bottom > window.innerHeight) btn.scrollIntoView({ block: 'center' }); + const r = btn.getBoundingClientRect(); + return { ok: true, name: labelOf(btn), + xPct: (r.left + r.width / 2) / window.innerWidth * 100, + yPct: (r.top + r.height / 2) / window.innerHeight * 100 }; })()""" diff --git a/backend/tests/test_browser_delete_script.py b/backend/tests/test_browser_delete_script.py index d23ee967..63ee182c 100644 --- a/backend/tests/test_browser_delete_script.py +++ b/backend/tests/test_browser_delete_script.py @@ -1,7 +1,9 @@ -"""BrowserDeleteItem tool logic (the JS is proven live on X; this pins the pure pieces: -the target is embedded safely, and the eval result maps to an honest removed/not-removed).""" +"""BrowserDeleteItem tool logic (the resolve JS is proven live; this pins the pure pieces and +the resolve->trusted-click->verify orchestration with a mocked tool runner).""" import json +import pytest + from backend.apps.agents.browser import browser_delete_script as d @@ -15,32 +17,110 @@ def test_flag_default_off(monkeypatch): def test_expression_embeds_target_json_safe(): # a target with quotes must not break out of the JS string literal tricky = 'he said "hi"; alert(1)' - expr = d.delete_item_expression(tricky) + expr = d.resolve_expression("more", tricky) assert json.dumps(tricky) in expr # embedded as a JSON literal assert 'const TARGET = ' + json.dumps(tricky) in expr + assert "const STEP = \"more\"" in expr -def test_parse_removed_true(): - r = d.parse_delete_result({"value": {"stage": "done", "ok": True, "msg": "item removed"}}) +def make_exec(step_results, click_ok=True, verify_after_refresh=None): + """Tool-runner mock: BrowserEvaluate returns the scripted resolve result for the step it was + called with (read out of the expression); BrowserClickPoint records real-input clicks; the + location probe and BrowserNavigate model the refresh-reverify pass.""" + calls = {"clicks": [], "steps": [], "navs": []} + + async def execute(tool, params, bid, tid): + if tool == "BrowserEvaluate": + expr = params["expression"] + if "location.href" in expr: + return {"value": {"href": "https://site.test/profile"}} + step = expr.split('const STEP = "', 1)[1].split('"', 1)[0] + calls["steps"].append(step) + if step == "verify" and calls["navs"] and verify_after_refresh is not None: + return {"value": verify_after_refresh} + return {"value": step_results[step]} + if tool == "BrowserClickPoint": + calls["clicks"].append((params["xPercent"], params["yPercent"])) + return {"ok": True} if click_ok else {"error": "no webview"} + if tool == "BrowserNavigate": + calls["navs"].append(params["url"]) + return {"ok": True} + return {"ok": True} + return execute, calls + + +POS = {"xPct": 50.0, "yPct": 40.0} + + +@pytest.mark.asyncio +async def test_full_flow_clicks_each_stage_with_real_input(): + ex, calls = make_exec({ + "more": {"ok": True, "stage": "more", **POS}, + "menuitem": {"ok": True, "stage": "menuitem", **POS}, + "confirm": {"ok": True, "stage": "confirm", **POS}, + "verify": {"ok": True, "stage": "verify"}, + }) + r = await d.run_delete("coffee notes abc123", "b1", "", ex) assert r == {"removed": True, "stage": "done", "msg": "item removed"} + assert len(calls["clicks"]) == 3 # more, menuitem, confirm: all real-input clicks + assert calls["steps"][-1] == "verify" -def test_parse_removed_false_still_present(): - r = d.parse_delete_result({"value": {"stage": "done", "ok": False, "msg": "still on the page"}}) +@pytest.mark.asyncio +async def test_verify_still_present_is_honest(): + # still-present survives the refresh re-verify too: honest not-removed, one refresh attempted + ex, calls = make_exec({ + "more": {"ok": True, "stage": "more", **POS}, + "menuitem": {"ok": True, "stage": "menuitem", **POS}, + "confirm": {"ok": True, "stage": "confirm", **POS}, + "verify": {"ok": False, "stage": "verify"}, + }) + r = await d.run_delete("coffee notes abc123", "b1", "", ex) assert r["removed"] is False and r["stage"] == "done" + assert calls["navs"] == ["https://site.test/profile"] -def test_parse_not_on_page(): - r = d.parse_delete_result({"value": {"stage": "find", "ok": False, "msg": "target text not on this page"}}) - assert r["removed"] is False and r["stage"] == "find" - - -def test_parse_text_wrapped_json(): - r = d.parse_delete_result({"text": json.dumps({"stage": "done", "ok": True, "msg": "item removed"})}) +@pytest.mark.asyncio +async def test_stale_tile_flips_removed_after_refresh(): + """Shreddit keeps the dead tile mounted until a reload; the refresh re-verify turns a real + server-side delete into removed=True instead of an honest-but-wrong still-present.""" + ex, calls = make_exec({ + "more": {"ok": True, "stage": "more", **POS}, + "menuitem": {"ok": True, "stage": "menuitem", **POS}, + "confirm": {"ok": True, "stage": "confirm", **POS}, + "verify": {"ok": False, "stage": "verify"}, + }, verify_after_refresh={"ok": True, "stage": "verify"}) + r = await d.run_delete("coffee notes abc123", "b1", "", ex) assert r["removed"] is True + assert len(calls["navs"]) == 1 -def test_parse_unreadable_is_honest_failure(): - assert d.parse_delete_result({"error": "eval blew up"})["removed"] is False - assert d.parse_delete_result({"value": "not a dict"})["removed"] is False - assert d.parse_delete_result("garbage")["removed"] is False +@pytest.mark.asyncio +async def test_missing_confirm_is_optional_verify_decides(): + ex, calls = make_exec({ + "more": {"ok": True, "stage": "more", **POS}, + "menuitem": {"ok": True, "stage": "menuitem", **POS}, + "confirm": {"ok": False, "stage": "confirm", "optional": True, "msg": "no confirm dialog appeared"}, + "verify": {"ok": True, "stage": "verify"}, + }) + r = await d.run_delete("coffee notes abc123", "b1", "", ex) + assert r["removed"] is True + assert len(calls["clicks"]) == 2 # confirm never clicked + + +@pytest.mark.asyncio +async def test_stage_failure_reports_stage_and_never_removed(): + ex, calls = make_exec({ + "more": {"ok": False, "stage": "find", "msg": "target text not on this page"}, + }) + r = await d.run_delete("coffee notes abc123", "b1", "", ex) + assert r == {"removed": False, "stage": "find", "msg": "target text not on this page"} + assert not calls["clicks"] + + +@pytest.mark.asyncio +async def test_unreadable_resolve_is_honest_failure(): + async def execute(tool, params, bid, tid): + return {"error": "eval blew up"} + r = await d.run_delete("coffee notes abc123", "b1", "", execute) + assert r["removed"] is False and r["stage"] == "eval" diff --git a/backend/tests/test_browser_submit_click.py b/backend/tests/test_browser_submit_click.py index 2cc6c07c..8ce511f0 100644 --- a/backend/tests/test_browser_submit_click.py +++ b/backend/tests/test_browser_submit_click.py @@ -29,14 +29,16 @@ def make_exec(eval_result): @pytest.mark.asyncio async def test_container_submit_tier_between_index_and_by_name(): """X's compose modal: covered feed rows behind the overlay eat the 60-row cap, so the modal's - own Post never reaches the picker (live 0/2 deliveries). The container-scoped JS tier clicks - the submit inside the composer's own container; by-name stays the last resort.""" - execute, calls = make_exec({"value": {"ok": True, "name": "post"}}) + own Post never reaches the picker (live 0/2 deliveries). The container tier resolves the + submit inside the composer's own container and clicks it with REAL input; by-name stays last.""" + execute, calls = make_exec({"value": {"ok": True, "name": "post", "xPct": 61.0, "yPct": 33.0}}) r = await ss.complete_send("[test] hello world r9-os", COMPOSER_FILLED_NO_SEND, "b1", "", execute, send_submit_index_in_state, composer_index=24) assert r["clicked"] is True and r["sent"] is True tools = [c[0] for c in calls["clicks"]] assert "BrowserEvaluate" in tools and "BrowserClickByName" not in tools + point = [c for c in calls["clicks"] if c[0] == "BrowserClickPoint"] + assert point and point[0][1] == {"xPercent": 61.0, "yPercent": 33.0} @pytest.mark.asyncio diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index 0dffc983..181b4ba1 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -716,16 +716,21 @@ async function handleClickPoint(wv: BrowserWebview, params: Record) } catch { /* use the element box as a fallback */ } const x = (cx / 100) * vw; const y = (cy / 100) * vh; + // hoverOnly: real mouse move, no press. Hover-revealed controls (reddit's post kebab only + // materializes on tile hover) need this; a click there would navigate into the post. + const hoverOnly = params.hoverOnly === true; try { await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseMoved', x, y }); - await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: 1 }); - if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs)); - await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: 1 }); + if (!hoverOnly) { + await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mousePressed', x, y, button, clickCount: 1 }); + if (holdMs > 0) await new Promise((r) => setTimeout(r, holdMs)); + await sendCdp(wv, 'Input.dispatchMouseEvent', { type: 'mouseReleased', x, y, button, clickCount: 1 }); + } } catch (err: any) { return { error: `Click point failed: ${err?.message || String(err)}` }; } return { - text: `Clicked at (${Math.round(x)}, ${Math.round(y)})${holdMs ? ` held ${holdMs}ms` : ''}.`, + text: `${hoverOnly ? 'Hovered' : 'Clicked'} at (${Math.round(x)}, ${Math.round(y)})${holdMs && !hoverOnly ? ` held ${holdMs}ms` : ''}.`, clickX: cx, clickY: cy, url: wv.getURL(), }; }