From 64675115f83cb256e2aa15e1b404f8346f28ade3 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 12 Jul 2026 19:29:07 -0700 Subject: [PATCH] [eric] browser: reveal-and-find reach primitive (OSW_COMPOSER_REVEAL, default off) = when no composer is painted the finder takes ONE reversible reveal action (click a scored compose-trigger, open the first list item, or scroll) and rescans, bounded; SPA route changes keep the JS context alive so X/LinkedIn/Reddit/YouTube stay drivable in one call; never clicks an irreversible control (hard-block send/submit/pay/delete); send-script forwards reveal only under the flag; 2 tests (reveal forwarded when on, false by default) --- .../agents/browser/browser_send_script.py | 9 +- backend/tests/test_browser_send_script.py | 31 ++++ frontend/src/shared/browserCommandHandler.ts | 138 ++++++++++++++---- 3 files changed, 147 insertions(+), 31 deletions(-) diff --git a/backend/apps/agents/browser/browser_send_script.py b/backend/apps/agents/browser/browser_send_script.py index a214c279..82403daf 100644 --- a/backend/apps/agents/browser/browser_send_script.py +++ b/backend/apps/agents/browser/browser_send_script.py @@ -180,11 +180,16 @@ async def run_send_script( # commit-check for a React contenteditable, whose text never reaches the AX value). # Flag-gated so the proven name path stays the default. if not composer and p_struct: - fc = await execute_tool("BrowserFindComposer", {"fill": payload}, browser_id, tab_id) + # OSW_COMPOSER_REVEAL: let the finder take one reversible reveal action (open the + # compose surface: a modal trigger, the first conversation, or a scroll) when the + # composer isn't painted yet. It never commits a send, only opens a surface. + p_reveal = os.environ.get("OSW_COMPOSER_REVEAL") == "1" + fc = await execute_tool("BrowserFindComposer", {"fill": payload, "reveal": p_reveal}, browser_id, tab_id) if isinstance(fc, dict) and fc.get("found") and fc.get("filled"): p_struct_selector = str(fc.get("selector") or "") logger.info(f"[browser-sendscript] structural composer role={fc.get('role')!r} " - f"score={fc.get('score')} nearSubmit={fc.get('nearSubmit')} filled+verified") + f"score={fc.get('score')} nearSubmit={fc.get('nearSubmit')} " + f"reveals={fc.get('reveals')} filled+verified") log.append({"tool": "BrowserFindComposer", "input": {"fill": ""}, "ok": True, "result_summary": f"structural composer {fc.get('role')!r} filled+verified"[:200], "elapsed_ms": 0}) composer = (-1, str(fc.get("role") or "composer")) diff --git a/backend/tests/test_browser_send_script.py b/backend/tests/test_browser_send_script.py index 6cc27e78..3a928d18 100644 --- a/backend/tests/test_browser_send_script.py +++ b/backend/tests/test_browser_send_script.py @@ -251,6 +251,37 @@ async def test_structural_off_by_default_never_calls_finder(monkeypatch): assert calls["find"] == 0 +@pytest.mark.asyncio +async def test_reveal_flag_passed_to_finder_when_enabled(monkeypatch): + """OSW_COMPOSER_REVEAL=1 lets the finder take a reversible reveal action: the send-script + must forward reveal=True to BrowserFindComposer (the composer isn't painted yet).""" + monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1") + monkeypatch.setenv("OSW_COMPOSER_REVEAL", "1") + monkeypatch.setenv("OSW_SENDSCRIPT_DRYRUN", "1") + ex, calls = make_struct_exec({"found": True, "filled": True, "role": "contenteditable", + "selector": '[data-osw-composer="1"]', "score": 6.0, + "nearSubmit": True, "reveals": ["trigger"]}) + r = await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state, + payload_in_textbox, payload_source=TASK, + current_url="https://www.linkedin.com/feed/") + assert r is not None and r["sent"] is False + find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer") + assert find_call[1].get("reveal") is True + + +@pytest.mark.asyncio +async def test_reveal_flag_off_by_default(monkeypatch): + """Struct on but reveal unset: the finder is still called, but reveal=False, so it only + scans what's painted and never clicks a trigger (the safe default).""" + monkeypatch.setenv("OSW_COMPOSER_STRUCT", "1") + monkeypatch.delenv("OSW_COMPOSER_REVEAL", raising=False) + ex, calls = make_struct_exec({"found": False}) + await ss.run_send_script(TASK, "b1", "", NAMELESS, ex, send_index_in_state, + payload_in_textbox, payload_source=TASK, current_url="https://example.com/") + find_call = next(c for c in calls["clicks"] if c[0] == "BrowserFindComposer") + assert find_call[1].get("reveal") is False + + def test_dryrun_report_encodes_the_gate_funnel(): """The coverage harness greps this one line for gate attribution: a staged composer reports composer=1, a bare page reports zeros, and armed/filled ride the booleans.""" diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index b2a12233..555a0954 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -345,12 +345,24 @@ async function handleType(wv: BrowserWebview, params: Record): Prom // with data-osw-composer so a follow-up type/click can target it by a stable selector. // With {fill}, it also types + reads back in the SAME in-page context (the only reliable // commit-check for a React-controlled contenteditable, whose value never reaches the AX tree). +// With {reveal:true}, when no composer is painted yet it takes ONE reversible reveal action +// (click a compose-trigger, open the first list item, or scroll) and rescans, up to a small +// bound. Reveal actions are same-document only: SPA route changes (pushState) keep this JS +// context alive, so X/LinkedIn/Reddit/YouTube stay drivable in one call; a full navigation +// just cuts the await short and the backend re-perceives. Reveal NEVER clicks an irreversible +// control (send/submit/pay/delete...): it only opens a surface, it never commits one. async function handleFindComposer(wv: BrowserWebview, params: Record): Promise> { const fill = params.fill != null ? String(params.fill) : null; const safeFill = JSON.stringify(fill); - const code = `(() => { + const reveal = params.reveal === true ? 'true' : 'false'; + const code = `(async () => { const SUBMIT = /\\b(send|post|reply|comment|tweet|publish|share|message)\\b/i; const SEARCH = /search|find\\b|filter|query|lookup|explore|jump to/i; + // Reversible compose openers: reveal a writing surface, never commit one. + const OPENER = /\\b(start a post|create( a)? post|new post|add a comment|write a comment|leave a comment|post a comment|write a review|create a review|new message|new chat|send a message|compose|reply|comment|tweet|write)\\b/i; + // Never clickable by reveal, even if the label also looks like an opener. + const HARDBLOCK = /\\b(send|submit|pay|buy|purchase|checkout|order|delete|remove|unfollow|unsubscribe|log ?out|sign ?out|report|block|confirm|deactivate|save)\\b/i; + const sleep = (ms) => new Promise((r) => setTimeout(r, ms)); const vis = (el) => { const r = el.getBoundingClientRect(); const s = getComputedStyle(el); return r.width >= 80 && r.height >= 16 && s.visibility !== 'hidden' @@ -358,38 +370,106 @@ async function handleFindComposer(wv: BrowserWebview, params: Record { - if (depth > 8 || out.length > 400) return out; + const deepAll = (root, out, depth) => { + if (depth > 8 || out.length > 4000) return out; let kids; try { kids = root.querySelectorAll('*'); } catch (e) { return out; } - for (const el of kids) { - if (el.matches && el.matches(SEL)) out.push(el); - if (el.shadowRoot) deep(el.shadowRoot, out, depth + 1); - } + for (const el of kids) { out.push(el); if (el.shadowRoot) deepAll(el.shadowRoot, out, depth + 1); } return out; }; - const cands = deep(document, [], 0); - let best = null, bestScore = 0, bestNear = false; - for (const el of cands) { - if (!vis(el) || el.readOnly || el.disabled) continue; - const label = ((el.getAttribute('aria-label')||'') + ' ' + (el.getAttribute('placeholder')||'') - + ' ' + (el.getAttribute('data-placeholder')||'') + ' ' + (el.getAttribute('name')||'')).trim(); - if (el.type === 'search' || SEARCH.test(label)) continue; - const rich = el.tagName === 'TEXTAREA' || el.isContentEditable || el.getAttribute('role') === 'textbox'; - const area = el.closest('form, [role="dialog"], [role="group"], section, main, [role="main"]') || document.body; - let near = false; - const btns = area.querySelectorAll('button, [role="button"], input[type="submit"]'); - for (const b of btns) { - const bt = ((b.textContent||'') + ' ' + (b.getAttribute('aria-label')||'')).trim(); - if (bt.length < 40 && SUBMIT.test(bt)) { near = true; break; } + const EDIT = 'textarea, [contenteditable="true"], [role="textbox"], input[type="text"]'; + const labelOf = (el) => ((el.getAttribute('aria-label')||'') + ' ' + (el.getAttribute('placeholder')||'') + + ' ' + (el.getAttribute('data-placeholder')||'') + ' ' + (el.getAttribute('name')||'')).trim(); + + const findBest = () => { + const all = deepAll(document, [], 0); + let best = null, bestScore = 0, bestNear = false; + for (const el of all) { + if (!el.matches || !el.matches(EDIT)) continue; + if (!vis(el) || el.readOnly || el.disabled) continue; + const label = labelOf(el); + if (el.type === 'search' || SEARCH.test(label)) continue; + const rich = el.tagName === 'TEXTAREA' || el.isContentEditable || el.getAttribute('role') === 'textbox'; + const area = el.closest('form, [role="dialog"], [role="group"], section, main, [role="main"]') || document.body; + let near = false; + for (const b of area.querySelectorAll('button, [role="button"], input[type="submit"]')) { + const bt = ((b.textContent||'') + ' ' + (b.getAttribute('aria-label')||'')).trim(); + if (bt.length < 40 && SUBMIT.test(bt)) { near = true; break; } + } + if (!rich && !near) continue; // a bare form input near nothing is not a composer + const r = el.getBoundingClientRect(); + const score = (el.isContentEditable ? 2 : 0) + (el.tagName === 'TEXTAREA' ? 2 : 0) + + (near ? 3 : 0) + Math.min((r.width * r.height) / 40000, 3) + (SUBMIT.test(label) ? 1 : 0); + if (score > bestScore) { bestScore = score; best = el; bestNear = near; } + } + return (best && bestScore >= 2) ? { el: best, score: bestScore, near: bestNear } : null; + }; + + const pollFind = async (ms) => { + const t0 = Date.now(); let h = findBest(); + while (!h && Date.now() - t0 < ms) { await sleep(140); h = findBest(); } + return h; + }; + + // The reveal actions, tried in yield order. Each returns true if it clicked/scrolled. + const clickTrigger = () => { + const all = deepAll(document, [], 0); + let best = null, bestScore = -1; + for (const el of all) { + const clickable = el.matches && el.matches('button, [role="button"], a[href], summary, [tabindex], [contenteditable="false"], div[class*="placeholder" i], span[class*="placeholder" i]'); + if (!clickable || !vis(el)) continue; + const txt = ((el.textContent||'') + ' ' + (el.getAttribute('aria-label')||'') + ' ' + (el.getAttribute('placeholder')||'')).trim(); + if (txt.length > 60 || !OPENER.test(txt) || HARDBLOCK.test(txt)) continue; + const r = el.getBoundingClientRect(); + // Prefer short-labelled, higher-on-page triggers (a real "Start a post" pill beats a + // long paragraph that happens to contain "reply"). + const score = (60 - txt.length) + Math.max(0, 600 - r.top) / 100; + if (score > bestScore) { bestScore = score; best = el; } + } + if (!best) return false; + best.scrollIntoView({ block: 'center', behavior: 'instant' }); + best.click(); + return true; + }; + const openFirstItem = () => { + const container = document.querySelector('[role="list"], [role="grid"], [role="feed"], main, [role="main"]') || document.body; + const items = container.querySelectorAll('[role="listitem"], [role="row"], [role="article"], article, li a[href], a[role="link"]'); + for (const it of items) { + if (!vis(it)) continue; + const r = it.getBoundingClientRect(); + if (r.top < 40 || r.top > window.innerHeight) continue; // skip sticky headers / offscreen + const target = it.matches('a[href], [role="link"]') ? it : (it.querySelector('a[href], [role="link"], [role="button"]') || it); + target.scrollIntoView({ block: 'center', behavior: 'instant' }); + target.click(); + return true; + } + return false; + }; + const scrollMain = () => { + const sc = document.scrollingElement || document.documentElement; + const before = sc.scrollTop; + sc.scrollBy(0, Math.round(window.innerHeight * 1.1)); + return sc.scrollTop !== before || true; + }; + + let hit = findBest(); + const acts = []; + if (!hit && ${reveal}) { + const reveals = [ + ['trigger', clickTrigger, 2000], + ['open-first', openFirstItem, 2000], + ['scroll', scrollMain, 1200], + ]; + for (const [name, act, waitMs] of reveals) { + let did = false; try { did = act(); } catch (e) { did = false; } + acts.push(name + (did ? '' : ':noop')); + if (!did) continue; + hit = await pollFind(waitMs); + if (hit) break; } - if (!rich && !near) continue; // a bare form input near nothing is not a composer - const r = el.getBoundingClientRect(); - const score = (el.isContentEditable ? 2 : 0) + (el.tagName === 'TEXTAREA' ? 2 : 0) - + (near ? 3 : 0) + Math.min((r.width * r.height) / 40000, 3) + (SUBMIT.test(label) ? 1 : 0); - if (score > bestScore) { bestScore = score; best = el; bestNear = near; } } - if (!best || bestScore < 2) return { found: false }; + if (!hit) return { found: false, reveals: acts }; + + const best = hit.el; best.setAttribute('data-osw-composer', '1'); let filled = false; if (${safeFill} != null) { @@ -404,7 +484,7 @@ async function handleFindComposer(wv: BrowserWebview, params: Record