diff --git a/frontend/src/shared/browserCommandHandler.ts b/frontend/src/shared/browserCommandHandler.ts index c2fe86a6..63dc8d4c 100644 --- a/frontend/src/shared/browserCommandHandler.ts +++ b/frontend/src/shared/browserCommandHandler.ts @@ -8,6 +8,7 @@ import { resolveInput } from './resolveUrl'; import { rankAndCapInteractives, type RankItem } from './interactiveRanking'; import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettle'; import { unwrapCdpEval } from './cdpEval'; +import { typeChars, type TypedKeys } from './typeChars'; let initialized = false; @@ -745,18 +746,17 @@ async function handleFindComposer(wv: BrowserWebview, params: Record { const safeSel = JSON.stringify(selector); await evalInPage(wv, `(() => { const el = document.querySelector(${safeSel}); if (!el) return false; el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus(); if (el.select) el.select(); document.execCommand('selectAll', false); document.execCommand('delete', false); return true; })()`); - for (const ch of text) { - wv.sendInputEvent({ type: 'char', keyCode: ch }); - } + const typed = await typeChars((m, p) => sendCdp(wv, m, p), text); + if (!typed.dispatched) return false; // Read back from the marked element OR the active element: editors like Reddit's swap the // node on activation, so the original selector can go stale even though the keystrokes landed // in whatever now holds focus. Checking both survives that re-render. @@ -1254,6 +1254,7 @@ async function clickBackendNode( // live on twitch, the composer was found and focused and the fill still errored, which cost // the site its entire write path. Clearing first, because a partial insert plus keystrokes is // how you post the same sentence twice. + let typed: TypedKeys = { dispatched: false, skipped: 'empty' }; try { const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId); await sendCdp(wv, 'Runtime.callFunctionOn', { @@ -1261,7 +1262,11 @@ async function clickBackendNode( functionDeclaration: 'function() { this.focus(); if (this.select) this.select(); document.execCommand("selectAll", false); document.execCommand("delete", false); }', }, sessionId); - for (const ch of opts.text) wv.sendInputEvent({ type: 'char', keyCode: ch }); + // Focus is set in the node's own frame above, but the key events themselves go to the ROOT + // target with no sessionId: Chromium routes keyboard input to whichever frame holds focus, + // and the Input domain is not reliably there on an OOPIF session. Sending them at the child + // would kill exactly the case this cares about, a composer inside an iframe. + typed = await typeChars((m, p) => sendCdp(wv, m, p), opts.text); } catch { /* verified below; the honest error covers this failing too */ } got = await readBack(); if (got !== null) return landedMsg(got, ' (via keystrokes)'); @@ -1275,6 +1280,11 @@ async function clickBackendNode( if (typeof live === 'string') return landedMsg(live, ' (via keystrokes)'); } catch { /* fall through to the honest error */ } flashField(wv, resolvedObjectId, sessionId, 'fail'); + // Saying "not even as real keystrokes" when we deliberately declined to send them is the + // kind of small lie that costs a debugging session. + if (typed.skipped === 'multiline') { + return { error: `Focused ${label} but this editor ignored both synthetic fills, and the text contains a line break, so real keystrokes were NOT attempted: pressing Enter in a composer can send it half-written. Fill it as a single line, or use a different element.` }; + } return { error: `Focused ${label} but the text did not register even as real keystrokes; the box may be a custom editor that rejects automation. Try a different element.` }; } return { text: `Focused ${label}; the cursor is in it now (type with BrowserPressKey, or pass a text arg to fill it in one call).` }; diff --git a/frontend/src/shared/typeChars.test.ts b/frontend/src/shared/typeChars.test.ts new file mode 100644 index 00000000..8bb9dba6 --- /dev/null +++ b/frontend/src/shared/typeChars.test.ts @@ -0,0 +1,100 @@ +// Run: node --test frontend/src/shared/typeChars.test.ts +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import { typeChars } from './typeChars.ts'; +import type { CdpDispatch } from './typeChars.ts'; + +interface Call { method: string; params: Record } + +function recorder(): { calls: Call[]; dispatch: CdpDispatch } { + const calls: Call[] = []; + const dispatch: CdpDispatch = async (method, params) => { + calls.push({ method, params: params as Record }); + return {}; + }; + return { calls, dispatch }; +} + +test('every character goes out as a CDP keyDown then keyUp, in order', async () => { + const { calls, dispatch } = recorder(); + const r = await typeChars(dispatch, 'hi'); + assert.deepEqual(r, { dispatched: true, skipped: '' }); + assert.equal(calls.length, 4); + assert.deepEqual(calls.map((c) => c.params.type), ['keyDown', 'keyUp', 'keyDown', 'keyUp']); + assert.deepEqual(calls.map((c) => c.params.key), ['h', 'h', 'i', 'i']); +}); + +test('the transport is CDP, never Electron sendInputEvent', async () => { + // The whole point of this module. sendInputEvent goes to whatever the OS thinks has focus, which + // is how the agent once typed into the user's notes app, and it is unawaitable, which is why the + // read-back after it always lost the race. If this ever regresses, the fill tier goes dead silent + // again: it fails with "did not register even as real keystrokes" and nobody sees a reason. + const { calls, dispatch } = recorder(); + await typeChars(dispatch, 'abc'); + assert.ok(calls.length > 0); + assert.ok(calls.every((c) => c.method === 'Input.dispatchKeyEvent'), 'every event must go over CDP'); +}); + +test('each dispatch is awaited, so a read-back after this cannot outrun the text', async () => { + // A fire-and-forget loop would let a slow first character land after a fast last one. Resolve in + // reverse-latency order and check the sequence still comes out forward. + const seen: string[] = []; + let delay = 30; + const dispatch: CdpDispatch = async (_m, params) => { + const wait = (delay -= 5); + await new Promise((res) => setTimeout(res, Math.max(0, wait))); + if (params.type === 'keyDown') seen.push(String(params.key)); + return {}; + }; + await typeChars(dispatch, 'abc'); + assert.deepEqual(seen, ['a', 'b', 'c']); +}); + +test('only the keyDown carries text, or every character types twice', async () => { + const { calls, dispatch } = recorder(); + await typeChars(dispatch, 'x'); + const [down, up] = calls; + assert.equal(down.params.text, 'x'); + assert.equal(down.params.unmodifiedText, 'x'); + assert.equal(up.params.text, undefined); +}); + +test('letters, digits, space and punctuation get the right code and key code', async () => { + const { calls, dispatch } = recorder(); + await typeChars(dispatch, 'a7 !'); + const downs = calls.filter((c) => c.params.type === 'keyDown'); + assert.deepEqual(downs.map((c) => c.params.code), ['KeyA', 'Digit7', 'Space', undefined]); + assert.deepEqual(downs.map((c) => c.params.windowsVirtualKeyCode), [65, 55, 32, 33]); +}); + +test('multi-line text dispatches NOTHING, because Enter in a composer sends the message', async () => { + const { calls, dispatch } = recorder(); + for (const text of ['line one\nline two', 'trailing\n', 'carriage\rreturn']) { + assert.deepEqual(await typeChars(dispatch, text), { dispatched: false, skipped: 'multiline' }); + } + assert.equal(calls.length, 0, 'a refused fill must not type a partial draft'); +}); + +test('empty text is a no-op, not an empty keystroke', async () => { + const { calls, dispatch } = recorder(); + assert.deepEqual(await typeChars(dispatch, ''), { dispatched: false, skipped: 'empty' }); + assert.equal(calls.length, 0); +}); + +test('a strict editor that ignores synthetic input still receives the whole string', async () => { + // Twitch's chat box, Reddit's Lexical, and X's composer are this shape: they own their state and + // commit only on a real key event, dropping Input.insertText and execCommand on the floor. This + // is the case the fill ladder's third tier exists for, and the case it silently failed for as + // long as it used sendInputEvent. The model below accepts nothing else on purpose. + let value = ''; + const strictEditor: CdpDispatch = async (method, params) => { + if (method === 'Input.insertText') return {}; + if (method === 'Runtime.callFunctionOn') return {}; + if (method === 'Input.dispatchKeyEvent' && params.type === 'keyDown' && params.text) { + value += String(params.text); + } + return {}; + }; + await typeChars(strictEditor, 'coverage probe alpha'); + assert.equal(value, 'coverage probe alpha'); +}); diff --git a/frontend/src/shared/typeChars.ts b/frontend/src/shared/typeChars.ts new file mode 100644 index 00000000..837d635a --- /dev/null +++ b/frontend/src/shared/typeChars.ts @@ -0,0 +1,50 @@ +// Type a string into the already-focused box as real, trusted key events over CDP. +// +// This file exists because the Electron sendInputEvent path it replaces was abandoned once +// already, in handlePressKey, for one reason: it delivers to whatever the OS thinks has focus +// rather than to this webview, and it is fire-and-forget, so a read-back races the text it is +// checking for. Both fill paths kept using it anyway, and the "via keystrokes" success line has +// never once appeared in a log. +// +// Multi-line text is refused rather than typed: Enter in a chat composer sends the message, and a +// fill tier that can post half a draft is worse than one that admits it gave up. + +/** One CDP command against the frame that owns the box being filled. */ +export type CdpDispatch = (method: string, params: Record) => Promise; + +export interface TypedKeys { + /** True only when every character was dispatched. */ + dispatched: boolean; + /** Why nothing was dispatched. Empty exactly when dispatched is true. */ + skipped: '' | 'empty' | 'multiline'; +} + +const NEWLINE: RegExp = /[\r\n]/; + +function codeFor(ch: string): string { + if (/[a-zA-Z]/.test(ch)) return `Key${ch.toUpperCase()}`; + if (/[0-9]/.test(ch)) return `Digit${ch}`; + return ch === ' ' ? 'Space' : ''; +} + +export async function typeChars(dispatch: CdpDispatch, text: string): Promise { + if (!text) return { dispatched: false, skipped: 'empty' }; + if (NEWLINE.test(text)) return { dispatched: false, skipped: 'multiline' }; + for (const ch of text) { + const vk: number = ch.toUpperCase().charCodeAt(0); + const code: string = codeFor(ch); + // text on the DOWN event is what actually inserts the character; keyUp carrying it too would + // type everything twice. + const down: Record = { + type: 'keyDown', key: ch, text: ch, unmodifiedText: ch, + windowsVirtualKeyCode: vk, nativeVirtualKeyCode: vk, + }; + const up: Record = { + type: 'keyUp', key: ch, windowsVirtualKeyCode: vk, nativeVirtualKeyCode: vk, + }; + if (code) { down.code = code; up.code = code; } + await dispatch('Input.dispatchKeyEvent', down); + await dispatch('Input.dispatchKeyEvent', up); + } + return { dispatched: true, skipped: '' }; +}