[eric] browser: fill a strict editor over CDP, since sendInputEvent never once landed

frontend/src/shared/browserCommandHandler.ts

@@ -8,6 +8,7 @@ import { resolveInput } from './resolveUrl';
 import { rankAndCapInteractives, type RankItem } from './interactiveRanking';
 import { shouldStopWaiting, SETTLE_POLL_MS, settleProbeJs } from './browserSettle';
 import { unwrapCdpEval } from './cdpEval';
+import { typeChars, type TypedKeys } from './typeChars';
 
 let initialized = false;
 
@@ -745,18 +746,17 @@ async function handleFindComposer(wv: BrowserWebview, params: Record<string, any
   return result;
 }
 
-// Type `text` into the element at `selector` as real OS-level key events (Electron
-// sendInputEvent 'char'), for editors that reject synthetic execCommand insertText. Focuses +
-// clears in-page first, types each char natively, then reads the value back in-page to verify.
+// Type `text` into the element at `selector` as real, trusted key events, for editors that reject
+// synthetic execCommand insertText. Focuses + clears in-page first, types each char, then reads the
+// value back in-page to verify.
 async function keystrokeFill(wv: BrowserWebview, selector: string, text: string): Promise<boolean> {
   const safeSel = JSON.stringify(selector);
   await evalInPage(wv, `(() => { const el = document.querySelector(${safeSel});
     if (!el) return false; el.scrollIntoView({ block: 'center', behavior: 'instant' }); el.focus();
     if (el.select) el.select(); document.execCommand('selectAll', false); document.execCommand('delete', false);
     return true; })()`);
-  for (const ch of text) {
-    wv.sendInputEvent({ type: 'char', keyCode: ch });
-  }
+  const typed = await typeChars((m, p) => sendCdp(wv, m, p), text);
+  if (!typed.dispatched) return false;
   // Read back from the marked element OR the active element: editors like Reddit's swap the
   // node on activation, so the original selector can go stale even though the keystrokes landed
   // in whatever now holds focus. Checking both survives that re-render.
@@ -1254,6 +1254,7 @@ async function clickBackendNode(
       // live on twitch, the composer was found and focused and the fill still errored, which cost
       // the site its entire write path. Clearing first, because a partial insert plus keystrokes is
       // how you post the same sentence twice.
+      let typed: TypedKeys = { dispatched: false, skipped: 'empty' };
       try {
         const t = await sendCdp(wv, 'DOM.resolveNode', { backendNodeId }, sessionId);
         await sendCdp(wv, 'Runtime.callFunctionOn', {
@@ -1261,7 +1262,11 @@ async function clickBackendNode(
           functionDeclaration:
             'function() { this.focus(); if (this.select) this.select(); document.execCommand("selectAll", false); document.execCommand("delete", false); }',
         }, sessionId);
-        for (const ch of opts.text) wv.sendInputEvent({ type: 'char', keyCode: ch });
+        // Focus is set in the node's own frame above, but the key events themselves go to the ROOT
+        // target with no sessionId: Chromium routes keyboard input to whichever frame holds focus,
+        // and the Input domain is not reliably there on an OOPIF session. Sending them at the child
+        // would kill exactly the case this cares about, a composer inside an iframe.
+        typed = await typeChars((m, p) => sendCdp(wv, m, p), opts.text);
       } catch { /* verified below; the honest error covers this failing too */ }
       got = await readBack();
       if (got !== null) return landedMsg(got, ' (via keystrokes)');
@@ -1275,6 +1280,11 @@ async function clickBackendNode(
         if (typeof live === 'string') return landedMsg(live, ' (via keystrokes)');
       } catch { /* fall through to the honest error */ }
       flashField(wv, resolvedObjectId, sessionId, 'fail');
+      // Saying "not even as real keystrokes" when we deliberately declined to send them is the
+      // kind of small lie that costs a debugging session.
+      if (typed.skipped === 'multiline') {
+        return { error: `Focused ${label} but this editor ignored both synthetic fills, and the text contains a line break, so real keystrokes were NOT attempted: pressing Enter in a composer can send it half-written. Fill it as a single line, or use a different element.` };
+      }
       return { error: `Focused ${label} but the text did not register even as real keystrokes; the box may be a custom editor that rejects automation. Try a different element.` };
     }
     return { text: `Focused ${label}; the cursor is in it now (type with BrowserPressKey, or pass a text arg to fill it in one call).` };

frontend/src/shared/typeChars.test.ts

@@ -0,0 +1,100 @@
+// Run: node --test frontend/src/shared/typeChars.test.ts
+import { test } from 'node:test';
+import assert from 'node:assert/strict';
+import { typeChars } from './typeChars.ts';
+import type { CdpDispatch } from './typeChars.ts';
+
+interface Call { method: string; params: Record<string, any> }
+
+function recorder(): { calls: Call[]; dispatch: CdpDispatch } {
+  const calls: Call[] = [];
+  const dispatch: CdpDispatch = async (method, params) => {
+    calls.push({ method, params: params as Record<string, any> });
+    return {};
+  };
+  return { calls, dispatch };
+}
+
+test('every character goes out as a CDP keyDown then keyUp, in order', async () => {
+  const { calls, dispatch } = recorder();
+  const r = await typeChars(dispatch, 'hi');
+  assert.deepEqual(r, { dispatched: true, skipped: '' });
+  assert.equal(calls.length, 4);
+  assert.deepEqual(calls.map((c) => c.params.type), ['keyDown', 'keyUp', 'keyDown', 'keyUp']);
+  assert.deepEqual(calls.map((c) => c.params.key), ['h', 'h', 'i', 'i']);
+});
+
+test('the transport is CDP, never Electron sendInputEvent', async () => {
+  // The whole point of this module. sendInputEvent goes to whatever the OS thinks has focus, which
+  // is how the agent once typed into the user's notes app, and it is unawaitable, which is why the
+  // read-back after it always lost the race. If this ever regresses, the fill tier goes dead silent
+  // again: it fails with "did not register even as real keystrokes" and nobody sees a reason.
+  const { calls, dispatch } = recorder();
+  await typeChars(dispatch, 'abc');
+  assert.ok(calls.length > 0);
+  assert.ok(calls.every((c) => c.method === 'Input.dispatchKeyEvent'), 'every event must go over CDP');
+});
+
+test('each dispatch is awaited, so a read-back after this cannot outrun the text', async () => {
+  // A fire-and-forget loop would let a slow first character land after a fast last one. Resolve in
+  // reverse-latency order and check the sequence still comes out forward.
+  const seen: string[] = [];
+  let delay = 30;
+  const dispatch: CdpDispatch = async (_m, params) => {
+    const wait = (delay -= 5);
+    await new Promise((res) => setTimeout(res, Math.max(0, wait)));
+    if (params.type === 'keyDown') seen.push(String(params.key));
+    return {};
+  };
+  await typeChars(dispatch, 'abc');
+  assert.deepEqual(seen, ['a', 'b', 'c']);
+});
+
+test('only the keyDown carries text, or every character types twice', async () => {
+  const { calls, dispatch } = recorder();
+  await typeChars(dispatch, 'x');
+  const [down, up] = calls;
+  assert.equal(down.params.text, 'x');
+  assert.equal(down.params.unmodifiedText, 'x');
+  assert.equal(up.params.text, undefined);
+});
+
+test('letters, digits, space and punctuation get the right code and key code', async () => {
+  const { calls, dispatch } = recorder();
+  await typeChars(dispatch, 'a7 !');
+  const downs = calls.filter((c) => c.params.type === 'keyDown');
+  assert.deepEqual(downs.map((c) => c.params.code), ['KeyA', 'Digit7', 'Space', undefined]);
+  assert.deepEqual(downs.map((c) => c.params.windowsVirtualKeyCode), [65, 55, 32, 33]);
+});
+
+test('multi-line text dispatches NOTHING, because Enter in a composer sends the message', async () => {
+  const { calls, dispatch } = recorder();
+  for (const text of ['line one\nline two', 'trailing\n', 'carriage\rreturn']) {
+    assert.deepEqual(await typeChars(dispatch, text), { dispatched: false, skipped: 'multiline' });
+  }
+  assert.equal(calls.length, 0, 'a refused fill must not type a partial draft');
+});
+
+test('empty text is a no-op, not an empty keystroke', async () => {
+  const { calls, dispatch } = recorder();
+  assert.deepEqual(await typeChars(dispatch, ''), { dispatched: false, skipped: 'empty' });
+  assert.equal(calls.length, 0);
+});
+
+test('a strict editor that ignores synthetic input still receives the whole string', async () => {
+  // Twitch's chat box, Reddit's Lexical, and X's composer are this shape: they own their state and
+  // commit only on a real key event, dropping Input.insertText and execCommand on the floor. This
+  // is the case the fill ladder's third tier exists for, and the case it silently failed for as
+  // long as it used sendInputEvent. The model below accepts nothing else on purpose.
+  let value = '';
+  const strictEditor: CdpDispatch = async (method, params) => {
+    if (method === 'Input.insertText') return {};
+    if (method === 'Runtime.callFunctionOn') return {};
+    if (method === 'Input.dispatchKeyEvent' && params.type === 'keyDown' && params.text) {
+      value += String(params.text);
+    }
+    return {};
+  };
+  await typeChars(strictEditor, 'coverage probe alpha');
+  assert.equal(value, 'coverage probe alpha');
+});

frontend/src/shared/typeChars.ts

@@ -0,0 +1,50 @@
+// Type a string into the already-focused box as real, trusted key events over CDP.
+//
+// This file exists because the Electron sendInputEvent path it replaces was abandoned once
+// already, in handlePressKey, for one reason: it delivers to whatever the OS thinks has focus
+// rather than to this webview, and it is fire-and-forget, so a read-back races the text it is
+// checking for. Both fill paths kept using it anyway, and the "via keystrokes" success line has
+// never once appeared in a log.
+//
+// Multi-line text is refused rather than typed: Enter in a chat composer sends the message, and a
+// fill tier that can post half a draft is worse than one that admits it gave up.
+
+/** One CDP command against the frame that owns the box being filled. */
+export type CdpDispatch = (method: string, params: Record<string, unknown>) => Promise<unknown>;
+
+export interface TypedKeys {
+  /** True only when every character was dispatched. */
+  dispatched: boolean;
+  /** Why nothing was dispatched. Empty exactly when dispatched is true. */
+  skipped: '' | 'empty' | 'multiline';
+}
+
+const NEWLINE: RegExp = /[\r\n]/;
+
+function codeFor(ch: string): string {
+  if (/[a-zA-Z]/.test(ch)) return `Key${ch.toUpperCase()}`;
+  if (/[0-9]/.test(ch)) return `Digit${ch}`;
+  return ch === ' ' ? 'Space' : '';
+}
+
+export async function typeChars(dispatch: CdpDispatch, text: string): Promise<TypedKeys> {
+  if (!text) return { dispatched: false, skipped: 'empty' };
+  if (NEWLINE.test(text)) return { dispatched: false, skipped: 'multiline' };
+  for (const ch of text) {
+    const vk: number = ch.toUpperCase().charCodeAt(0);
+    const code: string = codeFor(ch);
+    // text on the DOWN event is what actually inserts the character; keyUp carrying it too would
+    // type everything twice.
+    const down: Record<string, unknown> = {
+      type: 'keyDown', key: ch, text: ch, unmodifiedText: ch,
+      windowsVirtualKeyCode: vk, nativeVirtualKeyCode: vk,
+    };
+    const up: Record<string, unknown> = {
+      type: 'keyUp', key: ch, windowsVirtualKeyCode: vk, nativeVirtualKeyCode: vk,
+    };
+    if (code) { down.code = code; up.code = code; }
+    await dispatch('Input.dispatchKeyEvent', down);
+    await dispatch('Input.dispatchKeyEvent', up);
+  }
+  return { dispatched: true, skipped: '' };
+}