[eric] agents: an agent's click hands the physical cursor straight back, so a canvas app can't hold the user's mouse (ENG-310)

electron/main.js

@@ -169,6 +169,7 @@ const getPort = require('get-port');
 const http = require('http');
 const affiliateTracking = require('./affiliateTracking');
 const cdpRoutes = require('./cdp-routes');
+const { releaseAgentPointerLock } = require('./releaseAgentPointerLock');
 const workflowsLifecycle = require('./workflowsLifecycle');
 
 // Squirrel makes the APP create its own shortcuts: on --squirrel-install it must
@@ -4335,6 +4336,8 @@ async function sendCdpCommandSerialized(wcId, method, params, sessionId) {
 ipcMain.handle('send-cdp-command', async (_event, wcId, method, params, sessionId) => {
   try {
     const result = await sendCdpCommandSerialized(wcId, method, params, sessionId);
+    // A synthetic click can leave a canvas app holding the user's real cursor; hand it straight back (ENG-310).
+    releaseAgentPointerLock(sendCdpCommandSerialized, wcId, method, params);
     return { ok: true, result };
   } catch (err) {
     return { ok: false, error: err && err.message ? err.message : String(err) };

electron/releaseAgentPointerLock.js

@@ -0,0 +1,39 @@
+// The agent must never be able to swallow the user's real mouse (ENG-310).
+//
+// A CDP click is a trusted user gesture as far as Chromium is concerned, so a canvas app that calls
+// requestPointerLock() from its mousedown handler (every mouse-look game does) captures and HIDES the
+// physical cursor the instant the agent taps it. The user, who is off doing something else, finds
+// their pointer welded to a dashboard card for the rest of the run. Measured directly: an agent-
+// dispatched click on such a canvas left document.pointerLockElement=CANVAS.
+//
+// Denying the 'pointerLock' permission was the obvious fix and it is WRONG. Chromium caches that
+// decision per origin and asks exactly once, so whoever clicks first decides for the whole session:
+// deny for the agent and the user is denied too, forever, on their own app. Measured, both arms.
+//
+// So let the lock be granted and take it straight back off the agent's own click. Nothing about the
+// user's path changes: they still get pointer lock, from the same cached grant, whenever they click
+// it themselves. Exiting a lock needs no user gesture, which is what makes this side of it possible.
+
+/** The one command that can end with the cursor captured: the release that completes a synthetic click. */
+function isSyntheticClickRelease(method, params) {
+  return method === 'Input.dispatchMouseEvent' && !!params && params.type === 'mouseReleased';
+}
+
+const EXIT_EXPRESSION = 'document.pointerLockElement ? (document.exitPointerLock(), true) : false';
+
+/**
+ * Give the cursor back after a synthetic click. Fire and forget: a failure here means the eval did
+ * not land, which is no worse than not trying, and it must never fail the command it follows.
+ */
+function releaseAgentPointerLock(sendCdp, wcId, method, params) {
+  if (!isSyntheticClickRelease(method, params)) return false;
+  try {
+    const p = sendCdp(wcId, 'Runtime.evaluate', { expression: EXIT_EXPRESSION, returnByValue: true });
+    if (p && typeof p.catch === 'function') p.catch(() => {});
+  } catch {
+    // The surface went away mid-click; there is no cursor left to hand back.
+  }
+  return true;
+}
+
+module.exports = { releaseAgentPointerLock, isSyntheticClickRelease, EXIT_EXPRESSION };

electron/releaseAgentPointerLock.test.js

@@ -0,0 +1,94 @@
+// Run: node --test electron/releaseAgentPointerLock.test.js
+//
+// ENG-310: "the app agent takes over the physical cursor." Measured mechanism, not a guess: an
+// agent-dispatched click on a canvas that requests pointer lock left document.pointerLockElement=
+// CANVAS, i.e. the user's real mouse captured and hidden, and with this module wired the same click
+// logged ["locked","unlocked"] and ended at null.
+//
+// These pin the properties that decide whether it can come back: only the release half of a
+// synthetic CLICK triggers a handback (so reads and keystrokes cost nothing), a failure can never
+// break the command it follows, and main.js actually calls it.
+const test = require('node:test');
+const assert = require('node:assert/strict');
+const fs = require('node:fs');
+const path = require('node:path');
+
+const { releaseAgentPointerLock, isSyntheticClickRelease, EXIT_EXPRESSION } = require('./releaseAgentPointerLock');
+
+function recorder() {
+  const calls = [];
+  const send = (wcId, method, params) => {
+    calls.push({ wcId, method, params });
+    return Promise.resolve({});
+  };
+  return { calls, send };
+}
+
+test('the release half of a synthetic click hands the cursor back', () => {
+  const { calls, send } = recorder();
+  assert.equal(releaseAgentPointerLock(send, 42, 'Input.dispatchMouseEvent', { type: 'mouseReleased' }), true);
+  assert.equal(calls.length, 1);
+  assert.equal(calls[0].wcId, 42, 'the handback must go to the surface that was clicked');
+  assert.equal(calls[0].method, 'Runtime.evaluate');
+  assert.equal(calls[0].params.expression, EXIT_EXPRESSION);
+});
+
+test('the expression is a no-op when nothing is locked', () => {
+  // It runs after every agent click, so it must cost nothing on the overwhelmingly common path.
+  const doc = { pointerLockElement: null, exitPointerLock: () => { throw new Error('must not be called'); } };
+  const run = new Function('document', `return (${EXIT_EXPRESSION});`);
+  assert.equal(run(doc), false);
+});
+
+test('the expression exits exactly one lock when there is one', () => {
+  let exits = 0;
+  const doc = { pointerLockElement: {}, exitPointerLock: () => { exits += 1; } };
+  const run = new Function('document', `return (${EXIT_EXPRESSION});`);
+  assert.equal(run(doc), true);
+  assert.equal(exits, 1);
+});
+
+test('nothing else in a click triggers a handback, so one click costs one eval', () => {
+  const { calls, send } = recorder();
+  for (const params of [{ type: 'mouseMoved' }, { type: 'mousePressed' }, { type: 'mouseWheel' }]) {
+    assert.equal(releaseAgentPointerLock(send, 1, 'Input.dispatchMouseEvent', params), false);
+  }
+  assert.equal(calls.length, 0);
+});
+
+test('reads and keystrokes never pay for it', () => {
+  const { calls, send } = recorder();
+  for (const method of ['Page.captureScreenshot', 'Runtime.evaluate', 'Accessibility.getFullAXTree', 'Input.dispatchKeyEvent']) {
+    assert.equal(releaseAgentPointerLock(send, 1, method, { type: 'mouseReleased' }), false, `${method} must not trigger a handback`);
+  }
+  assert.equal(calls.length, 0);
+});
+
+test('missing or malformed params are ignored rather than thrown on', () => {
+  const { calls, send } = recorder();
+  assert.equal(releaseAgentPointerLock(send, 1, 'Input.dispatchMouseEvent', undefined), false);
+  assert.equal(releaseAgentPointerLock(send, 1, 'Input.dispatchMouseEvent', null), false);
+  assert.equal(calls.length, 0);
+});
+
+test('a handback that fails never breaks the click it follows', () => {
+  const thrower = () => { throw new Error('debugger detached'); };
+  assert.doesNotThrow(() => releaseAgentPointerLock(thrower, 1, 'Input.dispatchMouseEvent', { type: 'mouseReleased' }));
+  const rejecter = () => Promise.reject(new Error('target closed'));
+  assert.doesNotThrow(() => releaseAgentPointerLock(rejecter, 1, 'Input.dispatchMouseEvent', { type: 'mouseReleased' }));
+});
+
+test('the classifier is exact, never a prefix guess', () => {
+  assert.equal(isSyntheticClickRelease('Input.dispatchMouseEvent', { type: 'mouseReleased' }), true);
+  assert.equal(isSyntheticClickRelease('Input.dispatchMouseEventExtra', { type: 'mouseReleased' }), false);
+  assert.equal(isSyntheticClickRelease('Input.dispatchMouseEvent', { type: 'mouseReleasedish' }), false);
+});
+
+test('main.js calls it, and only on the agent CDP path', () => {
+  const src = fs.readFileSync(path.join(__dirname, 'main.js'), 'utf8');
+  assert.match(src, /releaseAgentPointerLock\(sendCdpCommandSerialized, wcId, method, params\)/,
+    'a handback nothing calls hands nothing back');
+  const handler = src.slice(src.indexOf("ipcMain.handle('send-cdp-command'"));
+  assert.ok(handler.indexOf('releaseAgentPointerLock') < handler.indexOf('return { ok: true, result }'),
+    'the handback belongs inside the agent command path, where a human click never goes');
+});