diff --git a/electron/main.js b/electron/main.js index 8e95e706..ae576ad6 100644 --- a/electron/main.js +++ b/electron/main.js @@ -169,6 +169,7 @@ const getPort = require('get-port'); const http = require('http'); const affiliateTracking = require('./affiliateTracking'); const cdpRoutes = require('./cdp-routes'); +const { releaseAgentPointerLock } = require('./releaseAgentPointerLock'); const workflowsLifecycle = require('./workflowsLifecycle'); // Squirrel makes the APP create its own shortcuts: on --squirrel-install it must @@ -4335,6 +4336,8 @@ async function sendCdpCommandSerialized(wcId, method, params, sessionId) { ipcMain.handle('send-cdp-command', async (_event, wcId, method, params, sessionId) => { try { const result = await sendCdpCommandSerialized(wcId, method, params, sessionId); + // A synthetic click can leave a canvas app holding the user's real cursor; hand it straight back (ENG-310). + releaseAgentPointerLock(sendCdpCommandSerialized, wcId, method, params); return { ok: true, result }; } catch (err) { return { ok: false, error: err && err.message ? err.message : String(err) }; diff --git a/electron/releaseAgentPointerLock.js b/electron/releaseAgentPointerLock.js new file mode 100644 index 00000000..734215bb --- /dev/null +++ b/electron/releaseAgentPointerLock.js @@ -0,0 +1,39 @@ +// The agent must never be able to swallow the user's real mouse (ENG-310). +// +// A CDP click is a trusted user gesture as far as Chromium is concerned, so a canvas app that calls +// requestPointerLock() from its mousedown handler (every mouse-look game does) captures and HIDES the +// physical cursor the instant the agent taps it. The user, who is off doing something else, finds +// their pointer welded to a dashboard card for the rest of the run. Measured directly: an agent- +// dispatched click on such a canvas left document.pointerLockElement=CANVAS. +// +// Denying the 'pointerLock' permission was the obvious fix and it is WRONG. Chromium caches that +// decision per origin and asks exactly once, so whoever clicks first decides for the whole session: +// deny for the agent and the user is denied too, forever, on their own app. Measured, both arms. +// +// So let the lock be granted and take it straight back off the agent's own click. Nothing about the +// user's path changes: they still get pointer lock, from the same cached grant, whenever they click +// it themselves. Exiting a lock needs no user gesture, which is what makes this side of it possible. + +/** The one command that can end with the cursor captured: the release that completes a synthetic click. */ +function isSyntheticClickRelease(method, params) { + return method === 'Input.dispatchMouseEvent' && !!params && params.type === 'mouseReleased'; +} + +const EXIT_EXPRESSION = 'document.pointerLockElement ? (document.exitPointerLock(), true) : false'; + +/** + * Give the cursor back after a synthetic click. Fire and forget: a failure here means the eval did + * not land, which is no worse than not trying, and it must never fail the command it follows. + */ +function releaseAgentPointerLock(sendCdp, wcId, method, params) { + if (!isSyntheticClickRelease(method, params)) return false; + try { + const p = sendCdp(wcId, 'Runtime.evaluate', { expression: EXIT_EXPRESSION, returnByValue: true }); + if (p && typeof p.catch === 'function') p.catch(() => {}); + } catch { + // The surface went away mid-click; there is no cursor left to hand back. + } + return true; +} + +module.exports = { releaseAgentPointerLock, isSyntheticClickRelease, EXIT_EXPRESSION }; diff --git a/electron/releaseAgentPointerLock.test.js b/electron/releaseAgentPointerLock.test.js new file mode 100644 index 00000000..4955c3ea --- /dev/null +++ b/electron/releaseAgentPointerLock.test.js @@ -0,0 +1,94 @@ +// Run: node --test electron/releaseAgentPointerLock.test.js +// +// ENG-310: "the app agent takes over the physical cursor." Measured mechanism, not a guess: an +// agent-dispatched click on a canvas that requests pointer lock left document.pointerLockElement= +// CANVAS, i.e. the user's real mouse captured and hidden, and with this module wired the same click +// logged ["locked","unlocked"] and ended at null. +// +// These pin the properties that decide whether it can come back: only the release half of a +// synthetic CLICK triggers a handback (so reads and keystrokes cost nothing), a failure can never +// break the command it follows, and main.js actually calls it. +const test = require('node:test'); +const assert = require('node:assert/strict'); +const fs = require('node:fs'); +const path = require('node:path'); + +const { releaseAgentPointerLock, isSyntheticClickRelease, EXIT_EXPRESSION } = require('./releaseAgentPointerLock'); + +function recorder() { + const calls = []; + const send = (wcId, method, params) => { + calls.push({ wcId, method, params }); + return Promise.resolve({}); + }; + return { calls, send }; +} + +test('the release half of a synthetic click hands the cursor back', () => { + const { calls, send } = recorder(); + assert.equal(releaseAgentPointerLock(send, 42, 'Input.dispatchMouseEvent', { type: 'mouseReleased' }), true); + assert.equal(calls.length, 1); + assert.equal(calls[0].wcId, 42, 'the handback must go to the surface that was clicked'); + assert.equal(calls[0].method, 'Runtime.evaluate'); + assert.equal(calls[0].params.expression, EXIT_EXPRESSION); +}); + +test('the expression is a no-op when nothing is locked', () => { + // It runs after every agent click, so it must cost nothing on the overwhelmingly common path. + const doc = { pointerLockElement: null, exitPointerLock: () => { throw new Error('must not be called'); } }; + const run = new Function('document', `return (${EXIT_EXPRESSION});`); + assert.equal(run(doc), false); +}); + +test('the expression exits exactly one lock when there is one', () => { + let exits = 0; + const doc = { pointerLockElement: {}, exitPointerLock: () => { exits += 1; } }; + const run = new Function('document', `return (${EXIT_EXPRESSION});`); + assert.equal(run(doc), true); + assert.equal(exits, 1); +}); + +test('nothing else in a click triggers a handback, so one click costs one eval', () => { + const { calls, send } = recorder(); + for (const params of [{ type: 'mouseMoved' }, { type: 'mousePressed' }, { type: 'mouseWheel' }]) { + assert.equal(releaseAgentPointerLock(send, 1, 'Input.dispatchMouseEvent', params), false); + } + assert.equal(calls.length, 0); +}); + +test('reads and keystrokes never pay for it', () => { + const { calls, send } = recorder(); + for (const method of ['Page.captureScreenshot', 'Runtime.evaluate', 'Accessibility.getFullAXTree', 'Input.dispatchKeyEvent']) { + assert.equal(releaseAgentPointerLock(send, 1, method, { type: 'mouseReleased' }), false, `${method} must not trigger a handback`); + } + assert.equal(calls.length, 0); +}); + +test('missing or malformed params are ignored rather than thrown on', () => { + const { calls, send } = recorder(); + assert.equal(releaseAgentPointerLock(send, 1, 'Input.dispatchMouseEvent', undefined), false); + assert.equal(releaseAgentPointerLock(send, 1, 'Input.dispatchMouseEvent', null), false); + assert.equal(calls.length, 0); +}); + +test('a handback that fails never breaks the click it follows', () => { + const thrower = () => { throw new Error('debugger detached'); }; + assert.doesNotThrow(() => releaseAgentPointerLock(thrower, 1, 'Input.dispatchMouseEvent', { type: 'mouseReleased' })); + const rejecter = () => Promise.reject(new Error('target closed')); + assert.doesNotThrow(() => releaseAgentPointerLock(rejecter, 1, 'Input.dispatchMouseEvent', { type: 'mouseReleased' })); +}); + +test('the classifier is exact, never a prefix guess', () => { + assert.equal(isSyntheticClickRelease('Input.dispatchMouseEvent', { type: 'mouseReleased' }), true); + assert.equal(isSyntheticClickRelease('Input.dispatchMouseEventExtra', { type: 'mouseReleased' }), false); + assert.equal(isSyntheticClickRelease('Input.dispatchMouseEvent', { type: 'mouseReleasedish' }), false); +}); + +test('main.js calls it, and only on the agent CDP path', () => { + const src = fs.readFileSync(path.join(__dirname, 'main.js'), 'utf8'); + assert.match(src, /releaseAgentPointerLock\(sendCdpCommandSerialized, wcId, method, params\)/, + 'a handback nothing calls hands nothing back'); + const handler = src.slice(src.indexOf("ipcMain.handle('send-cdp-command'")); + assert.ok(handler.indexOf('releaseAgentPointerLock') < handler.indexOf('return { ok: true, result }'), + 'the handback belongs inside the agent command path, where a human click never goes'); +});