From 85828f58542ebc85e65051e4bc5b56f673bb777c Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 26 Jul 2026 21:11:07 -0700 Subject: [PATCH] [eric] voice: native uiohook key tap gives keyboard true hold-to-talk on mac and windows, toggle fallback preserved --- electron/main.js | 50 +------- electron/package-lock.json | 28 ++++- electron/package.json | 3 +- electron/preload.js | 6 +- electron/voiceHotkey.js | 116 ++++++++++++++++++ .../sections/general/GeneralInterface.tsx | 10 +- .../shared/voice/VoiceDictationContext.tsx | 24 ++-- frontend/src/types/electron.d.ts | 2 + 8 files changed, 176 insertions(+), 63 deletions(-) create mode 100644 electron/voiceHotkey.js diff --git a/electron/main.js b/electron/main.js index 79e809ad..f8921af0 100644 --- a/electron/main.js +++ b/electron/main.js @@ -57,6 +57,7 @@ const os = require('os'); const fs = require('fs'); const hiddenBrowser = require('./hiddenBrowser'); const usageHarvest = require('./usageHarvest'); +const { installVoiceHotkey } = require('./voiceHotkey'); const getPort = require('get-port'); const http = require('http'); const affiliateTracking = require('./affiliateTracking'); @@ -1788,51 +1789,10 @@ app.whenReady().then(async () => { // Off-window mouse-release crash dodge (macOS). Safe to call before windows exist. installMacMouseClamp(); - // Voice dictation hotkey. globalShortcut can't see key-up, so hold-to-talk is impossible through it; - // the shortcut is only registered while OUR window is unfocused (background toggle). When the app is - // focused, MAIN watches before-input-event (sees every keyDown/keyUp incl. modifiers, regardless of - // which element or webview has focus, and immune to macOS's letter-keyup-under-Cmd suppression at the - // DOM layer) and relays voice:hold-down / voice:hold-up; the renderer maps those to hold-vs-toggle. - // F5 is the mic/dictation key printed on Mac keyboards (the fn/globe key itself is invisible to - // Electron without a native event tap), so it's the simple primary; the old combo stays as backup. - const VOICE_COMBOS = ['F5', 'CommandOrControl+Shift+D']; - const registerVoiceShortcut = () => { - for (const combo of VOICE_COMBOS) { - try { - if (!globalShortcut.isRegistered(combo)) { - globalShortcut.register(combo, () => { - if (mainWindow && !mainWindow.isDestroyed()) mainWindow.webContents.send('voice:toggle'); - }); - } - } catch (_) { /* a taken shortcut just means no global hotkey; the pill still works */ } - } - }; - registerVoiceShortcut(); - app.on('browser-window-focus', () => { for (const combo of VOICE_COMBOS) { try { globalShortcut.unregister(combo); } catch (_) {} } }); - app.on('browser-window-blur', registerVoiceShortcut); - - // Every discrete combo press relays (autorepeat filtered); the renderer toggles record state. - // No held/release tracking: macOS delivers neither the letter keyup nor modifier releases to ANY - // Chromium layer while Cmd is down (verified empirically), so keyboard hold-release is undetectable - // without a native event tap. Keyboard = press to start/stop; the mic buttons own true hold-to-talk. - const installVoiceHoldRelay = (contents) => { - contents.on('before-input-event', (event, input) => { - if (input.type !== 'keyDown' || input.isAutoRepeat) return; - const isD = (input.code === 'KeyD' || (input.key || '').toLowerCase() === 'd'); - const combo = (isD && (input.meta || input.control) && input.shift) || input.code === 'F5'; - if (combo) { - if (mainWindow && !mainWindow.isDestroyed()) mainWindow.webContents.send('voice:hold-down'); - event.preventDefault(); - } - }); - }; - // Installed via web-contents-created (not directly on mainWindow): createWindow() runs LATER in this - // whenReady sequence, so mainWindow is still undefined here. The hook catches the main window when it - // is born, and every webview guest too (guests swallow keys when a page has focus). - app.on('web-contents-created', (event, contents) => { - const t = contents.getType(); - if (t === 'window' || t === 'webview') installVoiceHoldRelay(contents); - }); + // Voice dictation hotkey (F5 / Cmd-Ctrl+Shift+D). Native uiohook key tap = true keyboard + // hold-to-talk on every platform; falls back to the old press-to-toggle when the tap can't run + // (module missing, or macOS without the Accessibility grant). Tiers live in voiceHotkey.js. + installVoiceHotkey(() => mainWindow); // PASSKEY SPIKE (macOS only): turn on the Secure-Enclave/Touch ID WebAuthn authenticator that Electron 42 added. Without this, isUserVerifyingPlatformAuthenticatorAvailable() is hardwired false (why the old reject-shim existed). keychainAccessGroup MUST match the keychain-access-groups entitlement (Y26NUZH4NG..webauthn) or this throws. Windows has no equivalent, so the reject-shim still runs there. if (process.platform === 'darwin' && typeof app.configureWebAuthn === 'function') { diff --git a/electron/package-lock.json b/electron/package-lock.json index 1781b23d..e6071cd5 100644 --- a/electron/package-lock.json +++ b/electron/package-lock.json @@ -8,9 +8,11 @@ "name": "openswarm", "version": "1.5.8", "hasInstallScript": true, + "license": "AGPL-3.0-only", "dependencies": { "electron-updater": "6.8.3", - "get-port": "5.1.1" + "get-port": "5.1.1", + "uiohook-napi": "^1.5.5" }, "devDependencies": { "@electron/notarize": "3.1.1", @@ -2724,6 +2726,17 @@ "node": "^20.17.0 || >=22.9.0" } }, + "node_modules/node-gyp-build": { + "version": "4.8.4", + "resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz", + "integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==", + "license": "MIT", + "bin": { + "node-gyp-build": "bin.js", + "node-gyp-build-optional": "optional.js", + "node-gyp-build-test": "build-test.js" + } + }, "node_modules/node-gyp/node_modules/isexe": { "version": "4.0.0", "resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz", @@ -3470,6 +3483,19 @@ "url": "https://github.com/sponsors/sindresorhus" } }, + "node_modules/uiohook-napi": { + "version": "1.5.5", + "resolved": "https://registry.npmjs.org/uiohook-napi/-/uiohook-napi-1.5.5.tgz", + "integrity": "sha512-oSlTdnECw2GBfsJPTbBQBeE4v/EXP0EZmX6BJq5nzH/JgFaBE8JpFwEA/kLhiEP7HxQw28FViWiYgdIZzWuuJQ==", + "hasInstallScript": true, + "license": "MIT", + "dependencies": { + "node-gyp-build": "^4.8.4" + }, + "engines": { + "node": ">= 16" + } + }, "node_modules/undici": { "version": "7.26.0", "resolved": "https://registry.npmjs.org/undici/-/undici-7.26.0.tgz", diff --git a/electron/package.json b/electron/package.json index a54422d9..8f9408db 100644 --- a/electron/package.json +++ b/electron/package.json @@ -20,7 +20,8 @@ }, "dependencies": { "electron-updater": "6.8.3", - "get-port": "5.1.1" + "get-port": "5.1.1", + "uiohook-napi": "^1.5.5" }, "devDependencies": { "@electron/notarize": "3.1.1", diff --git a/electron/preload.js b/electron/preload.js index 91a36f7a..2ed05e8e 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -75,7 +75,11 @@ contextBridge.exposeInMainWorld('openswarm', { }, // Reveal a diagnostics folder in Finder/Explorer (path validated in main; diagnostics dir only). revealBundle: (folderPath) => ipcRenderer.invoke('help:reveal-bundle', folderPath), - // Main-process hold relay (before-input-event): fires down/up for the combo regardless of DOM focus. + // True keyboard hold-to-talk needs the native key tap; renderers ask so Settings copy stays honest, + // and request triggers the macOS Accessibility prompt when the tap is blocked on permission. + voiceHoldCapable: () => ipcRenderer.invoke('voice:hold-capable'), + voiceRequestHoldPermission: () => ipcRenderer.invoke('voice:request-hold-permission'), + // Native-tap hold relay: real global key-down/key-up for the voice combo, focus-independent. onVoiceHold: (onDown, onUp) => { const down = () => onDown(); const up = () => onUp(); diff --git a/electron/voiceHotkey.js b/electron/voiceHotkey.js new file mode 100644 index 00000000..6d2542b4 --- /dev/null +++ b/electron/voiceHotkey.js @@ -0,0 +1,116 @@ +const { app, globalShortcut, ipcMain, systemPreferences } = require('electron'); + +// Voice dictation hotkey (F5 primary, Cmd/Ctrl+Shift+D backup), two tiers: +// +// NATIVE (uiohook-napi event tap): sees real key-down AND key-up globally, in or out of focus, +// immune to macOS's letter-keyup-under-Cmd suppression, so the keyboard gets TRUE hold-to-talk +// exactly like the mic buttons. Listen-only (never swallows keys from other apps). macOS needs +// the Accessibility grant; Windows needs nothing. +// +// FALLBACK (no native tap: module missing, load failure, or no macOS permission): the pre-tap +// behavior verbatim. globalShortcut toggles while our window is unfocused; a before-input relay +// toggles while focused (key-ups are undetectable there, proven empirically, so press-to-toggle). +// +// The before-input relay installs in BOTH tiers: with the tap active it only swallows the combo so +// pages/webviews never see F5 or the 'd', with the tap inactive it also sends the toggle. + +const VOICE_COMBOS = ['F5', 'CommandOrControl+Shift+D']; + +function installVoiceHotkey(getMainWindow) { + const send = (channel) => { + const win = getMainWindow(); + if (win && !win.isDestroyed()) win.webContents.send(channel); + }; + + let nativeTapActive = false; + + const tryStartNativeTap = () => { + try { + if (process.platform === 'darwin' && !systemPreferences.isTrustedAccessibilityClient(false)) { + console.log('[voice] no Accessibility grant, keyboard stays press-to-toggle'); + return false; + } + const { uIOhook, UiohookKey } = require('uiohook-napi'); + const MOD_KEYS = new Set([ + UiohookKey.Ctrl, UiohookKey.CtrlRight, UiohookKey.Shift, UiohookKey.ShiftRight, + UiohookKey.Meta, UiohookKey.MetaRight, + ]); + // 'f5' | 'combo' | null; one hold at a time, repeats and the other combo ignored while held. + let heldBy = null; + + uIOhook.on('keydown', (e) => { + if (heldBy) return; + if (e.keycode === UiohookKey.F5) { + heldBy = 'f5'; + send('voice:hold-down'); + } else if (e.keycode === UiohookKey.D && e.shiftKey && (e.metaKey || e.ctrlKey)) { + heldBy = 'combo'; + send('voice:hold-down'); + } + }); + uIOhook.on('keyup', (e) => { + if (!heldBy) return; + const releases = + (heldBy === 'f5' && e.keycode === UiohookKey.F5) || + (heldBy === 'combo' && (e.keycode === UiohookKey.D || MOD_KEYS.has(e.keycode))); + if (releases) { + heldBy = null; + send('voice:hold-up'); + } + }); + + uIOhook.start(); + app.on('will-quit', () => { try { uIOhook.stop(); } catch (_) {} }); + console.log('[voice] native key tap active, keyboard hold-to-talk enabled'); + return true; + } catch (e) { + console.log('[voice] native key tap unavailable (continuing with toggle):', e && e.message); + return false; + } + }; + nativeTapActive = tryStartNativeTap(); + + if (!nativeTapActive) { + const registerVoiceShortcut = () => { + for (const combo of VOICE_COMBOS) { + try { + if (!globalShortcut.isRegistered(combo)) { + globalShortcut.register(combo, () => send('voice:toggle')); + } + } catch (_) { /* a taken shortcut just means no global hotkey; the pill still works */ } + } + }; + registerVoiceShortcut(); + app.on('browser-window-focus', () => { for (const combo of VOICE_COMBOS) { try { globalShortcut.unregister(combo); } catch (_) {} } }); + app.on('browser-window-blur', registerVoiceShortcut); + } + + const installVoiceHoldRelay = (contents) => { + contents.on('before-input-event', (event, input) => { + if (input.type !== 'keyDown' || input.isAutoRepeat) return; + const isD = (input.code === 'KeyD' || (input.key || '').toLowerCase() === 'd'); + const combo = (isD && (input.meta || input.control) && input.shift) || input.code === 'F5'; + if (combo) { + if (!nativeTapActive) send('voice:toggle'); + event.preventDefault(); + } + }); + }; + // Installed via web-contents-created: the main window is born later in the whenReady sequence, and + // webview guests swallow keys when a page has focus, so every window/guest gets the relay. + app.on('web-contents-created', (event, contents) => { + const t = contents.getType(); + if (t === 'window' || t === 'webview') installVoiceHoldRelay(contents); + }); + + ipcMain.handle('voice:hold-capable', () => nativeTapActive); + // Settings' "Hold to talk" can trigger the real macOS Accessibility prompt; a restart picks it up. + ipcMain.handle('voice:request-hold-permission', () => { + if (process.platform === 'darwin' && !nativeTapActive) { + try { systemPreferences.isTrustedAccessibilityClient(true); } catch (_) {} + } + return nativeTapActive; + }); +} + +module.exports = { installVoiceHotkey }; diff --git a/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx b/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx index 7b6569fe..52132607 100644 --- a/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx +++ b/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx @@ -104,12 +104,18 @@ const GeneralInterface: React.FC<{ Dictation - How the mic button works. The mic key (F5) starts and stops. + How the mic button and the mic key (F5) work. { if (v !== null) setForm({ ...form, voice_hold_to_talk: v }); }} + onChange={(_, v) => { + if (v === null) return; + setForm({ ...form, voice_hold_to_talk: v }); + // Keyboard hold needs the native key tap; picking Hold on a Mac without the Accessibility + // grant fires the system prompt so the choice can actually take effect after a relaunch. + if (v) void window.openswarm?.voiceRequestHoldPermission?.(); + }} size="small" sx={{ '& .MuiToggleButton-root': { diff --git a/frontend/src/shared/voice/VoiceDictationContext.tsx b/frontend/src/shared/voice/VoiceDictationContext.tsx index 5ea37ce6..14d2dd89 100644 --- a/frontend/src/shared/voice/VoiceDictationContext.tsx +++ b/frontend/src/shared/voice/VoiceDictationContext.tsx @@ -47,20 +47,18 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode } }, [holdMode, stop]); - // The hotkey (Cmd/Ctrl+Shift+D) is press-to-start / press-to-stop, NOT hold: macOS delivers - // NEITHER the letter keyup nor the modifier releases to any Chromium layer while Cmd is held - // (proven empirically: DOM saw zero events, main's before-input-event saw only the first keyDown), - // so a keyboard hold-release is undetectable without a native event tap (uiohook class, the real - // fix, needs a packaged native module). Hold-to-talk lives on the mic buttons, which DO see - // pointerup. In-app presses arrive via main's before-input relay (works with webview focus and - // swallows the 'd' so it never types into a field); background presses via the global shortcut. + // Keyboard hotkey channels, matched to what the source can actually see: + // voice:hold-down/up come ONLY from main's native uiohook tap (real global key-up, so the keyboard + // gets the same hold-vs-toggle press semantics as the mic buttons); voice:toggle comes from the + // fallback tier (globalShortcut / before-input relay), where key-ups are undetectable, so each + // press toggles. Whichever tier main activated, the renderer just honors the channel it hears. useEffect(() => { - const off = (window as unknown as { openswarm?: { onVoiceHold?: (d: () => void, u: () => void) => () => void } }).openswarm?.onVoiceHold?.( - () => toggle(), - () => {}, - ); - return () => { off?.(); }; - }, [toggle]); + const bridge = window as unknown as { + openswarm?: { onVoiceHold?: (d: () => void, u: () => void) => () => void }; + }; + const offHold = bridge.openswarm?.onVoiceHold?.(pressStart, pressEnd); + return () => { offHold?.(); }; + }, [pressStart, pressEnd]); return ( diff --git a/frontend/src/types/electron.d.ts b/frontend/src/types/electron.d.ts index a48f26c3..55b62ef6 100644 --- a/frontend/src/types/electron.d.ts +++ b/frontend/src/types/electron.d.ts @@ -59,6 +59,8 @@ declare global { voiceTranscribe?: (wav: ArrayBuffer) => Promise<{ ok: boolean; text?: string; error?: string }>; voiceInject?: (text: string) => Promise<{ ok: boolean; pasted?: boolean; error?: string }>; onVoiceToggle?: (cb: () => void) => () => void; + voiceHoldCapable?: () => Promise; + voiceRequestHoldPermission?: () => Promise; onAuthUrl?: (cb: (url: string) => void) => () => void; onOauthClaim?: (cb: (url: string) => void) => () => void; }