mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-07 02:07:45 +02:00
[eric] voice: native uiohook key tap gives keyboard true hold-to-talk on mac and windows, toggle fallback preserved
This commit is contained in:
ciregenz
+5
-45
@@ -57,6 +57,7 @@ const os = require('os');
|
||||
const fs = require('fs');
|
||||
const hiddenBrowser = require('./hiddenBrowser');
|
||||
const usageHarvest = require('./usageHarvest');
|
||||
const { installVoiceHotkey } = require('./voiceHotkey');
|
||||
const getPort = require('get-port');
|
||||
const http = require('http');
|
||||
const affiliateTracking = require('./affiliateTracking');
|
||||
@@ -1788,51 +1789,10 @@ app.whenReady().then(async () => {
|
||||
// Off-window mouse-release crash dodge (macOS). Safe to call before windows exist.
|
||||
installMacMouseClamp();
|
||||
|
||||
// Voice dictation hotkey. globalShortcut can't see key-up, so hold-to-talk is impossible through it;
|
||||
// the shortcut is only registered while OUR window is unfocused (background toggle). When the app is
|
||||
// focused, MAIN watches before-input-event (sees every keyDown/keyUp incl. modifiers, regardless of
|
||||
// which element or webview has focus, and immune to macOS's letter-keyup-under-Cmd suppression at the
|
||||
// DOM layer) and relays voice:hold-down / voice:hold-up; the renderer maps those to hold-vs-toggle.
|
||||
// F5 is the mic/dictation key printed on Mac keyboards (the fn/globe key itself is invisible to
|
||||
// Electron without a native event tap), so it's the simple primary; the old combo stays as backup.
|
||||
const VOICE_COMBOS = ['F5', 'CommandOrControl+Shift+D'];
|
||||
const registerVoiceShortcut = () => {
|
||||
for (const combo of VOICE_COMBOS) {
|
||||
try {
|
||||
if (!globalShortcut.isRegistered(combo)) {
|
||||
globalShortcut.register(combo, () => {
|
||||
if (mainWindow && !mainWindow.isDestroyed()) mainWindow.webContents.send('voice:toggle');
|
||||
});
|
||||
}
|
||||
} catch (_) { /* a taken shortcut just means no global hotkey; the pill still works */ }
|
||||
}
|
||||
};
|
||||
registerVoiceShortcut();
|
||||
app.on('browser-window-focus', () => { for (const combo of VOICE_COMBOS) { try { globalShortcut.unregister(combo); } catch (_) {} } });
|
||||
app.on('browser-window-blur', registerVoiceShortcut);
|
||||
|
||||
// Every discrete combo press relays (autorepeat filtered); the renderer toggles record state.
|
||||
// No held/release tracking: macOS delivers neither the letter keyup nor modifier releases to ANY
|
||||
// Chromium layer while Cmd is down (verified empirically), so keyboard hold-release is undetectable
|
||||
// without a native event tap. Keyboard = press to start/stop; the mic buttons own true hold-to-talk.
|
||||
const installVoiceHoldRelay = (contents) => {
|
||||
contents.on('before-input-event', (event, input) => {
|
||||
if (input.type !== 'keyDown' || input.isAutoRepeat) return;
|
||||
const isD = (input.code === 'KeyD' || (input.key || '').toLowerCase() === 'd');
|
||||
const combo = (isD && (input.meta || input.control) && input.shift) || input.code === 'F5';
|
||||
if (combo) {
|
||||
if (mainWindow && !mainWindow.isDestroyed()) mainWindow.webContents.send('voice:hold-down');
|
||||
event.preventDefault();
|
||||
}
|
||||
});
|
||||
};
|
||||
// Installed via web-contents-created (not directly on mainWindow): createWindow() runs LATER in this
|
||||
// whenReady sequence, so mainWindow is still undefined here. The hook catches the main window when it
|
||||
// is born, and every webview guest too (guests swallow keys when a page has focus).
|
||||
app.on('web-contents-created', (event, contents) => {
|
||||
const t = contents.getType();
|
||||
if (t === 'window' || t === 'webview') installVoiceHoldRelay(contents);
|
||||
});
|
||||
// Voice dictation hotkey (F5 / Cmd-Ctrl+Shift+D). Native uiohook key tap = true keyboard
|
||||
// hold-to-talk on every platform; falls back to the old press-to-toggle when the tap can't run
|
||||
// (module missing, or macOS without the Accessibility grant). Tiers live in voiceHotkey.js.
|
||||
installVoiceHotkey(() => mainWindow);
|
||||
|
||||
// PASSKEY SPIKE (macOS only): turn on the Secure-Enclave/Touch ID WebAuthn authenticator that Electron 42 added. Without this, isUserVerifyingPlatformAuthenticatorAvailable() is hardwired false (why the old reject-shim existed). keychainAccessGroup MUST match the keychain-access-groups entitlement (Y26NUZH4NG.<bundle>.webauthn) or this throws. Windows has no equivalent, so the reject-shim still runs there.
|
||||
if (process.platform === 'darwin' && typeof app.configureWebAuthn === 'function') {
|
||||
|
||||
Generated
+27
-1
@@ -8,9 +8,11 @@
|
||||
"name": "openswarm",
|
||||
"version": "1.5.8",
|
||||
"hasInstallScript": true,
|
||||
"license": "AGPL-3.0-only",
|
||||
"dependencies": {
|
||||
"electron-updater": "6.8.3",
|
||||
"get-port": "5.1.1"
|
||||
"get-port": "5.1.1",
|
||||
"uiohook-napi": "^1.5.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@electron/notarize": "3.1.1",
|
||||
@@ -2724,6 +2726,17 @@
|
||||
"node": "^20.17.0 || >=22.9.0"
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp-build": {
|
||||
"version": "4.8.4",
|
||||
"resolved": "https://registry.npmjs.org/node-gyp-build/-/node-gyp-build-4.8.4.tgz",
|
||||
"integrity": "sha512-LA4ZjwlnUblHVgq0oBF3Jl/6h/Nvs5fzBLwdEF4nuxnFdsfajde4WfxtJr3CaiH+F6ewcIB/q4jQ4UzPyid+CQ==",
|
||||
"license": "MIT",
|
||||
"bin": {
|
||||
"node-gyp-build": "bin.js",
|
||||
"node-gyp-build-optional": "optional.js",
|
||||
"node-gyp-build-test": "build-test.js"
|
||||
}
|
||||
},
|
||||
"node_modules/node-gyp/node_modules/isexe": {
|
||||
"version": "4.0.0",
|
||||
"resolved": "https://registry.npmjs.org/isexe/-/isexe-4.0.0.tgz",
|
||||
@@ -3470,6 +3483,19 @@
|
||||
"url": "https://github.com/sponsors/sindresorhus"
|
||||
}
|
||||
},
|
||||
"node_modules/uiohook-napi": {
|
||||
"version": "1.5.5",
|
||||
"resolved": "https://registry.npmjs.org/uiohook-napi/-/uiohook-napi-1.5.5.tgz",
|
||||
"integrity": "sha512-oSlTdnECw2GBfsJPTbBQBeE4v/EXP0EZmX6BJq5nzH/JgFaBE8JpFwEA/kLhiEP7HxQw28FViWiYgdIZzWuuJQ==",
|
||||
"hasInstallScript": true,
|
||||
"license": "MIT",
|
||||
"dependencies": {
|
||||
"node-gyp-build": "^4.8.4"
|
||||
},
|
||||
"engines": {
|
||||
"node": ">= 16"
|
||||
}
|
||||
},
|
||||
"node_modules/undici": {
|
||||
"version": "7.26.0",
|
||||
"resolved": "https://registry.npmjs.org/undici/-/undici-7.26.0.tgz",
|
||||
|
||||
@@ -20,7 +20,8 @@
|
||||
},
|
||||
"dependencies": {
|
||||
"electron-updater": "6.8.3",
|
||||
"get-port": "5.1.1"
|
||||
"get-port": "5.1.1",
|
||||
"uiohook-napi": "^1.5.5"
|
||||
},
|
||||
"devDependencies": {
|
||||
"@electron/notarize": "3.1.1",
|
||||
|
||||
+5
-1
@@ -75,7 +75,11 @@ contextBridge.exposeInMainWorld('openswarm', {
|
||||
},
|
||||
// Reveal a diagnostics folder in Finder/Explorer (path validated in main; diagnostics dir only).
|
||||
revealBundle: (folderPath) => ipcRenderer.invoke('help:reveal-bundle', folderPath),
|
||||
// Main-process hold relay (before-input-event): fires down/up for the combo regardless of DOM focus.
|
||||
// True keyboard hold-to-talk needs the native key tap; renderers ask so Settings copy stays honest,
|
||||
// and request triggers the macOS Accessibility prompt when the tap is blocked on permission.
|
||||
voiceHoldCapable: () => ipcRenderer.invoke('voice:hold-capable'),
|
||||
voiceRequestHoldPermission: () => ipcRenderer.invoke('voice:request-hold-permission'),
|
||||
// Native-tap hold relay: real global key-down/key-up for the voice combo, focus-independent.
|
||||
onVoiceHold: (onDown, onUp) => {
|
||||
const down = () => onDown();
|
||||
const up = () => onUp();
|
||||
|
||||
@@ -0,0 +1,116 @@
|
||||
const { app, globalShortcut, ipcMain, systemPreferences } = require('electron');
|
||||
|
||||
// Voice dictation hotkey (F5 primary, Cmd/Ctrl+Shift+D backup), two tiers:
|
||||
//
|
||||
// NATIVE (uiohook-napi event tap): sees real key-down AND key-up globally, in or out of focus,
|
||||
// immune to macOS's letter-keyup-under-Cmd suppression, so the keyboard gets TRUE hold-to-talk
|
||||
// exactly like the mic buttons. Listen-only (never swallows keys from other apps). macOS needs
|
||||
// the Accessibility grant; Windows needs nothing.
|
||||
//
|
||||
// FALLBACK (no native tap: module missing, load failure, or no macOS permission): the pre-tap
|
||||
// behavior verbatim. globalShortcut toggles while our window is unfocused; a before-input relay
|
||||
// toggles while focused (key-ups are undetectable there, proven empirically, so press-to-toggle).
|
||||
//
|
||||
// The before-input relay installs in BOTH tiers: with the tap active it only swallows the combo so
|
||||
// pages/webviews never see F5 or the 'd', with the tap inactive it also sends the toggle.
|
||||
|
||||
const VOICE_COMBOS = ['F5', 'CommandOrControl+Shift+D'];
|
||||
|
||||
function installVoiceHotkey(getMainWindow) {
|
||||
const send = (channel) => {
|
||||
const win = getMainWindow();
|
||||
if (win && !win.isDestroyed()) win.webContents.send(channel);
|
||||
};
|
||||
|
||||
let nativeTapActive = false;
|
||||
|
||||
const tryStartNativeTap = () => {
|
||||
try {
|
||||
if (process.platform === 'darwin' && !systemPreferences.isTrustedAccessibilityClient(false)) {
|
||||
console.log('[voice] no Accessibility grant, keyboard stays press-to-toggle');
|
||||
return false;
|
||||
}
|
||||
const { uIOhook, UiohookKey } = require('uiohook-napi');
|
||||
const MOD_KEYS = new Set([
|
||||
UiohookKey.Ctrl, UiohookKey.CtrlRight, UiohookKey.Shift, UiohookKey.ShiftRight,
|
||||
UiohookKey.Meta, UiohookKey.MetaRight,
|
||||
]);
|
||||
// 'f5' | 'combo' | null; one hold at a time, repeats and the other combo ignored while held.
|
||||
let heldBy = null;
|
||||
|
||||
uIOhook.on('keydown', (e) => {
|
||||
if (heldBy) return;
|
||||
if (e.keycode === UiohookKey.F5) {
|
||||
heldBy = 'f5';
|
||||
send('voice:hold-down');
|
||||
} else if (e.keycode === UiohookKey.D && e.shiftKey && (e.metaKey || e.ctrlKey)) {
|
||||
heldBy = 'combo';
|
||||
send('voice:hold-down');
|
||||
}
|
||||
});
|
||||
uIOhook.on('keyup', (e) => {
|
||||
if (!heldBy) return;
|
||||
const releases =
|
||||
(heldBy === 'f5' && e.keycode === UiohookKey.F5) ||
|
||||
(heldBy === 'combo' && (e.keycode === UiohookKey.D || MOD_KEYS.has(e.keycode)));
|
||||
if (releases) {
|
||||
heldBy = null;
|
||||
send('voice:hold-up');
|
||||
}
|
||||
});
|
||||
|
||||
uIOhook.start();
|
||||
app.on('will-quit', () => { try { uIOhook.stop(); } catch (_) {} });
|
||||
console.log('[voice] native key tap active, keyboard hold-to-talk enabled');
|
||||
return true;
|
||||
} catch (e) {
|
||||
console.log('[voice] native key tap unavailable (continuing with toggle):', e && e.message);
|
||||
return false;
|
||||
}
|
||||
};
|
||||
nativeTapActive = tryStartNativeTap();
|
||||
|
||||
if (!nativeTapActive) {
|
||||
const registerVoiceShortcut = () => {
|
||||
for (const combo of VOICE_COMBOS) {
|
||||
try {
|
||||
if (!globalShortcut.isRegistered(combo)) {
|
||||
globalShortcut.register(combo, () => send('voice:toggle'));
|
||||
}
|
||||
} catch (_) { /* a taken shortcut just means no global hotkey; the pill still works */ }
|
||||
}
|
||||
};
|
||||
registerVoiceShortcut();
|
||||
app.on('browser-window-focus', () => { for (const combo of VOICE_COMBOS) { try { globalShortcut.unregister(combo); } catch (_) {} } });
|
||||
app.on('browser-window-blur', registerVoiceShortcut);
|
||||
}
|
||||
|
||||
const installVoiceHoldRelay = (contents) => {
|
||||
contents.on('before-input-event', (event, input) => {
|
||||
if (input.type !== 'keyDown' || input.isAutoRepeat) return;
|
||||
const isD = (input.code === 'KeyD' || (input.key || '').toLowerCase() === 'd');
|
||||
const combo = (isD && (input.meta || input.control) && input.shift) || input.code === 'F5';
|
||||
if (combo) {
|
||||
if (!nativeTapActive) send('voice:toggle');
|
||||
event.preventDefault();
|
||||
}
|
||||
});
|
||||
};
|
||||
// Installed via web-contents-created: the main window is born later in the whenReady sequence, and
|
||||
// webview guests swallow keys when a page has focus, so every window/guest gets the relay.
|
||||
app.on('web-contents-created', (event, contents) => {
|
||||
const t = contents.getType();
|
||||
if (t === 'window' || t === 'webview') installVoiceHoldRelay(contents);
|
||||
});
|
||||
|
||||
ipcMain.handle('voice:hold-capable', () => nativeTapActive);
|
||||
// Settings' "Hold to talk" can trigger the real macOS Accessibility prompt; a restart picks it up.
|
||||
ipcMain.handle('voice:request-hold-permission', () => {
|
||||
if (process.platform === 'darwin' && !nativeTapActive) {
|
||||
try { systemPreferences.isTrustedAccessibilityClient(true); } catch (_) {}
|
||||
}
|
||||
return nativeTapActive;
|
||||
});
|
||||
}
|
||||
|
||||
module.exports = { installVoiceHotkey };
|
||||
@@ -104,12 +104,18 @@ const GeneralInterface: React.FC<{
|
||||
<Box sx={inlineRowSx} {...settingSelectAttrs('voice_hold_to_talk', 'Dictation', 'Interface', 'Hold to talk, or tap to start and stop.')}>
|
||||
<Box sx={{ mr: 3 }}>
|
||||
<Typography sx={labelSx}>Dictation</Typography>
|
||||
<Typography sx={descSx}>How the mic button works. The mic key (F5) starts and stops.</Typography>
|
||||
<Typography sx={descSx}>How the mic button and the mic key (F5) work.</Typography>
|
||||
</Box>
|
||||
<ToggleButtonGroup
|
||||
value={form.voice_hold_to_talk ?? true}
|
||||
exclusive
|
||||
onChange={(_, v) => { if (v !== null) setForm({ ...form, voice_hold_to_talk: v }); }}
|
||||
onChange={(_, v) => {
|
||||
if (v === null) return;
|
||||
setForm({ ...form, voice_hold_to_talk: v });
|
||||
// Keyboard hold needs the native key tap; picking Hold on a Mac without the Accessibility
|
||||
// grant fires the system prompt so the choice can actually take effect after a relaunch.
|
||||
if (v) void window.openswarm?.voiceRequestHoldPermission?.();
|
||||
}}
|
||||
size="small"
|
||||
sx={{
|
||||
'& .MuiToggleButton-root': {
|
||||
|
||||
@@ -47,20 +47,18 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode
|
||||
}
|
||||
}, [holdMode, stop]);
|
||||
|
||||
// The hotkey (Cmd/Ctrl+Shift+D) is press-to-start / press-to-stop, NOT hold: macOS delivers
|
||||
// NEITHER the letter keyup nor the modifier releases to any Chromium layer while Cmd is held
|
||||
// (proven empirically: DOM saw zero events, main's before-input-event saw only the first keyDown),
|
||||
// so a keyboard hold-release is undetectable without a native event tap (uiohook class, the real
|
||||
// fix, needs a packaged native module). Hold-to-talk lives on the mic buttons, which DO see
|
||||
// pointerup. In-app presses arrive via main's before-input relay (works with webview focus and
|
||||
// swallows the 'd' so it never types into a field); background presses via the global shortcut.
|
||||
// Keyboard hotkey channels, matched to what the source can actually see:
|
||||
// voice:hold-down/up come ONLY from main's native uiohook tap (real global key-up, so the keyboard
|
||||
// gets the same hold-vs-toggle press semantics as the mic buttons); voice:toggle comes from the
|
||||
// fallback tier (globalShortcut / before-input relay), where key-ups are undetectable, so each
|
||||
// press toggles. Whichever tier main activated, the renderer just honors the channel it hears.
|
||||
useEffect(() => {
|
||||
const off = (window as unknown as { openswarm?: { onVoiceHold?: (d: () => void, u: () => void) => () => void } }).openswarm?.onVoiceHold?.(
|
||||
() => toggle(),
|
||||
() => {},
|
||||
);
|
||||
return () => { off?.(); };
|
||||
}, [toggle]);
|
||||
const bridge = window as unknown as {
|
||||
openswarm?: { onVoiceHold?: (d: () => void, u: () => void) => () => void };
|
||||
};
|
||||
const offHold = bridge.openswarm?.onVoiceHold?.(pressStart, pressEnd);
|
||||
return () => { offHold?.(); };
|
||||
}, [pressStart, pressEnd]);
|
||||
|
||||
return (
|
||||
<VoiceContext.Provider value={{ state, lastText, error, pct, feedback, toggle, pressStart, pressEnd, holdMode, volumeRef }}>
|
||||
|
||||
Vendored
+2
@@ -59,6 +59,8 @@ declare global {
|
||||
voiceTranscribe?: (wav: ArrayBuffer) => Promise<{ ok: boolean; text?: string; error?: string }>;
|
||||
voiceInject?: (text: string) => Promise<{ ok: boolean; pasted?: boolean; error?: string }>;
|
||||
onVoiceToggle?: (cb: () => void) => () => void;
|
||||
voiceHoldCapable?: () => Promise<boolean>;
|
||||
voiceRequestHoldPermission?: () => Promise<boolean>;
|
||||
onAuthUrl?: (cb: (url: string) => void) => () => void;
|
||||
onOauthClaim?: (cb: (url: string) => void) => () => void;
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user