From 7baba619cf9073a7addaf8b38cf2ab2aab7c2fbf Mon Sep 17 00:00:00 2001 From: ciregenz Date: Tue, 21 Jul 2026 23:06:58 -0700 Subject: [PATCH] [eric] voice: local whisper.cpp dictation engine (warm whisper-server + transcribe/inject/hotkey IPC) --- electron/main.js | 42 +++++++++++- electron/preload.js | 10 +++ electron/voice/textInjector.js | 39 +++++++++++ electron/voice/whisperService.js | 112 +++++++++++++++++++++++++++++++ 4 files changed, 202 insertions(+), 1 deletion(-) create mode 100644 electron/voice/textInjector.js create mode 100644 electron/voice/whisperService.js diff --git a/electron/main.js b/electron/main.js index f0116c14..9bb6bc8a 100644 --- a/electron/main.js +++ b/electron/main.js @@ -1,4 +1,6 @@ -const { app, components, BrowserWindow, ipcMain, shell, session, dialog, crashReporter, powerMonitor, Menu, clipboard } = require('electron'); +const { app, components, BrowserWindow, ipcMain, shell, session, dialog, crashReporter, powerMonitor, Menu, clipboard, globalShortcut } = require('electron'); +const whisperService = require('./voice/whisperService'); +const { injectText } = require('./voice/textInjector'); // Browser cards live in their own persistent partition so cookies/localStorage/IndexedDB survive reload + quit (Discord etc. stay logged in) and site data stays isolated from the app's defaultSession. The "clear browsing data" wipe nukes only this partition. MUST match BROWSER_PARTITION in frontend BrowserCard.tsx. const BROWSER_PARTITION = 'persist:openswarm-browser'; @@ -1786,6 +1788,14 @@ app.whenReady().then(async () => { // Off-window mouse-release crash dodge (macOS). Safe to call before windows exist. installMacMouseClamp(); + // Voice dictation toggle: press anywhere (even in another app) to start/stop dictating. globalShortcut + // can't see key-up, so this is a press-to-toggle, not hold-to-talk; the renderer owns the record state. + try { + globalShortcut.register('CommandOrControl+Shift+D', () => { + if (mainWindow && !mainWindow.isDestroyed()) mainWindow.webContents.send('voice:toggle'); + }); + } catch (_) { /* a taken shortcut just means no global hotkey; the pill still works */ } + // PASSKEY SPIKE (macOS only): turn on the Secure-Enclave/Touch ID WebAuthn authenticator that Electron 42 added. Without this, isUserVerifyingPlatformAuthenticatorAvailable() is hardwired false (why the old reject-shim existed). keychainAccessGroup MUST match the keychain-access-groups entitlement (Y26NUZH4NG..webauthn) or this throws. Windows has no equivalent, so the reject-shim still runs there. if (process.platform === 'darwin' && typeof app.configureWebAuthn === 'function') { try { @@ -2773,6 +2783,8 @@ app.on('before-quit', async (event) => { app.on('will-quit', () => { if (!isDev) killBackend(); + try { globalShortcut.unregisterAll(); } catch (_) {} + try { whisperService.stopServer(); } catch (_) {} }); app.on('activate', () => { @@ -2857,6 +2869,34 @@ ipcMain.handle = (channel, handler) => { }; ipcMain.handle('get-backend-port', () => backendPort); + +// ---- Voice dictation (local whisper.cpp) ---- +function voiceResourceDir() { + return getResourcePath('whisper'); +} +function voiceUserDataDir() { + try { return app.getPath('userData'); } catch (_) { return __dirname; } +} +// Renderer records the mic, encodes a 16kHz-mono WAV, and hands us the bytes; we run them through the +// warm whisper server and return text. Fail-soft: any error becomes { ok:false } so the pill can show +// a clean "couldn't hear that" instead of the app throwing. +ipcMain.handle('voice:transcribe', async (_e, wavArrayBuffer) => { + try { + const buf = Buffer.from(wavArrayBuffer); + const text = await whisperService.transcribe(voiceResourceDir(), voiceUserDataDir(), buf); + return { ok: true, text }; + } catch (err) { + return { ok: false, error: String(err && err.message ? err.message : err) }; + } +}); +// Warm the model ahead of the first phrase so dictation feels instant, not "1s to boot then type". +ipcMain.handle('voice:warmup', async () => { + try { await whisperService.ensureServer(voiceResourceDir(), voiceUserDataDir()); return { ok: true }; } catch (err) { return { ok: false, error: String(err && err.message ? err.message : err) }; } +}); +// Paste the text into the frontmost app (dictate-anywhere). Returns whether the OS paste actually fired. +ipcMain.handle('voice:inject', async (_e, text) => { + try { const pasted = await injectText(String(text || '')); return { ok: true, pasted }; } catch (err) { return { ok: false, error: String(err && err.message ? err.message : err) }; } +}); // Sync mirrors so preload.js can expose window.openswarm synchronously (no await), closing the race where React renders before the async exposure resolves and window.openswarm is briefly undefined. backendPort is assigned in app.whenReady before any BrowserWindow is created, so it is always set by the time preload runs. ipcMain.on('get-backend-port-sync', (event) => { event.returnValue = backendPort; }); ipcMain.on('get-webview-preload-path-sync', (event) => { diff --git a/electron/preload.js b/electron/preload.js index 7e814bee..c21fc629 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -62,6 +62,16 @@ contextBridge.exposeInMainWorld('openswarm', { // Clears cookies/cache/localStorage for the browser-card partition only (never the app's defaultSession). Logs you out of sites opened in browser cards. clearBrowserData: () => ipcRenderer.invoke('browser:clear-data'), connectSlack: () => ipcRenderer.invoke('connect-slack'), + // Voice dictation (local whisper.cpp). transcribe takes a 16kHz-mono WAV ArrayBuffer; inject pastes + // text into the frontmost app; warmup pre-loads the model; onVoiceToggle fires on the global hotkey. + voiceWarmup: () => ipcRenderer.invoke('voice:warmup'), + voiceTranscribe: (wavArrayBuffer) => ipcRenderer.invoke('voice:transcribe', wavArrayBuffer), + voiceInject: (text) => ipcRenderer.invoke('voice:inject', text), + onVoiceToggle: (cb) => { + const listener = () => cb(); + ipcRenderer.on('voice:toggle', listener); + return () => ipcRenderer.removeListener('voice:toggle', listener); + }, // Hands a vetted social platform's partition cookies to its session-backed MCP shim (allowlisted domains only, gated again in the main process). getPartitionCookies: (domain) => ipcRenderer.invoke('get-partition-cookies', domain), // Silently reads the user's own chatgpt.com/claude.ai history offscreen (no card) for onboarding personalization; main owns the injected script + gates the provider. diff --git a/electron/voice/textInjector.js b/electron/voice/textInjector.js new file mode 100644 index 00000000..575d73e7 --- /dev/null +++ b/electron/voice/textInjector.js @@ -0,0 +1,39 @@ +// Drop transcribed text into whatever app currently has focus, WhisperFlow-style. We can't synthesize +// raw keystrokes from a sandboxed renderer, so the durable trick is the clipboard: stash the user's +// existing clipboard, write our text, fire the OS paste chord, then restore the clipboard a beat later +// so we don't clobber what they had. macOS paste needs Accessibility permission (same wall clicky hits). + +const { clipboard } = require('electron'); +const { exec } = require('child_process'); + +function pasteFrontmost() { + return new Promise((resolve) => { + if (process.platform === 'darwin') { + exec('osascript -e \'tell application "System Events" to keystroke "v" using command down\'', (err) => resolve(!err)); + } else if (process.platform === 'win32') { + // SendWait "^v" = Ctrl+V into the focused control. + exec('powershell -NoProfile -Command "Add-Type -AssemblyName System.Windows.Forms; [System.Windows.Forms.SendKeys]::SendWait(\'^v\')"', (err) => resolve(!err)); + } else { + resolve(false); + } + }); +} + +// Write text to the clipboard and paste it into the focused field, then put the old clipboard back so +// dictation is non-destructive. Returns false if we couldn't fire the paste (e.g. no Accessibility grant). +async function injectText(text) { + if (!text) return false; + const previous = clipboard.readText(); + clipboard.writeText(text); + const pasted = await pasteFrontmost(); + // Restore after the paste has had time to read the clipboard. If the paste failed the text stays on + // the clipboard so the user can paste it by hand rather than losing the dictation entirely. + if (pasted) { + setTimeout(() => { + try { if (clipboard.readText() === text) clipboard.writeText(previous); } catch (_) {} + }, 400); + } + return pasted; +} + +module.exports = { injectText }; diff --git a/electron/voice/whisperService.js b/electron/voice/whisperService.js new file mode 100644 index 00000000..c9f4a452 --- /dev/null +++ b/electron/voice/whisperService.js @@ -0,0 +1,112 @@ +// Local speech-to-text via whisper.cpp, kept WARM so a phrase transcribes in ~0.2s instead of the +// ~16s cold-model-load a fresh CLI pays every time. We spawn `whisper-server` once (model loaded), +// then POST audio to it per utterance. Same "bundle a binary + manage its lifecycle" shape as the +// 9router subprocess: dev uses the system whisper.cpp, prod uses the per-arch binary + model we ship. + +const { spawn } = require('child_process'); +const path = require('path'); +const fs = require('fs'); + +const MODEL_FILE = 'ggml-base.en.bin'; + +// Resolve the whisper-server binary. Env override wins (dev convenience), then the bundled per-arch +// copy, then whatever is on PATH so a dev machine with `brew install whisper-cpp` just works. +function resolveBinary(resourceDir) { + if (process.env.OPENSWARM_WHISPER_BIN && fs.existsSync(process.env.OPENSWARM_WHISPER_BIN)) { + return process.env.OPENSWARM_WHISPER_BIN; + } + const exe = process.platform === 'win32' ? 'whisper-server.exe' : 'whisper-server'; + const bundled = path.join(resourceDir, exe); + if (fs.existsSync(bundled)) return bundled; + const brew = process.platform === 'win32' ? null : '/opt/homebrew/bin/whisper-server'; + if (brew && fs.existsSync(brew)) return brew; + return exe; // last resort: hope it is on PATH +} + +// Resolve the model file. Env override, then bundled, then a dev cache under the app's data dir. +function resolveModel(resourceDir, userDataDir) { + if (process.env.OPENSWARM_WHISPER_MODEL && fs.existsSync(process.env.OPENSWARM_WHISPER_MODEL)) { + return process.env.OPENSWARM_WHISPER_MODEL; + } + const bundled = path.join(resourceDir, MODEL_FILE); + if (fs.existsSync(bundled)) return bundled; + const cached = path.join(userDataDir, 'whisper', MODEL_FILE); + if (fs.existsSync(cached)) return cached; + return null; +} + +let proc = null; +let port = 0; +let readyPromise = null; + +function pickPort() { + // Fixed-ish high port; whisper-server has no ephemeral-port reporting, so we pick and probe. + return 8300 + Math.floor(Math.random() * 400); +} + +async function waitForReady(p, timeoutMs) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + try { + const res = await fetch(`http://127.0.0.1:${p}/`, { method: 'GET' }); + if (res.status) return true; // any HTTP answer means the socket is serving + } catch (_) { /* not up yet */ } + await new Promise((r) => setTimeout(r, 150)); + } + return false; +} + +// Boot the warm server once. resourceDir = where a packaged build put the binary+model; userDataDir +// = app.getPath('userData') for the dev cache. Returns the port, or throws with an actionable reason. +async function ensureServer(resourceDir, userDataDir) { + if (proc && port) return port; + if (readyPromise) return readyPromise; + readyPromise = (async () => { + const bin = resolveBinary(resourceDir); + const model = resolveModel(resourceDir, userDataDir); + if (!model) { + readyPromise = null; + throw new Error('no-model'); // caller surfaces a "voice model missing" state, never crashes + } + const p = pickPort(); + const child = spawn(bin, ['-m', model, '--port', String(p), '-nt', '--convert'], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.on('error', () => { proc = null; port = 0; }); + child.on('exit', () => { proc = null; port = 0; readyPromise = null; }); + const ok = await waitForReady(p, 20000); + if (!ok) { + try { child.kill(); } catch (_) {} + readyPromise = null; + throw new Error('server-timeout'); + } + proc = child; + port = p; + return p; + })(); + return readyPromise; +} + +// Transcribe a 16kHz-mono WAV buffer to text. The renderer records + encodes the WAV so the audio +// never crosses a CORS boundary; we POST from the main process where there is none. +async function transcribe(resourceDir, userDataDir, wavBuffer) { + const p = await ensureServer(resourceDir, userDataDir); + const form = new FormData(); + form.append('file', new Blob([wavBuffer], { type: 'audio/wav' }), 'audio.wav'); + form.append('response_format', 'text'); + const res = await fetch(`http://127.0.0.1:${p}/inference`, { method: 'POST', body: form }); + if (!res.ok) throw new Error(`whisper-http-${res.status}`); + const text = (await res.text()).trim(); + return text; +} + +function stopServer() { + if (proc) { + try { proc.kill(); } catch (_) {} + } + proc = null; + port = 0; + readyPromise = null; +} + +module.exports = { ensureServer, transcribe, stopServer, resolveBinary, resolveModel };