From 26e2a8e6d7fc7f2a00ed27fb8da5b28e433d0427 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Tue, 4 Aug 2026 00:10:53 -0700 Subject: [PATCH] [eric] voice: streaming dictation engine, phrases commit once and preview re-decodes only the open phrase --- electron/main.js | 26 +++++ electron/package.json | 2 +- electron/preload.js | 10 ++ electron/voice/streamSegmenter.js | 67 ++++++++++++ electron/voice/streamingSession.js | 140 ++++++++++++++++++++++++++ electron/voice/streamingVoice.test.js | 102 +++++++++++++++++++ 6 files changed, 346 insertions(+), 1 deletion(-) create mode 100644 electron/voice/streamSegmenter.js create mode 100644 electron/voice/streamingSession.js create mode 100644 electron/voice/streamingVoice.test.js diff --git a/electron/main.js b/electron/main.js index bb4ad227..0321c3dd 100644 --- a/electron/main.js +++ b/electron/main.js @@ -1,5 +1,6 @@ const { app, components, BrowserWindow, ipcMain, shell, session, dialog, crashReporter, powerMonitor, Menu, clipboard, globalShortcut } = require('electron'); const whisperService = require('./voice/whisperService'); +const { createStreamingSession } = require('./voice/streamingSession'); const whisperModels = require('./voice/whisperModels'); const { injectText } = require('./voice/textInjector'); @@ -3054,6 +3055,31 @@ ipcMain.handle('voice:set-model', (_e, id) => { ipcMain.handle('voice:inject', async (_e, text) => { try { const pasted = await injectText(String(text || '')); return { ok: true, pasted }; } catch (err) { return { ok: false, error: String(err && err.message ? err.message : err) }; } }); +// Streaming dictation: renderer streams worklet PCM here; the session re-decodes the open phrase on +// the warm server and pushes live partials back. One session at a time; a new start evicts the old. +let voiceStream = null; +ipcMain.handle('voice:stream-start', () => { + if (voiceStream) voiceStream.cancel(); + voiceStream = createStreamingSession({ + resourceDir: voiceResourceDir(), + userDataDir: voiceUserDataDir(), + onPartial: (p) => { if (mainWindow && !mainWindow.isDestroyed()) mainWindow.webContents.send('voice:partial', p); }, + }); + return { ok: true }; +}); +ipcMain.on('voice:stream-chunk', (_e, chunk) => { + try { if (voiceStream) voiceStream.pushChunk(Buffer.from(chunk)); } catch (_) { /* a bad chunk never kills the session */ } +}); +ipcMain.handle('voice:stream-stop', async () => { + const s = voiceStream; + voiceStream = null; + if (!s) return { ok: false, error: 'no-session' }; + try { return await s.stop(); } catch (err) { return { ok: false, error: String(err && err.message ? err.message : err) }; } +}); +ipcMain.on('voice:stream-cancel', () => { + if (voiceStream) voiceStream.cancel(); + voiceStream = null; +}); // Sync mirrors so preload.js can expose window.openswarm synchronously (no await), closing the race where React renders before the async exposure resolves and window.openswarm is briefly undefined. backendPort is assigned in app.whenReady before any BrowserWindow is created, so it is always set by the time preload runs. ipcMain.on('get-backend-port-sync', (event) => { event.returnValue = backendPort; }); ipcMain.on('get-webview-preload-path-sync', (event) => { diff --git a/electron/package.json b/electron/package.json index 44e4a0a6..62236943 100644 --- a/electron/package.json +++ b/electron/package.json @@ -15,7 +15,7 @@ "dist:win": "electron-builder --win --x64 --publish never", "dist:win:publish": "electron-builder --win --x64 --publish always", "dist:all": "electron-builder --mac --win --linux", - "test": "node --test affiliateTracking.test.js updateErrorMessage.test.js", + "test": "node --test affiliateTracking.test.js updateErrorMessage.test.js voice/streamingVoice.test.js", "test:mouseclamp": "bash native/mouseclamp/run-tests.sh" }, "dependencies": { diff --git a/electron/preload.js b/electron/preload.js index a4eb1689..65459774 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -69,6 +69,16 @@ contextBridge.exposeInMainWorld('openswarm', { voiceSetModel: (id) => ipcRenderer.invoke('voice:set-model', id), voiceTranscribe: (wavArrayBuffer) => ipcRenderer.invoke('voice:transcribe', wavArrayBuffer), voiceInject: (text) => ipcRenderer.invoke('voice:inject', text), + // Streaming dictation: chunks flow up fire-and-forget, live partials flow back down. + voiceStreamStart: () => ipcRenderer.invoke('voice:stream-start'), + voiceStreamChunk: (pcmArrayBuffer) => ipcRenderer.send('voice:stream-chunk', pcmArrayBuffer), + voiceStreamStop: () => ipcRenderer.invoke('voice:stream-stop'), + voiceStreamCancel: () => ipcRenderer.send('voice:stream-cancel'), + onVoicePartial: (cb) => { + const listener = (_event, payload) => cb(payload); + ipcRenderer.on('voice:partial', listener); + return () => ipcRenderer.removeListener('voice:partial', listener); + }, onVoiceToggle: (cb) => { const listener = () => cb(); ipcRenderer.on('voice:toggle', listener); diff --git a/electron/voice/streamSegmenter.js b/electron/voice/streamSegmenter.js new file mode 100644 index 00000000..ef6e101e --- /dev/null +++ b/electron/voice/streamSegmenter.js @@ -0,0 +1,67 @@ +// Phrase boundaries for streaming dictation: close a segment after real speech plus a short pause, +// so preview re-decodes stay bounded to the current phrase and closed phrases are never re-decoded. +// Constants come from shipping code, not guesses: the 0.004 RMS / 0.02 peak speech test matches the +// renderer's endpointer (TypeWhisper + openwhispr values), the 250ms-speech / 600ms-silence commit +// window is TypeWhisper-Windows' LegacyVad, and the 30s force-commit is whisper's native window. + +const FRAME_MS = 20; +const SPEECH_RMS = 0.004; +const SPEECH_PEAK = 0.02; +const MIN_SPEECH_MS = 250; +const BOUNDARY_SILENCE_MS = 600; +const MAX_SEGMENT_MS = 30000; + +// Feed Int16 PCM chunks; 'boundary' means commit the open segment now. hadSpeech() reports whether +// the segment being closed ever contained real speech (a silence-only segment must never be decoded, +// that is the classic "Thank you for watching" hallucination generator). +function createStreamSegmenter(sampleRate) { + const frameSize = Math.max(1, Math.round((sampleRate * FRAME_MS) / 1000)); + let speechMs = 0; + let silenceMs = 0; + let elapsedMs = 0; + let sumSquares = 0; + let peak = 0; + let framed = 0; + + return { + push(samples) { + for (let i = 0; i < samples.length; i++) { + const v = samples[i] / 0x8000; + sumSquares += v * v; + const mag = v < 0 ? -v : v; + if (mag > peak) peak = mag; + if (++framed < frameSize) continue; + const rms = Math.sqrt(sumSquares / frameSize); + const isSpeech = rms >= SPEECH_RMS && peak >= SPEECH_PEAK; + sumSquares = 0; + peak = 0; + framed = 0; + elapsedMs += FRAME_MS; + if (isSpeech) { + speechMs += FRAME_MS; + silenceMs = 0; + } else if (speechMs >= MIN_SPEECH_MS) { + silenceMs += FRAME_MS; + } + if ((speechMs >= MIN_SPEECH_MS && silenceMs >= BOUNDARY_SILENCE_MS) || elapsedMs >= MAX_SEGMENT_MS) { + return 'boundary'; + } + } + return 'open'; + }, + hadSpeech() { + return speechMs >= MIN_SPEECH_MS; + }, + // A boundary was acted on: start counting the next segment from zero. + reset() { + speechMs = 0; + silenceMs = 0; + elapsedMs = 0; + sumSquares = 0; + peak = 0; + framed = 0; + }, + }; +} + +module.exports = { createStreamSegmenter }; diff --git a/electron/voice/streamingSession.js b/electron/voice/streamingSession.js new file mode 100644 index 00000000..0eb0f364 --- /dev/null +++ b/electron/voice/streamingSession.js @@ -0,0 +1,140 @@ +// Streaming dictation over the SAME warm whisper-server the batch path uses: no engine swap, the +// renderer streams PCM here and this loop re-decodes the current open phrase every ~1.2s so partials +// appear live (openwhispr's preview-loop design). A phrase closed by the segmenter is decoded ONCE +// and committed forever, so per-tick decode cost is O(open phrase), never O(whole utterance). +// All state transitions are synchronous; only decodes are async and each carries the epoch it was +// started under, so a stale decode can never rewrite a later segment's text. + +const whisperService = require('./whisperService'); +const { createStreamSegmenter } = require('./streamSegmenter'); + +const PREVIEW_INTERVAL_MS = 1200; +// openwhispr's gate: silence must never buy a decode (cost) or a hallucinated caption (worse). +const PREVIEW_RMS_GATE = 0.002; +const SAMPLE_RATE = 16000; +// A speechless open buffer is trimmed so holding the hotkey in a quiet room can't grow memory forever. +const SILENT_KEEP_BYTES = SAMPLE_RATE * 2 * 2; +const SILENT_TRIM_BYTES = SAMPLE_RATE * 2 * 10; + +function wavFromPcm16(pcm) { + const buf = Buffer.alloc(44 + pcm.length); + buf.write('RIFF', 0); buf.writeUInt32LE(36 + pcm.length, 4); buf.write('WAVE', 8); + buf.write('fmt ', 12); buf.writeUInt32LE(16, 16); buf.writeUInt16LE(1, 20); buf.writeUInt16LE(1, 22); + buf.writeUInt32LE(SAMPLE_RATE, 24); buf.writeUInt32LE(SAMPLE_RATE * 2, 28); buf.writeUInt16LE(2, 32); buf.writeUInt16LE(16, 34); + buf.write('data', 36); buf.writeUInt32LE(pcm.length, 40); + pcm.copy(buf, 44); + return buf; +} + +function createStreamingSession({ resourceDir, userDataDir, onPartial, previewIntervalMs = PREVIEW_INTERVAL_MS }) { + let open = []; + let openBytes = 0; + const committed = []; + let tentative = ''; + let seq = 0; + let closedDown = false; + let degraded = false; + let inflight = null; + let skipNext = false; + let sinceTickSumSq = 0; + let sinceTickSamples = 0; + let segEpoch = 0; + // Segment finals must land in spoken order; preview decodes stay outside the chain, epoch-guarded. + let commitChain = Promise.resolve(); + const segmenter = createStreamSegmenter(SAMPLE_RATE); + const timer = setInterval(() => { void previewTick(); }, previewIntervalMs); + if (timer.unref) timer.unref(); + + function emit() { + seq += 1; + try { onPartial({ committed: committed.join(' ').trim(), tentative, seq }); } catch (_) { /* renderer gone */ } + } + + function decodePcm(pcm) { + return whisperService.transcribe(resourceDir, userDataDir, wavFromPcm16(pcm)); + } + + async function previewTick() { + if (closedDown || inflight) return; + if (skipNext) { skipNext = false; return; } + if (!openBytes || !segmenter.hadSpeech()) return; + const rms = sinceTickSamples ? Math.sqrt(sinceTickSumSq / sinceTickSamples) : 0; + sinceTickSumSq = 0; + sinceTickSamples = 0; + if (rms < PREVIEW_RMS_GATE) return; // nothing new was said; the last hypothesis stands + const pcm = Buffer.concat(open); + const epoch = segEpoch; + const t0 = Date.now(); + inflight = decodePcm(pcm) + .then((text) => { + if (closedDown || epoch !== segEpoch) return; // the segment closed mid-decode; its final wins + tentative = text; + emit(); + }) + .catch(() => { skipNext = true; }) + .finally(() => { + inflight = null; + // FluidVoice back-pressure: a decode that overran its interval earns the next tick off. + if (Date.now() - t0 > previewIntervalMs) skipNext = true; + }); + await inflight; + } + + // Synchronously seals the open buffer into a segment, then decodes it once on the ordered chain. + function closeOpenSegment() { + if (!openBytes) return; + const hadSpeech = segmenter.hadSpeech(); + const pcm = Buffer.concat(open); + open = []; + openBytes = 0; + segEpoch += 1; + tentative = ''; + segmenter.reset(); + if (!hadSpeech) { emit(); return; } + commitChain = commitChain.then(async () => { + if (inflight) await inflight; + try { + const text = await decodePcm(pcm); + if (text) committed.push(text); + } catch (_) { + degraded = true; // a lost phrase final means the caller must fall back to the full-clip decode + } + emit(); + }); + } + + return { + pushChunk(buf) { + if (closedDown || !buf || !buf.length) return; + open.push(buf); + openBytes += buf.length; + const i16 = new Int16Array(buf.buffer, buf.byteOffset, buf.length >> 1); + for (let i = 0; i < i16.length; i++) { + const v = i16[i] / 0x8000; + sinceTickSumSq += v * v; + } + sinceTickSamples += i16.length; + if (segmenter.push(i16) === 'boundary') { + closeOpenSegment(); + } else if (!segmenter.hadSpeech() && openBytes > SILENT_TRIM_BYTES) { + while (openBytes - open[0].length >= SILENT_KEEP_BYTES) openBytes -= open.shift().length; + } + }, + async stop() { + if (closedDown) return { ok: false, error: 'stopped' }; + clearInterval(timer); + closeOpenSegment(); + closedDown = true; + await commitChain; + return { ok: true, text: committed.join(' ').trim(), degraded }; + }, + cancel() { + clearInterval(timer); + closedDown = true; + open = []; + openBytes = 0; + }, + }; +} + +module.exports = { createStreamingSession, wavFromPcm16 }; diff --git a/electron/voice/streamingVoice.test.js b/electron/voice/streamingVoice.test.js new file mode 100644 index 00000000..5aa24dcc --- /dev/null +++ b/electron/voice/streamingVoice.test.js @@ -0,0 +1,102 @@ +const { test } = require('node:test'); +const assert = require('node:assert'); +const { createStreamSegmenter } = require('./streamSegmenter'); +const whisperService = require('./whisperService'); +const { createStreamingSession, wavFromPcm16 } = require('./streamingSession'); + +const RATE = 16000; + +function tone(ms, amplitude = 3000) { + const out = new Int16Array(Math.round((RATE * ms) / 1000)); + for (let i = 0; i < out.length; i++) out[i] = Math.round(Math.sin(i / 8) * amplitude); + return out; +} + +function silence(ms) { + return new Int16Array(Math.round((RATE * ms) / 1000)); +} + +function asBuffer(i16) { + return Buffer.from(i16.buffer, i16.byteOffset, i16.byteLength); +} + +test('segmenter: speech then a pause is a boundary; silence alone never is', () => { + const seg = createStreamSegmenter(RATE); + assert.strictEqual(seg.push(silence(5000)), 'open'); + assert.strictEqual(seg.hadSpeech(), false); + assert.strictEqual(seg.push(tone(400)), 'open'); + assert.strictEqual(seg.hadSpeech(), true); + assert.strictEqual(seg.push(silence(700)), 'boundary'); +}); + +test('segmenter: reset starts the next phrase from zero', () => { + const seg = createStreamSegmenter(RATE); + seg.push(tone(400)); + seg.push(silence(700)); + seg.reset(); + assert.strictEqual(seg.hadSpeech(), false); + assert.strictEqual(seg.push(silence(2000)), 'open'); +}); + +test('wavFromPcm16 writes a valid 16kHz mono RIFF header', () => { + const wav = wavFromPcm16(asBuffer(tone(100))); + assert.strictEqual(wav.toString('ascii', 0, 4), 'RIFF'); + assert.strictEqual(wav.readUInt32LE(24), RATE); + assert.strictEqual(wav.readUInt16LE(22), 1); + assert.strictEqual(wav.readUInt32LE(40), wav.length - 44); +}); + +function stubTranscribe(fn) { + const real = whisperService.transcribe; + whisperService.transcribe = fn; + return () => { whisperService.transcribe = real; }; +} + +test('session: phrase boundaries commit in order and stop() joins them', async () => { + let calls = 0; + const restore = stubTranscribe(async () => { calls += 1; return `phrase${calls}`; }); + const partials = []; + const s = createStreamingSession({ resourceDir: '', userDataDir: '', onPartial: (p) => partials.push(p), previewIntervalMs: 3600000 }); + s.pushChunk(asBuffer(tone(400))); + s.pushChunk(asBuffer(silence(700))); + s.pushChunk(asBuffer(tone(400))); + const out = await s.stop(); + restore(); + assert.strictEqual(out.ok, true); + assert.strictEqual(out.text, 'phrase1 phrase2'); + assert.strictEqual(out.degraded, false); + assert.strictEqual(partials[partials.length - 1].committed, 'phrase1 phrase2'); + const seqs = partials.map((p) => p.seq); + assert.deepStrictEqual(seqs, [...seqs].sort((a, b) => a - b)); +}); + +test('session: a silence-only recording never buys a decode', async () => { + let calls = 0; + const restore = stubTranscribe(async () => { calls += 1; return 'hallucination'; }); + const s = createStreamingSession({ resourceDir: '', userDataDir: '', onPartial: () => {}, previewIntervalMs: 3600000 }); + s.pushChunk(asBuffer(silence(3000))); + const out = await s.stop(); + restore(); + assert.strictEqual(calls, 0); + assert.strictEqual(out.text, ''); +}); + +test('session: a failed segment decode marks the result degraded', async () => { + const restore = stubTranscribe(async () => { throw new Error('server-timeout'); }); + const s = createStreamingSession({ resourceDir: '', userDataDir: '', onPartial: () => {}, previewIntervalMs: 3600000 }); + s.pushChunk(asBuffer(tone(400))); + const out = await s.stop(); + restore(); + assert.strictEqual(out.ok, true); + assert.strictEqual(out.degraded, true); +}); + +test('session: chunks after cancel are dropped and stop reports stopped', async () => { + const restore = stubTranscribe(async () => 'never'); + const s = createStreamingSession({ resourceDir: '', userDataDir: '', onPartial: () => {}, previewIntervalMs: 3600000 }); + s.cancel(); + s.pushChunk(asBuffer(tone(400))); + const out = await s.stop(); + restore(); + assert.strictEqual(out.ok, false); +});