// Streaming dictation over the SAME warm whisper-server the batch path uses: no engine swap, the // renderer streams PCM here and this loop re-decodes the current open phrase every ~1.2s so partials // appear live (openwhispr's preview-loop design). A phrase closed by the segmenter is decoded ONCE // and committed forever, so per-tick decode cost is O(open phrase), never O(whole utterance). // All state transitions are synchronous; only decodes are async and each carries the epoch it was // started under, so a stale decode can never rewrite a later segment's text. const whisperService = require('./whisperService'); const { createStreamSegmenter } = require('./streamSegmenter'); const PREVIEW_INTERVAL_MS = 1200; // openwhispr's gate: silence must never buy a decode (cost) or a hallucinated caption (worse). const PREVIEW_RMS_GATE = 0.002; const SAMPLE_RATE = 16000; // A speechless open buffer is trimmed so holding the hotkey in a quiet room can't grow memory forever. const SILENT_KEEP_BYTES = SAMPLE_RATE * 2 * 2; const SILENT_TRIM_BYTES = SAMPLE_RATE * 2 * 10; // Whisper captions non-speech in brackets/parens ("[ Background sounds ]", "(laughs)"); strip them so // neither the live preview nor a committed phrase ever carries a caption instead of dictation. function stripSoundCaptions(text) { return String(text || '').replace(/\[[^\]]*\]|\([^)]*\)|\*[^*]*\*|(?:^|\s)>>\s?/g, ' ').replace(/\s+/g, ' ').trim(); } function wavFromPcm16(pcm) { const buf = Buffer.alloc(44 + pcm.length); buf.write('RIFF', 0); buf.writeUInt32LE(36 + pcm.length, 4); buf.write('WAVE', 8); buf.write('fmt ', 12); buf.writeUInt32LE(16, 16); buf.writeUInt16LE(1, 20); buf.writeUInt16LE(1, 22); buf.writeUInt32LE(SAMPLE_RATE, 24); buf.writeUInt32LE(SAMPLE_RATE * 2, 28); buf.writeUInt16LE(2, 32); buf.writeUInt16LE(16, 34); buf.write('data', 36); buf.writeUInt32LE(pcm.length, 40); pcm.copy(buf, 44); return buf; } function createStreamingSession({ resourceDir, userDataDir, onPartial, previewIntervalMs = PREVIEW_INTERVAL_MS }) { let open = []; let openBytes = 0; const committed = []; let tentative = ''; let seq = 0; let closedDown = false; let degraded = false; let inflight = null; let skipNext = false; let sinceTickSumSq = 0; let sinceTickSamples = 0; let segEpoch = 0; // Segment finals must land in spoken order; preview decodes stay outside the chain, epoch-guarded. let commitChain = Promise.resolve(); const segmenter = createStreamSegmenter(SAMPLE_RATE); const timer = setInterval(() => { void previewTick(); }, previewIntervalMs); if (timer.unref) timer.unref(); function emit() { seq += 1; try { onPartial({ committed: committed.join(' ').trim(), tentative, seq }); } catch (_) { /* renderer gone */ } } function decodePcm(pcm) { return whisperService.transcribe(resourceDir, userDataDir, wavFromPcm16(pcm)); } async function previewTick() { if (closedDown || inflight) return; if (skipNext) { skipNext = false; return; } if (!openBytes || !segmenter.hadSpeech()) return; const rms = sinceTickSamples ? Math.sqrt(sinceTickSumSq / sinceTickSamples) : 0; sinceTickSumSq = 0; sinceTickSamples = 0; if (rms < PREVIEW_RMS_GATE) return; // nothing new was said; the last hypothesis stands const pcm = Buffer.concat(open); const epoch = segEpoch; const t0 = Date.now(); inflight = decodePcm(pcm) .then((text) => { if (closedDown || epoch !== segEpoch) return; // the segment closed mid-decode; its final wins tentative = stripSoundCaptions(text); emit(); }) .catch(() => { skipNext = true; }) .finally(() => { inflight = null; // FluidVoice back-pressure: a decode that overran its interval earns the next tick off. if (Date.now() - t0 > previewIntervalMs) skipNext = true; }); await inflight; } // Synchronously seals the open buffer into a segment, then decodes it once on the ordered chain. function closeOpenSegment() { if (!openBytes) return; const hadSpeech = segmenter.hadSpeech(); const pcm = Buffer.concat(open); open = []; openBytes = 0; segEpoch += 1; tentative = ''; segmenter.reset(); if (!hadSpeech) { emit(); return; } commitChain = commitChain.then(async () => { if (inflight) await inflight; try { const text = stripSoundCaptions(await decodePcm(pcm)); if (text) committed.push(text); } catch (_) { degraded = true; // a lost phrase final means the caller must fall back to the full-clip decode } emit(); }); } return { pushChunk(buf) { if (closedDown || !buf || !buf.length) return; open.push(buf); openBytes += buf.length; const i16 = new Int16Array(buf.buffer, buf.byteOffset, buf.length >> 1); for (let i = 0; i < i16.length; i++) { const v = i16[i] / 0x8000; sinceTickSumSq += v * v; } sinceTickSamples += i16.length; if (segmenter.push(i16) === 'boundary') { closeOpenSegment(); } else if (!segmenter.hadSpeech() && openBytes > SILENT_TRIM_BYTES) { while (openBytes - open[0].length >= SILENT_KEEP_BYTES) openBytes -= open.shift().length; } }, async stop() { if (closedDown) return { ok: false, error: 'stopped' }; clearInterval(timer); closeOpenSegment(); closedDown = true; await commitChain; return { ok: true, text: committed.join(' ').trim(), degraded }; }, cancel() { clearInterval(timer); closedDown = true; open = []; openBytes = 0; }, }; } module.exports = { createStreamingSession, wavFromPcm16, stripSoundCaptions };