diff --git a/electron/voice/whisperService.js b/electron/voice/whisperService.js index d43243da..baf7c03b 100644 --- a/electron/voice/whisperService.js +++ b/electron/voice/whisperService.js @@ -141,7 +141,7 @@ function p_reasonFrom(tail) { return pick ? `: ${pick.slice(0, 200)}` : ''; } -async function p_bootServer(resourceDir, userDataDir) { +async function p_bootServer(resourceDir, userDataDir, extended = true) { const bin = resolveBinary(resourceDir); const model = resolveModel(resourceDir, userDataDir); if (!model) { @@ -153,9 +153,12 @@ async function p_bootServer(resourceDir, userDataDir) { p_sweepStrays(bin); const p = await freePort(); // No --convert: our WAV is already 16kHz mono, and the flag makes whisper demand ffmpeg on PATH at boot; a Finder-launched app has no brew PATH, so it exited before ever binding the port. - // Beam search (5) over greedy: measurably fewer recognition errors at ~1.5x decode cost; Eric - // rates accuracy above stop-latency. + // Decode setup from the OSS-dictation survey (VoiceTypr/VoiceInk consensus): beam 5 over greedy, + // suppress-nst kills non-speech captions AT the decoder, no-context stops cross-segment + // hallucination carryover, flash-attn is a free Metal win. extended=false retries with the + // minimal set so an older binary missing a flag can never kill dictation. const args = ['-m', model, '--port', String(p), '-nt', '-bs', '5']; + if (extended) args.push('--suppress-nst', '--no-context', '--flash-attn'); // A multilingual model (no .en in the filename) auto-detects the spoken language per utterance. if (!path.basename(model).includes('.en')) args.push('-l', 'auto'); const child = spawn(bin, args, { @@ -184,6 +187,11 @@ async function p_bootServer(resourceDir, userDataDir) { if (!ok) { bootingChild = null; try { child.kill('SIGKILL'); } catch (_) {} + // An instantly-dead child with the extended flags is probably an older binary: retry minimal. + if (extended && (child.exitCode !== null || child.signalCode !== null)) { + console.log('[voice] extended decode flags rejected; retrying with the minimal set'); + return p_bootServer(resourceDir, userDataDir, false); + } // A dead child is not a slow one. Whisper can die in ~0.1s with exit code 0 (a missing ffmpeg on // a Finder-launched PATH does exactly that), so report ITS reason instantly instead of making the // user sit through the full ready budget for a process that was never coming back. @@ -237,6 +245,10 @@ async function transcribe(resourceDir, userDataDir, wavBuffer) { const form = new FormData(); form.append('file', new Blob([wavBuffer], { type: 'audio/wav' }), 'audio.wav'); form.append('response_format', 'text'); + // 0.2 + 0.2 fallback ladder is what VoiceTypr and VoiceInk ship; whisper's 0.0 greedy start + // retries into hallucination on marginal audio. + form.append('temperature', '0.2'); + form.append('temperature_inc', '0.2'); if (dictionaryPrompt) form.append('prompt', dictionaryPrompt); const res = await fetch(`http://127.0.0.1:${p}/inference`, { method: 'POST', body: form }); if (!res.ok) throw new Error(`whisper-http-${res.status}`); diff --git a/frontend/src/shared/voice/useVoiceDictation.ts b/frontend/src/shared/voice/useVoiceDictation.ts index 7d0832eb..e38e1b61 100644 --- a/frontend/src/shared/voice/useVoiceDictation.ts +++ b/frontend/src/shared/voice/useVoiceDictation.ts @@ -6,7 +6,7 @@ import { playVoiceCue } from './voiceCues'; import { injectAtFocus } from './injectAtFocus'; import { createSilenceDetector } from './createSilenceDetector'; import { pushDictation } from './voiceHistory'; -import { learnFromTranscript } from './voiceDictionary'; +import { learnFromTranscript, isDictionaryEcho } from './voiceDictionary'; import { getFocusedSurfaceHost, surfaceDisabled } from './voiceSurface'; import { store } from '@/shared/state/store'; import { createCaptureNode } from './createCaptureNode'; @@ -168,7 +168,11 @@ export function useVoiceDictation() { // Fire the OS mic prompt through the main process first: a packaged hardened-runtime build denies renderer getUserMedia outright until TCC granted (the prod dictation-dead cause, ENG-103). const micOk = await (window.openswarm as any)?.voiceRequestMicAccess?.() ?? true; if (micOk === false) { setError('mic-denied'); return; } - stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } }); + // autoGainControl OFF is the OSS-dictation consensus (Chromium's hidden AGC rides the mic and + // garbles levels mid-utterance; none of the five surveyed shipping apps allow any AGC), and + // noiseSuppression smears speech whisper handles better raw. Echo cancellation stays: we play + // cues out the speakers while the mic is hot. + stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: false, autoGainControl: false } }); ctx = new AudioContext({ sampleRate: VOICE_SAMPLE_RATE }); const source = ctx.createMediaStreamSource(stream); const chunks: Float32Array[] = []; @@ -239,20 +243,37 @@ export function useVoiceDictation() { } let res: { ok: boolean; text?: string; error?: string } | undefined; { - // Whisper hallucinates plausible punctuation on near-silence; a clip whose level never beat the streaming RMS gate gets "didn't catch that", never a decode. - let sumSq = 0; - for (let i = 0; i < samples.length; i += 4) sumSq += samples[i] * samples[i]; - const rms = Math.sqrt(sumSq / Math.max(1, Math.floor(samples.length / 4))); - if (rms < 0.002) { + // OpenWhispr's window gate: whole-clip RMS misses a clip that is silence plus one cough, so + // judge 100ms windows; no window with real speech energy means no decode, ever. + const win = Math.round(VOICE_SAMPLE_RATE * 0.1); + let peakRms = 0; + let speechWindow = false; + for (let off = 0; off + win <= samples.length; off += win) { + let sumSq = 0; + let peak = 0; + for (let i = off; i < off + win; i += 2) { const v = samples[i]; sumSq += v * v; if (Math.abs(v) > peak) peak = Math.abs(v); } + const wr = Math.sqrt(sumSq / (win / 2)); + if (wr > peakRms) peakRms = wr; + if (wr >= 0.003 && peak >= 0.02) speechWindow = true; + } + if (peakRms < 0.002 || (!speechWindow && peakRms < 0.006)) { res = { ok: true, text: '' }; } else { - const wav = encodeWav(samples); + // whisper.cpp asserts on sub-second buffers; zero-pad instead of rejecting (FluidVoice). + const padded = samples.length < VOICE_SAMPLE_RATE + ? (() => { const b = new Float32Array(VOICE_SAMPLE_RATE); b.set(samples); return b; })() + : samples; + const wav = encodeWav(padded); res = await window.openswarm?.voiceTranscribe?.(wav); if ((!res || !res.ok || !res.text) && streamedFallback) res = { ok: true, text: streamedFallback }; } } // Whisper captions non-speech in brackets/parens ("[ Background sounds ]", "(laughs)") and marks speaker turns with ">>"; those are annotations, not dictation. if (res?.ok && res.text) res = { ok: true, text: res.text.replace(/\[[^\]]*\]|\([^)]*\)|\*[^*]*\*|(?:^|\s)>>\s?/g, ' ').replace(/\s+/g, ' ').trim() }; + // Stutter collapse (Handy): three or more consecutive identical words are one decode loop, not speech. + if (res?.ok && res.text) res = { ok: true, text: res.text.replace(/\b([A-Za-z']+)(\s+\1\b){2,}/gi, '$1') }; + // Glossary echo (OpenWhispr): a transcript that is mostly the dictionary read back is the prompt leaking, not dictation. + if (res?.ok && res.text && isDictionaryEcho(res.text)) res = { ok: true, text: '' }; // A transcript with no letter or digit in it is a hallucination artifact (the lone comma), not dictation. if (res?.ok && res.text && !/[\p{L}\p{N}]/u.test(res.text)) res = { ok: true, text: '' }; // Wispr's command grammar, v1: saying only "scratch that" (or "delete/cancel that") throws the take away. diff --git a/frontend/src/shared/voice/voiceDictionary.ts b/frontend/src/shared/voice/voiceDictionary.ts index a90d5d1f..816d99f5 100644 --- a/frontend/src/shared/voice/voiceDictionary.ts +++ b/frontend/src/shared/voice/voiceDictionary.ts @@ -32,6 +32,7 @@ function pushMerged(): void { .map(([w]) => w); const manualWords = manual.split(',').map((w) => w.trim()).filter(Boolean); const merged = [...new Set([...manualWords, ...learned])].join(', '); + lastPushedMerged = merged; const bridge = window as unknown as { openswarm?: { voiceSetDictionary?: (words: string) => void } }; bridge.openswarm?.voiceSetDictionary?.(merged); } @@ -41,6 +42,21 @@ export function setManualDictionary(words: string): void { pushMerged(); } +let lastPushedMerged = ''; + +// OpenWhispr's echo test: the model sometimes reads the glossary prompt back as the "transcript". +// Mostly-dictionary words (90%) covering most of the dictionary (70%) = the prompt leaked. +export function isDictionaryEcho(text: string): boolean { + const dictWords = new Set(lastPushedMerged.toLowerCase().split(/[,\s]+/).filter(Boolean)); + if (dictWords.size === 0) return false; + const words = text.toLowerCase().split(/\s+/).map((w) => w.replace(/[^a-z']/g, '')).filter(Boolean); + if (words.length === 0) return false; + const unique = [...new Set(words)]; + const fromDict = unique.filter((w) => dictWords.has(w)).length; + const coverage = [...dictWords].filter((w) => words.includes(w)).length / dictWords.size; + return fromDict / unique.length >= 0.9 && coverage >= 0.7; +} + export function learnFromTranscript(text: string): void { try { const counts = readLearned();