WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-08-17 18:25:42 +02:00
Code Issues Packages Projects Releases Wiki Activity

[eric] voice: survey-grade accuracy layer, Chromium AGC off, suppress-nst+no-context+flash-attn+temp ladder, window energy gate, sub-second zero-pad, stutter and glossary-echo filters

Browse Source
This commit is contained in:
ciregenz
2026-08-06 00:22:32 -07:00
parent eb69fcaf84
commit ef594f737d
3 changed files with 60 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files
electron/voice/whisperService.js
+15 -3
View File
@@ -141,7 +141,7 @@ function p_reasonFrom(tail) {
return pick ? `: ${pick.slice(0, 200)}` : '';
}
async function p_bootServer(resourceDir, userDataDir) {
async function p_bootServer(resourceDir, userDataDir, extended = true) {
const bin = resolveBinary(resourceDir);
const model = resolveModel(resourceDir, userDataDir);
if (!model) {
@@ -153,9 +153,12 @@ async function p_bootServer(resourceDir, userDataDir) {
p_sweepStrays(bin);
const p = await freePort();
// No --convert: our WAV is already 16kHz mono, and the flag makes whisper demand ffmpeg on PATH at boot; a Finder-launched app has no brew PATH, so it exited before ever binding the port.
// Beam search (5) over greedy: measurably fewer recognition errors at ~1.5x decode cost; Eric
// rates accuracy above stop-latency.
// Decode setup from the OSS-dictation survey (VoiceTypr/VoiceInk consensus): beam 5 over greedy,
// suppress-nst kills non-speech captions AT the decoder, no-context stops cross-segment
// hallucination carryover, flash-attn is a free Metal win. extended=false retries with the
// minimal set so an older binary missing a flag can never kill dictation.
const args = ['-m', model, '--port', String(p), '-nt', '-bs', '5'];
if (extended) args.push('--suppress-nst', '--no-context', '--flash-attn');
// A multilingual model (no .en in the filename) auto-detects the spoken language per utterance.
if (!path.basename(model).includes('.en')) args.push('-l', 'auto');
const child = spawn(bin, args, {
@@ -184,6 +187,11 @@ async function p_bootServer(resourceDir, userDataDir) {
if (!ok) {
bootingChild = null;
try { child.kill('SIGKILL'); } catch (_) {}
// An instantly-dead child with the extended flags is probably an older binary: retry minimal.
if (extended && (child.exitCode !== null || child.signalCode !== null)) {
console.log('[voice] extended decode flags rejected; retrying with the minimal set');
return p_bootServer(resourceDir, userDataDir, false);
}
// A dead child is not a slow one. Whisper can die in ~0.1s with exit code 0 (a missing ffmpeg on
// a Finder-launched PATH does exactly that), so report ITS reason instantly instead of making the
// user sit through the full ready budget for a process that was never coming back.
@@ -237,6 +245,10 @@ async function transcribe(resourceDir, userDataDir, wavBuffer) {
const form = new FormData();
form.append('file', new Blob([wavBuffer], { type: 'audio/wav' }), 'audio.wav');
form.append('response_format', 'text');
// 0.2 + 0.2 fallback ladder is what VoiceTypr and VoiceInk ship; whisper's 0.0 greedy start
// retries into hallucination on marginal audio.
form.append('temperature', '0.2');
form.append('temperature_inc', '0.2');
if (dictionaryPrompt) form.append('prompt', dictionaryPrompt);
const res = await fetch(`http://127.0.0.1:${p}/inference`, { method: 'POST', body: form });
if (!res.ok) throw new Error(`whisper-http-${res.status}`);
frontend/src/shared/voice/useVoiceDictation.ts
+29 -8
View File
@@ -6,7 +6,7 @@ import { playVoiceCue } from './voiceCues';
import { injectAtFocus } from './injectAtFocus';
import { createSilenceDetector } from './createSilenceDetector';
import { pushDictation } from './voiceHistory';
import { learnFromTranscript } from './voiceDictionary';
import { learnFromTranscript, isDictionaryEcho } from './voiceDictionary';
import { getFocusedSurfaceHost, surfaceDisabled } from './voiceSurface';
import { store } from '@/shared/state/store';
import { createCaptureNode } from './createCaptureNode';
@@ -168,7 +168,11 @@ export function useVoiceDictation() {
// Fire the OS mic prompt through the main process first: a packaged hardened-runtime build denies renderer getUserMedia outright until TCC granted (the prod dictation-dead cause, ENG-103).
const micOk = await (window.openswarm as any)?.voiceRequestMicAccess?.() ?? true;
if (micOk === false) { setError('mic-denied'); return; }
stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } });
// autoGainControl OFF is the OSS-dictation consensus (Chromium's hidden AGC rides the mic and
// garbles levels mid-utterance; none of the five surveyed shipping apps allow any AGC), and
// noiseSuppression smears speech whisper handles better raw. Echo cancellation stays: we play
// cues out the speakers while the mic is hot.
stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: false, autoGainControl: false } });
ctx = new AudioContext({ sampleRate: VOICE_SAMPLE_RATE });
const source = ctx.createMediaStreamSource(stream);
const chunks: Float32Array[] = [];
@@ -239,20 +243,37 @@ export function useVoiceDictation() {
}
let res: { ok: boolean; text?: string; error?: string } | undefined;
{
// Whisper hallucinates plausible punctuation on near-silence; a clip whose level never beat the streaming RMS gate gets "didn't catch that", never a decode.
let sumSq = 0;
for (let i = 0; i < samples.length; i += 4) sumSq += samples[i] * samples[i];
const rms = Math.sqrt(sumSq / Math.max(1, Math.floor(samples.length / 4)));
if (rms < 0.002) {
// OpenWhispr's window gate: whole-clip RMS misses a clip that is silence plus one cough, so
// judge 100ms windows; no window with real speech energy means no decode, ever.
const win = Math.round(VOICE_SAMPLE_RATE * 0.1);
let peakRms = 0;
let speechWindow = false;
for (let off = 0; off + win <= samples.length; off += win) {
let sumSq = 0;
let peak = 0;
for (let i = off; i < off + win; i += 2) { const v = samples[i]; sumSq += v * v; if (Math.abs(v) > peak) peak = Math.abs(v); }
const wr = Math.sqrt(sumSq / (win / 2));
if (wr > peakRms) peakRms = wr;
if (wr >= 0.003 && peak >= 0.02) speechWindow = true;
}
if (peakRms < 0.002 || (!speechWindow && peakRms < 0.006)) {
res = { ok: true, text: '' };
} else {
const wav = encodeWav(samples);
// whisper.cpp asserts on sub-second buffers; zero-pad instead of rejecting (FluidVoice).
const padded = samples.length < VOICE_SAMPLE_RATE
? (() => { const b = new Float32Array(VOICE_SAMPLE_RATE); b.set(samples); return b; })()
: samples;
const wav = encodeWav(padded);
res = await window.openswarm?.voiceTranscribe?.(wav);
if ((!res || !res.ok || !res.text) && streamedFallback) res = { ok: true, text: streamedFallback };
}
}
// Whisper captions non-speech in brackets/parens ("[ Background sounds ]", "(laughs)") and marks speaker turns with ">>"; those are annotations, not dictation.
if (res?.ok && res.text) res = { ok: true, text: res.text.replace(/\[[^\]]*\]|\([^)]*\)|\*[^*]*\*|(?:^|\s)>>\s?/g, ' ').replace(/\s+/g, ' ').trim() };
// Stutter collapse (Handy): three or more consecutive identical words are one decode loop, not speech.
if (res?.ok && res.text) res = { ok: true, text: res.text.replace(/\b([A-Za-z']+)(\s+\1\b){2,}/gi, '$1') };
// Glossary echo (OpenWhispr): a transcript that is mostly the dictionary read back is the prompt leaking, not dictation.
if (res?.ok && res.text && isDictionaryEcho(res.text)) res = { ok: true, text: '' };
// A transcript with no letter or digit in it is a hallucination artifact (the lone comma), not dictation.
if (res?.ok && res.text && !/[\p{L}\p{N}]/u.test(res.text)) res = { ok: true, text: '' };
// Wispr's command grammar, v1: saying only "scratch that" (or "delete/cancel that") throws the take away.
frontend/src/shared/voice/voiceDictionary.ts
+16
View File
@@ -32,6 +32,7 @@ function pushMerged(): void {
.map(([w]) => w);
const manualWords = manual.split(',').map((w) => w.trim()).filter(Boolean);
const merged = [...new Set([...manualWords, ...learned])].join(', ');
lastPushedMerged = merged;
const bridge = window as unknown as { openswarm?: { voiceSetDictionary?: (words: string) => void } };
bridge.openswarm?.voiceSetDictionary?.(merged);
}
@@ -41,6 +42,21 @@ export function setManualDictionary(words: string): void {
pushMerged();
}
let lastPushedMerged = '';
// OpenWhispr's echo test: the model sometimes reads the glossary prompt back as the "transcript".
// Mostly-dictionary words (90%) covering most of the dictionary (70%) = the prompt leaked.
export function isDictionaryEcho(text: string): boolean {
const dictWords = new Set(lastPushedMerged.toLowerCase().split(/[,\s]+/).filter(Boolean));
if (dictWords.size === 0) return false;
const words = text.toLowerCase().split(/\s+/).map((w) => w.replace(/[^a-z']/g, '')).filter(Boolean);
if (words.length === 0) return false;
const unique = [...new Set(words)];
const fromDict = unique.filter((w) => dictWords.has(w)).length;
const coverage = [...dictWords].filter((w) => words.includes(w)).length / dictWords.size;
return fromDict / unique.length >= 0.9 && coverage >= 0.7;
}
export function learnFromTranscript(text: string): void {
try {
const counts = readLearned();