From e4196bc82a0cb114af4279bc496126fa9326dfe6 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Tue, 4 Aug 2026 00:30:44 -0700 Subject: [PATCH] [eric] voice: worklet ships as a real asset (blob module URLs are script-src, CSP-blocked) with ScriptProcessor fallback --- .../pcm-worklet.js} | 19 +++----- .../src/shared/voice/createCaptureNode.ts | 44 +++++++++++++++++++ .../src/shared/voice/useVoiceDictation.ts | 32 +++++--------- 3 files changed, 60 insertions(+), 35 deletions(-) rename frontend/{src/shared/voice/getPcmWorkletUrl.ts => public/pcm-worklet.js} (74%) create mode 100644 frontend/src/shared/voice/createCaptureNode.ts diff --git a/frontend/src/shared/voice/getPcmWorkletUrl.ts b/frontend/public/pcm-worklet.js similarity index 74% rename from frontend/src/shared/voice/getPcmWorkletUrl.ts rename to frontend/public/pcm-worklet.js index 58ea2997..9398ea69 100644 --- a/frontend/src/shared/voice/getPcmWorkletUrl.ts +++ b/frontend/public/pcm-worklet.js @@ -1,8 +1,7 @@ // OpenWhispr's capture worklet, ported near-verbatim (MIT): 800-sample Int16 buffers (50ms at // 16kHz) posted with transferables off the audio thread, plus a "stop" -> drain -> "flushed" -// handshake so the tail of an utterance is never lost at teardown. Replaces the deprecated -// main-thread ScriptProcessorNode. -const WORKLET_SOURCE = ` +// handshake so the tail of an utterance is never lost at teardown. A real static asset, not a blob +// URL: worklet module fetches obey script-src, and the app's CSP deliberately has no blob: there. const BUFFER_SIZE = 800; class PCMStreamingProcessor extends AudioWorkletProcessor { constructor() { @@ -11,14 +10,14 @@ class PCMStreamingProcessor extends AudioWorkletProcessor { this.offset = 0; this.stopped = false; this.port.onmessage = (event) => { - if (event.data === "stop") { + if (event.data === 'stop') { if (this.offset > 0) { const partial = this.buffer.slice(0, this.offset); this.port.postMessage(partial.buffer, [partial.buffer]); this.buffer = new Int16Array(BUFFER_SIZE); this.offset = 0; } - this.port.postMessage("flushed"); + this.port.postMessage('flushed'); this.stopped = true; } }; @@ -39,12 +38,4 @@ class PCMStreamingProcessor extends AudioWorkletProcessor { return true; } } -registerProcessor("pcm-streaming-processor", PCMStreamingProcessor); -`; - -let cachedUrl: string | null = null; - -export function getPcmWorkletUrl(): string { - if (!cachedUrl) cachedUrl = URL.createObjectURL(new Blob([WORKLET_SOURCE], { type: 'application/javascript' })); - return cachedUrl; -} +registerProcessor('pcm-streaming-processor', PCMStreamingProcessor); diff --git a/frontend/src/shared/voice/createCaptureNode.ts b/frontend/src/shared/voice/createCaptureNode.ts new file mode 100644 index 00000000..ded22b92 --- /dev/null +++ b/frontend/src/shared/voice/createCaptureNode.ts @@ -0,0 +1,44 @@ +// One capture interface, two implementations: the AudioWorklet (off-thread, 50ms Int16 chunks with +// a drain handshake) with the deprecated ScriptProcessor as the always-works fallback. Capture +// plumbing failing must degrade dictation, never kill it: the first worklet rollout died on CSP +// (blob: module URLs are script-src, not worker-src) and the generic catch read "mic broken". + +export interface CaptureNode { + node: AudioNode; + // Drains any buffered tail; resolves when the last samples have been delivered. + requestFlush: () => Promise; +} + +export async function createCaptureNode(ctx: AudioContext, onPcm: (pcm: Int16Array) => void): Promise { + try { + await ctx.audioWorklet.addModule(new URL('pcm-worklet.js', window.location.href).toString()); + const node = new AudioWorkletNode(ctx, 'pcm-streaming-processor'); + let flushResolve: () => void = () => {}; + const flushed = new Promise((resolve) => { flushResolve = resolve; }); + node.port.onmessage = (e: MessageEvent): void => { + if (e.data === 'flushed') { flushResolve(); return; } + onPcm(new Int16Array(e.data as ArrayBuffer)); + }; + return { + node, + requestFlush: () => { + try { node.port.postMessage('stop'); } catch (_) { /* node already gone */ } + return Promise.race([flushed, new Promise((resolve) => window.setTimeout(resolve, 1000))]); + }, + }; + } catch (err) { + console.warn('[voice] worklet capture unavailable, using ScriptProcessor:', err instanceof Error ? err.message : err); + const node = ctx.createScriptProcessor(4096, 1, 1); + node.onaudioprocess = (e): void => { + const data = e.inputBuffer.getChannelData(0); + const i16 = new Int16Array(data.length); + for (let i = 0; i < data.length; i++) { + const s = Math.max(-1, Math.min(1, data[i])); + i16[i] = s < 0 ? s * 0x8000 : s * 0x7fff; + } + onPcm(i16); + }; + // ScriptProcessor delivers synchronously per 4096-frame block; there is no tail to drain. + return { node, requestFlush: () => Promise.resolve() }; + } +} diff --git a/frontend/src/shared/voice/useVoiceDictation.ts b/frontend/src/shared/voice/useVoiceDictation.ts index e8748037..ff086fc8 100644 --- a/frontend/src/shared/voice/useVoiceDictation.ts +++ b/frontend/src/shared/voice/useVoiceDictation.ts @@ -4,7 +4,7 @@ import { encodeWav, VOICE_SAMPLE_RATE } from './encodeWav'; import { playVoiceCue } from './voiceCues'; import { injectAtFocus } from './injectAtFocus'; import { createSilenceDetector } from './createSilenceDetector'; -import { getPcmWorkletUrl } from './getPcmWorkletUrl'; +import { createCaptureNode } from './createCaptureNode'; export type VoiceState = 'idle' | 'recording' | 'transcribing' | 'preparing'; @@ -23,10 +23,10 @@ export interface VoicePartial { interface Recorder { ctx: AudioContext; stream: MediaStream; - node: AudioWorkletNode; + node: AudioNode; + requestFlush: () => Promise; source: MediaStreamAudioSourceNode; chunks: Float32Array[]; - flushed: Promise; streaming: boolean; } @@ -127,20 +127,13 @@ export function useVoiceDictation() { if (micOk === false) { setError('mic-denied'); return; } const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } }); const ctx = new AudioContext({ sampleRate: VOICE_SAMPLE_RATE }); - await ctx.audioWorklet.addModule(getPcmWorkletUrl()); const source = ctx.createMediaStreamSource(stream); - const node = new AudioWorkletNode(ctx, 'pcm-streaming-processor'); const chunks: Float32Array[] = []; const endpointer = hold ? null : createSilenceDetector(ctx.sampleRate); const streamRes = await window.openswarm?.voiceStreamStart?.(); const streaming = streamRes?.ok === true; - let flushResolve: () => void = () => {}; - const flushed = new Promise((resolve) => { flushResolve = resolve; }); - node.port.onmessage = (e: MessageEvent): void => { - if (e.data === 'flushed') { flushResolve(); return; } - const ab = e.data as ArrayBuffer; - if (streaming) window.openswarm?.voiceStreamChunk?.(ab); - const i16 = new Int16Array(ab); + const capture = await createCaptureNode(ctx, (i16) => { + if (streaming) window.openswarm?.voiceStreamChunk?.(i16.buffer as ArrayBuffer); const data = new Float32Array(i16.length); for (let i = 0; i < i16.length; i++) data[i] = i16[i] / 0x8000; chunks.push(data); @@ -150,10 +143,10 @@ export function useVoiceDictation() { const rms = Math.sqrt(sum / (data.length / 8)); volumeRef.current = volumeRef.current * 0.7 + Math.min(1, rms * 6) * 0.3; if (endpointer && endpointer.push(data) !== 'listening') void stopRef.current?.(); - }; - source.connect(node); - node.connect(ctx.destination); - recRef.current = { ctx, stream, node, source, chunks, flushed, streaming }; + }); + source.connect(capture.node); + capture.node.connect(ctx.destination); + recRef.current = { ctx, stream, node: capture.node, requestFlush: capture.requestFlush, source, chunks, streaming }; setState('recording'); playVoiceCue('start'); void (window.openswarm as { haptic?: (p: string) => Promise } | undefined)?.haptic?.('generic'); @@ -170,11 +163,8 @@ export function useVoiceDictation() { if (stateRef.current !== 'recording') return; const rec = recRef.current; const streaming = rec?.streaming === true; - // Drain the worklet's last partial buffer before teardown (1s watchdog: a wedged worklet costs 50ms of tail, not a hang). - if (rec) { - try { rec.node.port.postMessage('stop'); } catch (_) { /* node already gone */ } - await Promise.race([rec.flushed, new Promise((resolve) => window.setTimeout(resolve, 1000))]); - } + // Drain the capture's last partial buffer before teardown (worklet flush carries its own 1s watchdog). + if (rec) await rec.requestFlush(); const samples = teardown(); playVoiceCue('stop'); void (window.openswarm as { haptic?: (p: string) => Promise } | undefined)?.haptic?.('alignment');