diff --git a/electron/voice/whisperService.js b/electron/voice/whisperService.js index 03839697..83a87c5e 100644 --- a/electron/voice/whisperService.js +++ b/electron/voice/whisperService.js @@ -22,29 +22,37 @@ function downloadModel(dest) { download.error = null; try { fs.mkdirSync(path.dirname(dest), { recursive: true }); } catch (_) {} const tmp = `${dest}.part`; - const file = fs.createWriteStream(tmp); - const req = https.get(MODEL_URL, (res) => { - if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { - // Follow HuggingFace's CDN redirect once. - https.get(res.headers.location, (r2) => pipeTo(r2, file, tmp, dest)).on('error', onErr); - return; - } - pipeTo(res, file, tmp, dest); - }); - req.on('error', onErr); - function onErr(e) { download.active = false; download.error = String(e && e.message ? e.message : e); try { file.close(); fs.unlinkSync(tmp); } catch (_) {} } -} + try { fs.unlinkSync(tmp); } catch (_) {} -function pipeTo(res, file, tmp, dest) { - const total = Number(res.headers['content-length'] || 0); - let got = 0; - res.on('data', (c) => { got += c.length; if (total) download.pct = Math.round((got / total) * 100); }); - res.pipe(file); - file.on('finish', () => file.close(() => { - try { fs.renameSync(tmp, dest); download.pct = 100; } catch (e) { download.error = String(e); } - download.active = false; - })); - res.on('error', () => { download.active = false; download.error = 'stream-error'; try { fs.unlinkSync(tmp); } catch (_) {} }); + const fail = (msg) => { download.active = false; download.error = String(msg); try { fs.unlinkSync(tmp); } catch (_) {} }; + + // HuggingFace bounces resolve -> CDN -> signed URL, so follow redirects instead of assuming one hop. + const fetchUrl = (url, hops) => { + if (hops > 6) { fail('too-many-redirects'); return; } + const req = https.get(url, { headers: { 'User-Agent': 'openswarm-voice' } }, (res) => { + const code = res.statusCode || 0; + if (code >= 300 && code < 400 && res.headers.location) { + res.resume(); // drain so the socket frees + fetchUrl(new URL(res.headers.location, url).toString(), hops + 1); + return; + } + if (code !== 200) { res.resume(); fail(`http-${code}`); return; } + const total = Number(res.headers['content-length'] || 0); + let got = 0; + const file = fs.createWriteStream(tmp); + res.on('data', (c) => { got += c.length; if (total) download.pct = Math.round((got / total) * 100); }); + res.pipe(file); + file.on('finish', () => file.close(() => { + // A truncated download is worse than none: only accept a complete file. + if (total && got < total) { fail('truncated'); return; } + try { fs.renameSync(tmp, dest); download.pct = 100; download.active = false; } catch (e) { fail(e && e.message ? e.message : e); } + })); + res.on('error', () => fail('stream-error')); + file.on('error', () => fail('write-error')); + }); + req.on('error', (e) => fail(e && e.message ? e.message : e)); + }; + fetchUrl(MODEL_URL, 0); } function modelStatus() { @@ -98,37 +106,46 @@ async function waitForReady(p, timeoutMs) { return false; } +async function p_bootServer(resourceDir, userDataDir) { + const bin = resolveBinary(resourceDir); + const model = resolveModel(resourceDir, userDataDir); + if (!model) { + // Kick off a one-time background fetch so the NEXT dictation just works. + downloadModel(path.join(userDataDir, 'whisper', MODEL_FILE)); + throw new Error(download.active ? 'model-downloading' : 'no-model'); + } + const p = pickPort(); + const child = spawn(bin, ['-m', model, '--port', String(p), '-nt', '--convert'], { + stdio: ['ignore', 'pipe', 'pipe'], + }); + child.on('error', () => { proc = null; port = 0; }); + child.on('exit', () => { proc = null; port = 0; readyPromise = null; }); + const ok = await waitForReady(p, 20000); + if (!ok) { + try { child.kill(); } catch (_) {} + throw new Error('server-timeout'); + } + proc = child; + port = p; + return p; +} + // Boot the warm server once. resourceDir = where a packaged build put the binary+model; userDataDir // = app.getPath('userData') for the dev cache. Returns the port, or throws with an actionable reason. +// The readyPromise is cleared AFTER it settles, never synchronously inside the async body: the old +// code reset it inside the IIFE where the outer assignment immediately overwrote the null, pinning a +// settled-rejected promise forever so every later call kept throwing "model-downloading" even after +// the model finished. Clearing on rejection here lets the next call retry cleanly. async function ensureServer(resourceDir, userDataDir) { if (proc && port) return port; if (readyPromise) return readyPromise; - readyPromise = (async () => { - const bin = resolveBinary(resourceDir); - const model = resolveModel(resourceDir, userDataDir); - if (!model) { - readyPromise = null; - // Kick off a one-time background fetch to the dev cache so the NEXT dictation just works. - downloadModel(path.join(userDataDir, 'whisper', MODEL_FILE)); - throw new Error(download.active ? 'model-downloading' : 'no-model'); - } - const p = pickPort(); - const child = spawn(bin, ['-m', model, '--port', String(p), '-nt', '--convert'], { - stdio: ['ignore', 'pipe', 'pipe'], - }); - child.on('error', () => { proc = null; port = 0; }); - child.on('exit', () => { proc = null; port = 0; readyPromise = null; }); - const ok = await waitForReady(p, 20000); - if (!ok) { - try { child.kill(); } catch (_) {} - readyPromise = null; - throw new Error('server-timeout'); - } - proc = child; - port = p; - return p; - })(); - return readyPromise; + readyPromise = p_bootServer(resourceDir, userDataDir); + try { + return await readyPromise; + } catch (err) { + readyPromise = null; + throw err; + } } // Transcribe a 16kHz-mono WAV buffer to text. The renderer records + encodes the WAV so the audio diff --git a/frontend/src/shared/voice/VoiceDictationContext.tsx b/frontend/src/shared/voice/VoiceDictationContext.tsx index 169c96a5..37829909 100644 --- a/frontend/src/shared/voice/VoiceDictationContext.tsx +++ b/frontend/src/shared/voice/VoiceDictationContext.tsx @@ -1,5 +1,6 @@ import React, { createContext, useContext } from 'react'; -import { useVoiceDictation, VoiceState } from './useVoiceDictation'; +import { useVoiceDictation, VoiceState, VoiceFeedback } from './useVoiceDictation'; +import VoiceOverlay from './VoiceOverlay'; // One recorder for the whole app. Both mics (the Help pill and the spawn composer) plus the global // hotkey drive the SAME dictation session, so two mics can't fight over the microphone or show @@ -9,17 +10,19 @@ interface VoiceContextValue { lastText: string; error: string | null; pct: number; + feedback: VoiceFeedback | null; toggle: () => void; } -const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, toggle: () => {} }; +const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, feedback: null, toggle: () => {} }; const VoiceContext = createContext(NOOP); export function VoiceDictationProvider({ children }: { children: React.ReactNode }): React.ReactElement { - const { state, lastText, error, pct, toggle } = useVoiceDictation(); + const { state, lastText, error, pct, feedback, toggle } = useVoiceDictation(); return ( - + {children} + ); } diff --git a/frontend/src/shared/voice/VoiceOverlay.tsx b/frontend/src/shared/voice/VoiceOverlay.tsx new file mode 100644 index 00000000..e235d3ae --- /dev/null +++ b/frontend/src/shared/voice/VoiceOverlay.tsx @@ -0,0 +1,99 @@ +import React, { useEffect, useState } from 'react'; +import Box from '@mui/material/Box'; +import CircularProgress from '@mui/material/CircularProgress'; +import MicIcon from '@mui/icons-material/Mic'; +import CheckRoundedIcon from '@mui/icons-material/CheckRounded'; +import ContentPasteRoundedIcon from '@mui/icons-material/ContentPasteRounded'; +import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined'; +import { useVoice } from './VoiceDictationContext'; + +// The whole point: dictation must never look like "nothing happened." This floats a small status +// card above the composer for every phase (listening, transcribing, downloading the model) and shows +// the transcript + whether it was pasted or just copied. Non-interactive, auto-dismisses. +const FEEDBACK_MS = 4500; + +function feedbackIcon(icon: string): React.ReactElement { + if (icon === 'check') return ; + if (icon === 'clipboard') return ; + if (icon === 'mic') return ; + return ; +} + +const VoiceOverlay: React.FC = () => { + const { state, pct, feedback } = useVoice(); + const [showFeedback, setShowFeedback] = useState(false); + + useEffect(() => { + if (!feedback) return undefined; + setShowFeedback(true); + const t = setTimeout(() => setShowFeedback(false), FEEDBACK_MS); + return () => clearTimeout(t); + }, [feedback]); + + const live = state !== 'idle'; + const visible = live || (showFeedback && !!feedback); + if (!visible) return null; + + let content: React.ReactElement; + if (state === 'recording') { + content = ( + <> + + Listening + + + ); + } else if (state === 'transcribing') { + content = (<>Transcribing); + } else if (state === 'preparing') { + content = (<>Downloading voice model {pct}%); + } else if (feedback) { + content = ( + <> + {feedbackIcon(feedback.icon)} + + {feedback.text} + + + ); + } else { + return null; + } + + return ( + + {content} + + ); +}; + +export default VoiceOverlay; diff --git a/frontend/src/shared/voice/useVoiceDictation.ts b/frontend/src/shared/voice/useVoiceDictation.ts index d670f073..a648c002 100644 --- a/frontend/src/shared/voice/useVoiceDictation.ts +++ b/frontend/src/shared/voice/useVoiceDictation.ts @@ -15,11 +15,20 @@ interface Recorder { chunks: Float32Array[]; } +// One object per terminal outcome so the overlay's effect always re-fires (new identity every time). +export interface VoiceFeedback { + tone: 'ok' | 'warn' | 'error'; + icon: 'check' | 'clipboard' | 'mic' | 'info'; + text: string; + at: number; +} + export function useVoiceDictation() { const [state, setState] = useState('idle'); const [lastText, setLastText] = useState(''); const [error, setError] = useState(null); const [pct, setPct] = useState(0); + const [feedback, setFeedback] = useState(null); const recRef = useRef(null); const stateRef = useRef('idle'); stateRef.current = state; @@ -72,7 +81,10 @@ export function useVoiceDictation() { // Warm the model the moment recording begins so transcription is instant on stop. void window.openswarm?.voiceWarmup?.(); } catch (err) { - setError(err instanceof Error ? err.message : 'mic-unavailable'); + const msg = err instanceof Error ? err.message : 'mic-unavailable'; + setError(msg); + const denied = /NotAllowed|Permission|denied/i.test(msg); + setFeedback({ tone: 'error', icon: 'mic', text: denied ? 'Microphone access needed. Enable it in System Settings, Privacy, Microphone.' : 'Could not start the microphone.', at: Date.now() }); setState('idle'); } }, []); @@ -87,7 +99,13 @@ export function useVoiceDictation() { const res = await window.openswarm?.voiceTranscribe?.(wav); if (res?.ok && res.text) { setLastText(res.text); - await window.openswarm?.voiceInject?.(res.text); + const inj = await window.openswarm?.voiceInject?.(res.text); + setFeedback(inj?.pasted + ? { tone: 'ok', icon: 'check', text: res.text, at: Date.now() } + : { tone: 'ok', icon: 'clipboard', text: `${res.text} (copied, press Cmd+V)`, at: Date.now() }); + setState('idle'); + } else if (res?.ok && !res.text) { + setFeedback({ tone: 'warn', icon: 'info', text: "Didn't catch that. Try again.", at: Date.now() }); setState('idle'); } else if (res?.error === 'model-downloading' || res?.error === 'no-model') { // First use kicked off the model fetch; show progress and don't error out. @@ -95,10 +113,12 @@ export function useVoiceDictation() { pollModel(); } else { setError(res?.error || 'transcription-failed'); + setFeedback({ tone: 'error', icon: 'info', text: 'Voice transcription failed. Try again.', at: Date.now() }); setState('idle'); } } catch (err) { setError(err instanceof Error ? err.message : 'transcription-failed'); + setFeedback({ tone: 'error', icon: 'info', text: 'Voice transcription failed. Try again.', at: Date.now() }); setState('idle'); } }, [teardown, pollModel]); @@ -117,5 +137,5 @@ export function useVoiceDictation() { // A dangling recorder (unmount mid-capture) must release the mic. useEffect(() => () => { teardown(); }, [teardown]); - return { state, lastText, error, pct, toggle, start, stop }; + return { state, lastText, error, pct, feedback, toggle, start, stop }; }