diff --git a/electron/main.js b/electron/main.js index 9bb6bc8a..3f1ed372 100644 --- a/electron/main.js +++ b/electron/main.js @@ -2893,6 +2893,8 @@ ipcMain.handle('voice:transcribe', async (_e, wavArrayBuffer) => { ipcMain.handle('voice:warmup', async () => { try { await whisperService.ensureServer(voiceResourceDir(), voiceUserDataDir()); return { ok: true }; } catch (err) { return { ok: false, error: String(err && err.message ? err.message : err) }; } }); +// First-run model download progress so the pill can show "Preparing voice N%". +ipcMain.handle('voice:status', () => whisperService.modelStatus()); // Paste the text into the frontmost app (dictate-anywhere). Returns whether the OS paste actually fired. ipcMain.handle('voice:inject', async (_e, text) => { try { const pasted = await injectText(String(text || '')); return { ok: true, pasted }; } catch (err) { return { ok: false, error: String(err && err.message ? err.message : err) }; } diff --git a/electron/preload.js b/electron/preload.js index c21fc629..53cb6f3b 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -65,6 +65,7 @@ contextBridge.exposeInMainWorld('openswarm', { // Voice dictation (local whisper.cpp). transcribe takes a 16kHz-mono WAV ArrayBuffer; inject pastes // text into the frontmost app; warmup pre-loads the model; onVoiceToggle fires on the global hotkey. voiceWarmup: () => ipcRenderer.invoke('voice:warmup'), + voiceStatus: () => ipcRenderer.invoke('voice:status'), voiceTranscribe: (wavArrayBuffer) => ipcRenderer.invoke('voice:transcribe', wavArrayBuffer), voiceInject: (text) => ipcRenderer.invoke('voice:inject', text), onVoiceToggle: (cb) => { diff --git a/electron/voice/whisperService.js b/electron/voice/whisperService.js index c9f4a452..03839697 100644 --- a/electron/voice/whisperService.js +++ b/electron/voice/whisperService.js @@ -6,8 +6,50 @@ const { spawn } = require('child_process'); const path = require('path'); const fs = require('fs'); +const https = require('https'); const MODEL_FILE = 'ggml-base.en.bin'; +const MODEL_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin'; + +// First-run model fetch, so a dev build (or a prod build that shipped without the model) still works +// instead of dead-ending on "no model". Progress is exposed so the pill can say "Preparing voice 40%". +const download = { active: false, pct: 0, error: null }; + +function downloadModel(dest) { + if (download.active) return; + download.active = true; + download.pct = 0; + download.error = null; + try { fs.mkdirSync(path.dirname(dest), { recursive: true }); } catch (_) {} + const tmp = `${dest}.part`; + const file = fs.createWriteStream(tmp); + const req = https.get(MODEL_URL, (res) => { + if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) { + // Follow HuggingFace's CDN redirect once. + https.get(res.headers.location, (r2) => pipeTo(r2, file, tmp, dest)).on('error', onErr); + return; + } + pipeTo(res, file, tmp, dest); + }); + req.on('error', onErr); + function onErr(e) { download.active = false; download.error = String(e && e.message ? e.message : e); try { file.close(); fs.unlinkSync(tmp); } catch (_) {} } +} + +function pipeTo(res, file, tmp, dest) { + const total = Number(res.headers['content-length'] || 0); + let got = 0; + res.on('data', (c) => { got += c.length; if (total) download.pct = Math.round((got / total) * 100); }); + res.pipe(file); + file.on('finish', () => file.close(() => { + try { fs.renameSync(tmp, dest); download.pct = 100; } catch (e) { download.error = String(e); } + download.active = false; + })); + res.on('error', () => { download.active = false; download.error = 'stream-error'; try { fs.unlinkSync(tmp); } catch (_) {} }); +} + +function modelStatus() { + return { downloading: download.active, pct: download.pct, error: download.error }; +} // Resolve the whisper-server binary. Env override wins (dev convenience), then the bundled per-arch // copy, then whatever is on PATH so a dev machine with `brew install whisper-cpp` just works. @@ -66,7 +108,9 @@ async function ensureServer(resourceDir, userDataDir) { const model = resolveModel(resourceDir, userDataDir); if (!model) { readyPromise = null; - throw new Error('no-model'); // caller surfaces a "voice model missing" state, never crashes + // Kick off a one-time background fetch to the dev cache so the NEXT dictation just works. + downloadModel(path.join(userDataDir, 'whisper', MODEL_FILE)); + throw new Error(download.active ? 'model-downloading' : 'no-model'); } const p = pickPort(); const child = spawn(bin, ['-m', model, '--port', String(p), '-nt', '--convert'], { @@ -109,4 +153,4 @@ function stopServer() { readyPromise = null; } -module.exports = { ensureServer, transcribe, stopServer, resolveBinary, resolveModel }; +module.exports = { ensureServer, transcribe, stopServer, resolveBinary, resolveModel, modelStatus }; diff --git a/frontend/src/app/components/Layout/AppShell.tsx b/frontend/src/app/components/Layout/AppShell.tsx index e6b730a5..b932035c 100644 --- a/frontend/src/app/components/Layout/AppShell.tsx +++ b/frontend/src/app/components/Layout/AppShell.tsx @@ -10,6 +10,7 @@ import ListItemIcon from '@mui/material/ListItemIcon'; import ListItemText from '@mui/material/ListItemText'; import Menu from '@mui/material/Menu'; import MenuItem from '@mui/material/MenuItem'; +import { VoiceDictationProvider } from '@/shared/voice/VoiceDictationContext'; import Typography from '@mui/material/Typography'; import IconButton from '@mui/material/IconButton'; import Tooltip from '@mui/material/Tooltip'; @@ -1399,24 +1400,28 @@ const AppShell: React.FC = () => { ml: fsHideChrome ? 0 : '6px', borderRadius: fsHideChrome ? 0 : '14px', }}> - {/* Hidden (not unmounted) when the dashboard view is active so the persistent Dashboard layered above can take over. */} - - - + {/* One voice controller wraps BOTH the routed content and the persistent Dashboard host, so + the spawn-pill mic (which lives in the persistent host, not the Outlet) shares the recorder. */} + + {/* Hidden (not unmounted) when the dashboard view is active so the persistent Dashboard layered above can take over. */} + + + - {/* CSS-hidden on other routes so webviews + state survive nav. */} - {lastDashboardId && ( - - - - )} + {/* CSS-hidden on other routes so webviews + state survive nav. */} + {lastDashboardId && ( + + + + )} + diff --git a/frontend/src/app/pages/Dashboard/desktop/DesktopSpawnPill.tsx b/frontend/src/app/pages/Dashboard/desktop/DesktopSpawnPill.tsx index b0af5641..863cc55d 100644 --- a/frontend/src/app/pages/Dashboard/desktop/DesktopSpawnPill.tsx +++ b/frontend/src/app/pages/Dashboard/desktop/DesktopSpawnPill.tsx @@ -4,6 +4,9 @@ import Typography from '@mui/material/Typography'; import Tooltip from '@mui/material/Tooltip'; import AddRounded from '@mui/icons-material/AddRounded'; import MicNoneOutlinedIcon from '@mui/icons-material/MicNoneOutlined'; +import MicIcon from '@mui/icons-material/Mic'; +import CircularProgress from '@mui/material/CircularProgress'; +import { useVoice } from '@/shared/voice/VoiceDictationContext'; import GridViewRoundedIcon from '@mui/icons-material/GridViewRounded'; import StickyNote2OutlinedIcon from '@mui/icons-material/StickyNote2Outlined'; import HistoryRoundedIcon from '@mui/icons-material/HistoryRounded'; @@ -38,6 +41,11 @@ function DesktopSpawnPill({ }: DesktopSpawnPillProps): React.ReactElement { const [menuOpen, setMenuOpen] = useState(false); const rootRef = useRef(null); + const { state: voiceState, pct: voicePct, toggle: toggleVoice } = useVoice(); + const recording = voiceState === 'recording'; + const transcribing = voiceState === 'transcribing'; + const preparing = voiceState === 'preparing'; + const voiceBusy = transcribing || preparing; useEffect(() => { if (!menuOpen) return undefined; @@ -152,8 +160,11 @@ function DesktopSpawnPill({ > - + { e.stopPropagation(); if (!voiceBusy) toggleVoice(); }} sx={{ display: 'flex', alignItems: 'center', @@ -161,11 +172,16 @@ function DesktopSpawnPill({ width: 22, height: 22, borderRadius: '50%', - color: 'rgba(255,255,255,0.45)', - cursor: 'default', + color: recording ? '#ff8a8a' : 'rgba(255,255,255,0.6)', + cursor: voiceBusy ? 'default' : 'pointer', + '&:hover': { color: '#fff', background: 'rgba(255,255,255,0.12)' }, }} > - + {voiceBusy + ? + : recording + ? + : } diff --git a/frontend/src/app/pages/Dashboard/desktop/HelpPill.tsx b/frontend/src/app/pages/Dashboard/desktop/HelpPill.tsx index ea7e1c82..b167ca5b 100644 --- a/frontend/src/app/pages/Dashboard/desktop/HelpPill.tsx +++ b/frontend/src/app/pages/Dashboard/desktop/HelpPill.tsx @@ -7,16 +7,18 @@ import MicNoneOutlinedIcon from '@mui/icons-material/MicNoneOutlined'; import MicIcon from '@mui/icons-material/Mic'; import { useAppDispatch } from '@/shared/hooks'; import { addBrowserCard } from '@/shared/state/dashboardLayoutSlice'; -import { useVoiceDictation } from './useVoiceDictation'; +import { useVoice } from '@/shared/voice/VoiceDictationContext'; const HELP_URL = 'https://openswarm.com'; /** Top-right desktop pill: Help opens the docs; the mic dictates (local whisper) into the focused field. */ function HelpPill(): React.ReactElement { const dispatch = useAppDispatch(); - const { state, toggle } = useVoiceDictation(); + const { state, pct, toggle } = useVoice(); const recording = state === 'recording'; const transcribing = state === 'transcribing'; + const preparing = state === 'preparing'; + const busy = transcribing || preparing; return ( dispatch(addBrowserCard({ url: HELP_URL }))} > - {recording ? 'Listening' : transcribing ? 'Transcribing' : 'Help'} + {recording ? 'Listening' : transcribing ? 'Transcribing' : preparing ? `Preparing ${pct}%` : 'Help'} - + { e.stopPropagation(); if (!transcribing) toggle(); }} + onClick={(e) => { e.stopPropagation(); if (!busy) toggle(); }} > - {transcribing + {busy ? : recording ? diff --git a/frontend/src/shared/voice/VoiceDictationContext.tsx b/frontend/src/shared/voice/VoiceDictationContext.tsx new file mode 100644 index 00000000..169c96a5 --- /dev/null +++ b/frontend/src/shared/voice/VoiceDictationContext.tsx @@ -0,0 +1,31 @@ +import React, { createContext, useContext } from 'react'; +import { useVoiceDictation, VoiceState } from './useVoiceDictation'; + +// One recorder for the whole app. Both mics (the Help pill and the spawn composer) plus the global +// hotkey drive the SAME dictation session, so two mics can't fight over the microphone or show +// out-of-sync state. Mounted once near the app root. +interface VoiceContextValue { + state: VoiceState; + lastText: string; + error: string | null; + pct: number; + toggle: () => void; +} + +const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, toggle: () => {} }; +const VoiceContext = createContext(NOOP); + +export function VoiceDictationProvider({ children }: { children: React.ReactNode }): React.ReactElement { + const { state, lastText, error, pct, toggle } = useVoiceDictation(); + return ( + + {children} + + ); +} + +// A component rendered outside the provider (or a web build with no Electron bridge) gets the no-op, +// so mics still render and just do nothing rather than crashing. +export function useVoice(): VoiceContextValue { + return useContext(VoiceContext); +} diff --git a/frontend/src/app/pages/Dashboard/desktop/useVoiceDictation.ts b/frontend/src/shared/voice/useVoiceDictation.ts similarity index 74% rename from frontend/src/app/pages/Dashboard/desktop/useVoiceDictation.ts rename to frontend/src/shared/voice/useVoiceDictation.ts index fc68cbea..d670f073 100644 --- a/frontend/src/app/pages/Dashboard/desktop/useVoiceDictation.ts +++ b/frontend/src/shared/voice/useVoiceDictation.ts @@ -1,9 +1,9 @@ import { useCallback, useEffect, useRef, useState } from 'react'; -import { encodeWav, VOICE_SAMPLE_RATE } from '@/shared/voice/encodeWav'; +import { encodeWav, VOICE_SAMPLE_RATE } from './encodeWav'; -export type VoiceState = 'idle' | 'recording' | 'transcribing'; +export type VoiceState = 'idle' | 'recording' | 'transcribing' | 'preparing'; -// WhisperFlow-style push-to-dictate: toggle recording (global hotkey or the pill), speak, and the +// WhisperFlow-style push-to-dictate: toggle recording (global hotkey or a mic), speak, and the // transcribed text is pasted into whatever field has focus. Capture is 16kHz mono PCM so it feeds // whisper.cpp with no server-side resample. The recording path can only be proven with a real mic; // the encode -> transcribe -> inject half is exercised by the encoder round-trip test. @@ -19,10 +19,25 @@ export function useVoiceDictation() { const [state, setState] = useState('idle'); const [lastText, setLastText] = useState(''); const [error, setError] = useState(null); + const [pct, setPct] = useState(0); const recRef = useRef(null); const stateRef = useRef('idle'); stateRef.current = state; + // First-run: the model is downloading. Poll progress until it lands, then drop back to idle so the + // next click records for real. Never records while preparing, so nothing is lost to a dropped phrase. + const pollModel = useCallback((): void => { + const tick = async (): Promise => { + const st = await window.openswarm?.voiceStatus?.(); + if (!st) { setState('idle'); return; } + setPct(st.pct || 0); + if (st.error) { setError(st.error); setState('idle'); return; } + if (!st.downloading) { setState('idle'); return; } + setTimeout(() => { void tick(); }, 1000); + }; + void tick(); + }, []); + const teardown = useCallback((): Float32Array | null => { const rec = recRef.current; recRef.current = null; @@ -41,6 +56,7 @@ export function useVoiceDictation() { const start = useCallback(async (): Promise => { if (stateRef.current !== 'idle') return; + if (!window.openswarm?.voiceTranscribe) { setError('desktop-only'); return; } // no Electron bridge = web build setError(null); try { const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } }); @@ -72,15 +88,20 @@ export function useVoiceDictation() { if (res?.ok && res.text) { setLastText(res.text); await window.openswarm?.voiceInject?.(res.text); + setState('idle'); + } else if (res?.error === 'model-downloading' || res?.error === 'no-model') { + // First use kicked off the model fetch; show progress and don't error out. + setState('preparing'); + pollModel(); } else { setError(res?.error || 'transcription-failed'); + setState('idle'); } } catch (err) { setError(err instanceof Error ? err.message : 'transcription-failed'); - } finally { setState('idle'); } - }, [teardown]); + }, [teardown, pollModel]); const toggle = useCallback((): void => { if (stateRef.current === 'recording') void stop(); @@ -96,5 +117,5 @@ export function useVoiceDictation() { // A dangling recorder (unmount mid-capture) must release the mic. useEffect(() => () => { teardown(); }, [teardown]); - return { state, lastText, error, toggle, start, stop }; + return { state, lastText, error, pct, toggle, start, stop }; } diff --git a/frontend/src/types/electron.d.ts b/frontend/src/types/electron.d.ts index b57c8df9..a48f26c3 100644 --- a/frontend/src/types/electron.d.ts +++ b/frontend/src/types/electron.d.ts @@ -55,6 +55,7 @@ declare global { hardReset?: () => Promise; clearBrowserData?: () => Promise<{ ok: boolean }>; voiceWarmup?: () => Promise<{ ok: boolean; error?: string }>; + voiceStatus?: () => Promise<{ downloading: boolean; pct: number; error: string | null }>; voiceTranscribe?: (wav: ArrayBuffer) => Promise<{ ok: boolean; text?: string; error?: string }>; voiceInject?: (text: string) => Promise<{ ok: boolean; pasted?: boolean; error?: string }>; onVoiceToggle?: (cb: () => void) => () => void;