[eric] voice: wire both mics (help pill + spawn composer) to one shared recorder + first-run model download

electron/main.js

@@ -2893,6 +2893,8 @@ ipcMain.handle('voice:transcribe', async (_e, wavArrayBuffer) => {
 ipcMain.handle('voice:warmup', async () => {
   try { await whisperService.ensureServer(voiceResourceDir(), voiceUserDataDir()); return { ok: true }; } catch (err) { return { ok: false, error: String(err && err.message ? err.message : err) }; }
 });
+// First-run model download progress so the pill can show "Preparing voice N%".
+ipcMain.handle('voice:status', () => whisperService.modelStatus());
 // Paste the text into the frontmost app (dictate-anywhere). Returns whether the OS paste actually fired.
 ipcMain.handle('voice:inject', async (_e, text) => {
   try { const pasted = await injectText(String(text || '')); return { ok: true, pasted }; } catch (err) { return { ok: false, error: String(err && err.message ? err.message : err) }; }

electron/preload.js

@@ -65,6 +65,7 @@ contextBridge.exposeInMainWorld('openswarm', {
   // Voice dictation (local whisper.cpp). transcribe takes a 16kHz-mono WAV ArrayBuffer; inject pastes
   // text into the frontmost app; warmup pre-loads the model; onVoiceToggle fires on the global hotkey.
   voiceWarmup: () => ipcRenderer.invoke('voice:warmup'),
+  voiceStatus: () => ipcRenderer.invoke('voice:status'),
   voiceTranscribe: (wavArrayBuffer) => ipcRenderer.invoke('voice:transcribe', wavArrayBuffer),
   voiceInject: (text) => ipcRenderer.invoke('voice:inject', text),
   onVoiceToggle: (cb) => {

electron/voice/whisperService.js

@@ -6,8 +6,50 @@
 const { spawn } = require('child_process');
 const path = require('path');
 const fs = require('fs');
+const https = require('https');
 
 const MODEL_FILE = 'ggml-base.en.bin';
+const MODEL_URL = 'https://huggingface.co/ggerganov/whisper.cpp/resolve/main/ggml-base.en.bin';
+
+// First-run model fetch, so a dev build (or a prod build that shipped without the model) still works
+// instead of dead-ending on "no model". Progress is exposed so the pill can say "Preparing voice 40%".
+const download = { active: false, pct: 0, error: null };
+
+function downloadModel(dest) {
+  if (download.active) return;
+  download.active = true;
+  download.pct = 0;
+  download.error = null;
+  try { fs.mkdirSync(path.dirname(dest), { recursive: true }); } catch (_) {}
+  const tmp = `${dest}.part`;
+  const file = fs.createWriteStream(tmp);
+  const req = https.get(MODEL_URL, (res) => {
+    if (res.statusCode && res.statusCode >= 300 && res.statusCode < 400 && res.headers.location) {
+      // Follow HuggingFace's CDN redirect once.
+      https.get(res.headers.location, (r2) => pipeTo(r2, file, tmp, dest)).on('error', onErr);
+      return;
+    }
+    pipeTo(res, file, tmp, dest);
+  });
+  req.on('error', onErr);
+  function onErr(e) { download.active = false; download.error = String(e && e.message ? e.message : e); try { file.close(); fs.unlinkSync(tmp); } catch (_) {} }
+}
+
+function pipeTo(res, file, tmp, dest) {
+  const total = Number(res.headers['content-length'] || 0);
+  let got = 0;
+  res.on('data', (c) => { got += c.length; if (total) download.pct = Math.round((got / total) * 100); });
+  res.pipe(file);
+  file.on('finish', () => file.close(() => {
+    try { fs.renameSync(tmp, dest); download.pct = 100; } catch (e) { download.error = String(e); }
+    download.active = false;
+  }));
+  res.on('error', () => { download.active = false; download.error = 'stream-error'; try { fs.unlinkSync(tmp); } catch (_) {} });
+}
+
+function modelStatus() {
+  return { downloading: download.active, pct: download.pct, error: download.error };
+}
 
 // Resolve the whisper-server binary. Env override wins (dev convenience), then the bundled per-arch
 // copy, then whatever is on PATH so a dev machine with `brew install whisper-cpp` just works.
@@ -66,7 +108,9 @@ async function ensureServer(resourceDir, userDataDir) {
     const model = resolveModel(resourceDir, userDataDir);
     if (!model) {
       readyPromise = null;
-      throw new Error('no-model'); // caller surfaces a "voice model missing" state, never crashes
+      // Kick off a one-time background fetch to the dev cache so the NEXT dictation just works.
+      downloadModel(path.join(userDataDir, 'whisper', MODEL_FILE));
+      throw new Error(download.active ? 'model-downloading' : 'no-model');
     }
     const p = pickPort();
     const child = spawn(bin, ['-m', model, '--port', String(p), '-nt', '--convert'], {
@@ -109,4 +153,4 @@ function stopServer() {
   readyPromise = null;
 }
 
-module.exports = { ensureServer, transcribe, stopServer, resolveBinary, resolveModel };
+module.exports = { ensureServer, transcribe, stopServer, resolveBinary, resolveModel, modelStatus };

frontend/src/app/components/Layout/AppShell.tsx

@@ -10,6 +10,7 @@ import ListItemIcon from '@mui/material/ListItemIcon';
 import ListItemText from '@mui/material/ListItemText';
 import Menu from '@mui/material/Menu';
 import MenuItem from '@mui/material/MenuItem';
+import { VoiceDictationProvider } from '@/shared/voice/VoiceDictationContext';
 import Typography from '@mui/material/Typography';
 import IconButton from '@mui/material/IconButton';
 import Tooltip from '@mui/material/Tooltip';
@@ -1399,24 +1400,28 @@ const AppShell: React.FC = () => {
         ml: fsHideChrome ? 0 : '6px',
         borderRadius: fsHideChrome ? 0 : '14px',
       }}>
-        {/* Hidden (not unmounted) when the dashboard view is active so the persistent Dashboard layered above can take over. */}
-        <Box
-          sx={{
-            position: 'absolute',
-            inset: 0,
-            visibility: isDashboardViewActive ? 'hidden' : 'visible',
-            pointerEvents: isDashboardViewActive ? 'none' : 'auto',
-          }}
-        >
-          <Outlet />
-        </Box>
+        {/* One voice controller wraps BOTH the routed content and the persistent Dashboard host, so
+            the spawn-pill mic (which lives in the persistent host, not the Outlet) shares the recorder. */}
+        <VoiceDictationProvider>
+          {/* Hidden (not unmounted) when the dashboard view is active so the persistent Dashboard layered above can take over. */}
+          <Box
+            sx={{
+              position: 'absolute',
+              inset: 0,
+              visibility: isDashboardViewActive ? 'hidden' : 'visible',
+              pointerEvents: isDashboardViewActive ? 'none' : 'auto',
+            }}
+          >
+            <Outlet />
+          </Box>
 
-        {/* CSS-hidden on other routes so webviews + state survive nav. */}
-        {lastDashboardId && (
-          <DashboardHost visible={isDashboardViewActive}>
-            <Dashboard dashboardId={lastDashboardId} isActive={isDashboardViewActive} />
-          </DashboardHost>
-        )}
+          {/* CSS-hidden on other routes so webviews + state survive nav. */}
+          {lastDashboardId && (
+            <DashboardHost visible={isDashboardViewActive}>
+              <Dashboard dashboardId={lastDashboardId} isActive={isDashboardViewActive} />
+            </DashboardHost>
+          )}
+        </VoiceDictationProvider>
       </Box>
       </Box>
 

frontend/src/app/pages/Dashboard/desktop/DesktopSpawnPill.tsx

@@ -4,6 +4,9 @@ import Typography from '@mui/material/Typography';
 import Tooltip from '@mui/material/Tooltip';
 import AddRounded from '@mui/icons-material/AddRounded';
 import MicNoneOutlinedIcon from '@mui/icons-material/MicNoneOutlined';
+import MicIcon from '@mui/icons-material/Mic';
+import CircularProgress from '@mui/material/CircularProgress';
+import { useVoice } from '@/shared/voice/VoiceDictationContext';
 import GridViewRoundedIcon from '@mui/icons-material/GridViewRounded';
 import StickyNote2OutlinedIcon from '@mui/icons-material/StickyNote2Outlined';
 import HistoryRoundedIcon from '@mui/icons-material/HistoryRounded';
@@ -38,6 +41,11 @@ function DesktopSpawnPill({
 }: DesktopSpawnPillProps): React.ReactElement {
   const [menuOpen, setMenuOpen] = useState(false);
   const rootRef = useRef<HTMLDivElement | null>(null);
+  const { state: voiceState, pct: voicePct, toggle: toggleVoice } = useVoice();
+  const recording = voiceState === 'recording';
+  const transcribing = voiceState === 'transcribing';
+  const preparing = voiceState === 'preparing';
+  const voiceBusy = transcribing || preparing;
 
   useEffect(() => {
     if (!menuOpen) return undefined;
@@ -152,8 +160,11 @@ function DesktopSpawnPill({
         >
           <AddRounded sx={{ fontSize: 18 }} />
         </Box>
-        <Tooltip title="Voice input (coming soon)" placement="top" arrow>
+        <Tooltip title={recording ? 'Stop dictation' : preparing ? `Downloading voice model ${voicePct}%` : 'Dictate (Cmd+Shift+D)'} placement="top" arrow>
           <Box
+            role="button"
+            aria-label="Voice dictation"
+            onClick={(e) => { e.stopPropagation(); if (!voiceBusy) toggleVoice(); }}
             sx={{
               display: 'flex',
               alignItems: 'center',
@@ -161,11 +172,16 @@ function DesktopSpawnPill({
               width: 22,
               height: 22,
               borderRadius: '50%',
-              color: 'rgba(255,255,255,0.45)',
-              cursor: 'default',
+              color: recording ? '#ff8a8a' : 'rgba(255,255,255,0.6)',
+              cursor: voiceBusy ? 'default' : 'pointer',
+              '&:hover': { color: '#fff', background: 'rgba(255,255,255,0.12)' },
             }}
           >
-            <MicNoneOutlinedIcon sx={{ fontSize: 16 }} />
+            {voiceBusy
+              ? <CircularProgress size={14} thickness={5} sx={{ color: 'rgba(255,255,255,0.7)' }} />
+              : recording
+                ? <MicIcon sx={{ fontSize: 16 }} />
+                : <MicNoneOutlinedIcon sx={{ fontSize: 16 }} />}
           </Box>
         </Tooltip>
       </Box>

frontend/src/app/pages/Dashboard/desktop/HelpPill.tsx

@@ -7,16 +7,18 @@ import MicNoneOutlinedIcon from '@mui/icons-material/MicNoneOutlined';
 import MicIcon from '@mui/icons-material/Mic';
 import { useAppDispatch } from '@/shared/hooks';
 import { addBrowserCard } from '@/shared/state/dashboardLayoutSlice';
-import { useVoiceDictation } from './useVoiceDictation';
+import { useVoice } from '@/shared/voice/VoiceDictationContext';
 
 const HELP_URL = 'https://openswarm.com';
 
 /** Top-right desktop pill: Help opens the docs; the mic dictates (local whisper) into the focused field. */
 function HelpPill(): React.ReactElement {
   const dispatch = useAppDispatch();
-  const { state, toggle } = useVoiceDictation();
+  const { state, pct, toggle } = useVoice();
   const recording = state === 'recording';
   const transcribing = state === 'transcribing';
+  const preparing = state === 'preparing';
+  const busy = transcribing || preparing;
 
   return (
     <Box
@@ -39,14 +41,14 @@ function HelpPill(): React.ReactElement {
       onClick={() => dispatch(addBrowserCard({ url: HELP_URL }))}
     >
       <Typography sx={{ fontSize: '0.78rem', color: 'rgba(255,255,255,0.72)', fontWeight: 500 }}>
-        {recording ? 'Listening' : transcribing ? 'Transcribing' : 'Help'}
+        {recording ? 'Listening' : transcribing ? 'Transcribing' : preparing ? `Preparing ${pct}%` : 'Help'}
       </Typography>
-      <Tooltip title={recording ? 'Stop dictation' : 'Dictate (Cmd+Shift+D)'} placement="bottom" arrow>
+      <Tooltip title={recording ? 'Stop dictation' : preparing ? 'Downloading voice model' : 'Dictate (Cmd+Shift+D)'} placement="bottom" arrow>
         <Box
           sx={{ display: 'flex', alignItems: 'center', color: recording ? '#fff' : 'rgba(255,255,255,0.55)' }}
-          onClick={(e) => { e.stopPropagation(); if (!transcribing) toggle(); }}
+          onClick={(e) => { e.stopPropagation(); if (!busy) toggle(); }}
         >
-          {transcribing
+          {busy
             ? <CircularProgress size={13} thickness={5} sx={{ color: 'rgba(255,255,255,0.7)' }} />
             : recording
               ? <MicIcon sx={{ fontSize: 16 }} />

frontend/src/shared/voice/VoiceDictationContext.tsx

@@ -0,0 +1,31 @@
+import React, { createContext, useContext } from 'react';
+import { useVoiceDictation, VoiceState } from './useVoiceDictation';
+
+// One recorder for the whole app. Both mics (the Help pill and the spawn composer) plus the global
+// hotkey drive the SAME dictation session, so two mics can't fight over the microphone or show
+// out-of-sync state. Mounted once near the app root.
+interface VoiceContextValue {
+  state: VoiceState;
+  lastText: string;
+  error: string | null;
+  pct: number;
+  toggle: () => void;
+}
+
+const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, toggle: () => {} };
+const VoiceContext = createContext<VoiceContextValue>(NOOP);
+
+export function VoiceDictationProvider({ children }: { children: React.ReactNode }): React.ReactElement {
+  const { state, lastText, error, pct, toggle } = useVoiceDictation();
+  return (
+    <VoiceContext.Provider value={{ state, lastText, error, pct, toggle }}>
+      {children}
+    </VoiceContext.Provider>
+  );
+}
+
+// A component rendered outside the provider (or a web build with no Electron bridge) gets the no-op,
+// so mics still render and just do nothing rather than crashing.
+export function useVoice(): VoiceContextValue {
+  return useContext(VoiceContext);
+}

frontend/src/shared/voice/useVoiceDictation.ts

@@ -1,9 +1,9 @@
 import { useCallback, useEffect, useRef, useState } from 'react';
-import { encodeWav, VOICE_SAMPLE_RATE } from '@/shared/voice/encodeWav';
+import { encodeWav, VOICE_SAMPLE_RATE } from './encodeWav';
 
-export type VoiceState = 'idle' | 'recording' | 'transcribing';
+export type VoiceState = 'idle' | 'recording' | 'transcribing' | 'preparing';
 
-// WhisperFlow-style push-to-dictate: toggle recording (global hotkey or the pill), speak, and the
+// WhisperFlow-style push-to-dictate: toggle recording (global hotkey or a mic), speak, and the
 // transcribed text is pasted into whatever field has focus. Capture is 16kHz mono PCM so it feeds
 // whisper.cpp with no server-side resample. The recording path can only be proven with a real mic;
 // the encode -> transcribe -> inject half is exercised by the encoder round-trip test.
@@ -19,10 +19,25 @@ export function useVoiceDictation() {
   const [state, setState] = useState<VoiceState>('idle');
   const [lastText, setLastText] = useState<string>('');
   const [error, setError] = useState<string | null>(null);
+  const [pct, setPct] = useState<number>(0);
   const recRef = useRef<Recorder | null>(null);
   const stateRef = useRef<VoiceState>('idle');
   stateRef.current = state;
 
+  // First-run: the model is downloading. Poll progress until it lands, then drop back to idle so the
+  // next click records for real. Never records while preparing, so nothing is lost to a dropped phrase.
+  const pollModel = useCallback((): void => {
+    const tick = async (): Promise<void> => {
+      const st = await window.openswarm?.voiceStatus?.();
+      if (!st) { setState('idle'); return; }
+      setPct(st.pct || 0);
+      if (st.error) { setError(st.error); setState('idle'); return; }
+      if (!st.downloading) { setState('idle'); return; }
+      setTimeout(() => { void tick(); }, 1000);
+    };
+    void tick();
+  }, []);
+
   const teardown = useCallback((): Float32Array | null => {
     const rec = recRef.current;
     recRef.current = null;
@@ -41,6 +56,7 @@ export function useVoiceDictation() {
 
   const start = useCallback(async (): Promise<void> => {
     if (stateRef.current !== 'idle') return;
+    if (!window.openswarm?.voiceTranscribe) { setError('desktop-only'); return; } // no Electron bridge = web build
     setError(null);
     try {
       const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } });
@@ -72,15 +88,20 @@ export function useVoiceDictation() {
       if (res?.ok && res.text) {
         setLastText(res.text);
         await window.openswarm?.voiceInject?.(res.text);
+        setState('idle');
+      } else if (res?.error === 'model-downloading' || res?.error === 'no-model') {
+        // First use kicked off the model fetch; show progress and don't error out.
+        setState('preparing');
+        pollModel();
       } else {
         setError(res?.error || 'transcription-failed');
+        setState('idle');
       }
     } catch (err) {
       setError(err instanceof Error ? err.message : 'transcription-failed');
-    } finally {
       setState('idle');
     }
-  }, [teardown]);
+  }, [teardown, pollModel]);
 
   const toggle = useCallback((): void => {
     if (stateRef.current === 'recording') void stop();
@@ -96,5 +117,5 @@ export function useVoiceDictation() {
   // A dangling recorder (unmount mid-capture) must release the mic.
   useEffect(() => () => { teardown(); }, [teardown]);
 
-  return { state, lastText, error, toggle, start, stop };
+  return { state, lastText, error, pct, toggle, start, stop };
 }

frontend/src/types/electron.d.ts

@@ -55,6 +55,7 @@ declare global {
     hardReset?: () => Promise<void>;
     clearBrowserData?: () => Promise<{ ok: boolean }>;
     voiceWarmup?: () => Promise<{ ok: boolean; error?: string }>;
+    voiceStatus?: () => Promise<{ downloading: boolean; pct: number; error: string | null }>;
     voiceTranscribe?: (wav: ArrayBuffer) => Promise<{ ok: boolean; text?: string; error?: string }>;
     voiceInject?: (text: string) => Promise<{ ok: boolean; pasted?: boolean; error?: string }>;
     onVoiceToggle?: (cb: () => void) => () => void;