openswarm/electron/voice/whisperService.js

// Local speech-to-text via whisper.cpp, kept WARM so a phrase transcribes in ~0.2s instead of the
// ~16s cold-model-load a fresh CLI pays every time. We spawn `whisper-server` once (model loaded),
// then POST audio to it per utterance. Same "bundle a binary + manage its lifecycle" shape as the
// 9router subprocess: dev uses the system whisper.cpp, prod uses the per-arch binary + model we ship.

const { spawn } = require('child_process');
const path = require('path');
const fs = require('fs');
const net = require('net');
const os = require('os');
const whisperModels = require('./whisperModels');

// Which catalog model the user picked. Settings pushes it in; until then the catalog default wins.
let selectedModelId = whisperModels.DEFAULT_MODEL_ID;

function modelStatus() {
  return whisperModels.downloadStatus();
}

// Resolve the whisper-server binary. Env override wins (dev convenience), then the bundled per-arch
// copy, then whatever is on PATH so a dev machine with `brew install whisper-cpp` just works.
function resolveBinary(resourceDir) {
  if (process.env.OPENSWARM_WHISPER_BIN && fs.existsSync(process.env.OPENSWARM_WHISPER_BIN)) {
    return process.env.OPENSWARM_WHISPER_BIN;
  }
  const exe = process.platform === 'win32' ? 'whisper-server.exe' : 'whisper-server';
  const bundled = path.join(resourceDir, exe);
  if (fs.existsSync(bundled)) return bundled;
  const brew = process.platform === 'win32' ? null : '/opt/homebrew/bin/whisper-server';
  if (brew && fs.existsSync(brew)) return brew;
  return exe; // last resort: hope it is on PATH
}

function resolveModel(resourceDir, userDataDir) {
  return whisperModels.resolveModelFile(resourceDir, userDataDir, selectedModelId);
}

let proc = null;
let port = 0;
let readyPromise = null;
let idleTimer = null;
let loadedModelFile = null;

// A warm server holds ~210MB (measured, base.en). Free it after a long quiet spell; the reload that
// costs is the FIRST one on a cold page cache, and boot-warm already paid that.
const IDLE_UNLOAD_MS = 10 * 60 * 1000;

// Let the OS name a free port instead of guessing one. A guessed port that is already taken makes
// whisper exit(1) AND makes our readiness probe accept the squatter's reply as proof of life, so we
// would happily POST the user's audio at a stranger.
function freePort() {
  return new Promise((resolve, reject) => {
    const probe = net.createServer();
    probe.once('error', reject);
    probe.listen(0, '127.0.0.1', () => {
      const addr = probe.address();
      probe.close(() => resolve(addr.port));
    });
  });
}

async function waitForReady(child, p, timeoutMs) {
  const deadline = Date.now() + timeoutMs;
  while (Date.now() < deadline) {
    if (child.exitCode !== null || child.signalCode !== null) return false; // died mid-load; stop waiting
    try {
      const res = await fetch(`http://127.0.0.1:${p}/`, { method: 'GET' });
      if (res.status) return true; // the socket is ours and it is serving
    } catch (_) { /* not up yet */ }
    await new Promise((r) => setTimeout(r, 50));
  }
  return false;
}

function p_touchIdle() {
  if (idleTimer) clearTimeout(idleTimer);
  idleTimer = setTimeout(() => { idleTimer = null; stopServer(); }, IDLE_UNLOAD_MS);
  if (idleTimer.unref) idleTimer.unref(); // an idle countdown must never be the reason the app won't quit
}

// 44-byte RIFF header + silence, 16kHz mono, matching what the renderer sends.
function p_silentWav(seconds) {
  const samples = Math.round(16000 * seconds);
  const buf = Buffer.alloc(44 + samples * 2);
  buf.write('RIFF', 0); buf.writeUInt32LE(36 + samples * 2, 4); buf.write('WAVE', 8);
  buf.write('fmt ', 12); buf.writeUInt32LE(16, 16); buf.writeUInt16LE(1, 20); buf.writeUInt16LE(1, 22);
  buf.writeUInt32LE(16000, 24); buf.writeUInt32LE(32000, 28); buf.writeUInt16LE(2, 32); buf.writeUInt16LE(16, 34);
  buf.write('data', 36); buf.writeUInt32LE(samples * 2, 40);
  return buf;
}

// A loaded model is not a ready one: the first inference pays a one-off graph/kernel allocation
// (measured ~180ms on an M2). Spend it on silence at boot so the user's first phrase doesn't.
async function p_primeGraph(p) {
  try {
    const form = new FormData();
    form.append('file', new Blob([p_silentWav(0.5)], { type: 'audio/wav' }), 'warm.wav');
    form.append('response_format', 'text');
    const res = await fetch(`http://127.0.0.1:${p}/inference`, { method: 'POST', body: form });
    await res.text();
  } catch (_) { /* priming is an optimization; a failure just means the first phrase pays it */ }
}

// Pull the one line a human can act on out of whisper's chatty output.
function p_reasonFrom(tail) {
  const lines = tail.split('\n').map((l) => l.trim()).filter(Boolean);
  const blame = lines.filter((l) => /error|failed|invalid|unable|cannot|no such|not found/i.test(l));
  const pick = blame.length ? blame[blame.length - 1] : lines[lines.length - 1];
  return pick ? `: ${pick.slice(0, 200)}` : '';
}

async function p_bootServer(resourceDir, userDataDir) {
  const bin = resolveBinary(resourceDir);
  const model = resolveModel(resourceDir, userDataDir);
  if (!model) {
    // Kick off a one-time background fetch so the NEXT dictation just works.
    whisperModels.downloadModel(userDataDir, selectedModelId);
    throw new Error(whisperModels.downloadStatus().downloading ? 'model-downloading' : 'no-model');
  }
  loadedModelFile = model;
  const p = await freePort();
  // No --convert: our WAV is already 16kHz mono, and the flag makes whisper demand ffmpeg on PATH at boot; a Finder-launched app has no brew PATH, so it exited before ever binding the port.
  const child = spawn(bin, ['-m', model, '--port', String(p), '-nt'], {
    cwd: os.tmpdir(), // whisper writes per-request temp files beside cwd, and a Finder launch starts at the unwritable /
    // BOTH pipes: whisper writes its fatal reasons to STDOUT and then exits 0, so an ignored stdout
    // turns "ffmpeg is missing" into an unexplained failure. Draining also stops the pipe buffer
    // filling and blocking the child.
    stdio: ['ignore', 'pipe', 'pipe'],
  });
  // Keep a rolling tail rather than the last chunk: whisper prints its real reason and THEN keeps
  // banner-dumping, so "the most recent bytes" is reliably the least useful line it wrote.
  let tail = '';
  const drain = (c) => {
    const said = String(c);
    if (!said.trim()) return;
    tail = (tail + said).slice(-4000);
    console.log('[voice] whisper:', said.trim().slice(0, 400));
  };
  child.stdout.on('data', drain);
  child.stderr.on('data', drain);
  child.on('error', () => { proc = null; port = 0; });
  child.on('exit', (code) => { if (code) console.log(`[voice] whisper-server exited code=${code}`); proc = null; port = 0; readyPromise = null; });
  // Cold model load measured 15-38s on an M2; the old 20s budget timed out real first uses.
  const ok = await waitForReady(child, p, 60000);
  if (!ok) {
    try { child.kill(); } catch (_) {}
    // A dead child is not a slow one. Whisper can die in ~0.1s with exit code 0 (a missing ffmpeg on
    // a Finder-launched PATH does exactly that), so report ITS reason instantly instead of making the
    // user sit through the full ready budget for a process that was never coming back.
    if (child.exitCode !== null || child.signalCode !== null) {
      throw new Error(`whisper-exited-${child.exitCode}${p_reasonFrom(tail)}`);
    }
    throw new Error('server-timeout');
  }
  proc = child;
  port = p;
  await p_primeGraph(p);
  p_touchIdle();
  return p;
}

// Boot the warm server once. resourceDir = where a packaged build put the binary+model; userDataDir
// = app.getPath('userData') for the dev cache. Returns the port, or throws with an actionable reason.
// The readyPromise is cleared AFTER it settles, never synchronously inside the async body: the old
// code reset it inside the IIFE where the outer assignment immediately overwrote the null, pinning a
// settled-rejected promise forever so every later call kept throwing "model-downloading" even after
// the model finished. Clearing on rejection here lets the next call retry cleanly.
async function ensureServer(resourceDir, userDataDir) {
  // A warm server is only reusable if it holds the file we would load now: a model switch, or the
  // user's pick finishing its download while a fallback was serving, has to re-boot.
  if (proc && port && resolveModel(resourceDir, userDataDir) !== loadedModelFile) stopServer();
  if (proc && port) return port;
  if (readyPromise) return readyPromise;
  readyPromise = p_bootServer(resourceDir, userDataDir);
  try {
    return await readyPromise;
  } catch (err) {
    readyPromise = null;
    throw err;
  }
}

// Transcribe a 16kHz-mono WAV buffer to text. The renderer records + encodes the WAV so the audio
// never crosses a CORS boundary; we POST from the main process where there is none.
async function transcribe(resourceDir, userDataDir, wavBuffer) {
  const p = await ensureServer(resourceDir, userDataDir);
  p_touchIdle();
  const form = new FormData();
  form.append('file', new Blob([wavBuffer], { type: 'audio/wav' }), 'audio.wav');
  form.append('response_format', 'text');
  const res = await fetch(`http://127.0.0.1:${p}/inference`, { method: 'POST', body: form });
  if (!res.ok) throw new Error(`whisper-http-${res.status}`);
  const text = (await res.text()).trim();
  return text;
}

// Boot the model in the background at app start so the expensive FIRST load (cold page cache, and on
// a packaged build the OS's first-exec check of the bundled binary) never lands under a keypress.
// Deliberately refuses to download: a user who never dictates should not silently pull 148MB.
// Returns whether a warm was actually started, so the caller can log the honest reason.
function warmInBackground(resourceDir, userDataDir) {
  if (proc || readyPromise) return true; // already warm or warming; ensureServer dedupes anyway
  if (!resolveModel(resourceDir, userDataDir)) return false;
  ensureServer(resourceDir, userDataDir).catch(() => {});
  return true;
}

function stopServer() {
  if (idleTimer) { clearTimeout(idleTimer); idleTimer = null; }
  if (proc) {
    try { proc.kill(); } catch (_) {}
  }
  proc = null;
  port = 0;
  readyPromise = null;
  loadedModelFile = null;
}

// Settings picked a different model. Downloads it if missing; the running server keeps serving the
// old one until the new file is complete, so switching never leaves dictation dead in between.
function setModel(userDataDir, id) {
  const next = whisperModels.modelById(id).id;
  if (next === selectedModelId) return whisperModels.isInstalled(userDataDir, next);
  selectedModelId = next;
  if (whisperModels.isInstalled(userDataDir, next)) return true;
  whisperModels.downloadModel(userDataDir, next);
  return false;
}

function selectedModel() {
  return selectedModelId;
}

function isWarm() {
  return Boolean(proc && port);
}

// Sleep evicts the GPU-side state a warm server built, so the first phrase after a lid-open pays for
// it again. Re-prime on wake instead of restarting: same fix openwhispr and VoiceInk landed.
async function reprimeAfterWake() {
  if (!isWarm()) return false;
  await p_primeGraph(port);
  return true;
}

module.exports = { ensureServer, warmInBackground, reprimeAfterWake, transcribe, stopServer, isWarm, setModel, selectedModel, resolveBinary, resolveModel, modelStatus };