[eric] voice: text-landed chime cut, haptics toggle, full-clip decode wins at stop, beam search 5; dock tile says Browsers

This commit is contained in:
ciregenz
2026-08-06 00:02:52 -07:00
parent 69805dfbe7
commit 5130309a72
8 changed files with 40 additions and 26 deletions
+1
View File
@@ -98,6 +98,7 @@ class AppSettings(BaseModel):
# Personal glossary (comma-separated names/jargon) fed to whisper as a decode prompt so "Anthropic" never comes out "and Thropic".
dictation_dictionary: str = ""
dictation_sounds: bool = True
dictation_haptics: bool = True
# 0..1; the cue loudness Eric tuned by ear rides here instead of a hardcode.
dictation_sound_volume: float = 0.35
# Comma-separated hostnames (and app names) where dictation refuses to record while focused there.
+3 -1
View File
@@ -153,7 +153,9 @@ async function p_bootServer(resourceDir, userDataDir) {
p_sweepStrays(bin);
const p = await freePort();
// No --convert: our WAV is already 16kHz mono, and the flag makes whisper demand ffmpeg on PATH at boot; a Finder-launched app has no brew PATH, so it exited before ever binding the port.
const args = ['-m', model, '--port', String(p), '-nt'];
// Beam search (5) over greedy: measurably fewer recognition errors at ~1.5x decode cost; Eric
// rates accuracy above stop-latency.
const args = ['-m', model, '--port', String(p), '-nt', '-bs', '5'];
// A multilingual model (no .en in the filename) auto-detects the spoken language per utterance.
if (!path.basename(model).includes('.en')) args.push('-l', 'auto');
const child = spawn(bin, args, {
@@ -20,7 +20,7 @@ interface DockActionTilesProps {
function DockActionTiles({ tile, onAddBrowser, onApplications, onHoverAway }: DockActionTilesProps): React.ReactElement {
const dispatch = useAppDispatch();
const actions: { label: string; Icon: typeof Globe; act: () => void; divider?: boolean }[] = [
{ label: 'New browser', Icon: Globe, act: onAddBrowser },
{ label: 'Browsers', Icon: Globe, act: onAddBrowser },
{ label: 'Workflows', Icon: CalendarClock, act: () => dispatch(openWorkflowsApp()) },
{ label: 'Marketplace', Icon: Store, act: () => openMarketplace() },
{ label: 'Settings', Icon: Settings, act: () => dispatch(openSettingsCard()), divider: true },
@@ -84,10 +84,10 @@ const DictationSettings: React.FC<{
/>
</Box>
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_sounds', 'Dictation sounds', 'Interface', 'The start, stop, and text-landed cues.')}>
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_sounds', 'Dictation sounds', 'Interface', 'The start and stop cues.')}>
<Box sx={{ mr: 3 }}>
<Typography sx={labelSx}>Sounds</Typography>
<Typography sx={descSx}>The start, stop, and text-landed cues, and how loud they play.</Typography>
<Typography sx={descSx}>The start and stop cues, and how loud they play.</Typography>
</Box>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1.5 }}>
<Slider
@@ -107,6 +107,18 @@ const DictationSettings: React.FC<{
</Box>
</Box>
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_haptics', 'Dictation haptics', 'Interface', 'Trackpad taps on start and stop.')}>
<Box sx={{ mr: 3 }}>
<Typography sx={labelSx}>Haptics</Typography>
<Typography sx={descSx}>Trackpad taps when dictation starts and stops.</Typography>
</Box>
<Switch
size="small"
checked={form.dictation_haptics ?? true}
onChange={(e) => setForm({ ...form, dictation_haptics: e.target.checked })}
/>
</Box>
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_disabled_surfaces', 'Dictation off for sites', 'Interface', 'Sites where the dictation key does nothing.')}>
<Box sx={{ mr: 3 }}>
<Typography sx={labelSx}>Off for sites</Typography>
@@ -34,6 +34,7 @@ export interface AppSettings {
dictation_model?: string | null;
dictation_dictionary?: string;
dictation_sounds?: boolean;
dictation_haptics?: boolean;
dictation_sound_volume?: number;
dictation_disabled_surfaces?: string;
anthropic_api_key: string | null;
@@ -169,6 +170,7 @@ export const DEFAULT_SETTINGS: AppSettings = {
dictation_model: null,
dictation_dictionary: '',
dictation_sounds: true,
dictation_haptics: true,
dictation_sound_volume: 0.35,
dictation_disabled_surfaces: '',
anthropic_api_key: null,
+12 -12
View File
@@ -199,7 +199,7 @@ export function useVoiceDictation() {
recRef.current = { ctx, stream, node: capture.node, requestFlush: capture.requestFlush, source, chunks, streaming };
setState('recording');
playVoiceCue('start');
void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('generic');
if (store.getState().settings.data.dictation_haptics ?? true) void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('generic');
} catch (err) {
// Release whatever was acquired before the failure, or the OS mic indicator stays lit forever on a half-started session.
try { stream?.getTracks().forEach((t) => t.stop()); } catch { /* already dead */ }
@@ -220,7 +220,7 @@ export function useVoiceDictation() {
if (rec) await rec.requestFlush();
const samples = teardown();
playVoiceCue('stop');
void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('alignment');
if (store.getState().settings.data.dictation_haptics ?? true) void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('alignment');
setState('transcribing');
try {
if (!samples || samples.length < VOICE_SAMPLE_RATE * 0.2) { // < 0.2s = a misfire
@@ -229,14 +229,16 @@ export function useVoiceDictation() {
setState('idle');
return;
}
// The streamed assembly (each phrase decoded once at its boundary) is the fast path: stop only
// pays for the final open phrase. Any doubt (degraded, empty) falls back to one full-clip decode.
let res: { ok: boolean; text?: string; error?: string } | undefined;
// Accuracy first (Eric's call): phrases decoded in isolation lose whisper's cross-phrase
// context, which read as "sometimes fine, sometimes off". The FULL clip is always decoded at
// stop and wins; the streamed assembly only stands in if the batch decode fails outright.
let streamedFallback: string | null = null;
if (streaming) {
const sres = await window.openswarm?.voiceStreamStop?.();
if (sres?.ok && sres.text && !sres.degraded) res = { ok: true, text: sres.text };
if (sres?.ok && sres.text) streamedFallback = sres.text;
}
if (!res) {
let res: { ok: boolean; text?: string; error?: string } | undefined;
{
// Whisper hallucinates plausible punctuation on near-silence; a clip whose level never beat the streaming RMS gate gets "didn't catch that", never a decode.
let sumSq = 0;
for (let i = 0; i < samples.length; i += 4) sumSq += samples[i] * samples[i];
@@ -246,6 +248,7 @@ export function useVoiceDictation() {
} else {
const wav = encodeWav(samples);
res = await window.openswarm?.voiceTranscribe?.(wav);
if ((!res || !res.ok || !res.text) && streamedFallback) res = { ok: true, text: streamedFallback };
}
}
// Whisper captions non-speech in brackets/parens ("[ Background sounds ]", "(laughs)") and marks speaker turns with ">>"; those are annotations, not dictation.
@@ -275,12 +278,9 @@ export function useVoiceDictation() {
const target = injectAtFocus(text);
pushDictation(text, target || 'clipboard');
learnFromTranscript(text);
if (target) {
playVoiceCue('paste');
} else {
if (!target) {
const inj = await window.openswarm?.voiceInject?.(text);
if (inj?.pasted) playVoiceCue('paste');
else setFeedback({ tone: 'ok', icon: 'clipboard', text: `${text} (copied, press Cmd+V)`, at: Date.now() });
if (!inj?.pasted) setFeedback({ tone: 'ok', icon: 'clipboard', text: `${text} (copied, press Cmd+V)`, at: Date.now() });
}
setState('idle');
} else if (res?.ok && !res.text) {
File diff suppressed because one or more lines are too long
+6 -8
View File
@@ -1,12 +1,11 @@
import { VOICE_CUE_START, VOICE_CUE_STOP, VOICE_CUE_PASTE, VOICE_CUE_LOCK } from './voiceCueSounds';
import { VOICE_CUE_START, VOICE_CUE_STOP, VOICE_CUE_LOCK } from './voiceCueSounds';
// Eric picked these from Google's Material product sound set after a bake-off against Wispr Flow's
// real cues; the synth versions never survived an ear test. Files are embedded data URIs (see
// voiceCueSounds.ts), pre-instantiated so playback is instant on the press. The grammar mirrors
// Wispr's three-beat: tap in, tap out, and a rising completion the moment the text actually lands;
// lock marks a hands-free latch.
// voiceCueSounds.ts), pre-instantiated so playback is instant on the press. Start and stop taps plus the
// hands-free lock mark; Eric explicitly cut the text-landed chime.
type CueKind = 'start' | 'stop' | 'paste' | 'lock';
type CueKind = 'start' | 'stop' | 'lock';
// Pushed from Settings (dictation_sounds / dictation_sound_volume); defaults match the shipped feel.
let cueEnabled = true;
@@ -17,14 +16,13 @@ export function configureVoiceCues(enabled: boolean, volume: number): void {
cueVolume = Math.min(1, Math.max(0, volume));
for (const kind of Object.keys(p_players) as CueKind[]) {
const a = p_players[kind];
if (a) a.volume = kind === 'paste' ? cueVolume * 0.8 : cueVolume;
if (a) a.volume = cueVolume;
}
}
const SOURCES: Record<CueKind, string> = {
start: VOICE_CUE_START,
stop: VOICE_CUE_STOP,
paste: VOICE_CUE_PASTE,
lock: VOICE_CUE_LOCK,
};
@@ -34,7 +32,7 @@ function player(kind: CueKind): HTMLAudioElement {
let a = p_players[kind];
if (!a) {
a = new Audio(SOURCES[kind]);
a.volume = kind === 'paste' ? cueVolume * 0.8 : cueVolume;
a.volume = cueVolume;
p_players[kind] = a;
}
return a;