mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] voice: text-landed chime cut, haptics toggle, full-clip decode wins at stop, beam search 5; dock tile says Browsers
This commit is contained in:
@@ -98,6 +98,7 @@ class AppSettings(BaseModel):
|
||||
# Personal glossary (comma-separated names/jargon) fed to whisper as a decode prompt so "Anthropic" never comes out "and Thropic".
|
||||
dictation_dictionary: str = ""
|
||||
dictation_sounds: bool = True
|
||||
dictation_haptics: bool = True
|
||||
# 0..1; the cue loudness Eric tuned by ear rides here instead of a hardcode.
|
||||
dictation_sound_volume: float = 0.35
|
||||
# Comma-separated hostnames (and app names) where dictation refuses to record while focused there.
|
||||
|
||||
@@ -153,7 +153,9 @@ async function p_bootServer(resourceDir, userDataDir) {
|
||||
p_sweepStrays(bin);
|
||||
const p = await freePort();
|
||||
// No --convert: our WAV is already 16kHz mono, and the flag makes whisper demand ffmpeg on PATH at boot; a Finder-launched app has no brew PATH, so it exited before ever binding the port.
|
||||
const args = ['-m', model, '--port', String(p), '-nt'];
|
||||
// Beam search (5) over greedy: measurably fewer recognition errors at ~1.5x decode cost; Eric
|
||||
// rates accuracy above stop-latency.
|
||||
const args = ['-m', model, '--port', String(p), '-nt', '-bs', '5'];
|
||||
// A multilingual model (no .en in the filename) auto-detects the spoken language per utterance.
|
||||
if (!path.basename(model).includes('.en')) args.push('-l', 'auto');
|
||||
const child = spawn(bin, args, {
|
||||
|
||||
@@ -20,7 +20,7 @@ interface DockActionTilesProps {
|
||||
function DockActionTiles({ tile, onAddBrowser, onApplications, onHoverAway }: DockActionTilesProps): React.ReactElement {
|
||||
const dispatch = useAppDispatch();
|
||||
const actions: { label: string; Icon: typeof Globe; act: () => void; divider?: boolean }[] = [
|
||||
{ label: 'New browser', Icon: Globe, act: onAddBrowser },
|
||||
{ label: 'Browsers', Icon: Globe, act: onAddBrowser },
|
||||
{ label: 'Workflows', Icon: CalendarClock, act: () => dispatch(openWorkflowsApp()) },
|
||||
{ label: 'Marketplace', Icon: Store, act: () => openMarketplace() },
|
||||
{ label: 'Settings', Icon: Settings, act: () => dispatch(openSettingsCard()), divider: true },
|
||||
|
||||
@@ -84,10 +84,10 @@ const DictationSettings: React.FC<{
|
||||
/>
|
||||
</Box>
|
||||
|
||||
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_sounds', 'Dictation sounds', 'Interface', 'The start, stop, and text-landed cues.')}>
|
||||
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_sounds', 'Dictation sounds', 'Interface', 'The start and stop cues.')}>
|
||||
<Box sx={{ mr: 3 }}>
|
||||
<Typography sx={labelSx}>Sounds</Typography>
|
||||
<Typography sx={descSx}>The start, stop, and text-landed cues, and how loud they play.</Typography>
|
||||
<Typography sx={descSx}>The start and stop cues, and how loud they play.</Typography>
|
||||
</Box>
|
||||
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1.5 }}>
|
||||
<Slider
|
||||
@@ -107,6 +107,18 @@ const DictationSettings: React.FC<{
|
||||
</Box>
|
||||
</Box>
|
||||
|
||||
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_haptics', 'Dictation haptics', 'Interface', 'Trackpad taps on start and stop.')}>
|
||||
<Box sx={{ mr: 3 }}>
|
||||
<Typography sx={labelSx}>Haptics</Typography>
|
||||
<Typography sx={descSx}>Trackpad taps when dictation starts and stops.</Typography>
|
||||
</Box>
|
||||
<Switch
|
||||
size="small"
|
||||
checked={form.dictation_haptics ?? true}
|
||||
onChange={(e) => setForm({ ...form, dictation_haptics: e.target.checked })}
|
||||
/>
|
||||
</Box>
|
||||
|
||||
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_disabled_surfaces', 'Dictation off for sites', 'Interface', 'Sites where the dictation key does nothing.')}>
|
||||
<Box sx={{ mr: 3 }}>
|
||||
<Typography sx={labelSx}>Off for sites</Typography>
|
||||
|
||||
@@ -34,6 +34,7 @@ export interface AppSettings {
|
||||
dictation_model?: string | null;
|
||||
dictation_dictionary?: string;
|
||||
dictation_sounds?: boolean;
|
||||
dictation_haptics?: boolean;
|
||||
dictation_sound_volume?: number;
|
||||
dictation_disabled_surfaces?: string;
|
||||
anthropic_api_key: string | null;
|
||||
@@ -169,6 +170,7 @@ export const DEFAULT_SETTINGS: AppSettings = {
|
||||
dictation_model: null,
|
||||
dictation_dictionary: '',
|
||||
dictation_sounds: true,
|
||||
dictation_haptics: true,
|
||||
dictation_sound_volume: 0.35,
|
||||
dictation_disabled_surfaces: '',
|
||||
anthropic_api_key: null,
|
||||
|
||||
@@ -199,7 +199,7 @@ export function useVoiceDictation() {
|
||||
recRef.current = { ctx, stream, node: capture.node, requestFlush: capture.requestFlush, source, chunks, streaming };
|
||||
setState('recording');
|
||||
playVoiceCue('start');
|
||||
void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('generic');
|
||||
if (store.getState().settings.data.dictation_haptics ?? true) void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('generic');
|
||||
} catch (err) {
|
||||
// Release whatever was acquired before the failure, or the OS mic indicator stays lit forever on a half-started session.
|
||||
try { stream?.getTracks().forEach((t) => t.stop()); } catch { /* already dead */ }
|
||||
@@ -220,7 +220,7 @@ export function useVoiceDictation() {
|
||||
if (rec) await rec.requestFlush();
|
||||
const samples = teardown();
|
||||
playVoiceCue('stop');
|
||||
void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('alignment');
|
||||
if (store.getState().settings.data.dictation_haptics ?? true) void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('alignment');
|
||||
setState('transcribing');
|
||||
try {
|
||||
if (!samples || samples.length < VOICE_SAMPLE_RATE * 0.2) { // < 0.2s = a misfire
|
||||
@@ -229,14 +229,16 @@ export function useVoiceDictation() {
|
||||
setState('idle');
|
||||
return;
|
||||
}
|
||||
// The streamed assembly (each phrase decoded once at its boundary) is the fast path: stop only
|
||||
// pays for the final open phrase. Any doubt (degraded, empty) falls back to one full-clip decode.
|
||||
let res: { ok: boolean; text?: string; error?: string } | undefined;
|
||||
// Accuracy first (Eric's call): phrases decoded in isolation lose whisper's cross-phrase
|
||||
// context, which read as "sometimes fine, sometimes off". The FULL clip is always decoded at
|
||||
// stop and wins; the streamed assembly only stands in if the batch decode fails outright.
|
||||
let streamedFallback: string | null = null;
|
||||
if (streaming) {
|
||||
const sres = await window.openswarm?.voiceStreamStop?.();
|
||||
if (sres?.ok && sres.text && !sres.degraded) res = { ok: true, text: sres.text };
|
||||
if (sres?.ok && sres.text) streamedFallback = sres.text;
|
||||
}
|
||||
if (!res) {
|
||||
let res: { ok: boolean; text?: string; error?: string } | undefined;
|
||||
{
|
||||
// Whisper hallucinates plausible punctuation on near-silence; a clip whose level never beat the streaming RMS gate gets "didn't catch that", never a decode.
|
||||
let sumSq = 0;
|
||||
for (let i = 0; i < samples.length; i += 4) sumSq += samples[i] * samples[i];
|
||||
@@ -246,6 +248,7 @@ export function useVoiceDictation() {
|
||||
} else {
|
||||
const wav = encodeWav(samples);
|
||||
res = await window.openswarm?.voiceTranscribe?.(wav);
|
||||
if ((!res || !res.ok || !res.text) && streamedFallback) res = { ok: true, text: streamedFallback };
|
||||
}
|
||||
}
|
||||
// Whisper captions non-speech in brackets/parens ("[ Background sounds ]", "(laughs)") and marks speaker turns with ">>"; those are annotations, not dictation.
|
||||
@@ -275,12 +278,9 @@ export function useVoiceDictation() {
|
||||
const target = injectAtFocus(text);
|
||||
pushDictation(text, target || 'clipboard');
|
||||
learnFromTranscript(text);
|
||||
if (target) {
|
||||
playVoiceCue('paste');
|
||||
} else {
|
||||
if (!target) {
|
||||
const inj = await window.openswarm?.voiceInject?.(text);
|
||||
if (inj?.pasted) playVoiceCue('paste');
|
||||
else setFeedback({ tone: 'ok', icon: 'clipboard', text: `${text} (copied, press Cmd+V)`, at: Date.now() });
|
||||
if (!inj?.pasted) setFeedback({ tone: 'ok', icon: 'clipboard', text: `${text} (copied, press Cmd+V)`, at: Date.now() });
|
||||
}
|
||||
setState('idle');
|
||||
} else if (res?.ok && !res.text) {
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,12 +1,11 @@
|
||||
import { VOICE_CUE_START, VOICE_CUE_STOP, VOICE_CUE_PASTE, VOICE_CUE_LOCK } from './voiceCueSounds';
|
||||
import { VOICE_CUE_START, VOICE_CUE_STOP, VOICE_CUE_LOCK } from './voiceCueSounds';
|
||||
|
||||
// Eric picked these from Google's Material product sound set after a bake-off against Wispr Flow's
|
||||
// real cues; the synth versions never survived an ear test. Files are embedded data URIs (see
|
||||
// voiceCueSounds.ts), pre-instantiated so playback is instant on the press. The grammar mirrors
|
||||
// Wispr's three-beat: tap in, tap out, and a rising completion the moment the text actually lands;
|
||||
// lock marks a hands-free latch.
|
||||
// voiceCueSounds.ts), pre-instantiated so playback is instant on the press. Start and stop taps plus the
|
||||
// hands-free lock mark; Eric explicitly cut the text-landed chime.
|
||||
|
||||
type CueKind = 'start' | 'stop' | 'paste' | 'lock';
|
||||
type CueKind = 'start' | 'stop' | 'lock';
|
||||
|
||||
// Pushed from Settings (dictation_sounds / dictation_sound_volume); defaults match the shipped feel.
|
||||
let cueEnabled = true;
|
||||
@@ -17,14 +16,13 @@ export function configureVoiceCues(enabled: boolean, volume: number): void {
|
||||
cueVolume = Math.min(1, Math.max(0, volume));
|
||||
for (const kind of Object.keys(p_players) as CueKind[]) {
|
||||
const a = p_players[kind];
|
||||
if (a) a.volume = kind === 'paste' ? cueVolume * 0.8 : cueVolume;
|
||||
if (a) a.volume = cueVolume;
|
||||
}
|
||||
}
|
||||
|
||||
const SOURCES: Record<CueKind, string> = {
|
||||
start: VOICE_CUE_START,
|
||||
stop: VOICE_CUE_STOP,
|
||||
paste: VOICE_CUE_PASTE,
|
||||
lock: VOICE_CUE_LOCK,
|
||||
};
|
||||
|
||||
@@ -34,7 +32,7 @@ function player(kind: CueKind): HTMLAudioElement {
|
||||
let a = p_players[kind];
|
||||
if (!a) {
|
||||
a = new Audio(SOURCES[kind]);
|
||||
a.volume = kind === 'paste' ? cueVolume * 0.8 : cueVolume;
|
||||
a.volume = cueVolume;
|
||||
p_players[kind] = a;
|
||||
}
|
||||
return a;
|
||||
|
||||
Reference in New Issue
Block a user