[eric] voice: cue sound toggle+volume, per-site disable, learned proper nouns, quiet-speech AGC, favicon target chip, tone-matched polish; dead overlay hook off BrowserCard (ENG-169)

This commit is contained in:
ciregenz
2026-08-05 23:25:44 -07:00
parent 753903cc16
commit e548f8c4d8
12 changed files with 225 additions and 30 deletions
+5
View File
@@ -97,6 +97,11 @@ class AppSettings(BaseModel):
dictation_model: Optional[str] = None
# Personal glossary (comma-separated names/jargon) fed to whisper as a decode prompt so "Anthropic" never comes out "and Thropic".
dictation_dictionary: str = ""
dictation_sounds: bool = True
# 0..1; the cue loudness Eric tuned by ear rides here instead of a hardcode.
dictation_sound_volume: float = 0.35
# Comma-separated hostnames (and app names) where dictation refuses to record while focused there.
dictation_disabled_surfaces: str = ""
anthropic_api_key: Optional[str] = None
browser_homepage: str = "https://www.google.com"
# Opt-in: let a blocked browser agent borrow the sign-in you already have in your everyday
+4 -1
View File
@@ -27,7 +27,10 @@ P_POLISH_SYSTEM = (
"'new line'/'new paragraph' become real breaks, 'period'/'comma'/'question mark' become the "
"mark when clearly dictated as punctuation. NEVER add content, never answer questions in the "
"text, never translate, never wrap in quotes, never use em-dashes. Keep the speaker's words "
"and tone; this is transcription cleanup, not rewriting."
"and tone; this is transcription cleanup, not rewriting. Match formality to the destination "
"named in the bracket hint when one is present: email or document fields get complete "
"sentences and clean punctuation; chat or search fields keep casual phrasing and slang as "
"spoken. Never shift meaning either way."
)
POLISH_INPUT_CAP = 8_000
@@ -73,7 +73,6 @@ import {
} from '@/shared/browserCommandHandler';
import { resolveInput, isGoogleSearch } from '@/shared/resolveUrl';
import BrowserAgentOverlay from './BrowserAgentOverlay';
import { useOverlayScrollPassthrough } from '../hooks/interaction/useOverlayScrollPassthrough';
// Fixed light chrome for the macOS-window look; deliberately theme-independent, like a real browser window.
const CHROME_BG = '#f2eff5';
@@ -213,7 +212,6 @@ const BrowserCard: React.FC<Props> = ({
// Read via ref inside the webview-attach effect so a new onDoubleClick identity doesn't re-run that effect (which would re-register the webview).
const onDoubleClickRef = useRef(onDoubleClick);
onDoubleClickRef.current = onDoubleClick;
const scrollOverlayRef = useOverlayScrollPassthrough(isSelected);
const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage);
const elementSelectionCtx = useElementSelection();
const isElementSelectMode = elementSelectionCtx?.selectMode ?? false;
@@ -157,6 +157,43 @@ const GeneralInterface: React.FC<{
/>
</Box>
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_sounds', 'Dictation sounds', 'Interface', 'The start, stop, and text-landed cues.')}>
<Box sx={{ mr: 3 }}>
<Typography sx={labelSx}>Sounds</Typography>
<Typography sx={descSx}>The start, stop, and text-landed cues, and how loud they play.</Typography>
</Box>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1.5 }}>
<Slider
size="small"
min={0}
max={100}
disabled={!(form.dictation_sounds ?? true)}
value={Math.round((form.dictation_sound_volume ?? 0.35) * 100)}
onChange={(_, v) => setForm({ ...form, dictation_sound_volume: (v as number) / 100 })}
sx={{ width: 110 }}
/>
<Switch
size="small"
checked={form.dictation_sounds ?? true}
onChange={(e) => setForm({ ...form, dictation_sounds: e.target.checked })}
/>
</Box>
</Box>
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_disabled_surfaces', 'Dictation off for sites', 'Interface', 'Sites where the dictation key does nothing.')}>
<Box sx={{ mr: 3 }}>
<Typography sx={labelSx}>Off for sites</Typography>
<Typography sx={descSx}>Comma-separated hostnames where the dictation key refuses to record (it tells you instead of failing silently).</Typography>
</Box>
<TextField
size="small"
placeholder="docs.google.com, slack.com"
value={form.dictation_disabled_surfaces ?? ''}
onChange={(e) => setForm({ ...form, dictation_disabled_surfaces: e.target.value })}
sx={{ width: 280 }}
/>
</Box>
<Box sx={{ ...inlineRowSx, alignItems: 'flex-start' }} {...settingSelectAttrs('dictation_history', 'Dictation history', 'Interface', 'Your recent dictations, copyable.')}>
<Box sx={{ mr: 3, flexShrink: 0, width: 220 }}>
<Typography sx={labelSx}>History</Typography>
@@ -33,6 +33,9 @@ export interface AppSettings {
dictation_shortcut?: string | null;
dictation_model?: string | null;
dictation_dictionary?: string;
dictation_sounds?: boolean;
dictation_sound_volume?: number;
dictation_disabled_surfaces?: string;
anthropic_api_key: string | null;
openai_api_key?: string | null;
google_api_key?: string | null;
@@ -165,6 +168,9 @@ export const DEFAULT_SETTINGS: AppSettings = {
dictation_shortcut: null,
dictation_model: null,
dictation_dictionary: '',
dictation_sounds: true,
dictation_sound_volume: 0.35,
dictation_disabled_surfaces: '',
anthropic_api_key: null,
browser_homepage: 'https://duckduckgo.com',
browser_import_signins: false,
@@ -1,7 +1,8 @@
import React, { useCallback, useEffect, useRef } from 'react';
import { useAppSelector } from '@/shared/hooks';
import { useVoiceDictation } from './useVoiceDictation';
import { playVoiceCue } from './voiceCues';
import { playVoiceCue, configureVoiceCues } from './voiceCues';
import { setManualDictionary } from './voiceDictionary';
import { VoiceContext } from './voiceContext';
import VoiceOverlay from './VoiceOverlay';
@@ -10,7 +11,7 @@ import VoiceOverlay from './VoiceOverlay';
// out-of-sync state. Mounted once near the app root.
export function VoiceDictationProvider({ children }: { children: React.ReactNode }): React.ReactElement {
const { state, lastText, error, pct, feedback, partial, targetLabel, toggle, start, stop, cancel, notify, volumeRef } = useVoiceDictation();
const { state, lastText, error, pct, feedback, partial, target, toggle, start, stop, cancel, notify, volumeRef } = useVoiceDictation();
// fn is the dictation key, but macOS may still have its own Globe action bound (emoji picker on a quick tap); say so once.
const globeWarnedRef = useRef(false);
@@ -39,12 +40,18 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode
if (dictationModel) void window.openswarm?.voiceSetModel?.(dictationModel);
}, [dictationModel]);
// Personal glossary rides every decode as a whisper prompt; push on boot and on change.
// Personal glossary rides every decode as a whisper prompt (manual list merged with learned nouns).
const dictationDictionary = useAppSelector((s) => s.settings.data.dictation_dictionary ?? '');
useEffect(() => {
const bridge = window as unknown as { openswarm?: { voiceSetDictionary?: (words: string) => void } };
bridge.openswarm?.voiceSetDictionary?.(dictationDictionary);
setManualDictionary(dictationDictionary);
}, [dictationDictionary]);
// Cue sounds honor the user's toggle and loudness.
const cueSounds = useAppSelector((s) => s.settings.data.dictation_sounds ?? true);
const cueVolume = useAppSelector((s) => s.settings.data.dictation_sound_volume ?? 0.35);
useEffect(() => {
configureVoiceCues(cueSounds, cueVolume);
}, [cueSounds, cueVolume]);
const stateRef = useRef(state);
stateRef.current = state;
const heldRef = useRef(false);
@@ -111,7 +118,7 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode
const confirmRecording = useCallback((): void => { void stop(); }, [stop]);
return (
<VoiceContext.Provider value={{ state, lastText, error, pct, feedback, partial, targetLabel, toggle, pressStart, pressEnd, confirmRecording, cancelRecording: cancel, holdMode, volumeRef }}>
<VoiceContext.Provider value={{ state, lastText, error, pct, feedback, partial, target, toggle, pressStart, pressEnd, confirmRecording, cancelRecording: cancel, holdMode, volumeRef }}>
{children}
<VoiceOverlay />
</VoiceContext.Provider>
+5 -3
View File
@@ -183,7 +183,7 @@ const IdlePill: React.FC<{ onPressStart: () => void; onPressEnd: () => void }> =
};
const VoiceOverlay: React.FC = () => {
const { state, pct, feedback, partial, targetLabel, volumeRef, confirmRecording, cancelRecording, pressStart, pressEnd } = useVoice();
const { state, pct, feedback, partial, target, volumeRef, confirmRecording, cancelRecording, pressStart, pressEnd } = useVoice();
const [showFeedback, setShowFeedback] = useState(false);
// The capsule lingers past its state for one out-animation beat, so it glides back up instead of vanishing.
const [capsuleLeaving, setCapsuleLeaving] = useState(false);
@@ -224,8 +224,10 @@ const VoiceOverlay: React.FC = () => {
// Words once there are words; before that, the chip says WHERE they will land.
content = hasPartial
? <LiveTranscript committed={partial.committed} tentative={partial.tentative} />
: (state === 'recording' && targetLabel
? <Box component="span" sx={{ color: 'rgba(255,255,255,0.65)', fontSize: '0.75rem' }}>{'→'} {targetLabel}</Box>
: (state === 'recording' && target.label
? <Box component="span" sx={{ display: 'inline-flex', alignItems: 'center', gap: 0.6, color: 'rgba(255,255,255,0.65)', fontSize: '0.75rem' }}>
{'→'} {target.icon && <Box component="img" src={target.icon} alt="" sx={{ width: 13, height: 13, borderRadius: '3px' }} />} {target.label}
</Box>
: null);
} else if (state === 'preparing') {
content = (<><CircularProgress size={13} thickness={5} sx={{ color: 'rgba(255,255,255,0.7)' }} /><span>Downloading voice model {pct}%</span></>);
+50 -12
View File
@@ -6,6 +6,9 @@ import { playVoiceCue } from './voiceCues';
import { injectAtFocus } from './injectAtFocus';
import { createSilenceDetector } from './createSilenceDetector';
import { pushDictation } from './voiceHistory';
import { learnFromTranscript } from './voiceDictionary';
import { getFocusedSurfaceHost, surfaceDisabled } from './voiceSurface';
import { store } from '@/shared/state/store';
import { createCaptureNode } from './createCaptureNode';
export type VoiceState = 'idle' | 'recording' | 'transcribing' | 'preparing';
@@ -40,17 +43,30 @@ export interface VoiceFeedback {
at: number;
}
// Where the transcript will land RIGHT NOW, in the user's words; mirrors injectAtFocus's tiers so
// the capsule's target chip never promises a destination injection would not actually pick.
export function describeInjectTarget(): string {
// Where the transcript will land RIGHT NOW, in the user's words plus the surface's own icon;
// mirrors injectAtFocus's tiers so the chip never promises a destination injection would not pick.
export interface InjectTargetInfo {
label: string;
icon: string | null;
}
function browserTargetInfo(): InjectTargetInfo {
const browserId = getLastInteractedBrowser();
const card = browserId ? store.getState().dashboardLayout.browserCards[browserId] : undefined;
const tab = card?.tabs?.find((t) => t.id === card.activeTabId);
const host = (() => { try { return tab?.url ? new URL(tab.url).hostname : null; } catch { return null; } })();
return { label: host || 'browser page', icon: tab?.favicon || null };
}
export function describeInjectTarget(): InjectTargetInfo {
const a = document.activeElement as HTMLElement | null;
if (a && (a.tagName === 'INPUT' || a.tagName === 'TEXTAREA' || a.isContentEditable)) {
const hint = a.getAttribute('placeholder') || a.getAttribute('aria-label');
return hint ? hint.slice(0, 30) : 'text field';
return { label: hint ? hint.slice(0, 30) : 'text field', icon: null };
}
if (a && a.tagName === 'WEBVIEW') return 'browser page';
if (getLastInteractedBrowser()) return 'browser page';
return 'chat composer';
if (a && a.tagName === 'WEBVIEW') return browserTargetInfo();
if (getLastInteractedBrowser()) return browserTargetInfo();
return { label: 'chat composer', icon: null };
}
// Context hint for the polisher: what the user is dictating into (a field label, a page title), so
@@ -87,7 +103,7 @@ export function useVoiceDictation() {
const [pct, setPct] = useState<number>(0);
const [feedback, setFeedback] = useState<VoiceFeedback | null>(null);
const [partial, setPartial] = useState<VoicePartial | null>(null);
const [targetLabel, setTargetLabel] = useState<string>('');
const [target, setTarget] = useState<InjectTargetInfo>({ label: '', icon: null });
const partialSeqRef = useRef<number>(0);
const recRef = useRef<Recorder | null>(null);
const stateRef = useRef<VoiceState>('idle');
@@ -132,6 +148,15 @@ export function useVoiceDictation() {
const start = useCallback(async (hold = false): Promise<void> => {
if (stateRef.current !== 'idle') return;
if (!window.openswarm?.voiceTranscribe) { setError('desktop-only'); return; } // no Electron bridge = web build
// Per-surface disable: the user marked this site as no-dictation; refuse loudly, never silently.
const disabledList = store.getState().settings.data.dictation_disabled_surfaces ?? '';
if (disabledList) {
const host = getFocusedSurfaceHost();
if (surfaceDisabled(disabledList, host)) {
setFeedback({ tone: 'warn', icon: 'mic', text: `Dictation is off for ${host}. Change it in Settings > Interface.`, at: Date.now() });
return;
}
}
setError(null);
// Warm on the DOWN edge, before the mic prompt, so the model load overlaps the user starting to speak.
void window.openswarm?.voiceWarmup?.();
@@ -150,7 +175,19 @@ export function useVoiceDictation() {
const endpointer = hold ? null : createSilenceDetector(ctx.sampleRate);
const streamRes = await window.openswarm?.voiceStreamStart?.();
const streaming = streamRes?.ok === true;
// Whisper-mode: quiet speech gets a gentle, slow-moving boost (3x cap) so murmured dictation
// still clears the decode gates; loud input passes through untouched, and the gain glides so
// it can never pump. Applied to BOTH the streamed chunks and the stored clip.
let agcGain = 1;
const capture = await createCaptureNode(ctx, (i16) => {
let sumSq = 0;
for (let i = 0; i < i16.length; i += 8) { const v = i16[i] / 0x8000; sumSq += v * v; }
const rawRms = Math.sqrt(sumSq / Math.max(1, Math.floor(i16.length / 8)));
const desired = rawRms > 0.004 && rawRms < 0.03 ? Math.min(3, 0.06 / rawRms) : 1;
agcGain = agcGain * 0.9 + desired * 0.1;
if (agcGain > 1.02) {
for (let i = 0; i < i16.length; i++) i16[i] = Math.max(-32768, Math.min(32767, Math.round(i16[i] * agcGain)));
}
if (streaming) window.openswarm?.voiceStreamChunk?.(i16.buffer as ArrayBuffer);
const data = new Float32Array(i16.length);
for (let i = 0; i < i16.length; i++) data[i] = i16[i] / 0x8000;
@@ -246,6 +283,7 @@ export function useVoiceDictation() {
// fallback still speaks, because the user has to act (paste) to get the text.
const target = injectAtFocus(text);
pushDictation(text, target || 'clipboard');
learnFromTranscript(text);
if (target) {
playVoiceCue('paste');
} else {
@@ -317,9 +355,9 @@ export function useVoiceDictation() {
// The target chip tracks focus LIVE while recording: clicking into a field mid-dictation retargets
// injection (by design), and the chip must tell that truth as it happens.
useEffect(() => {
if (state !== 'recording') { setTargetLabel(''); return undefined; }
setTargetLabel(describeInjectTarget());
const onFocus = (): void => setTargetLabel(describeInjectTarget());
if (state !== 'recording') { setTarget({ label: '', icon: null }); return undefined; }
setTarget(describeInjectTarget());
const onFocus = (): void => setTarget(describeInjectTarget());
window.addEventListener('focusin', onFocus, true);
window.addEventListener('focusout', onFocus, true);
return () => {
@@ -336,5 +374,5 @@ export function useVoiceDictation() {
setFeedback({ tone: 'warn', icon: 'info', text, at: Date.now() });
}, []);
return { state, lastText, error, pct, feedback, partial, targetLabel, toggle, start, stop, cancel, notify, volumeRef };
return { state, lastText, error, pct, feedback, partial, target, toggle, start, stop, cancel, notify, volumeRef };
}
+3 -3
View File
@@ -10,8 +10,8 @@ export interface VoiceContextValue {
pct: number;
feedback: VoiceFeedback | null;
partial: VoicePartial | null;
// Where the transcript will land right now, in user words ("chat composer", a field's label).
targetLabel: string;
// Where the transcript will land right now, in user words plus the surface's icon.
target: { label: string; icon: string | null };
toggle: () => void;
// Mic-button press semantics that respect the hold/toggle setting: press starts (or toggles),
// release stops only in hold mode. Buttons wire onPointerDown/Up to these and stay mode-agnostic.
@@ -25,7 +25,7 @@ export interface VoiceContextValue {
}
const NOOP_REF = { current: 0 };
const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, feedback: null, partial: null, targetLabel: '', toggle: () => {}, pressStart: () => {}, pressEnd: () => {}, confirmRecording: () => {}, cancelRecording: () => {}, holdMode: true, volumeRef: NOOP_REF };
const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, feedback: null, partial: null, target: { label: '', icon: null }, toggle: () => {}, pressStart: () => {}, pressEnd: () => {}, confirmRecording: () => {}, cancelRecording: () => {}, holdMode: true, volumeRef: NOOP_REF };
export const VoiceContext = createContext<VoiceContextValue>(NOOP);
+15 -3
View File
@@ -6,10 +6,21 @@ import { VOICE_CUE_START, VOICE_CUE_STOP, VOICE_CUE_PASTE, VOICE_CUE_LOCK } from
// Wispr's three-beat: tap in, tap out, and a rising completion the moment the text actually lands;
// lock marks a hands-free latch.
const CUE_VOLUME = 0.35;
type CueKind = 'start' | 'stop' | 'paste' | 'lock';
// Pushed from Settings (dictation_sounds / dictation_sound_volume); defaults match the shipped feel.
let cueEnabled = true;
let cueVolume = 0.35;
export function configureVoiceCues(enabled: boolean, volume: number): void {
cueEnabled = enabled;
cueVolume = Math.min(1, Math.max(0, volume));
for (const kind of Object.keys(p_players) as CueKind[]) {
const a = p_players[kind];
if (a) a.volume = kind === 'paste' ? cueVolume * 0.8 : cueVolume;
}
}
const SOURCES: Record<CueKind, string> = {
start: VOICE_CUE_START,
stop: VOICE_CUE_STOP,
@@ -23,13 +34,14 @@ function player(kind: CueKind): HTMLAudioElement {
let a = p_players[kind];
if (!a) {
a = new Audio(SOURCES[kind]);
a.volume = kind === 'paste' ? CUE_VOLUME * 0.8 : CUE_VOLUME;
a.volume = kind === 'paste' ? cueVolume * 0.8 : cueVolume;
p_players[kind] = a;
}
return a;
}
export function playVoiceCue(kind: CueKind): void {
if (!cueEnabled) return;
try {
const a = player(kind);
a.currentTime = 0;
@@ -0,0 +1,56 @@
// The glossary that rides every whisper decode: the user's manual list merged with proper nouns
// LEARNED from their own dictations (a capitalized word that keeps showing up is a name worth
// biasing toward). All local; main receives one merged comma list via voiceSetDictionary.
const LEARNED_KEY = 'osw-dictation-learned';
const LEARNED_CAP = 40;
const MERGE_TOP = 20;
// Words that start sentences get capitalized for free; only mid-sentence capitals count as names.
const NOUN_RE = /(?<![.!?]\s)(?<!^)\b([A-Z][a-zA-Z]{2,}(?:'s)?)\b/g;
const COMMON = new Set(['The', 'This', 'That', 'What', 'When', 'Where', 'Which', 'And', 'But', 'For', 'Not', 'You', 'Your', 'They', 'Their', 'There', 'Then', 'Also', 'Okay', 'Yes', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']);
let manual = '';
function readLearned(): Record<string, number> {
try {
const parsed: unknown = JSON.parse(localStorage.getItem(LEARNED_KEY) || '{}');
return parsed && typeof parsed === 'object' ? (parsed as Record<string, number>) : {};
} catch {
return {};
}
}
function pushMerged(): void {
const counts = readLearned();
const learned = Object.entries(counts)
.filter(([, n]) => n >= 2)
.sort((a, b) => b[1] - a[1])
.slice(0, MERGE_TOP)
.map(([w]) => w);
const manualWords = manual.split(',').map((w) => w.trim()).filter(Boolean);
const merged = [...new Set([...manualWords, ...learned])].join(', ');
const bridge = window as unknown as { openswarm?: { voiceSetDictionary?: (words: string) => void } };
bridge.openswarm?.voiceSetDictionary?.(merged);
}
export function setManualDictionary(words: string): void {
manual = words || '';
pushMerged();
}
export function learnFromTranscript(text: string): void {
try {
const counts = readLearned();
for (const m of text.matchAll(NOUN_RE)) {
const w = m[1].replace(/'s$/, '');
if (COMMON.has(w)) continue;
counts[w] = (counts[w] || 0) + 1;
}
const trimmed = Object.fromEntries(
Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, LEARNED_CAP),
);
localStorage.setItem(LEARNED_KEY, JSON.stringify(trimmed));
pushMerged();
} catch { /* learning is a bonus, never a blocker */ }
}
+31
View File
@@ -0,0 +1,31 @@
import { store } from '@/shared/state/store';
import { getLastInteractedBrowser } from '@/shared/browserFocus';
import { getWebview } from '@/shared/browserRegistry';
// Which browser surface dictation would land in right now, as a hostname; null when the target is
// an in-app field. Mirrors injectAtFocus's browser tiers so the disable list judges the same
// destination injection would pick.
export function getFocusedSurfaceHost(): string | null {
const active = document.activeElement as HTMLElement | null;
let browserId: string | null = null;
if (active && active.tagName === 'WEBVIEW') {
const cards = store.getState().dashboardLayout.browserCards;
for (const id of Object.keys(cards)) {
if (getWebview(id) === (active as unknown)) { browserId = id; break; }
}
}
if (!browserId) browserId = getLastInteractedBrowser();
if (!browserId) return null;
const card = store.getState().dashboardLayout.browserCards[browserId];
const url = card?.tabs?.find((t) => t.id === card.activeTabId)?.url || card?.url;
if (!url) return null;
try { return new URL(url).hostname || null; } catch { return null; }
}
// "slack.com, docs.google.com" -> does the focused surface match any entry (exact or suffix).
export function surfaceDisabled(disabledList: string, host: string | null): boolean {
if (!host) return false;
const entries = disabledList.split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);
const h = host.toLowerCase();
return entries.some((e) => h === e || h.endsWith(`.${e}`));
}