diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index 4f47fb35..d313eb06 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -97,6 +97,11 @@ class AppSettings(BaseModel): dictation_model: Optional[str] = None # Personal glossary (comma-separated names/jargon) fed to whisper as a decode prompt so "Anthropic" never comes out "and Thropic". dictation_dictionary: str = "" + dictation_sounds: bool = True + # 0..1; the cue loudness Eric tuned by ear rides here instead of a hardcode. + dictation_sound_volume: float = 0.35 + # Comma-separated hostnames (and app names) where dictation refuses to record while focused there. + dictation_disabled_surfaces: str = "" anthropic_api_key: Optional[str] = None browser_homepage: str = "https://www.google.com" # Opt-in: let a blocked browser agent borrow the sign-in you already have in your everyday diff --git a/backend/apps/voice/polish.py b/backend/apps/voice/polish.py index ff49e929..aabc4bc4 100644 --- a/backend/apps/voice/polish.py +++ b/backend/apps/voice/polish.py @@ -27,7 +27,10 @@ P_POLISH_SYSTEM = ( "'new line'/'new paragraph' become real breaks, 'period'/'comma'/'question mark' become the " "mark when clearly dictated as punctuation. NEVER add content, never answer questions in the " "text, never translate, never wrap in quotes, never use em-dashes. Keep the speaker's words " - "and tone; this is transcription cleanup, not rewriting." + "and tone; this is transcription cleanup, not rewriting. Match formality to the destination " + "named in the bracket hint when one is present: email or document fields get complete " + "sentences and clean punctuation; chat or search fields keep casual phrasing and slang as " + "spoken. Never shift meaning either way." ) POLISH_INPUT_CAP = 8_000 diff --git a/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx b/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx index 7756b77d..27952c34 100644 --- a/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx +++ b/frontend/src/app/pages/Dashboard/cards/BrowserCard.tsx @@ -73,7 +73,6 @@ import { } from '@/shared/browserCommandHandler'; import { resolveInput, isGoogleSearch } from '@/shared/resolveUrl'; import BrowserAgentOverlay from './BrowserAgentOverlay'; -import { useOverlayScrollPassthrough } from '../hooks/interaction/useOverlayScrollPassthrough'; // Fixed light chrome for the macOS-window look; deliberately theme-independent, like a real browser window. const CHROME_BG = '#f2eff5'; @@ -213,7 +212,6 @@ const BrowserCard: React.FC = ({ // Read via ref inside the webview-attach effect so a new onDoubleClick identity doesn't re-run that effect (which would re-register the webview). const onDoubleClickRef = useRef(onDoubleClick); onDoubleClickRef.current = onDoubleClick; - const scrollOverlayRef = useOverlayScrollPassthrough(isSelected); const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage); const elementSelectionCtx = useElementSelection(); const isElementSelectMode = elementSelectionCtx?.selectMode ?? false; diff --git a/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx b/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx index 8527b19f..32d2862c 100644 --- a/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx +++ b/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx @@ -157,6 +157,43 @@ const GeneralInterface: React.FC<{ /> + + + Sounds + The start, stop, and text-landed cues, and how loud they play. + + + setForm({ ...form, dictation_sound_volume: (v as number) / 100 })} + sx={{ width: 110 }} + /> + setForm({ ...form, dictation_sounds: e.target.checked })} + /> + + + + + + Off for sites + Comma-separated hostnames where the dictation key refuses to record (it tells you instead of failing silently). + + setForm({ ...form, dictation_disabled_surfaces: e.target.value })} + sx={{ width: 280 }} + /> + + History diff --git a/frontend/src/shared/state/settingsSlice.ts b/frontend/src/shared/state/settingsSlice.ts index 344ccdb0..bffbd9cd 100644 --- a/frontend/src/shared/state/settingsSlice.ts +++ b/frontend/src/shared/state/settingsSlice.ts @@ -33,6 +33,9 @@ export interface AppSettings { dictation_shortcut?: string | null; dictation_model?: string | null; dictation_dictionary?: string; + dictation_sounds?: boolean; + dictation_sound_volume?: number; + dictation_disabled_surfaces?: string; anthropic_api_key: string | null; openai_api_key?: string | null; google_api_key?: string | null; @@ -165,6 +168,9 @@ export const DEFAULT_SETTINGS: AppSettings = { dictation_shortcut: null, dictation_model: null, dictation_dictionary: '', + dictation_sounds: true, + dictation_sound_volume: 0.35, + dictation_disabled_surfaces: '', anthropic_api_key: null, browser_homepage: 'https://duckduckgo.com', browser_import_signins: false, diff --git a/frontend/src/shared/voice/VoiceDictationContext.tsx b/frontend/src/shared/voice/VoiceDictationContext.tsx index 8dc07203..2d8042f0 100644 --- a/frontend/src/shared/voice/VoiceDictationContext.tsx +++ b/frontend/src/shared/voice/VoiceDictationContext.tsx @@ -1,7 +1,8 @@ import React, { useCallback, useEffect, useRef } from 'react'; import { useAppSelector } from '@/shared/hooks'; import { useVoiceDictation } from './useVoiceDictation'; -import { playVoiceCue } from './voiceCues'; +import { playVoiceCue, configureVoiceCues } from './voiceCues'; +import { setManualDictionary } from './voiceDictionary'; import { VoiceContext } from './voiceContext'; import VoiceOverlay from './VoiceOverlay'; @@ -10,7 +11,7 @@ import VoiceOverlay from './VoiceOverlay'; // out-of-sync state. Mounted once near the app root. export function VoiceDictationProvider({ children }: { children: React.ReactNode }): React.ReactElement { - const { state, lastText, error, pct, feedback, partial, targetLabel, toggle, start, stop, cancel, notify, volumeRef } = useVoiceDictation(); + const { state, lastText, error, pct, feedback, partial, target, toggle, start, stop, cancel, notify, volumeRef } = useVoiceDictation(); // fn is the dictation key, but macOS may still have its own Globe action bound (emoji picker on a quick tap); say so once. const globeWarnedRef = useRef(false); @@ -39,12 +40,18 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode if (dictationModel) void window.openswarm?.voiceSetModel?.(dictationModel); }, [dictationModel]); - // Personal glossary rides every decode as a whisper prompt; push on boot and on change. + // Personal glossary rides every decode as a whisper prompt (manual list merged with learned nouns). const dictationDictionary = useAppSelector((s) => s.settings.data.dictation_dictionary ?? ''); useEffect(() => { - const bridge = window as unknown as { openswarm?: { voiceSetDictionary?: (words: string) => void } }; - bridge.openswarm?.voiceSetDictionary?.(dictationDictionary); + setManualDictionary(dictationDictionary); }, [dictationDictionary]); + + // Cue sounds honor the user's toggle and loudness. + const cueSounds = useAppSelector((s) => s.settings.data.dictation_sounds ?? true); + const cueVolume = useAppSelector((s) => s.settings.data.dictation_sound_volume ?? 0.35); + useEffect(() => { + configureVoiceCues(cueSounds, cueVolume); + }, [cueSounds, cueVolume]); const stateRef = useRef(state); stateRef.current = state; const heldRef = useRef(false); @@ -111,7 +118,7 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode const confirmRecording = useCallback((): void => { void stop(); }, [stop]); return ( - + {children} diff --git a/frontend/src/shared/voice/VoiceOverlay.tsx b/frontend/src/shared/voice/VoiceOverlay.tsx index 9f8acf57..0c4295c4 100644 --- a/frontend/src/shared/voice/VoiceOverlay.tsx +++ b/frontend/src/shared/voice/VoiceOverlay.tsx @@ -183,7 +183,7 @@ const IdlePill: React.FC<{ onPressStart: () => void; onPressEnd: () => void }> = }; const VoiceOverlay: React.FC = () => { - const { state, pct, feedback, partial, targetLabel, volumeRef, confirmRecording, cancelRecording, pressStart, pressEnd } = useVoice(); + const { state, pct, feedback, partial, target, volumeRef, confirmRecording, cancelRecording, pressStart, pressEnd } = useVoice(); const [showFeedback, setShowFeedback] = useState(false); // The capsule lingers past its state for one out-animation beat, so it glides back up instead of vanishing. const [capsuleLeaving, setCapsuleLeaving] = useState(false); @@ -224,8 +224,10 @@ const VoiceOverlay: React.FC = () => { // Words once there are words; before that, the chip says WHERE they will land. content = hasPartial ? - : (state === 'recording' && targetLabel - ? {'→'} {targetLabel} + : (state === 'recording' && target.label + ? + {'→'} {target.icon && } {target.label} + : null); } else if (state === 'preparing') { content = (<>Downloading voice model {pct}%); diff --git a/frontend/src/shared/voice/useVoiceDictation.ts b/frontend/src/shared/voice/useVoiceDictation.ts index b94bc609..7f6f283d 100644 --- a/frontend/src/shared/voice/useVoiceDictation.ts +++ b/frontend/src/shared/voice/useVoiceDictation.ts @@ -6,6 +6,9 @@ import { playVoiceCue } from './voiceCues'; import { injectAtFocus } from './injectAtFocus'; import { createSilenceDetector } from './createSilenceDetector'; import { pushDictation } from './voiceHistory'; +import { learnFromTranscript } from './voiceDictionary'; +import { getFocusedSurfaceHost, surfaceDisabled } from './voiceSurface'; +import { store } from '@/shared/state/store'; import { createCaptureNode } from './createCaptureNode'; export type VoiceState = 'idle' | 'recording' | 'transcribing' | 'preparing'; @@ -40,17 +43,30 @@ export interface VoiceFeedback { at: number; } -// Where the transcript will land RIGHT NOW, in the user's words; mirrors injectAtFocus's tiers so -// the capsule's target chip never promises a destination injection would not actually pick. -export function describeInjectTarget(): string { +// Where the transcript will land RIGHT NOW, in the user's words plus the surface's own icon; +// mirrors injectAtFocus's tiers so the chip never promises a destination injection would not pick. +export interface InjectTargetInfo { + label: string; + icon: string | null; +} + +function browserTargetInfo(): InjectTargetInfo { + const browserId = getLastInteractedBrowser(); + const card = browserId ? store.getState().dashboardLayout.browserCards[browserId] : undefined; + const tab = card?.tabs?.find((t) => t.id === card.activeTabId); + const host = (() => { try { return tab?.url ? new URL(tab.url).hostname : null; } catch { return null; } })(); + return { label: host || 'browser page', icon: tab?.favicon || null }; +} + +export function describeInjectTarget(): InjectTargetInfo { const a = document.activeElement as HTMLElement | null; if (a && (a.tagName === 'INPUT' || a.tagName === 'TEXTAREA' || a.isContentEditable)) { const hint = a.getAttribute('placeholder') || a.getAttribute('aria-label'); - return hint ? hint.slice(0, 30) : 'text field'; + return { label: hint ? hint.slice(0, 30) : 'text field', icon: null }; } - if (a && a.tagName === 'WEBVIEW') return 'browser page'; - if (getLastInteractedBrowser()) return 'browser page'; - return 'chat composer'; + if (a && a.tagName === 'WEBVIEW') return browserTargetInfo(); + if (getLastInteractedBrowser()) return browserTargetInfo(); + return { label: 'chat composer', icon: null }; } // Context hint for the polisher: what the user is dictating into (a field label, a page title), so @@ -87,7 +103,7 @@ export function useVoiceDictation() { const [pct, setPct] = useState(0); const [feedback, setFeedback] = useState(null); const [partial, setPartial] = useState(null); - const [targetLabel, setTargetLabel] = useState(''); + const [target, setTarget] = useState({ label: '', icon: null }); const partialSeqRef = useRef(0); const recRef = useRef(null); const stateRef = useRef('idle'); @@ -132,6 +148,15 @@ export function useVoiceDictation() { const start = useCallback(async (hold = false): Promise => { if (stateRef.current !== 'idle') return; if (!window.openswarm?.voiceTranscribe) { setError('desktop-only'); return; } // no Electron bridge = web build + // Per-surface disable: the user marked this site as no-dictation; refuse loudly, never silently. + const disabledList = store.getState().settings.data.dictation_disabled_surfaces ?? ''; + if (disabledList) { + const host = getFocusedSurfaceHost(); + if (surfaceDisabled(disabledList, host)) { + setFeedback({ tone: 'warn', icon: 'mic', text: `Dictation is off for ${host}. Change it in Settings > Interface.`, at: Date.now() }); + return; + } + } setError(null); // Warm on the DOWN edge, before the mic prompt, so the model load overlaps the user starting to speak. void window.openswarm?.voiceWarmup?.(); @@ -150,7 +175,19 @@ export function useVoiceDictation() { const endpointer = hold ? null : createSilenceDetector(ctx.sampleRate); const streamRes = await window.openswarm?.voiceStreamStart?.(); const streaming = streamRes?.ok === true; + // Whisper-mode: quiet speech gets a gentle, slow-moving boost (3x cap) so murmured dictation + // still clears the decode gates; loud input passes through untouched, and the gain glides so + // it can never pump. Applied to BOTH the streamed chunks and the stored clip. + let agcGain = 1; const capture = await createCaptureNode(ctx, (i16) => { + let sumSq = 0; + for (let i = 0; i < i16.length; i += 8) { const v = i16[i] / 0x8000; sumSq += v * v; } + const rawRms = Math.sqrt(sumSq / Math.max(1, Math.floor(i16.length / 8))); + const desired = rawRms > 0.004 && rawRms < 0.03 ? Math.min(3, 0.06 / rawRms) : 1; + agcGain = agcGain * 0.9 + desired * 0.1; + if (agcGain > 1.02) { + for (let i = 0; i < i16.length; i++) i16[i] = Math.max(-32768, Math.min(32767, Math.round(i16[i] * agcGain))); + } if (streaming) window.openswarm?.voiceStreamChunk?.(i16.buffer as ArrayBuffer); const data = new Float32Array(i16.length); for (let i = 0; i < i16.length; i++) data[i] = i16[i] / 0x8000; @@ -246,6 +283,7 @@ export function useVoiceDictation() { // fallback still speaks, because the user has to act (paste) to get the text. const target = injectAtFocus(text); pushDictation(text, target || 'clipboard'); + learnFromTranscript(text); if (target) { playVoiceCue('paste'); } else { @@ -317,9 +355,9 @@ export function useVoiceDictation() { // The target chip tracks focus LIVE while recording: clicking into a field mid-dictation retargets // injection (by design), and the chip must tell that truth as it happens. useEffect(() => { - if (state !== 'recording') { setTargetLabel(''); return undefined; } - setTargetLabel(describeInjectTarget()); - const onFocus = (): void => setTargetLabel(describeInjectTarget()); + if (state !== 'recording') { setTarget({ label: '', icon: null }); return undefined; } + setTarget(describeInjectTarget()); + const onFocus = (): void => setTarget(describeInjectTarget()); window.addEventListener('focusin', onFocus, true); window.addEventListener('focusout', onFocus, true); return () => { @@ -336,5 +374,5 @@ export function useVoiceDictation() { setFeedback({ tone: 'warn', icon: 'info', text, at: Date.now() }); }, []); - return { state, lastText, error, pct, feedback, partial, targetLabel, toggle, start, stop, cancel, notify, volumeRef }; + return { state, lastText, error, pct, feedback, partial, target, toggle, start, stop, cancel, notify, volumeRef }; } diff --git a/frontend/src/shared/voice/voiceContext.ts b/frontend/src/shared/voice/voiceContext.ts index 97937311..6f9cab42 100644 --- a/frontend/src/shared/voice/voiceContext.ts +++ b/frontend/src/shared/voice/voiceContext.ts @@ -10,8 +10,8 @@ export interface VoiceContextValue { pct: number; feedback: VoiceFeedback | null; partial: VoicePartial | null; - // Where the transcript will land right now, in user words ("chat composer", a field's label). - targetLabel: string; + // Where the transcript will land right now, in user words plus the surface's icon. + target: { label: string; icon: string | null }; toggle: () => void; // Mic-button press semantics that respect the hold/toggle setting: press starts (or toggles), // release stops only in hold mode. Buttons wire onPointerDown/Up to these and stay mode-agnostic. @@ -25,7 +25,7 @@ export interface VoiceContextValue { } const NOOP_REF = { current: 0 }; -const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, feedback: null, partial: null, targetLabel: '', toggle: () => {}, pressStart: () => {}, pressEnd: () => {}, confirmRecording: () => {}, cancelRecording: () => {}, holdMode: true, volumeRef: NOOP_REF }; +const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, feedback: null, partial: null, target: { label: '', icon: null }, toggle: () => {}, pressStart: () => {}, pressEnd: () => {}, confirmRecording: () => {}, cancelRecording: () => {}, holdMode: true, volumeRef: NOOP_REF }; export const VoiceContext = createContext(NOOP); diff --git a/frontend/src/shared/voice/voiceCues.ts b/frontend/src/shared/voice/voiceCues.ts index ed985ceb..76b233d3 100644 --- a/frontend/src/shared/voice/voiceCues.ts +++ b/frontend/src/shared/voice/voiceCues.ts @@ -6,10 +6,21 @@ import { VOICE_CUE_START, VOICE_CUE_STOP, VOICE_CUE_PASTE, VOICE_CUE_LOCK } from // Wispr's three-beat: tap in, tap out, and a rising completion the moment the text actually lands; // lock marks a hands-free latch. -const CUE_VOLUME = 0.35; - type CueKind = 'start' | 'stop' | 'paste' | 'lock'; +// Pushed from Settings (dictation_sounds / dictation_sound_volume); defaults match the shipped feel. +let cueEnabled = true; +let cueVolume = 0.35; + +export function configureVoiceCues(enabled: boolean, volume: number): void { + cueEnabled = enabled; + cueVolume = Math.min(1, Math.max(0, volume)); + for (const kind of Object.keys(p_players) as CueKind[]) { + const a = p_players[kind]; + if (a) a.volume = kind === 'paste' ? cueVolume * 0.8 : cueVolume; + } +} + const SOURCES: Record = { start: VOICE_CUE_START, stop: VOICE_CUE_STOP, @@ -23,13 +34,14 @@ function player(kind: CueKind): HTMLAudioElement { let a = p_players[kind]; if (!a) { a = new Audio(SOURCES[kind]); - a.volume = kind === 'paste' ? CUE_VOLUME * 0.8 : CUE_VOLUME; + a.volume = kind === 'paste' ? cueVolume * 0.8 : cueVolume; p_players[kind] = a; } return a; } export function playVoiceCue(kind: CueKind): void { + if (!cueEnabled) return; try { const a = player(kind); a.currentTime = 0; diff --git a/frontend/src/shared/voice/voiceDictionary.ts b/frontend/src/shared/voice/voiceDictionary.ts new file mode 100644 index 00000000..ff0b0be0 --- /dev/null +++ b/frontend/src/shared/voice/voiceDictionary.ts @@ -0,0 +1,56 @@ +// The glossary that rides every whisper decode: the user's manual list merged with proper nouns +// LEARNED from their own dictations (a capitalized word that keeps showing up is a name worth +// biasing toward). All local; main receives one merged comma list via voiceSetDictionary. + +const LEARNED_KEY = 'osw-dictation-learned'; +const LEARNED_CAP = 40; +const MERGE_TOP = 20; + +// Words that start sentences get capitalized for free; only mid-sentence capitals count as names. +const NOUN_RE = /(? { + try { + const parsed: unknown = JSON.parse(localStorage.getItem(LEARNED_KEY) || '{}'); + return parsed && typeof parsed === 'object' ? (parsed as Record) : {}; + } catch { + return {}; + } +} + +function pushMerged(): void { + const counts = readLearned(); + const learned = Object.entries(counts) + .filter(([, n]) => n >= 2) + .sort((a, b) => b[1] - a[1]) + .slice(0, MERGE_TOP) + .map(([w]) => w); + const manualWords = manual.split(',').map((w) => w.trim()).filter(Boolean); + const merged = [...new Set([...manualWords, ...learned])].join(', '); + const bridge = window as unknown as { openswarm?: { voiceSetDictionary?: (words: string) => void } }; + bridge.openswarm?.voiceSetDictionary?.(merged); +} + +export function setManualDictionary(words: string): void { + manual = words || ''; + pushMerged(); +} + +export function learnFromTranscript(text: string): void { + try { + const counts = readLearned(); + for (const m of text.matchAll(NOUN_RE)) { + const w = m[1].replace(/'s$/, ''); + if (COMMON.has(w)) continue; + counts[w] = (counts[w] || 0) + 1; + } + const trimmed = Object.fromEntries( + Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, LEARNED_CAP), + ); + localStorage.setItem(LEARNED_KEY, JSON.stringify(trimmed)); + pushMerged(); + } catch { /* learning is a bonus, never a blocker */ } +} diff --git a/frontend/src/shared/voice/voiceSurface.ts b/frontend/src/shared/voice/voiceSurface.ts new file mode 100644 index 00000000..e0594c86 --- /dev/null +++ b/frontend/src/shared/voice/voiceSurface.ts @@ -0,0 +1,31 @@ +import { store } from '@/shared/state/store'; +import { getLastInteractedBrowser } from '@/shared/browserFocus'; +import { getWebview } from '@/shared/browserRegistry'; + +// Which browser surface dictation would land in right now, as a hostname; null when the target is +// an in-app field. Mirrors injectAtFocus's browser tiers so the disable list judges the same +// destination injection would pick. +export function getFocusedSurfaceHost(): string | null { + const active = document.activeElement as HTMLElement | null; + let browserId: string | null = null; + if (active && active.tagName === 'WEBVIEW') { + const cards = store.getState().dashboardLayout.browserCards; + for (const id of Object.keys(cards)) { + if (getWebview(id) === (active as unknown)) { browserId = id; break; } + } + } + if (!browserId) browserId = getLastInteractedBrowser(); + if (!browserId) return null; + const card = store.getState().dashboardLayout.browserCards[browserId]; + const url = card?.tabs?.find((t) => t.id === card.activeTabId)?.url || card?.url; + if (!url) return null; + try { return new URL(url).hostname || null; } catch { return null; } +} + +// "slack.com, docs.google.com" -> does the focused surface match any entry (exact or suffix). +export function surfaceDisabled(disabledList: string, host: string | null): boolean { + if (!host) return false; + const entries = disabledList.split(',').map((s) => s.trim().toLowerCase()).filter(Boolean); + const h = host.toLowerCase(); + return entries.some((e) => h === e || h.endsWith(`.${e}`)); +}