diff --git a/backend/apps/settings/models.py b/backend/apps/settings/models.py index da9d924f..4f47fb35 100644 --- a/backend/apps/settings/models.py +++ b/backend/apps/settings/models.py @@ -95,6 +95,8 @@ class AppSettings(BaseModel): voice_hold_to_talk: bool = True # Whisper model id from the desktop catalog (electron/voice/whisperModels.js); None = its default. dictation_model: Optional[str] = None + # Personal glossary (comma-separated names/jargon) fed to whisper as a decode prompt so "Anthropic" never comes out "and Thropic". + dictation_dictionary: str = "" anthropic_api_key: Optional[str] = None browser_homepage: str = "https://www.google.com" # Opt-in: let a blocked browser agent borrow the sign-in you already have in your everyday diff --git a/electron/main.js b/electron/main.js index 19c91f4e..e6057a11 100644 --- a/electron/main.js +++ b/electron/main.js @@ -3076,6 +3076,7 @@ ipcMain.handle('voice:set-model', (_e, id) => { if (ready) whisperService.warmInBackground(voiceResourceDir(), voiceUserDataDir()); return { ok: true, ready }; }); +ipcMain.on('voice:set-dictionary', (_e, words) => { whisperService.setDictionary(words); }); // Paste the text into the frontmost app (dictate-anywhere). Returns whether the OS paste actually fired. ipcMain.handle('voice:inject', async (_e, text) => { try { const pasted = await injectText(String(text || '')); return { ok: true, pasted }; } catch (err) { return { ok: false, error: String(err && err.message ? err.message : err) }; } diff --git a/electron/preload.js b/electron/preload.js index 259d4dc1..1f0c1389 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -70,6 +70,7 @@ contextBridge.exposeInMainWorld('openswarm', { // Settings' model picker: the catalog with install state, and switching (downloads on demand). voiceModels: () => ipcRenderer.invoke('voice:models'), voiceSetModel: (id) => ipcRenderer.invoke('voice:set-model', id), + voiceSetDictionary: (words) => ipcRenderer.send('voice:set-dictionary', words), voiceTranscribe: (wavArrayBuffer) => ipcRenderer.invoke('voice:transcribe', wavArrayBuffer), voiceInject: (text) => ipcRenderer.invoke('voice:inject', text), // Streaming dictation: chunks flow up fire-and-forget, live partials flow back down. diff --git a/electron/voice/whisperService.js b/electron/voice/whisperService.js index e5dc7fdc..c6a722bf 100644 --- a/electron/voice/whisperService.js +++ b/electron/voice/whisperService.js @@ -12,6 +12,14 @@ const whisperModels = require('./whisperModels'); // Which catalog model the user picked. Settings pushes it in; until then the catalog default wins. let selectedModelId = whisperModels.DEFAULT_MODEL_ID; +// The user's personal glossary, pushed from Settings; fed to whisper as a decode prompt so names +// and jargon bias recognition without any retraining (the classic initial-prompt trick). +let dictionaryPrompt = ''; + +function setDictionary(words) { + const clean = String(words || '').split(',').map((w) => w.trim()).filter(Boolean).slice(0, 60); + dictionaryPrompt = clean.length ? `Glossary: ${clean.join(', ')}.` : ''; +} function modelStatus() { return whisperModels.downloadStatus(); @@ -227,6 +235,7 @@ async function transcribe(resourceDir, userDataDir, wavBuffer) { const form = new FormData(); form.append('file', new Blob([wavBuffer], { type: 'audio/wav' }), 'audio.wav'); form.append('response_format', 'text'); + if (dictionaryPrompt) form.append('prompt', dictionaryPrompt); const res = await fetch(`http://127.0.0.1:${p}/inference`, { method: 'POST', body: form }); if (!res.ok) throw new Error(`whisper-http-${res.status}`); const text = (await res.text()).trim(); @@ -287,4 +296,4 @@ async function reprimeAfterWake() { return true; } -module.exports = { ensureServer, warmInBackground, reprimeAfterWake, transcribe, stopServer, isWarm, setModel, selectedModel, resolveBinary, resolveModel, modelStatus }; +module.exports = { ensureServer, warmInBackground, reprimeAfterWake, transcribe, stopServer, isWarm, setModel, setDictionary, selectedModel, resolveBinary, resolveModel, modelStatus }; diff --git a/frontend/src/app/pages/Dashboard/hooks/interaction/useDashboardInteractions.ts b/frontend/src/app/pages/Dashboard/hooks/interaction/useDashboardInteractions.ts index 30b7915b..a1600124 100644 --- a/frontend/src/app/pages/Dashboard/hooks/interaction/useDashboardInteractions.ts +++ b/frontend/src/app/pages/Dashboard/hooks/interaction/useDashboardInteractions.ts @@ -226,16 +226,9 @@ export function useDashboardInteractions({ if (e.button !== 0) return; if (isCardTarget(e.target, e.currentTarget)) return; report('dashboard', 'canvas_double_clicked'); - const vp = (e.currentTarget as HTMLElement).getBoundingClientRect(); - const cx = e.clientX - vp.left; - const cy = e.clientY - vp.top; - const cur = canvas.actions.getLiveState(); - const nextZoom = Math.max(0.15, cur.zoom * 0.55); - canvas.actions.animateTo({ - zoom: nextZoom, - panX: cx - ((cx - cur.panX) / cur.zoom) * nextZoom, - panY: cy - ((cy - cur.panY) / cur.zoom) * nextZoom, - }); + // Double-tap on empty space = show me everything (Eric's call): the same animated fit the + // overview affordances use, instead of the old blind 0.55x zoom-out that just lost people. + canvas.actions.fitToView(); }, [canvas.actions]); // Double-click a card → always expand + center + zoom (cancels pending collapse from single-click) diff --git a/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx b/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx index d3927870..8527b19f 100644 --- a/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx +++ b/frontend/src/app/pages/Settings/sections/general/GeneralInterface.tsx @@ -16,6 +16,7 @@ import type { SettingsStyles } from '../settingsStyles'; import { settingSelectAttrs } from '../settingSelect'; import ShortcutRecorderChip, { dictationDefaultCombo, comboDisplay } from './parts/ShortcutRecorderChip'; import DictationModelPicker from './parts/DictationModelPicker'; +import DictationHistoryList from './parts/DictationHistoryList'; const GeneralInterface: React.FC<{ form: AppSettings; @@ -142,6 +143,28 @@ const GeneralInterface: React.FC<{ /> + + + Dictionary + Comma-separated names and jargon (people, products, acronyms) that dictation should always spell right. + + setForm({ ...form, dictation_dictionary: e.target.value })} + sx={{ width: 280 }} + /> + + + + + History + Recent dictations, stored only on this machine. Copy one back if it landed in the wrong place. + + + + Canvas diff --git a/frontend/src/app/pages/Settings/sections/general/parts/DictationHistoryList.tsx b/frontend/src/app/pages/Settings/sections/general/parts/DictationHistoryList.tsx new file mode 100644 index 00000000..d2b21e3b --- /dev/null +++ b/frontend/src/app/pages/Settings/sections/general/parts/DictationHistoryList.tsx @@ -0,0 +1,53 @@ +import React, { useState } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import IconButton from '@mui/material/IconButton'; +import Button from '@mui/material/Button'; +import Tooltip from '@mui/material/Tooltip'; +import ContentCopyRoundedIcon from '@mui/icons-material/ContentCopyRounded'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { readDictationHistory, clearDictationHistory, DictationHistoryEntry } from '@/shared/voice/voiceHistory'; + +// The last dictations with one-click copy: rescue for a transcript that landed in the wrong field. +const DictationHistoryList: React.FC = () => { + const c = useClaudeTokens(); + const [entries, setEntries] = useState(readDictationHistory); + const [copiedAt, setCopiedAt] = useState(null); + + if (entries.length === 0) { + return Nothing dictated yet.; + } + + return ( + + {entries.slice(0, 8).map((e) => ( + + + {e.text} + + {new Date(e.at).toLocaleTimeString([], { hour: 'numeric', minute: '2-digit' })} · {e.target} + + + + { void navigator.clipboard.writeText(e.text); setCopiedAt(e.at); }} + sx={{ color: c.text.muted }} + > + + + + + ))} + + + ); +}; + +export default DictationHistoryList; diff --git a/frontend/src/shared/state/settingsSlice.ts b/frontend/src/shared/state/settingsSlice.ts index 776f7499..344ccdb0 100644 --- a/frontend/src/shared/state/settingsSlice.ts +++ b/frontend/src/shared/state/settingsSlice.ts @@ -32,6 +32,7 @@ export interface AppSettings { new_agent_shortcut: string; dictation_shortcut?: string | null; dictation_model?: string | null; + dictation_dictionary?: string; anthropic_api_key: string | null; openai_api_key?: string | null; google_api_key?: string | null; @@ -163,6 +164,7 @@ export const DEFAULT_SETTINGS: AppSettings = { new_agent_shortcut: 'Meta+l', dictation_shortcut: null, dictation_model: null, + dictation_dictionary: '', anthropic_api_key: null, browser_homepage: 'https://duckduckgo.com', browser_import_signins: false, diff --git a/frontend/src/shared/voice/VoiceDictationContext.tsx b/frontend/src/shared/voice/VoiceDictationContext.tsx index 734070fc..8dc07203 100644 --- a/frontend/src/shared/voice/VoiceDictationContext.tsx +++ b/frontend/src/shared/voice/VoiceDictationContext.tsx @@ -38,6 +38,13 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode useEffect(() => { if (dictationModel) void window.openswarm?.voiceSetModel?.(dictationModel); }, [dictationModel]); + + // Personal glossary rides every decode as a whisper prompt; push on boot and on change. + const dictationDictionary = useAppSelector((s) => s.settings.data.dictation_dictionary ?? ''); + useEffect(() => { + const bridge = window as unknown as { openswarm?: { voiceSetDictionary?: (words: string) => void } }; + bridge.openswarm?.voiceSetDictionary?.(dictationDictionary); + }, [dictationDictionary]); const stateRef = useRef(state); stateRef.current = state; const heldRef = useRef(false); diff --git a/frontend/src/shared/voice/useVoiceDictation.ts b/frontend/src/shared/voice/useVoiceDictation.ts index dd2a7063..b94bc609 100644 --- a/frontend/src/shared/voice/useVoiceDictation.ts +++ b/frontend/src/shared/voice/useVoiceDictation.ts @@ -5,6 +5,7 @@ import { encodeWav, VOICE_SAMPLE_RATE } from './encodeWav'; import { playVoiceCue } from './voiceCues'; import { injectAtFocus } from './injectAtFocus'; import { createSilenceDetector } from './createSilenceDetector'; +import { pushDictation } from './voiceHistory'; import { createCaptureNode } from './createCaptureNode'; export type VoiceState = 'idle' | 'recording' | 'transcribing' | 'preparing'; @@ -244,6 +245,7 @@ export function useVoiceDictation() { // Success is silent: the text landing at the cursor IS the feedback. Only the clipboard // fallback still speaks, because the user has to act (paste) to get the text. const target = injectAtFocus(text); + pushDictation(text, target || 'clipboard'); if (target) { playVoiceCue('paste'); } else { diff --git a/frontend/src/shared/voice/voiceHistory.ts b/frontend/src/shared/voice/voiceHistory.ts new file mode 100644 index 00000000..255cce47 --- /dev/null +++ b/frontend/src/shared/voice/voiceHistory.ts @@ -0,0 +1,34 @@ +// The last N dictations, local-only (localStorage): a transcript that landed somewhere wrong or got +// overwritten is recoverable without re-speaking it. Never synced, never sent anywhere. + +export interface DictationHistoryEntry { + text: string; + at: number; + target: string; +} + +const KEY = 'osw-dictation-history'; +const CAP = 20; + +export function readDictationHistory(): DictationHistoryEntry[] { + try { + const raw = localStorage.getItem(KEY); + const parsed: unknown = raw ? JSON.parse(raw) : []; + if (!Array.isArray(parsed)) return []; + return parsed.filter((e): e is DictationHistoryEntry => + !!e && typeof e === 'object' && typeof (e as DictationHistoryEntry).text === 'string' && typeof (e as DictationHistoryEntry).at === 'number'); + } catch { + return []; + } +} + +export function pushDictation(text: string, target: string): void { + try { + const next = [{ text, at: Date.now(), target }, ...readDictationHistory()].slice(0, CAP); + localStorage.setItem(KEY, JSON.stringify(next)); + } catch { /* quota or private mode; history is a convenience, never a blocker */ } +} + +export function clearDictationHistory(): void { + try { localStorage.removeItem(KEY); } catch { /* same */ } +}