[eric] dictation: app-scoped by default with a Dictate-anywhere opt-in, and the Input Monitoring prompt moves from install time to first dictation use (ENG-341)

This commit is contained in:
ciregenz
2026-08-18 19:10:08 -07:00
parent 123f71b8cb
commit 88d6c536cd
6 changed files with 54 additions and 6 deletions
+2
View File
@@ -95,6 +95,8 @@ class AppSettings(BaseModel):
# None = platform default (Cmd/Ctrl+Shift+D); parts format matches new_agent_shortcut.
dictation_shortcut: Optional[str] = None
voice_hold_to_talk: bool = True
# Dictation hotkey acts outside the app only when the user opts in (ENG-341).
dictation_works_anywhere: bool = False
# Whisper model id from the desktop catalog (electron/voice/whisperModels.js); None = its default.
dictation_model: Optional[str] = None
# Personal glossary (comma-separated names/jargon) fed to whisper as a decode prompt so "Anthropic" never comes out "and Thropic".
+1
View File
@@ -114,6 +114,7 @@ contextBridge.exposeInMainWorld('openswarm', {
// True keyboard hold-to-talk needs the native key tap; renderers ask so Settings copy stays honest,
// and request triggers the macOS Accessibility prompt when the tap is blocked on permission.
setVoiceHotkey: (combo) => ipcRenderer.send('voice:set-hotkey', combo),
setVoiceScope: (anywhere) => ipcRenderer.send('voice:set-scope', anywhere === true),
voiceHoldCapable: () => ipcRenderer.invoke('voice:hold-capable'),
voiceRequestHoldPermission: () => ipcRenderer.invoke('voice:request-hold-permission'),
voiceRequestMicAccess: () => ipcRenderer.invoke('voice:request-mic-access'),
+29 -5
View File
@@ -1,4 +1,4 @@
const { app, globalShortcut, ipcMain, systemPreferences } = require('electron');
const { app, BrowserWindow, globalShortcut, ipcMain, systemPreferences } = require('electron');
const { spawn, spawnSync } = require('child_process');
const path = require('path');
const fs = require('fs');
@@ -87,7 +87,11 @@ function uiohookKeycodeFor(key, UiohookKey) {
}
function installVoiceHotkey(getMainWindow) {
// App-scoped by default (ENG-341): hotkeys act only while an OpenSwarm window is focused, unless
// the user opts into dictate-anywhere in Settings.
let worksAnywhere = false;
const send = (channel) => {
if (!worksAnywhere && BrowserWindow.getFocusedWindow() === null) return;
const win = getMainWindow();
if (win && !win.isDestroyed()) win.webContents.send(channel);
};
@@ -123,6 +127,7 @@ function installVoiceHotkey(getMainWindow) {
// Fallback shortcut stays registered while unfocused until the primary tier proves alive.
const registerVoiceShortcut = () => {
if (!worksAnywhere) return;
if (primaryProven()) return;
if (registeredAccel === fallbackCombo.accel) return;
unregisterFallbackShortcut();
@@ -287,9 +292,19 @@ function installVoiceHotkey(getMainWindow) {
return false;
}
};
tryStartNativeTap();
startFnWatcher();
registerVoiceShortcut();
// Arming the native tiers is what raises macOS's Input Monitoring prompt, so a fresh install must
// NOT arm at boot (ENG-341): the prompt fires at first dictation use instead, with context.
let tiersArmed = false;
const armNativeTiers = () => {
if (tiersArmed) return;
tiersArmed = true;
tryStartNativeTap();
startFnWatcher();
registerVoiceShortcut();
};
try {
if (fs.existsSync(path.join(app.getPath('userData'), 'dictation-used'))) armNativeTiers();
} catch (_) {}
app.on('browser-window-focus', () => { unregisterFallbackShortcut(); pokeFnWatcher(); });
app.on('browser-window-blur', registerVoiceShortcut);
@@ -331,16 +346,24 @@ function installVoiceHotkey(getMainWindow) {
combo = next;
fallbackCombo = combo.special ? parseCombo(LEGACY_COMBO) : combo;
if (UiohookKeyRef && !combo.special) tapKeycode = uiohookKeycodeFor(combo.key, UiohookKeyRef);
startFnWatcher();
if (tiersArmed) startFnWatcher();
unregisterFallbackShortcut();
registerVoiceShortcut();
console.log('[voice] hotkey set to', combo.accel);
});
// Renderer pushes the dictate-anywhere setting on boot and on change.
ipcMain.on('voice:set-scope', (_e, anywhere) => {
worksAnywhere = anywhere === true;
if (!worksAnywhere) unregisterFallbackShortcut();
else if (tiersArmed && BrowserWindow.getFocusedWindow() === null) registerVoiceShortcut();
});
ipcMain.handle('voice:hold-capable', () => tapProven || fnProven);
// Settings' "Hold to talk" fires the Accessibility prompt; Input Monitoring has no Electron API,
// but a running tap makes macOS list the app in that pane for the user to flip.
ipcMain.handle('voice:request-hold-permission', () => {
armNativeTiers();
if (process.platform === 'darwin' && !tapProven) {
try { systemPreferences.isTrustedAccessibilityClient(true); } catch (_) {}
}
@@ -349,6 +372,7 @@ function installVoiceHotkey(getMainWindow) {
// Fires the real TCC mic prompt BEFORE the first capture: with the entitlement present but no
// prior grant, getUserMedia would still fail once and burn the user's first dictation attempt.
ipcMain.handle('voice:request-mic-access', async () => {
armNativeTiers();
if (process.platform !== 'darwin') return true;
try {
if (systemPreferences.getMediaAccessStatus('microphone') === 'granted') return true;
@@ -51,7 +51,7 @@ const DictationSettings: React.FC<{
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_shortcut', 'Dictation shortcut', 'Interface', 'Global hotkey that starts dictation.')}>
<Box sx={{ mr: 3 }}>
<Typography sx={labelSx}>Dictation shortcut</Typography>
<Typography sx={descSx}>Works anywhere, even with the app in the background.</Typography>
<Typography sx={descSx}>Starts dictation while OpenSwarm is focused.</Typography>
</Box>
<Box sx={{ display: 'flex', alignItems: 'center', gap: 1 }}>
{form.dictation_shortcut ? (
@@ -117,6 +117,18 @@ const DictationSettings: React.FC<{
</Box>
</Box>
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_works_anywhere', 'Dictate anywhere', 'Interface', 'Let the dictation shortcut work while other apps are focused.')}>
<Box sx={{ mr: 3 }}>
<Typography sx={labelSx}>Dictate anywhere</Typography>
<Typography sx={descSx}>Shortcut also works with OpenSwarm in the background, typing into other apps.</Typography>
</Box>
<Switch
size="small"
checked={form.dictation_works_anywhere ?? false}
onChange={(e) => setForm({ ...form, dictation_works_anywhere: e.target.checked })}
/>
</Box>
<Box sx={inlineRowSx} {...settingSelectAttrs('dictation_haptics', 'Dictation haptics', 'Interface', 'Trackpad taps on start and stop.')}>
<Box sx={{ mr: 3 }}>
<Typography sx={labelSx}>Haptics</Typography>
@@ -37,6 +37,7 @@ export interface AppSettings {
memory_enabled?: boolean;
agent_settings_write_enabled?: boolean;
dictation_haptics?: boolean;
dictation_works_anywhere?: boolean;
dictation_sound_volume?: number;
dictation_disabled_surfaces?: string;
anthropic_api_key: string | null;
@@ -179,6 +180,7 @@ export const DEFAULT_SETTINGS: AppSettings = {
memory_enabled: true,
agent_settings_write_enabled: true,
dictation_haptics: true,
dictation_works_anywhere: false,
dictation_sound_volume: 0.7,
dictation_disabled_surfaces: '',
anthropic_api_key: null,
@@ -29,12 +29,19 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode
const dictationShortcut = useAppSelector((s) => s.settings.data.dictation_shortcut ?? null);
const dictationModel = useAppSelector((s) => s.settings.data.dictation_model ?? null);
const dictationAnywhere = useAppSelector((s) => (s.settings.data as { dictation_works_anywhere?: boolean }).dictation_works_anywhere ?? false);
// Push the user's combo to main on boot and on change so every hotkey tier rebinds live.
useEffect(() => {
const bridge = window as unknown as { openswarm?: { setVoiceHotkey?: (combo: string | null) => void } };
bridge.openswarm?.setVoiceHotkey?.(dictationShortcut);
}, [dictationShortcut]);
// Same for the dictate-anywhere scope (ENG-341): app-scoped unless the user opted in.
useEffect(() => {
const bridge = window as unknown as { openswarm?: { setVoiceScope?: (anywhere: boolean) => void } };
bridge.openswarm?.setVoiceScope?.(dictationAnywhere);
}, [dictationAnywhere]);
// Main boots with the catalog default, so a user who picked something else has to say so on every
// launch or dictation quietly runs on the wrong model.
useEffect(() => {