From 5e2fe299a372e713e4881caa9c748ab9c2ae0df6 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Fri, 21 Aug 2026 13:20:57 -0700 Subject: [PATCH] [eric] dictation: a live fn watcher now MEANS a working tap, and a dead fn key says so (ENG-360) --- electron/native/fn-watcher.swift | 32 +++++++- electron/preload.js | 14 ++++ electron/voiceHotkey.js | 70 +++++++++++++++++- electron/voiceHotkeyProbe.test.js | 52 +++++++++++++ .../components/overlays/VoiceHotkeyToast.tsx | 74 +++++++++++++++++++ .../Dashboard/canvas/DashboardOverlays.tsx | 2 + 6 files changed, 237 insertions(+), 7 deletions(-) create mode 100644 frontend/src/app/components/overlays/VoiceHotkeyToast.tsx diff --git a/electron/native/fn-watcher.swift b/electron/native/fn-watcher.swift index da575bf2..e3a4e9c7 100644 --- a/electron/native/fn-watcher.swift +++ b/electron/native/fn-watcher.swift @@ -5,21 +5,43 @@ import CoreGraphics import Foundation import IOKit.hid -// --no-prompt: the boot-time probe must never raise the Input Monitoring TCC prompt (ENG-341); -// IOHIDCheckAccess answers silently, so granted machines arm and everyone else exits clean. -if CommandLine.arguments.contains("--no-prompt") - && IOHIDCheckAccess(kIOHIDRequestTypeListenEvent) != kIOHIDAccessTypeGranted { +// Permission is REPORTED, never inferred. A live process used to be the only evidence anyone had +// that the tap worked, and that read is simply wrong: tapCreate can hand back a port that never +// delivers an event, which is how a dead fn key looked identical to a key nobody pressed. +let isProbe = CommandLine.arguments.contains("--no-prompt") +var hidGranted = IOHIDCheckAccess(kIOHIDRequestTypeListenEvent) == kIOHIDAccessTypeGranted +// The probe stays silent (ENG-341 keeps the install flow prompt-free), but a user who is actually +// trying to dictate has earned the one prompt that can fix it. Without this the grant could only +// ever be discovered, never requested, so a denied machine stayed denied forever. +if !hidGranted && !isProbe { + hidGranted = IOHIDRequestAccess(kIOHIDRequestTypeListenEvent) +} +print(hidGranted ? "p granted" : "p denied") +fflush(stdout) + +// Without the grant the tap is DEAF, not absent: tapCreate happily returns a port that never +// delivers an event, which is exactly how a dead fn key passed for a live one. Refuse to run in +// that state so that "watcher alive" means "fn works" and nothing downstream has to guess. +if !hidGranted { print("e no-permission") fflush(stdout) exit(0) } var fnDown = false +// Proof the tap is on the wire at all. Without it, "no fn events" is ambiguous between a deaf tap +// and an untouched key, and that ambiguity is what made this unfixable from a log. +var wireAlive = false var tapRef: CFMachPort? var srcRef: CFRunLoopSource? let callback: CGEventTapCallBack = { _, type, event, _ in if type == .flagsChanged { + if !wireAlive { + wireAlive = true + print("w") + fflush(stdout) + } let keycode = event.getIntegerValueField(.keyboardEventKeycode) if keycode == 63 { let down = event.flags.contains(.maskSecondaryFn) @@ -60,6 +82,8 @@ guard armTap() else { fflush(stdout) exit(1) } +print("t ok") +fflush(stdout) // The parent pokes "r\n" when the app gains focus: another app's tap registered after ours sits // AHEAD of ours (head-insert) and can eat fn before we see it, with no disable event to catch, so diff --git a/electron/preload.js b/electron/preload.js index 941000d0..aee8eacb 100644 --- a/electron/preload.js +++ b/electron/preload.js @@ -127,6 +127,20 @@ contextBridge.exposeInMainWorld('openswarm', { ipcRenderer.on('voice:hold-up', up); return () => { ipcRenderer.removeListener('voice:hold-down', down); ipcRenderer.removeListener('voice:hold-up', up); }; }, + // Fires when the dictation primary (fn) cannot work, with the chord that does, and again if it + // recovers. Without a surface for this a denied grant was a key that silently did nothing forever. + onVoiceHotkeyIssue: (cb) => { + const bad = (_e, info) => cb({ ok: false, ...(info || {}) }); + const good = () => cb({ ok: true }); + ipcRenderer.on('voice:primary-unusable', bad); + ipcRenderer.on('voice:primary-usable', good); + return () => { + ipcRenderer.removeListener('voice:primary-unusable', bad); + ipcRenderer.removeListener('voice:primary-usable', good); + }; + }, + openInputMonitoringSettings: () => ipcRenderer.invoke('voice:open-input-monitoring'), + getVoiceHotkeyIssue: () => ipcRenderer.invoke('voice:hotkey-issue'), // Fires once at fn-watcher arm when macOS's own Globe-key action is still active (emoji picker on tap). onVoiceGlobeConflict: (cb) => { const h = () => cb(); diff --git a/electron/voiceHotkey.js b/electron/voiceHotkey.js index fd6e23b3..7d6e4bd1 100644 --- a/electron/voiceHotkey.js +++ b/electron/voiceHotkey.js @@ -1,4 +1,4 @@ -const { app, BrowserWindow, globalShortcut, ipcMain, systemPreferences } = require('electron'); +const { app, BrowserWindow, globalShortcut, ipcMain, shell, systemPreferences } = require('electron'); const { spawn, spawnSync } = require('child_process'); const path = require('path'); const fs = require('fs'); @@ -28,6 +28,8 @@ const fs = require('fs'); const DEFAULT_COMBO = process.platform === 'darwin' ? 'Fn' : process.platform === 'win32' ? 'Ctrl+Meta' : 'Ctrl+Shift+d'; const LEGACY_COMBO = process.platform === 'darwin' ? 'Meta+Shift+d' : 'Ctrl+Shift+d'; const TAP_FRESH_MS = 200; +// How long a tap may stay silent before we stop calling it "awaiting proof" and call it broken. +const FN_PROOF_GRACE_MS = 60_000; const FALLBACK_DEFER_MS = 90; // "Meta+Shift+d" (renderer parts format, same as new_agent_shortcut) -> matcher pieces. @@ -101,6 +103,11 @@ function installVoiceHotkey(getMainWindow) { let fallbackCombo = combo.special ? parseCombo(LEGACY_COMBO) : combo; let tapProven = false; let fnProven = false; + // What the watcher TOLD us, as opposed to what we guessed from it still being alive. + let fnPermission = 'unknown'; + let fnWireAlive = false; + let unusableNotified = false; + let lastHotkeyIssue = null; let lastTapKeyMs = 0; let registeredAccel = null; @@ -120,6 +127,23 @@ function installVoiceHotkey(getMainWindow) { }, FALLBACK_DEFER_MS); }; + // A dictation key that does nothing, forever, with nothing said, is the actual bug here: fn is + // the default, only the native watcher can see it, and app-scoped mode registers no global + // fallback, so a deaf tap left the user with no trigger AND no signal. Whenever the primary + // cannot serve, say so once and name the chord that does work right now. + const notifyPrimaryUnusable = (reason) => { + if (unusableNotified || primaryProven()) return; + unusableNotified = true; + // Remembered, not just fired: arming happens at boot and the renderer subscribes later, so a + // pure event is delivered to nobody and the warning goes silent again (caught live). + lastHotkeyIssue = { ok: false, reason, fallback: fallbackCombo.accel }; + console.log(`[voice] fn primary unusable (${reason}); dictation falls back to ${fallbackCombo.accel}`); + const win = getMainWindow(); + if (win && !win.isDestroyed()) { + win.webContents.send('voice:primary-unusable', { reason, fallback: fallbackCombo.accel }); + } + }; + // Only the tier that can actually SERVE the primary combo may retire the fallbacks: the uiohook // tap cannot see fn (keycode 63 is VC_UNDEFINED), so with an fn primary a proven tap must not // silence the legacy chord (caught live: focused Cmd+Shift+D went dead the moment any key flowed). @@ -142,7 +166,11 @@ function installVoiceHotkey(getMainWindow) { const startFnWatcher = (noPrompt) => { if (process.platform !== 'darwin' || combo.special !== 'fn' || fnProc) return; resolveFnWatcherBinary((bin) => { - if (!bin) { console.log('[voice] no fn watcher binary, legacy hotkey stays primary'); return; } + if (!bin) { + console.log('[voice] no fn watcher binary, legacy hotkey stays primary'); + notifyPrimaryUnusable('no-watcher-binary'); + return; + } if (combo.special !== 'fn' || fnProc) return; // rebound or raced while compiling startFnWatcherWith(bin, noPrompt === true); }); @@ -187,11 +215,27 @@ function installVoiceHotkey(getMainWindow) { while ((nl = buf.indexOf('\n')) >= 0) { const line = buf.slice(0, nl).trim(); buf = buf.slice(nl + 1); - if (line === 'd' || line === 'u') { + if (line === 'p granted' || line === 'p denied') { + fnPermission = line.slice(2); + console.log(`[voice] fn watcher Input Monitoring: ${fnPermission}`); + if (fnPermission === 'denied') notifyPrimaryUnusable('input-monitoring-denied'); + } else if (line === 't ok') { + console.log('[voice] fn watcher tap created'); + } else if (line === 'w') { + fnWireAlive = true; + console.log('[voice] fn watcher wire alive (tap is receiving key events)'); + } else if (line === 'd' || line === 'u') { if (!fnProven) { fnProven = true; unregisterFallbackShortcut(); console.log('[voice] fn watcher PROVEN (events flowing), fn hold-to-talk enabled'); + // Withdraw the fallback notice, or a key that started working keeps telling the user it is broken. + lastHotkeyIssue = null; + if (unusableNotified) { + unusableNotified = false; + const w = getMainWindow(); + if (w && !w.isDestroyed()) w.webContents.send('voice:primary-usable'); + } } if (combo.special === 'fn') send(line === 'd' ? 'voice:hold-down' : 'voice:hold-up'); } else if (line.startsWith('e')) { @@ -201,6 +245,7 @@ function installVoiceHotkey(getMainWindow) { }); fnProc.on('exit', (code) => { console.log(`[voice] fn watcher exited code=${code}; legacy hotkey resumes`); + notifyPrimaryUnusable(`watcher-exit-${code}`); fnProc = null; fnProven = false; registerVoiceShortcut(); @@ -210,6 +255,12 @@ function installVoiceHotkey(getMainWindow) { app.on('will-quit', () => { try { fnProc && fnProc.kill('SIGKILL'); } catch (_) {} }); } console.log('[voice] fn watcher armed (awaiting first event to prove Input Monitoring)'); + // Armed is not working. If the tap is still deaf after a spell of real use, that is a dead key, + // not a shy one, and the user deserves to hear it rather than keep pressing a key that no + // longer does anything (it regressed silently once already). + setTimeout(() => { + if (fnProc && !primaryProven() && !fnWireAlive) notifyPrimaryUnusable('tap-deaf'); + }, FN_PROOF_GRACE_MS); // macOS's own Globe-key action (emoji picker by default) fires on a quick fn tap alongside us; // tell the renderer once so it can point the user at "Press Globe key to: Do Nothing". require('child_process').exec('defaults read com.apple.HIToolbox AppleFnUsageType', (err, out) => { @@ -372,6 +423,19 @@ function installVoiceHotkey(getMainWindow) { else if (tiersArmed && BrowserWindow.getFocusedWindow() === null) registerVoiceShortcut(); }); + // The grant is the ONLY thing that fixes a denied fn key, and macOS will not re-prompt once the + // user has said no, so hand them the exact pane instead of a dead key and a shrug. + ipcMain.handle('voice:open-input-monitoring', () => { + if (process.platform !== 'darwin') return false; + try { + shell.openExternal('x-apple.systempreferences:com.apple.preference.security?Privacy_ListenEvent'); + return true; + } catch (_) { return false; } + }); + + // Pull, so a renderer that mounts (or reloads) after arming still learns the truth. + ipcMain.handle('voice:hotkey-issue', () => lastHotkeyIssue); + ipcMain.handle('voice:hold-capable', () => tapProven || fnProven); // Settings' "Hold to talk" fires the Accessibility prompt; Input Monitoring has no Electron API, // but a running tap makes macOS list the app in that pane for the user to flip. diff --git a/electron/voiceHotkeyProbe.test.js b/electron/voiceHotkeyProbe.test.js index 7f875ce3..8d142016 100644 --- a/electron/voiceHotkeyProbe.test.js +++ b/electron/voiceHotkeyProbe.test.js @@ -38,3 +38,55 @@ test('a focused bare Fn keydown arms the tiers and toggles', () => { test('the probe flag reaches the spawn argv', () => { assert.match(hotkeySrc, /spawn\(bin, noPrompt \? \['--no-prompt'\] : \[\]/); }); + +// ---- The deaf-tap class (Eric, exp.20, and the same "99% of devices" report) ---- +// tapCreate hands back a valid port even with Input Monitoring DENIED, and that tap then never +// delivers an event. So "the watcher process is alive" was read as "fn works" by every layer above +// it, and a permanently dead key looked identical to a key nobody had pressed yet. Measured live on +// a dev machine: `p denied` followed by a successful `t ok`. + +test('the watcher reports its permission instead of leaving it to be inferred', () => { + assert.match(swiftSrc, /print\(hidGranted \? "p granted" : "p denied"\)/, + 'permission must be stated on stdout, not guessed from the process still being alive'); +}); + +test('a watcher without the grant refuses to run, so alive means working', () => { + const guard = swiftSrc.split('if !hidGranted {')[1] || ''; + assert.match(guard, /e no-permission/); + assert.match(guard, /exit\(0\)/); + const guardAt = swiftSrc.indexOf('if !hidGranted {'); + assert.ok(guardAt > -1 && guardAt < swiftSrc.indexOf('guard armTap()'), + 'the refusal must come before the tap, or a deaf tap still gets created'); +}); + +test('the grant is REQUESTED on the intent path and never on the boot probe', () => { + assert.match(swiftSrc, /if !hidGranted && !isProbe \{\s*\n\s*hidGranted = IOHIDRequestAccess\(kIOHIDRequestTypeListenEvent\)/, + 'a denied machine can only be fixed by asking; the probe must still stay silent (ENG-341)'); +}); + +test('the tap proves it is on the wire, separately from any fn press', () => { + assert.match(swiftSrc, /if !wireAlive \{/); + assert.match(swiftSrc, /print\("w"\)/); +}); + +test('every way fn can fail tells the user which chord still works', () => { + for (const reason of ['input-monitoring-denied', 'no-watcher-binary', 'tap-deaf']) { + assert.ok(hotkeySrc.includes(reason), `unhandled fn failure mode: ${reason}`); + } + assert.match(hotkeySrc, /notifyPrimaryUnusable\(`watcher-exit-\$\{code\}`\)/, + 'a watcher that exits leaves no primary, so it must notify too'); + assert.match(hotkeySrc, /fallback: fallbackCombo\.accel/, + 'the notice is only useful if it names the chord that works'); +}); + +test('the notice is remembered, not just fired', () => { + // Arming happens at boot and the renderer subscribes later, so a pure event reaches nobody. + assert.match(hotkeySrc, /lastHotkeyIssue = \{ ok: false, reason, fallback: fallbackCombo\.accel \}/); + assert.match(hotkeySrc, /ipcMain\.handle\('voice:hotkey-issue', \(\) => lastHotkeyIssue\)/); +}); + +test('a key that starts working retracts its own warning', () => { + const provenBlock = hotkeySrc.split("fn watcher PROVEN")[1].slice(0, 400); + assert.match(provenBlock, /lastHotkeyIssue = null/); + assert.match(provenBlock, /voice:primary-usable/); +}); diff --git a/frontend/src/app/components/overlays/VoiceHotkeyToast.tsx b/frontend/src/app/components/overlays/VoiceHotkeyToast.tsx new file mode 100644 index 00000000..9875c340 --- /dev/null +++ b/frontend/src/app/components/overlays/VoiceHotkeyToast.tsx @@ -0,0 +1,74 @@ +// The fn/Globe key is the default dictation trigger and only the native watcher can see it, so a +// missing Input Monitoring grant used to leave a key that did nothing, forever, with nothing said. +// This is the surface for that: name the chord that works right now, and offer the pane that fixes it. + +import React from 'react'; +import Snackbar from '@mui/material/Snackbar'; +import Alert from '@mui/material/Alert'; +import Button from '@mui/material/Button'; +import IconButton from '@mui/material/IconButton'; +import CloseIcon from '@mui/icons-material/Close'; + +type Issue = { ok: boolean; reason?: string; fallback?: string }; + +const PRETTY: Record = { + 'input-monitoring-denied': 'OpenSwarm does not have Input Monitoring permission', + 'tap-deaf': 'the fn key is not reaching OpenSwarm', + 'no-watcher-binary': 'the fn key helper is missing', +}; + +export default function VoiceHotkeyToast() { + const [issue, setIssue] = React.useState(null); + const [dismissed, setDismissed] = React.useState(false); + + React.useEffect(() => { + const api = (window as any).openswarm; + if (!api || typeof api.onVoiceHotkeyIssue !== 'function') return; + // Ask first: arming happens before this mounts, so subscribing alone would miss the only send. + api.getVoiceHotkeyIssue?.().then((known: Issue | null) => { + if (known && known.ok === false) setIssue(known); + }).catch(() => {}); + return api.onVoiceHotkeyIssue((next: Issue) => { + // A key that starts working must retract its own warning, not keep crying wolf. + if (next.ok) { setIssue(null); setDismissed(false); return; } + setIssue(next); + }); + }, []); + + if (!issue || dismissed) return null; + + const why = PRETTY[issue.reason || ''] || 'the fn key is unavailable'; + const chord = (issue.fallback || 'Meta+Shift+D').replace('Meta', '⌘').replace('Shift', '⇧').replace(/\+/g, ''); + const canFix = issue.reason === 'input-monitoring-denied'; + + return ( + { if (reason !== 'clickaway') setDismissed(true); }} + anchorOrigin={{ vertical: 'bottom', horizontal: 'left' }} + > + + {canFix && ( + + )} + setDismissed(true)}> + + + + } + > + Dictation: {why}. Use {chord} for now. + + + ); +} diff --git a/frontend/src/app/pages/Dashboard/canvas/DashboardOverlays.tsx b/frontend/src/app/pages/Dashboard/canvas/DashboardOverlays.tsx index fe85adbf..619b4d7e 100644 --- a/frontend/src/app/pages/Dashboard/canvas/DashboardOverlays.tsx +++ b/frontend/src/app/pages/Dashboard/canvas/DashboardOverlays.tsx @@ -8,6 +8,7 @@ import WorkflowRunningToast from '@/app/pages/Workflows/WorkflowRunningToast'; import WorkflowNoticeToast from '@/app/pages/Workflows/WorkflowNoticeToast'; import MissedRunsToast from '@/app/pages/Workflows/MissedRunsToast'; import ProviderHealthToast from '@/app/components/overlays/ProviderHealthToast'; +import VoiceHotkeyToast from '@/app/components/overlays/VoiceHotkeyToast'; import ScheduleOfferToast from '@/app/components/nudges/ScheduleOfferToast'; import PrepKeepToast from '@/app/components/nudges/PrepKeepToast'; import type { AgentSession } from '@/shared/state/agentsSlice'; @@ -167,6 +168,7 @@ const DashboardOverlays: React.FC = ({ {/* Launch nudge when a subscription login died while the app was closed */} + {/* One-shot dependency beat: first completed personalized starter offers to become a weekly job */}