mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] voice: fn/Globe is the dictation key (native CGEvent watcher), Ctrl+Win chord on Windows, legacy chord backs both until proven
This commit is contained in:
@@ -0,0 +1,43 @@
|
||||
// Watches the macOS fn/Globe key globally and prints "d"/"u" per press/release. Exists because
|
||||
// libuiohook maps keycode 63 to VC_UNDEFINED, so no JS-side tap can ever see fn. Listen-only CGEvent
|
||||
// tap: needs the same Input Monitoring grant the app already requests, never swallows anything.
|
||||
import CoreGraphics
|
||||
import Foundation
|
||||
|
||||
var fnDown = false
|
||||
|
||||
let callback: CGEventTapCallBack = { _, type, event, _ in
|
||||
if type == .flagsChanged {
|
||||
let keycode = event.getIntegerValueField(.keyboardEventKeycode)
|
||||
if keycode == 63 {
|
||||
let down = event.flags.contains(.maskSecondaryFn)
|
||||
if down != fnDown {
|
||||
fnDown = down
|
||||
print(down ? "d" : "u")
|
||||
fflush(stdout)
|
||||
}
|
||||
}
|
||||
} else if type == .tapDisabledByTimeout || type == .tapDisabledByUserInput {
|
||||
// macOS pauses slow taps; re-enable or fn goes silently dead until relaunch.
|
||||
if let tap = tapRef { CGEvent.tapEnable(tap: tap, enable: true) }
|
||||
}
|
||||
return Unmanaged.passUnretained(event)
|
||||
}
|
||||
|
||||
var tapRef: CFMachPort?
|
||||
let mask = (CGEventMask(1) << CGEventType.flagsChanged.rawValue)
|
||||
guard let tap = CGEvent.tapCreate(
|
||||
tap: .cgSessionEventTap, place: .headInsertEventTap, options: .listenOnly,
|
||||
eventsOfInterest: mask, callback: callback, userInfo: nil
|
||||
) else {
|
||||
print("e tap-failed")
|
||||
fflush(stdout)
|
||||
exit(1)
|
||||
}
|
||||
tapRef = tap
|
||||
let src = CFMachPortCreateRunLoopSource(kCFAllocatorDefault, tap, 0)
|
||||
CFRunLoopAddSource(CFRunLoopGetCurrent(), src, .commonModes)
|
||||
CGEvent.tapEnable(tap: tap, enable: true)
|
||||
print("r")
|
||||
fflush(stdout)
|
||||
CFRunLoopRun()
|
||||
+118
-16
@@ -1,4 +1,7 @@
|
||||
const { app, globalShortcut, ipcMain, systemPreferences } = require('electron');
|
||||
const { spawn, spawnSync } = require('child_process');
|
||||
const path = require('path');
|
||||
const fs = require('fs');
|
||||
|
||||
// Voice dictation hotkey, user-rebindable (Settings > Interface > Dictation shortcut), two tiers:
|
||||
//
|
||||
@@ -19,13 +22,21 @@ const { app, globalShortcut, ipcMain, systemPreferences } = require('electron');
|
||||
// F5 is deliberately NOT a default: macOS's media-key layer routes it to Siri before any app sees
|
||||
// it. It stays bindable for users who have remapped that key at the OS level.
|
||||
|
||||
const DEFAULT_COMBO = process.platform === 'darwin' ? 'Meta+Shift+d' : 'Ctrl+Shift+d';
|
||||
// Wispr grammar: the fn/Globe key IS the dictation key on Mac; Windows gets the same bottom-corner
|
||||
// hold as Ctrl+Win (a bare laptop Fn never reaches the OS there). The old chord stays as the legacy
|
||||
// fallback tier until the primary proves alive, so a missing grant never strands dictation keyless.
|
||||
const DEFAULT_COMBO = process.platform === 'darwin' ? 'Fn' : process.platform === 'win32' ? 'Ctrl+Meta' : 'Ctrl+Shift+d';
|
||||
const LEGACY_COMBO = process.platform === 'darwin' ? 'Meta+Shift+d' : 'Ctrl+Shift+d';
|
||||
const TAP_FRESH_MS = 200;
|
||||
const FALLBACK_DEFER_MS = 90;
|
||||
|
||||
// "Meta+Shift+d" (renderer parts format, same as new_agent_shortcut) -> matcher pieces.
|
||||
// "Fn" and "Ctrl+Meta" are special: modifier-only triggers no accelerator grammar can express.
|
||||
function parseCombo(str) {
|
||||
const parts = String(str || DEFAULT_COMBO).split('+').filter(Boolean);
|
||||
const raw = String(str || DEFAULT_COMBO);
|
||||
if (raw === 'Fn') return { special: 'fn', key: '', mods: { meta: false, ctrl: false, alt: false, shift: false }, accel: 'Fn' };
|
||||
if (raw === 'Ctrl+Meta') return { special: 'ctrlmeta', key: '', mods: { meta: true, ctrl: true, alt: false, shift: false }, accel: 'Ctrl+Meta' };
|
||||
const parts = raw.split('+').filter(Boolean);
|
||||
const key = parts[parts.length - 1] || 'd';
|
||||
const mods = {
|
||||
meta: parts.includes('Meta'),
|
||||
@@ -40,7 +51,31 @@ function parseCombo(str) {
|
||||
mods.shift ? 'Shift' : null,
|
||||
key.length === 1 ? key.toUpperCase() : key,
|
||||
].filter(Boolean).join('+');
|
||||
return { key, mods, accel };
|
||||
return { special: null, key, mods, accel };
|
||||
}
|
||||
|
||||
// Resolve (or dev-compile) the native fn watcher, then call back with a path or null (legacy tiers
|
||||
// stay primary on null). The dev compile is async: a first-boot swiftc must never freeze startup.
|
||||
function resolveFnWatcherBinary(cb) {
|
||||
if (process.platform !== 'darwin') { cb(null); return; }
|
||||
const bundled = path.join(process.resourcesPath || '', 'fn-watcher');
|
||||
if (fs.existsSync(bundled)) { cb(bundled); return; }
|
||||
const src = path.join(__dirname, 'native', 'fn-watcher.swift');
|
||||
if (!fs.existsSync(src)) { cb(null); return; }
|
||||
const out = path.join(app.getPath('userData'), 'fn-watcher-bin');
|
||||
try { fs.mkdirSync(out, { recursive: true }); } catch (_) {}
|
||||
const bin = path.join(out, 'fn-watcher');
|
||||
try {
|
||||
if (fs.existsSync(bin) && fs.statSync(bin).mtimeMs >= fs.statSync(src).mtimeMs) { cb(bin); return; }
|
||||
} catch (_) { /* fall through to compile */ }
|
||||
const cc = spawn('swiftc', ['-O', '-o', bin, src], { stdio: ['ignore', 'ignore', 'pipe'] });
|
||||
let err = '';
|
||||
cc.stderr.on('data', (c) => { err = (err + String(c)).slice(-400); });
|
||||
cc.on('error', () => cb(null));
|
||||
cc.on('exit', (code) => {
|
||||
if (code !== 0) { console.log('[voice] fn watcher compile failed:', err.slice(0, 200)); cb(null); return; }
|
||||
cb(bin);
|
||||
});
|
||||
}
|
||||
|
||||
function uiohookKeycodeFor(key, UiohookKey) {
|
||||
@@ -58,7 +93,10 @@ function installVoiceHotkey(getMainWindow) {
|
||||
};
|
||||
|
||||
let combo = parseCombo(DEFAULT_COMBO);
|
||||
// Special combos (Fn, Ctrl+Meta) have no accelerator; the LEGACY chord backs them until proven.
|
||||
let fallbackCombo = combo.special ? parseCombo(LEGACY_COMBO) : combo;
|
||||
let tapProven = false;
|
||||
let fnProven = false;
|
||||
let lastTapKeyMs = 0;
|
||||
let registeredAccel = null;
|
||||
|
||||
@@ -76,16 +114,63 @@ function installVoiceHotkey(getMainWindow) {
|
||||
}, FALLBACK_DEFER_MS);
|
||||
};
|
||||
|
||||
// Fallback shortcut stays registered while unfocused until the tap proves alive.
|
||||
// Fallback shortcut stays registered while unfocused until a primary tier proves alive.
|
||||
const registerVoiceShortcut = () => {
|
||||
if (tapProven) return;
|
||||
if (registeredAccel === combo.accel) return;
|
||||
if (tapProven || fnProven) return;
|
||||
if (registeredAccel === fallbackCombo.accel) return;
|
||||
unregisterFallbackShortcut();
|
||||
try {
|
||||
if (globalShortcut.register(combo.accel, sendFallbackToggle)) registeredAccel = combo.accel;
|
||||
if (globalShortcut.register(fallbackCombo.accel, sendFallbackToggle)) registeredAccel = fallbackCombo.accel;
|
||||
} catch (_) { /* a taken shortcut just means no global hotkey; the pill still works */ }
|
||||
};
|
||||
|
||||
// ---- fn/Globe primary tier (macOS): the native watcher, since no JS tap can see keycode 63 ----
|
||||
let fnProc = null;
|
||||
const startFnWatcher = () => {
|
||||
if (process.platform !== 'darwin' || combo.special !== 'fn' || fnProc) return;
|
||||
resolveFnWatcherBinary((bin) => {
|
||||
if (!bin) { console.log('[voice] no fn watcher binary, legacy hotkey stays primary'); return; }
|
||||
if (combo.special !== 'fn' || fnProc) return; // rebound or raced while compiling
|
||||
startFnWatcherWith(bin);
|
||||
});
|
||||
};
|
||||
const startFnWatcherWith = (bin) => {
|
||||
try {
|
||||
fnProc = spawn(bin, [], { stdio: ['ignore', 'pipe', 'ignore'] });
|
||||
} catch (e) {
|
||||
console.log('[voice] fn watcher spawn failed:', e && e.message);
|
||||
fnProc = null;
|
||||
return;
|
||||
}
|
||||
let buf = '';
|
||||
fnProc.stdout.on('data', (c) => {
|
||||
buf += String(c);
|
||||
let nl;
|
||||
while ((nl = buf.indexOf('\n')) >= 0) {
|
||||
const line = buf.slice(0, nl).trim();
|
||||
buf = buf.slice(nl + 1);
|
||||
if (line === 'd' || line === 'u') {
|
||||
if (!fnProven) {
|
||||
fnProven = true;
|
||||
unregisterFallbackShortcut();
|
||||
console.log('[voice] fn watcher PROVEN (events flowing), fn hold-to-talk enabled');
|
||||
}
|
||||
if (combo.special === 'fn') send(line === 'd' ? 'voice:hold-down' : 'voice:hold-up');
|
||||
} else if (line.startsWith('e')) {
|
||||
console.log('[voice] fn watcher error:', line);
|
||||
}
|
||||
}
|
||||
});
|
||||
fnProc.on('exit', (code) => {
|
||||
console.log(`[voice] fn watcher exited code=${code}; legacy hotkey resumes`);
|
||||
fnProc = null;
|
||||
fnProven = false;
|
||||
registerVoiceShortcut();
|
||||
});
|
||||
app.on('will-quit', () => { try { fnProc && fnProc.kill('SIGKILL'); } catch (_) {} });
|
||||
console.log('[voice] fn watcher armed (awaiting first event to prove Input Monitoring)');
|
||||
};
|
||||
|
||||
let tapKeycode;
|
||||
let UiohookKeyRef = null;
|
||||
|
||||
@@ -121,7 +206,18 @@ function installVoiceHotkey(getMainWindow) {
|
||||
|
||||
uIOhook.on('keydown', (e) => {
|
||||
markAlive();
|
||||
if (held || tapKeycode === undefined) return;
|
||||
if (held) return;
|
||||
// Ctrl+Win chord (the Windows fn-equivalent): either modifier landing second completes it.
|
||||
if (combo.special === 'ctrlmeta') {
|
||||
const isMeta = e.keycode === UiohookKey.Meta || e.keycode === UiohookKey.MetaRight;
|
||||
const isCtrl = e.keycode === UiohookKey.Ctrl || e.keycode === UiohookKey.CtrlRight;
|
||||
if ((isMeta && e.ctrlKey) || (isCtrl && e.metaKey)) {
|
||||
held = true;
|
||||
send('voice:hold-down');
|
||||
}
|
||||
return;
|
||||
}
|
||||
if (tapKeycode === undefined) return;
|
||||
if (e.keycode === tapKeycode && modsMatch(e)) {
|
||||
held = true;
|
||||
send('voice:hold-down');
|
||||
@@ -146,27 +242,31 @@ function installVoiceHotkey(getMainWindow) {
|
||||
}
|
||||
};
|
||||
tryStartNativeTap();
|
||||
startFnWatcher();
|
||||
registerVoiceShortcut();
|
||||
app.on('browser-window-focus', unregisterFallbackShortcut);
|
||||
app.on('browser-window-blur', registerVoiceShortcut);
|
||||
|
||||
// The focused-window relay matches the FALLBACK chord: special primaries (fn, Ctrl+Win) are
|
||||
// invisible to renderer key events, their tiers prove themselves through native taps instead.
|
||||
const inputMatchesCombo = (input) => {
|
||||
const k = combo.key;
|
||||
const c = fallbackCombo;
|
||||
const k = c.key;
|
||||
const keyHit = k.length === 1
|
||||
? (input.code === `Key${k.toUpperCase()}` || (input.key || '').toLowerCase() === k.toLowerCase())
|
||||
: (input.code === k || input.key === k);
|
||||
return keyHit &&
|
||||
(!combo.mods.meta || input.meta) &&
|
||||
(!combo.mods.ctrl || input.control) &&
|
||||
(!combo.mods.alt || input.alt) &&
|
||||
(!combo.mods.shift || input.shift);
|
||||
(!c.mods.meta || input.meta) &&
|
||||
(!c.mods.ctrl || input.control) &&
|
||||
(!c.mods.alt || input.alt) &&
|
||||
(!c.mods.shift || input.shift);
|
||||
};
|
||||
|
||||
const installVoiceHoldRelay = (contents) => {
|
||||
contents.on('before-input-event', (event, input) => {
|
||||
if (input.type !== 'keyDown' || input.isAutoRepeat) return;
|
||||
if (inputMatchesCombo(input)) {
|
||||
if (!tapProven) sendFallbackToggle();
|
||||
if (!tapProven && !fnProven) sendFallbackToggle();
|
||||
event.preventDefault();
|
||||
}
|
||||
});
|
||||
@@ -183,13 +283,15 @@ function installVoiceHotkey(getMainWindow) {
|
||||
const next = parseCombo(comboStr);
|
||||
if (next.accel === combo.accel) return;
|
||||
combo = next;
|
||||
if (UiohookKeyRef) tapKeycode = uiohookKeycodeFor(combo.key, UiohookKeyRef);
|
||||
fallbackCombo = combo.special ? parseCombo(LEGACY_COMBO) : combo;
|
||||
if (UiohookKeyRef && !combo.special) tapKeycode = uiohookKeycodeFor(combo.key, UiohookKeyRef);
|
||||
startFnWatcher();
|
||||
unregisterFallbackShortcut();
|
||||
registerVoiceShortcut();
|
||||
console.log('[voice] hotkey set to', combo.accel);
|
||||
});
|
||||
|
||||
ipcMain.handle('voice:hold-capable', () => tapProven);
|
||||
ipcMain.handle('voice:hold-capable', () => tapProven || fnProven);
|
||||
// Settings' "Hold to talk" fires the Accessibility prompt; Input Monitoring has no Electron API,
|
||||
// but a running tap makes macOS list the app in that pane for the user to flip.
|
||||
ipcMain.handle('voice:request-hold-permission', () => {
|
||||
|
||||
Reference in New Issue
Block a user