mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-23 21:12:22 +02:00
[eric] voice: target chip shows where words will land, idle sliver at the top expands on hover as the always-there entry
This commit is contained in:
@@ -10,7 +10,7 @@ import VoiceOverlay from './VoiceOverlay';
|
||||
// out-of-sync state. Mounted once near the app root.
|
||||
|
||||
export function VoiceDictationProvider({ children }: { children: React.ReactNode }): React.ReactElement {
|
||||
const { state, lastText, error, pct, feedback, partial, toggle, start, stop, cancel, notify, volumeRef } = useVoiceDictation();
|
||||
const { state, lastText, error, pct, feedback, partial, targetLabel, toggle, start, stop, cancel, notify, volumeRef } = useVoiceDictation();
|
||||
|
||||
// fn is the dictation key, but macOS may still have its own Globe action bound (emoji picker on a quick tap); say so once.
|
||||
const globeWarnedRef = useRef(false);
|
||||
@@ -104,7 +104,7 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode
|
||||
const confirmRecording = useCallback((): void => { void stop(); }, [stop]);
|
||||
|
||||
return (
|
||||
<VoiceContext.Provider value={{ state, lastText, error, pct, feedback, partial, toggle, pressStart, pressEnd, confirmRecording, cancelRecording: cancel, holdMode, volumeRef }}>
|
||||
<VoiceContext.Provider value={{ state, lastText, error, pct, feedback, partial, targetLabel, toggle, pressStart, pressEnd, confirmRecording, cancelRecording: cancel, holdMode, volumeRef }}>
|
||||
{children}
|
||||
<VoiceOverlay />
|
||||
</VoiceContext.Provider>
|
||||
|
||||
@@ -147,8 +147,43 @@ const LiveTranscript: React.FC<{ committed: string; tentative: string }> = ({ co
|
||||
);
|
||||
};
|
||||
|
||||
// Always-there entry point at the top edge, Wispr's idle bar translated to our gloop position:
|
||||
// a sliver that expands on hover into mic + hotkey hint; press semantics match the hotkey.
|
||||
const IdlePill: React.FC<{ onPressStart: () => void; onPressEnd: () => void }> = ({ onPressStart, onPressEnd }) => {
|
||||
const [hover, setHover] = useState(false);
|
||||
return (
|
||||
<Box
|
||||
onMouseEnter={() => setHover(true)}
|
||||
onMouseLeave={() => setHover(false)}
|
||||
onMouseDown={(e) => e.preventDefault()}
|
||||
onPointerDown={onPressStart}
|
||||
onPointerUp={onPressEnd}
|
||||
role="button"
|
||||
aria-label="Start dictation"
|
||||
sx={{
|
||||
position: 'fixed', top: 6, left: '50%', transform: 'translateX(-50%)', zIndex: 2147482998,
|
||||
WebkitAppRegion: 'no-drag', cursor: 'pointer',
|
||||
display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 0.75,
|
||||
height: hover ? 24 : 8, minWidth: hover ? 74 : 44, px: hover ? 1.25 : 0,
|
||||
borderRadius: 999,
|
||||
background: hover ? 'rgba(18,16,24,0.92)' : 'rgba(120,116,134,0.28)',
|
||||
boxShadow: hover ? '0 6px 20px rgba(0,0,0,0.35), inset 0 0 0 1px rgba(255,255,255,0.08)' : 'inset 0 0 0 1px rgba(255,255,255,0.14)',
|
||||
transition: 'all 0.18s cubic-bezier(0.32, 0.72, 0, 1)',
|
||||
overflow: 'hidden',
|
||||
}}
|
||||
>
|
||||
{hover && (
|
||||
<>
|
||||
<MicIcon sx={{ fontSize: 13, color: 'rgba(255,255,255,0.85)' }} />
|
||||
<Box component="span" sx={{ color: 'rgba(255,255,255,0.6)', fontSize: '0.6875rem', fontWeight: 700, letterSpacing: '0.04em' }}>fn</Box>
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
const VoiceOverlay: React.FC = () => {
|
||||
const { state, pct, feedback, partial, volumeRef, confirmRecording, cancelRecording } = useVoice();
|
||||
const { state, pct, feedback, partial, targetLabel, volumeRef, confirmRecording, cancelRecording, pressStart, pressEnd } = useVoice();
|
||||
const [showFeedback, setShowFeedback] = useState(false);
|
||||
// The capsule lingers past its state for one out-animation beat, so it glides back up instead of vanishing.
|
||||
const [capsuleLeaving, setCapsuleLeaving] = useState(false);
|
||||
@@ -175,8 +210,9 @@ const VoiceOverlay: React.FC = () => {
|
||||
}, [feedback]);
|
||||
|
||||
const live = state !== 'idle';
|
||||
const desktop = !!(window as unknown as { openswarm?: { voiceTranscribe?: unknown } }).openswarm?.voiceTranscribe;
|
||||
const visible = live || capsuleMounted || (showFeedback && !!feedback);
|
||||
if (!visible) return null;
|
||||
if (!visible) return desktop ? <IdlePill onPressStart={pressStart} onPressEnd={pressEnd} /> : null;
|
||||
|
||||
const capsule = capsuleMounted ? (
|
||||
<VoiceCapsule transcribing={state === 'transcribing'} leaving={capsuleLeaving} volumeRef={volumeRef} onCancel={cancelRecording} onConfirm={confirmRecording} />
|
||||
@@ -185,8 +221,12 @@ const VoiceOverlay: React.FC = () => {
|
||||
const hasPartial = !!partial && !!(partial.committed || partial.tentative);
|
||||
let content: React.ReactElement | null;
|
||||
if (state === 'recording' || state === 'transcribing') {
|
||||
// The capsule says "listening"; the card appears only once there are live words to show.
|
||||
content = hasPartial ? <LiveTranscript committed={partial.committed} tentative={partial.tentative} /> : null;
|
||||
// Words once there are words; before that, the chip says WHERE they will land.
|
||||
content = hasPartial
|
||||
? <LiveTranscript committed={partial.committed} tentative={partial.tentative} />
|
||||
: (state === 'recording' && targetLabel
|
||||
? <Box component="span" sx={{ color: 'rgba(255,255,255,0.65)', fontSize: '0.75rem' }}>{'→'} {targetLabel}</Box>
|
||||
: null);
|
||||
} else if (state === 'preparing') {
|
||||
content = (<><CircularProgress size={13} thickness={5} sx={{ color: 'rgba(255,255,255,0.7)' }} /><span>Downloading voice model {pct}%</span></>);
|
||||
} else if (feedback) {
|
||||
@@ -210,7 +250,7 @@ const VoiceOverlay: React.FC = () => {
|
||||
sx={{
|
||||
// Hangs just under the droplet so the live words read as its tail.
|
||||
position: 'fixed',
|
||||
top: 52,
|
||||
top: 78,
|
||||
left: '50%',
|
||||
transform: 'translateX(-50%)',
|
||||
zIndex: 2147483000,
|
||||
|
||||
@@ -1,5 +1,6 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { API_BASE } from '@/shared/config';
|
||||
import { getLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
import { encodeWav, VOICE_SAMPLE_RATE } from './encodeWav';
|
||||
import { playVoiceCue } from './voiceCues';
|
||||
import { injectAtFocus } from './injectAtFocus';
|
||||
@@ -38,6 +39,19 @@ export interface VoiceFeedback {
|
||||
at: number;
|
||||
}
|
||||
|
||||
// Where the transcript will land RIGHT NOW, in the user's words; mirrors injectAtFocus's tiers so
|
||||
// the capsule's target chip never promises a destination injection would not actually pick.
|
||||
export function describeInjectTarget(): string {
|
||||
const a = document.activeElement as HTMLElement | null;
|
||||
if (a && (a.tagName === 'INPUT' || a.tagName === 'TEXTAREA' || a.isContentEditable)) {
|
||||
const hint = a.getAttribute('placeholder') || a.getAttribute('aria-label');
|
||||
return hint ? hint.slice(0, 30) : 'text field';
|
||||
}
|
||||
if (a && a.tagName === 'WEBVIEW') return 'browser page';
|
||||
if (getLastInteractedBrowser()) return 'browser page';
|
||||
return 'chat composer';
|
||||
}
|
||||
|
||||
// Context hint for the polisher: what the user is dictating into (a field label, a page title), so
|
||||
// names and jargon spell right. Never page CONTENT, just the one-line "where".
|
||||
function dictationContext(): string {
|
||||
@@ -72,6 +86,7 @@ export function useVoiceDictation() {
|
||||
const [pct, setPct] = useState<number>(0);
|
||||
const [feedback, setFeedback] = useState<VoiceFeedback | null>(null);
|
||||
const [partial, setPartial] = useState<VoicePartial | null>(null);
|
||||
const [targetLabel, setTargetLabel] = useState<string>('');
|
||||
const partialSeqRef = useRef<number>(0);
|
||||
const recRef = useRef<Recorder | null>(null);
|
||||
const stateRef = useRef<VoiceState>('idle');
|
||||
@@ -291,6 +306,20 @@ export function useVoiceDictation() {
|
||||
if (state === 'idle' || state === 'preparing') setPartial(null);
|
||||
}, [state]);
|
||||
|
||||
// The target chip tracks focus LIVE while recording: clicking into a field mid-dictation retargets
|
||||
// injection (by design), and the chip must tell that truth as it happens.
|
||||
useEffect(() => {
|
||||
if (state !== 'recording') { setTargetLabel(''); return undefined; }
|
||||
setTargetLabel(describeInjectTarget());
|
||||
const onFocus = (): void => setTargetLabel(describeInjectTarget());
|
||||
window.addEventListener('focusin', onFocus, true);
|
||||
window.addEventListener('focusout', onFocus, true);
|
||||
return () => {
|
||||
window.removeEventListener('focusin', onFocus, true);
|
||||
window.removeEventListener('focusout', onFocus, true);
|
||||
};
|
||||
}, [state]);
|
||||
|
||||
// A dangling recorder (unmount mid-capture) must release the mic.
|
||||
useEffect(() => () => { teardown(); }, [teardown]);
|
||||
|
||||
@@ -299,5 +328,5 @@ export function useVoiceDictation() {
|
||||
setFeedback({ tone: 'warn', icon: 'info', text, at: Date.now() });
|
||||
}, []);
|
||||
|
||||
return { state, lastText, error, pct, feedback, partial, toggle, start, stop, cancel, notify, volumeRef };
|
||||
return { state, lastText, error, pct, feedback, partial, targetLabel, toggle, start, stop, cancel, notify, volumeRef };
|
||||
}
|
||||
|
||||
@@ -10,17 +10,22 @@ export interface VoiceContextValue {
|
||||
pct: number;
|
||||
feedback: VoiceFeedback | null;
|
||||
partial: VoicePartial | null;
|
||||
// Where the transcript will land right now, in user words ("chat composer", a field's label).
|
||||
targetLabel: string;
|
||||
toggle: () => void;
|
||||
// Mic-button press semantics that respect the hold/toggle setting: press starts (or toggles),
|
||||
// release stops only in hold mode. Buttons wire onPointerDown/Up to these and stay mode-agnostic.
|
||||
pressStart: () => void;
|
||||
pressEnd: () => void;
|
||||
// The recording capsule's two endings: keep the take (transcribe + inject) or throw it away.
|
||||
confirmRecording: () => void;
|
||||
cancelRecording: () => void;
|
||||
holdMode: boolean;
|
||||
volumeRef: React.MutableRefObject<number>;
|
||||
}
|
||||
|
||||
const NOOP_REF = { current: 0 };
|
||||
const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, feedback: null, partial: null, toggle: () => {}, pressStart: () => {}, pressEnd: () => {}, holdMode: true, volumeRef: NOOP_REF };
|
||||
const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, feedback: null, partial: null, targetLabel: '', toggle: () => {}, pressStart: () => {}, pressEnd: () => {}, confirmRecording: () => {}, cancelRecording: () => {}, holdMode: true, volumeRef: NOOP_REF };
|
||||
|
||||
export const VoiceContext = createContext<VoiceContextValue>(NOOP);
|
||||
|
||||
|
||||
Reference in New Issue
Block a user