mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-30 11:49:50 +02:00
[eric] dictation: the paste target is snapshotted at press-start, so focus drift during decode cannot steal the transcript (ENG-176)
This commit is contained in:
@@ -1,5 +1,6 @@
|
||||
import { getLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
import { getWebview } from '@/shared/browserRegistry';
|
||||
import { takeInjectSnapshot, setInjectSnapshot } from './injectTargetSnapshot';
|
||||
|
||||
// Dictation lands where the user's cursor actually is, like every real dictation tool: a focused
|
||||
// in-app field gets the text typed in (undo-friendly, fires React input events), a focused browser
|
||||
@@ -7,7 +8,8 @@ import { getWebview } from '@/shared/browserRegistry';
|
||||
export type InjectTarget = 'field' | 'webview' | 'composer' | null;
|
||||
|
||||
export function injectAtFocus(text: string): InjectTarget {
|
||||
const active = document.activeElement as HTMLElement | null;
|
||||
const snap = takeInjectSnapshot();
|
||||
const active = snap.el || (document.activeElement as HTMLElement | null);
|
||||
if (active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA' || active.isContentEditable)) {
|
||||
try {
|
||||
active.focus();
|
||||
@@ -33,7 +35,7 @@ export function injectAtFocus(text: string): InjectTarget {
|
||||
try { void focusedTag.insertText(text); return 'webview'; } catch { /* fall through */ }
|
||||
}
|
||||
// Last-interacted browser card: the user clicked a page field, then hit the hotkey.
|
||||
const browserId = getLastInteractedBrowser();
|
||||
const browserId = snap.browserId || getLastInteractedBrowser();
|
||||
if (browserId) {
|
||||
const wv = getWebview(browserId) as unknown as { insertText?: (t: string) => Promise<void>; focus?: () => void } | undefined;
|
||||
if (wv?.insertText) {
|
||||
@@ -44,3 +46,11 @@ export function injectAtFocus(text: string): InjectTarget {
|
||||
window.dispatchEvent(new CustomEvent('openswarm:dictation-fallback', { detail: { text } }));
|
||||
return 'composer';
|
||||
}
|
||||
|
||||
/** Called at press-start so the words land where the user was looking, not where focus drifted. */
|
||||
export function snapshotInjectTarget(): void {
|
||||
setInjectSnapshot({
|
||||
el: document.activeElement as HTMLElement | null,
|
||||
browserId: getLastInteractedBrowser(),
|
||||
});
|
||||
}
|
||||
|
||||
@@ -0,0 +1,39 @@
|
||||
// ENG-176: the transcript belongs to the field the user was in when they STARTED speaking.
|
||||
// Run: node --test --experimental-strip-types frontend/src/shared/voice/injectTargetSnapshot.test.ts
|
||||
import { test, beforeEach } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { setInjectSnapshot, clearInjectSnapshot, takeInjectSnapshot, isUsableTarget } from './injectTargetSnapshot.ts';
|
||||
|
||||
const field = (connected = true) => ({ tagName: 'TEXTAREA', isConnected: connected, isContentEditable: false }) as unknown as HTMLElement;
|
||||
|
||||
beforeEach(() => clearInjectSnapshot());
|
||||
|
||||
test('the snapshotted field wins over whatever is focused later', () => {
|
||||
const a = field();
|
||||
setInjectSnapshot({ el: a, browserId: null });
|
||||
assert.equal(takeInjectSnapshot().el, a);
|
||||
});
|
||||
|
||||
test('a detached field is refused so injection falls back to live focus', () => {
|
||||
setInjectSnapshot({ el: field(false), browserId: null });
|
||||
assert.equal(takeInjectSnapshot().el, null);
|
||||
});
|
||||
|
||||
test('a non-typeable element never wins', () => {
|
||||
assert.equal(isUsableTarget({ tagName: 'DIV', isConnected: true, isContentEditable: false } as unknown as HTMLElement), false);
|
||||
assert.equal(isUsableTarget({ tagName: 'DIV', isConnected: true, isContentEditable: true } as unknown as HTMLElement), true);
|
||||
assert.equal(isUsableTarget({ tagName: 'WEBVIEW', isConnected: true, isContentEditable: false } as unknown as HTMLElement), true);
|
||||
});
|
||||
|
||||
test('taking consumes it, so one take can never leak into the next', () => {
|
||||
setInjectSnapshot({ el: field(), browserId: 'b1' });
|
||||
assert.equal(takeInjectSnapshot().browserId, 'b1');
|
||||
assert.equal(takeInjectSnapshot().el, null);
|
||||
assert.equal(takeInjectSnapshot().browserId, null);
|
||||
});
|
||||
|
||||
test('a cancelled take leaves nothing behind', () => {
|
||||
setInjectSnapshot({ el: field(), browserId: 'b2' });
|
||||
clearInjectSnapshot();
|
||||
assert.equal(takeInjectSnapshot().el, null);
|
||||
});
|
||||
@@ -0,0 +1,30 @@
|
||||
// Focus drifts during the seconds of decode + polish (click another field, another app), so the
|
||||
// dictation destination is snapshotted when the user starts speaking and preferred at paste time.
|
||||
export interface InjectSnapshot {
|
||||
el: HTMLElement | null;
|
||||
browserId: string | null;
|
||||
}
|
||||
|
||||
let p_snapshot: InjectSnapshot | null = null;
|
||||
|
||||
export function setInjectSnapshot(snap: InjectSnapshot): void {
|
||||
p_snapshot = snap;
|
||||
}
|
||||
|
||||
export function clearInjectSnapshot(): void {
|
||||
p_snapshot = null;
|
||||
}
|
||||
|
||||
/** A snapshot is only worth honoring while its element is still attached and still typeable. */
|
||||
export function isUsableTarget(el: HTMLElement | null): boolean {
|
||||
if (!el || !el.isConnected) return false;
|
||||
return el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.tagName === 'WEBVIEW' || el.isContentEditable;
|
||||
}
|
||||
|
||||
/** Consumes the snapshot: reading it once is the whole contract, so a stale one can never linger. */
|
||||
export function takeInjectSnapshot(): InjectSnapshot {
|
||||
const snap = p_snapshot;
|
||||
p_snapshot = null;
|
||||
if (!snap) return { el: null, browserId: null };
|
||||
return { el: isUsableTarget(snap.el) ? snap.el : null, browserId: snap.browserId };
|
||||
}
|
||||
@@ -3,7 +3,8 @@ import { API_BASE } from '@/shared/config';
|
||||
import { getLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
import { encodeWav, VOICE_SAMPLE_RATE } from './encodeWav';
|
||||
import { playVoiceCue } from './voiceCues';
|
||||
import { injectAtFocus } from './injectAtFocus';
|
||||
import { injectAtFocus, snapshotInjectTarget } from './injectAtFocus';
|
||||
import { clearInjectSnapshot } from './injectTargetSnapshot';
|
||||
import { createSilenceDetector } from './createSilenceDetector';
|
||||
import { pushDictation } from './voiceHistory';
|
||||
import { learnFromTranscript, isDictionaryEcho } from './voiceDictionary';
|
||||
@@ -158,6 +159,8 @@ export function useVoiceDictation() {
|
||||
}
|
||||
}
|
||||
setError(null);
|
||||
// Where the words belong is decided NOW, while the user is looking at it, not seconds later.
|
||||
snapshotInjectTarget();
|
||||
// Warm on the DOWN edge, before the mic prompt, so the model load overlaps the user starting to speak.
|
||||
void window.openswarm?.voiceWarmup?.();
|
||||
setPartial(null);
|
||||
@@ -328,6 +331,7 @@ export function useVoiceDictation() {
|
||||
// The capsule's X: throw the take away. No transcription, no cue, straight back to idle.
|
||||
const cancel = useCallback((): void => {
|
||||
if (stateRef.current !== 'recording') return;
|
||||
clearInjectSnapshot();
|
||||
const rec = recRef.current;
|
||||
if (rec?.streaming) window.openswarm?.voiceStreamCancel?.();
|
||||
teardown();
|
||||
|
||||
Reference in New Issue
Block a user