mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-09 11:17:44 +02:00
[eric] dictation: no cursor means your composer, not a new chat (ENG-378)
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01En8dRGsJPLrJCQBEkTH4Mp
This commit is contained in:
co-authored by
Claude Fable 5
parent
326574b995
commit
6c14b2d65b
@@ -18,6 +18,7 @@ import { AttachmentChips } from './AttachmentChips';
|
||||
import { EditorSurface } from './EditorSurface';
|
||||
import { ChatInputToolbar } from '../toolbar/ChatInputToolbar';
|
||||
import { ChatInputOverlays } from './ChatInputOverlays';
|
||||
import { setLastFocusedComposer } from '@/shared/composerFocus';
|
||||
|
||||
type ThinkingLevel = 'off' | 'low' | 'medium' | 'high' | 'auto';
|
||||
interface ModeConf { label: string; icon: React.ReactNode; color: string }
|
||||
@@ -113,6 +114,7 @@ export const ChatInputView: React.FC<Props> = (p) => {
|
||||
<Box
|
||||
ref={p.containerRef}
|
||||
data-osw-composer={p.sessionId ?? 'dashboard'}
|
||||
onFocusCapture={() => setLastFocusedComposer(p.sessionId ?? 'dashboard')}
|
||||
onDragOver={p.handleDragOver}
|
||||
onDragLeave={p.handleDragLeave}
|
||||
onDrop={p.handleDrop}
|
||||
|
||||
@@ -1,5 +1,7 @@
|
||||
import { useCallback, useEffect, useMemo, useRef, useState } from 'react';
|
||||
import { useAppSelector } from '@/shared/hooks';
|
||||
import { store } from '@/shared/state/store';
|
||||
import { selectViewportCoveringCardId } from '@/shared/state/dashboardLayoutSlice';
|
||||
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
|
||||
import { useElementSelection } from '@/app/components/editor/ElementSelectionContext';
|
||||
import { clipboardCardToSelectedElement } from '@/app/pages/AgentChat/ChatInput/hooks/pasteCards';
|
||||
@@ -239,12 +241,15 @@ export function useDashboardController(dashboardId: string, isActive: boolean) {
|
||||
}
|
||||
}, [toolbarOpen, toolbarPrefill, toolbarPrefillMode]);
|
||||
|
||||
// Dictation with no field focused lands HERE instead of vanishing: the composer opens with the transcript typed in, unsent.
|
||||
// Dictation with no field focused lands HERE instead of vanishing: the composer opens with the transcript typed in, unsent. Claiming = cancelling the event; a card that owns the screen hides this composer, so the words stay unclaimed and fall to the clipboard instead of into a toolbar nobody can see.
|
||||
useEffect(() => {
|
||||
if (!isActive) return;
|
||||
const onDictation = (e: Event): void => {
|
||||
const text = (e as CustomEvent).detail?.text;
|
||||
if (typeof text === 'string' && text.trim()) handleStarter(text);
|
||||
if (typeof text !== 'string' || !text.trim()) return;
|
||||
if (selectViewportCoveringCardId(store.getState())) return;
|
||||
e.preventDefault();
|
||||
handleStarter(text);
|
||||
};
|
||||
window.addEventListener('openswarm:dictation-fallback', onDictation);
|
||||
return () => window.removeEventListener('openswarm:dictation-fallback', onDictation);
|
||||
|
||||
@@ -0,0 +1,10 @@
|
||||
// The composer the user last typed in, by its data-osw-composer owner id. Dictation with no cursor anywhere lands there (expanding a collapsed card on the way) instead of in a brand-new chat, so a pending attachment or draft is never stranded.
|
||||
let lastFocusedComposerId: string | null = null;
|
||||
|
||||
export function setLastFocusedComposer(ownerId: string): void {
|
||||
lastFocusedComposerId = ownerId;
|
||||
}
|
||||
|
||||
export function getLastFocusedComposer(): string | null {
|
||||
return lastFocusedComposerId;
|
||||
}
|
||||
@@ -1,5 +1,9 @@
|
||||
import { getLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
import { getLastFocusedComposer } from '@/shared/composerFocus';
|
||||
import { getWebview } from '@/shared/browserRegistry';
|
||||
import { store } from '@/shared/state/store';
|
||||
import { expandSession } from '@/shared/state/agentsSlice';
|
||||
import { selectViewportCoveringCardId } from '@/shared/state/dashboardLayoutSlice';
|
||||
import { takeInjectSnapshot, setInjectSnapshot, isUsableTarget } from './injectTargetSnapshot';
|
||||
import { guestHasEditableFocus } from './guestHasEditableFocus';
|
||||
|
||||
@@ -8,6 +12,72 @@ import { guestHasEditableFocus } from './guestHasEditableFocus';
|
||||
// card forwards into the guest page's field, anything else falls back to the OS-level paste.
|
||||
export type InjectTarget = 'field' | 'webview' | 'composer' | null;
|
||||
|
||||
function isField(el: HTMLElement): boolean {
|
||||
return el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.isContentEditable;
|
||||
}
|
||||
|
||||
function insertIntoField(active: HTMLElement, text: string): boolean {
|
||||
try {
|
||||
active.focus();
|
||||
// execCommand keeps the undo stack and fires the input events React listens for; the manual
|
||||
// fallback covers fields where Chromium refuses the command (rare, e.g. type=number).
|
||||
const ok = document.execCommand('insertText', false, text);
|
||||
if (!ok && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA')) {
|
||||
const el = active as HTMLInputElement | HTMLTextAreaElement;
|
||||
const s = el.selectionStart ?? el.value.length;
|
||||
const e = el.selectionEnd ?? el.value.length;
|
||||
el.value = el.value.slice(0, s) + text + el.value.slice(e);
|
||||
el.selectionStart = el.selectionEnd = s + text.length;
|
||||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// A programmatic focus parks the caret at the START of a contenteditable, which would splice the dictation in front of an existing draft.
|
||||
function placeCaretAtEnd(el: HTMLElement): void {
|
||||
if (el.tagName === 'INPUT' || el.tagName === 'TEXTAREA') {
|
||||
const f = el as HTMLInputElement | HTMLTextAreaElement;
|
||||
f.selectionStart = f.selectionEnd = f.value.length;
|
||||
return;
|
||||
}
|
||||
const sel = window.getSelection();
|
||||
if (!sel) return;
|
||||
const range = document.createRange();
|
||||
range.selectNodeContents(el);
|
||||
range.collapse(false);
|
||||
sel.removeAllRanges();
|
||||
sel.addRange(range);
|
||||
}
|
||||
|
||||
function nextFrame(): Promise<void> {
|
||||
return new Promise((r) => requestAnimationFrame(() => r()));
|
||||
}
|
||||
|
||||
function composerEditor(ownerId: string): HTMLElement | null {
|
||||
const root = document.querySelector<HTMLElement>(`[data-osw-composer="${CSS.escape(ownerId)}"]`);
|
||||
return root?.querySelector<HTMLElement>('[contenteditable="true"], textarea, input') ?? null;
|
||||
}
|
||||
|
||||
// The one composer a dictation with no cursor can honestly mean: the card owning the screen (its only input), else the composer the user last typed in; a collapsed card is expanded first, and since its chat stays mounted across collapse the draft and pending attachments ride along.
|
||||
async function resolveComposerTarget(): Promise<HTMLElement | null> {
|
||||
const state = store.getState();
|
||||
const covering = selectViewportCoveringCardId(state);
|
||||
const ownerId = covering ?? getLastFocusedComposer();
|
||||
if (!ownerId) return null;
|
||||
const editor = composerEditor(ownerId);
|
||||
if (!editor) return null;
|
||||
if (editor.getClientRects().length === 0) {
|
||||
if (!state.agents.sessions[ownerId] || state.agents.expandedSessionIds.includes(ownerId)) return null;
|
||||
store.dispatch(expandSession(ownerId));
|
||||
for (let i = 0; i < 12 && editor.getClientRects().length === 0; i++) await nextFrame();
|
||||
if (editor.getClientRects().length === 0) return null;
|
||||
}
|
||||
return editor;
|
||||
}
|
||||
|
||||
export async function injectAtFocus(text: string): Promise<InjectTarget> {
|
||||
const snap = takeInjectSnapshot();
|
||||
// The cursor wins, not where you started. Wispr's grammar, and Eric's call: you dictate, you click
|
||||
@@ -17,25 +87,7 @@ export async function injectAtFocus(text: string): Promise<InjectTarget> {
|
||||
// nothing typeable (a button, the body) while you were talking.
|
||||
const live = document.activeElement as HTMLElement | null;
|
||||
const active = isUsableTarget(live) ? live : snap.el;
|
||||
if (active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA' || active.isContentEditable)) {
|
||||
try {
|
||||
active.focus();
|
||||
// execCommand keeps the undo stack and fires the input events React listens for; the manual
|
||||
// fallback covers fields where Chromium refuses the command (rare, e.g. type=number).
|
||||
const ok = document.execCommand('insertText', false, text);
|
||||
if (!ok && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA')) {
|
||||
const el = active as HTMLInputElement | HTMLTextAreaElement;
|
||||
const s = el.selectionStart ?? el.value.length;
|
||||
const e = el.selectionEnd ?? el.value.length;
|
||||
el.value = el.value.slice(0, s) + text + el.value.slice(e);
|
||||
el.selectionStart = el.selectionEnd = s + text.length;
|
||||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
return 'field';
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
if (active && isField(active)) return insertIntoField(active, text) ? 'field' : null;
|
||||
// A webview steals focus when the user clicks into a page, so activeElement IS the webview tag.
|
||||
// insertText silently no-ops when the guest has no focused editable, which read as the dictation
|
||||
// vanishing (ENG-254): confirm the guest target BEFORE claiming success, and await the insert so a
|
||||
@@ -53,9 +105,15 @@ export async function injectAtFocus(text: string): Promise<InjectTarget> {
|
||||
try { wv.focus?.(); await wv.insertText(text); return 'webview'; } catch { /* fall through */ }
|
||||
}
|
||||
}
|
||||
// No cursor anywhere: open the dashboard composer with the transcript typed in. Words are never dropped.
|
||||
window.dispatchEvent(new CustomEvent('openswarm:dictation-fallback', { detail: { text } }));
|
||||
return 'composer';
|
||||
const composer = await resolveComposerTarget();
|
||||
if (composer) {
|
||||
composer.focus();
|
||||
placeCaretAtEnd(composer);
|
||||
if (insertIntoField(composer, text)) return 'field';
|
||||
}
|
||||
// No composer anywhere: the dashboard toolbar opens with the transcript typed in and claims the event by cancelling it; unclaimed (no dashboard, or a card owns the screen) means nobody took the words, so the caller's clipboard fallback speaks instead of the text vanishing.
|
||||
const claimed = !window.dispatchEvent(new CustomEvent('openswarm:dictation-fallback', { detail: { text }, cancelable: true }));
|
||||
return claimed ? 'composer' : null;
|
||||
}
|
||||
|
||||
/** Called at press-start so the words land where the user was looking, not where focus drifted. */
|
||||
|
||||
@@ -4,7 +4,12 @@ import { test, beforeEach } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { setInjectSnapshot, clearInjectSnapshot, takeInjectSnapshot, isUsableTarget } from './injectTargetSnapshot.ts';
|
||||
|
||||
const field = (connected = true) => ({ tagName: 'TEXTAREA', isConnected: connected, isContentEditable: false }) as unknown as HTMLElement;
|
||||
// Rendered by default: the predicate refuses anything with no layout box (a composer inside a collapsed card is display:none).
|
||||
const el = (tagName: string, o: { connected?: boolean; editable?: boolean; rendered?: boolean } = {}) => ({
|
||||
tagName, isConnected: o.connected ?? true, isContentEditable: o.editable ?? false,
|
||||
getClientRects: () => ((o.rendered ?? true) ? [{}] : []),
|
||||
}) as unknown as HTMLElement;
|
||||
const field = (connected = true) => el('TEXTAREA', { connected });
|
||||
|
||||
beforeEach(() => clearInjectSnapshot());
|
||||
|
||||
@@ -20,9 +25,9 @@ test('a detached field is refused, so a dead origin can never be the destination
|
||||
});
|
||||
|
||||
test('a non-typeable element never wins', () => {
|
||||
assert.equal(isUsableTarget({ tagName: 'DIV', isConnected: true, isContentEditable: false } as unknown as HTMLElement), false);
|
||||
assert.equal(isUsableTarget({ tagName: 'DIV', isConnected: true, isContentEditable: true } as unknown as HTMLElement), true);
|
||||
assert.equal(isUsableTarget({ tagName: 'WEBVIEW', isConnected: true, isContentEditable: false } as unknown as HTMLElement), true);
|
||||
assert.equal(isUsableTarget(el('DIV')), false);
|
||||
assert.equal(isUsableTarget(el('DIV', { editable: true })), true);
|
||||
assert.equal(isUsableTarget(el('WEBVIEW')), true);
|
||||
});
|
||||
|
||||
test('taking consumes it, so one take can never leak into the next', () => {
|
||||
@@ -42,11 +47,20 @@ test('a cancelled take leaves nothing behind', () => {
|
||||
// injectAtFocus needs a live DOM, so what is pinned here is the predicate that decides whether the
|
||||
// live element is allowed to win at all. Getting this wrong is how the text lands in a stranger's box.
|
||||
test('a live click target only beats the origin when it is really typeable', () => {
|
||||
const typeable = { tagName: 'INPUT', isConnected: true, isContentEditable: false } as unknown as HTMLElement;
|
||||
const button = { tagName: 'BUTTON', isConnected: true, isContentEditable: false } as unknown as HTMLElement;
|
||||
const body = { tagName: 'BODY', isConnected: true, isContentEditable: false } as unknown as HTMLElement;
|
||||
const typeable = el('INPUT');
|
||||
const button = el('BUTTON');
|
||||
const body = el('BODY');
|
||||
assert.equal(isUsableTarget(typeable), true, 'clicking another field must take the text');
|
||||
assert.equal(isUsableTarget(button), false, 'clicking a button must NOT take the text');
|
||||
assert.equal(isUsableTarget(body), false, 'clicking empty space must NOT take the text');
|
||||
assert.equal(isUsableTarget(null), false);
|
||||
});
|
||||
|
||||
test('a hidden field is refused: a collapsed card\'s composer cannot take a caret', () => {
|
||||
const hidden = el('DIV', { editable: true, rendered: false });
|
||||
assert.equal(isUsableTarget(hidden), false);
|
||||
setInjectSnapshot({ el: hidden, browserId: null });
|
||||
const taken = takeInjectSnapshot();
|
||||
assert.equal(taken.el, null);
|
||||
assert.equal(taken.targetLost, true, 'it was aimed at and then hidden, which is a lost target, not no target');
|
||||
});
|
||||
|
||||
@@ -15,9 +15,9 @@ export function clearInjectSnapshot(): void {
|
||||
p_snapshot = null;
|
||||
}
|
||||
|
||||
/** A snapshot is only worth honoring while its element is still attached and still typeable. */
|
||||
/** A snapshot is only worth honoring while its element is still attached, still typeable, and still rendered: a composer inside a collapsed card is display:none and can't take a caret. */
|
||||
export function isUsableTarget(el: HTMLElement | null): boolean {
|
||||
if (!el || !el.isConnected) return false;
|
||||
if (!el || !el.isConnected || el.getClientRects().length === 0) return false;
|
||||
return el.tagName === 'INPUT' || el.tagName === 'TEXTAREA' || el.tagName === 'WEBVIEW' || el.isContentEditable;
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user