[eric] split: extract ChatInput attachments, editor handlers + send helpers

This commit is contained in:
ciregenz
2026-05-23 06:08:02 -07:00
parent 6002c06cb1
commit e72239ac41
5 changed files with 615 additions and 0 deletions
@@ -0,0 +1,38 @@
import { useElementSelection, SelectedElement } from '@/app/components/ElementSelectionContext';
import { getClipboardCards, clearClipboard } from '@/shared/dashboardClipboard';
/** Drains the dashboard clipboard into owner-scoped selected elements. Returns true if it consumed the paste. */
export function tryPasteClipboardCards(elementSelection: ReturnType<typeof useElementSelection>, ownerId: string): boolean {
const copied = getClipboardCards();
if (copied.length === 0 || !elementSelection) return false;
for (const card of copied) {
const semanticTypeMap: Record<string, SelectedElement['semanticType']> = {
agent: 'agent-card',
view: 'view-card',
browser: 'browser-card',
};
const semanticType = semanticTypeMap[card.type];
if (!semanticType) continue;
const labelMap: Record<string, string> = {
'agent-card': 'Agent',
'view-card': 'View',
'browser-card': 'Browser',
};
const semanticLabel = (labelMap[semanticType] || semanticType) + ': ' + card.name;
const el: SelectedElement = {
id: `sel-${Date.now()}-${Math.random().toString(36).slice(2, 8)}`,
selectorPath: `[data-select-type="${semanticType}"][data-select-id="${card.id}"]`,
tagName: 'DIV',
className: '',
outerHTML: '',
computedStyles: {},
boundingRect: { x: 0, y: 0, width: 0, height: 0 },
semanticType,
semanticLabel,
semanticData: { ...card.meta, selectId: card.id },
};
elementSelection.addElementForOwner(ownerId, el);
}
clearClipboard();
return true;
}
@@ -0,0 +1,122 @@
import { SelectedElement } from '@/app/components/ElementSelectionContext';
import { getWebview } from '@/shared/browserRegistry';
import { ContextPath } from '@/app/components/DirectoryBrowser';
import { AttachedImage } from './types';
import { SendBlock } from './useContextFiles';
type OutImage = { data: string; media_type: string };
interface SendBlockInputs {
trimmed: string;
currentModelCtx: number;
historyUsed: number;
contextPaths: ContextPath[];
sessionFrameworkOverhead: number;
}
// Pre-send dry-run guard. Sums every known component of next-turn input
// (history estimate from props, system prompt, framework/MCP overhead
// last reported by the API, attached file token estimates, and the
// prompt itself). If the sum exceeds 95% of the model's window, returns a
// block with concrete recovery actions instead of round-tripping to a
// doomed API call. Conservative on purpose: tokenizers differ across
// providers (char/4 is rough), so we leave 5% headroom plus the API's
// own response budget.
export function computeSendBlock({ trimmed, currentModelCtx, historyUsed, contextPaths, sessionFrameworkOverhead }: SendBlockInputs): NonNullable<SendBlock> | null {
const win = currentModelCtx;
const history = Math.max(0, historyUsed);
const filesSum = contextPaths.reduce((acc, cp) => acc + (cp.tokens || 0), 0);
const promptTokens = Math.ceil(trimmed.length / 4);
const framework = sessionFrameworkOverhead || 0;
const systemTokens = 0;
const estimate = history + framework + filesSum + promptTokens + systemTokens;
if (win > 0 && estimate > Math.floor(win * 0.95)) {
let largest: { path: string; tokens: number } | undefined;
for (const cp of contextPaths) {
if ((cp.tokens || 0) > (largest?.tokens || 0)) largest = { path: cp.path, tokens: cp.tokens || 0 };
}
return {
estimate, window: win,
history, system: systemTokens, framework, files: filesSum, prompt: promptTokens,
largestFile: largest,
};
}
return null;
}
/** Materialize blob-backed previews to base64 only at send; filters out empties. */
export async function materializeImages(images: AttachedImage[]): Promise<OutImage[]> {
if (images.length === 0) return [];
const all = await Promise.all(images.map(async (img) => {
if (img.data) return { data: img.data, media_type: img.media_type };
if (img._file) {
const base64 = await new Promise<string>((resolve, reject) => {
const reader = new FileReader();
reader.onload = () => {
const r = reader.result as string;
resolve(r.split(',')[1] ?? '');
};
reader.onerror = () => reject(reader.error || new Error('FileReader failed'));
reader.readAsDataURL(img._file!);
});
return { data: base64, media_type: img.media_type };
}
return { data: '', media_type: img.media_type };
}));
return all.filter((i) => i.data);
}
/** Appends a human-readable Selected UI Elements block to the prompt and pushes any element screenshots into allImages. */
export function appendSelectedElements(trimmed: string, selectedEls: SelectedElement[], allImages: OutImage[]): string {
if (selectedEls.length === 0) return trimmed;
const lines: string[] = ['\n\n---\nSelected UI Elements:\n'];
for (let i = 0; i < selectedEls.length; i++) {
const el = selectedEls[i];
if (el.semanticType === 'browser-card' && el.semanticData?.selectId) {
const wv = getWebview(el.semanticData.selectId as string);
const url = wv ? (el.semanticData.url || wv.getURL()) : (el.semanticData.url || '');
const title = wv ? (el.semanticData.name || wv.getTitle()) : (el.semanticLabel || '');
lines.push(`${i + 1}. [Browser Card] ${title}`);
lines.push(` browser_id: ${el.semanticData.selectId}`);
if (url) lines.push(` URL: ${url}`);
lines.push(` (Use BrowserAgent with this browser_id to interact with it, or CreateBrowserAgent for a new browser)`);
} else if (el.semanticType && el.semanticData) {
const typeLabel = {
'agent-card': 'Agent Card',
'message': 'Message',
'tool-call': 'Tool Call',
'tool-group': 'Tool Group',
'view-card': 'App Card',
'browser-card': 'Browser Card',
'dom-element': 'Element',
}[el.semanticType] || el.semanticType;
lines.push(`${i + 1}. [${typeLabel}] ${el.semanticLabel || ''}`);
const { selectId, ...rest } = el.semanticData;
if (selectId) lines.push(` ID: ${selectId}`);
const metaStr = Object.entries(rest)
.filter(([, v]) => v != null)
.map(([k, v]) => `${k}: ${typeof v === 'string' ? v : JSON.stringify(v)}`)
.join(', ');
if (metaStr) lines.push(` ${metaStr}`);
if (el.semanticType === 'agent-card' && selectId) {
lines.push(` (Use InvokeAgent with session_id "${selectId}" to query this agent with full conversation context)`);
}
} else {
const styleStr = Object.entries(el.computedStyles)
.map(([k, v]) => `${k}: ${v}`)
.join('; ');
lines.push(`${i + 1}. \`${el.selectorPath}\` (${el.tagName.toLowerCase()})`);
lines.push(` Selector: ${el.selectorPath}`);
lines.push(` HTML: ${el.outerHTML.length > 500 ? el.outerHTML.slice(0, 500) + '...' : el.outerHTML}`);
if (styleStr) lines.push(` Key styles: ${styleStr}`);
}
lines.push('');
if (el.screenshot) {
const base64 = el.screenshot.replace(/^data:image\/\w+;base64,/, '');
allImages.push({ data: base64, media_type: 'image/png' });
}
}
return trimmed + lines.join('\n');
}
@@ -0,0 +1,141 @@
import { useState, useCallback, useEffect, useMemo } from 'react';
import { ContextPath } from '@/app/components/DirectoryBrowser';
import { API_BASE, getAuthToken } from '@/shared/config';
import { ForcedToolGroup } from './types';
import { basename } from './helpers';
export type SendBlock = null | {
estimate: number;
window: number;
history: number;
system: number;
framework: number;
files: number;
prompt: number;
largestFile?: { path: string; tokens: number };
};
export function useContextFiles(
currentModelCtx: number,
model: string,
contextEstimate: { used: number; limit: number } | undefined,
sessionFrameworkOverhead: number,
) {
const [isUploading, setIsUploading] = useState(false);
const [contextPaths, setContextPaths] = useState<ContextPath[]>([]);
const [forcedTools, setForcedTools] = useState<ForcedToolGroup[]>([]);
const [copiedPathIdx, setCopiedPathIdx] = useState<number | null>(null);
const [oversizeQueue, setOversizeQueue] = useState<Array<{ path: string; name: string; tokens: number }>>([]);
const [summarizingPath, setSummarizingPath] = useState<string | null>(null);
const [summarizeError, setSummarizeError] = useState<string | null>(null);
const [sendBlock, setSendBlock] = useState<SendBlock>(null);
const uploadAndAttachFiles = useCallback(async (files: File[]) => {
if (files.length === 0) return;
setIsUploading(true);
try {
const formData = new FormData();
files.forEach((f) => formData.append('files', f));
const resp = await fetch(`${API_BASE}/settings/upload-files`, {
method: 'POST',
body: formData,
});
if (!resp.ok) throw new Error('Upload failed');
const data = await resp.json();
const halfCap = Math.floor(currentModelCtx * 0.5);
const oversize: Array<{ path: string; name: string; tokens: number }> = [];
const newPaths: ContextPath[] = (data.files || []).map((f: { path: string; name?: string; tokens?: number; kind?: 'text' | 'pdf' | 'image' | 'binary'; media_type?: string }) => {
const t = typeof f.tokens === 'number' ? f.tokens : 0;
if (t > halfCap) oversize.push({ path: f.path, name: f.name || basename(f.path) || 'file', tokens: t });
return { path: f.path, type: 'file' as const, tokens: t, kind: f.kind, media_type: f.media_type };
});
setContextPaths((prev) => [...prev, ...newPaths]);
if (oversize.length > 0) setOversizeQueue((q) => [...q, ...oversize]);
} catch (err) {
console.error('File upload failed:', err);
} finally {
setIsUploading(false);
}
}, [currentModelCtx]);
useEffect(() => {
const halfCap = Math.floor(currentModelCtx * 0.5);
const stillOversize: Array<{ path: string; name: string; tokens: number }> = [];
for (const cp of contextPaths) {
const t = cp.tokens || 0;
if (t > halfCap) {
const name = basename(cp.path) || cp.path;
stillOversize.push({ path: cp.path, name, tokens: t });
}
}
setOversizeQueue((q) => {
const next = stillOversize.filter((o) => !q.find((qq) => qq.path === o.path));
return [...q.filter((qq) => stillOversize.find((o) => o.path === qq.path)), ...next];
});
}, [currentModelCtx, contextPaths]);
const detachOversize = useCallback((path: string) => {
setContextPaths((prev) => prev.filter((cp) => cp.path !== path));
setOversizeQueue((q) => q.filter((o) => o.path !== path));
}, []);
const summarizeOversize = useCallback(async (path: string) => {
if (summarizingPath) return; // another summarize is in flight; ignore
setSummarizingPath(path);
try {
const tok = (() => { try { return getAuthToken(); } catch { return ''; } })();
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
if (tok) headers['Authorization'] = `Bearer ${tok}`;
const target = Math.min(8_000, Math.max(1_000, Math.floor(currentModelCtx * 0.05)));
const resp = await fetch(`${API_BASE}/settings/summarize-file`, {
method: 'POST', headers,
body: JSON.stringify({ path, target_tokens: target, primary_model: model }),
});
if (!resp.ok) {
let detail = `summarize failed (${resp.status})`;
try { const j = await resp.json(); if (j?.detail) detail = String(j.detail); } catch {}
throw new Error(detail);
}
const data = await resp.json();
const newPath: string = data.path;
const newTokens: number = data.tokens || 0;
setContextPaths((prev) => prev.map((cp) => cp.path === path ? { ...cp, path: newPath, tokens: newTokens, kind: 'text', media_type: 'text/plain' } : cp));
setOversizeQueue((q) => q.filter((o) => o.path !== path));
} catch (err) {
const msg = err instanceof Error ? err.message : 'summarize failed';
setSummarizeError(`${msg}. Detach the file or connect an aux provider in Settings.`);
} finally {
setSummarizingPath(null);
}
}, [currentModelCtx, model, summarizingPath]);
const pendingPayloadEstimate = useMemo(() => {
const history = Math.max(0, contextEstimate?.used ?? 0);
const filesSum = contextPaths.reduce((acc, cp) => acc + (cp.tokens || 0), 0);
return history + (sessionFrameworkOverhead || 0) + filesSum;
}, [contextEstimate, contextPaths, sessionFrameworkOverhead]);
const pendingKinds = useMemo(() => {
const set = new Set<string>();
for (const cp of contextPaths) {
if (cp.kind) set.add(cp.kind);
}
return set;
}, [contextPaths]);
return {
isUploading,
contextPaths, setContextPaths,
forcedTools, setForcedTools,
copiedPathIdx, setCopiedPathIdx,
oversizeQueue,
summarizingPath,
summarizeError, setSummarizeError,
sendBlock, setSendBlock,
uploadAndAttachFiles,
detachOversize,
summarizeOversize,
pendingPayloadEstimate,
pendingKinds,
};
}
@@ -0,0 +1,273 @@
import { useState, useRef, useCallback, RefObject } from 'react';
import { CommandPickerItem } from '@/app/components/CommandPicker';
import { useElementSelection } from '@/app/components/ElementSelectionContext';
import {
SKILL_PILL_ATTR,
AttachedSkill,
createSkillPillElement,
detectEditorTrigger,
TriggerState,
EMPTY_TRIGGER,
} from '@/app/components/richEditorUtils';
import { useAppDispatch } from '@/shared/hooks';
import { clearSessionMessages } from '@/shared/state/agentsSlice';
import { scheduleDraftSave } from './draftStore';
import { handleSlashCommand } from './slashCommands';
import { tryPasteClipboardCards } from './pasteCards';
import { ForcedToolGroup } from './types';
type Skill = { id: string; name: string; content: string };
interface Params {
editorRef: RefObject<HTMLDivElement>;
generalFileInputRef: RefObject<HTMLInputElement>;
ownerId: string;
sessionId?: string;
autoRunMode?: boolean;
c: any;
skills: Record<string, Skill>;
elementSelection: ReturnType<typeof useElementSelection>;
setHasContent: (v: boolean) => void;
setAttachedSkills: React.Dispatch<React.SetStateAction<Record<string, AttachedSkill>>>;
setForcedTools: React.Dispatch<React.SetStateAction<ForcedToolGroup[]>>;
onModeChange: (mode: string) => void;
addImageFiles: (files: FileList | File[]) => void;
uploadAndAttachFiles: (files: File[]) => void;
handleSend: () => void;
}
export function useEditorHandlers(p: Params) {
const {
editorRef, generalFileInputRef, ownerId, sessionId, autoRunMode, c, skills,
elementSelection, setHasContent, setAttachedSkills, setForcedTools, onModeChange,
addImageFiles, uploadAndAttachFiles, handleSend,
} = p;
const dispatch = useAppDispatch();
const [picker, setPicker] = useState<TriggerState>(EMPTY_TRIGGER);
const [isDragOver, setIsDragOver] = useState(false);
// Set by handlePaste before the synthetic input fires so handleInput skips post-input scans paste can't invalidate.
const justPastedRef = useRef(false);
const updateHasContent = useCallback(() => {
const editor = editorRef.current;
if (!editor) return;
const text = (editor.textContent || '').replace(/\u200B/g, '');
const hasPills = editor.querySelector(`[${SKILL_PILL_ATTR}]`) !== null;
setHasContent(text.trim().length > 0 || hasPills);
}, []);
const syncAttachedSkills = useCallback(() => {
const editor = editorRef.current;
if (!editor) return;
const pillIds = new Set(
Array.from(editor.querySelectorAll(`[${SKILL_PILL_ATTR}]`))
.map((el) => el.getAttribute(SKILL_PILL_ATTR))
.filter(Boolean) as string[],
);
setAttachedSkills((prev) => {
const prevKeys = Object.keys(prev);
if (prevKeys.length === pillIds.size && prevKeys.every((k) => pillIds.has(k))) return prev;
const next: Record<string, AttachedSkill> = {};
for (const [id, skill] of Object.entries(prev)) {
if (pillIds.has(id)) next[id] = skill;
}
return next;
});
}, []);
const removeSkillPill = useCallback((skillId: string) => {
const editor = editorRef.current;
if (!editor) return;
const pill = editor.querySelector(`[${SKILL_PILL_ATTR}="${skillId}"]`);
if (pill) pill.remove();
setAttachedSkills((prev) => {
const { [skillId]: _, ...rest } = prev;
return rest;
});
const text = (editor.textContent || '').replace(/\u200B/g, '');
const hasPills = editor.querySelector(`[${SKILL_PILL_ATTR}]`) !== null;
setHasContent(text.trim().length > 0 || hasPills);
editor.focus();
}, []);
const detectTrigger = useCallback(() => {
const result = detectEditorTrigger();
if (result) {
setPicker(result);
} else {
// Bailout when already hidden; otherwise spreading a new object re-renders all of ChatInput on every keystroke (~199ms input delay).
setPicker((prev) => prev.visible ? { ...prev, visible: false } : prev);
}
}, []);
const handleInput = useCallback(() => {
if (justPastedRef.current) {
justPastedRef.current = false;
setHasContent(true);
scheduleDraftSave(ownerId, () => editorRef.current?.innerHTML ?? '');
return;
}
updateHasContent();
detectTrigger();
syncAttachedSkills();
scheduleDraftSave(ownerId, () => editorRef.current?.innerHTML ?? '');
}, [updateHasContent, detectTrigger, syncAttachedSkills, ownerId]);
const handleEditorClick = useCallback(() => {
detectTrigger();
}, [detectTrigger]);
const handlePickerSelect = (item: CommandPickerItem) => {
setPicker((prev) => ({ ...prev, visible: false }));
const editor = editorRef.current;
if (!editor) return;
editor.focus();
const { triggerNode, triggerOffset, filter } = picker;
if (triggerNode && triggerNode.parentNode && editor.contains(triggerNode)) {
const endOffset = Math.min(triggerOffset + 1 + filter.length, triggerNode.length);
const range = document.createRange();
range.setStart(triggerNode, triggerOffset);
range.setEnd(triggerNode, endOffset);
range.deleteContents();
const sel = window.getSelection();
if (sel) { sel.removeAllRanges(); sel.addRange(range); }
}
if (item.type === 'skill') {
const skill = skills[item.id];
if (!skill) return;
if (editor.querySelector(`[${SKILL_PILL_ATTR}="${skill.id}"]`)) return;
const pill = createSkillPillElement(
{ id: skill.id, name: skill.name, content: skill.content },
removeSkillPill,
c.font.mono,
c.status.error,
);
const sel = window.getSelection();
if (sel && sel.rangeCount > 0) {
const range = sel.getRangeAt(0);
range.collapse(false);
range.insertNode(pill);
const spacer = document.createTextNode('\u200B');
pill.after(spacer);
const newRange = document.createRange();
newRange.setStartAfter(spacer);
newRange.collapse(true);
sel.removeAllRanges();
sel.addRange(newRange);
}
setAttachedSkills((prev) => ({
...prev,
[skill.id]: { id: skill.id, name: skill.name, content: skill.content },
}));
} else if (item.type === 'mode') {
onModeChange(item.id);
} else if (item.type === 'context') {
if (item.command === 'file') {
generalFileInputRef.current?.click();
} else if (item.toolNames && item.toolNames.length > 0) {
setForcedTools((prev) => [...prev, { label: item.name, tools: item.toolNames!, icon: item.icon, iconKey: item.iconKey }]);
} else {
document.execCommand('insertText', false, `@${item.command} `);
}
}
updateHasContent();
setTimeout(() => editor.focus(), 0);
};
const handleKeyDown = (e: React.KeyboardEvent) => {
if (picker.visible && ['ArrowDown', 'ArrowUp', 'Escape', 'Tab', 'Enter'].includes(e.key)) {
e.preventDefault();
return;
}
if ((e.ctrlKey || e.metaKey) && ['b', 'i', 'u'].includes(e.key.toLowerCase())) {
e.preventDefault();
return;
}
if ((e.ctrlKey || e.metaKey) && e.key.toLowerCase() === 'l' && !e.shiftKey && !e.altKey) {
e.preventDefault();
if (sessionId) {
handleSlashCommand('/clear', sessionId).catch(() => {});
dispatch(clearSessionMessages(sessionId));
}
const editor = editorRef.current;
if (editor) {
editor.innerHTML = '';
updateHasContent();
}
return;
}
if (e.key === 'Enter' && !e.shiftKey && !autoRunMode) {
e.preventDefault();
handleSend();
}
};
const handlePaste = useCallback((e: React.ClipboardEvent) => {
if (tryPasteClipboardCards(elementSelection, ownerId)) {
e.preventDefault();
return;
}
const items = e.clipboardData?.items;
if (!items) return;
const imageFiles: File[] = [];
for (let i = 0; i < items.length; i++) {
if (items[i].type.startsWith('image/')) {
const file = items[i].getAsFile();
if (file) imageFiles.push(file);
}
}
if (imageFiles.length > 0) {
e.preventDefault();
addImageFiles(imageFiles);
return;
}
e.preventDefault();
const plain = e.clipboardData.getData('text/plain');
if (plain) {
justPastedRef.current = true;
document.execCommand('insertText', false, plain);
}
}, [addImageFiles, elementSelection, ownerId]);
const handleDragOver = useCallback((e: React.DragEvent) => {
e.preventDefault();
e.stopPropagation();
if (e.dataTransfer.types.includes('Files')) {
setIsDragOver(true);
}
}, []);
const handleDragLeave = useCallback((e: React.DragEvent) => {
e.preventDefault();
e.stopPropagation();
setIsDragOver(false);
}, []);
const handleDrop = useCallback((e: React.DragEvent) => {
e.preventDefault();
e.stopPropagation();
setIsDragOver(false);
if (e.dataTransfer.files.length === 0) return;
const allFiles = Array.from(e.dataTransfer.files);
const imageFiles = allFiles.filter((f) => f.type.startsWith('image/'));
const otherFiles = allFiles.filter((f) => !f.type.startsWith('image/'));
if (imageFiles.length > 0) addImageFiles(imageFiles);
if (otherFiles.length > 0) uploadAndAttachFiles(otherFiles);
}, [addImageFiles, uploadAndAttachFiles]);
return {
picker, setPicker,
isDragOver,
updateHasContent,
handleInput, handleEditorClick, handlePickerSelect, handleKeyDown, handlePaste,
handleDragOver, handleDragLeave, handleDrop,
};
}
@@ -0,0 +1,41 @@
import { useState, useRef, useCallback, useEffect } from 'react';
import { AttachedImage } from './types';
export function useImageAttachments() {
const [images, setImages] = useState<AttachedImage[]>([]);
// Ref so unmount cleanup revokes the latest blob: preview URLs.
const imagesRef = useRef(images);
imagesRef.current = images;
useEffect(() => () => {
for (const img of imagesRef.current) {
if (img.preview?.startsWith('blob:')) {
try { URL.revokeObjectURL(img.preview); } catch {}
}
}
}, []);
const [lightboxSrc, setLightboxSrc] = useState<string | null>(null);
const addImageFiles = useCallback((files: FileList | File[]) => {
// Preview via blob: URL; base64 only materializes at send (saves ~2.7MB JS heap per attachment).
Array.from(files).forEach((file) => {
if (!file.type.startsWith('image/')) return;
const previewUrl = URL.createObjectURL(file);
setImages((prev) => [
...prev,
{ data: '', media_type: file.type, preview: previewUrl, _file: file } as AttachedImage,
]);
});
}, []);
const removeImage = useCallback((idx: number) => {
setImages((prev) => {
const removed = prev[idx];
if (removed?.preview?.startsWith('blob:')) {
try { URL.revokeObjectURL(removed.preview); } catch {}
}
return prev.filter((_, i) => i !== idx);
});
}, []);
return { images, setImages, addImageFiles, removeImage, lightboxSrc, setLightboxSrc };
}