[eric] agents: collapsed pill pins the session's artifact (latest ShowUI widget, else live spawned-browser shot) under the narrator pill

This commit is contained in:
ciregenz
2026-07-19 22:02:08 -07:00
parent 3ff3b0bcc6
commit 27ff21176c
5 changed files with 86 additions and 12 deletions
@@ -0,0 +1,17 @@
import React from 'react';
import WeatherWidget from './WeatherWidget';
import PlanWidget from './PlanWidget';
import StatsWidget from './StatsWidget';
import LinksWidget from './LinksWidget';
import type { ShowUiPayload } from './showUiPayload';
/** One switch for every surface that renders a ShowUI payload (chat bubble, pill artifact). */
function ShowUiWidgetView({ payload }: { payload: ShowUiPayload }): React.ReactElement | null {
if (payload.component === 'weather') return <WeatherWidget props={payload.props} />;
if (payload.component === 'plan') return <PlanWidget props={payload.props} />;
if (payload.component === 'stats') return <StatsWidget props={payload.props} />;
if (payload.component === 'links') return <LinksWidget props={payload.props} />;
return null;
}
export default ShowUiWidgetView;
@@ -3,10 +3,7 @@ import Box from '@mui/material/Box';
import ToolCallBubble from '../tool-bubbles/ToolCallBubble';
import type { ToolPair } from '../tool-bubbles/ToolCallBubble';
import { parseShowUiPayload } from './showUiPayload';
import WeatherWidget from './WeatherWidget';
import PlanWidget from './PlanWidget';
import StatsWidget from './StatsWidget';
import LinksWidget from './LinksWidget';
import ShowUiWidgetView from './ShowUiWidgetView';
interface ToolUiBubbleProps {
pair: ToolPair;
@@ -25,10 +22,7 @@ function ToolUiBubble({ pair, sessionId, isPending, suppressReveal }: ToolUiBubb
}
return (
<Box sx={{ my: 1, contain: 'layout style' }} data-select-type="tool-ui" data-select-id={pair.id} data-select-meta={JSON.stringify({ component: payload.component })}>
{payload.component === 'weather' && <WeatherWidget props={payload.props} />}
{payload.component === 'plan' && <PlanWidget props={payload.props} />}
{payload.component === 'stats' && <StatsWidget props={payload.props} />}
{payload.component === 'links' && <LinksWidget props={payload.props} />}
<ShowUiWidgetView payload={payload} />
</Box>
);
}
@@ -68,10 +68,26 @@ export function isShowUiPair(pair: ToolPair): boolean {
return /(^|__)ShowUI$/.test(tool);
}
/** Latest ShowUI payload anywhere in a transcript; the collapsed card pins this artifact under its pill. */
export function extractLatestShowUi(messages: Array<{ role: string; content: any }>): ShowUiPayload | null {
for (let i = messages.length - 1; i >= 0; i--) {
const msg = messages[i];
if (msg.role !== 'tool_call') continue;
const tool = typeof msg.content === 'object' ? String(msg.content?.tool || '') : '';
if (!/(^|__)ShowUI$/.test(tool)) continue;
const parsed = parseShowUiInput(msg.content?.input);
if (parsed) return parsed;
}
return null;
}
/** Strict parse of a ShowUI tool_call's input; null on any mismatch so the caller falls back to the plain bubble. */
export function parseShowUiPayload(pair: ToolPair): ShowUiPayload | null {
const content = typeof pair.call.content === 'object' ? pair.call.content : null;
const input = content?.input;
return parseShowUiInput(content?.input);
}
function parseShowUiInput(input: unknown): ShowUiPayload | null {
if (!input || typeof input !== 'object') return null;
const component = String((input as { component?: unknown }).component || '');
const props = (input as { props?: unknown }).props;
@@ -37,6 +37,8 @@ import WindowControls from './WindowControls';
import { useTiledStyle } from './tileZones';
import AgentNarratorPill from '../desktop/AgentNarratorPill';
import { extractLatestTodos } from '../desktop/agentTodos';
import { extractLatestShowUi } from '@/app/pages/AgentChat/tool-ui/showUiPayload';
import { getWebview } from '@/shared/browserRegistry';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import { QuestionForm } from '@/app/pages/AgentChat/shell/ApprovalBar';
import AgentChat from '@/app/pages/AgentChat/AgentChat';
@@ -691,10 +693,40 @@ const AgentCard: React.FC<Props> = ({
// Desktop-shell narrator pill: a collapsed card with nothing to ask renders as the minimal pill
// (live turn label + plan checklist); approvals and drafts keep the full card so their UI has a home.
const todos = useMemo(() => extractLatestTodos(session.messages || []), [session.messages]);
const pillArtifact = useMemo(() => extractLatestShowUi(session.messages || []), [session.messages]);
const pillMode = !expanded && !hasPending && !isDraft && !tileZone;
const pillLabel = session.turn_label?.label || displayChatTitle(session);
const pillRunning = session.status === 'running';
// f7's collapsed state: a session that spawned a browser shows that window under the pill.
const spawnedBrowserId = useAppSelector((s) => {
for (const bc of Object.values(s.dashboardLayout.browserCards)) {
if (bc.spawned_by === session.id) return bc.browser_id;
}
return null;
});
const [browserShot, setBrowserShot] = useState<string | null>(null);
useEffect(() => {
if (!pillMode || pillArtifact || !spawnedBrowserId) {
setBrowserShot(null);
return undefined;
}
let cancelled = false;
const capture = (): void => {
const wv = getWebview(spawnedBrowserId);
const p = wv?.capturePage?.();
if (p && typeof (p as Promise<unknown>).then === 'function') {
(p as Promise<{ toDataURL(): string }>)
.then((img) => { if (!cancelled) setBrowserShot(img.toDataURL()); })
.catch(() => undefined);
}
};
capture();
// Refresh while the agent is driving so the shot tracks the page; parked cards keep the last frame.
const timer = pillRunning ? window.setInterval(capture, 5000) : null;
return () => { cancelled = true; if (timer) window.clearInterval(timer); };
}, [pillMode, pillArtifact, spawnedBrowserId, pillRunning]);
const noTransition = isDragging || isResizing || (isSelected && !!multiDragDelta);
const mdDx = (!isDragging && isSelected && multiDragDelta) ? multiDragDelta.dx : 0;
@@ -926,6 +958,8 @@ const AgentCard: React.FC<Props> = ({
label={pillLabel}
running={pillRunning}
todos={todos}
artifact={pillArtifact}
browserShot={browserShot}
selected={isSelected}
highlighted={isHighlighted}
/>
@@ -3,12 +3,16 @@ import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import CheckIcon from '@mui/icons-material/Check';
import DashboardGlyph from '../canvas/DashboardGlyph';
import ShowUiWidgetView from '@/app/pages/AgentChat/tool-ui/ShowUiWidgetView';
import type { ShowUiPayload } from '@/app/pages/AgentChat/tool-ui/showUiPayload';
import type { AgentTodoItem } from './agentTodos';
interface AgentNarratorPillProps {
label: string;
running: boolean;
todos: AgentTodoItem[] | null;
artifact: ShowUiPayload | null;
browserShot: string | null;
selected: boolean;
highlighted: boolean;
}
@@ -17,8 +21,8 @@ const GLASS = 'rgba(24,14,32,0.8)';
const GLASS_BLUR = 'blur(18px) saturate(150%)';
const MAX_VISIBLE_TODOS = 4;
/** Collapsed running agent as the desktop narrator pill, with its live plan hanging below. */
function AgentNarratorPill({ label, running, todos, selected, highlighted }: AgentNarratorPillProps): React.ReactElement {
/** Collapsed agent as the desktop narrator pill; below it, the best artifact wins: widget > browser shot > plan > Thinking. */
function AgentNarratorPill({ label, running, todos, artifact, browserShot, selected, highlighted }: AgentNarratorPillProps): React.ReactElement {
const visibleTodos = (todos || []).slice(0, MAX_VISIBLE_TODOS);
const hiddenCount = (todos?.length || 0) - visibleTodos.length;
const ring = selected || highlighted ? { outline: '2px solid #3b82f6', outlineOffset: '2px' } : undefined;
@@ -48,7 +52,16 @@ function AgentNarratorPill({ label, running, todos, selected, highlighted }: Age
</Typography>
</Box>
{visibleTodos.length > 0 ? (
{artifact ? (
<ShowUiWidgetView payload={artifact} />
) : browserShot ? (
<Box
component="img"
src={browserShot}
alt=""
sx={{ width: 300, display: 'block', borderRadius: '10px', boxShadow: '0 10px 30px rgba(0,0,0,0.35)' }}
/>
) : visibleTodos.length > 0 ? (
<Box
sx={{
borderRadius: '16px',