From 27ff21176c04427b30c0cd881b7a03bde06aecd9 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 19 Jul 2026 22:02:08 -0700 Subject: [PATCH] [eric] agents: collapsed pill pins the session's artifact (latest ShowUI widget, else live spawned-browser shot) under the narrator pill --- .../AgentChat/tool-ui/ShowUiWidgetView.tsx | 17 ++++++++++ .../pages/AgentChat/tool-ui/ToolUiBubble.tsx | 10 ++---- .../pages/AgentChat/tool-ui/showUiPayload.ts | 18 +++++++++- .../app/pages/Dashboard/cards/AgentCard.tsx | 34 +++++++++++++++++++ .../Dashboard/desktop/AgentNarratorPill.tsx | 19 +++++++++-- 5 files changed, 86 insertions(+), 12 deletions(-) create mode 100644 frontend/src/app/pages/AgentChat/tool-ui/ShowUiWidgetView.tsx diff --git a/frontend/src/app/pages/AgentChat/tool-ui/ShowUiWidgetView.tsx b/frontend/src/app/pages/AgentChat/tool-ui/ShowUiWidgetView.tsx new file mode 100644 index 00000000..e62f0b02 --- /dev/null +++ b/frontend/src/app/pages/AgentChat/tool-ui/ShowUiWidgetView.tsx @@ -0,0 +1,17 @@ +import React from 'react'; +import WeatherWidget from './WeatherWidget'; +import PlanWidget from './PlanWidget'; +import StatsWidget from './StatsWidget'; +import LinksWidget from './LinksWidget'; +import type { ShowUiPayload } from './showUiPayload'; + +/** One switch for every surface that renders a ShowUI payload (chat bubble, pill artifact). */ +function ShowUiWidgetView({ payload }: { payload: ShowUiPayload }): React.ReactElement | null { + if (payload.component === 'weather') return ; + if (payload.component === 'plan') return ; + if (payload.component === 'stats') return ; + if (payload.component === 'links') return ; + return null; +} + +export default ShowUiWidgetView; diff --git a/frontend/src/app/pages/AgentChat/tool-ui/ToolUiBubble.tsx b/frontend/src/app/pages/AgentChat/tool-ui/ToolUiBubble.tsx index c8567566..292b55aa 100644 --- a/frontend/src/app/pages/AgentChat/tool-ui/ToolUiBubble.tsx +++ b/frontend/src/app/pages/AgentChat/tool-ui/ToolUiBubble.tsx @@ -3,10 +3,7 @@ import Box from '@mui/material/Box'; import ToolCallBubble from '../tool-bubbles/ToolCallBubble'; import type { ToolPair } from '../tool-bubbles/ToolCallBubble'; import { parseShowUiPayload } from './showUiPayload'; -import WeatherWidget from './WeatherWidget'; -import PlanWidget from './PlanWidget'; -import StatsWidget from './StatsWidget'; -import LinksWidget from './LinksWidget'; +import ShowUiWidgetView from './ShowUiWidgetView'; interface ToolUiBubbleProps { pair: ToolPair; @@ -25,10 +22,7 @@ function ToolUiBubble({ pair, sessionId, isPending, suppressReveal }: ToolUiBubb } return ( - {payload.component === 'weather' && } - {payload.component === 'plan' && } - {payload.component === 'stats' && } - {payload.component === 'links' && } + ); } diff --git a/frontend/src/app/pages/AgentChat/tool-ui/showUiPayload.ts b/frontend/src/app/pages/AgentChat/tool-ui/showUiPayload.ts index 4f13fe1a..21f962ee 100644 --- a/frontend/src/app/pages/AgentChat/tool-ui/showUiPayload.ts +++ b/frontend/src/app/pages/AgentChat/tool-ui/showUiPayload.ts @@ -68,10 +68,26 @@ export function isShowUiPair(pair: ToolPair): boolean { return /(^|__)ShowUI$/.test(tool); } +/** Latest ShowUI payload anywhere in a transcript; the collapsed card pins this artifact under its pill. */ +export function extractLatestShowUi(messages: Array<{ role: string; content: any }>): ShowUiPayload | null { + for (let i = messages.length - 1; i >= 0; i--) { + const msg = messages[i]; + if (msg.role !== 'tool_call') continue; + const tool = typeof msg.content === 'object' ? String(msg.content?.tool || '') : ''; + if (!/(^|__)ShowUI$/.test(tool)) continue; + const parsed = parseShowUiInput(msg.content?.input); + if (parsed) return parsed; + } + return null; +} + /** Strict parse of a ShowUI tool_call's input; null on any mismatch so the caller falls back to the plain bubble. */ export function parseShowUiPayload(pair: ToolPair): ShowUiPayload | null { const content = typeof pair.call.content === 'object' ? pair.call.content : null; - const input = content?.input; + return parseShowUiInput(content?.input); +} + +function parseShowUiInput(input: unknown): ShowUiPayload | null { if (!input || typeof input !== 'object') return null; const component = String((input as { component?: unknown }).component || ''); const props = (input as { props?: unknown }).props; diff --git a/frontend/src/app/pages/Dashboard/cards/AgentCard.tsx b/frontend/src/app/pages/Dashboard/cards/AgentCard.tsx index e0a0a1e9..41042f74 100644 --- a/frontend/src/app/pages/Dashboard/cards/AgentCard.tsx +++ b/frontend/src/app/pages/Dashboard/cards/AgentCard.tsx @@ -37,6 +37,8 @@ import WindowControls from './WindowControls'; import { useTiledStyle } from './tileZones'; import AgentNarratorPill from '../desktop/AgentNarratorPill'; import { extractLatestTodos } from '../desktop/agentTodos'; +import { extractLatestShowUi } from '@/app/pages/AgentChat/tool-ui/showUiPayload'; +import { getWebview } from '@/shared/browserRegistry'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { QuestionForm } from '@/app/pages/AgentChat/shell/ApprovalBar'; import AgentChat from '@/app/pages/AgentChat/AgentChat'; @@ -691,10 +693,40 @@ const AgentCard: React.FC = ({ // Desktop-shell narrator pill: a collapsed card with nothing to ask renders as the minimal pill // (live turn label + plan checklist); approvals and drafts keep the full card so their UI has a home. const todos = useMemo(() => extractLatestTodos(session.messages || []), [session.messages]); + const pillArtifact = useMemo(() => extractLatestShowUi(session.messages || []), [session.messages]); const pillMode = !expanded && !hasPending && !isDraft && !tileZone; const pillLabel = session.turn_label?.label || displayChatTitle(session); const pillRunning = session.status === 'running'; + // f7's collapsed state: a session that spawned a browser shows that window under the pill. + const spawnedBrowserId = useAppSelector((s) => { + for (const bc of Object.values(s.dashboardLayout.browserCards)) { + if (bc.spawned_by === session.id) return bc.browser_id; + } + return null; + }); + const [browserShot, setBrowserShot] = useState(null); + useEffect(() => { + if (!pillMode || pillArtifact || !spawnedBrowserId) { + setBrowserShot(null); + return undefined; + } + let cancelled = false; + const capture = (): void => { + const wv = getWebview(spawnedBrowserId); + const p = wv?.capturePage?.(); + if (p && typeof (p as Promise).then === 'function') { + (p as Promise<{ toDataURL(): string }>) + .then((img) => { if (!cancelled) setBrowserShot(img.toDataURL()); }) + .catch(() => undefined); + } + }; + capture(); + // Refresh while the agent is driving so the shot tracks the page; parked cards keep the last frame. + const timer = pillRunning ? window.setInterval(capture, 5000) : null; + return () => { cancelled = true; if (timer) window.clearInterval(timer); }; + }, [pillMode, pillArtifact, spawnedBrowserId, pillRunning]); + const noTransition = isDragging || isResizing || (isSelected && !!multiDragDelta); const mdDx = (!isDragging && isSelected && multiDragDelta) ? multiDragDelta.dx : 0; @@ -926,6 +958,8 @@ const AgentCard: React.FC = ({ label={pillLabel} running={pillRunning} todos={todos} + artifact={pillArtifact} + browserShot={browserShot} selected={isSelected} highlighted={isHighlighted} /> diff --git a/frontend/src/app/pages/Dashboard/desktop/AgentNarratorPill.tsx b/frontend/src/app/pages/Dashboard/desktop/AgentNarratorPill.tsx index 6f1d0d59..bf5d8302 100644 --- a/frontend/src/app/pages/Dashboard/desktop/AgentNarratorPill.tsx +++ b/frontend/src/app/pages/Dashboard/desktop/AgentNarratorPill.tsx @@ -3,12 +3,16 @@ import Box from '@mui/material/Box'; import Typography from '@mui/material/Typography'; import CheckIcon from '@mui/icons-material/Check'; import DashboardGlyph from '../canvas/DashboardGlyph'; +import ShowUiWidgetView from '@/app/pages/AgentChat/tool-ui/ShowUiWidgetView'; +import type { ShowUiPayload } from '@/app/pages/AgentChat/tool-ui/showUiPayload'; import type { AgentTodoItem } from './agentTodos'; interface AgentNarratorPillProps { label: string; running: boolean; todos: AgentTodoItem[] | null; + artifact: ShowUiPayload | null; + browserShot: string | null; selected: boolean; highlighted: boolean; } @@ -17,8 +21,8 @@ const GLASS = 'rgba(24,14,32,0.8)'; const GLASS_BLUR = 'blur(18px) saturate(150%)'; const MAX_VISIBLE_TODOS = 4; -/** Collapsed running agent as the desktop narrator pill, with its live plan hanging below. */ -function AgentNarratorPill({ label, running, todos, selected, highlighted }: AgentNarratorPillProps): React.ReactElement { +/** Collapsed agent as the desktop narrator pill; below it, the best artifact wins: widget > browser shot > plan > Thinking. */ +function AgentNarratorPill({ label, running, todos, artifact, browserShot, selected, highlighted }: AgentNarratorPillProps): React.ReactElement { const visibleTodos = (todos || []).slice(0, MAX_VISIBLE_TODOS); const hiddenCount = (todos?.length || 0) - visibleTodos.length; const ring = selected || highlighted ? { outline: '2px solid #3b82f6', outlineOffset: '2px' } : undefined; @@ -48,7 +52,16 @@ function AgentNarratorPill({ label, running, todos, selected, highlighted }: Age - {visibleTodos.length > 0 ? ( + {artifact ? ( + + ) : browserShot ? ( + + ) : visibleTodos.length > 0 ? (