mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-08 18:57:43 +02:00
[eric] agents: collapsed pill pins the session's artifact (latest ShowUI widget, else live spawned-browser shot) under the narrator pill
This commit is contained in:
@@ -0,0 +1,17 @@
|
||||
import React from 'react';
|
||||
import WeatherWidget from './WeatherWidget';
|
||||
import PlanWidget from './PlanWidget';
|
||||
import StatsWidget from './StatsWidget';
|
||||
import LinksWidget from './LinksWidget';
|
||||
import type { ShowUiPayload } from './showUiPayload';
|
||||
|
||||
/** One switch for every surface that renders a ShowUI payload (chat bubble, pill artifact). */
|
||||
function ShowUiWidgetView({ payload }: { payload: ShowUiPayload }): React.ReactElement | null {
|
||||
if (payload.component === 'weather') return <WeatherWidget props={payload.props} />;
|
||||
if (payload.component === 'plan') return <PlanWidget props={payload.props} />;
|
||||
if (payload.component === 'stats') return <StatsWidget props={payload.props} />;
|
||||
if (payload.component === 'links') return <LinksWidget props={payload.props} />;
|
||||
return null;
|
||||
}
|
||||
|
||||
export default ShowUiWidgetView;
|
||||
@@ -3,10 +3,7 @@ import Box from '@mui/material/Box';
|
||||
import ToolCallBubble from '../tool-bubbles/ToolCallBubble';
|
||||
import type { ToolPair } from '../tool-bubbles/ToolCallBubble';
|
||||
import { parseShowUiPayload } from './showUiPayload';
|
||||
import WeatherWidget from './WeatherWidget';
|
||||
import PlanWidget from './PlanWidget';
|
||||
import StatsWidget from './StatsWidget';
|
||||
import LinksWidget from './LinksWidget';
|
||||
import ShowUiWidgetView from './ShowUiWidgetView';
|
||||
|
||||
interface ToolUiBubbleProps {
|
||||
pair: ToolPair;
|
||||
@@ -25,10 +22,7 @@ function ToolUiBubble({ pair, sessionId, isPending, suppressReveal }: ToolUiBubb
|
||||
}
|
||||
return (
|
||||
<Box sx={{ my: 1, contain: 'layout style' }} data-select-type="tool-ui" data-select-id={pair.id} data-select-meta={JSON.stringify({ component: payload.component })}>
|
||||
{payload.component === 'weather' && <WeatherWidget props={payload.props} />}
|
||||
{payload.component === 'plan' && <PlanWidget props={payload.props} />}
|
||||
{payload.component === 'stats' && <StatsWidget props={payload.props} />}
|
||||
{payload.component === 'links' && <LinksWidget props={payload.props} />}
|
||||
<ShowUiWidgetView payload={payload} />
|
||||
</Box>
|
||||
);
|
||||
}
|
||||
|
||||
@@ -68,10 +68,26 @@ export function isShowUiPair(pair: ToolPair): boolean {
|
||||
return /(^|__)ShowUI$/.test(tool);
|
||||
}
|
||||
|
||||
/** Latest ShowUI payload anywhere in a transcript; the collapsed card pins this artifact under its pill. */
|
||||
export function extractLatestShowUi(messages: Array<{ role: string; content: any }>): ShowUiPayload | null {
|
||||
for (let i = messages.length - 1; i >= 0; i--) {
|
||||
const msg = messages[i];
|
||||
if (msg.role !== 'tool_call') continue;
|
||||
const tool = typeof msg.content === 'object' ? String(msg.content?.tool || '') : '';
|
||||
if (!/(^|__)ShowUI$/.test(tool)) continue;
|
||||
const parsed = parseShowUiInput(msg.content?.input);
|
||||
if (parsed) return parsed;
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Strict parse of a ShowUI tool_call's input; null on any mismatch so the caller falls back to the plain bubble. */
|
||||
export function parseShowUiPayload(pair: ToolPair): ShowUiPayload | null {
|
||||
const content = typeof pair.call.content === 'object' ? pair.call.content : null;
|
||||
const input = content?.input;
|
||||
return parseShowUiInput(content?.input);
|
||||
}
|
||||
|
||||
function parseShowUiInput(input: unknown): ShowUiPayload | null {
|
||||
if (!input || typeof input !== 'object') return null;
|
||||
const component = String((input as { component?: unknown }).component || '');
|
||||
const props = (input as { props?: unknown }).props;
|
||||
|
||||
@@ -37,6 +37,8 @@ import WindowControls from './WindowControls';
|
||||
import { useTiledStyle } from './tileZones';
|
||||
import AgentNarratorPill from '../desktop/AgentNarratorPill';
|
||||
import { extractLatestTodos } from '../desktop/agentTodos';
|
||||
import { extractLatestShowUi } from '@/app/pages/AgentChat/tool-ui/showUiPayload';
|
||||
import { getWebview } from '@/shared/browserRegistry';
|
||||
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
|
||||
import { QuestionForm } from '@/app/pages/AgentChat/shell/ApprovalBar';
|
||||
import AgentChat from '@/app/pages/AgentChat/AgentChat';
|
||||
@@ -691,10 +693,40 @@ const AgentCard: React.FC<Props> = ({
|
||||
// Desktop-shell narrator pill: a collapsed card with nothing to ask renders as the minimal pill
|
||||
// (live turn label + plan checklist); approvals and drafts keep the full card so their UI has a home.
|
||||
const todos = useMemo(() => extractLatestTodos(session.messages || []), [session.messages]);
|
||||
const pillArtifact = useMemo(() => extractLatestShowUi(session.messages || []), [session.messages]);
|
||||
const pillMode = !expanded && !hasPending && !isDraft && !tileZone;
|
||||
const pillLabel = session.turn_label?.label || displayChatTitle(session);
|
||||
const pillRunning = session.status === 'running';
|
||||
|
||||
// f7's collapsed state: a session that spawned a browser shows that window under the pill.
|
||||
const spawnedBrowserId = useAppSelector((s) => {
|
||||
for (const bc of Object.values(s.dashboardLayout.browserCards)) {
|
||||
if (bc.spawned_by === session.id) return bc.browser_id;
|
||||
}
|
||||
return null;
|
||||
});
|
||||
const [browserShot, setBrowserShot] = useState<string | null>(null);
|
||||
useEffect(() => {
|
||||
if (!pillMode || pillArtifact || !spawnedBrowserId) {
|
||||
setBrowserShot(null);
|
||||
return undefined;
|
||||
}
|
||||
let cancelled = false;
|
||||
const capture = (): void => {
|
||||
const wv = getWebview(spawnedBrowserId);
|
||||
const p = wv?.capturePage?.();
|
||||
if (p && typeof (p as Promise<unknown>).then === 'function') {
|
||||
(p as Promise<{ toDataURL(): string }>)
|
||||
.then((img) => { if (!cancelled) setBrowserShot(img.toDataURL()); })
|
||||
.catch(() => undefined);
|
||||
}
|
||||
};
|
||||
capture();
|
||||
// Refresh while the agent is driving so the shot tracks the page; parked cards keep the last frame.
|
||||
const timer = pillRunning ? window.setInterval(capture, 5000) : null;
|
||||
return () => { cancelled = true; if (timer) window.clearInterval(timer); };
|
||||
}, [pillMode, pillArtifact, spawnedBrowserId, pillRunning]);
|
||||
|
||||
const noTransition = isDragging || isResizing || (isSelected && !!multiDragDelta);
|
||||
|
||||
const mdDx = (!isDragging && isSelected && multiDragDelta) ? multiDragDelta.dx : 0;
|
||||
@@ -926,6 +958,8 @@ const AgentCard: React.FC<Props> = ({
|
||||
label={pillLabel}
|
||||
running={pillRunning}
|
||||
todos={todos}
|
||||
artifact={pillArtifact}
|
||||
browserShot={browserShot}
|
||||
selected={isSelected}
|
||||
highlighted={isHighlighted}
|
||||
/>
|
||||
|
||||
@@ -3,12 +3,16 @@ import Box from '@mui/material/Box';
|
||||
import Typography from '@mui/material/Typography';
|
||||
import CheckIcon from '@mui/icons-material/Check';
|
||||
import DashboardGlyph from '../canvas/DashboardGlyph';
|
||||
import ShowUiWidgetView from '@/app/pages/AgentChat/tool-ui/ShowUiWidgetView';
|
||||
import type { ShowUiPayload } from '@/app/pages/AgentChat/tool-ui/showUiPayload';
|
||||
import type { AgentTodoItem } from './agentTodos';
|
||||
|
||||
interface AgentNarratorPillProps {
|
||||
label: string;
|
||||
running: boolean;
|
||||
todos: AgentTodoItem[] | null;
|
||||
artifact: ShowUiPayload | null;
|
||||
browserShot: string | null;
|
||||
selected: boolean;
|
||||
highlighted: boolean;
|
||||
}
|
||||
@@ -17,8 +21,8 @@ const GLASS = 'rgba(24,14,32,0.8)';
|
||||
const GLASS_BLUR = 'blur(18px) saturate(150%)';
|
||||
const MAX_VISIBLE_TODOS = 4;
|
||||
|
||||
/** Collapsed running agent as the desktop narrator pill, with its live plan hanging below. */
|
||||
function AgentNarratorPill({ label, running, todos, selected, highlighted }: AgentNarratorPillProps): React.ReactElement {
|
||||
/** Collapsed agent as the desktop narrator pill; below it, the best artifact wins: widget > browser shot > plan > Thinking. */
|
||||
function AgentNarratorPill({ label, running, todos, artifact, browserShot, selected, highlighted }: AgentNarratorPillProps): React.ReactElement {
|
||||
const visibleTodos = (todos || []).slice(0, MAX_VISIBLE_TODOS);
|
||||
const hiddenCount = (todos?.length || 0) - visibleTodos.length;
|
||||
const ring = selected || highlighted ? { outline: '2px solid #3b82f6', outlineOffset: '2px' } : undefined;
|
||||
@@ -48,7 +52,16 @@ function AgentNarratorPill({ label, running, todos, selected, highlighted }: Age
|
||||
</Typography>
|
||||
</Box>
|
||||
|
||||
{visibleTodos.length > 0 ? (
|
||||
{artifact ? (
|
||||
<ShowUiWidgetView payload={artifact} />
|
||||
) : browserShot ? (
|
||||
<Box
|
||||
component="img"
|
||||
src={browserShot}
|
||||
alt=""
|
||||
sx={{ width: 300, display: 'block', borderRadius: '10px', boxShadow: '0 10px 30px rgba(0,0,0,0.35)' }}
|
||||
/>
|
||||
) : visibleTodos.length > 0 ? (
|
||||
<Box
|
||||
sx={{
|
||||
borderRadius: '16px',
|
||||
|
||||
Reference in New Issue
Block a user