From 79b9d3b4c29fa2c6c7c91acd883f9b8aa57097c5 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Fri, 24 Jul 2026 13:14:38 -0700 Subject: [PATCH] [eric] chat: live browser + app embeds inside the transcript, one click pops them onto the canvas --- .../src/app/pages/AgentChat/AgentChat.tsx | 4 + .../AgentChat/shell/InlineSurfaceEmbeds.tsx | 172 ++++++++++++++++++ .../src/shared/state/dashboardLayoutSlice.ts | 11 ++ 3 files changed, 187 insertions(+) create mode 100644 frontend/src/app/pages/AgentChat/shell/InlineSurfaceEmbeds.tsx diff --git a/frontend/src/app/pages/AgentChat/AgentChat.tsx b/frontend/src/app/pages/AgentChat/AgentChat.tsx index 59f3e593..2fff1fca 100644 --- a/frontend/src/app/pages/AgentChat/AgentChat.tsx +++ b/frontend/src/app/pages/AgentChat/AgentChat.tsx @@ -50,6 +50,7 @@ import { fetchModes } from '@/shared/state/modesSlice'; import { createSessionWs, acquireSessionWs, releaseSessionWs, seedSessionSeq } from '@/shared/ws/WebSocketManager'; import StreamingBubble from './bubbles/StreamingBubble'; import WelcomeQuickReplies from './WelcomeQuickReplies'; +import InlineSurfaceEmbeds from './shell/InlineSurfaceEmbeds'; import { useWelcomeGreeting } from './useWelcomeGreeting'; import { THINKING_LABELS } from './thinkingLabels'; import MessageBubble from './bubbles/MessageBubble'; @@ -1916,6 +1917,9 @@ const AgentChat: React.FC = ({ sessionId: sessionIdProp, onClose )} + {/* The surfaces this agent is driving, live inside the chat: browser snapshots + built + apps, each one click from popping out onto the canvas. */} + {!isDraft && id && } {/* First-run welcome chips: sit UNDER the streamed greeting, appear once it finishes, vanish the moment the user answers. The greeting itself is a real assistant bubble. */} {session.is_welcome_draft && isDraft && welcomeGreetingDone && !session.messages.some((m) => m.role === 'user') && ( diff --git a/frontend/src/app/pages/AgentChat/shell/InlineSurfaceEmbeds.tsx b/frontend/src/app/pages/AgentChat/shell/InlineSurfaceEmbeds.tsx new file mode 100644 index 00000000..dcc307ac --- /dev/null +++ b/frontend/src/app/pages/AgentChat/shell/InlineSurfaceEmbeds.tsx @@ -0,0 +1,172 @@ +import React, { useEffect, useMemo, useState } from 'react'; +import Box from '@mui/material/Box'; +import Typography from '@mui/material/Typography'; +import { motion } from 'framer-motion'; +import LanguageIcon from '@mui/icons-material/Language'; +import GridViewRoundedIcon from '@mui/icons-material/GridViewRounded'; +import OpenInFullRoundedIcon from '@mui/icons-material/OpenInFullRounded'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; +import { clearTiledCard, focusBrowserCard, focusViewCard } from '@/shared/state/dashboardLayoutSlice'; +import { getWebview } from '@/shared/browserRegistry'; +import type { ClaudeTokens } from '@/shared/styles/claudeTokens'; + +// The desktop-redesign frame: the surfaces an agent is driving live INSIDE its chat. Each linked +// browser card renders as a live snapshot embed (same visual grammar as the tool cards: rounded, +// bordered, quiet chrome header) that repaints while the agent works; each built app renders as an +// open row. Clicking pops the real card out on the canvas (exiting full size view first if needed), +// which IS the "drag it out" gesture without inventing a second windowing system inside the chat. +const SNAPSHOT_MS = 2500; + +function useBrowserSnapshot(browserId: string, live: boolean): string | null { + const [shot, setShot] = useState(null); + useEffect(() => { + let dead = false; + let timer = 0; + const grab = async (): Promise => { + try { + const wv = getWebview(browserId) as unknown as { capturePage?: () => Promise<{ toDataURL(): string }> } | undefined; + if (wv?.capturePage) { + // capturePage can hang on off-screen guests (Electron 42); race a timeout so the poll never wedges. + const img = await Promise.race([ + wv.capturePage(), + new Promise((res) => window.setTimeout(() => res(null), 1200)), + ]); + if (!dead && img) setShot(img.toDataURL()); + } + } catch { /* snapshot is best-effort */ } + // Live sessions repaint forever; idle ones retry a few times so a webview that mounts late + // (fullscreen hiding the canvas, load races) still yields one real frame instead of a blank. + tries += 1; + if (!dead && (live || (tries < 6))) timer = window.setTimeout(() => { void grab(); }, SNAPSHOT_MS); + }; + let tries = 0; + void grab(); + return () => { dead = true; window.clearTimeout(timer); }; + }, [browserId, live]); + return shot; +} + +const BrowserEmbed: React.FC<{ c: ClaudeTokens; browserId: string; title: string; url: string; live: boolean; onOpen: () => void }> = ({ c, browserId, title, url, live, onOpen }) => { + const shot = useBrowserSnapshot(browserId, live); + return ( + + + + + + {title || 'Browser'} + + + {url} + + {live && ( + + )} + + + Open on canvas + + + {shot ? ( + + ) : ( + + {live ? 'Loading page view...' : 'Page view unavailable'} + + )} + + + ); +}; + +const InlineSurfaceEmbeds: React.FC<{ c: ClaudeTokens; sessionId: string; fullscreen?: boolean }> = ({ c, sessionId, fullscreen }) => { + const dispatch = useAppDispatch(); + const browserCards = useAppSelector((s) => s.dashboardLayout.browserCards); + const outputs = useAppSelector((s) => s.outputs.items); + const sessions = useAppSelector((s) => s.agents.sessions); + const sessionStatus = sessions[sessionId]?.status; + const live = sessionStatus === 'running' || sessionStatus === 'waiting_approval'; + + const linkedBrowsers = useMemo(() => { + // A chat's browsers arrive two ways: cards this session spawned directly (spawned_by), and cards + // driven by its browser sub-agents (child session's browser_id). Union both, keyed by card id. + const childBrowserIds = new Set( + Object.values(sessions) + .filter((s) => s.parent_session_id === sessionId && s.browser_id) + .map((s) => s.browser_id as string), + ); + return Object.values(browserCards).filter( + (bc) => bc.spawned_by === sessionId || childBrowserIds.has(bc.browser_id), + ); + }, [browserCards, sessions, sessionId]); + const linkedApps = useMemo( + () => Object.values(outputs).filter((o) => o.session_id === sessionId), + [outputs, sessionId], + ); + + if (linkedBrowsers.length === 0 && linkedApps.length === 0) return null; + + const popOut = (fire: () => void): void => { + // Full size view hides the canvas; step out first so "open on canvas" lands somewhere visible. + if (fullscreen) dispatch(clearTiledCard(sessionId)); + fire(); + }; + + return ( + + {linkedBrowsers.map((bc) => { + const tab = bc.tabs.find((t) => t.id === bc.activeTabId) ?? bc.tabs[0]; + return ( + popOut(() => dispatch(focusBrowserCard(bc.browser_id)))} + /> + ); + })} + {linkedApps.map((o) => ( + popOut(() => dispatch(focusViewCard(o.id)))} + sx={{ + display: 'flex', alignItems: 'center', gap: 1.25, px: 1.25, py: 1, + border: `1px solid ${c.border.subtle}`, borderRadius: 2, cursor: 'pointer', + bgcolor: c.bg.elevated, transition: 'border-color 150ms', + '&:hover': { borderColor: c.border.strong }, + }} + > + + + + {o.name || 'App'} + + Built in this chat + + + + Open + + + ))} + + ); +}; + +export default InlineSurfaceEmbeds; diff --git a/frontend/src/shared/state/dashboardLayoutSlice.ts b/frontend/src/shared/state/dashboardLayoutSlice.ts index fc0b992c..713c4dfc 100644 --- a/frontend/src/shared/state/dashboardLayoutSlice.ts +++ b/frontend/src/shared/state/dashboardLayoutSlice.ts @@ -918,6 +918,15 @@ const dashboardLayoutSlice = createSlice({ state.pendingFocusBrowserId = null; }, + // Camera-focus an EXISTING card (the inline chat embeds' "show on canvas"); the lifecycle + // pendingFocus effects own the fit + highlight + clear, same as a fresh add. + focusBrowserCard(state, action: PayloadAction) { + if (state.browserCards[action.payload]) state.pendingFocusBrowserId = action.payload; + }, + focusViewCard(state, action: PayloadAction) { + if (state.viewCards[action.payload]) state.pendingFocusViewCardId = action.payload; + }, + addBrowserCardFromBackend(state, action: PayloadAction) { const card = action.payload; if (state.browserCards[card.browser_id]) return; @@ -1833,6 +1842,8 @@ export const { clearAllTiles, clearCardWindowState, clearPendingFocusBrowserId, + focusBrowserCard, + focusViewCard, activateViewCardPreview, clearPendingFocusViewCardId, addWorkflowCard,