[aidan] ui/ux: canvas navigation (#85)

* dashboard: pan canvas via middle-drag and horizontal scroll over browser/chat

- Middle-mouse drag inside a browser webview is forwarded as canvas pan
  (previously eaten silently by the guest compositor).
- Horizontal-dominant wheel inside a browser webview pans the canvas
  when no ancestor in the guest page can absorb horizontal scroll;
  otherwise the page handles it.
- Horizontal-dominant wheel over a chat panel pans the canvas (chat has
  no horizontal scroller). Vertical scroll is unchanged.

Plain vertical scroll and ctrl/meta+wheel zoom over a browser stay with
chromium's defaults.

* browser: drop in-guest zoom locks so pinch and cmd+wheel zoom the page

setVisualZoomLevelLimits(1, 1) blocked pinch-to-zoom inside the webview, and
setZoomFactor(1) on every dom-ready reset any user page zoom. Both were added
back when canvas zoom was supposed to take over for ctrl+wheel; with that
reverted, chromium's native zoom handlers should run unimpeded.

* [aidan] fix: prev. commit fix. restore ctrl+wheel canvas zoom over webview cards

* [aidan] ui/ux: app navigation

* [aidan] ui/ux: gate webview preload forwarding on app interactive mode

When a view card is "interactive" (user clicked into the app), the preload stops forwarding ctrl+wheel, middle-mouse-drag, and horizontal-scroll-pan gestures so the embedded app gets every event. A mousedown notifier reports in-guest clicks to the host so it can flip the card into interactive mode without intercepting the click itself.
This commit is contained in:
Aidan
2026-06-14 07:55:34 -07:00
committed by GitHub
parent f08f71313f
commit fd10171ac1
8 changed files with 280 additions and 47 deletions
+121 -25
View File
@@ -136,40 +136,92 @@ try {
}
});
// When the host marks this webview as "interactive" (user clicked into the
// app), the preload stops forwarding wheel/middle gestures to the canvas
// and lets the app handle everything. Host pushes via webview.send.
let isInteractive = false;
try {
ipcRenderer.on('openswarm:set-interactive', (_event, payload) => {
isInteractive = !!(payload && payload.interactive);
});
} catch (_) {}
// First in-guest mousedown tells the host to activate interact mode. Never
// preventDefault so the click still reaches the app (Minecraft etc).
const onMouseDownNotify = (e) => {
if (isInteractive) return;
try { ipcRenderer.sendToHost('app-clicked', { button: e.button }); } catch (_) {}
};
window.addEventListener('mousedown', onMouseDownNotify, { capture: true });
// ---------------------------------------------------------------------------
// Canvas zoom passthrough (ctrl/meta + wheel)
// Horizontal scroll passthrough to canvas pan
//
// A <webview> is an out-of-process Chromium guest; wheel events that
// originate inside it never bubble to the embedding renderer. Without
// intercepting here, ctrl+wheel over a browser card just zooms the
// embedded page (Chromium's default) and the dashboard canvas never
// sees the gesture — issue #27.
//
// Capture-phase + passive:false so we run before the page's own listeners
// and can preventDefault to suppress the in-page page-zoom. We then
// forward the gesture (deltaY + guest-local cursor coords) to the host
// via sendToHost; BrowserCard's ipc-message handler turns it back into a
// synthetic WheelEvent dispatched from the webview element, which bubbles
// naturally to useCanvasControls' wheel listener.
// <webview> is an out-of-process guest; wheel events inside it never bubble
// to the embedding renderer. Vertical scroll and ctrl/meta+wheel zoom stay
// with the page (chromium default). A horizontal-dominant gesture, however,
// should pan the dashboard canvas if the guest page has nothing horizontal
// to scroll, to match the behavior over chat panels (which never have a
// horizontal scroller and always pan the canvas).
const pageCanScrollX = (node, dx) => {
let t = node;
while (t) {
const sw = t.scrollWidth || 0;
const cw = t.clientWidth || 0;
if (sw > cw) {
let style;
try { style = getComputedStyle(t); } catch (_) {}
const ox = style ? style.overflowX : 'visible';
if (ox === 'auto' || ox === 'scroll') {
const atRight = t.scrollLeft + cw >= sw - 1;
const atLeft = t.scrollLeft <= 1;
const atBoundary = (dx > 0 && atRight) || (dx < 0 && atLeft);
if (!atBoundary) return true;
}
}
t = t.parentElement;
}
const docEl = document.scrollingElement || document.documentElement;
if (docEl && docEl.scrollWidth > docEl.clientWidth) {
const atRight = docEl.scrollLeft + docEl.clientWidth >= docEl.scrollWidth - 1;
const atLeft = docEl.scrollLeft <= 1;
const atBoundary = (dx > 0 && atRight) || (dx < 0 && atLeft);
if (!atBoundary) return true;
}
return false;
};
const onWheelCapture = (e) => {
if (!(e.ctrlKey || e.metaKey)) return;
if (isInteractive) return;
if (e.ctrlKey || e.metaKey) {
e.preventDefault();
e.stopPropagation();
const iw = window.innerWidth || 1;
const ih = window.innerHeight || 1;
try {
ipcRenderer.sendToHost('canvas-wheel-zoom', {
deltaY: e.deltaY,
deltaMode: e.deltaMode,
fracX: Math.max(0, Math.min(1, e.clientX / iw)),
fracY: Math.max(0, Math.min(1, e.clientY / ih)),
});
} catch (_) {}
return;
}
// Vertical-dominant scroll stays with the page.
if (Math.abs(e.deltaX) <= Math.abs(e.deltaY)) return;
// Horizontal-dominant: defer to the page if anything inside can absorb
// it; otherwise forward to the host as a canvas pan.
if (pageCanScrollX(e.target, e.deltaX)) return;
e.preventDefault();
e.stopPropagation();
try {
console.warn('[openswarm:webview-preload] ctrl+wheel intercept → sendToHost', {
deltaY: e.deltaY,
clientX: e.clientX,
clientY: e.clientY,
});
ipcRenderer.sendToHost('canvas-wheel-zoom', {
ipcRenderer.sendToHost('canvas-wheel-pan', {
deltaX: e.deltaX,
deltaY: e.deltaY,
deltaMode: e.deltaMode,
clientX: e.clientX,
clientY: e.clientY,
});
} catch (err) {
console.warn('[openswarm:webview-preload] sendToHost failed', err);
}
} catch (_) {}
};
// Listen on both window and document in capture phase so we run before any
// page-level handler that might swallow the event. passive:false is required
@@ -177,6 +229,50 @@ try {
window.addEventListener('wheel', onWheelCapture, { capture: true, passive: false });
document.addEventListener('wheel', onWheelCapture, { capture: true, passive: false });
// ---------------------------------------------------------------------------
// Middle-mouse-button drag → canvas pan
//
// Empty canvas and agent cards already get middle-button pan because the
// event bubbles to the dashboard's mousedown handler. <webview> is a
// separate compositor layer that eats mouse events, so middle-drag over a
// browser silently did nothing. Intercept here and forward the per-event
// movement as a pan delta through the existing canvas-wheel-pan channel
// (negated, since drag pans panX += dx while wheel pans panX -= dx).
// Always pans regardless of capture state — middle-drag is unambiguously
// a canvas gesture.
let middleDragging = false;
const onMouseDownMiddle = (e) => {
if (isInteractive) return;
if (e.button !== 1) return;
e.preventDefault();
e.stopPropagation();
middleDragging = true;
};
const onMouseMoveMiddle = (e) => {
if (!middleDragging) return;
e.preventDefault();
e.stopPropagation();
const dx = e.movementX || 0;
const dy = e.movementY || 0;
if (dx === 0 && dy === 0) return;
try {
ipcRenderer.sendToHost('canvas-wheel-pan', { deltaX: -dx, deltaY: -dy, deltaMode: 0 });
} catch (_) {}
};
const onMouseUpMiddle = (e) => {
if (e.button !== 1) return;
middleDragging = false;
};
// Chromium starts auxiliary-scroll on middle-click; auxclick prevents that.
const onAuxClickSuppress = (e) => {
if (isInteractive) return;
if (e.button === 1) { e.preventDefault(); e.stopPropagation(); }
};
window.addEventListener('mousedown', onMouseDownMiddle, { capture: true });
window.addEventListener('mousemove', onMouseMoveMiddle, { capture: true });
window.addEventListener('mouseup', onMouseUpMiddle, { capture: true });
window.addEventListener('auxclick', onAuxClickSuppress, { capture: true });
// ---------------------------------------------------------------------------
// Double-click to fit the browser card (parity with agent-chat dblclick).
//
@@ -665,6 +665,9 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
// Without this early-out the unconditional stopPropagation below kills
// ctrl+wheel and the canvas listener never fires.
if (e.ctrlKey || e.metaKey) return;
// Horizontal-dominant gestures must also reach the canvas so a sideways
// swipe pans the dashboard (chat has no horizontal scroll to absorb).
if (Math.abs(e.deltaX) > Math.abs(e.deltaY)) return;
const atTop = el.scrollTop <= 0;
const atBottom = el.scrollTop + el.clientHeight >= el.scrollHeight - 1;
const scrollingDown = e.deltaY > 0;
@@ -303,7 +303,6 @@ const BrowserCard: React.FC<Props> = ({
// (the historical Windows mount segfault). Clear the crash-safety marker.
if (isWindows) markWindowsWebviewSurvived();
wv.loadURL(targetUrl).catch(() => {});
// Lock guest zoom at 1.0 so ctrl+wheel never triggers Chromium's in-page zoom; canvas zoom takes over (issue #27).
try {
(wv as any).setVisualZoomLevelLimits?.(1, 1);
(wv as any).setZoomFactor?.(1);
@@ -329,18 +328,30 @@ const BrowserCard: React.FC<Props> = ({
} else if (e?.channel === 'browser-dblclick') {
onDoubleClickRef.current?.(browserId, 'browser');
} else if (e?.channel === 'canvas-wheel-zoom') {
// Convert guest coords to doc coords and dispatch a CustomEvent; synthetic WheelEvent bubble was unreliable through GuestView.
const payload = e.args?.[0] || {};
const wvRect = wv.getBoundingClientRect();
const docX = wvRect.left + (payload.clientX ?? 0);
const docY = wvRect.top + (payload.clientY ?? 0);
const fx = typeof payload.fracX === 'number' ? payload.fracX : 0.5;
const fy = typeof payload.fracY === 'number' ? payload.fracY : 0.5;
window.dispatchEvent(
new CustomEvent('openswarm:canvas-wheel-zoom', {
detail: {
deltaY: payload.deltaY ?? 0,
deltaMode: payload.deltaMode ?? 0,
clientX: docX,
clientY: docY,
clientX: wvRect.left + fx * wvRect.width,
clientY: wvRect.top + fy * wvRect.height,
},
}),
);
} else if (e?.channel === 'canvas-wheel-pan') {
// Plain wheel inside an unselected webview never bubbles out; the
// preload forwards it here so the dashboard canvas can pan.
const payload = e.args?.[0] || {};
window.dispatchEvent(
new CustomEvent('openswarm:canvas-wheel-pan', {
detail: {
deltaX: payload.deltaX ?? 0,
deltaY: payload.deltaY ?? 0,
deltaMode: payload.deltaMode ?? 0,
},
}),
);
@@ -9,8 +9,8 @@ import RestartAltIcon from '@mui/icons-material/RestartAlt';
import CloseIcon from '@mui/icons-material/Close';
import GridViewRoundedIcon from '@mui/icons-material/GridViewRounded';
import { Output, SERVE_BASE } from '@/shared/state/outputsSlice';
import { setViewCardPosition, setViewCardSize, removeViewCard } from '@/shared/state/dashboardLayoutSlice';
import { useAppDispatch } from '@/shared/hooks';
import { setViewCardPosition, setViewCardSize, removeViewCard, setActiveViewCardId } from '@/shared/state/dashboardLayoutSlice';
import { useAppDispatch, useAppSelector } from '@/shared/hooks';
import { API_BASE, getAuthToken } from '@/shared/config';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
import ViewPreview, { ViewPreviewHandle } from '@/app/pages/Views/ViewPreview';
@@ -75,6 +75,23 @@ const DashboardViewCard: React.FC<Props> = ({
const dispatch = useAppDispatch();
const scrollOverlayRef = useOverlayScrollPassthrough(isSelected);
const previewRef = useRef<ViewPreviewHandle>(null);
const activeViewCardId = useAppSelector((s) => s.dashboardLayout.activeViewCardId);
const interactive = activeViewCardId === output.id;
// Deselecting the card exits interact mode (click anywhere else on canvas).
useEffect(() => {
if (!isSelected && interactive) dispatch(setActiveViewCardId(null));
}, [isSelected, interactive, dispatch]);
// Escape exits interact mode.
useEffect(() => {
if (!interactive) return;
const onKey = (e: KeyboardEvent) => {
if (e.key === 'Escape') dispatch(setActiveViewCardId(null));
};
window.addEventListener('keydown', onKey);
return () => window.removeEventListener('keydown', onKey);
}, [interactive, dispatch]);
const [inputData] = useState<Record<string, any>>(() => getDefault(output.input_schema));
const [backendResult] = useState<Record<string, any> | null>(null);
@@ -288,7 +305,9 @@ const DashboardViewCard: React.FC<Props> = ({
borderRadius: `${c.radius.lg}px`,
border: isHighlighted
? `2px solid ${c.accent.primary}`
: isSelected ? '2px solid #3b82f6' : `1px solid ${c.border.medium}`,
: interactive
? `2px solid ${c.accent.primary}`
: isSelected ? '2px solid #3b82f6' : `1px solid ${c.border.medium}`,
bgcolor: c.bg.surface,
boxShadow: isHighlighted
? `0 0 0 3px ${c.accent.primary}50, 0 0 20px ${c.accent.primary}35, 0 0 40px ${c.accent.primary}15`
@@ -406,6 +425,8 @@ const DashboardViewCard: React.FC<Props> = ({
output={output}
inputData={inputData}
backendResult={backendResult}
interactive={interactive}
onAppClicked={() => dispatch(setActiveViewCardId(output.id))}
/>
</Box>
@@ -487,7 +508,9 @@ const DashboardOutputPreview: React.FC<{
output: Output;
inputData: Record<string, any>;
backendResult: any;
}> = ({ previewRef, output, inputData, backendResult }) => {
interactive: boolean;
onAppClicked: () => void;
}> = ({ previewRef, output, inputData, backendResult, interactive, onAppClicked }) => {
const tokens = useClaudeTokens();
const dispatch = useAppDispatch();
const workspaceId = output.workspace_id ?? null;
@@ -587,6 +610,8 @@ const DashboardOutputPreview: React.FC<{
frontendCode={output.files?.['index.html'] ?? ''}
inputData={inputData}
backendResult={backendResult}
interactive={interactive}
onAppClicked={onAppClicked}
/>
);
};
@@ -261,6 +261,15 @@ export function useCanvasControls(zoomSensitivity: number = 50, contentBounds?:
// Re-read scrollHeight/clientHeight; cached decision is structural, scroll position is dynamic.
const canScrollY = target.scrollHeight > target.clientHeight;
const canScrollX = target.scrollWidth > target.clientWidth;
// Horizontal-dominant gestures over a container that only scrolls
// vertically (e.g., chat) should pan the canvas instead of being
// silently absorbed by the child's no-op horizontal handling.
if (Math.abs(dx) > Math.abs(dy) && !canScrollX) {
target = target.parentElement;
continue;
}
const atYBoundary = !canScrollY ||
(dy > 0 && target.scrollTop + target.clientHeight >= target.scrollHeight - 1) ||
(dy < 0 && target.scrollTop <= 1);
@@ -302,11 +311,6 @@ export function useCanvasControls(zoomSensitivity: number = 50, contentBounds?:
el.addEventListener('wheel', onWheel, { passive: false });
// ctrl/meta+wheel events that originate inside an Electron <webview>
// never bubble out of the guest into the host DOM, so the wheel
// listener above can't see them. BrowserCard's preload-bridge
// forwards those gestures via this CustomEvent (issue #27); we run
// the same zoom-around-cursor math the wheel handler uses.
const onForwardedZoom = (e: Event) => {
const detail = (e as CustomEvent).detail || {};
const dy = detail.deltaMode === 1 ? detail.deltaY * 40 : detail.deltaY;
@@ -324,9 +328,27 @@ export function useCanvasControls(zoomSensitivity: number = 50, contentBounds?:
};
window.addEventListener('openswarm:canvas-wheel-zoom', onForwardedZoom);
// Plain wheel inside a webview can't bubble out either; the preload
// forwards horizontal-dominant scrolls as a pan when the guest page
// has nothing to scroll horizontally, plus middle-mouse drag deltas.
const onForwardedPan = (e: Event) => {
const detail = (e as CustomEvent).detail || {};
const dy = detail.deltaMode === 1 ? (detail.deltaY ?? 0) * 40 : (detail.deltaY ?? 0);
const dx = detail.deltaMode === 1 ? (detail.deltaX ?? 0) * 40 : (detail.deltaX ?? 0);
if (inertiaFrameRef.current) {
cancelAnimationFrame(inertiaFrameRef.current);
inertiaFrameRef.current = null;
}
pendingPanDx += dx;
pendingPanDy += dy;
scheduleWheelFlush();
};
window.addEventListener('openswarm:canvas-wheel-pan', onForwardedPan);
return () => {
el.removeEventListener('wheel', onWheel);
window.removeEventListener('openswarm:canvas-wheel-zoom', onForwardedZoom);
window.removeEventListener('openswarm:canvas-wheel-pan', onForwardedPan);
if (wheelRafId != null) cancelAnimationFrame(wheelRafId);
if (wheelIdleTimer != null) clearTimeout(wheelIdleTimer);
// Don't leave the flag stuck on if the canvas unmounts mid-gesture.
@@ -22,6 +22,8 @@ export function useOverlayScrollPassthrough(active: boolean) {
dy *= 20;
}
const horizontalDominant = Math.abs(dx) > Math.abs(dy);
let node = underneath as HTMLElement | null;
while (node) {
if (node.tagName === 'WEBVIEW') {
@@ -52,6 +54,14 @@ export function useOverlayScrollPassthrough(active: boolean) {
node.scrollWidth > node.clientWidth &&
(cs.overflowX === 'auto' || cs.overflowX === 'scroll');
// Horizontal-dominant gesture over a vertically-only scrollable
// container: don't absorb it (scrollBy with dx would be a no-op).
// Let it bubble to the canvas wheel handler so the canvas pans.
if (horizontalDominant && !canScrollX) {
node = node.parentElement;
continue;
}
if (canScrollY || canScrollX) {
e.stopPropagation();
e.preventDefault();
+64 -7
View File
@@ -55,6 +55,10 @@ interface Props {
onConsoleMessage?: (level: string, text: string) => void;
/** Fires once the embedded app has actually painted, so cold-start placeholders don't unmount during the vite-ready to first-paint gap. */
onContentLoad?: () => void;
/** True when the user has clicked into the app; preload stops forwarding canvas gestures and lets the app handle all events. */
interactive?: boolean;
/** Fired when the preload reports a mousedown inside the guest, so the host can flip the card into interactive mode. */
onAppClicked?: () => void;
}
function buildSrcdoc(
@@ -92,6 +96,8 @@ const ViewPreview = forwardRef<ViewPreviewHandle, Props>(({
style,
onConsoleMessage,
onContentLoad,
interactive = false,
onAppClicked,
}, ref) => {
const iframeRef = useRef<HTMLIFrameElement>(null);
const webviewRef = useRef<any>(null);
@@ -219,22 +225,72 @@ const ViewPreview = forwardRef<ViewPreviewHandle, Props>(({
}
}, [srcdoc, useWebview]);
// Forward webview-console events (preload wraps console.*) to onConsoleMessage; iframe path has no equivalent.
// Listen for preload IPC: console forwarding, canvas wheel forwarding
// (matches BrowserCard so apps share the same dashboard pan/zoom defaults),
// and the app-clicked notification that flips the card into interact mode.
useEffect(() => {
if (!useWebview || !onConsoleMessage) return;
if (!useWebview) return;
const wv = webviewRef.current;
if (!wv) return;
const handler = (e: any) => {
if (e?.channel !== 'webview-console') return;
const arg = Array.isArray(e.args) ? e.args[0] : undefined;
if (!arg) return;
onConsoleMessage(arg.level || 'log', arg.text || '');
if (e?.channel === 'webview-console') {
if (!onConsoleMessage) return;
const arg = Array.isArray(e.args) ? e.args[0] : undefined;
if (!arg) return;
onConsoleMessage(arg.level || 'log', arg.text || '');
return;
}
if (e?.channel === 'canvas-wheel-zoom') {
const payload = e.args?.[0] || {};
const wvRect = wv.getBoundingClientRect();
const fx = typeof payload.fracX === 'number' ? payload.fracX : 0.5;
const fy = typeof payload.fracY === 'number' ? payload.fracY : 0.5;
window.dispatchEvent(
new CustomEvent('openswarm:canvas-wheel-zoom', {
detail: {
deltaY: payload.deltaY ?? 0,
deltaMode: payload.deltaMode ?? 0,
clientX: wvRect.left + fx * wvRect.width,
clientY: wvRect.top + fy * wvRect.height,
},
}),
);
return;
}
if (e?.channel === 'canvas-wheel-pan') {
const payload = e.args?.[0] || {};
window.dispatchEvent(
new CustomEvent('openswarm:canvas-wheel-pan', {
detail: {
deltaX: payload.deltaX ?? 0,
deltaY: payload.deltaY ?? 0,
deltaMode: payload.deltaMode ?? 0,
},
}),
);
return;
}
if (e?.channel === 'app-clicked') {
onAppClicked?.();
return;
}
};
wv.addEventListener?.('ipc-message', handler);
return () => {
try { wv.removeEventListener?.('ipc-message', handler); } catch (_e) {}
};
}, [useWebview, onConsoleMessage, iframeSrc]);
}, [useWebview, onConsoleMessage, onAppClicked, iframeSrc]);
// Mirror `interactive` into a ref so the once-per-load did-finish-load
// listener can read the latest value when it pushes initial state.
const interactiveRef = useRef(interactive);
interactiveRef.current = interactive;
useEffect(() => {
if (!useWebview) return;
const wv = webviewRef.current;
if (!wv) return;
try { wv.send?.('openswarm:set-interactive', { interactive }); } catch (_e) {}
}, [useWebview, interactive]);
// Webviews use did-finish-load instead of onLoad; did-fail-load retries with 500ms to 5s backoff (Vite may not have bound yet when frontend_url arrives).
useEffect(() => {
@@ -256,6 +312,7 @@ const ViewPreview = forwardRef<ViewPreviewHandle, Props>(({
retryDelay = 500;
cancelRetry();
handleNavigationLoad();
try { wv.send?.('openswarm:set-interactive', { interactive: interactiveRef.current }); } catch (_) {}
};
const onFail = (e: any) => {
// Guard on isMainFrame (subresource 404s fire too) and ERR_ABORTED (user-cancel).
@@ -98,6 +98,8 @@ export interface DashboardLayoutState {
suspendedBrowserCards: Record<string, { dataUrl: string; capturedAt: number }>;
/** Transient: spawned cards that are about to be removed; surfaces the fade + Keep pill. */
endingBrowserCards: Record<string, { status: 'completed' | 'error'; at: number }>;
/** Transient: id of the view card the user has clicked into; preload stops forwarding canvas gestures while set. */
activeViewCardId: string | null;
}
const initialState: DashboardLayoutState = {
@@ -116,6 +118,7 @@ const initialState: DashboardLayoutState = {
pendingFocusNoteId: null,
suspendedBrowserCards: {},
endingBrowserCards: {},
activeViewCardId: null,
};
interface LayoutPayload {
@@ -565,6 +568,11 @@ const dashboardLayoutSlice = createSlice({
removeViewCard(state, action: PayloadAction<string>) {
delete state.viewCards[action.payload];
if (state.activeViewCardId === action.payload) state.activeViewCardId = null;
},
setActiveViewCardId(state, action: PayloadAction<string | null>) {
state.activeViewCardId = action.payload;
},
addBrowserCard(state, action: PayloadAction<{ url: string; expandedSessionIds?: string[] }>) {
@@ -1080,6 +1088,7 @@ export const {
setViewCardPosition,
setViewCardSize,
removeViewCard,
setActiveViewCardId,
addBrowserCard,
addBrowserCardFromBackend,
setBrowserCardPosition,