wip: browser hover-capture zoom + canvas pan/scroll improvements

Bundles two strands of work for archival:
- Hover/select-to-capture system that routes ctrl+wheel over a selected
  or long-hovered browser into in-page setZoomFactor instead of canvas zoom.
- Middle-mouse pan + horizontal-scroll-to-canvas-pan over browser and chat.

The first strand is being reverted on the working branch; this commit
preserves it on browser-zoom-features-archive.
This commit is contained in:
abccodes
2026-06-13 19:35:11 -07:00
parent cee7a94559
commit c15a46e43e
7 changed files with 347 additions and 21 deletions
+121 -17
View File
@@ -137,38 +137,100 @@ try {
});
// ---------------------------------------------------------------------------
// Canvas zoom passthrough (ctrl/meta + wheel)
// Canvas wheel passthrough (zoom + pan)
//
// A <webview> is an out-of-process Chromium guest; wheel events that
// originate inside it never bubble to the embedding renderer. Without
// intercepting here, ctrl+wheel over a browser card just zooms the
// embedded page (Chromium's default) and the dashboard canvas never
// sees the gesture issue #27.
// sees the gesture (issue #27).
//
// Capture-phase + passive:false so we run before the page's own listeners
// and can preventDefault to suppress the in-page page-zoom. We then
// forward the gesture (deltaY + guest-local cursor coords) to the host
// via sendToHost; BrowserCard's ipc-message handler turns it back into a
// synthetic WheelEvent dispatched from the webview element, which bubbles
// naturally to useCanvasControls' wheel listener.
// Capture-state lives in the host (BrowserCard); when this browser is
// "captured" (single-selected or hovered >3s) the user wants zoom/scroll
// to act on the page itself. Otherwise zoom and pan should drive the
// dashboard canvas. The host pushes capture-state changes to us via
// webview.send('openswarm:set-capture-state', { captures }) so we can
// decide what to do with each event without a round-trip.
let captureState = { captures: false };
try {
ipcRenderer.on('openswarm:set-capture-state', (_event, payload) => {
captureState = { captures: !!(payload && payload.captures) };
});
} catch (_) {}
// True when an ancestor of `node` (or the document) can scroll horizontally
// with the given dx direction. Used to decide whether a horizontal wheel
// gesture should pan the dashboard canvas or scroll the page itself.
const pageCanScrollX = (node, dx) => {
let t = node;
while (t) {
const sw = t.scrollWidth || 0;
const cw = t.clientWidth || 0;
if (sw > cw) {
let style;
try { style = getComputedStyle(t); } catch (_) {}
const ox = style ? style.overflowX : 'visible';
if (ox === 'auto' || ox === 'scroll') {
const atRight = t.scrollLeft + cw >= sw - 1;
const atLeft = t.scrollLeft <= 1;
const atBoundary = (dx > 0 && atRight) || (dx < 0 && atLeft);
if (!atBoundary) return true;
}
}
t = t.parentElement;
}
const docEl = document.scrollingElement || document.documentElement;
if (docEl && docEl.scrollWidth > docEl.clientWidth) {
const atRight = docEl.scrollLeft + docEl.clientWidth >= docEl.scrollWidth - 1;
const atLeft = docEl.scrollLeft <= 1;
const atBoundary = (dx > 0 && atRight) || (dx < 0 && atLeft);
if (!atBoundary) return true;
}
return false;
};
const onWheelCapture = (e) => {
if (!(e.ctrlKey || e.metaKey)) return;
const isPinch = !!(e.ctrlKey || e.metaKey);
if (!isPinch) {
// Vertical-dominant scroll always stays with the page.
if (Math.abs(e.deltaX) <= Math.abs(e.deltaY)) return;
// Horizontal-dominant: defer to the page if anything inside can absorb
// it; otherwise pan the dashboard canvas (matches chat behavior).
if (pageCanScrollX(e.target, e.deltaX)) return;
e.preventDefault();
e.stopPropagation();
try {
ipcRenderer.sendToHost('canvas-wheel-pan', {
deltaX: e.deltaX,
deltaY: e.deltaY,
deltaMode: e.deltaMode,
});
} catch (_) {}
return;
}
// Always preventDefault: chromium's in-page zoom would fight whatever
// the host decides to do. The host's ipc-message handler does either
// canvas zoom (not captured) or webview.setZoomFactor (captured).
e.preventDefault();
e.stopPropagation();
// Send the cursor as a FRACTION of the guest viewport. The host's
// <webview>.getBoundingClientRect() reports the on-screen rect after
// every CSS transform, so (frac * wvRect) is the exact screen pixel
// position regardless of canvas pan/zoom. Forwarding raw clientX
// pixels broke zoom-around-cursor at non-1 canvas zooms because the
// host can't reliably reconstruct the guest->screen scale.
const iw = window.innerWidth || 1;
const ih = window.innerHeight || 1;
try {
console.warn('[openswarm:webview-preload] ctrl+wheel intercept → sendToHost', {
deltaY: e.deltaY,
clientX: e.clientX,
clientY: e.clientY,
});
ipcRenderer.sendToHost('canvas-wheel-zoom', {
deltaY: e.deltaY,
deltaMode: e.deltaMode,
clientX: e.clientX,
clientY: e.clientY,
fracX: Math.max(0, Math.min(1, e.clientX / iw)),
fracY: Math.max(0, Math.min(1, e.clientY / ih)),
captured: captureState.captures,
});
} catch (err) {
console.warn('[openswarm:webview-preload] sendToHost failed', err);
console.warn('[openswarm:webview-preload] zoom sendToHost failed', err);
}
};
// Listen on both window and document in capture phase so we run before any
@@ -177,6 +239,48 @@ try {
window.addEventListener('wheel', onWheelCapture, { capture: true, passive: false });
document.addEventListener('wheel', onWheelCapture, { capture: true, passive: false });
// ---------------------------------------------------------------------------
// Middle-mouse-button drag → canvas pan
//
// Empty canvas and agent cards already get middle-button pan because the
// event bubbles to the dashboard's mousedown handler. <webview> is a
// separate compositor layer that eats mouse events, so middle-drag over a
// browser silently did nothing. Intercept here and forward the per-event
// movement as a pan delta through the existing canvas-wheel-pan channel
// (negated, since drag pans panX += dx while wheel pans panX -= dx).
// Always pans regardless of capture state — middle-drag is unambiguously
// a canvas gesture.
let middleDragging = false;
const onMouseDownMiddle = (e) => {
if (e.button !== 1) return;
e.preventDefault();
e.stopPropagation();
middleDragging = true;
};
const onMouseMoveMiddle = (e) => {
if (!middleDragging) return;
e.preventDefault();
e.stopPropagation();
const dx = e.movementX || 0;
const dy = e.movementY || 0;
if (dx === 0 && dy === 0) return;
try {
ipcRenderer.sendToHost('canvas-wheel-pan', { deltaX: -dx, deltaY: -dy, deltaMode: 0 });
} catch (_) {}
};
const onMouseUpMiddle = (e) => {
if (e.button !== 1) return;
middleDragging = false;
};
// Chromium starts auxiliary-scroll on middle-click; auxclick prevents that.
const onAuxClickSuppress = (e) => {
if (e.button === 1) { e.preventDefault(); e.stopPropagation(); }
};
window.addEventListener('mousedown', onMouseDownMiddle, { capture: true });
window.addEventListener('mousemove', onMouseMoveMiddle, { capture: true });
window.addEventListener('mouseup', onMouseUpMiddle, { capture: true });
window.addEventListener('auxclick', onAuxClickSuppress, { capture: true });
// ---------------------------------------------------------------------------
// Double-click to fit the browser card (parity with agent-chat dblclick).
//
@@ -664,6 +664,9 @@ const AgentChat: React.FC<AgentChatProps> = ({ sessionId: sessionIdProp, onClose
// Without this early-out the unconditional stopPropagation below kills
// ctrl+wheel and the canvas listener never fires.
if (e.ctrlKey || e.metaKey) return;
// Horizontal-dominant gestures must also reach the canvas so a sideways
// swipe pans the dashboard (chat has no horizontal scroll to absorb).
if (Math.abs(e.deltaX) > Math.abs(e.deltaY)) return;
const atTop = el.scrollTop <= 0;
const atBottom = el.scrollTop + el.clientHeight >= el.scrollHeight - 1;
const scrollingDown = e.deltaY > 0;
@@ -36,6 +36,11 @@ import { parseMcpToolName, getMcpShortAction } from '@/shared/mcpToolMeta';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
import { useDashboardActive } from '@/shared/hooks/useDashboardActive';
import { useOverlayScrollPassthrough } from '../hooks/interaction/useOverlayScrollPassthrough';
import {
markHovered as markCaptureHovered,
markUnhovered as markCaptureUnhovered,
useReportCardSelection,
} from '../hooks/interaction/useCardCaptureState';
import { useStreamingMessage } from '@/shared/state/streamingSlice';
import { isCanvasInteractionActive, onCanvasInteractionEnd } from '@/shared/canvasInteractionState';
import { getAgentWorkTime, fmtSeconds } from '@/shared/agentWorkTime';
@@ -241,6 +246,7 @@ const AgentCard: React.FC<Props> = ({
return s;
}, [session.model, modelsByProvider]);
const scrollOverlayRef = useOverlayScrollPassthrough(isSelected);
useReportCardSelection(session.id, 'agent', isSelected);
const cardBoxRef = useRef<HTMLDivElement>(null);
// Ref so ResizeObserver sees latest value without re-attaching when active flips.
@@ -556,6 +562,8 @@ const AgentCard: React.FC<Props> = ({
if (justDraggedRef.current) return;
onCardSelect?.(session.id, 'agent', e.shiftKey);
}}
onPointerEnter={() => markCaptureHovered(session.id, 'agent')}
onPointerLeave={() => markCaptureUnhovered(session.id)}
onDoubleClick={(e: React.MouseEvent) => {
e.stopPropagation();
onDoubleClick?.(session.id, 'agent');
@@ -51,6 +51,12 @@ import { getActionLabel } from '@/shared/browserCommandHandler';
import { resolveInput, isGoogleSearch } from '@/shared/resolveUrl';
import BrowserAgentOverlay from './BrowserAgentOverlay';
import { useOverlayScrollPassthrough } from '../hooks/interaction/useOverlayScrollPassthrough';
import {
markHovered as markCaptureHovered,
markUnhovered as markCaptureUnhovered,
useReportCardSelection,
useCardCapture,
} from '../hooks/interaction/useCardCaptureState';
import { useElementSelection } from '@/app/components/editor/ElementSelectionContext';
type ResizeDir = 'n' | 's' | 'e' | 'w' | 'ne' | 'nw' | 'se' | 'sw';
@@ -198,6 +204,8 @@ const BrowserCard: React.FC<Props> = ({
const onDoubleClickRef = useRef(onDoubleClick);
onDoubleClickRef.current = onDoubleClick;
const scrollOverlayRef = useOverlayScrollPassthrough(isSelected);
useReportCardSelection(browserId, 'browser', isSelected);
const capture = useCardCapture(browserId, 'browser');
const browserHomepage = useAppSelector((state) => state.settings.data.browser_homepage);
const elementSelectionCtx = useElementSelection();
const isElementSelectMode = elementSelectionCtx?.selectMode ?? false;
@@ -262,6 +270,15 @@ const BrowserCard: React.FC<Props> = ({
const webviewMap = useRef<Map<string, WebviewElement>>(new Map());
const initializedTabs = useRef(new Set<string>());
const tabBarRef = useRef<HTMLDivElement>(null);
// In-page zoom factor we drive ourselves when this browser is "captured".
// Per-tab so each tab keeps its own zoom across activations within the session.
const tabZoomFactors = useRef<Map<string, number>>(new Map());
const MIN_PAGE_ZOOM = 0.25;
const MAX_PAGE_ZOOM = 5;
// Mirror capture.captures into a ref so dom-ready can read the current value
// (the useEffect that pushes capture state can race the webview's first ready).
const captureRef = useRef(false);
captureRef.current = capture.captures;
useEffect(() => {
setRegistryActiveTab(browserId, activeTabId);
@@ -303,11 +320,16 @@ const BrowserCard: React.FC<Props> = ({
// (the historical Windows mount segfault). Clear the crash-safety marker.
if (isWindows) markWindowsWebviewSurvived();
wv.loadURL(targetUrl).catch(() => {});
// Lock guest zoom at 1.0 so ctrl+wheel never triggers Chromium's in-page zoom; canvas zoom takes over (issue #27).
// Pin visual (pinch) zoom inside the guest so chromium never fights us;
// we drive page zoom ourselves via setZoomFactor from canvas-wheel-zoom
// when this card is "captured" (selected or hovered >3s).
try {
(wv as any).setVisualZoomLevelLimits?.(1, 1);
(wv as any).setZoomFactor?.(1);
const initial = tabZoomFactors.current.get(tabId) ?? 1;
(wv as any).setZoomFactor?.(initial);
} catch (_) {}
// Sync initial capture state to the preload now that ipc is live.
try { (wv as any).send?.('openswarm:set-capture-state', { captures: captureRef.current }); } catch (_) {}
};
wv.addEventListener('dom-ready', doLoad, { once: true });
cleanups.push(() => wv.removeEventListener('dom-ready', doLoad));
@@ -331,9 +353,27 @@ const BrowserCard: React.FC<Props> = ({
} else if (e?.channel === 'canvas-wheel-zoom') {
// Convert guest coords to doc coords and dispatch a CustomEvent; synthetic WheelEvent bubble was unreliable through GuestView.
const payload = e.args?.[0] || {};
if (payload.captured) {
// In-page zoom: drive setZoomFactor directly so chromium scales
// the page itself (Wikipedia text gets bigger), not the canvas.
const deltaY = payload.deltaMode === 1 ? (payload.deltaY ?? 0) * 40 : (payload.deltaY ?? 0);
const current = tabZoomFactors.current.get(tabId) ?? 1;
const factor = Math.pow(2, -deltaY * 0.005);
const next = Math.max(MIN_PAGE_ZOOM, Math.min(MAX_PAGE_ZOOM, current * factor));
tabZoomFactors.current.set(tabId, next);
try { (wv as any).setZoomFactor?.(next); } catch (_) {}
return;
}
const wvRect = wv.getBoundingClientRect();
const docX = wvRect.left + (payload.clientX ?? 0);
const docY = wvRect.top + (payload.clientY ?? 0);
// The preload sends fracX/Y (cursor as a fraction of guest viewport).
// wvRect is the on-screen rect after all CSS transforms, so frac *
// wvRect.size lands on the exact cursor pixel without us needing to
// reconstruct any guest->screen scale. (Forwarding raw guest pixels
// broke zoom-around-cursor at non-1 canvas zooms.)
const fx = typeof payload.fracX === 'number' ? payload.fracX : 0.5;
const fy = typeof payload.fracY === 'number' ? payload.fracY : 0.5;
const docX = wvRect.left + fx * wvRect.width;
const docY = wvRect.top + fy * wvRect.height;
window.dispatchEvent(
new CustomEvent('openswarm:canvas-wheel-zoom', {
detail: {
@@ -344,6 +384,19 @@ const BrowserCard: React.FC<Props> = ({
},
}),
);
} else if (e?.channel === 'canvas-wheel-pan') {
// Plain wheel inside an unselected webview never bubbles out; the
// preload forwards it here so the dashboard canvas can pan.
const payload = e.args?.[0] || {};
window.dispatchEvent(
new CustomEvent('openswarm:canvas-wheel-pan', {
detail: {
deltaX: payload.deltaX ?? 0,
deltaY: payload.deltaY ?? 0,
deltaMode: payload.deltaMode ?? 0,
},
}),
);
}
};
@@ -416,6 +469,15 @@ const BrowserCard: React.FC<Props> = ({
// eslint-disable-next-line react-hooks/exhaustive-deps
}, [tabIdKey, browserId, dispatch, updateTabLocal, suspendedSnap]);
// Push capture-state changes to every live webview so the preload knows
// whether to forward wheel events to canvas or let the page handle them.
useEffect(() => {
if (!isElectron) return;
for (const wv of webviewMap.current.values()) {
try { (wv as any).send?.('openswarm:set-capture-state', { captures: capture.captures }); } catch (_) {}
}
}, [capture.captures, tabIdKey]);
const navigate = useCallback((targetUrl: string) => {
const finalUrl = resolveInput(targetUrl);
setUrlBarValue(finalUrl);
@@ -743,6 +805,8 @@ const BrowserCard: React.FC<Props> = ({
data-select-id={browserId}
data-select-meta={JSON.stringify({ name: activeTitle || 'Browser', url: activeUrl })}
onPointerDownCapture={() => onBringToFront?.(browserId, 'browser')}
onPointerEnter={() => markCaptureHovered(browserId, 'browser')}
onPointerLeave={() => markCaptureUnhovered(browserId)}
onClick={(e: React.MouseEvent) => {
if (justDraggedRef.current) return;
onCardSelect?.(browserId, 'browser', e.shiftKey);
@@ -261,6 +261,15 @@ export function useCanvasControls(zoomSensitivity: number = 50, contentBounds?:
// Re-read scrollHeight/clientHeight; cached decision is structural, scroll position is dynamic.
const canScrollY = target.scrollHeight > target.clientHeight;
const canScrollX = target.scrollWidth > target.clientWidth;
// Horizontal-dominant gestures over a container that only scrolls
// vertically (e.g., chat) should pan the canvas instead of being
// silently absorbed by the child's no-op horizontal handling.
if (Math.abs(dx) > Math.abs(dy) && !canScrollX) {
target = target.parentElement;
continue;
}
const atYBoundary = !canScrollY ||
(dy > 0 && target.scrollTop + target.clientHeight >= target.scrollHeight - 1) ||
(dy < 0 && target.scrollTop <= 1);
@@ -324,9 +333,26 @@ export function useCanvasControls(zoomSensitivity: number = 50, contentBounds?:
};
window.addEventListener('openswarm:canvas-wheel-zoom', onForwardedZoom);
// Plain wheel inside a webview can't bubble out either; the preload
// forwards it as a pan when the browser isn't captured.
const onForwardedPan = (e: Event) => {
const detail = (e as CustomEvent).detail || {};
const dy = detail.deltaMode === 1 ? (detail.deltaY ?? 0) * 40 : (detail.deltaY ?? 0);
const dx = detail.deltaMode === 1 ? (detail.deltaX ?? 0) * 40 : (detail.deltaX ?? 0);
if (inertiaFrameRef.current) {
cancelAnimationFrame(inertiaFrameRef.current);
inertiaFrameRef.current = null;
}
pendingPanDx += dx;
pendingPanDy += dy;
scheduleWheelFlush();
};
window.addEventListener('openswarm:canvas-wheel-pan', onForwardedPan);
return () => {
el.removeEventListener('wheel', onWheel);
window.removeEventListener('openswarm:canvas-wheel-zoom', onForwardedZoom);
window.removeEventListener('openswarm:canvas-wheel-pan', onForwardedPan);
if (wheelRafId != null) cancelAnimationFrame(wheelRafId);
if (wheelIdleTimer != null) clearTimeout(wheelIdleTimer);
// Don't leave the flag stuck on if the canvas unmounts mid-gesture.
@@ -0,0 +1,111 @@
import { useEffect, useState, useSyncExternalStore } from 'react';
export type CapturableCardType = 'browser' | 'agent';
export const HOVER_CAPTURE_MS = 5000;
interface InternalState {
hoveredId: string | null;
hoveredType: CapturableCardType | null;
hoverStartMs: number;
selectedIds: Map<string, CapturableCardType>;
}
let state: InternalState = {
hoveredId: null,
hoveredType: null,
hoverStartMs: 0,
selectedIds: new Map(),
};
const listeners = new Set<() => void>();
let hoverTimer: ReturnType<typeof setTimeout> | null = null;
function notify() {
for (const l of listeners) l();
}
function subscribe(l: () => void): () => void {
listeners.add(l);
return () => { listeners.delete(l); };
}
export function markHovered(id: string, type: CapturableCardType) {
if (state.hoveredId === id && state.hoveredType === type) return;
state = { ...state, hoveredId: id, hoveredType: type, hoverStartMs: performance.now() };
if (hoverTimer) clearTimeout(hoverTimer);
hoverTimer = setTimeout(() => { hoverTimer = null; notify(); }, HOVER_CAPTURE_MS);
notify();
}
export function markUnhovered(id: string) {
if (state.hoveredId !== id) return;
state = { ...state, hoveredId: null, hoveredType: null, hoverStartMs: 0 };
if (hoverTimer) { clearTimeout(hoverTimer); hoverTimer = null; }
notify();
}
export function setCardSelected(id: string, type: CapturableCardType, selected: boolean) {
const prev = state.selectedIds.get(id);
if (selected && prev === type) return;
if (!selected && prev === undefined) return;
const next = new Map(state.selectedIds);
if (selected) next.set(id, type); else next.delete(id);
state = { ...state, selectedIds: next };
notify();
}
export interface CaptureSnapshot {
hoveredId: string | null;
hoveredType: CapturableCardType | null;
capturedId: string | null;
capturedType: CapturableCardType | null;
}
export function getCaptureState(): CaptureSnapshot {
const { hoveredId, hoveredType, hoverStartMs, selectedIds } = state;
if (!hoveredId || !hoveredType) {
return { hoveredId: null, hoveredType: null, capturedId: null, capturedType: null };
}
const isSingleSelected = selectedIds.size === 1 && selectedIds.get(hoveredId) === hoveredType;
const elapsed = performance.now() - hoverStartMs;
const captures = isSingleSelected || elapsed >= HOVER_CAPTURE_MS;
return {
hoveredId,
hoveredType,
capturedId: captures ? hoveredId : null,
capturedType: captures ? hoveredType : null,
};
}
function snapshotForId(id: string, type: CapturableCardType): { isHovered: boolean; captures: boolean } {
const snap = getCaptureState();
return {
isHovered: snap.hoveredId === id && snap.hoveredType === type,
captures: snap.capturedId === id && snap.capturedType === type,
};
}
// Per-id subscription. useSyncExternalStore requires a stable snapshot reference
// when nothing has changed, so memoize the {isHovered, captures} tuple per call.
export function useCardCapture(id: string, type: CapturableCardType): { isHovered: boolean; captures: boolean } {
const [{ getSnap }] = useState(() => {
let cached: { isHovered: boolean; captures: boolean } = snapshotForId(id, type);
const getSnapFn = () => {
const next = snapshotForId(id, type);
if (next.isHovered === cached.isHovered && next.captures === cached.captures) return cached;
cached = next;
return cached;
};
return { getSnap: getSnapFn };
});
return useSyncExternalStore(subscribe, getSnap, getSnap);
}
// Mirror a card's `isSelected` prop into the module's selection map so the
// synchronous getCaptureState() reader (called from wheel handlers) stays correct.
export function useReportCardSelection(id: string, type: CapturableCardType, isSelected: boolean) {
useEffect(() => {
setCardSelected(id, type, isSelected);
return () => { setCardSelected(id, type, false); };
}, [id, type, isSelected]);
}
@@ -22,6 +22,8 @@ export function useOverlayScrollPassthrough(active: boolean) {
dy *= 20;
}
const horizontalDominant = Math.abs(dx) > Math.abs(dy);
let node = underneath as HTMLElement | null;
while (node) {
if (node.tagName === 'WEBVIEW') {
@@ -52,6 +54,14 @@ export function useOverlayScrollPassthrough(active: boolean) {
node.scrollWidth > node.clientWidth &&
(cs.overflowX === 'auto' || cs.overflowX === 'scroll');
// Horizontal-dominant gesture over a vertically-only scrollable
// container: don't absorb it (scrollBy with dx would be a no-op).
// Let it bubble to the canvas wheel handler so the canvas pans.
if (horizontalDominant && !canScrollX) {
node = node.parentElement;
continue;
}
if (canScrollY || canScrollX) {
e.stopPropagation();
e.preventDefault();