mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-30 05:24:50 +02:00
Merge remote-tracking branch 'origin/eric/dev' into eric/browser-merged
# Conflicts: # electron/package-lock.json
This commit is contained in:
@@ -2,7 +2,7 @@ import { getWebview, findWebviewByDomain, hasDomReady, markDomReady, isPendingLo
|
||||
import { shouldSelfHealClick } from './selfHealClick';
|
||||
import { FP_EXPR, clickEffect } from './clickEffect';
|
||||
import { store } from './state/store';
|
||||
import { resumeBrowserCard, focusBrowserCard } from './state/dashboardLayoutSlice';
|
||||
import { resumeBrowserCard } from './state/dashboardLayoutSlice';
|
||||
import { dashboardWs } from './ws/WebSocketManager';
|
||||
import { resolveInput } from './resolveUrl';
|
||||
import { rankAndCapInteractives, type RankItem } from './interactiveRanking';
|
||||
@@ -148,8 +148,6 @@ async function removeAnnotations(wv: BrowserWebview): Promise<void> {
|
||||
// while a card at x=-15185 timed out on guest capturePage, on host capturePage, AND on CDP
|
||||
// Page.captureScreenshot in both fromSurface modes. Panning the card into view is not a nicety,
|
||||
// it is the only thing that makes the pixels exist.
|
||||
const ONSCREEN_POLL_MS = 60;
|
||||
const ONSCREEN_WAIT_MS = 1500;
|
||||
// A sliver poking over the edge composites the sliver, not the page, so ask for a real chunk of card.
|
||||
const ONSCREEN_MIN_PX = 80;
|
||||
|
||||
@@ -163,23 +161,17 @@ function p_cardIsOnScreen(wv: BrowserWebview): boolean {
|
||||
&& p_overlap(r.top, r.bottom, window.innerHeight) >= ONSCREEN_MIN_PX;
|
||||
}
|
||||
|
||||
/** Pan the canvas to the card if it is out of view. '' once it is in view, else why it never will be. */
|
||||
async function ensureCardOnScreen(wv: BrowserWebview, browserId: string): Promise<string> {
|
||||
/** '' when the card is visible enough to capture, else why not. Never moves the camera: yanking the user's view because the agent wants a screenshot was the top annoyance report, so an off-screen card degrades to reading tools instead. */
|
||||
function ensureCardOnScreen(wv: BrowserWebview): string {
|
||||
if (p_cardIsOnScreen(wv)) return '';
|
||||
store.dispatch(focusBrowserCard(browserId));
|
||||
const deadline = Date.now() + ONSCREEN_WAIT_MS;
|
||||
while (Date.now() < deadline) {
|
||||
await new Promise((r) => setTimeout(r, ONSCREEN_POLL_MS));
|
||||
if (p_cardIsOnScreen(wv)) return '';
|
||||
}
|
||||
return 'the browser card is off screen and the canvas could not pan to it (it may live on another dashboard)';
|
||||
return 'the browser card is off screen (screenshots need it visible; the user can bring it into view)';
|
||||
}
|
||||
|
||||
async function handleScreenshot(wv: BrowserWebview, browserId: string, params?: Record<string, any>): Promise<Record<string, any>> {
|
||||
const p_t0 = Date.now();
|
||||
const p_stages: string[] = [];
|
||||
const p_mark = (stage: string): void => { p_stages.push(`${stage}:${Date.now() - p_t0}`); };
|
||||
const p_hidden = await ensureCardOnScreen(wv, browserId);
|
||||
const p_hidden = ensureCardOnScreen(wv);
|
||||
p_mark('onscreen');
|
||||
if (p_hidden) {
|
||||
return {
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { store } from '@/shared/state/store';
|
||||
|
||||
/** Copy the chat's latest assistant answer to the clipboard; true when something was copied.
|
||||
* Menu rows (card context menu, dock tiles) call this statelessly. */
|
||||
export function copySessionResponse(sessionId: string): boolean {
|
||||
const session = store.getState().agents.sessions[sessionId];
|
||||
const last = [...(session?.messages ?? [])]
|
||||
.reverse()
|
||||
.find((m) => m.role === 'assistant' && typeof m.content === 'string' && m.content.trim());
|
||||
if (!last) return false;
|
||||
void navigator.clipboard.writeText(last.content as string);
|
||||
return true;
|
||||
}
|
||||
@@ -41,6 +41,7 @@ export function useRuntimePreviewUrl(opts: RuntimePreviewOptions): RuntimePrevie
|
||||
}
|
||||
let cancelled = false;
|
||||
let ws: WebSocket | null = null;
|
||||
let retryTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
setFrontendUrl(null);
|
||||
setIsNewMode(false);
|
||||
setIsHydrating(true);
|
||||
@@ -53,7 +54,8 @@ export function useRuntimePreviewUrl(opts: RuntimePreviewOptions): RuntimePrevie
|
||||
const headers: Record<string, string> = { 'Content-Type': 'application/json' };
|
||||
if (auth) headers.Authorization = `Bearer ${auth}`;
|
||||
|
||||
(async () => {
|
||||
const connect = async (): Promise<void> => {
|
||||
if (cancelled) return;
|
||||
try {
|
||||
await fetch(`${API_BASE}/outputs/workspace/${workspaceId}/runtime/start?instance=${instance}`, {
|
||||
method: 'POST',
|
||||
@@ -85,14 +87,23 @@ export function useRuntimePreviewUrl(opts: RuntimePreviewOptions): RuntimePrevie
|
||||
// Malformed frame; safe to drop.
|
||||
}
|
||||
};
|
||||
// A backend restart kills the runtime AND this socket; without re-attaching the card sits
|
||||
// on a dead port forever, so drop the stale URL (placeholder over a dead webview) and retry.
|
||||
ws.onclose = () => {
|
||||
if (cancelled) return;
|
||||
setFrontendUrl(null);
|
||||
retryTimer = setTimeout(() => { void connect(); }, 2000);
|
||||
};
|
||||
} catch (_) {
|
||||
// WS construction failed; caller stays in "no preview yet" state.
|
||||
retryTimer = setTimeout(() => { void connect(); }, 2000);
|
||||
}
|
||||
})();
|
||||
};
|
||||
void connect();
|
||||
|
||||
return () => {
|
||||
cancelled = true;
|
||||
clearTimeout(hydrationTimer);
|
||||
if (retryTimer) clearTimeout(retryTimer);
|
||||
try { ws?.close(); } catch (_) {}
|
||||
setFrontendUrl(null);
|
||||
setIsNewMode(false);
|
||||
|
||||
@@ -0,0 +1,13 @@
|
||||
import { store } from '@/shared/state/store';
|
||||
|
||||
// A browser an agent is actively working: its guest clicks are the AGENT's, not the user's, so they
|
||||
// must not steal selection, last-interacted targeting (Ctrl+R, zoom, dictation fallback), or z-order.
|
||||
export function isAgentDrivenBrowser(browserId: string): boolean {
|
||||
const st = store.getState();
|
||||
const working = (s?: { status?: string }) => !!s && (s.status === 'running' || s.status === 'waiting_approval');
|
||||
const glow = st.dashboardLayout.glowingBrowserCards[browserId];
|
||||
return (
|
||||
Object.values(st.agents.sessions).some((s) => s.browser_id === browserId && working(s)) ||
|
||||
(!!glow && !glow.fading && working(st.agents.sessions[glow.sourceId]))
|
||||
);
|
||||
}
|
||||
@@ -1,4 +1,11 @@
|
||||
// Native notifications for agent completion; fires only when document is hidden (user switched away).
|
||||
import { store } from '@/shared/state/store';
|
||||
|
||||
function notificationsEnabled(kind: 'agent' | 'workflow'): boolean {
|
||||
const d = store.getState().settings.data as { notify_agent_completion?: boolean; notify_workflow_runs?: boolean };
|
||||
return kind === 'agent' ? d.notify_agent_completion !== false : d.notify_workflow_runs !== false;
|
||||
}
|
||||
|
||||
const FIRED_RECENTLY = new Set<string>();
|
||||
const COOLDOWN_MS = 30_000;
|
||||
|
||||
@@ -25,6 +32,7 @@ export interface AgentCompletionPayload {
|
||||
|
||||
export function notifyAgentCompletion(p: AgentCompletionPayload): void {
|
||||
if (typeof document === 'undefined') return;
|
||||
if (!notificationsEnabled('agent')) return;
|
||||
// Same-window: skip noise (hidden = tab-switched, minimized, or another BrowserWindow in front).
|
||||
if (!document.hidden) return;
|
||||
if (typeof Notification === 'undefined') return;
|
||||
@@ -100,6 +108,7 @@ function workflowBodyFor(p: WorkflowRunNotification): string {
|
||||
|
||||
/** Native OS notification for a finished workflow run. Prefers the Electron main process, which reaches Notification Center even with the window hidden or the renderer backgrounded; the renderer's own Notification API is the browser-only fallback and it only fires when the tab is hidden. */
|
||||
export function notifyWorkflowRun(p: WorkflowRunNotification): void {
|
||||
if (!notificationsEnabled('workflow')) return;
|
||||
const bridge = typeof window !== 'undefined' ? window.openswarm : undefined;
|
||||
if (!bridge?.notify) {
|
||||
notifyAgentCompletion({
|
||||
|
||||
@@ -25,10 +25,13 @@ export const DEFAULT_WORKFLOW_CARD_H = 520;
|
||||
// Open at the same default footprint as a browser/view card so it lands at a comfortable size automatically.
|
||||
export const DEFAULT_WORKFLOWS_HUB_W = DEFAULT_BROWSER_CARD_W;
|
||||
export const DEFAULT_WORKFLOWS_HUB_H = DEFAULT_BROWSER_CARD_H;
|
||||
export const DEFAULT_SETTINGS_CARD_W = 900;
|
||||
export const DEFAULT_SETTINGS_CARD_H = 640;
|
||||
export const DEFAULT_SETTINGS_CARD_W = DEFAULT_BROWSER_CARD_W;
|
||||
export const DEFAULT_SETTINGS_CARD_H = DEFAULT_BROWSER_CARD_H;
|
||||
// The two singleton windows have no card map to key off, so they own these fixed ids everywhere (selection, minimize, z-order).
|
||||
export const SETTINGS_CARD_ID = 'settings';
|
||||
export const MARKETPLACE_CARD_ID = 'marketplace';
|
||||
export const DEFAULT_MARKETPLACE_CARD_W = DEFAULT_BROWSER_CARD_W;
|
||||
export const DEFAULT_MARKETPLACE_CARD_H = DEFAULT_BROWSER_CARD_H;
|
||||
export const WORKFLOWS_HUB_ID = 'workflows-hub';
|
||||
export const EXPANDED_CARD_MIN_H = 620;
|
||||
export const GRID_GAP = 24;
|
||||
@@ -37,7 +40,7 @@ export const WORKFLOW_CARD_GAP = 140;
|
||||
const GRID_ORIGIN = { x: 40, y: 100 };
|
||||
const GRID_COLS_FALLBACK = 4;
|
||||
|
||||
export type CardType = 'agent' | 'view' | 'browser' | 'workflow' | 'workflows-hub' | 'workflows-monitor' | 'settings';
|
||||
export type CardType = 'agent' | 'view' | 'browser' | 'workflow' | 'workflows-hub' | 'workflows-monitor' | 'settings' | 'marketplace';
|
||||
|
||||
export interface CardPosition {
|
||||
session_id: string;
|
||||
@@ -175,6 +178,18 @@ export interface DashboardLayoutState {
|
||||
settingsCard: WorkflowsHubPosition | null;
|
||||
/** Transient: signals Dashboard to pan/zoom to the Settings window on open. */
|
||||
pendingFocusSettingsCard: boolean;
|
||||
/** Transient deep-link: which settings tab to land on; SettingsBody consumes and clears it. */
|
||||
settingsRequestedTab: string | null;
|
||||
/** Marketplace as an on-canvas window (singleton), same lifecycle as the Settings window. */
|
||||
marketplaceCard: WorkflowsHubPosition | null;
|
||||
/** Transient: signals Dashboard to pan/zoom to the Marketplace window on open. */
|
||||
pendingFocusMarketplaceCard: boolean;
|
||||
/** Transient deep-link: which marketplace view to land on; the card consumes and clears it. */
|
||||
marketplaceRequestedTab: string | null;
|
||||
/** Focus-order overrides, keyed by card id (singletons use their literal ids). bringToFront writes ONLY here, so a click never churns card-object identities; effective z = zOrders[id] ?? the base stamped at creation. */
|
||||
zOrders: Record<string, number>;
|
||||
/** Creation-order ledger, oldest first, across all content card types; the trash's no-selection press pops the newest. Persisted with the layout (zOrder can't stand in, it re-bumps on every focus). */
|
||||
creationOrder: string[];
|
||||
}
|
||||
|
||||
export interface WorkflowsRunContext {
|
||||
@@ -216,6 +231,12 @@ const initialState: DashboardLayoutState = {
|
||||
workflowsRunContext: null,
|
||||
settingsCard: null,
|
||||
pendingFocusSettingsCard: false,
|
||||
settingsRequestedTab: null,
|
||||
marketplaceCard: null,
|
||||
pendingFocusMarketplaceCard: false,
|
||||
marketplaceRequestedTab: null,
|
||||
creationOrder: [],
|
||||
zOrders: {},
|
||||
};
|
||||
|
||||
interface LayoutPayload {
|
||||
@@ -225,6 +246,24 @@ interface LayoutPayload {
|
||||
workflowCards: Record<string, WorkflowCardPosition>;
|
||||
workflowsHub: WorkflowsHubPosition | null;
|
||||
expandedSessionIds: string[];
|
||||
creationOrder: string[];
|
||||
// Optional because savers omit it: the save thunk reads the live map from state itself.
|
||||
zOrders?: Record<string, number>;
|
||||
}
|
||||
|
||||
function ledgerAdd(ledger: string[], id: string): void {
|
||||
if (!ledger.includes(id)) ledger.push(id);
|
||||
}
|
||||
|
||||
function ledgerRemove(ledger: string[], id: string): void {
|
||||
const i = ledger.indexOf(id);
|
||||
if (i !== -1) ledger.splice(i, 1);
|
||||
}
|
||||
|
||||
function ledgerRekey(ledger: string[], oldId: string, newId: string): void {
|
||||
const i = ledger.indexOf(oldId);
|
||||
if (i !== -1) ledger[i] = newId;
|
||||
else ledgerAdd(ledger, newId);
|
||||
}
|
||||
|
||||
function generateTabId(): string {
|
||||
@@ -261,6 +300,8 @@ export const fetchLayout = createAsyncThunk(
|
||||
workflowCards: (layout.workflow_cards ?? {}) as Record<string, WorkflowCardPosition>,
|
||||
workflowsHub: (layout.workflows_hub ?? null) as WorkflowsHubPosition | null,
|
||||
expandedSessionIds: (layout.expanded_session_ids ?? []) as string[],
|
||||
creationOrder: (layout.creation_order ?? []) as string[],
|
||||
zOrders: (layout.z_orders ?? {}) as Record<string, number>,
|
||||
} satisfies LayoutPayload;
|
||||
},
|
||||
);
|
||||
@@ -273,8 +314,14 @@ export const saveLayout = createAsyncThunk(
|
||||
'dashboardLayout/save',
|
||||
async (payload: SaveLayoutPayload, { getState }) => {
|
||||
// Never persist a layout this client never successfully loaded; a failed boot fetch otherwise saves the pristine empty store over the server's real layout (the wipe class).
|
||||
const armed = (getState() as { dashboardLayout: { saveArmed: boolean } }).dashboardLayout.saveArmed;
|
||||
if (!armed) return payload;
|
||||
const dl = (getState() as { dashboardLayout: { saveArmed: boolean; zOrders: Record<string, number> } }).dashboardLayout;
|
||||
if (!dl.saveArmed) return payload;
|
||||
// Prune focus overrides to ids that still exist, so removed cards can't grow the map forever.
|
||||
const liveZ: Record<string, number> = {};
|
||||
for (const [zid, z] of Object.entries(dl.zOrders)) {
|
||||
if (payload.cards[zid] || payload.viewCards[zid] || payload.browserCards[zid] || payload.workflowCards[zid]
|
||||
|| zid === 'settings' || zid === 'marketplace' || zid === 'workflows-hub' || zid === 'workflows-monitor') liveZ[zid] = z;
|
||||
}
|
||||
await fetch(`${DASHBOARDS_API}/${payload.dashboardId}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -286,6 +333,8 @@ export const saveLayout = createAsyncThunk(
|
||||
workflow_cards: payload.workflowCards,
|
||||
workflows_hub: payload.workflowsHub,
|
||||
expanded_session_ids: payload.expandedSessionIds,
|
||||
creation_order: payload.creationOrder,
|
||||
z_orders: liveZ,
|
||||
},
|
||||
}),
|
||||
});
|
||||
@@ -338,6 +387,9 @@ function collectOccupiedRects(
|
||||
if (state.settingsCard) {
|
||||
rects.push({ x: state.settingsCard.x, y: state.settingsCard.y, w: state.settingsCard.width, h: state.settingsCard.height });
|
||||
}
|
||||
if (state.marketplaceCard) {
|
||||
rects.push({ x: state.marketplaceCard.x, y: state.marketplaceCard.y, w: state.marketplaceCard.width, h: state.marketplaceCard.height });
|
||||
}
|
||||
return rects;
|
||||
}
|
||||
|
||||
@@ -586,9 +638,13 @@ function addMissingCards<T extends { x: number; y: number; width: number; height
|
||||
|
||||
// One docked surface per chat: the slot is a single rect, so docking a new browser/app releases
|
||||
// whatever was previously docked there (it falls back to its stored free position).
|
||||
function clearOtherDocks(state: { browserCards: Record<string, BrowserCardPosition>; viewCards: Record<string, ViewCardPosition> }, sessionId: string): void {
|
||||
function clearOtherDocks(state: { browserCards: Record<string, BrowserCardPosition>; viewCards: Record<string, ViewCardPosition> }, sessionId: string, keepBrowserId?: string): void {
|
||||
for (const bc of Object.values(state.browserCards)) {
|
||||
if (bc.docked_to === sessionId) bc.docked_to = null;
|
||||
if (bc.docked_to !== sessionId) continue;
|
||||
// The card being (re-)docked must survive its own clear: layout sync re-asserts docks.
|
||||
if (bc.browser_id === keepBrowserId) continue;
|
||||
// Corpse cleanup happens in WebSocketManager via removeBrowserCardCleanly, NOT here: a bare store delete gets resurrected by the backend layout sync as a free card.
|
||||
bc.docked_to = null;
|
||||
}
|
||||
for (const vc of Object.values(state.viewCards)) {
|
||||
if (vc.docked_to === sessionId) vc.docked_to = null;
|
||||
@@ -676,6 +732,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
height,
|
||||
zOrder: state.nextZOrder++,
|
||||
};
|
||||
ledgerAdd(state.creationOrder, sessionId);
|
||||
},
|
||||
|
||||
bringToFront(
|
||||
@@ -683,52 +740,30 @@ const dashboardLayoutSlice = createSlice({
|
||||
action: PayloadAction<{ id: string; type: CardType }>,
|
||||
) {
|
||||
const { id, type } = action.payload;
|
||||
// Compute the current top zOrder across ALL card types so we can short-circuit when the target is already on top. Without this guard, every click on a card (which fires onPointerDownCapture + onClick + onDoubleClick) bumps zOrder and triggers a Redux mutation. That mutation cascades into a re-render that unmounts inputs mid-keystroke, causing the workflow card's title / description / step textareas to lose focus on every click.
|
||||
let maxZ = 0;
|
||||
let currentZ = 0;
|
||||
const tally = (z: number | undefined) => { if (typeof z === 'number' && z > maxZ) maxZ = z; };
|
||||
for (const c of Object.values(state.cards)) tally(c.zOrder);
|
||||
for (const c of Object.values(state.viewCards)) tally(c.zOrder);
|
||||
for (const c of Object.values(state.browserCards)) tally(c.zOrder);
|
||||
for (const c of Object.values(state.workflowCards)) tally(c.zOrder);
|
||||
if (state.workflowsHub) tally(state.workflowsHub.zOrder);
|
||||
if (state.workflowsMonitorCard) tally(state.workflowsMonitorCard.zOrder);
|
||||
if (state.settingsCard) tally(state.settingsCard.zOrder);
|
||||
if (type === 'agent') currentZ = state.cards[id]?.zOrder ?? 0;
|
||||
else if (type === 'view') currentZ = state.viewCards[id]?.zOrder ?? 0;
|
||||
else if (type === 'workflow') currentZ = state.workflowCards[id]?.zOrder ?? 0;
|
||||
else if (type === 'workflows-hub') currentZ = state.workflowsHub?.zOrder ?? 0;
|
||||
else if (type === 'workflows-monitor') currentZ = state.workflowsMonitorCard?.zOrder ?? 0;
|
||||
else if (type === 'settings') currentZ = state.settingsCard?.zOrder ?? 0;
|
||||
else currentZ = state.browserCards[id]?.zOrder ?? 0;
|
||||
if (currentZ >= maxZ) return; // Already on top: no-op.
|
||||
|
||||
const z = state.nextZOrder++;
|
||||
if (type === 'agent') {
|
||||
const card = state.cards[id];
|
||||
if (card) card.zOrder = z;
|
||||
} else if (type === 'view') {
|
||||
const card = state.viewCards[id];
|
||||
if (card) card.zOrder = z;
|
||||
} else if (type === 'workflow') {
|
||||
const card = state.workflowCards[id];
|
||||
if (card) card.zOrder = z;
|
||||
} else if (type === 'workflows-hub') {
|
||||
if (state.workflowsHub) state.workflowsHub.zOrder = z;
|
||||
} else if (type === 'workflows-monitor') {
|
||||
if (state.workflowsMonitorCard) state.workflowsMonitorCard.zOrder = z;
|
||||
} else if (type === 'settings') {
|
||||
if (state.settingsCard) state.settingsCard.zOrder = z;
|
||||
} else {
|
||||
const card = state.browserCards[id];
|
||||
if (card) card.zOrder = z;
|
||||
}
|
||||
// Focus writes ONLY the override map: the old form mutated the card object, which replaced its
|
||||
// dict, re-rendered the controller + tethers + dock + minimap, and armed a layout PUT +
|
||||
// thumbnail capture on every press (CANVAS_LAG_NOTES item 33). The nextZOrder-1 holder is
|
||||
// always the top card, so the already-on-top guard (workflow textarea focus-loss) is one compare.
|
||||
let base = 0;
|
||||
if (type === 'agent') base = state.cards[id]?.zOrder ?? 0;
|
||||
else if (type === 'view') base = state.viewCards[id]?.zOrder ?? 0;
|
||||
else if (type === 'workflow') base = state.workflowCards[id]?.zOrder ?? 0;
|
||||
else if (type === 'workflows-hub') base = state.workflowsHub?.zOrder ?? 0;
|
||||
else if (type === 'workflows-monitor') base = state.workflowsMonitorCard?.zOrder ?? 0;
|
||||
else if (type === 'settings') base = state.settingsCard?.zOrder ?? 0;
|
||||
else if (type === 'marketplace') base = state.marketplaceCard?.zOrder ?? 0;
|
||||
else base = state.browserCards[id]?.zOrder ?? 0;
|
||||
const effective = state.zOrders[id] ?? base;
|
||||
// Zero is the legacy every-card tie (backend-synced cards arrive without z); never short-circuit on it.
|
||||
if (effective > 0 && effective === state.nextZOrder - 1) return;
|
||||
state.zOrders[id] = state.nextZOrder++;
|
||||
},
|
||||
|
||||
removeCard(state, action: PayloadAction<string>) {
|
||||
delete state.cards[action.payload];
|
||||
delete state.tiledCards[action.payload];
|
||||
delete state.minimizedCards[action.payload];
|
||||
ledgerRemove(state.creationOrder, action.payload);
|
||||
},
|
||||
|
||||
reconcileSessions(
|
||||
@@ -745,6 +780,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
// A dead card must never keep owning a tile: an orphaned 'fullscreen' entry hides ALL chrome until reload.
|
||||
delete state.tiledCards[id];
|
||||
delete state.minimizedCards[id];
|
||||
ledgerRemove(state.creationOrder, id);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -768,6 +804,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
zOrder: state.nextZOrder++,
|
||||
};
|
||||
}
|
||||
ledgerAdd(state.creationOrder, id);
|
||||
}
|
||||
},
|
||||
|
||||
@@ -783,7 +820,8 @@ const dashboardLayoutSlice = createSlice({
|
||||
const hub = state.workflowsHub;
|
||||
const mon = state.workflowsMonitorCard;
|
||||
const settings = state.settingsCard;
|
||||
const total = agentCards.length + viewCards.length + bCards.length + wCards.length + (hub ? 1 : 0) + (mon ? 1 : 0) + (settings ? 1 : 0);
|
||||
const market = state.marketplaceCard;
|
||||
const total = agentCards.length + viewCards.length + bCards.length + wCards.length + (hub ? 1 : 0) + (mon ? 1 : 0) + (settings ? 1 : 0) + (market ? 1 : 0);
|
||||
if (total === 0) return;
|
||||
|
||||
const allItems = [
|
||||
@@ -794,6 +832,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
...(hub ? [{ kind: 'workflows-hub' as const, id: 'workflows-hub', x: hub.x, y: hub.y, storedW: hub.width, storedH: hub.height }] : []),
|
||||
...(mon ? [{ kind: 'workflows-monitor' as const, id: 'workflows-monitor', x: mon.x, y: mon.y, storedW: mon.width, storedH: mon.height }] : []),
|
||||
...(settings ? [{ kind: 'settings' as const, id: SETTINGS_CARD_ID, x: settings.x, y: settings.y, storedW: settings.width, storedH: settings.height }] : []),
|
||||
...(market ? [{ kind: 'marketplace' as const, id: MARKETPLACE_CARD_ID, x: market.x, y: market.y, storedW: market.width, storedH: market.height }] : []),
|
||||
];
|
||||
allItems.sort((a, b) => a.y - b.y || a.x - b.x);
|
||||
|
||||
@@ -827,6 +866,8 @@ const dashboardLayoutSlice = createSlice({
|
||||
if (state.workflowsMonitorCard) { state.workflowsMonitorCard.x = pos.x; state.workflowsMonitorCard.y = pos.y; }
|
||||
} else if (item.kind === 'settings') {
|
||||
if (state.settingsCard) { state.settingsCard.x = pos.x; state.settingsCard.y = pos.y; }
|
||||
} else if (item.kind === 'marketplace') {
|
||||
if (state.marketplaceCard) { state.marketplaceCard.x = pos.x; state.marketplaceCard.y = pos.y; }
|
||||
} else {
|
||||
const card = state.browserCards[item.id];
|
||||
if (card) { card.x = pos.x; card.y = pos.y; }
|
||||
@@ -883,6 +924,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
docked_to: (parentSessionId && (clearOtherDocks(state, parentSessionId), parentSessionId)) || null,
|
||||
preview_deferred: previewDeferred || undefined,
|
||||
};
|
||||
ledgerAdd(state.creationOrder, cardKey);
|
||||
state.pendingFocusViewCardId = cardKey;
|
||||
},
|
||||
|
||||
@@ -921,6 +963,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
delete state.viewCards[action.payload];
|
||||
delete state.tiledCards[action.payload];
|
||||
delete state.minimizedCards[action.payload];
|
||||
ledgerRemove(state.creationOrder, action.payload);
|
||||
if (state.activeViewCardId === action.payload) state.activeViewCardId = null;
|
||||
},
|
||||
|
||||
@@ -948,6 +991,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
// Born onto the current dashboard so it shows there and only there, never bleeding onto every dashboard while it waits for the first layout save to tag it.
|
||||
dashboard_id: getLastDashboardId() ?? undefined,
|
||||
};
|
||||
ledgerAdd(state.creationOrder, id);
|
||||
state.pendingFocusBrowserId = id;
|
||||
},
|
||||
|
||||
@@ -973,7 +1017,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
setBrowserDocked(state, action: PayloadAction<{ browserId: string; dockedTo: string | null }>) {
|
||||
const bc = state.browserCards[action.payload.browserId];
|
||||
if (!bc) return;
|
||||
if (action.payload.dockedTo) clearOtherDocks(state, action.payload.dockedTo);
|
||||
if (action.payload.dockedTo) clearOtherDocks(state, action.payload.dockedTo, action.payload.browserId);
|
||||
bc.docked_to = action.payload.dockedTo;
|
||||
},
|
||||
addBrowserCardFromBackend(state, action: PayloadAction<BrowserCardPosition>) {
|
||||
@@ -994,6 +1038,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
// An agent-spawned card must carry its home dashboard or it renders on EVERY dashboard; trust the backend's tag, fall back to the current dashboard so an old/untagged payload can't bleed.
|
||||
dashboard_id: card.dashboard_id ?? getLastDashboardId() ?? undefined,
|
||||
};
|
||||
ledgerAdd(state.creationOrder, card.browser_id);
|
||||
},
|
||||
|
||||
setBrowserCardPosition(
|
||||
@@ -1023,6 +1068,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
delete state.endingBrowserCards[action.payload];
|
||||
delete state.tiledCards[action.payload];
|
||||
delete state.minimizedCards[action.payload];
|
||||
ledgerRemove(state.creationOrder, action.payload);
|
||||
},
|
||||
|
||||
markBrowserCardEnding(
|
||||
@@ -1098,6 +1144,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
zOrder: state.nextZOrder++,
|
||||
source_session_id: sourceSessionId || null,
|
||||
};
|
||||
ledgerAdd(state.creationOrder, workflowId);
|
||||
state.pendingFocusWorkflowId = workflowId;
|
||||
},
|
||||
|
||||
@@ -1127,6 +1174,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
// Rule 7: a dead card must never keep owning a tile; a stale entry poisons every reader of it.
|
||||
delete state.tiledCards[action.payload];
|
||||
delete state.minimizedCards[action.payload];
|
||||
ledgerRemove(state.creationOrder, action.payload);
|
||||
},
|
||||
|
||||
// Rekey draft- id to the server-assigned id without visually hopping the card.
|
||||
@@ -1139,6 +1187,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
if (!card) return;
|
||||
delete state.workflowCards[oldId];
|
||||
state.workflowCards[newId] = { ...card, workflow_id: newId };
|
||||
ledgerRekey(state.creationOrder, oldId, newId);
|
||||
if (state.pendingFocusWorkflowId === oldId) state.pendingFocusWorkflowId = newId;
|
||||
},
|
||||
|
||||
@@ -1263,7 +1312,8 @@ const dashboardLayoutSlice = createSlice({
|
||||
},
|
||||
|
||||
// Settings is an on-canvas window like the Workflows app, not a modal: opening it creates or raises that card and pans to it.
|
||||
openSettingsCard(state, action: PayloadAction<{ expandedSessionIds?: string[] } | undefined>) {
|
||||
openSettingsCard(state, action: PayloadAction<{ tab?: string; expandedSessionIds?: string[] } | undefined>) {
|
||||
state.settingsRequestedTab = action.payload?.tab ?? null;
|
||||
if (state.settingsCard) {
|
||||
state.settingsCard.zOrder = state.nextZOrder++;
|
||||
delete state.minimizedCards[SETTINGS_CARD_ID];
|
||||
@@ -1284,6 +1334,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
|
||||
closeSettingsCard(state) {
|
||||
state.settingsCard = null;
|
||||
state.settingsRequestedTab = null;
|
||||
delete state.minimizedCards[SETTINGS_CARD_ID];
|
||||
delete state.tiledCards[SETTINGS_CARD_ID];
|
||||
state.pendingFocusSettingsCard = false;
|
||||
@@ -1293,6 +1344,10 @@ const dashboardLayoutSlice = createSlice({
|
||||
state.pendingFocusSettingsCard = false;
|
||||
},
|
||||
|
||||
clearSettingsRequestedTab(state) {
|
||||
state.settingsRequestedTab = null;
|
||||
},
|
||||
|
||||
setSettingsCardPosition(state, action: PayloadAction<{ x: number; y: number }>) {
|
||||
if (!state.settingsCard) return;
|
||||
state.settingsCard.x = action.payload.x;
|
||||
@@ -1305,6 +1360,55 @@ const dashboardLayoutSlice = createSlice({
|
||||
state.settingsCard.height = Math.max(460, action.payload.height);
|
||||
},
|
||||
|
||||
// Marketplace mirrors the Settings window: an on-canvas singleton, opened or raised in place.
|
||||
openMarketplaceCard(state, action: PayloadAction<{ tab?: string; expandedSessionIds?: string[] } | undefined>) {
|
||||
state.marketplaceRequestedTab = action.payload?.tab ?? null;
|
||||
if (state.marketplaceCard) {
|
||||
state.marketplaceCard.zOrder = state.nextZOrder++;
|
||||
delete state.minimizedCards[MARKETPLACE_CARD_ID];
|
||||
state.pendingFocusMarketplaceCard = true;
|
||||
return;
|
||||
}
|
||||
const rects = collectOccupiedRects(state, action.payload?.expandedSessionIds);
|
||||
const pos = findOpenGridCell(rects, DEFAULT_MARKETPLACE_CARD_W, DEFAULT_MARKETPLACE_CARD_H);
|
||||
state.marketplaceCard = {
|
||||
x: pos.x,
|
||||
y: pos.y,
|
||||
width: DEFAULT_MARKETPLACE_CARD_W,
|
||||
height: DEFAULT_MARKETPLACE_CARD_H,
|
||||
zOrder: state.nextZOrder++,
|
||||
};
|
||||
state.pendingFocusMarketplaceCard = true;
|
||||
},
|
||||
|
||||
closeMarketplaceCard(state) {
|
||||
state.marketplaceCard = null;
|
||||
state.marketplaceRequestedTab = null;
|
||||
delete state.minimizedCards[MARKETPLACE_CARD_ID];
|
||||
delete state.tiledCards[MARKETPLACE_CARD_ID];
|
||||
state.pendingFocusMarketplaceCard = false;
|
||||
},
|
||||
|
||||
clearPendingFocusMarketplaceCard(state) {
|
||||
state.pendingFocusMarketplaceCard = false;
|
||||
},
|
||||
|
||||
clearMarketplaceRequestedTab(state) {
|
||||
state.marketplaceRequestedTab = null;
|
||||
},
|
||||
|
||||
setMarketplaceCardPosition(state, action: PayloadAction<{ x: number; y: number }>) {
|
||||
if (!state.marketplaceCard) return;
|
||||
state.marketplaceCard.x = action.payload.x;
|
||||
state.marketplaceCard.y = action.payload.y;
|
||||
},
|
||||
|
||||
setMarketplaceCardSize(state, action: PayloadAction<{ width: number; height: number }>) {
|
||||
if (!state.marketplaceCard) return;
|
||||
state.marketplaceCard.width = Math.max(760, action.payload.width);
|
||||
state.marketplaceCard.height = Math.max(520, action.payload.height);
|
||||
},
|
||||
|
||||
pasteBrowserCard(
|
||||
state,
|
||||
action: PayloadAction<{
|
||||
@@ -1563,6 +1667,11 @@ const dashboardLayoutSlice = createSlice({
|
||||
state.settingsCard.x += dx;
|
||||
state.settingsCard.y += dy;
|
||||
}
|
||||
} else if (item.type === 'marketplace') {
|
||||
if (state.marketplaceCard) {
|
||||
state.marketplaceCard.x += dx;
|
||||
state.marketplaceCard.y += dy;
|
||||
}
|
||||
} else {
|
||||
const card = state.browserCards[item.id];
|
||||
if (card) {
|
||||
@@ -1710,6 +1819,10 @@ const dashboardLayoutSlice = createSlice({
|
||||
state.workflowsHub = null;
|
||||
state.settingsCard = null;
|
||||
state.pendingFocusSettingsCard = false;
|
||||
state.settingsRequestedTab = null;
|
||||
state.marketplaceCard = null;
|
||||
state.pendingFocusMarketplaceCard = false;
|
||||
state.marketplaceRequestedTab = null;
|
||||
state.closedCardPositions = {};
|
||||
state.glowingBrowserCards = {};
|
||||
state.glowingAgentCards = {};
|
||||
@@ -1720,6 +1833,8 @@ const dashboardLayoutSlice = createSlice({
|
||||
state.suspendedBrowserCards = keptSuspended;
|
||||
state.endingBrowserCards = {};
|
||||
state.pendingFocusWorkflowId = null;
|
||||
// Per-dashboard, same as the cards it tracks; fetchLayout.fulfilled rebuilds it for the new dashboard.
|
||||
state.creationOrder = [];
|
||||
},
|
||||
|
||||
},
|
||||
@@ -1782,7 +1897,25 @@ const dashboardLayoutSlice = createSlice({
|
||||
if (!w.zOrder) w.zOrder = 0;
|
||||
if (w.zOrder > maxZ) maxZ = w.zOrder;
|
||||
}
|
||||
state.zOrders = { ...state.zOrders, ...(action.payload.zOrders ?? {}) };
|
||||
for (const z of Object.values(state.zOrders)) { if (z > maxZ) maxZ = z; }
|
||||
state.nextZOrder = maxZ + 1;
|
||||
|
||||
// Ledger rebuild: persisted order filtered to live ids, then unledgered survivors by zOrder (legacy layouts, drift). Keep-alive browsers homed on OTHER dashboards stay out: the trash must never delete a card the user can't see.
|
||||
const zOf = (id: string): number =>
|
||||
state.zOrders[id] ?? state.cards[id]?.zOrder ?? state.viewCards[id]?.zOrder ?? state.browserCards[id]?.zOrder ?? state.workflowCards[id]?.zOrder ?? 0;
|
||||
const live = new Set<string>([
|
||||
...Object.keys(state.cards),
|
||||
...Object.keys(state.viewCards),
|
||||
...Object.keys(state.workflowCards),
|
||||
...Object.entries(state.browserCards)
|
||||
.filter(([, bc]) => !bc.dashboard_id || bc.dashboard_id === ownerDashboardId)
|
||||
.map(([id]) => id),
|
||||
]);
|
||||
const persisted = action.payload.creationOrder.filter((id) => live.has(id));
|
||||
const ledgered = new Set(persisted);
|
||||
const rest = [...live].filter((id) => !ledgered.has(id)).sort((a, b) => zOf(a) - zOf(b));
|
||||
state.creationOrder = [...persisted, ...rest];
|
||||
})
|
||||
.addCase(fetchLayout.rejected, (state) => {
|
||||
state.loading = false;
|
||||
@@ -1808,10 +1941,12 @@ const dashboardLayoutSlice = createSlice({
|
||||
const id = payload.sessionId;
|
||||
if (state.cards[id]) delete state.cards[id];
|
||||
if (state.closedCardPositions[id]) delete state.closedCardPositions[id];
|
||||
ledgerRemove(state.creationOrder, id);
|
||||
})
|
||||
.addCase(deleteWorkflowFulfilledAction, (state, action) => {
|
||||
const id = action.payload;
|
||||
if (id && state.workflowCards[id]) delete state.workflowCards[id];
|
||||
if (id) ledgerRemove(state.creationOrder, id);
|
||||
})
|
||||
.addCase(launchAndSendFirstMessage.fulfilled, (state, action) => {
|
||||
const { draftId, session } = action.payload;
|
||||
@@ -1819,6 +1954,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
if (card) {
|
||||
delete state.cards[draftId];
|
||||
state.cards[session.id] = { ...card, session_id: session.id, zOrder: state.nextZOrder++ };
|
||||
ledgerRekey(state.creationOrder, draftId, session.id);
|
||||
}
|
||||
// The zone rides the re-key too: left behind, a tiled draft pops out of its tile AND strands an
|
||||
// entry no reader can ever clear (a stranded 'fullscreen' hides the whole shell until reload).
|
||||
@@ -1925,7 +2061,14 @@ export const {
|
||||
clearPendingFocusWorkflowsHub,
|
||||
openSettingsCard,
|
||||
closeSettingsCard,
|
||||
openMarketplaceCard,
|
||||
closeMarketplaceCard,
|
||||
clearPendingFocusMarketplaceCard,
|
||||
clearMarketplaceRequestedTab,
|
||||
setMarketplaceCardPosition,
|
||||
setMarketplaceCardSize,
|
||||
clearPendingFocusSettingsCard,
|
||||
clearSettingsRequestedTab,
|
||||
setSettingsCardPosition,
|
||||
setSettingsCardSize,
|
||||
recordClosedCard,
|
||||
@@ -1961,7 +2104,8 @@ export const selectFullscreenCardId = (state: { dashboardLayout: DashboardLayout
|
||||
const id = entry[0];
|
||||
// Belt over the reducer hygiene: an entry whose card is gone (any removal path) must not hold the app in fullscreen.
|
||||
const exists = id in s.cards || id in s.viewCards || id in s.browserCards || id in s.workflowCards
|
||||
|| (id === WORKFLOWS_HUB_ID && !!s.workflowsHub) || (id === SETTINGS_CARD_ID && !!s.settingsCard);
|
||||
|| (id === WORKFLOWS_HUB_ID && !!s.workflowsHub) || (id === SETTINGS_CARD_ID && !!s.settingsCard)
|
||||
|| (id === MARKETPLACE_CARD_ID && !!s.marketplaceCard);
|
||||
return exists && !s.minimizedCards[id] ? id : null;
|
||||
};
|
||||
|
||||
|
||||
@@ -35,6 +35,8 @@ export function displayChatTitle(session: AgentSession | null | undefined): stri
|
||||
if (session.name && !isLegacyAutoName(session.name)) {
|
||||
return session.name;
|
||||
}
|
||||
// A spawned sub-agent's first message is its parent's raw instruction; titling the card with it reads as a leak.
|
||||
if (session.parent_session_id) return 'Sub-agent';
|
||||
const firstUserMsg = session.messages?.find((m) => m.role === 'user');
|
||||
const firstUserContent = firstUserMsg && typeof firstUserMsg.content === 'string'
|
||||
? firstUserMsg.content
|
||||
|
||||
@@ -3,23 +3,6 @@ import { API_BASE } from '@/shared/config';
|
||||
|
||||
const SETTINGS_API = `${API_BASE}/settings`;
|
||||
|
||||
export const DEFAULT_SYSTEM_PROMPT =
|
||||
`You are a personal AI assistant running inside OpenSwarm.\n\n` +
|
||||
`## Tool Priority\n` +
|
||||
`When a dedicated MCP tool exists for a task, use it directly. Do not use the browser for things MCP tools can handle.\n` +
|
||||
`Priority order:\n` +
|
||||
`1. MCP tools first (Reddit, Google Workspace, etc.); fastest and most reliable\n` +
|
||||
`2. WebSearch / WebFetch for general web lookups without a dedicated MCP\n` +
|
||||
`3. BrowserAgent only when you need to visually interact with a website, fill forms, or do something no other tool can handle\n\n` +
|
||||
`## Tool Call Style\n` +
|
||||
`Default: do not narrate routine tool calls. Just call the tool.\n` +
|
||||
`Narrate only when it helps: multi-step work, complex problems, or when the user explicitly asks.\n` +
|
||||
`Keep narration brief. Use plain language.\n\n` +
|
||||
`## Interaction Style\n` +
|
||||
`Be direct and action-oriented. Do not ask clarifying questions unless genuinely ambiguous; ` +
|
||||
`make reasonable assumptions and act. If you need to ask, use the AskUserQuestion tool.\n` +
|
||||
`Do not over-explain what you are about to do. Just do it and show the results.`;
|
||||
|
||||
export interface CustomProvider {
|
||||
name: string;
|
||||
base_url: string;
|
||||
@@ -49,6 +32,12 @@ export interface AppSettings {
|
||||
new_agent_shortcut: string;
|
||||
dictation_shortcut?: string | null;
|
||||
dictation_model?: string | null;
|
||||
dictation_dictionary?: string;
|
||||
dictation_sounds?: boolean;
|
||||
memory_enabled?: boolean;
|
||||
dictation_haptics?: boolean;
|
||||
dictation_sound_volume?: number;
|
||||
dictation_disabled_surfaces?: string;
|
||||
anthropic_api_key: string | null;
|
||||
openai_api_key?: string | null;
|
||||
google_api_key?: string | null;
|
||||
@@ -60,6 +49,9 @@ export interface AppSettings {
|
||||
expand_new_chats_in_dashboard: boolean;
|
||||
auto_reveal_sub_agents: boolean;
|
||||
dev_mode: boolean;
|
||||
/** Notification toggles read before firing native notifications. */
|
||||
notify_agent_completion?: boolean;
|
||||
notify_workflow_runs?: boolean;
|
||||
allow_experimental_updates: boolean;
|
||||
/** Managed subscription state; surfaces only when user has subscribed via cloud. */
|
||||
connection_mode?: 'own_key' | 'openswarm-pro' | 'free-trial';
|
||||
@@ -148,9 +140,6 @@ interface SettingsState {
|
||||
/** The first fetch finished, either way. Only the paint gate wants this, so a dead backend shows a
|
||||
* window with an honest error instead of a blank one. */
|
||||
settled: boolean;
|
||||
modalOpen: boolean;
|
||||
/** When non-null, Settings opens to this tab instead of 'general'. */
|
||||
initialTab: string | null;
|
||||
/** In-flight form edits preserved across modal close/reopen; null = synced with `data`. */
|
||||
draft: AppSettings | null;
|
||||
/** Tab the user was on when they closed the modal with unsaved edits. */
|
||||
@@ -160,11 +149,13 @@ interface SettingsState {
|
||||
latestWriteId: string | null;
|
||||
/** False until the boot free-trial mint attempt settles; gates the no-model banner. */
|
||||
freeTrialArmSettled: boolean;
|
||||
/** The backend's shipped default prompt, fetched on demand; null until it lands. */
|
||||
defaultSystemPrompt: string | null;
|
||||
}
|
||||
|
||||
/** Baseline for every required settings field. Spread under any backend payload so an older saved shape that predates a newer field (e.g. new_agent_shortcut) can't surface as undefined and crash a consumer. */
|
||||
export const DEFAULT_SETTINGS: AppSettings = {
|
||||
default_system_prompt: DEFAULT_SYSTEM_PROMPT,
|
||||
default_system_prompt: null,
|
||||
default_folder: null,
|
||||
default_model: 'opus-5',
|
||||
default_mode: 'agent',
|
||||
@@ -178,6 +169,12 @@ export const DEFAULT_SETTINGS: AppSettings = {
|
||||
new_agent_shortcut: 'Meta+l',
|
||||
dictation_shortcut: null,
|
||||
dictation_model: null,
|
||||
dictation_dictionary: '',
|
||||
dictation_sounds: true,
|
||||
memory_enabled: true,
|
||||
dictation_haptics: true,
|
||||
dictation_sound_volume: 0.7,
|
||||
dictation_disabled_surfaces: '',
|
||||
anthropic_api_key: null,
|
||||
browser_homepage: 'https://duckduckgo.com',
|
||||
browser_import_signins: false,
|
||||
@@ -193,12 +190,11 @@ const initialState: SettingsState = {
|
||||
loading: false,
|
||||
loaded: false,
|
||||
settled: false,
|
||||
modalOpen: false,
|
||||
initialTab: null,
|
||||
draft: null,
|
||||
draftTab: null,
|
||||
latestWriteId: null,
|
||||
freeTrialArmSettled: false,
|
||||
defaultSystemPrompt: null,
|
||||
};
|
||||
|
||||
export const fetchSettings = createAsyncThunk('settings/fetch', async () => {
|
||||
@@ -220,6 +216,16 @@ export const updateSettingsPatch = createAsyncThunk(
|
||||
}
|
||||
);
|
||||
|
||||
// The default lives in ONE place (the backend); shipping a frontend copy rotted once already and made Reset show forever.
|
||||
export const fetchDefaultSystemPrompt = createAsyncThunk(
|
||||
'settings/fetchDefaultSystemPrompt',
|
||||
async () => {
|
||||
const res = await fetch(`${SETTINGS_API}/default-system-prompt`);
|
||||
const data = await res.json();
|
||||
return data.default_system_prompt as string;
|
||||
}
|
||||
);
|
||||
|
||||
export const resetSystemPrompt = createAsyncThunk(
|
||||
'settings/resetSystemPrompt',
|
||||
async () => {
|
||||
@@ -313,14 +319,6 @@ const settingsSlice = createSlice({
|
||||
name: 'settings',
|
||||
initialState,
|
||||
reducers: {
|
||||
openSettingsModal(state, action: PayloadAction<string | undefined>) {
|
||||
state.modalOpen = true;
|
||||
state.initialTab = action.payload ?? null;
|
||||
},
|
||||
closeSettingsModal(state) {
|
||||
state.modalOpen = false;
|
||||
state.initialTab = null;
|
||||
},
|
||||
/** Persist in-flight form edits + tab so they survive modal close; clearDraft drops the marker. */
|
||||
setDraft(state, action: PayloadAction<{ form: AppSettings; tab: string }>) {
|
||||
state.draft = action.payload.form;
|
||||
@@ -369,6 +367,9 @@ const settingsSlice = createSlice({
|
||||
state.draft = null;
|
||||
state.draftTab = null;
|
||||
})
|
||||
.addCase(fetchDefaultSystemPrompt.fulfilled, (state, action) => {
|
||||
state.defaultSystemPrompt = action.payload;
|
||||
})
|
||||
.addCase(resetSystemPrompt.fulfilled, (state, action) => {
|
||||
state.latestWriteId = action.meta.requestId;
|
||||
state.data = { ...DEFAULT_SETTINGS, ...action.payload };
|
||||
@@ -382,5 +383,5 @@ const settingsSlice = createSlice({
|
||||
},
|
||||
});
|
||||
|
||||
export const { openSettingsModal, closeSettingsModal, setDraft, clearDraft, markFreeTrialArmSettled } = settingsSlice.actions;
|
||||
export const { setDraft, clearDraft, markFreeTrialArmSettled } = settingsSlice.actions;
|
||||
export default settingsSlice.reducer;
|
||||
|
||||
@@ -16,6 +16,12 @@ export interface Skill {
|
||||
source?: string;
|
||||
folder?: string;
|
||||
version?: string;
|
||||
/** Folder skill with files beyond SKILL.md; the detail page offers a file picker for those. */
|
||||
has_supporting_files?: boolean;
|
||||
/** Detail-page toggle: disabled skills stay installed but leave the agent's skill list. */
|
||||
enabled?: boolean;
|
||||
/** SKILL.md mtime (epoch seconds); the settings table's Last updated column. */
|
||||
updated_at?: number;
|
||||
}
|
||||
|
||||
interface SkillsState {
|
||||
|
||||
@@ -208,9 +208,11 @@ interface State {
|
||||
runControlPending: Record<string, WorkflowRunControlAction>;
|
||||
deleted: Workflow[];
|
||||
deletedLoading: boolean;
|
||||
/** Bumped when a workflow WRITE settles server-side. The optimistic pending update changes the schedule fingerprint BEFORE the PATCH commits, so an occurrences fetch keyed only on the fingerprint reads pre-commit truth and never re-asks; this seq forces one authoritative refetch per settled write. */
|
||||
settleSeq: number;
|
||||
}
|
||||
|
||||
const initialState: State = { items: {}, runs: {}, openCards: {}, loaded: false, loading: false, paused: false, active: [], cloudSmsEnabled: false, allRuns: [], allRunsLoading: false, runningToast: null, noticeToast: null, runControlPending: {}, deleted: [], deletedLoading: false };
|
||||
const initialState: State = { items: {}, runs: {}, openCards: {}, loaded: false, loading: false, paused: false, active: [], cloudSmsEnabled: false, allRuns: [], allRunsLoading: false, runningToast: null, noticeToast: null, runControlPending: {}, deleted: [], deletedLoading: false, settleSeq: 0 };
|
||||
|
||||
function mergeRunIntoState(state: State, r: WorkflowRun) {
|
||||
const arr = state.runs[r.workflow_id] || [];
|
||||
@@ -601,9 +603,9 @@ const slice = createSlice({
|
||||
const cur = state.items[id];
|
||||
if (cur) state.items[id] = { ...cur, ...patch };
|
||||
})
|
||||
.addCase(updateWorkflow.fulfilled, (state, action) => { state.items[action.payload.id] = action.payload; })
|
||||
.addCase(commitDraft.fulfilled, (state, action) => { state.items[action.payload.id] = action.payload; })
|
||||
.addCase(discardDraft.fulfilled, (state, action) => { state.items[action.payload.id] = action.payload; })
|
||||
.addCase(updateWorkflow.fulfilled, (state, action) => { state.items[action.payload.id] = action.payload; state.settleSeq += 1; })
|
||||
.addCase(commitDraft.fulfilled, (state, action) => { state.items[action.payload.id] = action.payload; state.settleSeq += 1; })
|
||||
.addCase(discardDraft.fulfilled, (state, action) => { state.items[action.payload.id] = action.payload; state.settleSeq += 1; })
|
||||
.addCase(deleteWorkflow.rejected, (state, action) => {
|
||||
state.noticeToast = typeof action.payload === 'string' ? action.payload : "Couldn't delete this workflow. Try again in a moment.";
|
||||
})
|
||||
|
||||
@@ -117,6 +117,9 @@ export const ThemeProvider: React.FC<{ children: React.ReactNode }> = ({ childre
|
||||
|
||||
const tokens = useMemo(() => withAccent(mode === 'dark' ? darkTokens : lightTokens, accent, mode), [mode, accent]);
|
||||
|
||||
// The native window was created boot-dark; keep it on the theme's page color so live-resize filler never flashes dark under a light UI.
|
||||
useEffect(() => { void window.openswarm?.setWindowBackground?.(tokens.bg.page); }, [tokens.bg.page]);
|
||||
|
||||
// Stable identities: SettingsLoader's "apply settings.theme" effect lists setMode in its deps, so a setter that changed every render made that effect re-fire on each toggle and re-assert the OLD persisted theme until the debounced save caught up: live theme snapped back for ~900ms = the switch flicker.
|
||||
const toggleMode = useCallback(() => setModeState((m) => (m === 'light' ? 'dark' : 'light')), []);
|
||||
const setMode = useCallback((m: ThemeMode) => setModeState(m), []);
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
import { useEffect, useState } from 'react';
|
||||
|
||||
// The grain PNG with the slider's opacity BAKED INTO its alpha, so grain can ride the wash element
|
||||
// as a second background layer: one raster means a GPU-evicted tile drops wash AND grain together
|
||||
// and falls back to the same flat tint, instead of the grain-only cutoff seam (the ENG-151 family).
|
||||
const cache = new Map<number, string>();
|
||||
let sourceImage: HTMLImageElement | null = null;
|
||||
|
||||
export function useGrainTileUrl(opacity: number): string | null {
|
||||
const key = Math.round(Math.max(0, Math.min(1, opacity)) * 100) / 100;
|
||||
const [url, setUrl] = useState<string | null>(cache.get(key) ?? null);
|
||||
|
||||
useEffect(() => {
|
||||
if (key <= 0) { setUrl(null); return; }
|
||||
const hit = cache.get(key);
|
||||
if (hit) { setUrl(hit); return; }
|
||||
let alive = true;
|
||||
const bake = (img: HTMLImageElement): void => {
|
||||
const cv = document.createElement('canvas');
|
||||
cv.width = img.naturalWidth;
|
||||
cv.height = img.naturalHeight;
|
||||
const ctx = cv.getContext('2d');
|
||||
if (!ctx) return;
|
||||
ctx.globalAlpha = key;
|
||||
ctx.drawImage(img, 0, 0);
|
||||
const baked = `url("${cv.toDataURL('image/png')}")`;
|
||||
cache.set(key, baked);
|
||||
if (alive) setUrl(baked);
|
||||
};
|
||||
if (sourceImage && sourceImage.complete) {
|
||||
bake(sourceImage);
|
||||
} else {
|
||||
const img = sourceImage ?? new Image();
|
||||
sourceImage = img;
|
||||
// addEventListener, never onload=: with two consumers (shell + canvas) the second onload
|
||||
// assignment silently erased the first, and that instance stayed grainless forever.
|
||||
img.addEventListener('load', () => bake(img), { once: true });
|
||||
if (!img.src) img.src = './grain-texture.png';
|
||||
if (img.complete && img.naturalWidth > 0) bake(img);
|
||||
}
|
||||
return () => { alive = false; };
|
||||
}, [key]);
|
||||
|
||||
return key > 0 ? url : null;
|
||||
}
|
||||
@@ -13,6 +13,28 @@ export function washBackgroundUrl(stops: string[], washOpacity: number): string
|
||||
return `url("data:image/svg+xml,${svg.replace(/#/g, '%23').replace(/'/g, '%27')}")`;
|
||||
}
|
||||
|
||||
function mixHex(a: string, b: string, t: number): string {
|
||||
const pa = parseInt(a.slice(1), 16);
|
||||
const pb = parseInt(b.slice(1), 16);
|
||||
const ch = (shift: number): number => Math.round(((pa >> shift) & 0xff) * (1 - t) + ((pb >> shift) & 0xff) * t);
|
||||
return `#${((ch(16) << 16) | (ch(8) << 8) | ch(0)).toString(16).padStart(6, '0')}`;
|
||||
}
|
||||
|
||||
// The canvas wash, pre-blended over the page color so the layer is OPAQUE: identical pixels to the translucent version, but Chromium can then paint the declared background-color for any tile it evicted or hasn't rastered yet, instead of raw page white/black.
|
||||
export function washOpaqueBackgroundUrl(stops: string[], washOpacity: number, pageBg: string): string {
|
||||
const alpha = Math.max(0, Math.min(1, washOpacity));
|
||||
const blended = stops.map((hex) => mixHex(pageBg, hex, alpha));
|
||||
return washBackgroundUrl(blended, 1);
|
||||
}
|
||||
|
||||
// What an evicted/unrastered wash tile should paint as: the wash's mean tint, never raw page color.
|
||||
export function washUnderlayColor(stops: string[], washOpacity: number, pageBg: string): string {
|
||||
const alpha = Math.max(0, Math.min(1, washOpacity));
|
||||
if (stops.length === 0) return pageBg;
|
||||
const mean = stops.reduce((acc, hex, i) => (i === 0 ? hex : mixHex(acc, hex, 1 / (i + 1))), stops[0]);
|
||||
return mixHex(pageBg, mean, alpha);
|
||||
}
|
||||
|
||||
// Stock wallpaper when the user hasn't picked an accent yet: a designed blue-to-cream-to-pink
|
||||
// gradient (contrasting stops), so a fresh install and the onboarding stage never look flat/white.
|
||||
export const DEFAULT_WASH_STOPS = ['#B7CDEA', '#EFE0D2', '#E7BDD1'];
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
import React, { useCallback, useEffect, useRef } from 'react';
|
||||
import { useAppSelector } from '@/shared/hooks';
|
||||
import { useVoiceDictation } from './useVoiceDictation';
|
||||
import { playVoiceCue, configureVoiceCues } from './voiceCues';
|
||||
import { setManualDictionary } from './voiceDictionary';
|
||||
import { VoiceContext } from './voiceContext';
|
||||
import VoiceOverlay from './VoiceOverlay';
|
||||
|
||||
@@ -9,7 +11,19 @@ import VoiceOverlay from './VoiceOverlay';
|
||||
// out-of-sync state. Mounted once near the app root.
|
||||
|
||||
export function VoiceDictationProvider({ children }: { children: React.ReactNode }): React.ReactElement {
|
||||
const { state, lastText, error, pct, feedback, toggle, start, stop, volumeRef } = useVoiceDictation();
|
||||
const { state, lastText, error, pct, feedback, partial, target, toggle, start, stop, cancel, notify, volumeRef } = useVoiceDictation();
|
||||
|
||||
// fn is the dictation key, but macOS may still have its own Globe action bound (emoji picker on a quick tap); say so once.
|
||||
const globeWarnedRef = useRef(false);
|
||||
useEffect(() => {
|
||||
const bridge = window as unknown as { openswarm?: { onVoiceGlobeConflict?: (cb: () => void) => () => void } };
|
||||
const off = bridge.openswarm?.onVoiceGlobeConflict?.(() => {
|
||||
if (globeWarnedRef.current) return;
|
||||
globeWarnedRef.current = true;
|
||||
notify('Tip: set System Settings > Keyboard > "Press Globe key to" to "Do Nothing" so fn only dictates.');
|
||||
});
|
||||
return () => { off?.(); };
|
||||
}, [notify]);
|
||||
const holdMode = useAppSelector((s) => s.settings.data.voice_hold_to_talk ?? true);
|
||||
const dictationShortcut = useAppSelector((s) => s.settings.data.dictation_shortcut ?? null);
|
||||
const dictationModel = useAppSelector((s) => s.settings.data.dictation_model ?? null);
|
||||
@@ -25,17 +39,34 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode
|
||||
useEffect(() => {
|
||||
if (dictationModel) void window.openswarm?.voiceSetModel?.(dictationModel);
|
||||
}, [dictationModel]);
|
||||
|
||||
// Personal glossary rides every decode as a whisper prompt (manual list merged with learned nouns).
|
||||
const dictationDictionary = useAppSelector((s) => s.settings.data.dictation_dictionary ?? '');
|
||||
useEffect(() => {
|
||||
setManualDictionary(dictationDictionary);
|
||||
}, [dictationDictionary]);
|
||||
|
||||
// Cue sounds honor the user's toggle and loudness.
|
||||
const cueSounds = useAppSelector((s) => s.settings.data.dictation_sounds ?? true);
|
||||
const cueVolume = useAppSelector((s) => s.settings.data.dictation_sound_volume ?? 0.7);
|
||||
useEffect(() => {
|
||||
configureVoiceCues(cueSounds, cueVolume);
|
||||
}, [cueSounds, cueVolume]);
|
||||
const stateRef = useRef(state);
|
||||
stateRef.current = state;
|
||||
const heldRef = useRef(false);
|
||||
|
||||
// A quick tap releases before start() flips state to 'recording': the session latches hands-free
|
||||
// (Wispr's lock). The lock cue lands once recording is actually live, confirming the latch.
|
||||
const latchedRef = useRef(false);
|
||||
|
||||
const pressStart = useCallback((): void => {
|
||||
if (holdMode) {
|
||||
// A TAP while recording must stop: the quick tap's press-end fires before the async start
|
||||
// flips state to 'recording', so without this the mic could be started by a click but never
|
||||
// stopped by one.
|
||||
if (stateRef.current === 'recording') { heldRef.current = false; void stop(); return; }
|
||||
if (stateRef.current === 'idle') { heldRef.current = true; void start(true); }
|
||||
if (stateRef.current === 'idle') { heldRef.current = true; latchedRef.current = false; void start(true); }
|
||||
} else {
|
||||
toggle();
|
||||
}
|
||||
@@ -45,9 +76,20 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode
|
||||
if (holdMode && heldRef.current) {
|
||||
heldRef.current = false;
|
||||
if (stateRef.current === 'recording') void stop();
|
||||
else latchedRef.current = true;
|
||||
}
|
||||
}, [holdMode, stop]);
|
||||
|
||||
useEffect(() => {
|
||||
if (state === 'recording' && latchedRef.current) {
|
||||
latchedRef.current = false;
|
||||
const t = setTimeout(() => playVoiceCue('lock'), 160);
|
||||
return () => clearTimeout(t);
|
||||
}
|
||||
if (state === 'idle') latchedRef.current = false;
|
||||
return undefined;
|
||||
}, [state]);
|
||||
|
||||
// Keyboard hotkey channels, matched to what the source can actually see:
|
||||
// voice:hold-down/up come ONLY from main's native uiohook tap (real global key-up, so the keyboard
|
||||
// gets the same hold-vs-toggle press semantics as the mic buttons); voice:toggle comes from the
|
||||
@@ -61,8 +103,22 @@ export function VoiceDictationProvider({ children }: { children: React.ReactNode
|
||||
return () => { offHold?.(); };
|
||||
}, [pressStart, pressEnd]);
|
||||
|
||||
// Wispr grammar: Esc while the mic is hot throws the take away.
|
||||
useEffect(() => {
|
||||
const onKey = (e: KeyboardEvent): void => {
|
||||
if (e.key === 'Escape' && stateRef.current === 'recording') {
|
||||
e.stopPropagation();
|
||||
cancel();
|
||||
}
|
||||
};
|
||||
window.addEventListener('keydown', onKey, true);
|
||||
return () => window.removeEventListener('keydown', onKey, true);
|
||||
}, [cancel]);
|
||||
|
||||
const confirmRecording = useCallback((): void => { void stop(); }, [stop]);
|
||||
|
||||
return (
|
||||
<VoiceContext.Provider value={{ state, lastText, error, pct, feedback, toggle, pressStart, pressEnd, holdMode, volumeRef }}>
|
||||
<VoiceContext.Provider value={{ state, lastText, error, pct, feedback, partial, target, toggle, pressStart, pressEnd, confirmRecording, cancelRecording: cancel, holdMode, volumeRef }}>
|
||||
{children}
|
||||
<VoiceOverlay />
|
||||
</VoiceContext.Provider>
|
||||
|
||||
@@ -1,143 +1,43 @@
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import Box from '@mui/material/Box';
|
||||
import IconButton from '@mui/material/IconButton';
|
||||
import CircularProgress from '@mui/material/CircularProgress';
|
||||
import MicIcon from '@mui/icons-material/Mic';
|
||||
import CloseRoundedIcon from '@mui/icons-material/CloseRounded';
|
||||
import CheckRoundedIcon from '@mui/icons-material/CheckRounded';
|
||||
import ContentPasteRoundedIcon from '@mui/icons-material/ContentPasteRounded';
|
||||
import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined';
|
||||
import { useThemeAccent } from '@/shared/styles/ThemeContext';
|
||||
import { useVoice } from './voiceContext';
|
||||
|
||||
// WhisperFlow-style presence: while the mic is hot, an aurora breathes up from the bottom edge in
|
||||
// the COMPLEMENT of the user's accent (opposite hue = always visible against their canvas, never
|
||||
// camouflaged by it), as a three-hue gradient whose lobes morph with the live mic level. Imperative
|
||||
// rAF writes only (opacity + transform), so 60Hz voice never re-renders React.
|
||||
// WhisperFlow's recording grammar, and nothing else: one small capsule at bottom center holding
|
||||
// cancel (X), a live waveform, and confirm (check). No aurora, no full-screen dressing; the capsule
|
||||
// IS the "mic is hot" signal. Canvas bars are driven imperatively off the mic-level ref so 60Hz
|
||||
// audio never re-renders React.
|
||||
const BAR_COUNT = 18;
|
||||
|
||||
function hexToHue(hex: string): { h: number; s: number; l: number } {
|
||||
const m = /^#?([0-9a-f]{6})$/i.exec(hex.trim());
|
||||
if (!m) return { h: 250, s: 70, l: 60 };
|
||||
const n = parseInt(m[1], 16);
|
||||
const r = ((n >> 16) & 255) / 255, g = ((n >> 8) & 255) / 255, b = (n & 255) / 255;
|
||||
const max = Math.max(r, g, b), min = Math.min(r, g, b);
|
||||
const l = (max + min) / 2;
|
||||
if (max === min) return { h: 0, s: 0, l: l * 100 };
|
||||
const d = max - min;
|
||||
const sat = l > 0.5 ? d / (2 - max - min) : d / (max + min);
|
||||
let h = 0;
|
||||
if (max === r) h = ((g - b) / d + (g < b ? 6 : 0)) * 60;
|
||||
else if (max === g) h = ((b - r) / d + 2) * 60;
|
||||
else h = ((r - g) / d + 4) * 60;
|
||||
return { h, s: sat * 100, l: l * 100 };
|
||||
}
|
||||
|
||||
// Interleaved wisps, like colored gases: neighbors always carry a DIFFERENT hue and overlap ~70%,
|
||||
// so mixture happens everywhere instead of three tinted patches sitting in a row.
|
||||
const LOBES = [
|
||||
{ left: '18%', width: '80vw', height: 200, drift: 0.9, phase: 0.0, hue: 0 },
|
||||
{ left: '34%', width: '95vw', height: 165, drift: 0.65, phase: 2.1, hue: 2 },
|
||||
{ left: '47%', width: '85vw', height: 210, drift: 1.1, phase: 4.2, hue: 1 },
|
||||
{ left: '60%', width: '100vw', height: 170, drift: 0.8, phase: 1.3, hue: 0 },
|
||||
{ left: '73%', width: '85vw', height: 205, drift: 1.2, phase: 3.4, hue: 2 },
|
||||
{ left: '88%', width: '78vw', height: 175, drift: 0.7, phase: 5.1, hue: 1 },
|
||||
];
|
||||
|
||||
const VoiceAurora: React.FC<{ volumeRef: React.MutableRefObject<number> }> = ({ volumeRef }) => {
|
||||
const { accent } = useThemeAccent();
|
||||
const lobeRefs = useRef<Array<HTMLDivElement | null>>([]);
|
||||
const smooth = useRef<number[]>([0, 0, 0]);
|
||||
useEffect(() => {
|
||||
const reduced = window.matchMedia('(prefers-reduced-motion: reduce)').matches;
|
||||
let raf = 0;
|
||||
const tick = (): void => {
|
||||
const v = Math.min(1, volumeRef.current);
|
||||
const t = performance.now() / 1000;
|
||||
LOBES.forEach((lobe, i) => {
|
||||
const el = lobeRefs.current[i];
|
||||
if (!el) return;
|
||||
// Lerp toward the live level so the shape glides instead of jittering; a slow sinusoid keeps
|
||||
// it undulating at steady volume, and volume swells both the wave and the lobe height.
|
||||
smooth.current[i] += (v - smooth.current[i]) * 0.16;
|
||||
const sv = smooth.current[i];
|
||||
const wave = reduced ? 0 : Math.sin(t * lobe.drift + lobe.phase) * (0.08 + sv * 0.1);
|
||||
el.style.opacity = String(0.3 + sv * 0.6);
|
||||
el.style.transform = `translateX(-50%) scale(${1 + wave * 0.6}, ${0.45 + sv * 1.25 + wave})`;
|
||||
});
|
||||
raf = requestAnimationFrame(tick);
|
||||
};
|
||||
raf = requestAnimationFrame(tick);
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}, [volumeRef]);
|
||||
|
||||
// Opposite side of the wheel from the chosen accent, with two flanking hues for a real gradient.
|
||||
const { h } = hexToHue(accent || '#6b62f0');
|
||||
// Wide, softly randomized spread around the complement: three tight neon bands read segmented
|
||||
// (the red/orange/yellow stripe complaint); jittered hues + lower saturation blend like a real
|
||||
// aurora while staying distinguishable. Jitter is per-mount so each dictation feels alive.
|
||||
const jitter = React.useMemo(() => [Math.random() * 16 - 8, Math.random() * 12 - 6, Math.random() * 16 - 8], []);
|
||||
const c0 = `hsl(${(h + 158 + jitter[0] + 360) % 360} 68% 63%)`;
|
||||
const c1 = `hsl(${(h + 180 + jitter[1]) % 360} 72% 60%)`;
|
||||
const c2 = `hsl(${(h + 202 + jitter[2]) % 360} 68% 65%)`;
|
||||
return (
|
||||
<div style={{ position: 'fixed', left: 0, right: 0, bottom: 0, height: 170, zIndex: 2147482999, pointerEvents: 'none', overflow: 'visible' }}>
|
||||
{LOBES.map((lobe, i) => {
|
||||
const col = [c0, c1, c2][lobe.hue];
|
||||
return (
|
||||
<div
|
||||
key={i}
|
||||
ref={(el) => { lobeRefs.current[i] = el; }}
|
||||
style={{
|
||||
position: 'absolute', bottom: -40, left: lobe.left, width: lobe.width, height: lobe.height,
|
||||
transform: 'translateX(-50%) scale(1, 0.45)', transformOrigin: 'bottom center',
|
||||
background: `radial-gradient(ellipse at 50% 100%, ${col} 0%, ${col}00 70%)`,
|
||||
filter: 'blur(52px)', opacity: 0.17, willChange: 'transform, opacity',
|
||||
}}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute', left: 0, right: 0, bottom: 0, height: 90,
|
||||
background: `linear-gradient(to right, ${c1}33, ${c1}55, ${c1}33)`,
|
||||
maskImage: 'linear-gradient(to top, black 0%, transparent 100%)',
|
||||
WebkitMaskImage: 'linear-gradient(to top, black 0%, transparent 100%)',
|
||||
filter: 'blur(10px)',
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
// WhisperFlow's signature: a small capsule drops from the TOP edge while the mic is hot, carrying a
|
||||
// live waveform. Canvas bars driven straight off the mic level ring buffer, imperative rAF only.
|
||||
const BAR_COUNT = 26;
|
||||
|
||||
const VoiceTab: React.FC<{ volumeRef: React.MutableRefObject<number> }> = ({ volumeRef }) => {
|
||||
const Waveform: React.FC<{ volumeRef: React.MutableRefObject<number> }> = ({ volumeRef }) => {
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const history = useRef<number[]>(new Array(BAR_COUNT).fill(0.06));
|
||||
const history = useRef<number[]>(new Array(BAR_COUNT).fill(0.05));
|
||||
useEffect(() => {
|
||||
let raf = 0;
|
||||
let frame = 0;
|
||||
const draw = (): void => {
|
||||
const canvas = canvasRef.current;
|
||||
if (canvas) {
|
||||
const g = canvas.getContext('2d');
|
||||
if (g) {
|
||||
// Shift one bar every other frame so the wave scrolls readably at 60Hz input.
|
||||
frame += 1;
|
||||
if (frame % 2 === 0) {
|
||||
history.current.push(Math.min(1, 0.08 + volumeRef.current * 1.6));
|
||||
history.current.shift();
|
||||
}
|
||||
// Every frame, with a perceptual curve: speech snaps the bars up the way Wispr's do.
|
||||
history.current.push(Math.min(1, Math.pow(volumeRef.current * 2.1, 0.75)));
|
||||
history.current.shift();
|
||||
const w = canvas.width;
|
||||
const h = canvas.height;
|
||||
g.clearRect(0, 0, w, h);
|
||||
const bw = w / BAR_COUNT;
|
||||
for (let i = 0; i < BAR_COUNT; i++) {
|
||||
const level = history.current[i];
|
||||
const bh = Math.max(3, level * (h - 4));
|
||||
const x = i * bw + bw * 0.25;
|
||||
const level = Math.max(0.08, history.current[i]);
|
||||
const bh = Math.max(2, level * (h - 2));
|
||||
const x = i * bw + bw * 0.3;
|
||||
g.fillStyle = `rgba(255,255,255,${0.35 + level * 0.6})`;
|
||||
const bwid = bw * 0.5;
|
||||
const bwid = bw * 0.4;
|
||||
const y = (h - bh) / 2;
|
||||
g.beginPath();
|
||||
g.roundRect(x, y, bwid, bh, bwid / 2);
|
||||
@@ -150,27 +50,75 @@ const VoiceTab: React.FC<{ volumeRef: React.MutableRefObject<number> }> = ({ vol
|
||||
raf = requestAnimationFrame(draw);
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}, [volumeRef]);
|
||||
return (
|
||||
<Box
|
||||
sx={{
|
||||
position: 'fixed', top: 10, left: '50%', zIndex: 2147483001, pointerEvents: 'none',
|
||||
display: 'flex', alignItems: 'center', gap: 1, px: 1.5, py: 0.75, borderRadius: 999,
|
||||
background: 'rgba(18,12,26,0.88)',
|
||||
backdropFilter: 'blur(18px) saturate(150%)', WebkitBackdropFilter: 'blur(18px) saturate(150%)',
|
||||
boxShadow: '0 8px 28px rgba(0,0,0,0.4)',
|
||||
'@keyframes vtab-in': { from: { opacity: 0, transform: 'translate(-50%, -14px)' }, to: { opacity: 1, transform: 'translate(-50%, 0)' } },
|
||||
animation: 'vtab-in 0.22s cubic-bezier(0.2, 0.8, 0.2, 1) both',
|
||||
}}
|
||||
>
|
||||
<MicIcon sx={{ fontSize: 14, color: 'rgba(255,255,255,0.75)' }} />
|
||||
<canvas ref={canvasRef} width={132} height={22} style={{ display: 'block', width: 132, height: 22 }} />
|
||||
</Box>
|
||||
);
|
||||
return <canvas ref={canvasRef} width={88} height={16} style={{ display: 'block', width: 88, height: 16 }} />;
|
||||
};
|
||||
|
||||
// The whole point: dictation must never look like "nothing happened." This floats a small status
|
||||
// card above the composer for every phase (listening, transcribing, downloading the model) and shows
|
||||
// the transcript + whether it was pasted or just copied. Non-interactive, auto-dismisses.
|
||||
const VoiceCapsule: React.FC<{
|
||||
transcribing: boolean;
|
||||
leaving: boolean;
|
||||
volumeRef: React.MutableRefObject<number>;
|
||||
onCancel: () => void;
|
||||
onConfirm: () => void;
|
||||
}> = ({ transcribing, leaving, volumeRef, onCancel, onConfirm }) => (
|
||||
<Box
|
||||
// Buttons must never steal focus: a mousedown that moved focus here would redirect the
|
||||
// transcript away from the field the user is dictating into.
|
||||
onMouseDown={(e) => e.preventDefault()}
|
||||
sx={{
|
||||
// Inline with the Help pill (top 14); the no-drag island is what beats the frameless drag strip here.
|
||||
position: 'fixed', top: 14, left: '50%', transform: 'translateX(-50%)', zIndex: 2147483001,
|
||||
WebkitAppRegion: 'no-drag',
|
||||
display: 'flex', alignItems: 'center', gap: 0.75, pl: 0.5, pr: 0.5, py: 0.5,
|
||||
borderRadius: '14px 14px 22px 22px',
|
||||
background: 'rgba(18,16,24,0.92)',
|
||||
backdropFilter: 'blur(18px) saturate(150%)', WebkitBackdropFilter: 'blur(18px) saturate(150%)',
|
||||
boxShadow: '0 10px 32px rgba(0,0,0,0.45), inset 0 0 0 1px rgba(255,255,255,0.07)',
|
||||
transformOrigin: 'top center',
|
||||
'@keyframes vgloop-in': {
|
||||
'0%': { opacity: 0, transform: 'translate(-50%, -135%) scale(0.9, 0.68)' },
|
||||
'55%': { opacity: 1, transform: 'translate(-50%, 2%) scale(0.955, 1.16)' },
|
||||
'78%': { transform: 'translate(-50%, 0) scale(1.035, 0.93)' },
|
||||
'100%': { transform: 'translate(-50%, 0) scale(1, 1)' },
|
||||
},
|
||||
'@keyframes vgloop-out': {
|
||||
'0%': { opacity: 1, transform: 'translate(-50%, 0) scale(1, 1)' },
|
||||
'30%': { transform: 'translate(-50%, 6%) scale(0.955, 1.1)' },
|
||||
'100%': { opacity: 0, transform: 'translate(-50%, -135%) scale(0.9, 0.68)' },
|
||||
},
|
||||
animation: leaving
|
||||
? 'vgloop-out 0.3s cubic-bezier(0.32, 0.72, 0, 1) both'
|
||||
: 'vgloop-in 0.38s cubic-bezier(0.3, 1.2, 0.4, 1) both',
|
||||
'@media (prefers-reduced-motion: reduce)': { animation: 'none', opacity: leaving ? 0 : 1 },
|
||||
}}
|
||||
>
|
||||
<IconButton
|
||||
size="small"
|
||||
aria-label="Cancel dictation"
|
||||
onClick={onCancel}
|
||||
disabled={transcribing}
|
||||
sx={{ width: 26, height: 26, color: 'rgba(255,255,255,0.75)', bgcolor: 'rgba(255,255,255,0.09)', '&:hover': { bgcolor: 'rgba(255,255,255,0.18)' } }}
|
||||
>
|
||||
<CloseRoundedIcon sx={{ fontSize: 14 }} />
|
||||
</IconButton>
|
||||
<Box sx={{ px: 0.5, display: 'flex', alignItems: 'center', minWidth: 88, justifyContent: 'center' }}>
|
||||
{transcribing
|
||||
? <CircularProgress size={13} thickness={5} sx={{ color: 'rgba(255,255,255,0.7)' }} />
|
||||
: <Waveform volumeRef={volumeRef} />}
|
||||
</Box>
|
||||
<IconButton
|
||||
size="small"
|
||||
aria-label="Finish dictation"
|
||||
onClick={onConfirm}
|
||||
disabled={transcribing}
|
||||
sx={{ width: 26, height: 26, color: '#17141d', bgcolor: 'rgba(255,255,255,0.95)', '&:hover': { bgcolor: '#ffffff' }, '&.Mui-disabled': { bgcolor: 'rgba(255,255,255,0.4)', color: '#17141d' } }}
|
||||
>
|
||||
<CheckRoundedIcon sx={{ fontSize: 15 }} />
|
||||
</IconButton>
|
||||
</Box>
|
||||
);
|
||||
|
||||
// The floating status card above the capsule: live transcript while speaking, download progress,
|
||||
// and terminal feedback (clipboard fallback, errors). Non-interactive, auto-dismisses.
|
||||
const FEEDBACK_MS = 4500;
|
||||
|
||||
function feedbackIcon(icon: string): React.ReactElement {
|
||||
@@ -180,9 +128,78 @@ function feedbackIcon(icon: string): React.ReactElement {
|
||||
return <InfoOutlinedIcon sx={{ fontSize: 15, color: 'rgba(255,255,255,0.8)' }} />;
|
||||
}
|
||||
|
||||
// Live transcript preview: committed phrases solid, the in-flight hypothesis dimmed. Tail-clamped
|
||||
// so the newest words are always the visible ones (openwhispr's preview overlay behavior).
|
||||
const PREVIEW_TAIL_CHARS = 220;
|
||||
|
||||
const LiveTranscript: React.FC<{ committed: string; tentative: string }> = ({ committed, tentative }) => {
|
||||
const total = committed.length + tentative.length;
|
||||
const over = total - PREVIEW_TAIL_CHARS;
|
||||
const shownCommitted = over > 0 ? `…${committed.slice(Math.min(over, committed.length))}` : committed;
|
||||
return (
|
||||
<Box component="span" sx={{ maxWidth: 560, lineHeight: 1.45, whiteSpace: 'normal' }}>
|
||||
<Box component="span">{shownCommitted}</Box>
|
||||
{tentative && (
|
||||
<Box component="span" sx={{ opacity: 0.55 }}>{shownCommitted ? ' ' : ''}{tentative}</Box>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
// Always-there entry point at the top edge, Wispr's idle bar translated to our gloop position:
|
||||
// a sliver that expands on hover into mic + hotkey hint; press semantics match the hotkey.
|
||||
const IdlePill: React.FC<{ onPressStart: () => void; onPressEnd: () => void }> = ({ onPressStart, onPressEnd }) => {
|
||||
const [hover, setHover] = useState(false);
|
||||
return (
|
||||
<Box
|
||||
onMouseEnter={() => setHover(true)}
|
||||
onMouseLeave={() => setHover(false)}
|
||||
onMouseDown={(e) => e.preventDefault()}
|
||||
onPointerDown={onPressStart}
|
||||
onPointerUp={onPressEnd}
|
||||
role="button"
|
||||
aria-label="Start dictation"
|
||||
sx={{
|
||||
position: 'fixed', top: hover ? 14 : 23, left: '50%', transform: 'translateX(-50%)', zIndex: 2147482998,
|
||||
WebkitAppRegion: 'no-drag', cursor: 'pointer',
|
||||
display: 'flex', alignItems: 'center', justifyContent: 'center', gap: 0.75,
|
||||
height: hover ? 24 : 8, minWidth: hover ? 74 : 44, px: hover ? 1.25 : 0,
|
||||
borderRadius: 999,
|
||||
background: hover ? 'rgba(18,16,24,0.92)' : 'rgba(120,116,134,0.28)',
|
||||
boxShadow: hover ? '0 6px 20px rgba(0,0,0,0.35), inset 0 0 0 1px rgba(255,255,255,0.08)' : 'inset 0 0 0 1px rgba(255,255,255,0.14)',
|
||||
transition: 'all 0.18s cubic-bezier(0.32, 0.72, 0, 1)',
|
||||
overflow: 'hidden',
|
||||
}}
|
||||
>
|
||||
{hover && (
|
||||
<>
|
||||
<MicIcon sx={{ fontSize: 13, color: 'rgba(255,255,255,0.85)' }} />
|
||||
<Box component="span" sx={{ color: 'rgba(255,255,255,0.6)', fontSize: '0.6875rem', fontWeight: 700, letterSpacing: '0.04em' }}>fn</Box>
|
||||
</>
|
||||
)}
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
const VoiceOverlay: React.FC = () => {
|
||||
const { state, pct, feedback, volumeRef } = useVoice();
|
||||
const { state, pct, feedback, partial, target, volumeRef, confirmRecording, cancelRecording, pressStart, pressEnd } = useVoice();
|
||||
const [showFeedback, setShowFeedback] = useState(false);
|
||||
// The capsule lingers past its state for one out-animation beat, so it glides back up instead of vanishing.
|
||||
const [capsuleLeaving, setCapsuleLeaving] = useState(false);
|
||||
const capsuleAlive = state === 'recording' || state === 'transcribing';
|
||||
const [capsuleMounted, setCapsuleMounted] = useState(capsuleAlive);
|
||||
|
||||
useEffect(() => {
|
||||
if (capsuleAlive) {
|
||||
setCapsuleMounted(true);
|
||||
setCapsuleLeaving(false);
|
||||
return undefined;
|
||||
}
|
||||
if (!capsuleMounted) return undefined;
|
||||
setCapsuleLeaving(true);
|
||||
const t = setTimeout(() => { setCapsuleMounted(false); setCapsuleLeaving(false); }, 270);
|
||||
return () => clearTimeout(t);
|
||||
}, [capsuleAlive, capsuleMounted]);
|
||||
|
||||
useEffect(() => {
|
||||
if (!feedback) return undefined;
|
||||
@@ -192,21 +209,25 @@ const VoiceOverlay: React.FC = () => {
|
||||
}, [feedback]);
|
||||
|
||||
const live = state !== 'idle';
|
||||
const visible = live || (showFeedback && !!feedback);
|
||||
if (!visible) return null;
|
||||
const aurora = state === 'recording' ? (
|
||||
<>
|
||||
<VoiceAurora volumeRef={volumeRef} />
|
||||
<VoiceTab volumeRef={volumeRef} />
|
||||
</>
|
||||
const desktop = !!(window as unknown as { openswarm?: { voiceTranscribe?: unknown } }).openswarm?.voiceTranscribe;
|
||||
const visible = live || capsuleMounted || (showFeedback && !!feedback);
|
||||
if (!visible) return desktop ? <IdlePill onPressStart={pressStart} onPressEnd={pressEnd} /> : null;
|
||||
|
||||
const capsule = capsuleMounted ? (
|
||||
<VoiceCapsule transcribing={state === 'transcribing'} leaving={capsuleLeaving} volumeRef={volumeRef} onCancel={cancelRecording} onConfirm={confirmRecording} />
|
||||
) : null;
|
||||
|
||||
const hasPartial = !!partial && !!(partial.committed || partial.tentative);
|
||||
let content: React.ReactElement | null;
|
||||
if (state === 'recording') {
|
||||
// The aurora IS the listening indicator; a pill on top of it read as clutter.
|
||||
content = null;
|
||||
} else if (state === 'transcribing') {
|
||||
content = (<><CircularProgress size={13} thickness={5} sx={{ color: 'rgba(255,255,255,0.7)' }} /><span>Transcribing</span></>);
|
||||
if (state === 'recording' || state === 'transcribing') {
|
||||
// Words once there are words; before that, the chip says WHERE they will land.
|
||||
content = hasPartial
|
||||
? <LiveTranscript committed={partial.committed} tentative={partial.tentative} />
|
||||
: (state === 'recording' && target.label
|
||||
? <Box component="span" sx={{ display: 'inline-flex', alignItems: 'center', gap: 0.6, color: 'rgba(255,255,255,0.65)', fontSize: '0.75rem' }}>
|
||||
{'→'} {target.icon && <Box component="img" src={target.icon} alt="" sx={{ width: 13, height: 13, borderRadius: '3px' }} />} {target.label}
|
||||
</Box>
|
||||
: null);
|
||||
} else if (state === 'preparing') {
|
||||
content = (<><CircularProgress size={13} thickness={5} sx={{ color: 'rgba(255,255,255,0.7)' }} /><span>Downloading voice model {pct}%</span></>);
|
||||
} else if (feedback) {
|
||||
@@ -219,17 +240,18 @@ const VoiceOverlay: React.FC = () => {
|
||||
</>
|
||||
);
|
||||
} else {
|
||||
return null;
|
||||
content = null;
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
{aurora}
|
||||
{capsule}
|
||||
{content && (
|
||||
<Box
|
||||
sx={{
|
||||
// Hangs just under the droplet so the live words read as its tail.
|
||||
position: 'fixed',
|
||||
bottom: 84,
|
||||
top: 64,
|
||||
left: '50%',
|
||||
transform: 'translateX(-50%)',
|
||||
zIndex: 2147483000,
|
||||
@@ -248,7 +270,7 @@ const VoiceOverlay: React.FC = () => {
|
||||
color: 'rgba(255,255,255,0.92)',
|
||||
fontSize: '0.8125rem',
|
||||
fontWeight: 500,
|
||||
'@keyframes vin': { from: { opacity: 0, transform: 'translate(-50%, 6px)' }, to: { opacity: 1, transform: 'translate(-50%, 0)' } },
|
||||
'@keyframes vin': { from: { opacity: 0, transform: 'translate(-50%, -6px)' }, to: { opacity: 1, transform: 'translate(-50%, 0)' } },
|
||||
animation: 'vin 0.16s ease-out',
|
||||
}}
|
||||
>
|
||||
|
||||
@@ -0,0 +1,44 @@
|
||||
// One capture interface, two implementations: the AudioWorklet (off-thread, 50ms Int16 chunks with
|
||||
// a drain handshake) with the deprecated ScriptProcessor as the always-works fallback. Capture
|
||||
// plumbing failing must degrade dictation, never kill it: the first worklet rollout died on CSP
|
||||
// (blob: module URLs are script-src, not worker-src) and the generic catch read "mic broken".
|
||||
|
||||
export interface CaptureNode {
|
||||
node: AudioNode;
|
||||
// Drains any buffered tail; resolves when the last samples have been delivered.
|
||||
requestFlush: () => Promise<void>;
|
||||
}
|
||||
|
||||
export async function createCaptureNode(ctx: AudioContext, onPcm: (pcm: Int16Array) => void): Promise<CaptureNode> {
|
||||
try {
|
||||
await ctx.audioWorklet.addModule(new URL('pcm-worklet.js', window.location.href).toString());
|
||||
const node = new AudioWorkletNode(ctx, 'pcm-streaming-processor');
|
||||
let flushResolve: () => void = () => {};
|
||||
const flushed = new Promise<void>((resolve) => { flushResolve = resolve; });
|
||||
node.port.onmessage = (e: MessageEvent): void => {
|
||||
if (e.data === 'flushed') { flushResolve(); return; }
|
||||
onPcm(new Int16Array(e.data as ArrayBuffer));
|
||||
};
|
||||
return {
|
||||
node,
|
||||
requestFlush: () => {
|
||||
try { node.port.postMessage('stop'); } catch (_) { /* node already gone */ }
|
||||
return Promise.race([flushed, new Promise<void>((resolve) => window.setTimeout(resolve, 1000))]);
|
||||
},
|
||||
};
|
||||
} catch (err) {
|
||||
console.warn('[voice] worklet capture unavailable, using ScriptProcessor:', err instanceof Error ? err.message : err);
|
||||
const node = ctx.createScriptProcessor(4096, 1, 1);
|
||||
node.onaudioprocess = (e): void => {
|
||||
const data = e.inputBuffer.getChannelData(0);
|
||||
const i16 = new Int16Array(data.length);
|
||||
for (let i = 0; i < data.length; i++) {
|
||||
const s = Math.max(-1, Math.min(1, data[i]));
|
||||
i16[i] = s < 0 ? s * 0x8000 : s * 0x7fff;
|
||||
}
|
||||
onPcm(i16);
|
||||
};
|
||||
// ScriptProcessor delivers synchronously per 4096-frame block; there is no tail to drain.
|
||||
return { node, requestFlush: () => Promise.resolve() };
|
||||
}
|
||||
}
|
||||
@@ -4,7 +4,7 @@ import { getWebview } from '@/shared/browserRegistry';
|
||||
// Dictation lands where the user's cursor actually is, like every real dictation tool: a focused
|
||||
// in-app field gets the text typed in (undo-friendly, fires React input events), a focused browser
|
||||
// card forwards into the guest page's field, anything else falls back to the OS-level paste.
|
||||
export type InjectTarget = 'field' | 'webview' | null;
|
||||
export type InjectTarget = 'field' | 'webview' | 'composer' | null;
|
||||
|
||||
export function injectAtFocus(text: string): InjectTarget {
|
||||
const active = document.activeElement as HTMLElement | null;
|
||||
@@ -40,5 +40,7 @@ export function injectAtFocus(text: string): InjectTarget {
|
||||
try { wv.focus?.(); void wv.insertText(text); return 'webview'; } catch { /* fall through */ }
|
||||
}
|
||||
}
|
||||
return null;
|
||||
// No cursor anywhere: open the dashboard composer with the transcript typed in. Words are never dropped.
|
||||
window.dispatchEvent(new CustomEvent('openswarm:dictation-fallback', { detail: { text } }));
|
||||
return 'composer';
|
||||
}
|
||||
|
||||
@@ -1,22 +1,38 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { API_BASE } from '@/shared/config';
|
||||
import { getLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
import { encodeWav, VOICE_SAMPLE_RATE } from './encodeWav';
|
||||
import { playVoiceCue } from './voiceCues';
|
||||
import { injectAtFocus } from './injectAtFocus';
|
||||
import { createSilenceDetector } from './createSilenceDetector';
|
||||
import { pushDictation } from './voiceHistory';
|
||||
import { learnFromTranscript, isDictionaryEcho } from './voiceDictionary';
|
||||
import { getFocusedSurfaceHost, surfaceDisabled } from './voiceSurface';
|
||||
import { store } from '@/shared/state/store';
|
||||
import { createCaptureNode } from './createCaptureNode';
|
||||
|
||||
export type VoiceState = 'idle' | 'recording' | 'transcribing' | 'preparing';
|
||||
|
||||
// Live transcript preview: committed phrases are decoded once and never rewritten; the tentative
|
||||
// tail is the latest hypothesis for the phrase still being spoken.
|
||||
export interface VoicePartial {
|
||||
committed: string;
|
||||
tentative: string;
|
||||
}
|
||||
|
||||
// WhisperFlow-style push-to-dictate: toggle recording (global hotkey or a mic), speak, and the
|
||||
// transcribed text is pasted into whatever field has focus. Capture is 16kHz mono PCM so it feeds
|
||||
// whisper.cpp with no server-side resample. The recording path can only be proven with a real mic;
|
||||
// transcribed text is pasted into whatever field has focus. Capture is 16kHz mono PCM off the audio
|
||||
// thread (AudioWorklet) so it feeds whisper.cpp with no server-side resample, and the same chunks
|
||||
// stream to main for live partials. The recording path can only be proven with a real mic;
|
||||
// the encode -> transcribe -> inject half is exercised by the encoder round-trip test.
|
||||
interface Recorder {
|
||||
ctx: AudioContext;
|
||||
stream: MediaStream;
|
||||
node: ScriptProcessorNode;
|
||||
node: AudioNode;
|
||||
requestFlush: () => Promise<void>;
|
||||
source: MediaStreamAudioSourceNode;
|
||||
chunks: Float32Array[];
|
||||
streaming: boolean;
|
||||
}
|
||||
|
||||
// One object per terminal outcome so the overlay's effect always re-fires (new identity every time).
|
||||
@@ -27,6 +43,32 @@ export interface VoiceFeedback {
|
||||
at: number;
|
||||
}
|
||||
|
||||
// Where the transcript will land RIGHT NOW, in the user's words plus the surface's own icon;
|
||||
// mirrors injectAtFocus's tiers so the chip never promises a destination injection would not pick.
|
||||
export interface InjectTargetInfo {
|
||||
label: string;
|
||||
icon: string | null;
|
||||
}
|
||||
|
||||
function browserTargetInfo(): InjectTargetInfo {
|
||||
const browserId = getLastInteractedBrowser();
|
||||
const card = browserId ? store.getState().dashboardLayout.browserCards[browserId] : undefined;
|
||||
const tab = card?.tabs?.find((t) => t.id === card.activeTabId);
|
||||
const host = (() => { try { return tab?.url ? new URL(tab.url).hostname : null; } catch { return null; } })();
|
||||
return { label: host || 'browser page', icon: tab?.favicon || null };
|
||||
}
|
||||
|
||||
export function describeInjectTarget(): InjectTargetInfo {
|
||||
const a = document.activeElement as HTMLElement | null;
|
||||
if (a && (a.tagName === 'INPUT' || a.tagName === 'TEXTAREA' || a.isContentEditable)) {
|
||||
const hint = a.getAttribute('placeholder') || a.getAttribute('aria-label');
|
||||
return { label: hint ? hint.slice(0, 30) : 'text field', icon: null };
|
||||
}
|
||||
if (a && a.tagName === 'WEBVIEW') return browserTargetInfo();
|
||||
if (getLastInteractedBrowser()) return browserTargetInfo();
|
||||
return { label: 'chat composer', icon: null };
|
||||
}
|
||||
|
||||
// Context hint for the polisher: what the user is dictating into (a field label, a page title), so
|
||||
// names and jargon spell right. Never page CONTENT, just the one-line "where".
|
||||
function dictationContext(): string {
|
||||
@@ -60,6 +102,9 @@ export function useVoiceDictation() {
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [pct, setPct] = useState<number>(0);
|
||||
const [feedback, setFeedback] = useState<VoiceFeedback | null>(null);
|
||||
const [partial, setPartial] = useState<VoicePartial | null>(null);
|
||||
const [target, setTarget] = useState<InjectTargetInfo>({ label: '', icon: null });
|
||||
const partialSeqRef = useRef<number>(0);
|
||||
const recRef = useRef<Recorder | null>(null);
|
||||
const stateRef = useRef<VoiceState>('idle');
|
||||
// Live mic level (0..1) for the aurora; a ref, not state, so 60Hz visuals never re-render React.
|
||||
@@ -103,33 +148,66 @@ export function useVoiceDictation() {
|
||||
const start = useCallback(async (hold = false): Promise<void> => {
|
||||
if (stateRef.current !== 'idle') return;
|
||||
if (!window.openswarm?.voiceTranscribe) { setError('desktop-only'); return; } // no Electron bridge = web build
|
||||
// Per-surface disable: the user marked this site as no-dictation; refuse loudly, never silently.
|
||||
const disabledList = store.getState().settings.data.dictation_disabled_surfaces ?? '';
|
||||
if (disabledList) {
|
||||
const host = getFocusedSurfaceHost();
|
||||
if (surfaceDisabled(disabledList, host)) {
|
||||
setFeedback({ tone: 'warn', icon: 'mic', text: `Dictation is off for ${host}. Change it in Settings > Interface.`, at: Date.now() });
|
||||
return;
|
||||
}
|
||||
}
|
||||
setError(null);
|
||||
// Warm on the DOWN edge, before the mic prompt, so the model load overlaps the user starting to speak.
|
||||
void window.openswarm?.voiceWarmup?.();
|
||||
setPartial(null);
|
||||
partialSeqRef.current = 0;
|
||||
let stream: MediaStream | null = null;
|
||||
let ctx: AudioContext | null = null;
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } });
|
||||
const ctx = new AudioContext({ sampleRate: VOICE_SAMPLE_RATE });
|
||||
// Fire the OS mic prompt through the main process first: a packaged hardened-runtime build denies renderer getUserMedia outright until TCC granted (the prod dictation-dead cause, ENG-103).
|
||||
const micOk = await (window.openswarm as any)?.voiceRequestMicAccess?.() ?? true;
|
||||
if (micOk === false) { setError('mic-denied'); return; }
|
||||
// autoGainControl OFF is the OSS-dictation consensus (Chromium's hidden AGC rides the mic and
|
||||
// garbles levels mid-utterance; none of the five surveyed shipping apps allow any AGC), and
|
||||
// noiseSuppression smears speech whisper handles better raw. Echo cancellation stays: we play
|
||||
// cues out the speakers while the mic is hot.
|
||||
stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: false, autoGainControl: false } });
|
||||
ctx = new AudioContext({ sampleRate: VOICE_SAMPLE_RATE });
|
||||
const source = ctx.createMediaStreamSource(stream);
|
||||
const node = ctx.createScriptProcessor(4096, 1, 1);
|
||||
const chunks: Float32Array[] = [];
|
||||
const endpointer = hold ? null : createSilenceDetector(ctx.sampleRate);
|
||||
node.onaudioprocess = (e): void => {
|
||||
const data = e.inputBuffer.getChannelData(0);
|
||||
chunks.push(new Float32Array(data));
|
||||
// RMS per chunk drives the aurora; smoothed so it breathes instead of flickering.
|
||||
const streamRes = await window.openswarm?.voiceStreamStart?.();
|
||||
const streaming = streamRes?.ok === true;
|
||||
// No AGC: boosting "quiet" speech clipped NORMAL speech into distortion and wrecked accuracy
|
||||
// (Eric's garbled-history report). Whisper handles real levels fine; a whisper-mode boost can
|
||||
// only come back as an opt-in with a proper peak limiter, never inline on the hot path.
|
||||
const capture = await createCaptureNode(ctx, (i16) => {
|
||||
if (streaming) window.openswarm?.voiceStreamChunk?.(i16.buffer as ArrayBuffer);
|
||||
const data = new Float32Array(i16.length);
|
||||
for (let i = 0; i < i16.length; i++) data[i] = i16[i] / 0x8000;
|
||||
chunks.push(data);
|
||||
// RMS per chunk drives the waveform; fast attack + slower release, like a real peak meter,
|
||||
// so the bars snap to speech instead of lagging behind it.
|
||||
let sum = 0;
|
||||
for (let i = 0; i < data.length; i += 8) sum += data[i] * data[i];
|
||||
const rms = Math.sqrt(sum / (data.length / 8));
|
||||
volumeRef.current = volumeRef.current * 0.7 + Math.min(1, rms * 6) * 0.3;
|
||||
const level = Math.min(1, rms * 7);
|
||||
volumeRef.current = level > volumeRef.current
|
||||
? volumeRef.current * 0.35 + level * 0.65
|
||||
: volumeRef.current * 0.78 + level * 0.22;
|
||||
if (endpointer && endpointer.push(data) !== 'listening') void stopRef.current?.();
|
||||
};
|
||||
source.connect(node);
|
||||
node.connect(ctx.destination);
|
||||
recRef.current = { ctx, stream, node, source, chunks };
|
||||
});
|
||||
source.connect(capture.node);
|
||||
capture.node.connect(ctx.destination);
|
||||
recRef.current = { ctx, stream, node: capture.node, requestFlush: capture.requestFlush, source, chunks, streaming };
|
||||
setState('recording');
|
||||
playVoiceCue('start');
|
||||
void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('generic');
|
||||
// Warm the model the moment recording begins so transcription is instant on stop.
|
||||
void window.openswarm?.voiceWarmup?.();
|
||||
if (store.getState().settings.data.dictation_haptics ?? true) void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('generic');
|
||||
} catch (err) {
|
||||
// Release whatever was acquired before the failure, or the OS mic indicator stays lit forever on a half-started session.
|
||||
try { stream?.getTracks().forEach((t) => t.stop()); } catch { /* already dead */ }
|
||||
try { void ctx?.close(); } catch { /* already closed */ }
|
||||
const msg = err instanceof Error ? err.message : 'mic-unavailable';
|
||||
setError(msg);
|
||||
const denied = /NotAllowed|Permission|denied/i.test(msg);
|
||||
@@ -140,14 +218,70 @@ export function useVoiceDictation() {
|
||||
|
||||
const stop = useCallback(async (): Promise<void> => {
|
||||
if (stateRef.current !== 'recording') return;
|
||||
const rec = recRef.current;
|
||||
const streaming = rec?.streaming === true;
|
||||
// Drain the capture's last partial buffer before teardown (worklet flush carries its own 1s watchdog).
|
||||
if (rec) await rec.requestFlush();
|
||||
const samples = teardown();
|
||||
playVoiceCue('stop');
|
||||
void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('alignment');
|
||||
if (store.getState().settings.data.dictation_haptics ?? true) void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('alignment');
|
||||
setState('transcribing');
|
||||
try {
|
||||
if (!samples || samples.length < VOICE_SAMPLE_RATE * 0.2) { setState('idle'); return; } // < 0.2s = a misfire
|
||||
const wav = encodeWav(samples);
|
||||
const res = await window.openswarm?.voiceTranscribe?.(wav);
|
||||
if (!samples || samples.length < VOICE_SAMPLE_RATE * 0.2) { // < 0.2s = a misfire
|
||||
if (streaming) window.openswarm?.voiceStreamCancel?.();
|
||||
setPartial(null);
|
||||
setState('idle');
|
||||
return;
|
||||
}
|
||||
// Accuracy first (Eric's call): phrases decoded in isolation lose whisper's cross-phrase
|
||||
// context, which read as "sometimes fine, sometimes off". The FULL clip is always decoded at
|
||||
// stop and wins; the streamed assembly only stands in if the batch decode fails outright.
|
||||
let streamedFallback: string | null = null;
|
||||
if (streaming) {
|
||||
const sres = await window.openswarm?.voiceStreamStop?.();
|
||||
if (sres?.ok && sres.text) streamedFallback = sres.text;
|
||||
}
|
||||
let res: { ok: boolean; text?: string; error?: string } | undefined;
|
||||
{
|
||||
// OpenWhispr's window gate: whole-clip RMS misses a clip that is silence plus one cough, so
|
||||
// judge 100ms windows; no window with real speech energy means no decode, ever.
|
||||
const win = Math.round(VOICE_SAMPLE_RATE * 0.1);
|
||||
let peakRms = 0;
|
||||
let speechWindow = false;
|
||||
for (let off = 0; off + win <= samples.length; off += win) {
|
||||
let sumSq = 0;
|
||||
let peak = 0;
|
||||
for (let i = off; i < off + win; i += 2) { const v = samples[i]; sumSq += v * v; if (Math.abs(v) > peak) peak = Math.abs(v); }
|
||||
const wr = Math.sqrt(sumSq / (win / 2));
|
||||
if (wr > peakRms) peakRms = wr;
|
||||
if (wr >= 0.003 && peak >= 0.02) speechWindow = true;
|
||||
}
|
||||
if (peakRms < 0.002 || (!speechWindow && peakRms < 0.006)) {
|
||||
res = { ok: true, text: '' };
|
||||
} else {
|
||||
// whisper.cpp asserts on sub-second buffers; zero-pad instead of rejecting (FluidVoice).
|
||||
const padded = samples.length < VOICE_SAMPLE_RATE
|
||||
? (() => { const b = new Float32Array(VOICE_SAMPLE_RATE); b.set(samples); return b; })()
|
||||
: samples;
|
||||
const wav = encodeWav(padded);
|
||||
res = await window.openswarm?.voiceTranscribe?.(wav);
|
||||
if ((!res || !res.ok || !res.text) && streamedFallback) res = { ok: true, text: streamedFallback };
|
||||
}
|
||||
}
|
||||
// Whisper captions non-speech in brackets/parens ("[ Background sounds ]", "(laughs)") and marks speaker turns with ">>"; those are annotations, not dictation.
|
||||
if (res?.ok && res.text) res = { ok: true, text: res.text.replace(/\[[^\]]*\]|\([^)]*\)|\*[^*]*\*|(?:^|\s)>>\s?/g, ' ').replace(/\s+/g, ' ').trim() };
|
||||
// Stutter collapse (Handy): three or more consecutive identical words are one decode loop, not speech.
|
||||
if (res?.ok && res.text) res = { ok: true, text: res.text.replace(/\b([A-Za-z']+)(\s+\1\b){2,}/gi, '$1') };
|
||||
// Glossary echo (OpenWhispr): a transcript that is mostly the dictionary read back is the prompt leaking, not dictation.
|
||||
if (res?.ok && res.text && isDictionaryEcho(res.text)) res = { ok: true, text: '' };
|
||||
// A transcript with no letter or digit in it is a hallucination artifact (the lone comma), not dictation.
|
||||
if (res?.ok && res.text && !/[\p{L}\p{N}]/u.test(res.text)) res = { ok: true, text: '' };
|
||||
// Wispr's command grammar, v1: saying only "scratch that" (or "delete/cancel that") throws the take away.
|
||||
if (res?.ok && res.text && /^(scratch|delete|cancel) that[.!?]?$/i.test(res.text.trim())) {
|
||||
setFeedback({ tone: 'ok', icon: 'check', text: 'Scratched.', at: Date.now() });
|
||||
setState('idle');
|
||||
return;
|
||||
}
|
||||
if (res?.ok && res.text) {
|
||||
// WhisperFlow-style cleanup: punctuation + filler words via the cheap aux tier, fail-open to
|
||||
// the raw transcript on any error/timeout so dictation never breaks with the aux down.
|
||||
@@ -163,6 +297,8 @@ export function useVoiceDictation() {
|
||||
// Success is silent: the text landing at the cursor IS the feedback. Only the clipboard
|
||||
// fallback still speaks, because the user has to act (paste) to get the text.
|
||||
const target = injectAtFocus(text);
|
||||
pushDictation(text, target || 'clipboard');
|
||||
learnFromTranscript(text);
|
||||
if (!target) {
|
||||
const inj = await window.openswarm?.voiceInject?.(text);
|
||||
if (!inj?.pasted) setFeedback({ tone: 'ok', icon: 'clipboard', text: `${text} (copied, press Cmd+V)`, at: Date.now() });
|
||||
@@ -189,6 +325,16 @@ export function useVoiceDictation() {
|
||||
|
||||
stopRef.current = stop;
|
||||
|
||||
// The capsule's X: throw the take away. No transcription, no cue, straight back to idle.
|
||||
const cancel = useCallback((): void => {
|
||||
if (stateRef.current !== 'recording') return;
|
||||
const rec = recRef.current;
|
||||
if (rec?.streaming) window.openswarm?.voiceStreamCancel?.();
|
||||
teardown();
|
||||
setPartial(null);
|
||||
setState('idle');
|
||||
}, [teardown]);
|
||||
|
||||
const toggle = useCallback((): void => {
|
||||
if (stateRef.current === 'recording') void stop();
|
||||
else if (stateRef.current === 'idle') void start();
|
||||
@@ -200,8 +346,45 @@ export function useVoiceDictation() {
|
||||
return () => { off?.(); };
|
||||
}, [toggle]);
|
||||
|
||||
// Live partials from the main-process streaming session; the seq guard drops anything a stale
|
||||
// session (previous recording's in-flight decode) manages to emit after a new one started.
|
||||
useEffect(() => {
|
||||
const off = window.openswarm?.onVoicePartial?.((p) => {
|
||||
if (p.seq <= partialSeqRef.current) return;
|
||||
partialSeqRef.current = p.seq;
|
||||
if (stateRef.current === 'recording' || stateRef.current === 'transcribing') {
|
||||
setPartial({ committed: p.committed, tentative: p.tentative });
|
||||
}
|
||||
});
|
||||
return () => { off?.(); };
|
||||
}, []);
|
||||
|
||||
// The preview belongs to a live session only; any terminal state clears it in one place.
|
||||
useEffect(() => {
|
||||
if (state === 'idle' || state === 'preparing') setPartial(null);
|
||||
}, [state]);
|
||||
|
||||
// The target chip tracks focus LIVE while recording: clicking into a field mid-dictation retargets
|
||||
// injection (by design), and the chip must tell that truth as it happens.
|
||||
useEffect(() => {
|
||||
if (state !== 'recording') { setTarget({ label: '', icon: null }); return undefined; }
|
||||
setTarget(describeInjectTarget());
|
||||
const onFocus = (): void => setTarget(describeInjectTarget());
|
||||
window.addEventListener('focusin', onFocus, true);
|
||||
window.addEventListener('focusout', onFocus, true);
|
||||
return () => {
|
||||
window.removeEventListener('focusin', onFocus, true);
|
||||
window.removeEventListener('focusout', onFocus, true);
|
||||
};
|
||||
}, [state]);
|
||||
|
||||
// A dangling recorder (unmount mid-capture) must release the mic.
|
||||
useEffect(() => () => { teardown(); }, [teardown]);
|
||||
|
||||
return { state, lastText, error, pct, feedback, toggle, start, stop, volumeRef };
|
||||
// One-line notices from outside the session flow (Globe-key conflict, permission hints) ride the same bubble.
|
||||
const notify = useCallback((text: string): void => {
|
||||
setFeedback({ tone: 'warn', icon: 'info', text, at: Date.now() });
|
||||
}, []);
|
||||
|
||||
return { state, lastText, error, pct, feedback, partial, target, toggle, start, stop, cancel, notify, volumeRef };
|
||||
}
|
||||
|
||||
@@ -1,5 +1,5 @@
|
||||
import React, { createContext, useContext } from 'react';
|
||||
import { VoiceState, VoiceFeedback } from './useVoiceDictation';
|
||||
import { VoiceState, VoiceFeedback, VoicePartial } from './useVoiceDictation';
|
||||
|
||||
// The context lives below both the provider and the overlay so neither imports the other
|
||||
// (VoiceDictationContext renders VoiceOverlay; both reach down here instead of sideways).
|
||||
@@ -9,17 +9,23 @@ export interface VoiceContextValue {
|
||||
error: string | null;
|
||||
pct: number;
|
||||
feedback: VoiceFeedback | null;
|
||||
partial: VoicePartial | null;
|
||||
// Where the transcript will land right now, in user words plus the surface's icon.
|
||||
target: { label: string; icon: string | null };
|
||||
toggle: () => void;
|
||||
// Mic-button press semantics that respect the hold/toggle setting: press starts (or toggles),
|
||||
// release stops only in hold mode. Buttons wire onPointerDown/Up to these and stay mode-agnostic.
|
||||
pressStart: () => void;
|
||||
pressEnd: () => void;
|
||||
// The recording capsule's two endings: keep the take (transcribe + inject) or throw it away.
|
||||
confirmRecording: () => void;
|
||||
cancelRecording: () => void;
|
||||
holdMode: boolean;
|
||||
volumeRef: React.MutableRefObject<number>;
|
||||
}
|
||||
|
||||
const NOOP_REF = { current: 0 };
|
||||
const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, feedback: null, toggle: () => {}, pressStart: () => {}, pressEnd: () => {}, holdMode: true, volumeRef: NOOP_REF };
|
||||
const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, feedback: null, partial: null, target: { label: '', icon: null }, toggle: () => {}, pressStart: () => {}, pressEnd: () => {}, confirmRecording: () => {}, cancelRecording: () => {}, holdMode: true, volumeRef: NOOP_REF };
|
||||
|
||||
export const VoiceContext = createContext<VoiceContextValue>(NOOP);
|
||||
|
||||
|
||||
File diff suppressed because one or more lines are too long
@@ -1,46 +1,48 @@
|
||||
// WhisperFlow-grade start/stop chimes, synthesized in WebAudio so they ship weightless and always
|
||||
// match: a soft two-note rise on start ("I'm listening"), the mirrored fall on stop. Sine + gentle
|
||||
// lowpass + fast attack / exponential release = crisp but soothing, never a system beep.
|
||||
import { VOICE_CUE_START, VOICE_CUE_STOP, VOICE_CUE_LOCK } from './voiceCueSounds';
|
||||
|
||||
let ctx: AudioContext | null = null;
|
||||
// Eric picked these from Google's Material product sound set after a bake-off against Wispr Flow's
|
||||
// real cues; the synth versions never survived an ear test. Files are embedded data URIs (see
|
||||
// voiceCueSounds.ts), pre-instantiated so playback is instant on the press. Start and stop taps plus the
|
||||
// hands-free lock mark; Eric explicitly cut the text-landed chime.
|
||||
|
||||
function ensureCtx(): AudioContext | null {
|
||||
type CueKind = 'start' | 'stop' | 'lock';
|
||||
|
||||
// Pushed from Settings (dictation_sounds / dictation_sound_volume); defaults match the shipped feel.
|
||||
let cueEnabled = true;
|
||||
let cueVolume = 0.7;
|
||||
|
||||
export function configureVoiceCues(enabled: boolean, volume: number): void {
|
||||
cueEnabled = enabled;
|
||||
cueVolume = Math.min(1, Math.max(0, volume));
|
||||
for (const kind of Object.keys(p_players) as CueKind[]) {
|
||||
const a = p_players[kind];
|
||||
if (a) a.volume = cueVolume;
|
||||
}
|
||||
}
|
||||
|
||||
const SOURCES: Record<CueKind, string> = {
|
||||
start: VOICE_CUE_START,
|
||||
stop: VOICE_CUE_STOP,
|
||||
lock: VOICE_CUE_LOCK,
|
||||
};
|
||||
|
||||
const p_players: Partial<Record<CueKind, HTMLAudioElement>> = {};
|
||||
|
||||
function player(kind: CueKind): HTMLAudioElement {
|
||||
let a = p_players[kind];
|
||||
if (!a) {
|
||||
a = new Audio(SOURCES[kind]);
|
||||
a.volume = cueVolume;
|
||||
p_players[kind] = a;
|
||||
}
|
||||
return a;
|
||||
}
|
||||
|
||||
export function playVoiceCue(kind: CueKind): void {
|
||||
if (!cueEnabled) return;
|
||||
try {
|
||||
if (!ctx || ctx.state === 'closed') ctx = new AudioContext();
|
||||
if (ctx.state === 'suspended') void ctx.resume();
|
||||
return ctx;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function blip(ac: AudioContext, freq: number, at: number, dur: number, peak: number): void {
|
||||
const osc = ac.createOscillator();
|
||||
const gain = ac.createGain();
|
||||
const lp = ac.createBiquadFilter();
|
||||
lp.type = 'lowpass';
|
||||
lp.frequency.value = 2400;
|
||||
osc.type = 'sine';
|
||||
osc.frequency.setValueAtTime(freq, at);
|
||||
gain.gain.setValueAtTime(0, at);
|
||||
gain.gain.linearRampToValueAtTime(peak, at + 0.012);
|
||||
gain.gain.exponentialRampToValueAtTime(0.0004, at + dur);
|
||||
osc.connect(lp);
|
||||
lp.connect(gain);
|
||||
gain.connect(ac.destination);
|
||||
osc.start(at);
|
||||
osc.stop(at + dur + 0.02);
|
||||
}
|
||||
|
||||
export function playVoiceCue(kind: 'start' | 'stop'): void {
|
||||
const ac = ensureCtx();
|
||||
if (!ac) return;
|
||||
const t = ac.currentTime + 0.01;
|
||||
if (kind === 'start') {
|
||||
blip(ac, 587, t, 0.16, 0.055);
|
||||
blip(ac, 880, t + 0.085, 0.2, 0.05);
|
||||
} else {
|
||||
blip(ac, 880, t, 0.14, 0.045);
|
||||
blip(ac, 587, t + 0.075, 0.22, 0.05);
|
||||
}
|
||||
const a = player(kind);
|
||||
a.currentTime = 0;
|
||||
void a.play();
|
||||
} catch { /* a missing audio device must never break dictation */ }
|
||||
}
|
||||
|
||||
@@ -0,0 +1,74 @@
|
||||
// The glossary that rides every whisper decode: the user's manual list merged with proper nouns
|
||||
// LEARNED from their own dictations (a capitalized word that keeps showing up is a name worth
|
||||
// biasing toward). All local; main receives one merged comma list via voiceSetDictionary.
|
||||
|
||||
// v2: the v1 store learned junk from garbled transcripts and fed it BACK into decoding (a
|
||||
// degradation loop); new key orphans it, and the merge bar is higher.
|
||||
const LEARNED_KEY = 'osw-dictation-learned-v2';
|
||||
const LEARNED_CAP = 40;
|
||||
const MERGE_TOP = 12;
|
||||
|
||||
// Words that start sentences get capitalized for free; only mid-sentence capitals count as names.
|
||||
const NOUN_RE = /(?<![.!?]\s)(?<!^)\b([A-Z][a-zA-Z]{2,}(?:'s)?)\b/g;
|
||||
const COMMON = new Set(['The', 'This', 'That', 'What', 'When', 'Where', 'Which', 'And', 'But', 'For', 'Not', 'You', 'Your', 'They', 'Their', 'There', 'Then', 'Also', 'Okay', 'Yes', 'Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']);
|
||||
|
||||
let manual = '';
|
||||
|
||||
function readLearned(): Record<string, number> {
|
||||
try {
|
||||
const parsed: unknown = JSON.parse(localStorage.getItem(LEARNED_KEY) || '{}');
|
||||
return parsed && typeof parsed === 'object' ? (parsed as Record<string, number>) : {};
|
||||
} catch {
|
||||
return {};
|
||||
}
|
||||
}
|
||||
|
||||
function pushMerged(): void {
|
||||
const counts = readLearned();
|
||||
const learned = Object.entries(counts)
|
||||
.filter(([w, n]) => n >= 3 && w.length >= 4)
|
||||
.sort((a, b) => b[1] - a[1])
|
||||
.slice(0, MERGE_TOP)
|
||||
.map(([w]) => w);
|
||||
const manualWords = manual.split(',').map((w) => w.trim()).filter(Boolean);
|
||||
const merged = [...new Set([...manualWords, ...learned])].join(', ');
|
||||
lastPushedMerged = merged;
|
||||
const bridge = window as unknown as { openswarm?: { voiceSetDictionary?: (words: string) => void } };
|
||||
bridge.openswarm?.voiceSetDictionary?.(merged);
|
||||
}
|
||||
|
||||
export function setManualDictionary(words: string): void {
|
||||
manual = words || '';
|
||||
pushMerged();
|
||||
}
|
||||
|
||||
let lastPushedMerged = '';
|
||||
|
||||
// OpenWhispr's echo test: the model sometimes reads the glossary prompt back as the "transcript".
|
||||
// Mostly-dictionary words (90%) covering most of the dictionary (70%) = the prompt leaked.
|
||||
export function isDictionaryEcho(text: string): boolean {
|
||||
const dictWords = new Set(lastPushedMerged.toLowerCase().split(/[,\s]+/).filter(Boolean));
|
||||
if (dictWords.size === 0) return false;
|
||||
const words = text.toLowerCase().split(/\s+/).map((w) => w.replace(/[^a-z']/g, '')).filter(Boolean);
|
||||
if (words.length === 0) return false;
|
||||
const unique = [...new Set(words)];
|
||||
const fromDict = unique.filter((w) => dictWords.has(w)).length;
|
||||
const coverage = [...dictWords].filter((w) => words.includes(w)).length / dictWords.size;
|
||||
return fromDict / unique.length >= 0.9 && coverage >= 0.7;
|
||||
}
|
||||
|
||||
export function learnFromTranscript(text: string): void {
|
||||
try {
|
||||
const counts = readLearned();
|
||||
for (const m of text.matchAll(NOUN_RE)) {
|
||||
const w = m[1].replace(/'s$/, '');
|
||||
if (COMMON.has(w)) continue;
|
||||
counts[w] = (counts[w] || 0) + 1;
|
||||
}
|
||||
const trimmed = Object.fromEntries(
|
||||
Object.entries(counts).sort((a, b) => b[1] - a[1]).slice(0, LEARNED_CAP),
|
||||
);
|
||||
localStorage.setItem(LEARNED_KEY, JSON.stringify(trimmed));
|
||||
pushMerged();
|
||||
} catch { /* learning is a bonus, never a blocker */ }
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
// The last N dictations, local-only (localStorage): a transcript that landed somewhere wrong or got
|
||||
// overwritten is recoverable without re-speaking it. Never synced, never sent anywhere.
|
||||
|
||||
export interface DictationHistoryEntry {
|
||||
text: string;
|
||||
at: number;
|
||||
target: string;
|
||||
}
|
||||
|
||||
const KEY = 'osw-dictation-history';
|
||||
const CAP = 20;
|
||||
|
||||
export function readDictationHistory(): DictationHistoryEntry[] {
|
||||
try {
|
||||
const raw = localStorage.getItem(KEY);
|
||||
const parsed: unknown = raw ? JSON.parse(raw) : [];
|
||||
if (!Array.isArray(parsed)) return [];
|
||||
return parsed.filter((e): e is DictationHistoryEntry =>
|
||||
!!e && typeof e === 'object' && typeof (e as DictationHistoryEntry).text === 'string' && typeof (e as DictationHistoryEntry).at === 'number');
|
||||
} catch {
|
||||
return [];
|
||||
}
|
||||
}
|
||||
|
||||
export function pushDictation(text: string, target: string): void {
|
||||
try {
|
||||
const next = [{ text, at: Date.now(), target }, ...readDictationHistory()].slice(0, CAP);
|
||||
localStorage.setItem(KEY, JSON.stringify(next));
|
||||
} catch { /* quota or private mode; history is a convenience, never a blocker */ }
|
||||
}
|
||||
|
||||
export function clearDictationHistory(): void {
|
||||
try { localStorage.removeItem(KEY); } catch { /* same */ }
|
||||
}
|
||||
@@ -0,0 +1,31 @@
|
||||
import { store } from '@/shared/state/store';
|
||||
import { getLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
import { getWebview } from '@/shared/browserRegistry';
|
||||
|
||||
// Which browser surface dictation would land in right now, as a hostname; null when the target is
|
||||
// an in-app field. Mirrors injectAtFocus's browser tiers so the disable list judges the same
|
||||
// destination injection would pick.
|
||||
export function getFocusedSurfaceHost(): string | null {
|
||||
const active = document.activeElement as HTMLElement | null;
|
||||
let browserId: string | null = null;
|
||||
if (active && active.tagName === 'WEBVIEW') {
|
||||
const cards = store.getState().dashboardLayout.browserCards;
|
||||
for (const id of Object.keys(cards)) {
|
||||
if (getWebview(id) === (active as unknown)) { browserId = id; break; }
|
||||
}
|
||||
}
|
||||
if (!browserId) browserId = getLastInteractedBrowser();
|
||||
if (!browserId) return null;
|
||||
const card = store.getState().dashboardLayout.browserCards[browserId];
|
||||
const url = card?.tabs?.find((t) => t.id === card.activeTabId)?.url || card?.url;
|
||||
if (!url) return null;
|
||||
try { return new URL(url).hostname || null; } catch { return null; }
|
||||
}
|
||||
|
||||
// "slack.com, docs.google.com" -> does the focused surface match any entry (exact or suffix).
|
||||
export function surfaceDisabled(disabledList: string, host: string | null): boolean {
|
||||
if (!host) return false;
|
||||
const entries = disabledList.split(',').map((s) => s.trim().toLowerCase()).filter(Boolean);
|
||||
const h = host.toLowerCase();
|
||||
return entries.some((e) => h === e || h.endsWith(`.${e}`));
|
||||
}
|
||||
@@ -121,6 +121,9 @@ class WebSocketManager {
|
||||
WebSocketManager._flushTimer = setTimeout(WebSocketManager._flushMessages, 250);
|
||||
}
|
||||
|
||||
// First moment a flush was deferred for a live drag; bounds the damming so a stuck drag class can't buffer forever.
|
||||
private static _dragDeferredAt = 0;
|
||||
|
||||
private static _flushMessages = () => {
|
||||
if (!WebSocketManager._flushScheduled) return; // the other trigger already drained this batch
|
||||
WebSocketManager._flushScheduled = false;
|
||||
@@ -129,6 +132,18 @@ class WebSocketManager {
|
||||
WebSocketManager._flushTimer = null;
|
||||
}
|
||||
if (WebSocketManager._messageQueue.length === 0) return;
|
||||
// A live card drag owns the main thread: WS-driven renders mid-drag are what made dragging a
|
||||
// working agent feel laggy, and nobody reads streaming tokens while holding a card. Buffer until
|
||||
// the pointer settles, hard-capped by time and queue depth.
|
||||
if (document.body.classList.contains('dashboard-marquee-active')) {
|
||||
if (!WebSocketManager._dragDeferredAt) WebSocketManager._dragDeferredAt = Date.now();
|
||||
if (Date.now() - WebSocketManager._dragDeferredAt < 2000 && WebSocketManager._messageQueue.length < 500) {
|
||||
WebSocketManager._flushScheduled = true;
|
||||
WebSocketManager._flushTimer = setTimeout(WebSocketManager._flushMessages, 100);
|
||||
return;
|
||||
}
|
||||
}
|
||||
WebSocketManager._dragDeferredAt = 0;
|
||||
const batch = WebSocketManager._messageQueue;
|
||||
WebSocketManager._messageQueue = [];
|
||||
// unstable_batchedUpdates collapses all dispatches inside the callback into a single React render. Available in React 17; React 18's automatic batching covers this too, but explicit wrap remains correct in both and protects against future batching-context changes.
|
||||
@@ -409,6 +424,17 @@ class WebSocketManager {
|
||||
if (data.status === 'running' && session_id) {
|
||||
store.dispatch(trackAgentNotification(session_id));
|
||||
}
|
||||
// Native OS notification when an agent finishes while the user is elsewhere: workflows already had this; long chat tasks deserve the same "it's done" tap on both platforms. Sub-agents stay silent (their parent's finish is the story).
|
||||
if (data.status === 'completed' && session_id && document.hidden) {
|
||||
const p_sess2 = data.session ?? store.getState().agents.sessions[session_id];
|
||||
if (p_sess2 && !p_sess2.parent_session_id && p_sess2.mode !== 'browser-agent') {
|
||||
void (window as any).openswarm?.notify?.({
|
||||
title: 'Agent finished',
|
||||
body: (p_sess2.name && p_sess2.name !== 'Untitled' ? p_sess2.name : 'Your task is done.').slice(0, 200),
|
||||
deepLink: `openswarm://session/${session_id}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// An AppAgent driving an app card announces itself only via this status event (no card_added like browsers), so light the app card here. Keyed by the parent chat like browser glows, so the same terminal fade below clears it.
|
||||
const p_sess = data.session;
|
||||
@@ -817,6 +843,12 @@ class WebSocketManager {
|
||||
let glowLabel = 'Use Browser';
|
||||
if (parentCard) {
|
||||
pos = placeBrowserBesideChat(layoutState, parentCard, parentId, browserCard.width, browserCard.height, browserCard.browser_id);
|
||||
// A replacement browser buries its predecessor: the agent spins a fresh card when a tab dies, and the displaced spawned sibling must be torn down through the FULL path (a bare store delete gets resurrected by the backend layout sync as a free stacked window).
|
||||
for (const old of Object.values(layoutState.browserCards)) {
|
||||
if (old.browser_id !== browserCard.browser_id && old.spawned_by === parentId && !old.keep_open) {
|
||||
void import('@/shared/browserTeardown').then(({ removeBrowserCardCleanly }) => removeBrowserCardCleanly(old.browser_id, store.dispatch));
|
||||
}
|
||||
}
|
||||
// Default home is INSIDE the chat: the card overlays the chat's dock slot while the
|
||||
// chat is expanded; the beside-chat spot stays the undock/collapse fallback.
|
||||
store.dispatch(setBrowserDocked({ browserId: browserCard.browser_id, dockedTo: parentId }));
|
||||
|
||||
Reference in New Issue
Block a user