mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-30 13:34:50 +02:00
[eric] merge eric/onboarding into eric/dev; version 1.6.0
This commit is contained in:
ciregenz
@@ -166,6 +166,20 @@ function p_cleanSummaryText(s: string): string {
|
||||
|
||||
export function getMcpInputSummary(input: any, action?: string, serverSlug?: string): string {
|
||||
if (!input || typeof input !== 'object') return '';
|
||||
// Web tools read as SOURCES, not payload dumps: a quoted query for a search, a bare domain for a
|
||||
// page read. Lives here so every caller (compact rows, labels, approvals) gets the clean form.
|
||||
const p_act = (action || '').toLowerCase();
|
||||
if (p_act === 'websearch') {
|
||||
const q = input.query ?? input.search_term;
|
||||
if (typeof q === 'string' && q.trim()) return `"${p_cleanSummaryText(q.trim())}"`;
|
||||
}
|
||||
if (p_act === 'webfetch' && typeof input.url === 'string' && input.url) {
|
||||
try {
|
||||
return new URL(input.url).host.replace(/^www\./, '');
|
||||
} catch {
|
||||
return input.url.replace(/^https?:\/\//, '').split(/[/?#]/)[0];
|
||||
}
|
||||
}
|
||||
if (serverSlug === 'openswarm-schedule' || (action && getWorkflowToolLabel(action))) {
|
||||
const workflowSummary = compactWorkflowSchedule(input);
|
||||
if (workflowSummary) return workflowSummary;
|
||||
|
||||
@@ -0,0 +1,45 @@
|
||||
// Dev-only responsiveness watchdog: regressions show up as numbers in the console, not vibes.
|
||||
interface LongTaskStats {
|
||||
count: number;
|
||||
totalMs: number;
|
||||
worstMs: number;
|
||||
reset: () => void;
|
||||
}
|
||||
|
||||
declare global {
|
||||
interface Window {
|
||||
OSW_PERF?: LongTaskStats;
|
||||
}
|
||||
}
|
||||
|
||||
const NOISY_TASK_MS = 150;
|
||||
|
||||
export function startLongTaskLogger(): void {
|
||||
if (process.env.NODE_ENV === 'production') return;
|
||||
if (typeof PerformanceObserver === 'undefined') return;
|
||||
if (!PerformanceObserver.supportedEntryTypes?.includes('longtask')) return;
|
||||
|
||||
const stats: LongTaskStats = {
|
||||
count: 0,
|
||||
totalMs: 0,
|
||||
worstMs: 0,
|
||||
reset: () => {
|
||||
stats.count = 0;
|
||||
stats.totalMs = 0;
|
||||
stats.worstMs = 0;
|
||||
},
|
||||
};
|
||||
window.OSW_PERF = stats;
|
||||
|
||||
const observer = new PerformanceObserver((list: PerformanceObserverEntryList) => {
|
||||
for (const entry of list.getEntries()) {
|
||||
stats.count += 1;
|
||||
stats.totalMs += entry.duration;
|
||||
if (entry.duration > stats.worstMs) stats.worstMs = entry.duration;
|
||||
if (entry.duration >= NOISY_TASK_MS) {
|
||||
console.debug(`[perf] long task ${Math.round(entry.duration)}ms (total ${stats.count} tasks, ${Math.round(stats.totalMs)}ms)`);
|
||||
}
|
||||
}
|
||||
});
|
||||
observer.observe({ entryTypes: ['longtask'] });
|
||||
}
|
||||
@@ -0,0 +1,28 @@
|
||||
// Run: node --test frontend/src/shared/providerUsage.test.ts
|
||||
import { test } from 'node:test';
|
||||
import assert from 'node:assert/strict';
|
||||
import { summarizeUsage, type ProviderUsage } from './providerUsage.ts';
|
||||
|
||||
test('failed or null read yields empty summary', () => {
|
||||
assert.equal(summarizeUsage(null), '');
|
||||
assert.equal(summarizeUsage({ ok: false, total: 0, titles: [], memories: [] }), '');
|
||||
});
|
||||
|
||||
test('summary leads with memory, names scale, and lists recent topics', () => {
|
||||
const u: ProviderUsage = {
|
||||
ok: true,
|
||||
total: 812,
|
||||
titles: ['Swift concurrency', 'Deadlift form check', 'Tax on RSUs'],
|
||||
memories: ['Has an Akita', 'Squats 495'],
|
||||
};
|
||||
const s = summarizeUsage(u);
|
||||
assert.match(s, /812 past AI conversations/);
|
||||
assert.match(s, /Has an Akita; Squats 495/);
|
||||
assert.match(s, /Swift concurrency; Deadlift form check; Tax on RSUs/);
|
||||
});
|
||||
|
||||
test('summary is hard-capped so a heavy user never ships a wall of PII', () => {
|
||||
const titles = Array.from({ length: 1000 }, (_, i) => `conversation number ${i} about a very specific topic`);
|
||||
const s = summarizeUsage({ ok: true, total: 1000, titles, memories: [] });
|
||||
assert.ok(s.length <= 4000, `summary length ${s.length} exceeded 4000`);
|
||||
});
|
||||
@@ -0,0 +1,41 @@
|
||||
// The user's provider chat history is read offscreen in the main process (see electron/usageHarvest.js), which owns the injected script + the partition session. This module holds only the shared shape + the pure summarizer that turns the raw read into the compact profile block prep sees. The raw read is dropped after; only this summary travels.
|
||||
|
||||
export type UsageProvider = 'codex' | 'claude' | 'gemini';
|
||||
|
||||
export interface ProviderUsage {
|
||||
ok: boolean;
|
||||
total: number;
|
||||
// We stop fetching titles early (prep only needs ~150), so total is a floor when this is set.
|
||||
capped?: boolean;
|
||||
titles: string[];
|
||||
memories: string[];
|
||||
// FULL text of the most recent few conversations (their real asks + the exchange). The payload:
|
||||
// titles are the vague label, this is the substance a clustering pass turns into a real profile.
|
||||
convos?: { title: string; text: string }[];
|
||||
}
|
||||
|
||||
// ~130K chars is ~32K tokens: a generous budget for the full-convo block, since a downstream clustering
|
||||
// pass distills it before the reveal (so a couple cents, not a bloated final prompt).
|
||||
const TOTAL_CONVO_CHARS = 130000;
|
||||
|
||||
// Turn the raw read into the profile block for prep: memory facts, the scale, recent titles for breadth,
|
||||
// then the FULL recent conversations (the payload). Bounded so even a heavy user can't ship a wall of PII;
|
||||
// the backend then distills it to a tight "who is this person" profile.
|
||||
export function summarizeUsage(u: ProviderUsage | null): string {
|
||||
if (!u || !u.ok) return '';
|
||||
const parts: string[] = [];
|
||||
if (u.total > 0) parts.push(`They have ${u.total}${u.capped ? '+' : ''} past AI conversations.`);
|
||||
if (u.memories.length > 0) parts.push('Facts their AI remembers about them: ' + u.memories.join('; '));
|
||||
if (u.titles.length > 0) parts.push('Recent conversation titles (breadth): ' + u.titles.slice(0, 150).join('; '));
|
||||
if (u.convos && u.convos.length > 0) {
|
||||
const block: string[] = [];
|
||||
let used = 0;
|
||||
for (const cv of u.convos) {
|
||||
if (used + cv.text.length > TOTAL_CONVO_CHARS) break;
|
||||
block.push(cv.text);
|
||||
used += cv.text.length;
|
||||
}
|
||||
if (block.length) parts.push('Full text of their most recent conversations (their real asks + the exchange):\n\n' + block.join('\n\n---\n\n'));
|
||||
}
|
||||
return parts.join('\n');
|
||||
}
|
||||
@@ -1,13 +1,13 @@
|
||||
import { Search, Hammer, Globe, Plug } from 'lucide-react';
|
||||
import { Search, Hammer, Globe, CalendarClock } from 'lucide-react';
|
||||
import type { LucideIcon } from 'lucide-react';
|
||||
|
||||
// Two-level starters shared by the empty-state and the first-run welcome chat: pick a category, then a concrete prompt. Chosen to SHOWCASE what only OpenSwarm can do, and to feel PERSONAL: the agents can see the user's own computer/files, drive the browser, plug into their apps (MCPs), build real apps, and run agents in parallel, none of which a plain chatbot can do out of the box. Many prompts deliberately touch the user's own stuff so it matters to them. One-click-runnable (no [placeholders]); reads plainly for a non-dev. All run as a normal agent; the 'build' category (target 'app-builder') just prefills the composer in the welcome chat instead of auto-sending, since the agent builds the app in-place (it calls CreateApp and the live card drops on the canvas).
|
||||
// Two-level starters shared by the empty-state and the first-run welcome chat: pick a category, then a concrete prompt. Chosen to SHOWCASE what only OpenSwarm can do, and to feel PERSONAL: the agents can see the user's own computer/files, drive the browser, build real apps, and run recurring routines on a schedule, none of which a plain chatbot can do out of the box. Many prompts deliberately touch the user's own stuff so it matters to them. One-click-runnable (no [placeholders]); reads plainly for a non-dev. Labels name the END RESULT, not the mechanism. The 'build' category (target 'app-builder') prefills the composer since the agent builds the app in-place (CreateApp drops a live card); the 'schedule' category (target 'schedule') hands off to the automation chips, which turn a pick into a real scheduled workflow.
|
||||
export type StarterCategory = {
|
||||
id: string;
|
||||
label: string;
|
||||
Icon: LucideIcon;
|
||||
prompts: string[];
|
||||
target?: 'app-builder';
|
||||
target?: 'app-builder' | 'schedule';
|
||||
};
|
||||
|
||||
export const STARTER_CATEGORIES: StarterCategory[] = [
|
||||
@@ -42,13 +42,10 @@ export const STARTER_CATEGORIES: StarterCategory[] = [
|
||||
],
|
||||
},
|
||||
{
|
||||
// MCPs: plug your real tools in and let agents work across them.
|
||||
id: 'connect', label: 'Connect your apps', Icon: Plug,
|
||||
prompts: [
|
||||
'Summarize my Gmail inbox and flag what actually needs a reply',
|
||||
'Turn my Notion notes into a clear action plan',
|
||||
'Look at my calendar and lay out a realistic plan for my week',
|
||||
'Pull a sheet from my Google Drive and chart what matters',
|
||||
],
|
||||
// Recurring routines on a real schedule: the pick hands off to AutomationChips (target 'schedule'),
|
||||
// which shows THIS user's tailored automations and turns a click into a scheduled workflow. Label
|
||||
// names the payoff (a brief waiting for you), not the machinery. No inline prompts: the chips own them.
|
||||
id: 'schedule', label: 'Daily brief', Icon: CalendarClock, target: 'schedule',
|
||||
prompts: [],
|
||||
},
|
||||
];
|
||||
|
||||
@@ -144,6 +144,8 @@ export interface AgentConfig {
|
||||
target_directory?: string;
|
||||
dashboard_id?: string;
|
||||
selected_app_output_ids?: string[];
|
||||
// Onboarding's unattended audit runs read-only over the user's real files (Edit/Bash hard-blocked).
|
||||
read_only?: boolean;
|
||||
}
|
||||
|
||||
export interface HistorySession {
|
||||
@@ -502,7 +504,10 @@ export const handleApproval = createAsyncThunk(
|
||||
export const closeSession = createAsyncThunk(
|
||||
'agents/closeSession',
|
||||
async ({ sessionId }: { sessionId: string }) => {
|
||||
await fetch(`${AGENTS_API}/sessions/${sessionId}/close`, { method: 'POST' });
|
||||
// Drafts are client-only; the server has no such session and would 404.
|
||||
if (!sessionId.startsWith('draft-')) {
|
||||
await fetch(`${AGENTS_API}/sessions/${sessionId}/close`, { method: 'POST' });
|
||||
}
|
||||
return sessionId;
|
||||
}
|
||||
);
|
||||
|
||||
@@ -52,6 +52,11 @@ export interface ViewCardPosition {
|
||||
height: number;
|
||||
zOrder: number;
|
||||
parent_session_id?: string | null;
|
||||
/** Chat session this app preview lives inside (renders over the chat's dock slot); null/absent = free card. */
|
||||
docked_to?: string | null;
|
||||
// Reveal-born apps start as a light "built for you, click to open" card so the onboarding curtain
|
||||
// lifts INSTANTLY instead of booting a live Vite preview in-frame. Cleared on first click -> boots.
|
||||
preview_deferred?: boolean;
|
||||
}
|
||||
|
||||
// Record key + card identity for a view card. The primary keeps the bare output_id so persisted layouts and every existing by-output lookup stay valid; secondaries append #N.
|
||||
@@ -81,6 +86,8 @@ export interface BrowserCardPosition {
|
||||
keep_open?: boolean;
|
||||
/** Dashboard this card belongs to; cards render and persist only on their owning dashboard. */
|
||||
dashboard_id?: string;
|
||||
/** Chat session this browser lives inside (renders over the chat's dock slot); null/absent = free card. */
|
||||
docked_to?: string | null;
|
||||
}
|
||||
|
||||
export interface WorkflowCardPosition {
|
||||
@@ -100,6 +107,8 @@ export interface WorkflowsHubPosition {
|
||||
width: number;
|
||||
height: number;
|
||||
zOrder: number;
|
||||
// Full size view: the card fills the whole dashboard (reuses the fullscreen tile geometry).
|
||||
fullscreen?: boolean;
|
||||
}
|
||||
|
||||
|
||||
@@ -144,10 +153,16 @@ export interface DashboardLayoutState {
|
||||
recentlyClosed: ClosedCard[];
|
||||
glowingBrowserCards: Record<string, { sourceId: string; fading: boolean; label?: string }>;
|
||||
glowingAgentCards: Record<string, { sourceId: string; fading: boolean; sourceYRatio?: number; label?: string }>;
|
||||
/** Window controls: cards collapsed to a title pill (many at once). Keyed by any card id (session/note/browser/view/workflow). */
|
||||
minimizedCards: Record<string, boolean>;
|
||||
/** macOS-style tiling: card id -> zone ('fullscreen' | 'fill' | 'left'|'right'|'top'|'bottom' | 'tl'|'tr'|'bl'|'br' | 't3l'|'t3c'|'t3r'). A tiled card renders at that viewport region (webview stays mounted); 'fullscreen' also hides the app chrome. */
|
||||
tiledCards: Record<string, string>;
|
||||
persistedExpandedSessionIds: string[];
|
||||
nextZOrder: number;
|
||||
loading: boolean;
|
||||
initialized: boolean;
|
||||
/** True only after a SUCCESSFUL layout fetch for the current dashboard; saveLayout is a no-op until then (a failed boot fetch must never wipe the server layout). */
|
||||
saveArmed: boolean;
|
||||
/** Transient: new browser card id; Dashboard pans/zooms to it then clears via clearPendingFocusBrowserId. */
|
||||
pendingFocusBrowserId: string | null;
|
||||
// Set when a view card is opened from outside the canvas (sidebar app click / toolbar picker) so the dashboard fits+highlights it on arrival; holds the card key.
|
||||
@@ -193,10 +208,13 @@ const initialState: DashboardLayoutState = {
|
||||
recentlyClosed: [],
|
||||
glowingBrowserCards: {},
|
||||
glowingAgentCards: {},
|
||||
minimizedCards: {},
|
||||
tiledCards: {},
|
||||
persistedExpandedSessionIds: [],
|
||||
nextZOrder: 1,
|
||||
loading: false,
|
||||
initialized: false,
|
||||
saveArmed: false,
|
||||
pendingFocusBrowserId: null,
|
||||
pendingFocusViewCardId: null,
|
||||
pendingFocusNoteId: null,
|
||||
@@ -231,6 +249,8 @@ export const fetchLayout = createAsyncThunk(
|
||||
// isReconnect distinguishes a socket-reconnect recovery refetch (merge, keep live positions) from a fresh mount/switch load (replace, snapshot is the user's saved layout). Passed explicitly, not inferred from state, so a stale in-flight fetch from a previous dashboard can't be misread as a merge.
|
||||
async ({ dashboardId }: { dashboardId: string; isReconnect?: boolean }) => {
|
||||
const res = await fetch(`${DASHBOARDS_API}/${dashboardId}`);
|
||||
// A non-2xx body silently parsing to "no layout" is how a healthy dashboard gets wiped: the empty result gets marked initialized and the next debounced save persists it.
|
||||
if (!res.ok) throw new Error(`layout fetch failed: ${res.status}`);
|
||||
const data = await res.json();
|
||||
const layout = data.layout ?? {};
|
||||
const browserCards = (layout.browser_cards ?? {}) as Record<string, any>;
|
||||
@@ -265,7 +285,10 @@ interface SaveLayoutPayload extends LayoutPayload {
|
||||
|
||||
export const saveLayout = createAsyncThunk(
|
||||
'dashboardLayout/save',
|
||||
async (payload: SaveLayoutPayload) => {
|
||||
async (payload: SaveLayoutPayload, { getState }) => {
|
||||
// Never persist a layout this client never successfully loaded; a failed boot fetch otherwise saves the pristine empty store over the server's real layout (the wipe class).
|
||||
const armed = (getState() as { dashboardLayout: { saveArmed: boolean } }).dashboardLayout.saveArmed;
|
||||
if (!armed) return payload;
|
||||
await fetch(`${DASHBOARDS_API}/${payload.dashboardId}`, {
|
||||
method: 'PUT',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
@@ -548,10 +571,49 @@ function addMissingCards<T extends { x: number; y: number; width: number; height
|
||||
}
|
||||
}
|
||||
|
||||
// One docked surface per chat: the slot is a single rect, so docking a new browser/app releases
|
||||
// whatever was previously docked there (it falls back to its stored free position).
|
||||
function clearOtherDocks(state: { browserCards: Record<string, BrowserCardPosition>; viewCards: Record<string, ViewCardPosition> }, sessionId: string): void {
|
||||
for (const bc of Object.values(state.browserCards)) {
|
||||
if (bc.docked_to === sessionId) bc.docked_to = null;
|
||||
}
|
||||
for (const vc of Object.values(state.viewCards)) {
|
||||
if (vc.docked_to === sessionId) vc.docked_to = null;
|
||||
}
|
||||
}
|
||||
|
||||
const dashboardLayoutSlice = createSlice({
|
||||
name: 'dashboardLayout',
|
||||
initialState,
|
||||
reducers: {
|
||||
// Window controls (traffic lights). Minimize toggles a per-card pill; tiling snaps a card to a
|
||||
// macOS-style viewport zone (green = 'fill'). Minimizing an un-tiles and vice-versa, so a card
|
||||
// is never both pill'd and tiled at once.
|
||||
toggleMinimizeCard(state, action: PayloadAction<{ cardId: string }>) {
|
||||
const id = action.payload.cardId;
|
||||
if (state.minimizedCards[id]) {
|
||||
delete state.minimizedCards[id];
|
||||
} else {
|
||||
state.minimizedCards[id] = true;
|
||||
if (state.tiledCards[id]) delete state.tiledCards[id];
|
||||
}
|
||||
},
|
||||
setTiledCard(state, action: PayloadAction<{ cardId: string; zone: string }>) {
|
||||
const { cardId, zone } = action.payload;
|
||||
state.tiledCards[cardId] = zone;
|
||||
if (state.minimizedCards[cardId]) delete state.minimizedCards[cardId];
|
||||
},
|
||||
clearTiledCard(state, action: PayloadAction<string>) {
|
||||
if (state.tiledCards[action.payload]) delete state.tiledCards[action.payload];
|
||||
},
|
||||
clearAllTiles(state) {
|
||||
state.tiledCards = {};
|
||||
},
|
||||
clearCardWindowState(state, action: PayloadAction<string>) {
|
||||
const id = action.payload;
|
||||
if (state.minimizedCards[id]) delete state.minimizedCards[id];
|
||||
if (state.tiledCards[id]) delete state.tiledCards[id];
|
||||
},
|
||||
setCardPosition(
|
||||
state,
|
||||
action: PayloadAction<{ sessionId: string; x: number; y: number }>
|
||||
@@ -652,6 +714,8 @@ const dashboardLayoutSlice = createSlice({
|
||||
|
||||
removeCard(state, action: PayloadAction<string>) {
|
||||
delete state.cards[action.payload];
|
||||
delete state.tiledCards[action.payload];
|
||||
delete state.minimizedCards[action.payload];
|
||||
},
|
||||
|
||||
reconcileSessions(
|
||||
@@ -665,6 +729,9 @@ const dashboardLayoutSlice = createSlice({
|
||||
if (!liveIds.has(id)) {
|
||||
state.closedCardPositions[id] = { ...state.cards[id] };
|
||||
delete state.cards[id];
|
||||
// A dead card must never keep owning a tile: an orphaned 'fullscreen' entry hides ALL chrome until reload.
|
||||
delete state.tiledCards[id];
|
||||
delete state.minimizedCards[id];
|
||||
}
|
||||
}
|
||||
|
||||
@@ -756,8 +823,10 @@ const dashboardLayoutSlice = createSlice({
|
||||
x?: number; y?: number; width?: number; height?: number;
|
||||
// Open ANOTHER independent instance of an already-open app instead of no-op'ing.
|
||||
newInstance?: boolean;
|
||||
// Reveal-only: start as a light "click to open" card instead of booting the preview eagerly.
|
||||
previewDeferred?: boolean;
|
||||
}>) {
|
||||
const { outputId, expandedSessionIds, parentSessionId, x, y, width, height, newInstance } = action.payload;
|
||||
const { outputId, expandedSessionIds, parentSessionId, x, y, width, height, newInstance, previewDeferred } = action.payload;
|
||||
let instance = 1;
|
||||
if (state.viewCards[outputId]) {
|
||||
if (!newInstance) return;
|
||||
@@ -793,10 +862,19 @@ const dashboardLayoutSlice = createSlice({
|
||||
height: h,
|
||||
zOrder: state.nextZOrder++,
|
||||
parent_session_id: parentSessionId || null,
|
||||
// An agent-built app defaults to living INSIDE its chat, same as spawned browsers.
|
||||
docked_to: (parentSessionId && (clearOtherDocks(state, parentSessionId), parentSessionId)) || null,
|
||||
preview_deferred: previewDeferred || undefined,
|
||||
};
|
||||
state.pendingFocusViewCardId = cardKey;
|
||||
},
|
||||
|
||||
// First click on a reveal-parked app card: drop the defer so its live preview boots.
|
||||
activateViewCardPreview(state, action: PayloadAction<string>) {
|
||||
const card = state.viewCards[action.payload];
|
||||
if (card && card.preview_deferred) card.preview_deferred = undefined;
|
||||
},
|
||||
|
||||
clearPendingFocusViewCardId(state) {
|
||||
state.pendingFocusViewCardId = null;
|
||||
},
|
||||
@@ -824,6 +902,8 @@ const dashboardLayoutSlice = createSlice({
|
||||
|
||||
removeViewCard(state, action: PayloadAction<string>) {
|
||||
delete state.viewCards[action.payload];
|
||||
delete state.tiledCards[action.payload];
|
||||
delete state.minimizedCards[action.payload];
|
||||
if (state.activeViewCardId === action.payload) state.activeViewCardId = null;
|
||||
},
|
||||
|
||||
@@ -858,6 +938,27 @@ const dashboardLayoutSlice = createSlice({
|
||||
state.pendingFocusBrowserId = null;
|
||||
},
|
||||
|
||||
// Camera-focus an EXISTING card (the inline chat embeds' "show on canvas"); the lifecycle
|
||||
// pendingFocus effects own the fit + highlight + clear, same as a fresh add.
|
||||
focusBrowserCard(state, action: PayloadAction<string>) {
|
||||
if (state.browserCards[action.payload]) state.pendingFocusBrowserId = action.payload;
|
||||
},
|
||||
focusViewCard(state, action: PayloadAction<string>) {
|
||||
if (state.viewCards[action.payload]) state.pendingFocusViewCardId = action.payload;
|
||||
},
|
||||
|
||||
setViewDocked(state, action: PayloadAction<{ cardKey: string; dockedTo: string | null }>) {
|
||||
const vc = state.viewCards[action.payload.cardKey];
|
||||
if (!vc) return;
|
||||
if (action.payload.dockedTo) clearOtherDocks(state, action.payload.dockedTo);
|
||||
vc.docked_to = action.payload.dockedTo;
|
||||
},
|
||||
setBrowserDocked(state, action: PayloadAction<{ browserId: string; dockedTo: string | null }>) {
|
||||
const bc = state.browserCards[action.payload.browserId];
|
||||
if (!bc) return;
|
||||
if (action.payload.dockedTo) clearOtherDocks(state, action.payload.dockedTo);
|
||||
bc.docked_to = action.payload.dockedTo;
|
||||
},
|
||||
addBrowserCardFromBackend(state, action: PayloadAction<BrowserCardPosition>) {
|
||||
const card = action.payload;
|
||||
if (state.browserCards[card.browser_id]) return;
|
||||
@@ -903,6 +1004,8 @@ const dashboardLayoutSlice = createSlice({
|
||||
delete state.browserCards[action.payload];
|
||||
delete state.suspendedBrowserCards[action.payload];
|
||||
delete state.endingBrowserCards[action.payload];
|
||||
delete state.tiledCards[action.payload];
|
||||
delete state.minimizedCards[action.payload];
|
||||
},
|
||||
|
||||
markBrowserCardEnding(
|
||||
@@ -1077,6 +1180,13 @@ const dashboardLayoutSlice = createSlice({
|
||||
state.workflowsMonitorCard = null;
|
||||
},
|
||||
|
||||
toggleWorkflowsHubFullscreen(state) {
|
||||
if (state.workflowsHub) {
|
||||
state.workflowsHub.fullscreen = !state.workflowsHub.fullscreen;
|
||||
state.workflowsHub.zOrder = state.nextZOrder++;
|
||||
}
|
||||
},
|
||||
|
||||
clearWorkflowsAppTarget(state) {
|
||||
state.workflowsAppTarget = null;
|
||||
},
|
||||
@@ -1442,6 +1552,8 @@ const dashboardLayoutSlice = createSlice({
|
||||
|
||||
removeNote(state, action: PayloadAction<string>) {
|
||||
delete state.notes[action.payload];
|
||||
delete state.tiledCards[action.payload];
|
||||
delete state.minimizedCards[action.payload];
|
||||
},
|
||||
|
||||
clearPendingFocusNoteId(state) {
|
||||
@@ -1572,6 +1684,8 @@ const dashboardLayoutSlice = createSlice({
|
||||
},
|
||||
|
||||
resetLayout(state, action: PayloadAction<{ keepBrowserIds?: string[] } | undefined>) {
|
||||
state.tiledCards = {};
|
||||
state.minimizedCards = {};
|
||||
// Keep the recently-used (keep-alive) browser cards mounted across a dashboard switch so their webContents + sessionStorage survive (logged-in sites stay logged in); everything else is wiped for the fresh load. Their suspend entry rides along so a parked one isn't silently dropped.
|
||||
const keep = new Set(action.payload?.keepBrowserIds || []);
|
||||
const keptBrowsers: typeof state.browserCards = {};
|
||||
@@ -1592,6 +1706,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
state.persistedExpandedSessionIds = [];
|
||||
state.nextZOrder = 1;
|
||||
state.initialized = false;
|
||||
state.saveArmed = false;
|
||||
state.pendingFocusNoteId = null;
|
||||
state.suspendedBrowserCards = keptSuspended;
|
||||
state.endingBrowserCards = {};
|
||||
@@ -1609,6 +1724,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
// A fresh mount/switch load replaces (the snapshot is the user's saved layout, authoritative). A reconnect refetch (useDashboardLifecycle line ~90) recovers cards lost in a socket gap and must MERGE, blind- replacing there clobbered the live, collision-placed positions of cards already on canvas (the overlap / vanish under load while many browsers spawn). The caller says which; never inferred from state.
|
||||
const isReconnectRefetch = action.meta.arg.isReconnect === true;
|
||||
state.initialized = true;
|
||||
state.saveArmed = true;
|
||||
const ownerDashboardId = action.meta.arg.dashboardId;
|
||||
if (!isReconnectRefetch) {
|
||||
state.cards = action.payload.cards;
|
||||
@@ -1667,6 +1783,7 @@ const dashboardLayoutSlice = createSlice({
|
||||
})
|
||||
.addCase(fetchLayout.rejected, (state) => {
|
||||
state.loading = false;
|
||||
// Fail-open for RENDERING only; saveArmed stays false so this client can never persist the empty layout it booted with over the server's real one (the wipe that hit 2026-07-20).
|
||||
state.initialized = true;
|
||||
})
|
||||
.addCase(fetchSessionRejectedAction, (state, action) => {
|
||||
@@ -1699,6 +1816,9 @@ const dashboardLayoutSlice = createSlice({
|
||||
for (const bc of Object.values(state.browserCards)) {
|
||||
if (bc.spawned_by !== session.id) continue;
|
||||
const pos = placeBrowserBesideChat(state, parentCard, session.id, bc.width, bc.height, bc.browser_id);
|
||||
// Default home is inside the chat, same as the non-racing spawn path.
|
||||
clearOtherDocks(state, session.id);
|
||||
bc.docked_to = session.id;
|
||||
bc.x = pos.x;
|
||||
bc.y = pos.y;
|
||||
state.glowingBrowserCards[bc.browser_id] = { sourceId: session.id, fading: false, label: 'Use Browser' };
|
||||
@@ -1724,6 +1844,8 @@ export const {
|
||||
setActiveViewCardId,
|
||||
addBrowserCard,
|
||||
addBrowserCardFromBackend,
|
||||
setBrowserDocked,
|
||||
setViewDocked,
|
||||
setBrowserCardPosition,
|
||||
setBrowserCardSize,
|
||||
removeBrowserCard,
|
||||
@@ -1751,7 +1873,15 @@ export const {
|
||||
setGlowingAgentCard,
|
||||
fadeGlowingAgentCard,
|
||||
clearGlowingAgentCard,
|
||||
toggleMinimizeCard,
|
||||
setTiledCard,
|
||||
clearTiledCard,
|
||||
clearAllTiles,
|
||||
clearCardWindowState,
|
||||
clearPendingFocusBrowserId,
|
||||
focusBrowserCard,
|
||||
focusViewCard,
|
||||
activateViewCardPreview,
|
||||
clearPendingFocusViewCardId,
|
||||
addWorkflowCard,
|
||||
setWorkflowCardPosition,
|
||||
@@ -1763,6 +1893,7 @@ export const {
|
||||
closeWorkflowsHub,
|
||||
openWorkflowsApp,
|
||||
closeWorkflowsApp,
|
||||
toggleWorkflowsHubFullscreen,
|
||||
clearWorkflowsAppTarget,
|
||||
openWorkflowMonitor,
|
||||
closeWorkflowMonitor,
|
||||
@@ -1805,4 +1936,14 @@ export const reopenLastClosed = createAsyncThunk(
|
||||
}
|
||||
);
|
||||
|
||||
export const selectFullscreenCardId = (state: { dashboardLayout: DashboardLayoutState }): string | null => {
|
||||
const s = state.dashboardLayout;
|
||||
const entry = Object.entries(s.tiledCards).find(([, zone]) => zone === 'fullscreen');
|
||||
if (!entry) return null;
|
||||
const id = entry[0];
|
||||
// Belt over the reducer hygiene: an entry whose card is gone (any removal path) must not hold the app in fullscreen.
|
||||
const exists = id in s.cards || id in s.viewCards || id in s.browserCards || id in s.notes || id in s.workflowCards;
|
||||
return exists ? id : null;
|
||||
};
|
||||
|
||||
export default dashboardLayoutSlice.reducer;
|
||||
|
||||
@@ -3,6 +3,17 @@ import type { PersonalizedStarter } from '@/shared/state/settingsSlice';
|
||||
|
||||
// Transient bridge between the v3 flow overlay and the dashboard: the overlay finishes, stashes the prep payload here, and the dashboard's reveal hook consumes it exactly once to seed the canvas.
|
||||
|
||||
export interface PreppedJob {
|
||||
/** Agent-session id for audit/app/research jobs; '' for a scheduled workflow (keyed by workflowId instead). */
|
||||
sessionId: string;
|
||||
title: string;
|
||||
kind: 'audit' | 'app' | 'schedule' | 'research' | 'browser';
|
||||
/** The one-clause "why we started this for you", shown in the reveal note. */
|
||||
reason?: string;
|
||||
/** Set for kind 'schedule': the created scheduled workflow, placed as a workflow card + deletable on discard. */
|
||||
workflowId?: string;
|
||||
}
|
||||
|
||||
export interface OnboardingV3State {
|
||||
/** True while the full-screen flow owns the window; suppresses onboarding v2. */
|
||||
flowActive: boolean;
|
||||
@@ -11,6 +22,12 @@ export interface OnboardingV3State {
|
||||
greeting: string | null;
|
||||
starters: PersonalizedStarter[];
|
||||
scanSummary: string | null;
|
||||
/** Read-only audit prompt to launch as a live agent at reveal; null = chips only. */
|
||||
autoPrompt: string | null;
|
||||
/** Background jobs launched mid-flow (audit + app build); the keep/discard toast owns their fate. */
|
||||
prepped: PreppedJob[];
|
||||
/** Canvas center the reveal composed around; lets the app view card be born at the arc-end spot. */
|
||||
revealAnchor: { cx: number; cy: number } | null;
|
||||
}
|
||||
|
||||
const initialState: OnboardingV3State = {
|
||||
@@ -19,6 +36,9 @@ const initialState: OnboardingV3State = {
|
||||
greeting: null,
|
||||
starters: [],
|
||||
scanSummary: null,
|
||||
autoPrompt: null,
|
||||
prepped: [],
|
||||
revealAnchor: null,
|
||||
};
|
||||
|
||||
const onboardingV3Slice = createSlice({
|
||||
@@ -28,18 +48,31 @@ const onboardingV3Slice = createSlice({
|
||||
setFlowActive(state, action: PayloadAction<boolean>) {
|
||||
state.flowActive = action.payload;
|
||||
},
|
||||
stageReveal(state, action: PayloadAction<{ greeting: string | null; starters: PersonalizedStarter[]; scanSummary: string | null }>) {
|
||||
stageReveal(state, action: PayloadAction<{ greeting: string | null; starters: PersonalizedStarter[]; scanSummary: string | null; autoPrompt: string | null }>) {
|
||||
state.revealPending = true;
|
||||
state.greeting = action.payload.greeting;
|
||||
state.starters = action.payload.starters;
|
||||
state.scanSummary = action.payload.scanSummary;
|
||||
state.autoPrompt = action.payload.autoPrompt;
|
||||
},
|
||||
addPreppedJob(state, action: PayloadAction<PreppedJob>) {
|
||||
const key = action.payload.workflowId || action.payload.sessionId;
|
||||
if (!state.prepped.some((j) => (j.workflowId || j.sessionId) === key)) state.prepped.push(action.payload);
|
||||
},
|
||||
setRevealAnchor(state, action: PayloadAction<{ cx: number; cy: number }>) {
|
||||
state.revealAnchor = action.payload;
|
||||
},
|
||||
clearPrepped(state) {
|
||||
state.prepped = [];
|
||||
state.revealAnchor = null;
|
||||
},
|
||||
clearReveal(state) {
|
||||
state.revealPending = false;
|
||||
state.scanSummary = null;
|
||||
state.autoPrompt = null;
|
||||
},
|
||||
},
|
||||
});
|
||||
|
||||
export const { setFlowActive, stageReveal, clearReveal } = onboardingV3Slice.actions;
|
||||
export const { setFlowActive, stageReveal, clearReveal, addPreppedJob, setRevealAnchor, clearPrepped } = onboardingV3Slice.actions;
|
||||
export default onboardingV3Slice.reducer;
|
||||
|
||||
@@ -45,6 +45,7 @@ export interface AppSettings {
|
||||
zoom_sensitivity: number;
|
||||
theme: 'light' | 'dark';
|
||||
new_agent_shortcut: string;
|
||||
dictation_shortcut?: string | null;
|
||||
anthropic_api_key: string | null;
|
||||
openai_api_key?: string | null;
|
||||
google_api_key?: string | null;
|
||||
@@ -71,6 +72,7 @@ export interface AppSettings {
|
||||
openswarm_usage_cached?: SubscriptionUsage | null;
|
||||
/** Identity populated by /api/auth/signin-activate; Stripe checkout also fills these. */
|
||||
user_id?: string | null;
|
||||
user_name?: string | null;
|
||||
user_email?: string | null;
|
||||
signin_method?: 'google' | 'email' | 'stripe' | null;
|
||||
/** Anonymous device id (first-run generated); stitches anon to authed PostHog Persons. */
|
||||
@@ -79,13 +81,39 @@ export interface AppSettings {
|
||||
onboarding_v3?: string | null;
|
||||
/** User-picked accent hex from the onboarding theme pad; null = stock accent. */
|
||||
accent_color?: string | null;
|
||||
/** Multi-stop gradient from the theme pad (2-3 hexes); washes the canvas. */
|
||||
accent_gradient?: string[] | null;
|
||||
/** Global text-size ratio. Scales the root font-size so every rem-based size grows/shrinks together, no layout breakage. 1 = default. */
|
||||
ui_font_scale?: number;
|
||||
/** Dictation activation: true = hold the hotkey/mic to record and release to stop (WhisperFlow style); false = tap toggles. */
|
||||
voice_hold_to_talk?: boolean;
|
||||
personalized_greeting?: string | null;
|
||||
/** The short one-glance identity hook shown in the reveal's focal beat (greeting is the longer read). */
|
||||
personalized_headline?: string | null;
|
||||
personalized_starters?: PersonalizedStarter[];
|
||||
personalized_automations?: PersonalizedAutomation[];
|
||||
/** The hero's two-level menu: 4 general categories, each with 4 starters tailored to this user. */
|
||||
personalized_menu?: PersonalizedMenu | null;
|
||||
personalized_usage_summary?: string | null;
|
||||
}
|
||||
|
||||
export interface PersonalizedStarter {
|
||||
title: string;
|
||||
prompt: string;
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface PersonalizedAutomation {
|
||||
title: string;
|
||||
prompt: string;
|
||||
cadence: 'daily' | 'weekday' | 'weekly';
|
||||
}
|
||||
|
||||
export interface PersonalizedMenu {
|
||||
computer: PersonalizedStarter[];
|
||||
research: PersonalizedStarter[];
|
||||
web: PersonalizedStarter[];
|
||||
build: PersonalizedStarter[];
|
||||
}
|
||||
|
||||
export interface ActivateSubscriptionPayload {
|
||||
@@ -129,13 +157,16 @@ interface SettingsState {
|
||||
export const DEFAULT_SETTINGS: AppSettings = {
|
||||
default_system_prompt: DEFAULT_SYSTEM_PROMPT,
|
||||
default_folder: null,
|
||||
default_model: 'sonnet',
|
||||
default_model: 'opus-5',
|
||||
default_mode: 'agent',
|
||||
default_max_turns: null,
|
||||
default_thinking_level: 'auto',
|
||||
zoom_sensitivity: 50,
|
||||
ui_font_scale: 1,
|
||||
voice_hold_to_talk: true,
|
||||
theme: 'light',
|
||||
new_agent_shortcut: 'Meta+l',
|
||||
dictation_shortcut: null,
|
||||
anthropic_api_key: null,
|
||||
browser_homepage: 'https://duckduckgo.com',
|
||||
auto_select_mode_on_new_agent: false,
|
||||
|
||||
@@ -119,13 +119,21 @@ export const { setSubscriptionStatus, markSubscriptionConnected, hideProviderHea
|
||||
const EMPTY_CONNECTIONS: SubscriptionConnection[] = [];
|
||||
|
||||
/** Unwraps the polymorphic `providers` shape (modern object vs legacy array). */
|
||||
let p_lastRows: SubscriptionConnection[] | null = null;
|
||||
let p_lastFiltered: SubscriptionConnection[] | null = null;
|
||||
|
||||
export function selectSubscriptionConnections(
|
||||
state: WithSubscriptions,
|
||||
): SubscriptionConnection[] {
|
||||
const providers = state.subscriptions.status?.providers;
|
||||
if (!providers) return EMPTY_CONNECTIONS;
|
||||
if (Array.isArray(providers)) return providers;
|
||||
return providers.connections ?? EMPTY_CONNECTIONS;
|
||||
const rows = Array.isArray(providers) ? providers : (providers.connections ?? EMPTY_CONNECTIONS);
|
||||
// The free-trial/Pro lane is an OpenSwarm-managed router row, not a connection the USER made; counting it made a fresh trial look "connected" (dead Connect-beat rows, premature onConnected, phantom green rings). Memoized per rows identity so useSelector consumers don't churn.
|
||||
if (rows === p_lastRows && p_lastFiltered) return p_lastFiltered;
|
||||
const filtered = rows.filter((p) => !/\(OpenSwarm-managed\)/.test(p.name ?? ''));
|
||||
p_lastRows = rows;
|
||||
p_lastFiltered = filtered.length === rows.length ? rows : filtered;
|
||||
return p_lastFiltered;
|
||||
}
|
||||
|
||||
export function hasAnyActiveSubscription(state: WithSubscriptions): boolean {
|
||||
|
||||
@@ -7,13 +7,39 @@ interface ThemeContextValue {
|
||||
mode: ThemeMode;
|
||||
tokens: ClaudeTokens;
|
||||
accent: string | null;
|
||||
gradient: string[] | null;
|
||||
washOpacity: number;
|
||||
grain: number;
|
||||
toggleMode: () => void;
|
||||
setMode: (mode: ThemeMode) => void;
|
||||
setAccent: (hex: string | null) => void;
|
||||
setGradient: (stops: string[] | null) => void;
|
||||
setWashOpacity: (v: number) => void;
|
||||
setGrain: (v: number) => void;
|
||||
}
|
||||
|
||||
const STORAGE_KEY = 'self-swarm-theme-mode';
|
||||
const ACCENT_STORAGE_KEY = 'self-swarm-theme-accent';
|
||||
const GRADIENT_STORAGE_KEY = 'self-swarm-theme-gradient';
|
||||
const WASH_OPACITY_KEY = 'self-swarm-theme-wash-opacity';
|
||||
const GRAIN_KEY = 'self-swarm-theme-grain';
|
||||
export const DEFAULT_WASH_OPACITY = 0.17;
|
||||
|
||||
function getInitialNum(key: string, def: number): number {
|
||||
try {
|
||||
const v = parseFloat(localStorage.getItem(key) ?? '');
|
||||
if (Number.isFinite(v) && v >= 0 && v <= 1) return v;
|
||||
} catch {}
|
||||
return def;
|
||||
}
|
||||
|
||||
function getInitialGradient(): string[] | null {
|
||||
try {
|
||||
const stored = JSON.parse(localStorage.getItem(GRADIENT_STORAGE_KEY) ?? 'null');
|
||||
if (Array.isArray(stored) && stored.length > 1 && stored.every((v) => /^#[0-9a-f]{6}$/i.test(v))) return stored;
|
||||
} catch {}
|
||||
return null;
|
||||
}
|
||||
|
||||
function getInitialAccent(): string | null {
|
||||
try {
|
||||
@@ -46,14 +72,23 @@ const ThemeContext = createContext<ThemeContextValue>({
|
||||
mode: 'light',
|
||||
tokens: lightTokens,
|
||||
accent: null,
|
||||
gradient: null,
|
||||
washOpacity: DEFAULT_WASH_OPACITY,
|
||||
grain: 0,
|
||||
toggleMode: () => {},
|
||||
setMode: () => {},
|
||||
setAccent: () => {},
|
||||
setGradient: () => {},
|
||||
setWashOpacity: () => {},
|
||||
setGrain: () => {},
|
||||
});
|
||||
|
||||
export const ThemeProvider: React.FC<{ children: React.ReactNode }> = ({ children }) => {
|
||||
const [mode, setModeState] = useState<ThemeMode>(getInitialMode);
|
||||
const [accent, setAccentState] = useState<string | null>(getInitialAccent);
|
||||
const [gradient, setGradientState] = useState<string[] | null>(getInitialGradient);
|
||||
const [washOpacity, setWashOpacityState] = useState<number>(() => getInitialNum(WASH_OPACITY_KEY, DEFAULT_WASH_OPACITY));
|
||||
const [grain, setGrainState] = useState<number>(() => getInitialNum(GRAIN_KEY, 0));
|
||||
const firstMount = useRef(true);
|
||||
|
||||
useEffect(() => {
|
||||
@@ -70,24 +105,52 @@ export const ThemeProvider: React.FC<{ children: React.ReactNode }> = ({ childre
|
||||
} catch {}
|
||||
}, [accent]);
|
||||
|
||||
useEffect(() => {
|
||||
try {
|
||||
if (gradient) localStorage.setItem(GRADIENT_STORAGE_KEY, JSON.stringify(gradient));
|
||||
else localStorage.removeItem(GRADIENT_STORAGE_KEY);
|
||||
} catch {}
|
||||
}, [gradient]);
|
||||
|
||||
useEffect(() => { try { localStorage.setItem(WASH_OPACITY_KEY, String(washOpacity)); } catch {} }, [washOpacity]);
|
||||
useEffect(() => { try { localStorage.setItem(GRAIN_KEY, String(grain)); } catch {} }, [grain]);
|
||||
|
||||
const tokens = useMemo(() => withAccent(mode === 'dark' ? darkTokens : lightTokens, accent, mode), [mode, accent]);
|
||||
|
||||
// Stable identities: SettingsLoader's "apply settings.theme" effect lists setMode in its deps, so a setter that changed every render made that effect re-fire on each toggle and re-assert the OLD persisted theme until the debounced save caught up: live theme snapped back for ~900ms = the switch flicker.
|
||||
const toggleMode = useCallback(() => setModeState((m) => (m === 'light' ? 'dark' : 'light')), []);
|
||||
const setMode = useCallback((m: ThemeMode) => setModeState(m), []);
|
||||
const setAccent = useCallback((hex: string | null) => setAccentState(hex), []);
|
||||
const setGradient = useCallback((stops: string[] | null) => setGradientState(stops), []);
|
||||
const setWashOpacity = useCallback((v: number) => setWashOpacityState(Math.min(1, Math.max(0, v))), []);
|
||||
const setGrain = useCallback((v: number) => setGrainState(Math.min(1, Math.max(0, v))), []);
|
||||
|
||||
const value = useMemo(() => ({ mode, tokens, accent, toggleMode, setMode, setAccent }), [mode, tokens, accent, toggleMode, setMode, setAccent]);
|
||||
const value = useMemo(() => ({ mode, tokens, accent, gradient, washOpacity, grain, toggleMode, setMode, setAccent, setGradient, setWashOpacity, setGrain }), [mode, tokens, accent, gradient, washOpacity, grain, toggleMode, setMode, setAccent, setGradient, setWashOpacity, setGrain]);
|
||||
|
||||
return <ThemeContext.Provider value={value}>{children}</ThemeContext.Provider>;
|
||||
};
|
||||
|
||||
export const useClaudeTokens = (): ClaudeTokens => useContext(ThemeContext).tokens;
|
||||
|
||||
/** Forces dark tokens for a subtree: desktop-shell glass panels stay dark even in light mode. */
|
||||
export const DarkTokensScope: React.FC<{ children: React.ReactNode }> = ({ children }) => {
|
||||
const ctx = useContext(ThemeContext);
|
||||
const value = useMemo(() => ({
|
||||
...ctx,
|
||||
mode: 'dark' as ThemeMode,
|
||||
tokens: withAccent(darkTokens, ctx.accent, 'dark'),
|
||||
}), [ctx]);
|
||||
return <ThemeContext.Provider value={value}>{children}</ThemeContext.Provider>;
|
||||
};
|
||||
export const useThemeMode = () => {
|
||||
const { mode, toggleMode, setMode } = useContext(ThemeContext);
|
||||
return { mode, toggleMode, setMode };
|
||||
};
|
||||
export const useThemeAccent = () => {
|
||||
const { accent, setAccent } = useContext(ThemeContext);
|
||||
return { accent, setAccent };
|
||||
const { accent, setAccent, gradient, setGradient } = useContext(ThemeContext);
|
||||
return { accent, setAccent, gradient, setGradient };
|
||||
};
|
||||
export const useThemeWash = () => {
|
||||
const { washOpacity, grain, setWashOpacity, setGrain } = useContext(ThemeContext);
|
||||
return { washOpacity, grain, setWashOpacity, setGrain };
|
||||
};
|
||||
|
||||
@@ -1,3 +1,30 @@
|
||||
// The type scale. OpenSwarm had ~30 near-duplicate hardcoded rem sizes (0.78/0.8/0.82/0.85...), which
|
||||
// reads as noise; snap to these 7 steps so type feels intentional and consistent. px comments assume a
|
||||
// 16px root. Use c.font.size.* instead of a raw rem string in anything you touch.
|
||||
export interface FontSizeScale {
|
||||
micro: string; // 10px, densest chrome (badges, kbd hints)
|
||||
xxs: string; // 11px, tiny meta / timestamps
|
||||
xs: string; // 12px, captions, meta, micro labels
|
||||
sm: string; // 13px, secondary / small body
|
||||
base: string; // 14px, default body + most UI text
|
||||
md: string; // 16px, emphasized body, inputs
|
||||
lg: string; // 18px, subheads
|
||||
xl: string; // 22px, section headings
|
||||
display: string; // 28px, hero / empty-state headline
|
||||
}
|
||||
|
||||
export const fontSize: FontSizeScale = {
|
||||
micro: '0.625rem',
|
||||
xxs: '0.6875rem',
|
||||
xs: '0.75rem',
|
||||
sm: '0.8125rem',
|
||||
base: '0.875rem',
|
||||
md: '1rem',
|
||||
lg: '1.125rem',
|
||||
xl: '1.375rem',
|
||||
display: '1.75rem',
|
||||
};
|
||||
|
||||
export interface ClaudeTokens {
|
||||
bg: { page: string; surface: string; elevated: string; secondary: string; inverse: string };
|
||||
text: { primary: string; secondary: string; tertiary: string; muted: string; inverse: string; ghost: string };
|
||||
@@ -7,7 +34,7 @@ export interface ClaudeTokens {
|
||||
radius: { xs: number; sm: number; md: number; lg: number; xl: number; full: number };
|
||||
status: { success: string; successBg: string; warning: string; warningBg: string; error: string; errorBg: string; info: string; infoBg: string };
|
||||
user: { bubble: string };
|
||||
font: { sans: string; mono: string };
|
||||
font: { sans: string; mono: string; size: FontSizeScale };
|
||||
transition: string;
|
||||
}
|
||||
|
||||
@@ -56,8 +83,9 @@ export const lightTokens: ClaudeTokens = {
|
||||
},
|
||||
user: { bubble: '#DDD9CE' },
|
||||
font: {
|
||||
sans: '"Anthropic Sans", ui-serif, Georgia, Cambria, "Times New Roman", Times, serif',
|
||||
sans: '"Inter Variable", Inter, ui-sans-serif, -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", Arial, sans-serif',
|
||||
mono: 'ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace',
|
||||
size: fontSize,
|
||||
},
|
||||
transition: 'all 150ms cubic-bezier(0.165, 0.85, 0.45, 1)',
|
||||
};
|
||||
@@ -107,8 +135,9 @@ export const darkTokens: ClaudeTokens = {
|
||||
},
|
||||
user: { bubble: '#393937' },
|
||||
font: {
|
||||
sans: '"Anthropic Sans", ui-serif, Georgia, Cambria, "Times New Roman", Times, serif',
|
||||
sans: '"Inter Variable", Inter, ui-sans-serif, -apple-system, BlinkMacSystemFont, "Segoe UI", "Helvetica Neue", Arial, sans-serif',
|
||||
mono: 'ui-monospace, SFMono-Regular, "SF Mono", Menlo, Consolas, monospace',
|
||||
size: fontSize,
|
||||
},
|
||||
transition: 'all 150ms cubic-bezier(0.165, 0.85, 0.45, 1)',
|
||||
};
|
||||
|
||||
@@ -0,0 +1,4 @@
|
||||
// Zen-style film grain: a static 220px PNG tile of sparse BIPOLAR speckle (some pixels lighten,
|
||||
// some darken, most near-transparent), so at full slider it reads as texture in the paint rather
|
||||
// than gray haze on top. Opacity IS the grain slider, exactly like Zen's --zen-grainy-background-opacity.
|
||||
export const GRAIN_URL = 'url("./grain-texture.png")';
|
||||
@@ -0,0 +1,24 @@
|
||||
// Theme wash as an SVG IMAGE, not a CSS linear-gradient: Chromium caches a decoded image as a GPU
|
||||
// texture, while a window-sized procedural gradient re-rasterizes on resize and, under GPU memory
|
||||
// pressure (webviews, external monitors), those rasters get DROPPED and paint as a half/blank
|
||||
// rectangle (the same class as the 1.5.9 dot-grid white-patch bug; see DashboardCanvas's grid note).
|
||||
export function washBackgroundUrl(stops: string[], washOpacity: number): string {
|
||||
const alpha = Math.max(0, Math.min(1, washOpacity));
|
||||
const stopEls = stops.map((hex, i) => {
|
||||
const offset = stops.length > 1 ? (i / (stops.length - 1)) * 100 : 100;
|
||||
return `<stop offset='${offset}%' stop-color='${hex}' stop-opacity='${alpha}'/>`;
|
||||
}).join('');
|
||||
// x2/y2 approximate the CSS 115deg direction (25 degrees below horizontal).
|
||||
const svg = `<svg xmlns='http://www.w3.org/2000/svg' width='100' height='100' preserveAspectRatio='none'><defs><linearGradient id='w' x1='0%' y1='0%' x2='90%' y2='42%'>${stopEls}</linearGradient></defs><rect width='100' height='100' fill='url(%23w)'/></svg>`;
|
||||
return `url("data:image/svg+xml,${svg.replace(/#/g, '%23').replace(/'/g, '%27')}")`;
|
||||
}
|
||||
|
||||
// Stock wallpaper when the user hasn't picked an accent yet: a designed blue-to-cream-to-pink
|
||||
// gradient (contrasting stops), so a fresh install and the onboarding stage never look flat/white.
|
||||
export const DEFAULT_WASH_STOPS = ['#B7CDEA', '#EFE0D2', '#E7BDD1'];
|
||||
|
||||
export function effectiveWashStops(gradient: string[] | null, accent: string | null): string[] {
|
||||
if (gradient && gradient.length > 0) return gradient;
|
||||
if (accent) return [accent];
|
||||
return DEFAULT_WASH_STOPS;
|
||||
}
|
||||
@@ -0,0 +1,63 @@
|
||||
import React, { useCallback, useEffect, useRef } from 'react';
|
||||
import { useAppSelector } from '@/shared/hooks';
|
||||
import { useVoiceDictation } from './useVoiceDictation';
|
||||
import { VoiceContext } from './voiceContext';
|
||||
import VoiceOverlay from './VoiceOverlay';
|
||||
|
||||
// One recorder for the whole app. Both mics (the Help pill and the spawn composer) plus the global
|
||||
// hotkey drive the SAME dictation session, so two mics can't fight over the microphone or show
|
||||
// out-of-sync state. Mounted once near the app root.
|
||||
|
||||
export function VoiceDictationProvider({ children }: { children: React.ReactNode }): React.ReactElement {
|
||||
const { state, lastText, error, pct, feedback, toggle, start, stop, volumeRef } = useVoiceDictation();
|
||||
const holdMode = useAppSelector((s) => s.settings.data.voice_hold_to_talk ?? true);
|
||||
const dictationShortcut = useAppSelector((s) => s.settings.data.dictation_shortcut ?? null);
|
||||
|
||||
// Push the user's combo to main on boot and on change so every hotkey tier rebinds live.
|
||||
useEffect(() => {
|
||||
const bridge = window as unknown as { openswarm?: { setVoiceHotkey?: (combo: string | null) => void } };
|
||||
bridge.openswarm?.setVoiceHotkey?.(dictationShortcut);
|
||||
}, [dictationShortcut]);
|
||||
const stateRef = useRef(state);
|
||||
stateRef.current = state;
|
||||
const heldRef = useRef(false);
|
||||
|
||||
const pressStart = useCallback((): void => {
|
||||
if (holdMode) {
|
||||
// A TAP while recording must stop: the quick tap's press-end fires before the async start
|
||||
// flips state to 'recording', so without this the mic could be started by a click but never
|
||||
// stopped by one.
|
||||
if (stateRef.current === 'recording') { heldRef.current = false; void stop(); return; }
|
||||
if (stateRef.current === 'idle') { heldRef.current = true; void start(); }
|
||||
} else {
|
||||
toggle();
|
||||
}
|
||||
}, [holdMode, start, stop, toggle]);
|
||||
|
||||
const pressEnd = useCallback((): void => {
|
||||
if (holdMode && heldRef.current) {
|
||||
heldRef.current = false;
|
||||
if (stateRef.current === 'recording') void stop();
|
||||
}
|
||||
}, [holdMode, stop]);
|
||||
|
||||
// Keyboard hotkey channels, matched to what the source can actually see:
|
||||
// voice:hold-down/up come ONLY from main's native uiohook tap (real global key-up, so the keyboard
|
||||
// gets the same hold-vs-toggle press semantics as the mic buttons); voice:toggle comes from the
|
||||
// fallback tier (globalShortcut / before-input relay), where key-ups are undetectable, so each
|
||||
// press toggles. Whichever tier main activated, the renderer just honors the channel it hears.
|
||||
useEffect(() => {
|
||||
const bridge = window as unknown as {
|
||||
openswarm?: { onVoiceHold?: (d: () => void, u: () => void) => () => void };
|
||||
};
|
||||
const offHold = bridge.openswarm?.onVoiceHold?.(pressStart, pressEnd);
|
||||
return () => { offHold?.(); };
|
||||
}, [pressStart, pressEnd]);
|
||||
|
||||
return (
|
||||
<VoiceContext.Provider value={{ state, lastText, error, pct, feedback, toggle, pressStart, pressEnd, holdMode, volumeRef }}>
|
||||
{children}
|
||||
<VoiceOverlay />
|
||||
</VoiceContext.Provider>
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,262 @@
|
||||
import React, { useEffect, useRef, useState } from 'react';
|
||||
import Box from '@mui/material/Box';
|
||||
import CircularProgress from '@mui/material/CircularProgress';
|
||||
import MicIcon from '@mui/icons-material/Mic';
|
||||
import CheckRoundedIcon from '@mui/icons-material/CheckRounded';
|
||||
import ContentPasteRoundedIcon from '@mui/icons-material/ContentPasteRounded';
|
||||
import InfoOutlinedIcon from '@mui/icons-material/InfoOutlined';
|
||||
import { useThemeAccent } from '@/shared/styles/ThemeContext';
|
||||
import { useVoice } from './voiceContext';
|
||||
|
||||
// WhisperFlow-style presence: while the mic is hot, an aurora breathes up from the bottom edge in
|
||||
// the COMPLEMENT of the user's accent (opposite hue = always visible against their canvas, never
|
||||
// camouflaged by it), as a three-hue gradient whose lobes morph with the live mic level. Imperative
|
||||
// rAF writes only (opacity + transform), so 60Hz voice never re-renders React.
|
||||
|
||||
function hexToHue(hex: string): { h: number; s: number; l: number } {
|
||||
const m = /^#?([0-9a-f]{6})$/i.exec(hex.trim());
|
||||
if (!m) return { h: 250, s: 70, l: 60 };
|
||||
const n = parseInt(m[1], 16);
|
||||
const r = ((n >> 16) & 255) / 255, g = ((n >> 8) & 255) / 255, b = (n & 255) / 255;
|
||||
const max = Math.max(r, g, b), min = Math.min(r, g, b);
|
||||
const l = (max + min) / 2;
|
||||
if (max === min) return { h: 0, s: 0, l: l * 100 };
|
||||
const d = max - min;
|
||||
const sat = l > 0.5 ? d / (2 - max - min) : d / (max + min);
|
||||
let h = 0;
|
||||
if (max === r) h = ((g - b) / d + (g < b ? 6 : 0)) * 60;
|
||||
else if (max === g) h = ((b - r) / d + 2) * 60;
|
||||
else h = ((r - g) / d + 4) * 60;
|
||||
return { h, s: sat * 100, l: l * 100 };
|
||||
}
|
||||
|
||||
// Interleaved wisps, like colored gases: neighbors always carry a DIFFERENT hue and overlap ~70%,
|
||||
// so mixture happens everywhere instead of three tinted patches sitting in a row.
|
||||
const LOBES = [
|
||||
{ left: '18%', width: '80vw', height: 200, drift: 0.9, phase: 0.0, hue: 0 },
|
||||
{ left: '34%', width: '95vw', height: 165, drift: 0.65, phase: 2.1, hue: 2 },
|
||||
{ left: '47%', width: '85vw', height: 210, drift: 1.1, phase: 4.2, hue: 1 },
|
||||
{ left: '60%', width: '100vw', height: 170, drift: 0.8, phase: 1.3, hue: 0 },
|
||||
{ left: '73%', width: '85vw', height: 205, drift: 1.2, phase: 3.4, hue: 2 },
|
||||
{ left: '88%', width: '78vw', height: 175, drift: 0.7, phase: 5.1, hue: 1 },
|
||||
];
|
||||
|
||||
const VoiceAurora: React.FC<{ volumeRef: React.MutableRefObject<number> }> = ({ volumeRef }) => {
|
||||
const { accent } = useThemeAccent();
|
||||
const lobeRefs = useRef<Array<HTMLDivElement | null>>([]);
|
||||
const smooth = useRef<number[]>([0, 0, 0]);
|
||||
useEffect(() => {
|
||||
const reduced = window.matchMedia('(prefers-reduced-motion: reduce)').matches;
|
||||
let raf = 0;
|
||||
const tick = (): void => {
|
||||
const v = Math.min(1, volumeRef.current);
|
||||
const t = performance.now() / 1000;
|
||||
LOBES.forEach((lobe, i) => {
|
||||
const el = lobeRefs.current[i];
|
||||
if (!el) return;
|
||||
// Lerp toward the live level so the shape glides instead of jittering; a slow sinusoid keeps
|
||||
// it undulating at steady volume, and volume swells both the wave and the lobe height.
|
||||
smooth.current[i] += (v - smooth.current[i]) * 0.16;
|
||||
const sv = smooth.current[i];
|
||||
const wave = reduced ? 0 : Math.sin(t * lobe.drift + lobe.phase) * (0.08 + sv * 0.1);
|
||||
el.style.opacity = String(0.3 + sv * 0.6);
|
||||
el.style.transform = `translateX(-50%) scale(${1 + wave * 0.6}, ${0.45 + sv * 1.25 + wave})`;
|
||||
});
|
||||
raf = requestAnimationFrame(tick);
|
||||
};
|
||||
raf = requestAnimationFrame(tick);
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}, [volumeRef]);
|
||||
|
||||
// Opposite side of the wheel from the chosen accent, with two flanking hues for a real gradient.
|
||||
const { h } = hexToHue(accent || '#6b62f0');
|
||||
// Wide, softly randomized spread around the complement: three tight neon bands read segmented
|
||||
// (the red/orange/yellow stripe complaint); jittered hues + lower saturation blend like a real
|
||||
// aurora while staying distinguishable. Jitter is per-mount so each dictation feels alive.
|
||||
const jitter = React.useMemo(() => [Math.random() * 16 - 8, Math.random() * 12 - 6, Math.random() * 16 - 8], []);
|
||||
const c0 = `hsl(${(h + 158 + jitter[0] + 360) % 360} 68% 63%)`;
|
||||
const c1 = `hsl(${(h + 180 + jitter[1]) % 360} 72% 60%)`;
|
||||
const c2 = `hsl(${(h + 202 + jitter[2]) % 360} 68% 65%)`;
|
||||
return (
|
||||
<div style={{ position: 'fixed', left: 0, right: 0, bottom: 0, height: 170, zIndex: 2147482999, pointerEvents: 'none', overflow: 'visible' }}>
|
||||
{LOBES.map((lobe, i) => {
|
||||
const col = [c0, c1, c2][lobe.hue];
|
||||
return (
|
||||
<div
|
||||
key={i}
|
||||
ref={(el) => { lobeRefs.current[i] = el; }}
|
||||
style={{
|
||||
position: 'absolute', bottom: -40, left: lobe.left, width: lobe.width, height: lobe.height,
|
||||
transform: 'translateX(-50%) scale(1, 0.45)', transformOrigin: 'bottom center',
|
||||
background: `radial-gradient(ellipse at 50% 100%, ${col} 0%, ${col}00 70%)`,
|
||||
filter: 'blur(52px)', opacity: 0.17, willChange: 'transform, opacity',
|
||||
}}
|
||||
/>
|
||||
);
|
||||
})}
|
||||
<div
|
||||
style={{
|
||||
position: 'absolute', left: 0, right: 0, bottom: 0, height: 90,
|
||||
background: `linear-gradient(to right, ${c1}33, ${c1}55, ${c1}33)`,
|
||||
maskImage: 'linear-gradient(to top, black 0%, transparent 100%)',
|
||||
WebkitMaskImage: 'linear-gradient(to top, black 0%, transparent 100%)',
|
||||
filter: 'blur(10px)',
|
||||
}}
|
||||
/>
|
||||
</div>
|
||||
);
|
||||
};
|
||||
|
||||
// WhisperFlow's signature: a small capsule drops from the TOP edge while the mic is hot, carrying a
|
||||
// live waveform. Canvas bars driven straight off the mic level ring buffer, imperative rAF only.
|
||||
const BAR_COUNT = 26;
|
||||
|
||||
const VoiceTab: React.FC<{ volumeRef: React.MutableRefObject<number> }> = ({ volumeRef }) => {
|
||||
const canvasRef = useRef<HTMLCanvasElement | null>(null);
|
||||
const history = useRef<number[]>(new Array(BAR_COUNT).fill(0.06));
|
||||
useEffect(() => {
|
||||
let raf = 0;
|
||||
let frame = 0;
|
||||
const draw = (): void => {
|
||||
const canvas = canvasRef.current;
|
||||
if (canvas) {
|
||||
const g = canvas.getContext('2d');
|
||||
if (g) {
|
||||
// Shift one bar every other frame so the wave scrolls readably at 60Hz input.
|
||||
frame += 1;
|
||||
if (frame % 2 === 0) {
|
||||
history.current.push(Math.min(1, 0.08 + volumeRef.current * 1.6));
|
||||
history.current.shift();
|
||||
}
|
||||
const w = canvas.width;
|
||||
const h = canvas.height;
|
||||
g.clearRect(0, 0, w, h);
|
||||
const bw = w / BAR_COUNT;
|
||||
for (let i = 0; i < BAR_COUNT; i++) {
|
||||
const level = history.current[i];
|
||||
const bh = Math.max(3, level * (h - 4));
|
||||
const x = i * bw + bw * 0.25;
|
||||
g.fillStyle = `rgba(255,255,255,${0.35 + level * 0.6})`;
|
||||
const bwid = bw * 0.5;
|
||||
const y = (h - bh) / 2;
|
||||
g.beginPath();
|
||||
g.roundRect(x, y, bwid, bh, bwid / 2);
|
||||
g.fill();
|
||||
}
|
||||
}
|
||||
}
|
||||
raf = requestAnimationFrame(draw);
|
||||
};
|
||||
raf = requestAnimationFrame(draw);
|
||||
return () => cancelAnimationFrame(raf);
|
||||
}, [volumeRef]);
|
||||
return (
|
||||
<Box
|
||||
sx={{
|
||||
position: 'fixed', top: 10, left: '50%', zIndex: 2147483001, pointerEvents: 'none',
|
||||
display: 'flex', alignItems: 'center', gap: 1, px: 1.5, py: 0.75, borderRadius: 999,
|
||||
background: 'rgba(18,12,26,0.88)',
|
||||
backdropFilter: 'blur(18px) saturate(150%)', WebkitBackdropFilter: 'blur(18px) saturate(150%)',
|
||||
boxShadow: '0 8px 28px rgba(0,0,0,0.4)',
|
||||
'@keyframes vtab-in': { from: { opacity: 0, transform: 'translate(-50%, -14px)' }, to: { opacity: 1, transform: 'translate(-50%, 0)' } },
|
||||
animation: 'vtab-in 0.22s cubic-bezier(0.2, 0.8, 0.2, 1) both',
|
||||
}}
|
||||
>
|
||||
<MicIcon sx={{ fontSize: 14, color: 'rgba(255,255,255,0.75)' }} />
|
||||
<canvas ref={canvasRef} width={132} height={22} style={{ display: 'block', width: 132, height: 22 }} />
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
// The whole point: dictation must never look like "nothing happened." This floats a small status
|
||||
// card above the composer for every phase (listening, transcribing, downloading the model) and shows
|
||||
// the transcript + whether it was pasted or just copied. Non-interactive, auto-dismisses.
|
||||
const FEEDBACK_MS = 4500;
|
||||
|
||||
function feedbackIcon(icon: string): React.ReactElement {
|
||||
if (icon === 'check') return <CheckRoundedIcon sx={{ fontSize: 16, color: '#4ade80' }} />;
|
||||
if (icon === 'clipboard') return <ContentPasteRoundedIcon sx={{ fontSize: 15, color: 'rgba(255,255,255,0.8)' }} />;
|
||||
if (icon === 'mic') return <MicIcon sx={{ fontSize: 16, color: '#ff8a8a' }} />;
|
||||
return <InfoOutlinedIcon sx={{ fontSize: 15, color: 'rgba(255,255,255,0.8)' }} />;
|
||||
}
|
||||
|
||||
const VoiceOverlay: React.FC = () => {
|
||||
const { state, pct, feedback, volumeRef } = useVoice();
|
||||
const [showFeedback, setShowFeedback] = useState(false);
|
||||
|
||||
useEffect(() => {
|
||||
if (!feedback) return undefined;
|
||||
setShowFeedback(true);
|
||||
const t = setTimeout(() => setShowFeedback(false), FEEDBACK_MS);
|
||||
return () => clearTimeout(t);
|
||||
}, [feedback]);
|
||||
|
||||
const live = state !== 'idle';
|
||||
const visible = live || (showFeedback && !!feedback);
|
||||
if (!visible) return null;
|
||||
const aurora = state === 'recording' ? (
|
||||
<>
|
||||
<VoiceAurora volumeRef={volumeRef} />
|
||||
<VoiceTab volumeRef={volumeRef} />
|
||||
</>
|
||||
) : null;
|
||||
|
||||
let content: React.ReactElement | null;
|
||||
if (state === 'recording') {
|
||||
// The aurora IS the listening indicator; a pill on top of it read as clutter.
|
||||
content = null;
|
||||
} else if (state === 'transcribing') {
|
||||
content = (<><CircularProgress size={13} thickness={5} sx={{ color: 'rgba(255,255,255,0.7)' }} /><span>Transcribing</span></>);
|
||||
} else if (state === 'preparing') {
|
||||
content = (<><CircularProgress size={13} thickness={5} sx={{ color: 'rgba(255,255,255,0.7)' }} /><span>Downloading voice model {pct}%</span></>);
|
||||
} else if (feedback) {
|
||||
content = (
|
||||
<>
|
||||
{feedbackIcon(feedback.icon)}
|
||||
<Box component="span" sx={{ maxWidth: 420, overflow: 'hidden', textOverflow: 'ellipsis', whiteSpace: 'nowrap' }}>
|
||||
{feedback.text}
|
||||
</Box>
|
||||
</>
|
||||
);
|
||||
} else {
|
||||
return null;
|
||||
}
|
||||
|
||||
return (
|
||||
<>
|
||||
{aurora}
|
||||
{content && (
|
||||
<Box
|
||||
sx={{
|
||||
position: 'fixed',
|
||||
bottom: 84,
|
||||
left: '50%',
|
||||
transform: 'translateX(-50%)',
|
||||
zIndex: 2147483000,
|
||||
pointerEvents: 'none',
|
||||
display: 'flex',
|
||||
alignItems: 'center',
|
||||
gap: 1,
|
||||
px: 1.75,
|
||||
py: 0.9,
|
||||
maxWidth: '80vw',
|
||||
borderRadius: 999,
|
||||
background: 'rgba(22,12,34,0.9)',
|
||||
backdropFilter: 'blur(20px) saturate(160%)',
|
||||
WebkitBackdropFilter: 'blur(20px) saturate(160%)',
|
||||
boxShadow: '0 8px 28px rgba(0,0,0,0.4)',
|
||||
color: 'rgba(255,255,255,0.92)',
|
||||
fontSize: '0.8125rem',
|
||||
fontWeight: 500,
|
||||
'@keyframes vin': { from: { opacity: 0, transform: 'translate(-50%, 6px)' }, to: { opacity: 1, transform: 'translate(-50%, 0)' } },
|
||||
animation: 'vin 0.16s ease-out',
|
||||
}}
|
||||
>
|
||||
{content}
|
||||
</Box>
|
||||
)}
|
||||
</>
|
||||
);
|
||||
};
|
||||
|
||||
export default VoiceOverlay;
|
||||
@@ -0,0 +1,38 @@
|
||||
// whisper.cpp wants 16kHz mono 16-bit PCM WAV. We capture at 16kHz already (the AudioContext is created
|
||||
// at that rate), so this is just the float32 -> int16 pack plus the 44-byte RIFF header. Kept dead simple:
|
||||
// one channel, no resampling here, so there is nothing to get subtly wrong.
|
||||
|
||||
const SAMPLE_RATE = 16000;
|
||||
|
||||
export function encodeWav(samples: Float32Array): ArrayBuffer {
|
||||
const buffer = new ArrayBuffer(44 + samples.length * 2);
|
||||
const view = new DataView(buffer);
|
||||
|
||||
const writeStr = (offset: number, s: string): void => {
|
||||
for (let i = 0; i < s.length; i++) view.setUint8(offset + i, s.charCodeAt(i));
|
||||
};
|
||||
|
||||
writeStr(0, 'RIFF');
|
||||
view.setUint32(4, 36 + samples.length * 2, true);
|
||||
writeStr(8, 'WAVE');
|
||||
writeStr(12, 'fmt ');
|
||||
view.setUint32(16, 16, true); // PCM chunk size
|
||||
view.setUint16(20, 1, true); // format = PCM
|
||||
view.setUint16(22, 1, true); // channels = mono
|
||||
view.setUint32(24, SAMPLE_RATE, true);
|
||||
view.setUint32(28, SAMPLE_RATE * 2, true); // byte rate
|
||||
view.setUint16(32, 2, true); // block align
|
||||
view.setUint16(34, 16, true); // bits per sample
|
||||
writeStr(36, 'data');
|
||||
view.setUint32(40, samples.length * 2, true);
|
||||
|
||||
let offset = 44;
|
||||
for (let i = 0; i < samples.length; i++) {
|
||||
const clamped = Math.max(-1, Math.min(1, samples[i]));
|
||||
view.setInt16(offset, clamped < 0 ? clamped * 0x8000 : clamped * 0x7fff, true);
|
||||
offset += 2;
|
||||
}
|
||||
return buffer;
|
||||
}
|
||||
|
||||
export const VOICE_SAMPLE_RATE = SAMPLE_RATE;
|
||||
@@ -0,0 +1,44 @@
|
||||
import { getLastInteractedBrowser } from '@/shared/browserFocus';
|
||||
import { getWebview } from '@/shared/browserRegistry';
|
||||
|
||||
// Dictation lands where the user's cursor actually is, like every real dictation tool: a focused
|
||||
// in-app field gets the text typed in (undo-friendly, fires React input events), a focused browser
|
||||
// card forwards into the guest page's field, anything else falls back to the OS-level paste.
|
||||
export type InjectTarget = 'field' | 'webview' | null;
|
||||
|
||||
export function injectAtFocus(text: string): InjectTarget {
|
||||
const active = document.activeElement as HTMLElement | null;
|
||||
if (active && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA' || active.isContentEditable)) {
|
||||
try {
|
||||
active.focus();
|
||||
// execCommand keeps the undo stack and fires the input events React listens for; the manual
|
||||
// fallback covers fields where Chromium refuses the command (rare, e.g. type=number).
|
||||
const ok = document.execCommand('insertText', false, text);
|
||||
if (!ok && (active.tagName === 'INPUT' || active.tagName === 'TEXTAREA')) {
|
||||
const el = active as HTMLInputElement | HTMLTextAreaElement;
|
||||
const s = el.selectionStart ?? el.value.length;
|
||||
const e = el.selectionEnd ?? el.value.length;
|
||||
el.value = el.value.slice(0, s) + text + el.value.slice(e);
|
||||
el.selectionStart = el.selectionEnd = s + text.length;
|
||||
el.dispatchEvent(new Event('input', { bubbles: true }));
|
||||
}
|
||||
return 'field';
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
// A webview steals focus when the user clicks into a page, so activeElement IS the webview tag.
|
||||
const focusedTag = active && active.tagName === 'WEBVIEW' ? (active as unknown as { insertText?: (t: string) => Promise<void> }) : null;
|
||||
if (focusedTag?.insertText) {
|
||||
try { void focusedTag.insertText(text); return 'webview'; } catch { /* fall through */ }
|
||||
}
|
||||
// Last-interacted browser card: the user clicked a page field, then hit the hotkey.
|
||||
const browserId = getLastInteractedBrowser();
|
||||
if (browserId) {
|
||||
const wv = getWebview(browserId) as unknown as { insertText?: (t: string) => Promise<void>; focus?: () => void } | undefined;
|
||||
if (wv?.insertText) {
|
||||
try { wv.focus?.(); void wv.insertText(text); return 'webview'; } catch { /* fall through */ }
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
@@ -0,0 +1,198 @@
|
||||
import { useCallback, useEffect, useRef, useState } from 'react';
|
||||
import { API_BASE } from '@/shared/config';
|
||||
import { encodeWav, VOICE_SAMPLE_RATE } from './encodeWav';
|
||||
import { playVoiceCue } from './voiceCues';
|
||||
import { injectAtFocus } from './injectAtFocus';
|
||||
|
||||
export type VoiceState = 'idle' | 'recording' | 'transcribing' | 'preparing';
|
||||
|
||||
// WhisperFlow-style push-to-dictate: toggle recording (global hotkey or a mic), speak, and the
|
||||
// transcribed text is pasted into whatever field has focus. Capture is 16kHz mono PCM so it feeds
|
||||
// whisper.cpp with no server-side resample. The recording path can only be proven with a real mic;
|
||||
// the encode -> transcribe -> inject half is exercised by the encoder round-trip test.
|
||||
interface Recorder {
|
||||
ctx: AudioContext;
|
||||
stream: MediaStream;
|
||||
node: ScriptProcessorNode;
|
||||
source: MediaStreamAudioSourceNode;
|
||||
chunks: Float32Array[];
|
||||
}
|
||||
|
||||
// One object per terminal outcome so the overlay's effect always re-fires (new identity every time).
|
||||
export interface VoiceFeedback {
|
||||
tone: 'ok' | 'warn' | 'error';
|
||||
icon: 'check' | 'clipboard' | 'mic' | 'info';
|
||||
text: string;
|
||||
at: number;
|
||||
}
|
||||
|
||||
// Context hint for the polisher: what the user is dictating into (a field label, a page title), so
|
||||
// names and jargon spell right. Never page CONTENT, just the one-line "where".
|
||||
function dictationContext(): string {
|
||||
const active = document.activeElement as HTMLElement | null;
|
||||
const hint = active?.getAttribute?.('placeholder') || active?.getAttribute?.('aria-label') || '';
|
||||
return [hint, document.title].filter(Boolean).join(' - ').slice(0, 200);
|
||||
}
|
||||
|
||||
async function polishText(raw: string): Promise<string> {
|
||||
try {
|
||||
const ctl = new AbortController();
|
||||
const timer = window.setTimeout(() => ctl.abort(), 6500);
|
||||
const res = await fetch(`${API_BASE}/voice/polish`, {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ text: raw, context: dictationContext() }),
|
||||
signal: ctl.signal,
|
||||
});
|
||||
window.clearTimeout(timer);
|
||||
if (!res.ok) return raw;
|
||||
const data = (await res.json()) as { text?: string };
|
||||
return data.text?.trim() || raw;
|
||||
} catch {
|
||||
return raw;
|
||||
}
|
||||
}
|
||||
|
||||
export function useVoiceDictation() {
|
||||
const [state, setState] = useState<VoiceState>('idle');
|
||||
const [lastText, setLastText] = useState<string>('');
|
||||
const [error, setError] = useState<string | null>(null);
|
||||
const [pct, setPct] = useState<number>(0);
|
||||
const [feedback, setFeedback] = useState<VoiceFeedback | null>(null);
|
||||
const recRef = useRef<Recorder | null>(null);
|
||||
const stateRef = useRef<VoiceState>('idle');
|
||||
// Live mic level (0..1) for the aurora; a ref, not state, so 60Hz visuals never re-render React.
|
||||
const volumeRef = useRef<number>(0);
|
||||
stateRef.current = state;
|
||||
|
||||
// First-run: the model is downloading. Poll progress until it lands, then drop back to idle so the
|
||||
// next click records for real. Never records while preparing, so nothing is lost to a dropped phrase.
|
||||
const pollModel = useCallback((): void => {
|
||||
const tick = async (): Promise<void> => {
|
||||
const st = await window.openswarm?.voiceStatus?.();
|
||||
if (!st) { setState('idle'); return; }
|
||||
setPct(st.pct || 0);
|
||||
if (st.error) { setError(st.error); setState('idle'); return; }
|
||||
if (!st.downloading) { setState('idle'); return; }
|
||||
setTimeout(() => { void tick(); }, 1000);
|
||||
};
|
||||
void tick();
|
||||
}, []);
|
||||
|
||||
const teardown = useCallback((): Float32Array | null => {
|
||||
const rec = recRef.current;
|
||||
recRef.current = null;
|
||||
if (!rec) return null;
|
||||
try { rec.node.disconnect(); } catch (_) { /* already gone */ }
|
||||
try { rec.source.disconnect(); } catch (_) { /* already gone */ }
|
||||
try { rec.stream.getTracks().forEach((t) => t.stop()); } catch (_) { /* already gone */ }
|
||||
try { void rec.ctx.close(); } catch (_) { /* already gone */ }
|
||||
const total = rec.chunks.reduce((n, c) => n + c.length, 0);
|
||||
if (!total) return new Float32Array(0);
|
||||
const out = new Float32Array(total);
|
||||
let off = 0;
|
||||
for (const c of rec.chunks) { out.set(c, off); off += c.length; }
|
||||
return out;
|
||||
}, []);
|
||||
|
||||
const start = useCallback(async (): Promise<void> => {
|
||||
if (stateRef.current !== 'idle') return;
|
||||
if (!window.openswarm?.voiceTranscribe) { setError('desktop-only'); return; } // no Electron bridge = web build
|
||||
setError(null);
|
||||
try {
|
||||
const stream = await navigator.mediaDevices.getUserMedia({ audio: { channelCount: 1, echoCancellation: true, noiseSuppression: true } });
|
||||
const ctx = new AudioContext({ sampleRate: VOICE_SAMPLE_RATE });
|
||||
const source = ctx.createMediaStreamSource(stream);
|
||||
const node = ctx.createScriptProcessor(4096, 1, 1);
|
||||
const chunks: Float32Array[] = [];
|
||||
node.onaudioprocess = (e): void => {
|
||||
const data = e.inputBuffer.getChannelData(0);
|
||||
chunks.push(new Float32Array(data));
|
||||
// RMS per chunk drives the aurora; smoothed so it breathes instead of flickering.
|
||||
let sum = 0;
|
||||
for (let i = 0; i < data.length; i += 8) sum += data[i] * data[i];
|
||||
const rms = Math.sqrt(sum / (data.length / 8));
|
||||
volumeRef.current = volumeRef.current * 0.7 + Math.min(1, rms * 6) * 0.3;
|
||||
};
|
||||
source.connect(node);
|
||||
node.connect(ctx.destination);
|
||||
recRef.current = { ctx, stream, node, source, chunks };
|
||||
setState('recording');
|
||||
playVoiceCue('start');
|
||||
void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('generic');
|
||||
// Warm the model the moment recording begins so transcription is instant on stop.
|
||||
void window.openswarm?.voiceWarmup?.();
|
||||
} catch (err) {
|
||||
const msg = err instanceof Error ? err.message : 'mic-unavailable';
|
||||
setError(msg);
|
||||
const denied = /NotAllowed|Permission|denied/i.test(msg);
|
||||
setFeedback({ tone: 'error', icon: 'mic', text: denied ? 'Microphone access needed. Enable it in System Settings, Privacy, Microphone.' : 'Could not start the microphone.', at: Date.now() });
|
||||
setState('idle');
|
||||
}
|
||||
}, []);
|
||||
|
||||
const stop = useCallback(async (): Promise<void> => {
|
||||
if (stateRef.current !== 'recording') return;
|
||||
const samples = teardown();
|
||||
playVoiceCue('stop');
|
||||
void (window.openswarm as { haptic?: (p: string) => Promise<boolean> } | undefined)?.haptic?.('alignment');
|
||||
setState('transcribing');
|
||||
try {
|
||||
if (!samples || samples.length < VOICE_SAMPLE_RATE * 0.2) { setState('idle'); return; } // < 0.2s = a misfire
|
||||
const wav = encodeWav(samples);
|
||||
const res = await window.openswarm?.voiceTranscribe?.(wav);
|
||||
if (res?.ok && res.text) {
|
||||
// WhisperFlow-style cleanup: punctuation + filler words via the cheap aux tier, fail-open to
|
||||
// the raw transcript on any error/timeout so dictation never breaks with the aux down.
|
||||
// Latency: short phrases don't need the aux cleanup (whisper's raw output is fine), and a
|
||||
// slow aux must never hold the cursor hostage; 900ms is the most polish is allowed to cost.
|
||||
const raw = res.text.trim();
|
||||
const text = raw.split(/\s+/).length <= 6
|
||||
? raw
|
||||
: await Promise.race([polishText(raw), new Promise<string>((r) => window.setTimeout(() => r(raw), 900))]);
|
||||
setLastText(text);
|
||||
// Land the text where the user's cursor is: focused field, then focused browser page, then
|
||||
// the OS paste fallback (other apps). The floating bubble is just confirmation, not the output.
|
||||
// Success is silent: the text landing at the cursor IS the feedback. Only the clipboard
|
||||
// fallback still speaks, because the user has to act (paste) to get the text.
|
||||
const target = injectAtFocus(text);
|
||||
if (!target) {
|
||||
const inj = await window.openswarm?.voiceInject?.(text);
|
||||
if (!inj?.pasted) setFeedback({ tone: 'ok', icon: 'clipboard', text: `${text} (copied, press Cmd+V)`, at: Date.now() });
|
||||
}
|
||||
setState('idle');
|
||||
} else if (res?.ok && !res.text) {
|
||||
setFeedback({ tone: 'warn', icon: 'info', text: "Didn't catch that. Try again.", at: Date.now() });
|
||||
setState('idle');
|
||||
} else if (res?.error === 'model-downloading' || res?.error === 'no-model') {
|
||||
// First use kicked off the model fetch; show progress and don't error out.
|
||||
setState('preparing');
|
||||
pollModel();
|
||||
} else {
|
||||
setError(res?.error || 'transcription-failed');
|
||||
setFeedback({ tone: 'error', icon: 'info', text: 'Voice transcription failed. Try again.', at: Date.now() });
|
||||
setState('idle');
|
||||
}
|
||||
} catch (err) {
|
||||
setError(err instanceof Error ? err.message : 'transcription-failed');
|
||||
setFeedback({ tone: 'error', icon: 'info', text: 'Voice transcription failed. Try again.', at: Date.now() });
|
||||
setState('idle');
|
||||
}
|
||||
}, [teardown, pollModel]);
|
||||
|
||||
const toggle = useCallback((): void => {
|
||||
if (stateRef.current === 'recording') void stop();
|
||||
else if (stateRef.current === 'idle') void start();
|
||||
}, [start, stop]);
|
||||
|
||||
// Global hotkey (CommandOrControl+Shift+D) routes here from the main process.
|
||||
useEffect(() => {
|
||||
const off = window.openswarm?.onVoiceToggle?.(() => toggle());
|
||||
return () => { off?.(); };
|
||||
}, [toggle]);
|
||||
|
||||
// A dangling recorder (unmount mid-capture) must release the mic.
|
||||
useEffect(() => () => { teardown(); }, [teardown]);
|
||||
|
||||
return { state, lastText, error, pct, feedback, toggle, start, stop, volumeRef };
|
||||
}
|
||||
@@ -0,0 +1,30 @@
|
||||
import React, { createContext, useContext } from 'react';
|
||||
import { VoiceState, VoiceFeedback } from './useVoiceDictation';
|
||||
|
||||
// The context lives below both the provider and the overlay so neither imports the other
|
||||
// (VoiceDictationContext renders VoiceOverlay; both reach down here instead of sideways).
|
||||
export interface VoiceContextValue {
|
||||
state: VoiceState;
|
||||
lastText: string;
|
||||
error: string | null;
|
||||
pct: number;
|
||||
feedback: VoiceFeedback | null;
|
||||
toggle: () => void;
|
||||
// Mic-button press semantics that respect the hold/toggle setting: press starts (or toggles),
|
||||
// release stops only in hold mode. Buttons wire onPointerDown/Up to these and stay mode-agnostic.
|
||||
pressStart: () => void;
|
||||
pressEnd: () => void;
|
||||
holdMode: boolean;
|
||||
volumeRef: React.MutableRefObject<number>;
|
||||
}
|
||||
|
||||
const NOOP_REF = { current: 0 };
|
||||
const NOOP: VoiceContextValue = { state: 'idle', lastText: '', error: null, pct: 0, feedback: null, toggle: () => {}, pressStart: () => {}, pressEnd: () => {}, holdMode: true, volumeRef: NOOP_REF };
|
||||
|
||||
export const VoiceContext = createContext<VoiceContextValue>(NOOP);
|
||||
|
||||
// A component rendered outside the provider (or a web build with no Electron bridge) gets the no-op,
|
||||
// so mics still render and just do nothing rather than crashing.
|
||||
export function useVoice(): VoiceContextValue {
|
||||
return useContext(VoiceContext);
|
||||
}
|
||||
@@ -0,0 +1,46 @@
|
||||
// WhisperFlow-grade start/stop chimes, synthesized in WebAudio so they ship weightless and always
|
||||
// match: a soft two-note rise on start ("I'm listening"), the mirrored fall on stop. Sine + gentle
|
||||
// lowpass + fast attack / exponential release = crisp but soothing, never a system beep.
|
||||
|
||||
let ctx: AudioContext | null = null;
|
||||
|
||||
function ensureCtx(): AudioContext | null {
|
||||
try {
|
||||
if (!ctx || ctx.state === 'closed') ctx = new AudioContext();
|
||||
if (ctx.state === 'suspended') void ctx.resume();
|
||||
return ctx;
|
||||
} catch {
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
function blip(ac: AudioContext, freq: number, at: number, dur: number, peak: number): void {
|
||||
const osc = ac.createOscillator();
|
||||
const gain = ac.createGain();
|
||||
const lp = ac.createBiquadFilter();
|
||||
lp.type = 'lowpass';
|
||||
lp.frequency.value = 2400;
|
||||
osc.type = 'sine';
|
||||
osc.frequency.setValueAtTime(freq, at);
|
||||
gain.gain.setValueAtTime(0, at);
|
||||
gain.gain.linearRampToValueAtTime(peak, at + 0.012);
|
||||
gain.gain.exponentialRampToValueAtTime(0.0004, at + dur);
|
||||
osc.connect(lp);
|
||||
lp.connect(gain);
|
||||
gain.connect(ac.destination);
|
||||
osc.start(at);
|
||||
osc.stop(at + dur + 0.02);
|
||||
}
|
||||
|
||||
export function playVoiceCue(kind: 'start' | 'stop'): void {
|
||||
const ac = ensureCtx();
|
||||
if (!ac) return;
|
||||
const t = ac.currentTime + 0.01;
|
||||
if (kind === 'start') {
|
||||
blip(ac, 587, t, 0.16, 0.055);
|
||||
blip(ac, 880, t + 0.085, 0.2, 0.05);
|
||||
} else {
|
||||
blip(ac, 880, t, 0.14, 0.045);
|
||||
blip(ac, 587, t + 0.075, 0.22, 0.05);
|
||||
}
|
||||
}
|
||||
@@ -28,7 +28,7 @@ import {
|
||||
clearTurnLabel,
|
||||
} from '../state/agentsSlice';
|
||||
import { streamStart, streamDelta, streamEnd, clearStreamingForSession } from '../state/streamingSlice';
|
||||
import { addBrowserCardFromBackend, markBrowserCardEnding, keepBrowserCardOpen, placeBesideCard, placeBelowCard, placeBrowserBesideChat, setBrowserCardPosition, setGlowingBrowserCards, fadeGlowingBrowserCards, clearGlowingBrowserCards, removeBrowserCard, GRID_GAP, WORKFLOW_CARD_GAP, openWorkflowsApp, openWorkflowMonitor } from '../state/dashboardLayoutSlice';
|
||||
import { addBrowserCardFromBackend, setBrowserDocked, markBrowserCardEnding, keepBrowserCardOpen, placeBesideCard, placeBelowCard, placeBrowserBesideChat, setBrowserCardPosition, setGlowingBrowserCards, fadeGlowingBrowserCards, clearGlowingBrowserCards, removeBrowserCard, GRID_GAP, WORKFLOW_CARD_GAP, openWorkflowsApp, openWorkflowMonitor } from '../state/dashboardLayoutSlice';
|
||||
import { upsertOutput } from '../state/outputsSlice';
|
||||
import { fetchSettings } from '../state/settingsSlice';
|
||||
import { displaySessionName } from '../state/sessionDisplay';
|
||||
@@ -847,6 +847,9 @@ class WebSocketManager {
|
||||
let glowLabel = 'Use Browser';
|
||||
if (parentCard) {
|
||||
pos = placeBrowserBesideChat(layoutState, parentCard, parentId, browserCard.width, browserCard.height, browserCard.browser_id);
|
||||
// Default home is INSIDE the chat: the card overlays the chat's dock slot while the
|
||||
// chat is expanded; the beside-chat spot stays the undock/collapse fallback.
|
||||
store.dispatch(setBrowserDocked({ browserId: browserCard.browser_id, dockedTo: parentId }));
|
||||
} else if (sess?.workflow_run_id && layoutState.workflowsMonitorCard) {
|
||||
pos = placeBesideCard(layoutState, layoutState.workflowsMonitorCard, browserCard.width, browserCard.height, undefined, exclude, WORKFLOW_CARD_GAP, true);
|
||||
} else if (sess?.workflow_edit_id && layoutState.workflowsHub) {
|
||||
|
||||
Reference in New Issue
Block a user