mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] crash: safe-mode loop breaker; two dirty exits in ten minutes boots with webviews parked as screenshots and app runtimes held until clicked, with a crash fingerprint from the newest minidump
This commit is contained in:
@@ -645,6 +645,8 @@ function spawnCrashWatchdog() {
|
||||
}
|
||||
|
||||
function writeCleanQuitLock() {
|
||||
// Session lock clears on EVERY platform; only the watchdog half below is mac-specific.
|
||||
try { fs.unlinkSync(SESSION_RUNNING_LOCK); } catch (_) {}
|
||||
if (process.platform !== 'darwin') return;
|
||||
try {
|
||||
if (!fs.existsSync(CRASH_WATCHDOG_SUPPORT_DIR)) fs.mkdirSync(CRASH_WATCHDOG_SUPPORT_DIR, { recursive: true });
|
||||
@@ -652,6 +654,54 @@ function writeCleanQuitLock() {
|
||||
} catch (_) {}
|
||||
}
|
||||
|
||||
// Safe-mode loop breaker (ENG-228). A session lock written at boot and cleared on clean quit makes
|
||||
// dirty exits detectable without any crash handler firing; two dirty exits inside ten minutes means
|
||||
// relaunching keeps rebuilding the exact state that dies, so the NEXT boot restores layout with
|
||||
// webviews parked as screenshots until clicked. Also grabs a crash fingerprint (exception name +
|
||||
// address from the newest Crashpad minidump) so the renderer chip and diagnostics can say WHAT died.
|
||||
const SESSION_RUNNING_LOCK = path.join(CRASH_WATCHDOG_SUPPORT_DIR, 'session-running.lock');
|
||||
const DIRTY_EXITS_LOG = path.join(CRASH_WATCHDOG_SUPPORT_DIR, 'dirty-exits.json');
|
||||
const SAFE_MODE_WINDOW_MS = 10 * 60 * 1000;
|
||||
const SAFE_MODE_THRESHOLD = 2;
|
||||
let safeModeInfo = { safeMode: false, dirtyCount: 0, fingerprint: null };
|
||||
|
||||
function scanCrashFingerprint(sinceMs) {
|
||||
try {
|
||||
const { newDumpsSince } = require('./crashDumpScan');
|
||||
const crashpadDir = path.join(CRASH_WATCHDOG_SUPPORT_DIR, 'Crashpad', 'completed');
|
||||
const dumps = newDumpsSince(crashpadDir, sinceMs, 1);
|
||||
if (!dumps || !dumps.length) return null;
|
||||
const d = dumps[0];
|
||||
return { exception: d.exception_name || null, code: d.exception_code || null, address: d.exception_address || null, mtime: d.mtime_ms || null };
|
||||
} catch (_) { return null; }
|
||||
}
|
||||
|
||||
function detectDirtyExitAndArmSafeMode() {
|
||||
try {
|
||||
if (!fs.existsSync(CRASH_WATCHDOG_SUPPORT_DIR)) fs.mkdirSync(CRASH_WATCHDOG_SUPPORT_DIR, { recursive: true });
|
||||
let lastBootTs = 0;
|
||||
const wasDirty = fs.existsSync(SESSION_RUNNING_LOCK);
|
||||
if (wasDirty) {
|
||||
try { lastBootTs = parseInt(fs.readFileSync(SESSION_RUNNING_LOCK, 'utf-8'), 10) || 0; } catch (_) {}
|
||||
}
|
||||
let stamps = [];
|
||||
try { stamps = JSON.parse(fs.readFileSync(DIRTY_EXITS_LOG, 'utf-8')); } catch (_) {}
|
||||
const cutoff = Date.now() - SAFE_MODE_WINDOW_MS;
|
||||
stamps = (Array.isArray(stamps) ? stamps : []).filter((t) => typeof t === 'number' && t > cutoff);
|
||||
if (wasDirty) stamps.push(Date.now());
|
||||
try { fs.writeFileSync(DIRTY_EXITS_LOG, JSON.stringify(stamps)); } catch (_) {}
|
||||
safeModeInfo.dirtyCount = stamps.length;
|
||||
safeModeInfo.safeMode = stamps.length >= SAFE_MODE_THRESHOLD;
|
||||
if (wasDirty) safeModeInfo.fingerprint = scanCrashFingerprint(lastBootTs || cutoff);
|
||||
fs.writeFileSync(SESSION_RUNNING_LOCK, String(Date.now()));
|
||||
if (wasDirty) console.log('[safe-mode] dirty exit detected; count=', safeModeInfo.dirtyCount, 'safeMode=', safeModeInfo.safeMode, 'fingerprint=', JSON.stringify(safeModeInfo.fingerprint));
|
||||
} catch (e) {
|
||||
console.warn('[safe-mode] detect failed:', e && e.message);
|
||||
}
|
||||
}
|
||||
|
||||
ipcMain.handle('get-safe-mode', () => safeModeInfo);
|
||||
|
||||
// Quit-cause forensics. On a real quit (Cmd+Q, dock Quit, app.quit()) Electron
|
||||
// fires before-quit BEFORE any window 'close' events; a window closing on its
|
||||
// own (Cmd+W, red X, programmatic close) fires 'close' with quitInitiated
|
||||
@@ -2019,6 +2069,7 @@ app.whenReady().then(async () => {
|
||||
// Spawn the Mac crash watchdog. Detached process; if it fails to spawn the
|
||||
// app continues normally (silent fail by design). Guards inside the
|
||||
// watchdog itself prevent false-positive relaunches.
|
||||
detectDirtyExitAndArmSafeMode();
|
||||
spawnCrashWatchdog();
|
||||
|
||||
// Off-window mouse-release crash dodge (macOS). Safe to call before windows exist.
|
||||
|
||||
@@ -47,6 +47,7 @@ contextBridge.exposeInMainWorld('openswarm', {
|
||||
|
||||
// Phase 2 provenance: { sha, shortSha, builtAt, channel } for the About panel.
|
||||
getBuildInfo: () => ipcRenderer.invoke('get-build-info'),
|
||||
getSafeMode: () => ipcRenderer.invoke('get-safe-mode'),
|
||||
|
||||
// Phase 0 boot instrumentation: renderer calls this exactly once, when the
|
||||
// first streamed token of the first agent response paints. Fire-and-forget
|
||||
|
||||
@@ -25,6 +25,7 @@ import { ackRun, runWorkflowNow } from '@/shared/state/workflowsSlice';
|
||||
import { setPendingBrowserUrl } from '@/shared/state/tempStateSlice';
|
||||
import { fetchOutputs } from '@/shared/state/outputsSlice';
|
||||
import UpdateReadyPill from '@/app/components/Layout/UpdateReadyPill';
|
||||
import SafeModePill from '@/app/components/Layout/SafeModePill';
|
||||
import WhatsNewCard from '@/app/components/Layout/WhatsNewCard';
|
||||
import ShareRequestHost from '@/app/components/share/ShareRequestHost';
|
||||
import AppToolGrantHost from '@/app/components/apps/AppToolGrantHost';
|
||||
@@ -560,6 +561,7 @@ const AppShell: React.FC = () => {
|
||||
</Collapse>
|
||||
|
||||
{!fsHideChrome && <UpdateReadyPill />}
|
||||
{!fsHideChrome && <SafeModePill />}
|
||||
|
||||
<Box sx={{ display: 'flex', flex: 1, minHeight: 0 }}>
|
||||
{/* Sidebar excised: dashboards live in the Spaces strip (hover the top edge; right-click a tile for rename/duplicate/delete). */}
|
||||
|
||||
@@ -0,0 +1,41 @@
|
||||
import React, { useState } from 'react';
|
||||
import Box from '@mui/material/Box';
|
||||
import Typography from '@mui/material/Typography';
|
||||
import ShieldOutlinedIcon from '@mui/icons-material/ShieldOutlined';
|
||||
import CloseIcon from '@mui/icons-material/Close';
|
||||
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
|
||||
import { safeModeInfo } from '@/shared/safeMode';
|
||||
|
||||
// Shown only when the loop breaker armed (two dirty exits in ten minutes, ENG-228): tells the user
|
||||
// why their browsers and apps boot paused instead of looking silently broken. Dismiss is session-only.
|
||||
const SafeModePill: React.FC = () => {
|
||||
const c = useClaudeTokens();
|
||||
const [dismissed, setDismissed] = useState(false);
|
||||
const info = safeModeInfo();
|
||||
if (!info.safeMode || dismissed) return null;
|
||||
return (
|
||||
<Box
|
||||
sx={{
|
||||
position: 'fixed', top: 34, left: '50%', transform: 'translateX(-50%)',
|
||||
zIndex: 1400, WebkitAppRegion: 'no-drag',
|
||||
display: 'flex', alignItems: 'center', gap: 1,
|
||||
px: 1.5, py: 0.75, borderRadius: '10px',
|
||||
bgcolor: c.bg.surface, border: `1px solid ${c.border.subtle}`,
|
||||
boxShadow: '0 4px 16px rgba(0,0,0,0.18)',
|
||||
}}
|
||||
>
|
||||
<ShieldOutlinedIcon sx={{ fontSize: 16, color: c.text.tertiary }} />
|
||||
<Typography sx={{ fontSize: '0.8125rem', color: c.text.primary }}>
|
||||
Recovered after repeated crashes. Browsers and apps are paused; click one to resume it.
|
||||
</Typography>
|
||||
<Box
|
||||
role="button" aria-label="Dismiss safe mode notice" onClick={() => setDismissed(true)}
|
||||
sx={{ display: 'flex', alignItems: 'center', cursor: 'pointer', color: c.text.tertiary, '&:hover': { color: c.text.primary } }}
|
||||
>
|
||||
<CloseIcon sx={{ fontSize: 14 }} />
|
||||
</Box>
|
||||
</Box>
|
||||
);
|
||||
};
|
||||
|
||||
export default SafeModePill;
|
||||
@@ -2,6 +2,7 @@
|
||||
|
||||
import { useEffect, useRef, useState } from 'react';
|
||||
import { API_BASE, getAuthToken } from '@/shared/config';
|
||||
import { isSafeMode } from '@/shared/safeMode';
|
||||
|
||||
export interface RuntimeLogLine {
|
||||
source: 'backend' | 'runtime';
|
||||
@@ -56,13 +57,16 @@ export function useRuntimePreviewUrl(opts: RuntimePreviewOptions): RuntimePrevie
|
||||
|
||||
const connect = async (): Promise<void> => {
|
||||
if (cancelled) return;
|
||||
try {
|
||||
await fetch(`${API_BASE}/outputs/workspace/${workspaceId}/runtime/start?instance=${instance}`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
});
|
||||
} catch (_) {
|
||||
// Spawn errors surface via the log WS; don't double-report.
|
||||
// Safe mode (ENG-228): after repeated dirty exits, app runtimes don't auto-boot on card mount; the card's restart button is the explicit resume, so a crash loop can't respawn the surface storm.
|
||||
if (!isSafeMode()) {
|
||||
try {
|
||||
await fetch(`${API_BASE}/outputs/workspace/${workspaceId}/runtime/start?instance=${instance}`, {
|
||||
method: 'POST',
|
||||
headers,
|
||||
});
|
||||
} catch (_) {
|
||||
// Spawn errors surface via the log WS; don't double-report.
|
||||
}
|
||||
}
|
||||
if (cancelled) return;
|
||||
try {
|
||||
|
||||
@@ -0,0 +1,26 @@
|
||||
// Safe-mode loop breaker (ENG-228): after two dirty exits in ten minutes the main process arms
|
||||
// safe mode, and this boot restores layout with webviews parked as screenshots until clicked, so a
|
||||
// relaunch stops rebuilding the exact state that crashed. Resolved once at bundle eval; the IPC
|
||||
// round-trip finishes long before the layout fetch that first reads it.
|
||||
export interface SafeModeInfo {
|
||||
safeMode: boolean;
|
||||
dirtyCount: number;
|
||||
fingerprint: { exception: string | null; code: number | null; address: number | null } | null;
|
||||
}
|
||||
|
||||
let cached: SafeModeInfo = { safeMode: false, dirtyCount: 0, fingerprint: null };
|
||||
|
||||
const api = (window as unknown as { openswarm?: { getSafeMode?: () => Promise<SafeModeInfo> } }).openswarm;
|
||||
if (api?.getSafeMode) {
|
||||
void api.getSafeMode().then((info) => {
|
||||
if (info && typeof info.safeMode === 'boolean') cached = info;
|
||||
}).catch(() => {});
|
||||
}
|
||||
|
||||
export function safeModeInfo(): SafeModeInfo {
|
||||
return cached;
|
||||
}
|
||||
|
||||
export function isSafeMode(): boolean {
|
||||
return cached.safeMode;
|
||||
}
|
||||
@@ -3,6 +3,7 @@ import { launchAndSendFirstMessage, resumeSession, collapseSession, collapseAllS
|
||||
import { untileClosedChats } from './untileClosedChats';
|
||||
import { API_BASE } from '@/shared/config';
|
||||
import { getLastDashboardId } from '@/shared/lastDashboardId';
|
||||
import { isSafeMode } from '@/shared/safeMode';
|
||||
|
||||
// fetchSession 404/410 strips the layout card to stop AgentChat remount-loop. Matched by string to avoid circular import.
|
||||
const fetchSessionRejectedAction = createAction<
|
||||
@@ -1886,6 +1887,12 @@ const dashboardLayoutSlice = createSlice({
|
||||
for (const id of Object.keys(state.tiledCards)) {
|
||||
if (!tileOwnerExists(state, id)) delete state.tiledCards[id];
|
||||
}
|
||||
// Safe mode (ENG-228): after repeated dirty exits, every browser webview boots parked as a screenshot; clicking a card resumes it (the existing suspend/resume path), so a crash loop can't rebuild the surface storm that killed the last session.
|
||||
if (isSafeMode()) {
|
||||
for (const id of Object.keys(state.browserCards)) {
|
||||
if (!state.suspendedBrowserCards[id]) state.suspendedBrowserCards[id] = { dataUrl: '', capturedAt: 0 };
|
||||
}
|
||||
}
|
||||
// Pre-1.7.6 profiles can persist several live 'fullscreen' entries; the selector crowns the first, so every OTHER card's drag guard compares against the wrong owner and lets the drag through. One owner, same rule as the write reducer.
|
||||
let fsOwner: string | null = null;
|
||||
for (const [id, zone] of Object.entries(state.tiledCards)) {
|
||||
|
||||
Reference in New Issue
Block a user