[eric] crash: safe-mode loop breaker; two dirty exits in ten minutes boots with webviews parked as screenshots and app runtimes held until clicked, with a crash fingerprint from the newest minidump

This commit is contained in:
ciregenz
2026-08-10 08:36:47 -07:00
parent 2d5f35ad6e
commit df8af72125
7 changed files with 139 additions and 7 deletions
+51
View File
@@ -645,6 +645,8 @@ function spawnCrashWatchdog() {
}
function writeCleanQuitLock() {
// Session lock clears on EVERY platform; only the watchdog half below is mac-specific.
try { fs.unlinkSync(SESSION_RUNNING_LOCK); } catch (_) {}
if (process.platform !== 'darwin') return;
try {
if (!fs.existsSync(CRASH_WATCHDOG_SUPPORT_DIR)) fs.mkdirSync(CRASH_WATCHDOG_SUPPORT_DIR, { recursive: true });
@@ -652,6 +654,54 @@ function writeCleanQuitLock() {
} catch (_) {}
}
// Safe-mode loop breaker (ENG-228). A session lock written at boot and cleared on clean quit makes
// dirty exits detectable without any crash handler firing; two dirty exits inside ten minutes means
// relaunching keeps rebuilding the exact state that dies, so the NEXT boot restores layout with
// webviews parked as screenshots until clicked. Also grabs a crash fingerprint (exception name +
// address from the newest Crashpad minidump) so the renderer chip and diagnostics can say WHAT died.
const SESSION_RUNNING_LOCK = path.join(CRASH_WATCHDOG_SUPPORT_DIR, 'session-running.lock');
const DIRTY_EXITS_LOG = path.join(CRASH_WATCHDOG_SUPPORT_DIR, 'dirty-exits.json');
const SAFE_MODE_WINDOW_MS = 10 * 60 * 1000;
const SAFE_MODE_THRESHOLD = 2;
let safeModeInfo = { safeMode: false, dirtyCount: 0, fingerprint: null };
function scanCrashFingerprint(sinceMs) {
try {
const { newDumpsSince } = require('./crashDumpScan');
const crashpadDir = path.join(CRASH_WATCHDOG_SUPPORT_DIR, 'Crashpad', 'completed');
const dumps = newDumpsSince(crashpadDir, sinceMs, 1);
if (!dumps || !dumps.length) return null;
const d = dumps[0];
return { exception: d.exception_name || null, code: d.exception_code || null, address: d.exception_address || null, mtime: d.mtime_ms || null };
} catch (_) { return null; }
}
function detectDirtyExitAndArmSafeMode() {
try {
if (!fs.existsSync(CRASH_WATCHDOG_SUPPORT_DIR)) fs.mkdirSync(CRASH_WATCHDOG_SUPPORT_DIR, { recursive: true });
let lastBootTs = 0;
const wasDirty = fs.existsSync(SESSION_RUNNING_LOCK);
if (wasDirty) {
try { lastBootTs = parseInt(fs.readFileSync(SESSION_RUNNING_LOCK, 'utf-8'), 10) || 0; } catch (_) {}
}
let stamps = [];
try { stamps = JSON.parse(fs.readFileSync(DIRTY_EXITS_LOG, 'utf-8')); } catch (_) {}
const cutoff = Date.now() - SAFE_MODE_WINDOW_MS;
stamps = (Array.isArray(stamps) ? stamps : []).filter((t) => typeof t === 'number' && t > cutoff);
if (wasDirty) stamps.push(Date.now());
try { fs.writeFileSync(DIRTY_EXITS_LOG, JSON.stringify(stamps)); } catch (_) {}
safeModeInfo.dirtyCount = stamps.length;
safeModeInfo.safeMode = stamps.length >= SAFE_MODE_THRESHOLD;
if (wasDirty) safeModeInfo.fingerprint = scanCrashFingerprint(lastBootTs || cutoff);
fs.writeFileSync(SESSION_RUNNING_LOCK, String(Date.now()));
if (wasDirty) console.log('[safe-mode] dirty exit detected; count=', safeModeInfo.dirtyCount, 'safeMode=', safeModeInfo.safeMode, 'fingerprint=', JSON.stringify(safeModeInfo.fingerprint));
} catch (e) {
console.warn('[safe-mode] detect failed:', e && e.message);
}
}
ipcMain.handle('get-safe-mode', () => safeModeInfo);
// Quit-cause forensics. On a real quit (Cmd+Q, dock Quit, app.quit()) Electron
// fires before-quit BEFORE any window 'close' events; a window closing on its
// own (Cmd+W, red X, programmatic close) fires 'close' with quitInitiated
@@ -2019,6 +2069,7 @@ app.whenReady().then(async () => {
// Spawn the Mac crash watchdog. Detached process; if it fails to spawn the
// app continues normally (silent fail by design). Guards inside the
// watchdog itself prevent false-positive relaunches.
detectDirtyExitAndArmSafeMode();
spawnCrashWatchdog();
// Off-window mouse-release crash dodge (macOS). Safe to call before windows exist.
+1
View File
@@ -47,6 +47,7 @@ contextBridge.exposeInMainWorld('openswarm', {
// Phase 2 provenance: { sha, shortSha, builtAt, channel } for the About panel.
getBuildInfo: () => ipcRenderer.invoke('get-build-info'),
getSafeMode: () => ipcRenderer.invoke('get-safe-mode'),
// Phase 0 boot instrumentation: renderer calls this exactly once, when the
// first streamed token of the first agent response paints. Fire-and-forget
@@ -25,6 +25,7 @@ import { ackRun, runWorkflowNow } from '@/shared/state/workflowsSlice';
import { setPendingBrowserUrl } from '@/shared/state/tempStateSlice';
import { fetchOutputs } from '@/shared/state/outputsSlice';
import UpdateReadyPill from '@/app/components/Layout/UpdateReadyPill';
import SafeModePill from '@/app/components/Layout/SafeModePill';
import WhatsNewCard from '@/app/components/Layout/WhatsNewCard';
import ShareRequestHost from '@/app/components/share/ShareRequestHost';
import AppToolGrantHost from '@/app/components/apps/AppToolGrantHost';
@@ -560,6 +561,7 @@ const AppShell: React.FC = () => {
</Collapse>
{!fsHideChrome && <UpdateReadyPill />}
{!fsHideChrome && <SafeModePill />}
<Box sx={{ display: 'flex', flex: 1, minHeight: 0 }}>
{/* Sidebar excised: dashboards live in the Spaces strip (hover the top edge; right-click a tile for rename/duplicate/delete). */}
@@ -0,0 +1,41 @@
import React, { useState } from 'react';
import Box from '@mui/material/Box';
import Typography from '@mui/material/Typography';
import ShieldOutlinedIcon from '@mui/icons-material/ShieldOutlined';
import CloseIcon from '@mui/icons-material/Close';
import { useClaudeTokens } from '@/shared/styles/ThemeContext';
import { safeModeInfo } from '@/shared/safeMode';
// Shown only when the loop breaker armed (two dirty exits in ten minutes, ENG-228): tells the user
// why their browsers and apps boot paused instead of looking silently broken. Dismiss is session-only.
const SafeModePill: React.FC = () => {
const c = useClaudeTokens();
const [dismissed, setDismissed] = useState(false);
const info = safeModeInfo();
if (!info.safeMode || dismissed) return null;
return (
<Box
sx={{
position: 'fixed', top: 34, left: '50%', transform: 'translateX(-50%)',
zIndex: 1400, WebkitAppRegion: 'no-drag',
display: 'flex', alignItems: 'center', gap: 1,
px: 1.5, py: 0.75, borderRadius: '10px',
bgcolor: c.bg.surface, border: `1px solid ${c.border.subtle}`,
boxShadow: '0 4px 16px rgba(0,0,0,0.18)',
}}
>
<ShieldOutlinedIcon sx={{ fontSize: 16, color: c.text.tertiary }} />
<Typography sx={{ fontSize: '0.8125rem', color: c.text.primary }}>
Recovered after repeated crashes. Browsers and apps are paused; click one to resume it.
</Typography>
<Box
role="button" aria-label="Dismiss safe mode notice" onClick={() => setDismissed(true)}
sx={{ display: 'flex', alignItems: 'center', cursor: 'pointer', color: c.text.tertiary, '&:hover': { color: c.text.primary } }}
>
<CloseIcon sx={{ fontSize: 14 }} />
</Box>
</Box>
);
};
export default SafeModePill;
@@ -2,6 +2,7 @@
import { useEffect, useRef, useState } from 'react';
import { API_BASE, getAuthToken } from '@/shared/config';
import { isSafeMode } from '@/shared/safeMode';
export interface RuntimeLogLine {
source: 'backend' | 'runtime';
@@ -56,13 +57,16 @@ export function useRuntimePreviewUrl(opts: RuntimePreviewOptions): RuntimePrevie
const connect = async (): Promise<void> => {
if (cancelled) return;
try {
await fetch(`${API_BASE}/outputs/workspace/${workspaceId}/runtime/start?instance=${instance}`, {
method: 'POST',
headers,
});
} catch (_) {
// Spawn errors surface via the log WS; don't double-report.
// Safe mode (ENG-228): after repeated dirty exits, app runtimes don't auto-boot on card mount; the card's restart button is the explicit resume, so a crash loop can't respawn the surface storm.
if (!isSafeMode()) {
try {
await fetch(`${API_BASE}/outputs/workspace/${workspaceId}/runtime/start?instance=${instance}`, {
method: 'POST',
headers,
});
} catch (_) {
// Spawn errors surface via the log WS; don't double-report.
}
}
if (cancelled) return;
try {
+26
View File
@@ -0,0 +1,26 @@
// Safe-mode loop breaker (ENG-228): after two dirty exits in ten minutes the main process arms
// safe mode, and this boot restores layout with webviews parked as screenshots until clicked, so a
// relaunch stops rebuilding the exact state that crashed. Resolved once at bundle eval; the IPC
// round-trip finishes long before the layout fetch that first reads it.
export interface SafeModeInfo {
safeMode: boolean;
dirtyCount: number;
fingerprint: { exception: string | null; code: number | null; address: number | null } | null;
}
let cached: SafeModeInfo = { safeMode: false, dirtyCount: 0, fingerprint: null };
const api = (window as unknown as { openswarm?: { getSafeMode?: () => Promise<SafeModeInfo> } }).openswarm;
if (api?.getSafeMode) {
void api.getSafeMode().then((info) => {
if (info && typeof info.safeMode === 'boolean') cached = info;
}).catch(() => {});
}
export function safeModeInfo(): SafeModeInfo {
return cached;
}
export function isSafeMode(): boolean {
return cached.safeMode;
}
@@ -3,6 +3,7 @@ import { launchAndSendFirstMessage, resumeSession, collapseSession, collapseAllS
import { untileClosedChats } from './untileClosedChats';
import { API_BASE } from '@/shared/config';
import { getLastDashboardId } from '@/shared/lastDashboardId';
import { isSafeMode } from '@/shared/safeMode';
// fetchSession 404/410 strips the layout card to stop AgentChat remount-loop. Matched by string to avoid circular import.
const fetchSessionRejectedAction = createAction<
@@ -1886,6 +1887,12 @@ const dashboardLayoutSlice = createSlice({
for (const id of Object.keys(state.tiledCards)) {
if (!tileOwnerExists(state, id)) delete state.tiledCards[id];
}
// Safe mode (ENG-228): after repeated dirty exits, every browser webview boots parked as a screenshot; clicking a card resumes it (the existing suspend/resume path), so a crash loop can't rebuild the surface storm that killed the last session.
if (isSafeMode()) {
for (const id of Object.keys(state.browserCards)) {
if (!state.suspendedBrowserCards[id]) state.suspendedBrowserCards[id] = { dataUrl: '', capturedAt: 0 };
}
}
// Pre-1.7.6 profiles can persist several live 'fullscreen' entries; the selector crowns the first, so every OTHER card's drag guard compares against the wrong owner and lets the drag through. One owner, same rule as the write reducer.
let fsOwner: string | null = null;
for (const [id, zone] of Object.entries(state.tiledCards)) {