diff --git a/backend/apps/service/service.py b/backend/apps/service/service.py index b7c12422..6ff1c36f 100644 --- a/backend/apps/service/service.py +++ b/backend/apps/service/service.py @@ -215,6 +215,9 @@ async def service_lifespan(): yield + # Graceful shutdown just began; if it wedges (the ENG-223 quit race), the fuse hard-exits the tree. + from backend.apps.service.shutdown_fuse import arm_shutdown_fuse + arm_shutdown_fuse() if p_pulse_task: p_pulse_task.cancel() try: diff --git a/backend/apps/service/shutdown_fuse.py b/backend/apps/service/shutdown_fuse.py new file mode 100644 index 00000000..36a3c464 --- /dev/null +++ b/backend/apps/service/shutdown_fuse.py @@ -0,0 +1,58 @@ +"""SIGTERM arms a hard exit fuse (ENG-223): the quit path can wedge mid-shutdown (measured live: a +quit racing a pending update left uvicorn hung 8+ minutes with its whole agent-CLI tree orphaned at +~700MB), so if graceful shutdown has not finished FUSE_S after TERM, the fuse kills our process +tree and exits. A daemon thread, so a wedged event loop cannot block it.""" + +import os +import subprocess +import threading +from typing import List + +from typeguard import typechecked + +FUSE_S = 10.0 + + +@typechecked +def p_descendant_pids() -> List[int]: + pids: List[int] = [] + frontier: List[int] = [os.getpid()] + for depth in range(6): + next_frontier: List[int] = [] + for parent in frontier: + try: + out = subprocess.run( + ["pgrep", "-P", str(parent)], capture_output=True, text=True, timeout=2, + ).stdout + except Exception: + continue + for tok in out.split(): + try: + next_frontier.append(int(tok)) + except ValueError: + pass + pids.extend(next_frontier) + if not next_frontier: + break + frontier = next_frontier + return pids + + +@typechecked +def p_burn() -> None: + for pid in p_descendant_pids(): + try: + os.kill(pid, 9) + except Exception: + pass + os._exit(0) + + +@typechecked +def arm_shutdown_fuse() -> None: + """Called at lifespan-shutdown START (already past TERM), so no signal handling: just the timer. Touching signal.signal here would clobber uvicorn's asyncio-installed handlers.""" + if os.name == "nt": + return + timer = threading.Timer(FUSE_S, p_burn) + timer.daemon = True + timer.start() diff --git a/electron/main.js b/electron/main.js index 4d588868..60c50905 100644 --- a/electron/main.js +++ b/electron/main.js @@ -113,11 +113,16 @@ process.on('unhandledRejection', (reason) => { }); // child-process-gone fires for GPU/utility/renderer process deaths. The GPU one is especially useful: a GPU crash forces the renderer to recover its compositor, and that recovery can itself crash on Windows. +let gpuCrashCount = 0; app.on('child-process-gone', (_event, details) => { console.error('[diag][main:child-process-gone]', JSON.stringify(details)); // Clean exits and user kills are not crashes; reporting them would bury the real ones. if (details && details.reason && details.reason !== 'clean-exit' && details.reason !== 'killed') { crashReports.writeCrashReport('child-process-gone', details); + // Three GPU deaths in one session: the compositor is losing on this machine, so the NEXT boot runs software rendering (one boot only; the marker is consumed at startup). ENG-228. + if (details.type === 'GPU' && ++gpuCrashCount >= 3) { + try { fs.writeFileSync(GPU_FALLBACK_MARKER, String(Date.now())); } catch (_) {} + } } }); // Platform-split auto-updater: electron-updater on Mac (full-featured), Electron's @@ -602,6 +607,18 @@ async function startFrontendServer() { }); } +const GPU_FALLBACK_MARKER = path.join(os.homedir(), 'Library', 'Application Support', 'openswarm', 'gpu-fallback.marker'); +let reducedGraphicsThisBoot = false; +// Must run BEFORE app ready: a marker from last session's repeated GPU crashes buys ONE boot of software rendering, then normal service resumes (the marker is consumed here). ENG-228. +try { + if (process.platform === 'darwin' && fs.existsSync(GPU_FALLBACK_MARKER)) { + fs.unlinkSync(GPU_FALLBACK_MARKER); + app.disableHardwareAcceleration(); + reducedGraphicsThisBoot = true; + console.warn('[gpu-fallback] repeated GPU crashes last session; this boot uses software rendering'); + } +} catch (_) {} + const isPackaged = app.isPackaged; const isDev = process.env.ELECTRON_DEV === '1'; @@ -700,7 +717,7 @@ function detectDirtyExitAndArmSafeMode() { } } -ipcMain.handle('get-safe-mode', () => safeModeInfo); +ipcMain.handle('get-safe-mode', () => ({ ...safeModeInfo, reducedGraphics: reducedGraphicsThisBoot })); // Quit-cause forensics. On a real quit (Cmd+Q, dock Quit, app.quit()) Electron // fires before-quit BEFORE any window 'close' events; a window closing on its @@ -1201,7 +1218,7 @@ async function startBackend() { backendProcess = spawn( pythonPath, - ['-m', 'uvicorn', 'backend.main:app', '--host', '127.0.0.1', '--port', String(backendPort)], + ['-m', 'uvicorn', 'backend.main:app', '--host', '127.0.0.1', '--port', String(backendPort), '--timeout-graceful-shutdown', '8'], { cwd: projectRoot, env, diff --git a/frontend/src/app/components/Layout/SafeModePill.tsx b/frontend/src/app/components/Layout/SafeModePill.tsx index b7228c53..258c8b01 100644 --- a/frontend/src/app/components/Layout/SafeModePill.tsx +++ b/frontend/src/app/components/Layout/SafeModePill.tsx @@ -12,7 +12,7 @@ const SafeModePill: React.FC = () => { const c = useClaudeTokens(); const [dismissed, setDismissed] = useState(false); const info = safeModeInfo(); - if (!info.safeMode || dismissed) return null; + if ((!info.safeMode && !info.reducedGraphics) || dismissed) return null; return ( { > - Recovered after repeated crashes. Browsers and apps are paused; click one to resume it. + {info.safeMode + ? 'Recovered after repeated crashes. Browsers and apps are paused; click one to resume it.' + : 'Running in reduced graphics mode after repeated graphics crashes. Restart to return to full speed.'} setDismissed(true)} diff --git a/frontend/src/shared/safeMode.ts b/frontend/src/shared/safeMode.ts index 2a0f8779..c8c3f6e9 100644 --- a/frontend/src/shared/safeMode.ts +++ b/frontend/src/shared/safeMode.ts @@ -6,6 +6,7 @@ export interface SafeModeInfo { safeMode: boolean; dirtyCount: number; fingerprint: { exception: string | null; code: number | null; address: number | null } | null; + reducedGraphics?: boolean; } let cached: SafeModeInfo = { safeMode: false, dirtyCount: 0, fingerprint: null };