From b56e91b1ebf78e5704661313b67d91fe5bb82825 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Mon, 10 Aug 2026 20:35:07 -0700 Subject: [PATCH] [eric] resilience: api fetches ride backend restarts (bounded retry), a dead backend shows an honest reconnecting pill and self-heals --- .../src/app/components/Layout/AppShell.tsx | 3 + .../components/Layout/ReconnectingPill.tsx | 69 ++++++++++++++++++ frontend/src/shared/backendConnection.ts | 66 +++++++++++++++++ frontend/src/shared/config.ts | 71 +++++++++++++++++-- 4 files changed, 203 insertions(+), 6 deletions(-) create mode 100644 frontend/src/app/components/Layout/ReconnectingPill.tsx create mode 100644 frontend/src/shared/backendConnection.ts diff --git a/frontend/src/app/components/Layout/AppShell.tsx b/frontend/src/app/components/Layout/AppShell.tsx index 427739ff..e4fd2c20 100644 --- a/frontend/src/app/components/Layout/AppShell.tsx +++ b/frontend/src/app/components/Layout/AppShell.tsx @@ -25,6 +25,7 @@ import { ackRun, runWorkflowNow } from '@/shared/state/workflowsSlice'; import { setPendingBrowserUrl } from '@/shared/state/tempStateSlice'; import { fetchOutputs } from '@/shared/state/outputsSlice'; import UpdateReadyPill from '@/app/components/Layout/UpdateReadyPill'; +import ReconnectingPill from '@/app/components/Layout/ReconnectingPill'; import SafeModePill from '@/app/components/Layout/SafeModePill'; import WhatsNewCard from '@/app/components/Layout/WhatsNewCard'; import ShareRequestHost from '@/app/components/share/ShareRequestHost'; @@ -561,6 +562,8 @@ const AppShell: React.FC = () => { {!fsHideChrome && } + {/* Deliberately NOT behind fsHideChrome: a dead backend must be visible even in fullscreen, or the app reads as frozen (ENG-242). */} + {!fsHideChrome && } diff --git a/frontend/src/app/components/Layout/ReconnectingPill.tsx b/frontend/src/app/components/Layout/ReconnectingPill.tsx new file mode 100644 index 00000000..135c5f6d --- /dev/null +++ b/frontend/src/app/components/Layout/ReconnectingPill.tsx @@ -0,0 +1,69 @@ +import React, { useEffect, useState } from 'react'; +import Box from '@mui/material/Box'; +import Grow from '@mui/material/Grow'; +import Typography from '@mui/material/Typography'; +import CircularProgress from '@mui/material/CircularProgress'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { backendReachable, onBackendReachability } from '@/shared/backendConnection'; + +// Honest "the local backend went away" state, so an unreachable backend never reads as a silent +// forever-spinner (ENG-242). The interceptor's background probe self-heals; this only tells the +// user what is happening while it does, and lets them force a reload if they are impatient. +const ReconnectingPill: React.FC = () => { + const c = useClaudeTokens(); + const [reachable, setReachable] = useState(true); + // Only show after a short grace so a normal ~4s backend respawn heals invisibly; the pill is + // for the case that actually worried the user, a backend that stays gone. + const [showable, setShowable] = useState(false); + + useEffect(() => { + setReachable(backendReachable()); + return onBackendReachability(setReachable); + }, []); + + useEffect(() => { + if (reachable) { setShowable(false); return undefined; } + const t = setTimeout(() => setShowable(true), 6000); + return () => clearTimeout(t); + }, [reachable]); + + const show = !reachable && showable; + + return ( + + window.location.reload()} + role="button" + aria-label="Reconnecting to OpenSwarm; click to reload" + sx={{ + position: 'fixed', + bottom: 16, + left: '50%', + transform: 'translateX(-50%)', + zIndex: 1400, + display: 'flex', + alignItems: 'center', + gap: 1, + px: 1.75, + py: 1, + borderRadius: 999, + cursor: 'pointer', + WebkitAppRegion: 'no-drag', + background: c.bg.elevated, + border: `1px solid ${c.border.strong}`, + boxShadow: c.shadow.lg, + } as object} + > + + + Reconnecting to OpenSwarm… + + + click to reload + + + + ); +}; + +export default ReconnectingPill; diff --git a/frontend/src/shared/backendConnection.ts b/frontend/src/shared/backendConnection.ts new file mode 100644 index 00000000..61e88b89 --- /dev/null +++ b/frontend/src/shared/backendConnection.ts @@ -0,0 +1,66 @@ +// One source of truth for "is our local backend answering", fed by the global fetch +// interceptor (config.ts) and consumed by the reconnecting pill. Exists so an unreachable +// backend can never again present as a silent forever-spinner (ENG-241/ENG-242): every +// consumer reads the same signal, and a background probe self-heals the moment it returns. + +type ReachabilityListener = (reachable: boolean) => void; + +let reachableNow = true; +let failStreak = 0; +const listeners = new Set(); +let probeTimer: ReturnType | null = null; +// Injected by config.ts with the UN-intercepted fetch so probes never recurse into retry logic. +let prober: (() => Promise) | null = null; + +export function backendReachable(): boolean { + return reachableNow; +} + +export function onBackendReachability(cb: ReachabilityListener): () => void { + listeners.add(cb); + return () => { listeners.delete(cb); }; +} + +export function setBackendProber(fn: () => Promise): void { + prober = fn; +} + +function emit(value: boolean): void { + listeners.forEach((l) => { try { l(value); } catch { /* a listener must never break the signal */ } }); +} + +function startProbe(): void { + if (probeTimer || !prober) return; + probeTimer = setInterval(() => { + void (prober as () => Promise)() + .then(() => noteBackendSuccess()) + .catch(() => { /* still down; keep probing */ }); + }, 1500); +} + +function stopProbe(): void { + if (probeTimer) { clearInterval(probeTimer); probeTimer = null; } +} + +export function noteBackendFailure(): void { + failStreak++; + // Two consecutive failures = down. One lone failure is never a state flip, so a single + // dropped request can't flash the reconnecting UI. + if (reachableNow && failStreak >= 2) { + reachableNow = false; + emit(false); + startProbe(); + } +} + +export function noteBackendSuccess(): void { + failStreak = 0; + if (!reachableNow) { + reachableNow = true; + stopProbe(); + emit(true); + } +} + +// Harness/debug handle: lets a live session (CDP, support) read the signal without a store import. +(window as unknown as { __OSW_CONN?: object }).__OSW_CONN = { backendReachable, onBackendReachability }; diff --git a/frontend/src/shared/config.ts b/frontend/src/shared/config.ts index 019d7219..0982add1 100644 --- a/frontend/src/shared/config.ts +++ b/frontend/src/shared/config.ts @@ -1,3 +1,5 @@ +import { noteBackendFailure, noteBackendSuccess, setBackendProber } from '@/shared/backendConnection'; + const _w = window as any; // Prefer the preload-injected port; if it's missing (preload raced the backend port being picked), re-query the live value before falling back to 8324. The bare 8324 guess is wrong on any machine where the backend landed on a fallback port (e.g. 8324 was held by a leftover backend); see the self-heal below. const port = @@ -83,6 +85,25 @@ function _installAuthFetchInterceptor() { (window as any).__OPENSWARM_FETCH_PATCHED__ = true; const originalFetch = window.fetch.bind(window); + // Reachability probe uses the RAW fetch: any HTTP response (401 included) proves the backend + // is back, and it must never recurse into the retry/dedupe logic below. + setBackendProber(() => originalFetch(`http://${host}:${port}/`, { signal: AbortSignal.timeout(2000), cache: 'no-store' })); + + // Loopback calls answer in ms; anything past this is a dead/wedged backend, and an unbounded + // hang here is exactly the silent forever-spinner class (ENG-241). Generous enough for a big + // .swarm export, still bounded. Callers with their own AbortSignal keep it via AbortSignal.any. + const ATTEMPT_TIMEOUT_MS = 30000; + // Spans the measured ~4.2s packaged-backend respawn (kill-to-listening), so a GET fired the + // instant the backend dies succeeds on the last attempt instead of surfacing a one-off error. + const GET_RETRY_DELAYS_MS = [500, 1500, 2600]; + const isTransientStatus = (s: number) => s === 502 || s === 503 || s === 504; + + const attemptInit = (finalInit: RequestInit | undefined): RequestInit => { + const timeout = AbortSignal.timeout(ATTEMPT_TIMEOUT_MS); + const callerSignal = finalInit?.signal; + return { ...(finalInit ?? {}), signal: callerSignal ? AbortSignal.any([callerSignal, timeout]) : timeout }; + }; + window.fetch = async function patchedFetch(input: RequestInfo | URL, init?: RequestInit): Promise { try { const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : (input as Request).url; @@ -106,9 +127,18 @@ function _installAuthFetchInterceptor() { ?? (input instanceof Request ? input.method : 'GET') ).toUpperCase(); - // Only GET is safe to dedupe/cache; mutations could collapse intentional double-clicks. + // Mutations never auto-retry (not idempotent) and keep their own timing (some POSTs are + // legitimately slow); they still feed the reachability signal so the UI stays honest. if (method !== 'GET') { - return originalFetch(input, finalInit); + try { + const resp = await originalFetch(input, finalInit); + noteBackendSuccess(); + return resp; + } catch (err) { + noteBackendFailure(); + _maybeHealBackendPort(); + throw err; + } } const cacheKey = `GET ${url}`; @@ -126,7 +156,36 @@ function _installAuthFetchInterceptor() { return resp.clone(); } - const promise = originalFetch(input, finalInit).then((resp) => { + // Bounded retry: a backend respawn (measured ~4.2s door-to-door) or a transient 5xx must + // not permanently fail an idempotent read; a genuinely dead backend fails fast and flips + // the reachability signal instead of hanging forever. + const runWithRetry = async (): Promise => { + let lastErr: unknown = null; + for (let attempt = 0; attempt <= GET_RETRY_DELAYS_MS.length; attempt++) { + try { + const resp = await originalFetch(input, attemptInit(finalInit)); + if (isTransientStatus(resp.status) && attempt < GET_RETRY_DELAYS_MS.length) { + await new Promise((r) => setTimeout(r, GET_RETRY_DELAYS_MS[attempt])); + continue; + } + noteBackendSuccess(); + return resp; + } catch (err) { + lastErr = err; + // A caller-driven abort is a real answer, never something to retry through. + if (finalInit?.signal?.aborted) throw err; + if (attempt < GET_RETRY_DELAYS_MS.length) { + await new Promise((r) => setTimeout(r, GET_RETRY_DELAYS_MS[attempt])); + continue; + } + } + } + noteBackendFailure(); + _maybeHealBackendPort(); + throw lastErr; + }; + + const promise = runWithRetry().then((resp) => { if (resp.ok) { _cachedFetches.set(cacheKey, { resp: resp.clone(), @@ -142,9 +201,9 @@ function _installAuthFetchInterceptor() { } finally { _inflightFetches.delete(cacheKey); } - } catch { - // A network failure reaching our backend may mean we're on a stale port. - _maybeHealBackendPort(); + } catch (err) { + // Interceptor plumbing must never turn a workable request into a failure; fall through raw. + if (err instanceof TypeError || (err as Error)?.name === 'AbortError' || (err as Error)?.name === 'TimeoutError') throw err; return originalFetch(input, init); } };