[eric] resilience: api fetches ride backend restarts (bounded retry), a dead backend shows an honest reconnecting pill and self-heals

frontend/src/app/components/Layout/AppShell.tsx

@@ -25,6 +25,7 @@ import { ackRun, runWorkflowNow } from '@/shared/state/workflowsSlice';
 import { setPendingBrowserUrl } from '@/shared/state/tempStateSlice';
 import { fetchOutputs } from '@/shared/state/outputsSlice';
 import UpdateReadyPill from '@/app/components/Layout/UpdateReadyPill';
+import ReconnectingPill from '@/app/components/Layout/ReconnectingPill';
 import SafeModePill from '@/app/components/Layout/SafeModePill';
 import WhatsNewCard from '@/app/components/Layout/WhatsNewCard';
 import ShareRequestHost from '@/app/components/share/ShareRequestHost';
@@ -561,6 +562,8 @@ const AppShell: React.FC = () => {
       </Collapse>
 
       {!fsHideChrome && <UpdateReadyPill />}
+      {/* Deliberately NOT behind fsHideChrome: a dead backend must be visible even in fullscreen, or the app reads as frozen (ENG-242). */}
+      <ReconnectingPill />
       {!fsHideChrome && <SafeModePill />}
 
       <Box sx={{ display: 'flex', flex: 1, minHeight: 0 }}>

frontend/src/app/components/Layout/ReconnectingPill.tsx

@@ -0,0 +1,69 @@
+import React, { useEffect, useState } from 'react';
+import Box from '@mui/material/Box';
+import Grow from '@mui/material/Grow';
+import Typography from '@mui/material/Typography';
+import CircularProgress from '@mui/material/CircularProgress';
+import { useClaudeTokens } from '@/shared/styles/ThemeContext';
+import { backendReachable, onBackendReachability } from '@/shared/backendConnection';
+
+// Honest "the local backend went away" state, so an unreachable backend never reads as a silent
+// forever-spinner (ENG-242). The interceptor's background probe self-heals; this only tells the
+// user what is happening while it does, and lets them force a reload if they are impatient.
+const ReconnectingPill: React.FC = () => {
+  const c = useClaudeTokens();
+  const [reachable, setReachable] = useState(true);
+  // Only show after a short grace so a normal ~4s backend respawn heals invisibly; the pill is
+  // for the case that actually worried the user, a backend that stays gone.
+  const [showable, setShowable] = useState(false);
+
+  useEffect(() => {
+    setReachable(backendReachable());
+    return onBackendReachability(setReachable);
+  }, []);
+
+  useEffect(() => {
+    if (reachable) { setShowable(false); return undefined; }
+    const t = setTimeout(() => setShowable(true), 6000);
+    return () => clearTimeout(t);
+  }, [reachable]);
+
+  const show = !reachable && showable;
+
+  return (
+    <Grow in={show} unmountOnExit>
+      <Box
+        onClick={() => window.location.reload()}
+        role="button"
+        aria-label="Reconnecting to OpenSwarm; click to reload"
+        sx={{
+          position: 'fixed',
+          bottom: 16,
+          left: '50%',
+          transform: 'translateX(-50%)',
+          zIndex: 1400,
+          display: 'flex',
+          alignItems: 'center',
+          gap: 1,
+          px: 1.75,
+          py: 1,
+          borderRadius: 999,
+          cursor: 'pointer',
+          WebkitAppRegion: 'no-drag',
+          background: c.bg.elevated,
+          border: `1px solid ${c.border.strong}`,
+          boxShadow: c.shadow.lg,
+        } as object}
+      >
+        <CircularProgress size={14} sx={{ color: c.text.secondary }} />
+        <Typography sx={{ fontSize: '0.8125rem', color: c.text.primary, fontWeight: 500 }}>
+          Reconnecting to OpenSwarm…
+        </Typography>
+        <Typography sx={{ fontSize: '0.75rem', color: c.text.tertiary }}>
+          click to reload
+        </Typography>
+      </Box>
+    </Grow>
+  );
+};
+
+export default ReconnectingPill;

frontend/src/shared/backendConnection.ts

@@ -0,0 +1,66 @@
+// One source of truth for "is our local backend answering", fed by the global fetch
+// interceptor (config.ts) and consumed by the reconnecting pill. Exists so an unreachable
+// backend can never again present as a silent forever-spinner (ENG-241/ENG-242): every
+// consumer reads the same signal, and a background probe self-heals the moment it returns.
+
+type ReachabilityListener = (reachable: boolean) => void;
+
+let reachableNow = true;
+let failStreak = 0;
+const listeners = new Set<ReachabilityListener>();
+let probeTimer: ReturnType<typeof setInterval> | null = null;
+// Injected by config.ts with the UN-intercepted fetch so probes never recurse into retry logic.
+let prober: (() => Promise<unknown>) | null = null;
+
+export function backendReachable(): boolean {
+  return reachableNow;
+}
+
+export function onBackendReachability(cb: ReachabilityListener): () => void {
+  listeners.add(cb);
+  return () => { listeners.delete(cb); };
+}
+
+export function setBackendProber(fn: () => Promise<unknown>): void {
+  prober = fn;
+}
+
+function emit(value: boolean): void {
+  listeners.forEach((l) => { try { l(value); } catch { /* a listener must never break the signal */ } });
+}
+
+function startProbe(): void {
+  if (probeTimer || !prober) return;
+  probeTimer = setInterval(() => {
+    void (prober as () => Promise<unknown>)()
+      .then(() => noteBackendSuccess())
+      .catch(() => { /* still down; keep probing */ });
+  }, 1500);
+}
+
+function stopProbe(): void {
+  if (probeTimer) { clearInterval(probeTimer); probeTimer = null; }
+}
+
+export function noteBackendFailure(): void {
+  failStreak++;
+  // Two consecutive failures = down. One lone failure is never a state flip, so a single
+  // dropped request can't flash the reconnecting UI.
+  if (reachableNow && failStreak >= 2) {
+    reachableNow = false;
+    emit(false);
+    startProbe();
+  }
+}
+
+export function noteBackendSuccess(): void {
+  failStreak = 0;
+  if (!reachableNow) {
+    reachableNow = true;
+    stopProbe();
+    emit(true);
+  }
+}
+
+// Harness/debug handle: lets a live session (CDP, support) read the signal without a store import.
+(window as unknown as { __OSW_CONN?: object }).__OSW_CONN = { backendReachable, onBackendReachability };

frontend/src/shared/config.ts

@@ -1,3 +1,5 @@
+import { noteBackendFailure, noteBackendSuccess, setBackendProber } from '@/shared/backendConnection';
+
 const _w = window as any;
 // Prefer the preload-injected port; if it's missing (preload raced the backend port being picked), re-query the live value before falling back to 8324. The bare 8324 guess is wrong on any machine where the backend landed on a fallback port (e.g. 8324 was held by a leftover backend); see the self-heal below.
 const port =
@@ -83,6 +85,25 @@ function _installAuthFetchInterceptor() {
   (window as any).__OPENSWARM_FETCH_PATCHED__ = true;
 
   const originalFetch = window.fetch.bind(window);
+  // Reachability probe uses the RAW fetch: any HTTP response (401 included) proves the backend
+  // is back, and it must never recurse into the retry/dedupe logic below.
+  setBackendProber(() => originalFetch(`http://${host}:${port}/`, { signal: AbortSignal.timeout(2000), cache: 'no-store' }));
+
+  // Loopback calls answer in ms; anything past this is a dead/wedged backend, and an unbounded
+  // hang here is exactly the silent forever-spinner class (ENG-241). Generous enough for a big
+  // .swarm export, still bounded. Callers with their own AbortSignal keep it via AbortSignal.any.
+  const ATTEMPT_TIMEOUT_MS = 30000;
+  // Spans the measured ~4.2s packaged-backend respawn (kill-to-listening), so a GET fired the
+  // instant the backend dies succeeds on the last attempt instead of surfacing a one-off error.
+  const GET_RETRY_DELAYS_MS = [500, 1500, 2600];
+  const isTransientStatus = (s: number) => s === 502 || s === 503 || s === 504;
+
+  const attemptInit = (finalInit: RequestInit | undefined): RequestInit => {
+    const timeout = AbortSignal.timeout(ATTEMPT_TIMEOUT_MS);
+    const callerSignal = finalInit?.signal;
+    return { ...(finalInit ?? {}), signal: callerSignal ? AbortSignal.any([callerSignal, timeout]) : timeout };
+  };
+
   window.fetch = async function patchedFetch(input: RequestInfo | URL, init?: RequestInit): Promise<Response> {
     try {
       const url = typeof input === 'string' ? input : input instanceof URL ? input.toString() : (input as Request).url;
@@ -106,9 +127,18 @@ function _installAuthFetchInterceptor() {
         ?? (input instanceof Request ? input.method : 'GET')
       ).toUpperCase();
 
-      // Only GET is safe to dedupe/cache; mutations could collapse intentional double-clicks.
+      // Mutations never auto-retry (not idempotent) and keep their own timing (some POSTs are
+      // legitimately slow); they still feed the reachability signal so the UI stays honest.
       if (method !== 'GET') {
-        return originalFetch(input, finalInit);
+        try {
+          const resp = await originalFetch(input, finalInit);
+          noteBackendSuccess();
+          return resp;
+        } catch (err) {
+          noteBackendFailure();
+          _maybeHealBackendPort();
+          throw err;
+        }
       }
 
       const cacheKey = `GET ${url}`;
@@ -126,7 +156,36 @@ function _installAuthFetchInterceptor() {
         return resp.clone();
       }
 
-      const promise = originalFetch(input, finalInit).then((resp) => {
+      // Bounded retry: a backend respawn (measured ~4.2s door-to-door) or a transient 5xx must
+      // not permanently fail an idempotent read; a genuinely dead backend fails fast and flips
+      // the reachability signal instead of hanging forever.
+      const runWithRetry = async (): Promise<Response> => {
+        let lastErr: unknown = null;
+        for (let attempt = 0; attempt <= GET_RETRY_DELAYS_MS.length; attempt++) {
+          try {
+            const resp = await originalFetch(input, attemptInit(finalInit));
+            if (isTransientStatus(resp.status) && attempt < GET_RETRY_DELAYS_MS.length) {
+              await new Promise((r) => setTimeout(r, GET_RETRY_DELAYS_MS[attempt]));
+              continue;
+            }
+            noteBackendSuccess();
+            return resp;
+          } catch (err) {
+            lastErr = err;
+            // A caller-driven abort is a real answer, never something to retry through.
+            if (finalInit?.signal?.aborted) throw err;
+            if (attempt < GET_RETRY_DELAYS_MS.length) {
+              await new Promise((r) => setTimeout(r, GET_RETRY_DELAYS_MS[attempt]));
+              continue;
+            }
+          }
+        }
+        noteBackendFailure();
+        _maybeHealBackendPort();
+        throw lastErr;
+      };
+
+      const promise = runWithRetry().then((resp) => {
         if (resp.ok) {
           _cachedFetches.set(cacheKey, {
             resp: resp.clone(),
@@ -142,9 +201,9 @@ function _installAuthFetchInterceptor() {
       } finally {
         _inflightFetches.delete(cacheKey);
       }
-    } catch {
-      // A network failure reaching our backend may mean we're on a stale port.
-      _maybeHealBackendPort();
+    } catch (err) {
+      // Interceptor plumbing must never turn a workable request into a failure; fall through raw.
+      if (err instanceof TypeError || (err as Error)?.name === 'AbortError' || (err as Error)?.name === 'TimeoutError') throw err;
       return originalFetch(input, init);
     }
   };