[eric] providers: boot-time login-health pill (dead sub token -> reconnect toast, no more silent codex deaths)

backend/apps/agents/agents.py

@@ -488,6 +488,24 @@ async def subscriptions_exchange(body: dict):
         raise HTTPException(status_code=500, detail=str(e))
 
 
+@agents.router.get("/subscriptions/health")
+async def subscriptions_health():
+    """Boot-time login-health check: 1-token probe per ACTIVE subscription lane, reporting only
+    definitive auth-death (the silent credential-rot class). `skipped` when the router isn't up yet
+    so the frontend can retry once instead of reading 'all healthy' off a cold boot."""
+    from backend.apps.nine_router import is_running, get_providers
+    from backend.apps.nine_router.subscription_health import probe_subscription_health
+    if not is_running():
+        return {"dead": [], "skipped": True}
+    try:
+        connections = await get_providers()
+        dead = await probe_subscription_health(connections)
+        return {"dead": dead, "skipped": False}
+    except Exception as e:
+        logger.debug(f"subscription health probe failed: {e}")
+        return {"dead": [], "skipped": True}
+
+
 @agents.router.get("/subscriptions/models")
 async def subscriptions_models():
     """List all models available through connected subscriptions."""

backend/apps/nine_router/subscription_health.py

@@ -0,0 +1,114 @@
+"""Boot-time subscription health probe: catches a provider login that died while the app was
+closed (refresh-token rotation, the "Breaking codex" class) so the UI can offer reconnect BEFORE
+the user burns a failed turn discovering it. Probes SUBSCRIPTION lanes only (1 token of sub quota,
+never a billable API key), and only a definitive auth-shaped 401/403 counts as dead: transient
+429/5xx/timeouts stay silent so the pill can never cry wolf. Kill switch: OPENSWARM_BOOT_HEALTH=0."""
+
+import asyncio
+import logging
+import os
+import time
+from typing import Dict, List, Optional
+
+import httpx
+from typeguard import typechecked
+
+from backend.apps.nine_router.process import NINE_ROUTER_URL, is_running
+
+logger = logging.getLogger(__name__)
+
+PREFIX_BY_PROVIDER: Dict[str, str] = {
+    "claude": "cc/",
+    "codex": "cx/",
+    "gemini-cli": "gemini/",
+    "antigravity": "ag/",
+}
+LABEL_BY_PROVIDER: Dict[str, str] = {
+    "claude": "Claude",
+    "codex": "ChatGPT",
+    "gemini-cli": "Gemini",
+    "antigravity": "Gemini (Antigravity)",
+}
+P_AUTH_DEAD_MARKERS = ("authentication", "expired", "sign in", "signing in", "invalid_grant", "unauthorized", "invalid authentication")
+P_PROBE_TIMEOUT_S = 25.0
+P_CACHE_TTL_S = 300.0
+
+p_probe_lock = asyncio.Lock()
+p_cached_result: Optional[List[Dict[str, str]]] = None
+p_cached_at: float = 0.0
+
+
+@typechecked
+def health_probe_enabled() -> bool:
+    return os.environ.get("OPENSWARM_BOOT_HEALTH", "1") != "0"
+
+
+@typechecked
+def classify_auth_dead(status_code: int, body_text: str) -> bool:
+    """Dead ONLY on a definitive auth failure; anything ambiguous reads healthy (silence beats a false reconnect prompt)."""
+    if status_code not in (401, 403):
+        return False
+    low = body_text.lower()
+    return any(m in low for m in P_AUTH_DEAD_MARKERS)
+
+
+@typechecked
+async def p_pick_probe_model(client: httpx.AsyncClient, prefix: str) -> Optional[str]:
+    try:
+        r = await client.get(f"{NINE_ROUTER_URL}/v1/models")
+        if r.status_code != 200:
+            return None
+        ids = [m.get("id") for m in (r.json().get("data") or []) if isinstance(m, dict)]
+        for i in ids:
+            if isinstance(i, str) and i.startswith(prefix):
+                return i
+    except Exception:
+        return None
+    return None
+
+
+@typechecked
+async def p_probe_one(client: httpx.AsyncClient, model: str) -> Optional[bool]:
+    """True = auth dead, False = healthy, None = inconclusive (never reported)."""
+    try:
+        r = await client.post(
+            f"{NINE_ROUTER_URL}/v1/messages",
+            json={"model": model, "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]},
+            headers={"x-api-key": "9router", "anthropic-version": "2023-06-01"},
+        )
+        if r.status_code < 400:
+            return False
+        return True if classify_auth_dead(r.status_code, r.text or "") else None
+    except Exception:
+        return None
+
+
+@typechecked
+async def probe_subscription_health(connections: List[Dict]) -> List[Dict[str, str]]:
+    """Probe each active subscription connection with a 1-token turn; returns [{provider, label}]
+    for the definitively auth-dead ones. Cached for 5 minutes; concurrent callers share one run."""
+    global p_cached_result, p_cached_at
+    if not health_probe_enabled() or not is_running():
+        return []
+    async with p_probe_lock:
+        if p_cached_result is not None and time.monotonic() - p_cached_at < P_CACHE_TTL_S:
+            return p_cached_result
+        subs = [
+            c for c in connections
+            if isinstance(c, dict) and c.get("provider") in PREFIX_BY_PROVIDER and c.get("isActive")
+        ]
+        dead: List[Dict[str, str]] = []
+        if subs:
+            async with httpx.AsyncClient(timeout=P_PROBE_TIMEOUT_S) as client:
+                for c in subs:
+                    provider = str(c.get("provider"))
+                    model = await p_pick_probe_model(client, PREFIX_BY_PROVIDER[provider])
+                    if not model:
+                        continue
+                    verdict = await p_probe_one(client, model)
+                    if verdict is True:
+                        dead.append({"provider": provider, "label": LABEL_BY_PROVIDER[provider]})
+                        logger.info(f"[sub-health] {provider}: auth dead (reconnect needed)")
+        p_cached_result = dead
+        p_cached_at = time.monotonic()
+        return dead

backend/tests/test_subscription_health.py

@@ -0,0 +1,88 @@
+"""Boot-time subscription health probe: only a definitive auth-shaped 401/403 reads as dead
+(transients stay silent so the reconnect pill can't cry wolf), probes cover only active
+subscription lanes, and results are cached so a double boot-fetch can't double-spend probes."""
+
+import asyncio
+import json
+from typing import Dict, List, Optional
+
+import backend.apps.nine_router.subscription_health as sh
+
+
+def test_classify_auth_dead_is_conservative():
+    assert sh.classify_auth_dead(401, '{"error": "authentication token is expired"}')
+    assert sh.classify_auth_dead(403, "Unauthorized access")
+    assert sh.classify_auth_dead(401, "Invalid authentication credentials")
+    assert not sh.classify_auth_dead(401, "weird opaque body")  # 401 without an auth marker stays silent
+    assert not sh.classify_auth_dead(429, "rate limit expired")  # non-auth status never fires
+    assert not sh.classify_auth_dead(500, "authentication service down")
+    assert not sh.classify_auth_dead(200, "expired")
+
+
+class FakeResponse:
+    def __init__(self, status_code: int, payload: Dict):
+        self.status_code = status_code
+        self.text = json.dumps(payload)
+        self.p_payload = payload
+
+    def json(self) -> Dict:
+        return self.p_payload
+
+
+class FakeClient:
+    """cc/ lane healthy, cx/ lane auth-dead, gemini/ lane rate-limited (must stay silent)."""
+
+    def __init__(self, counter: List[int], **kwargs):
+        self.counter = counter
+
+    async def __aenter__(self) -> "FakeClient":
+        return self
+
+    async def __aexit__(self, *exc) -> None:
+        return None
+
+    async def get(self, url: str, **kw) -> FakeResponse:
+        return FakeResponse(200, {"data": [
+            {"id": "cc/claude-haiku"}, {"id": "cx/gpt-5.4"}, {"id": "gemini/gemini-2.5-flash"},
+        ]})
+
+    async def post(self, url: str, **kw) -> FakeResponse:
+        self.counter[0] += 1
+        model = kw.get("json", {}).get("model", "")
+        if model.startswith("cx/"):
+            return FakeResponse(401, {"error": {"message": "authentication token is expired, try signing in again"}})
+        if model.startswith("gemini/"):
+            return FakeResponse(429, {"error": {"message": "rate limited"}})
+        return FakeResponse(200, {"content": []})
+
+
+CONNS = [
+    {"provider": "claude", "isActive": True},
+    {"provider": "codex", "isActive": True},
+    {"provider": "gemini-cli", "isActive": True},
+    {"provider": "openrouter", "isActive": True},  # not a sub lane; never probed
+    {"provider": "codex", "isActive": False},  # inactive; never probed
+]
+
+
+def test_probe_reports_only_definitive_death_and_caches(monkeypatch):
+    counter = [0]
+    monkeypatch.setattr(sh, "is_running", lambda: True)
+    monkeypatch.setattr(sh.httpx, "AsyncClient", lambda **kw: FakeClient(counter, **kw))
+    monkeypatch.setattr(sh, "p_cached_result", None)
+    monkeypatch.setattr(sh, "p_cached_at", 0.0)
+    dead = asyncio.run(sh.probe_subscription_health(CONNS))
+    assert dead == [{"provider": "codex", "label": "ChatGPT"}]
+    assert counter[0] == 3  # claude + codex + gemini-cli probed; openrouter/inactive skipped
+    # Second call within the TTL serves the cache; no new probe spend.
+    dead2 = asyncio.run(sh.probe_subscription_health(CONNS))
+    assert dead2 == dead
+    assert counter[0] == 3
+
+
+def test_probe_disabled_or_router_down(monkeypatch):
+    monkeypatch.setattr(sh, "is_running", lambda: False)
+    assert asyncio.run(sh.probe_subscription_health(CONNS)) == []
+    monkeypatch.setattr(sh, "is_running", lambda: True)
+    monkeypatch.setenv("OPENSWARM_BOOT_HEALTH", "0")
+    assert asyncio.run(sh.probe_subscription_health(CONNS)) == []

frontend/src/app/components/overlays/ProviderHealthToast.tsx

@@ -0,0 +1,63 @@
+// Bottom-left nudge shown at launch when a subscription login died while the app was closed (silent token rotation): names the provider(s) and jumps straight to Settings -> Models to reconnect. Stays put until the user acts; the X dismisses it.
+
+import React from 'react';
+import Snackbar from '@mui/material/Snackbar';
+import Alert from '@mui/material/Alert';
+import Button from '@mui/material/Button';
+import IconButton from '@mui/material/IconButton';
+import CloseIcon from '@mui/icons-material/Close';
+import { useClaudeTokens } from '@/shared/styles/ThemeContext';
+import { useAppDispatch, useAppSelector } from '@/shared/hooks';
+import { hideProviderHealthToast } from '@/shared/state/subscriptionsSlice';
+import { openSettingsModal } from '@/shared/state/settingsSlice';
+
+export default function ProviderHealthToast() {
+  const c = useClaudeTokens();
+  const dispatch = useAppDispatch();
+  const open = useAppSelector((s) => s.subscriptions.healthToastOpen);
+  const dead = useAppSelector((s) => s.subscriptions.healthDead);
+
+  const onReconnect = React.useCallback(() => {
+    dispatch(openSettingsModal('models'));
+    dispatch(hideProviderHealthToast());
+  }, [dispatch]);
+
+  const labels = dead.map((d) => d.label).join(' and ');
+
+  return (
+    <Snackbar
+      open={open && dead.length > 0}
+      autoHideDuration={null}
+      onClose={() => dispatch(hideProviderHealthToast())}
+      anchorOrigin={{ vertical: 'bottom', horizontal: 'left' }}
+    >
+      <Alert
+        icon={false}
+        severity="warning"
+        sx={{
+          bgcolor: c.bg.surface,
+          color: c.text.primary,
+          border: `1px solid ${c.border.medium}`,
+          '& .MuiAlert-action': { alignItems: 'center', pt: 0 },
+        }}
+        action={
+          <>
+            <Button size="small" onClick={onReconnect} sx={{ color: c.accent.primary, fontWeight: 700 }}>
+              Reconnect
+            </Button>
+            <IconButton
+              size="small"
+              aria-label="Dismiss"
+              onClick={() => dispatch(hideProviderHealthToast())}
+              sx={{ color: c.text.muted, ml: 0.25, '&:hover': { color: c.text.primary } }}
+            >
+              <CloseIcon fontSize="small" />
+            </IconButton>
+          </>
+        }
+      >
+        Your {labels} login{dead.length > 1 ? 's have' : ' has'} expired; chats on {dead.length > 1 ? 'them' : 'it'} will fail until you reconnect.
+      </Alert>
+    </Snackbar>
+  );
+}

frontend/src/app/pages/Dashboard/canvas/DashboardOverlays.tsx

@@ -6,6 +6,7 @@ import CardSearchPalette from '../controls/CardSearchPalette';
 import DirectionHints from '../controls/DirectionHints';
 import WorkflowRunningToast from '@/app/pages/Workflows/WorkflowRunningToast';
 import MissedRunsToast from '@/app/pages/Workflows/MissedRunsToast';
+import ProviderHealthToast from '@/app/components/overlays/ProviderHealthToast';
 import type { AgentSession } from '@/shared/state/agentsSlice';
 import type {
   CardPosition,
@@ -152,6 +153,9 @@ const DashboardOverlays: React.FC<DashboardOverlaysProps> = ({
 
       {/* Launch nudge when scheduled runs elapsed while the app was closed */}
       <MissedRunsToast />
+
+      {/* Launch nudge when a subscription login died while the app was closed */}
+      <ProviderHealthToast />
     </>
   );
 };

frontend/src/app/pages/Dashboard/hooks/lifecycle/useDashboardLifecycle.ts

@@ -25,6 +25,7 @@ import { fetchOutputs, type Output } from '@/shared/state/outputsSlice';
 import { generateDashboardName } from '@/shared/state/dashboardsSlice';
 import { fetchWorkflows, fetchAllRuns, fetchActiveRuns } from '@/shared/state/workflowsSlice';
 import { fetchMissedRuns } from '@/shared/state/missedRunsSlice';
+import { fetchProviderHealth } from '@/shared/state/subscriptionsSlice';
 import { dashboardWs } from '@/shared/ws/WebSocketManager';
 import { initBrowserCommandHandler } from '@/shared/browserCommandHandler';
 import { getKeepAliveBrowserIds } from '@/shared/browserFocus';
@@ -81,6 +82,14 @@ export function useDashboardLifecycle({
     if (!isActive || missedRunsCheckedThisSession) return;
     missedRunsCheckedThisSession = true;
     dispatch(fetchMissedRuns());
+    // Login-health check rides the same once-per-launch gate; delayed so the lazy router has time to boot, with ONE retry when the probe reports it wasn't up yet.
+    const t = setTimeout(async () => {
+      try {
+        const res = await dispatch(fetchProviderHealth()).unwrap();
+        if (res.skipped) setTimeout(() => { dispatch(fetchProviderHealth()); }, 45_000);
+      } catch { /* probe is best-effort; silence on failure */ }
+    }, 12_000);
+    return () => clearTimeout(t);
   }, [isActive, dispatch]);
 
   // Track dashboard engagement time

frontend/src/shared/state/subscriptionsSlice.ts

@@ -17,8 +17,15 @@ export interface SubscriptionStatus {
   [key: string]: any;
 }
 
+export interface DeadProvider {
+  provider: string;
+  label: string;
+}
+
 export interface SubscriptionsState {
   status: SubscriptionStatus | null;
+  healthDead: DeadProvider[];
+  healthToastOpen: boolean;
 }
 
 // Minimal slice shape for selectors; avoids circular type import from store.ts.
@@ -26,6 +33,8 @@ type WithSubscriptions = { subscriptions: SubscriptionsState };
 
 const initialState: SubscriptionsState = {
   status: null,
+  healthDead: [],
+  healthToastOpen: false,
 };
 
 /** Mirror /agents/subscriptions/status into Redux; preserveTransient debounces is_running() false negatives. */
@@ -44,6 +53,15 @@ export const fetchSubscriptionStatus = createAsyncThunk(
   },
 );
 
+/** Boot-time login-health check; `skipped` means the router wasn't up yet, caller may retry once. */
+export const fetchProviderHealth = createAsyncThunk(
+  'subscriptions/fetchHealth',
+  async (): Promise<{ dead: DeadProvider[]; skipped: boolean }> => {
+    const r = await fetch(`${API_BASE}/agents/subscriptions/health`);
+    return (await r.json()) as { dead: DeadProvider[]; skipped: boolean };
+  },
+);
+
 const subscriptionsSlice = createSlice({
   name: 'subscriptions',
   initialState,
@@ -72,15 +90,23 @@ const subscriptionsSlice = createSlice({
         state.status.providers = { connections: conns };
       }
     },
+    hideProviderHealthToast(state) {
+      state.healthToastOpen = false;
+    },
   },
   extraReducers: (builder) => {
     builder.addCase(fetchSubscriptionStatus.fulfilled, (state, action) => {
       state.status = action.payload;
     });
+    builder.addCase(fetchProviderHealth.fulfilled, (state, action) => {
+      if (action.payload.skipped) return;
+      state.healthDead = action.payload.dead ?? [];
+      state.healthToastOpen = state.healthDead.length > 0;
+    });
   },
 });
 
-export const { setSubscriptionStatus, markSubscriptionConnected } = subscriptionsSlice.actions;
+export const { setSubscriptionStatus, markSubscriptionConnected, hideProviderHealthToast } = subscriptionsSlice.actions;
 
 // Stable empty ref so the selector doesn't hand back a fresh [] each call (forces needless rerenders).
 const EMPTY_CONNECTIONS: SubscriptionConnection[] = [];