diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 6ac4deae..34291509 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -488,6 +488,24 @@ async def subscriptions_exchange(body: dict): raise HTTPException(status_code=500, detail=str(e)) +@agents.router.get("/subscriptions/health") +async def subscriptions_health(): + """Boot-time login-health check: 1-token probe per ACTIVE subscription lane, reporting only + definitive auth-death (the silent credential-rot class). `skipped` when the router isn't up yet + so the frontend can retry once instead of reading 'all healthy' off a cold boot.""" + from backend.apps.nine_router import is_running, get_providers + from backend.apps.nine_router.subscription_health import probe_subscription_health + if not is_running(): + return {"dead": [], "skipped": True} + try: + connections = await get_providers() + dead = await probe_subscription_health(connections) + return {"dead": dead, "skipped": False} + except Exception as e: + logger.debug(f"subscription health probe failed: {e}") + return {"dead": [], "skipped": True} + + @agents.router.get("/subscriptions/models") async def subscriptions_models(): """List all models available through connected subscriptions.""" diff --git a/backend/apps/nine_router/subscription_health.py b/backend/apps/nine_router/subscription_health.py new file mode 100644 index 00000000..a805b7cf --- /dev/null +++ b/backend/apps/nine_router/subscription_health.py @@ -0,0 +1,114 @@ +"""Boot-time subscription health probe: catches a provider login that died while the app was +closed (refresh-token rotation, the "Breaking codex" class) so the UI can offer reconnect BEFORE +the user burns a failed turn discovering it. Probes SUBSCRIPTION lanes only (1 token of sub quota, +never a billable API key), and only a definitive auth-shaped 401/403 counts as dead: transient +429/5xx/timeouts stay silent so the pill can never cry wolf. Kill switch: OPENSWARM_BOOT_HEALTH=0.""" + +import asyncio +import logging +import os +import time +from typing import Dict, List, Optional + +import httpx +from typeguard import typechecked + +from backend.apps.nine_router.process import NINE_ROUTER_URL, is_running + +logger = logging.getLogger(__name__) + +PREFIX_BY_PROVIDER: Dict[str, str] = { + "claude": "cc/", + "codex": "cx/", + "gemini-cli": "gemini/", + "antigravity": "ag/", +} +LABEL_BY_PROVIDER: Dict[str, str] = { + "claude": "Claude", + "codex": "ChatGPT", + "gemini-cli": "Gemini", + "antigravity": "Gemini (Antigravity)", +} +P_AUTH_DEAD_MARKERS = ("authentication", "expired", "sign in", "signing in", "invalid_grant", "unauthorized", "invalid authentication") +P_PROBE_TIMEOUT_S = 25.0 +P_CACHE_TTL_S = 300.0 + +p_probe_lock = asyncio.Lock() +p_cached_result: Optional[List[Dict[str, str]]] = None +p_cached_at: float = 0.0 + + +@typechecked +def health_probe_enabled() -> bool: + return os.environ.get("OPENSWARM_BOOT_HEALTH", "1") != "0" + + +@typechecked +def classify_auth_dead(status_code: int, body_text: str) -> bool: + """Dead ONLY on a definitive auth failure; anything ambiguous reads healthy (silence beats a false reconnect prompt).""" + if status_code not in (401, 403): + return False + low = body_text.lower() + return any(m in low for m in P_AUTH_DEAD_MARKERS) + + +@typechecked +async def p_pick_probe_model(client: httpx.AsyncClient, prefix: str) -> Optional[str]: + try: + r = await client.get(f"{NINE_ROUTER_URL}/v1/models") + if r.status_code != 200: + return None + ids = [m.get("id") for m in (r.json().get("data") or []) if isinstance(m, dict)] + for i in ids: + if isinstance(i, str) and i.startswith(prefix): + return i + except Exception: + return None + return None + + +@typechecked +async def p_probe_one(client: httpx.AsyncClient, model: str) -> Optional[bool]: + """True = auth dead, False = healthy, None = inconclusive (never reported).""" + try: + r = await client.post( + f"{NINE_ROUTER_URL}/v1/messages", + json={"model": model, "max_tokens": 1, "messages": [{"role": "user", "content": "hi"}]}, + headers={"x-api-key": "9router", "anthropic-version": "2023-06-01"}, + ) + if r.status_code < 400: + return False + return True if classify_auth_dead(r.status_code, r.text or "") else None + except Exception: + return None + + +@typechecked +async def probe_subscription_health(connections: List[Dict]) -> List[Dict[str, str]]: + """Probe each active subscription connection with a 1-token turn; returns [{provider, label}] + for the definitively auth-dead ones. Cached for 5 minutes; concurrent callers share one run.""" + global p_cached_result, p_cached_at + if not health_probe_enabled() or not is_running(): + return [] + async with p_probe_lock: + if p_cached_result is not None and time.monotonic() - p_cached_at < P_CACHE_TTL_S: + return p_cached_result + subs = [ + c for c in connections + if isinstance(c, dict) and c.get("provider") in PREFIX_BY_PROVIDER and c.get("isActive") + ] + dead: List[Dict[str, str]] = [] + if subs: + async with httpx.AsyncClient(timeout=P_PROBE_TIMEOUT_S) as client: + for c in subs: + provider = str(c.get("provider")) + model = await p_pick_probe_model(client, PREFIX_BY_PROVIDER[provider]) + if not model: + continue + verdict = await p_probe_one(client, model) + if verdict is True: + dead.append({"provider": provider, "label": LABEL_BY_PROVIDER[provider]}) + logger.info(f"[sub-health] {provider}: auth dead (reconnect needed)") + p_cached_result = dead + p_cached_at = time.monotonic() + return dead diff --git a/backend/tests/test_subscription_health.py b/backend/tests/test_subscription_health.py new file mode 100644 index 00000000..806fcefc --- /dev/null +++ b/backend/tests/test_subscription_health.py @@ -0,0 +1,88 @@ +"""Boot-time subscription health probe: only a definitive auth-shaped 401/403 reads as dead +(transients stay silent so the reconnect pill can't cry wolf), probes cover only active +subscription lanes, and results are cached so a double boot-fetch can't double-spend probes.""" + +import asyncio +import json +from typing import Dict, List, Optional + +import backend.apps.nine_router.subscription_health as sh + + +def test_classify_auth_dead_is_conservative(): + assert sh.classify_auth_dead(401, '{"error": "authentication token is expired"}') + assert sh.classify_auth_dead(403, "Unauthorized access") + assert sh.classify_auth_dead(401, "Invalid authentication credentials") + assert not sh.classify_auth_dead(401, "weird opaque body") # 401 without an auth marker stays silent + assert not sh.classify_auth_dead(429, "rate limit expired") # non-auth status never fires + assert not sh.classify_auth_dead(500, "authentication service down") + assert not sh.classify_auth_dead(200, "expired") + + +class FakeResponse: + def __init__(self, status_code: int, payload: Dict): + self.status_code = status_code + self.text = json.dumps(payload) + self.p_payload = payload + + def json(self) -> Dict: + return self.p_payload + + +class FakeClient: + """cc/ lane healthy, cx/ lane auth-dead, gemini/ lane rate-limited (must stay silent).""" + + def __init__(self, counter: List[int], **kwargs): + self.counter = counter + + async def __aenter__(self) -> "FakeClient": + return self + + async def __aexit__(self, *exc) -> None: + return None + + async def get(self, url: str, **kw) -> FakeResponse: + return FakeResponse(200, {"data": [ + {"id": "cc/claude-haiku"}, {"id": "cx/gpt-5.4"}, {"id": "gemini/gemini-2.5-flash"}, + ]}) + + async def post(self, url: str, **kw) -> FakeResponse: + self.counter[0] += 1 + model = kw.get("json", {}).get("model", "") + if model.startswith("cx/"): + return FakeResponse(401, {"error": {"message": "authentication token is expired, try signing in again"}}) + if model.startswith("gemini/"): + return FakeResponse(429, {"error": {"message": "rate limited"}}) + return FakeResponse(200, {"content": []}) + + +CONNS = [ + {"provider": "claude", "isActive": True}, + {"provider": "codex", "isActive": True}, + {"provider": "gemini-cli", "isActive": True}, + {"provider": "openrouter", "isActive": True}, # not a sub lane; never probed + {"provider": "codex", "isActive": False}, # inactive; never probed +] + + +def test_probe_reports_only_definitive_death_and_caches(monkeypatch): + counter = [0] + monkeypatch.setattr(sh, "is_running", lambda: True) + monkeypatch.setattr(sh.httpx, "AsyncClient", lambda **kw: FakeClient(counter, **kw)) + monkeypatch.setattr(sh, "p_cached_result", None) + monkeypatch.setattr(sh, "p_cached_at", 0.0) + dead = asyncio.run(sh.probe_subscription_health(CONNS)) + assert dead == [{"provider": "codex", "label": "ChatGPT"}] + assert counter[0] == 3 # claude + codex + gemini-cli probed; openrouter/inactive skipped + # Second call within the TTL serves the cache; no new probe spend. + dead2 = asyncio.run(sh.probe_subscription_health(CONNS)) + assert dead2 == dead + assert counter[0] == 3 + + +def test_probe_disabled_or_router_down(monkeypatch): + monkeypatch.setattr(sh, "is_running", lambda: False) + assert asyncio.run(sh.probe_subscription_health(CONNS)) == [] + monkeypatch.setattr(sh, "is_running", lambda: True) + monkeypatch.setenv("OPENSWARM_BOOT_HEALTH", "0") + assert asyncio.run(sh.probe_subscription_health(CONNS)) == [] diff --git a/frontend/src/app/components/overlays/ProviderHealthToast.tsx b/frontend/src/app/components/overlays/ProviderHealthToast.tsx new file mode 100644 index 00000000..4683e726 --- /dev/null +++ b/frontend/src/app/components/overlays/ProviderHealthToast.tsx @@ -0,0 +1,63 @@ +// Bottom-left nudge shown at launch when a subscription login died while the app was closed (silent token rotation): names the provider(s) and jumps straight to Settings -> Models to reconnect. Stays put until the user acts; the X dismisses it. + +import React from 'react'; +import Snackbar from '@mui/material/Snackbar'; +import Alert from '@mui/material/Alert'; +import Button from '@mui/material/Button'; +import IconButton from '@mui/material/IconButton'; +import CloseIcon from '@mui/icons-material/Close'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; +import { hideProviderHealthToast } from '@/shared/state/subscriptionsSlice'; +import { openSettingsModal } from '@/shared/state/settingsSlice'; + +export default function ProviderHealthToast() { + const c = useClaudeTokens(); + const dispatch = useAppDispatch(); + const open = useAppSelector((s) => s.subscriptions.healthToastOpen); + const dead = useAppSelector((s) => s.subscriptions.healthDead); + + const onReconnect = React.useCallback(() => { + dispatch(openSettingsModal('models')); + dispatch(hideProviderHealthToast()); + }, [dispatch]); + + const labels = dead.map((d) => d.label).join(' and '); + + return ( + 0} + autoHideDuration={null} + onClose={() => dispatch(hideProviderHealthToast())} + anchorOrigin={{ vertical: 'bottom', horizontal: 'left' }} + > + + + dispatch(hideProviderHealthToast())} + sx={{ color: c.text.muted, ml: 0.25, '&:hover': { color: c.text.primary } }} + > + + + + } + > + Your {labels} login{dead.length > 1 ? 's have' : ' has'} expired; chats on {dead.length > 1 ? 'them' : 'it'} will fail until you reconnect. + + + ); +} diff --git a/frontend/src/app/pages/Dashboard/canvas/DashboardOverlays.tsx b/frontend/src/app/pages/Dashboard/canvas/DashboardOverlays.tsx index 64473f54..350e9b3e 100644 --- a/frontend/src/app/pages/Dashboard/canvas/DashboardOverlays.tsx +++ b/frontend/src/app/pages/Dashboard/canvas/DashboardOverlays.tsx @@ -6,6 +6,7 @@ import CardSearchPalette from '../controls/CardSearchPalette'; import DirectionHints from '../controls/DirectionHints'; import WorkflowRunningToast from '@/app/pages/Workflows/WorkflowRunningToast'; import MissedRunsToast from '@/app/pages/Workflows/MissedRunsToast'; +import ProviderHealthToast from '@/app/components/overlays/ProviderHealthToast'; import type { AgentSession } from '@/shared/state/agentsSlice'; import type { CardPosition, @@ -152,6 +153,9 @@ const DashboardOverlays: React.FC = ({ {/* Launch nudge when scheduled runs elapsed while the app was closed */} + + {/* Launch nudge when a subscription login died while the app was closed */} + ); }; diff --git a/frontend/src/app/pages/Dashboard/hooks/lifecycle/useDashboardLifecycle.ts b/frontend/src/app/pages/Dashboard/hooks/lifecycle/useDashboardLifecycle.ts index eb30d497..90e79f6c 100644 --- a/frontend/src/app/pages/Dashboard/hooks/lifecycle/useDashboardLifecycle.ts +++ b/frontend/src/app/pages/Dashboard/hooks/lifecycle/useDashboardLifecycle.ts @@ -25,6 +25,7 @@ import { fetchOutputs, type Output } from '@/shared/state/outputsSlice'; import { generateDashboardName } from '@/shared/state/dashboardsSlice'; import { fetchWorkflows, fetchAllRuns, fetchActiveRuns } from '@/shared/state/workflowsSlice'; import { fetchMissedRuns } from '@/shared/state/missedRunsSlice'; +import { fetchProviderHealth } from '@/shared/state/subscriptionsSlice'; import { dashboardWs } from '@/shared/ws/WebSocketManager'; import { initBrowserCommandHandler } from '@/shared/browserCommandHandler'; import { getKeepAliveBrowserIds } from '@/shared/browserFocus'; @@ -81,6 +82,14 @@ export function useDashboardLifecycle({ if (!isActive || missedRunsCheckedThisSession) return; missedRunsCheckedThisSession = true; dispatch(fetchMissedRuns()); + // Login-health check rides the same once-per-launch gate; delayed so the lazy router has time to boot, with ONE retry when the probe reports it wasn't up yet. + const t = setTimeout(async () => { + try { + const res = await dispatch(fetchProviderHealth()).unwrap(); + if (res.skipped) setTimeout(() => { dispatch(fetchProviderHealth()); }, 45_000); + } catch { /* probe is best-effort; silence on failure */ } + }, 12_000); + return () => clearTimeout(t); }, [isActive, dispatch]); // Track dashboard engagement time diff --git a/frontend/src/shared/state/subscriptionsSlice.ts b/frontend/src/shared/state/subscriptionsSlice.ts index 335294df..388566ca 100644 --- a/frontend/src/shared/state/subscriptionsSlice.ts +++ b/frontend/src/shared/state/subscriptionsSlice.ts @@ -17,8 +17,15 @@ export interface SubscriptionStatus { [key: string]: any; } +export interface DeadProvider { + provider: string; + label: string; +} + export interface SubscriptionsState { status: SubscriptionStatus | null; + healthDead: DeadProvider[]; + healthToastOpen: boolean; } // Minimal slice shape for selectors; avoids circular type import from store.ts. @@ -26,6 +33,8 @@ type WithSubscriptions = { subscriptions: SubscriptionsState }; const initialState: SubscriptionsState = { status: null, + healthDead: [], + healthToastOpen: false, }; /** Mirror /agents/subscriptions/status into Redux; preserveTransient debounces is_running() false negatives. */ @@ -44,6 +53,15 @@ export const fetchSubscriptionStatus = createAsyncThunk( }, ); +/** Boot-time login-health check; `skipped` means the router wasn't up yet, caller may retry once. */ +export const fetchProviderHealth = createAsyncThunk( + 'subscriptions/fetchHealth', + async (): Promise<{ dead: DeadProvider[]; skipped: boolean }> => { + const r = await fetch(`${API_BASE}/agents/subscriptions/health`); + return (await r.json()) as { dead: DeadProvider[]; skipped: boolean }; + }, +); + const subscriptionsSlice = createSlice({ name: 'subscriptions', initialState, @@ -72,15 +90,23 @@ const subscriptionsSlice = createSlice({ state.status.providers = { connections: conns }; } }, + hideProviderHealthToast(state) { + state.healthToastOpen = false; + }, }, extraReducers: (builder) => { builder.addCase(fetchSubscriptionStatus.fulfilled, (state, action) => { state.status = action.payload; }); + builder.addCase(fetchProviderHealth.fulfilled, (state, action) => { + if (action.payload.skipped) return; + state.healthDead = action.payload.dead ?? []; + state.healthToastOpen = state.healthDead.length > 0; + }); }, }); -export const { setSubscriptionStatus, markSubscriptionConnected } = subscriptionsSlice.actions; +export const { setSubscriptionStatus, markSubscriptionConnected, hideProviderHealthToast } = subscriptionsSlice.actions; // Stable empty ref so the selector doesn't hand back a fresh [] each call (forces needless rerenders). const EMPTY_CONNECTIONS: SubscriptionConnection[] = [];