diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 5d912bdc..1ae67167 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -864,7 +864,15 @@ async def list_models(): if visible: result[provider_name] = visible + # Availability answers "can you use this right now"; the CATALOG answers "does this model still exist". + # Only the second one may retire a user's pinned model, or a router bounce silently moves their chat to another vendor (ENG-386). + p_catalog: list[str] = [ + m["value"] for p_models in BUILTIN_MODELS.values() for m in p_models if m.get("value") + ] + p_catalog_complete = True + # Fetch OpenRouter catalog directly (independent of 9Router) so picker fills the moment a key lands. + or_models: list[dict] = [] if has_openrouter_key: try: from backend.apps.agents.providers.registry import fetch_openrouter_models @@ -872,6 +880,10 @@ async def list_models(): except Exception as e: logger.debug(f"OpenRouter catalog fetch failed: {e}") or_models = [] + # A key we could not enumerate means we cannot vouch for the catalog, so nothing may be retired from it this tick. + if not or_models: + p_catalog_complete = False + p_catalog += [m["value"] for m in or_models if m.get("value")] if or_models: by_vendor: dict[str, list[dict]] = {} from backend.apps.agents.providers.registry import ( @@ -941,6 +953,7 @@ async def list_models(): "billing_kind": "api_key", "tiers": [3, 3, 1], }) + p_catalog += [e["value"] for e in entries] if entries: result[cp_name] = entries @@ -954,7 +967,12 @@ async def list_models(): hr["billing_kind"] = "free" result["Anthropic"] = haiku_rows - return {"models": result, "notes": notes} + return { + "models": result, + "notes": notes, + "known_values": sorted(set(p_catalog)), + "catalog_complete": p_catalog_complete, + } @agents.router.post("/subscriptions/disconnect") diff --git a/backend/tests/test_model_catalog_vs_availability.py b/backend/tests/test_model_catalog_vs_availability.py new file mode 100644 index 00000000..242aea65 --- /dev/null +++ b/backend/tests/test_model_catalog_vs_availability.py @@ -0,0 +1,60 @@ +"""A model's ABSENCE from today's picker is not evidence it was retired. + +The bug class (ENG-386, seen live on a field install): /models is intersected with available creds +and with 9Router's in-process provider state, so a router bounce or a provider cooldown drops whole +vendors out of the payload. The renderer read "not in the list" as "retired", rewrote every session +pinned to that vendor onto the default model, and PERSISTED it. A chat the user had put on GPT came +back on Claude, and then collected Anthropic policy blocks the user could not explain. + +The seal: the payload carries a catalog that depends on neither creds nor router state. Availability +answers "can you use this right now"; only the catalog answers "does this still exist", and only the +catalog may retire a session's model. +""" + +import asyncio +from unittest.mock import patch + +from backend.apps.agents.agents import list_models +from backend.apps.settings.models import AppSettings, CustomProvider + + +def p_run(cfg: AppSettings, *, router_up: bool = False): + with patch("backend.apps.settings.settings.load_settings", return_value=cfg), \ + patch("backend.apps.nine_router.is_running", return_value=router_up): + return asyncio.run(list_models()) + + +def test_a_provider_dropout_does_not_empty_the_catalog(): + """The whole point: with nothing connected, availability collapses but the catalog does not.""" + result = p_run(AppSettings()) + available = {m["value"] for rows in result["models"].values() for m in rows} + known = set(result["known_values"]) + + assert known, "the catalog must never be empty, or the renderer has no way to tell gone from unreachable" + assert known - available, "a dropout must leave models known-but-unavailable, which is exactly the state that used to read as retired" + + +def test_subscription_models_survive_a_router_bounce_in_the_catalog(): + """9Router stamps provider state in-process, so a bounce un-connects every subscription lane.""" + up = p_run(AppSettings(), router_up=True) + down = p_run(AppSettings(), router_up=False) + assert set(down["known_values"]) == set(up["known_values"]), "the catalog must not move when the router does" + + +def test_a_custom_providers_models_are_in_the_catalog(): + cfg = AppSettings(custom_providers=[ + CustomProvider(name="Ollama Cloud", base_url="https://ollama.com/v1", api_key="x", + models=[{"value": "gpt-oss:120b", "label": "gpt-oss:120b"}]), + ]) + assert "custom/ollama-cloud/gpt-oss:120b" in p_run(cfg)["known_values"] + + +def test_an_unenumerable_provider_marks_the_catalog_incomplete(): + """A key we could not enumerate means we cannot vouch for the list, so nothing may be retired from it.""" + cfg = AppSettings(openrouter_api_key="sk-or-broken") + with patch("backend.apps.agents.providers.registry.fetch_openrouter_models", + side_effect=RuntimeError("network down")): + result = p_run(cfg) + assert result["catalog_complete"] is False + + assert p_run(AppSettings())["catalog_complete"] is True diff --git a/frontend/src/app/Main.tsx b/frontend/src/app/Main.tsx index 218e12d9..79dcb934 100644 --- a/frontend/src/app/Main.tsx +++ b/frontend/src/app/Main.tsx @@ -354,6 +354,9 @@ const DefaultModelGuard: React.FC<{ children: React.ReactNode }> = ({ children } const settingsLoaded = useAppSelector((s) => s.settings.loaded); const byProvider = useAppSelector((s) => s.models.byProvider); const modelsLoaded = useAppSelector((s) => s.models.loaded); + // Availability is a live fact (a router bounce or a provider cooldown drops models out of /models); the catalog is a static one. Retiring a user's model on availability silently moved whole chats to another vendor, which is how a GPT chat started returning Anthropic policy blocks (ENG-386). + const knownValues = useAppSelector((s) => s.models.knownValues); + const catalogComplete = useAppSelector((s) => s.models.catalogComplete); // Until 9Router answers, /models omits subscription models, so the saved default can look "no longer available" when it's really just not loaded yet. Reconciling then would clobber a real sub user's default down to a fallback (and persist it). Only reconcile against the complete list. const nineRouterUp = useAppSelector((s) => s.subscriptions.status?.running === true); // A primitive fingerprint, not the sessions map: subscribing the app ROOT to whole sessions re-rendered it on every stream tick; this only changes when some session's MODEL changes. @@ -372,9 +375,11 @@ const DefaultModelGuard: React.FC<{ children: React.ReactNode }> = ({ children } if (pendingRef.current) return; if (Object.keys(byProvider).length === 0) return; + if (!catalogComplete || knownValues.length === 0) return; const flat = Object.values(byProvider).flat(); const currentExists = flat.some((m) => m.value === settings.default_model); if (currentExists) return; + if (knownValues.includes(settings.default_model)) return; // Nothing real connected means the synthesized free row is the whole list; persisting it would brand haiku as the user's default forever (ENG-343). if (!flat.some((m) => m.billing_kind !== 'free')) return; @@ -389,7 +394,7 @@ const DefaultModelGuard: React.FC<{ children: React.ReactNode }> = ({ children } pendingRef.current = false; }); setSessionSwitch({ toFreeTrial: connectionMode === 'free-trial', runs: freeTrialRemaining ?? null, toLabel: fallback.label }); - }, [settingsLoaded, modelsLoaded, nineRouterUp, connectionMode, freeTrialRemaining, byProvider, settings, dispatch]); + }, [settingsLoaded, modelsLoaded, nineRouterUp, connectionMode, freeTrialRemaining, byProvider, knownValues, catalogComplete, settings, dispatch]); // Same staleness per session: a session pinned to a now-gone model (e.g. gpt-5.4-api after its key is disconnected) snags on the next send since the send carries that model, so reconcile open sessions to the valid default/fallback and warn once. useEffect(() => { @@ -402,10 +407,13 @@ const DefaultModelGuard: React.FC<{ children: React.ReactNode }> = ({ children } if (valid.size === 0) return; const fallback = pickFallbackModel(byProvider); if (!fallback) return; + // A model absent from the catalog is genuinely gone; one merely absent from today's list is a provider we cannot reach this second, and moving the chat off it is never ours to do silently. + if (!catalogComplete || knownValues.length === 0) return; + const known = new Set(knownValues); const target = valid.has(settings.default_model) ? settings.default_model : fallback.value; let switched = false; for (const sess of Object.values(store.getState().agents.sessions)) { - if (sess.model && !valid.has(sess.model)) { + if (sess.model && !known.has(sess.model)) { // The switch is store-only, and the metadata poll re-hydrates the dead model from disk every // few seconds, so re-announcing meant the banner returned forever for anyone holding a chat // pinned to a retired model. Fix it every time (the next send must carry a live model), tell @@ -423,7 +431,7 @@ const DefaultModelGuard: React.FC<{ children: React.ReactNode }> = ({ children } const toLabel = flat.find((m) => m.value === target)?.label ?? target; setSessionSwitch({ toFreeTrial: connectionMode === 'free-trial', runs: freeTrialRemaining ?? null, toLabel }); } - }, [settingsLoaded, modelsLoaded, nineRouterUp, connectionMode, freeTrialRemaining, byProvider, sessionModelsKey, settings, dispatch]); + }, [settingsLoaded, modelsLoaded, nineRouterUp, connectionMode, freeTrialRemaining, byProvider, knownValues, catalogComplete, sessionModelsKey, settings, dispatch]); return ( <> diff --git a/frontend/src/shared/state/modelsSlice.ts b/frontend/src/shared/state/modelsSlice.ts index 639e3276..29e8e0db 100644 --- a/frontend/src/shared/state/modelsSlice.ts +++ b/frontend/src/shared/state/modelsSlice.ts @@ -20,12 +20,18 @@ export interface ModelOption { interface ModelsState { byProvider: Record; + /** Every model that EXISTS, independent of creds or router state. Availability says "usable now"; only this says "still exists". */ + knownValues: string[]; + /** False when a configured provider could not be enumerated, so nothing may be retired from the catalog this tick. */ + catalogComplete: boolean; loaded: boolean; failed: boolean; } const initialState: ModelsState = { byProvider: {}, + knownValues: [], + catalogComplete: false, loaded: false, failed: false, }; @@ -34,8 +40,11 @@ export const fetchModels = createAsyncThunk('models/fetchModels', async () => { const res = await fetch(`${AGENTS_API}/models`); if (!res.ok) throw new Error('Failed to fetch models'); const data = await res.json(); - const models = data.models || data; - return models as Record; + return { + byProvider: (data.models || data) as Record, + knownValues: (data.known_values ?? []) as string[], + catalogComplete: data.catalog_complete !== false, + }; }); const modelsSlice = createSlice({ @@ -45,7 +54,9 @@ const modelsSlice = createSlice({ extraReducers: (builder) => { builder .addCase(fetchModels.fulfilled, (state, action) => { - state.byProvider = action.payload; + state.byProvider = action.payload.byProvider; + state.knownValues = action.payload.knownValues; + state.catalogComplete = action.payload.catalogComplete; state.loaded = true; }) .addCase(fetchModels.rejected, (state) => {