[eric] agents: never spend a turn on a lane the router gave up on, and stop our own retries stacking cards

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014wtspwSFzZmjCx9UNPAorQ
This commit is contained in:
ciregenz
2026-08-20 09:20:46 -07:00
co-authored by Claude Opus 5
parent 58e376496e
commit 9a7570092f
6 changed files with 368 additions and 21 deletions
+20
View File
@@ -207,6 +207,26 @@ class AgentManager(SessionLifecycle, SessionPersistence, Messaging, SessionContr
p_router_model_id = p_resolve_model_id_early(session.model, load_settings())
p_api_type_for_session = p_get_api_type_early(session.model)
# Never spend a turn on a lane the router has already given up on: the live 2026-08-20 drill burned two minutes and six cards on a credential that had been dead for 89 hours. This heals it if it can, and tells the truth immediately if it cannot.
from backend.apps.agents.manager.run.lane_preflight import preflight_lane
p_lane_problem = await preflight_lane(p_router_model_id, session)
if p_lane_problem:
p_card = Message(role="system", content=p_lane_problem, branch_id=session.active_branch_id)
from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card
absorb_repeat_card(session, p_card)
session.status = "error"
await ws_manager.send_to_session(session_id, "agent:auth_error", {
"session_id": session_id,
"reason": "credential_expired",
"message": p_lane_problem,
"model": session.model,
})
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": p_card.model_dump(mode="json"),
})
return
builtin_perms = load_builtin_permissions()
# Builtins default to always_allow (frictionless); path_gate still force-prompts on catastrophic patterns (rm -rf), OS-scheduling, and sensitive paths, so poisoned-email -> destructive-command is still caught. Flip Bash to "ask" in the UI for a prompt on every command. Bind turn + stderr first: build_agent_options can raise early (no provider) and the except hands both to handle_run_error.
+2
View File
@@ -153,6 +153,8 @@ class AgentSession(BaseModel):
# Consecutive dirty deaths this session was MID-TURN for; the crash auto-resume breaker (hermes #30719 pairing: auto-resume must never outrun its circuit breaker).
crash_interrupt_count: int = 0
# Outage rounds spent on this ask: the in-turn ladder covers only 335s, and the work is checkpointed, so a longer drop is waited out rather than ending the task.
# True when the preflight found the router had already given up on this lane, so an auth failure this turn is a dead credential, not a rotation window worth waiting out.
lane_credential_dead: bool = False
reconnect_attempts: int = 0
# True while a turn is parked waiting for the connection back; persisted so a quit DURING the wait is still an owed turn at next boot.
awaiting_reconnect: bool = False
@@ -35,12 +35,19 @@ logger = logging.getLogger(__name__)
@typechecked
def p_absorb_repeat_card(session: AgentSession, error_msg: Message) -> None:
def absorb_repeat_card(session: AgentSession, error_msg: Message) -> None:
"""Append the error card, unless the branch tail is already the IDENTICAL card with nothing
after it: a retry ladder re-failing the same way then bumps the existing card instead of
stacking a wall of clones (field screenshot 2026-08-19). A user message in between always
yields a fresh card, each ask deserves its own honest answer."""
p_tail = [m for m in session.messages if getattr(m, "branch_id", None) in (None, session.active_branch_id)]
yields a fresh card, each ask deserves its own honest answer.
HIDDEN messages do not count as that user message. Our own self-heal continuations are hidden
user-role sends, so counting them re-opened the exact clone wall this exists to stop: a live
codex drill on 2026-08-20 produced FIVE identical "still refreshing" cards, one per retry,
because each retry's hidden prompt had displaced the previous card from the tail."""
p_tail = [m for m in session.messages
if getattr(m, "branch_id", None) in (None, session.active_branch_id)
and not getattr(m, "hidden", False)]
if p_tail and p_tail[-1].role == "system" and p_tail[-1].content == error_msg.content:
error_msg.id = p_tail[-1].id
p_tail[-1].timestamp = error_msg.timestamp
@@ -97,7 +104,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
"with a larger window."
)
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
p_absorb_repeat_card(session, error_msg)
absorb_repeat_card(session, error_msg)
p_ovf_payload = {
"session_id": session_id,
"reason": "long_context_required" if p_tier_gate else "context_overflow",
@@ -144,7 +151,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
"api.anthropic.com; retrying won't help until one of those changes."
)
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
p_absorb_repeat_card(session, error_msg)
absorb_repeat_card(session, error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
@@ -160,7 +167,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
"are kept either way."
)
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
p_absorb_repeat_card(session, error_msg)
absorb_repeat_card(session, error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
@@ -216,7 +223,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
"OpenSwarm Pro."
)
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
p_absorb_repeat_card(session, error_msg)
absorb_repeat_card(session, error_msg)
await ws_manager.send_to_session(session_id, "agent:free_trial_exhausted", {
"session_id": session_id,
"message": friendly_msg,
@@ -248,7 +255,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
"or starting a fresh chat about this topic, usually clears it."
)
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
p_absorb_repeat_card(session, error_msg)
absorb_repeat_card(session, error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
@@ -263,7 +270,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
"another option in Settings → Models."
)
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
p_absorb_repeat_card(session, error_msg)
absorb_repeat_card(session, error_msg)
await ws_manager.send_to_session(session_id, "agent:out_of_credits", {
"session_id": session_id,
"message": friendly_msg,
@@ -283,6 +290,25 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or "401" in p_combined)
)
# Every sub lane gets ONE silent self-heal before any card; only a missing credential (config problem, retry fails identically) goes straight to the card. Codex waits out its rotation window first.
# A lane the router had ALREADY given up on before this turn is a dead credential, so the
# rotation story is false and the wait is doomed; say the true thing straight away.
if getattr(session, "lane_credential_dead", False):
from backend.apps.agents.manager.run.lane_preflight import RECONNECT_COPY
p_prov = (session.provider or "").lower()
friendly_msg = RECONNECT_COPY.get(
p_prov,
"This model's sign-in expired and could not be renewed. Reconnect it in Settings, "
"then Models. Waiting will not clear this one.")
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
absorb_repeat_card(session, error_msg)
await ws_manager.send_to_session(session_id, "agent:auth_error", {
"session_id": session_id, "reason": "credential_expired",
"message": friendly_msg, "model": session.model,
})
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": error_msg.model_dump(mode="json"),
})
return
if "no credentials for provider" not in p_combined:
from backend.apps.agents.manager.streaming.auth_retry import try_auth_self_heal
if try_auth_self_heal(session, delay_s=75 if p_codex_rotation else 5):
@@ -341,7 +367,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
)
reason = "anthropic_auth_invalid"
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
p_absorb_repeat_card(session, error_msg)
absorb_repeat_card(session, error_msg)
try:
from backend.apps.service.client import submit_diagnostic
submit_diagnostic({
@@ -368,7 +394,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
# Upstream rejected the model code itself (e.g. Codex 1211 on a ChatGPT plan that lacks our GPT ids). Track it; the friendly "add an API key / pick another model" card is rendered frontend-side.
p_report_model_error("unknown_model", session_id, session, turn, e, p_stderr_tail)
error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id)
p_absorb_repeat_card(session, error_msg)
absorb_repeat_card(session, error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
@@ -378,7 +404,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
# where the fix is entirely on our side of the wire.
p_report_model_error("router_unavailable", session_id, session, turn, e, p_stderr_tail)
error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id)
p_absorb_repeat_card(session, error_msg)
absorb_repeat_card(session, error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
@@ -400,7 +426,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
if p_cause and "check stderr" in str(e).lower():
p_card_text += f"\n\nRuntime log tail:\n{p_cause}"
error_msg = Message(role="system", content=p_card_text, branch_id=session.active_branch_id)
p_absorb_repeat_card(session, error_msg)
absorb_repeat_card(session, error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
@@ -0,0 +1,135 @@
"""Do not spend a user's turn on a lane we already know is dead.
Live drill, 2026-08-20: a codex credential that expired 89 HOURS earlier produced "GPT subscription
token just rotated (automatic, every couple minutes), retrying automatically, no action needed", a
75 second wait, a doomed retry, and then five identical cards. Zero files read. Every word of that
was wrong, and the evidence to know better was already sitting in 9Router's own provider list:
`testStatus: "unavailable"` with `errorCode: 401`, published before we spend anything.
So this looks first, and it tries to fix it before it complains:
healthy -> say nothing, cost nothing, dispatch as normal
sticky-dead, first -> bounce the router ONCE (re-reads db.json, clears the in-process
`unavailable` stamp and modelLock cooldowns), then DISPATCH ANYWAY and let
the turn itself be the verdict. If it goes through, the user never learns
anything happened.
sticky-dead, again -> one accurate sentence, immediately. No invented rotation window, no wait
the user has already tried by hand, no promise of a self-heal that cannot
happen.
The bounce is NOT allowed to declare success on its own, and that mistake is worth recording: the
first version re-read the health flag afterwards and called a cleared stamp a recovery. But a fresh
router starts with no stamp, so the check passed for a credential that was still dead, and the turn
hit the same 401 seconds later. A restart clears the accusation, not the cause. Only a real
dispatch can tell you whether a credential works, so that is what decides it now.
The bounce is the same one ENG-315 already runs at the connect chokepoint, and is documented safe
mid-session; the in-flight kill drill on 2026-08-20 confirmed a live turn survives one.
"""
import logging
import time
from typing import Dict, Optional, TYPE_CHECKING
from typeguard import typechecked
if TYPE_CHECKING:
from backend.apps.agents.core.models import AgentSession
logger = logging.getLogger(__name__)
# Router prefix -> the provider name its connection is filed under.
P_PREFIX_PROVIDER = {"cc/": "claude", "cx/": "codex", "gc/": "antigravity", "ag/": "antigravity"}
# A bounce restarts a process every other session shares, so one per lane per window, never per turn.
BOUNCE_COOLDOWN_S = 300
LAST_BOUNCE: Dict[str, float] = {}
RECONNECT_COPY = {
"codex": ("Your ChatGPT subscription needs reconnecting: the saved sign-in expired and could "
"not be renewed. Open Settings, then Models, and click Reconnect on the OpenAI / GPT "
"row. Waiting will not clear this one."),
"claude": ("Your Claude subscription needs reconnecting: the saved sign-in expired and could "
"not be renewed. Open Settings, then Models, and click Reconnect on the Claude "
"Pro / Max row. Waiting will not clear this one."),
"antigravity": ("Your Google sign-in needs reconnecting: the saved credential expired and "
"could not be renewed. Open Settings, then Models, and reconnect the Google "
"row. Waiting will not clear this one."),
}
@typechecked
def provider_for_model(resolved_model: str) -> Optional[str]:
"""The router connection a resolved model id will dispatch through, or None when the call does not go through the router at all (direct API keys own their own errors)."""
for prefix, provider in P_PREFIX_PROVIDER.items():
if resolved_model.startswith(prefix):
return provider
return None
@typechecked
def connection_is_dead(conn: Dict) -> bool:
"""A connection the router has given up on. Deliberately narrow: only the states that mean a dispatch is guaranteed to fail, never a slow or merely idle one."""
if conn.get("testStatus") == "unavailable":
return True
return conn.get("errorCode") in (401, 403)
async def dead_connection(provider: str) -> Optional[Dict]:
"""The provider's connection if the router considers it dead, else None. Never raises: a preflight that cannot read health must let the turn proceed, because guessing "dead" would ground a working lane."""
try:
from backend.apps.nine_router import get_providers
for conn in await get_providers():
if conn.get("provider") != provider:
continue
return conn if connection_is_dead(conn) else None
except Exception:
logger.debug("lane preflight could not read provider health; proceeding", exc_info=True)
return None
async def preflight_lane(resolved_model: str,
session: Optional["AgentSession"] = None) -> Optional[str]:
"""None when the turn should proceed, or the sentence to show the user when it should not.
Returning a message here is a decision NOT to spend the turn, which is only correct because the
alternative was measured: a guaranteed 401, misleading copy, and a wait the user has already
tried themselves.
"""
provider = provider_for_model(resolved_model)
if provider is None:
return None
dead = await dead_connection(provider)
if dead is None:
return None
# Whatever happens next, an auth failure on THIS turn is a dead credential, not a rotation
# window: the router had already given up before we sent anything.
if session is not None:
try:
session.lane_credential_dead = True
except Exception:
pass
now = time.time()
if now - LAST_BOUNCE.get(provider, 0.0) >= BOUNCE_COOLDOWN_S:
LAST_BOUNCE[provider] = now
logger.warning(
f"lane preflight: {provider} is {dead.get('testStatus')} (errorCode={dead.get('errorCode')}); "
"bouncing the router once, then letting the turn decide"
)
try:
from backend.apps.nine_router.bounce_after_connect import bounce_router_after_connect
await bounce_router_after_connect(provider)
except Exception:
logger.debug("lane preflight bounce failed", exc_info=True)
# Deliberately no post-bounce health re-read: see the module docstring. Dispatch is the test.
return None
return RECONNECT_COPY.get(
provider,
"This model's sign-in expired and could not be renewed. Reconnect it in Settings, then "
"Models. Waiting will not clear this one.",
)
+46 -8
View File
@@ -3,7 +3,7 @@ identical "hit a snag" clones (field screenshot 2026-08-19); a user message in b
earns a fresh card."""
from backend.apps.agents.core.models import AgentSession, Message
from backend.apps.agents.manager.run.handle_run_error import p_absorb_repeat_card
from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card
def p_card(text: str) -> Message:
@@ -13,30 +13,68 @@ def p_card(text: str) -> Message:
def test_identical_consecutive_card_is_absorbed():
s = AgentSession(name="t", model="sonnet")
first = p_card("That one failed.")
p_absorb_repeat_card(s, first)
absorb_repeat_card(s, first)
repeat = p_card("That one failed.")
p_absorb_repeat_card(s, repeat)
absorb_repeat_card(s, repeat)
assert len(s.messages) == 1
assert repeat.id == first.id, "the bump must reuse the id so the frontend updates in place"
def test_a_user_message_in_between_earns_a_fresh_card():
s = AgentSession(name="t", model="sonnet")
p_absorb_repeat_card(s, p_card("That one failed."))
absorb_repeat_card(s, p_card("That one failed."))
s.messages.append(Message(role="user", content="try again", branch_id="main"))
p_absorb_repeat_card(s, p_card("That one failed."))
absorb_repeat_card(s, p_card("That one failed."))
assert len([m for m in s.messages if m.role == "system"]) == 2
def test_different_error_text_always_appends():
s = AgentSession(name="t", model="sonnet")
p_absorb_repeat_card(s, p_card("Error A"))
p_absorb_repeat_card(s, p_card("Error B"))
absorb_repeat_card(s, p_card("Error A"))
absorb_repeat_card(s, p_card("Error B"))
assert len(s.messages) == 2
def test_other_branch_cards_do_not_mask():
s = AgentSession(name="t", model="sonnet")
s.messages.append(Message(role="system", content="Same text", branch_id="side"))
p_absorb_repeat_card(s, p_card("Same text"))
absorb_repeat_card(s, p_card("Same text"))
assert len(s.messages) == 2, "a card on another branch is invisible here and must not absorb"
def test_our_own_hidden_retries_do_not_earn_fresh_cards():
"""A live codex drill (2026-08-20) produced FIVE identical "still refreshing" cards on one ask.
Each self-heal retry sends a HIDDEN user-role continuation, which displaced the previous card
from the tail, so the dedup saw "a user message came in" and appended a clone. Our own
machinery was manufacturing the wall it was written to prevent."""
from backend.apps.agents.core.models import AgentSession, Message
from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card
s = AgentSession(name="t", model="gpt-5.6", dashboard_id="d")
s.messages.append(Message(role="user", content="do the thing", branch_id=s.active_branch_id))
card = "GPT subscription token is still refreshing."
for _ in range(5):
absorb_repeat_card(s, Message(role="system", content=card, branch_id=s.active_branch_id))
# what every self-heal retry does next
s.messages.append(Message(role="user", content="[Automated message] retry",
branch_id=s.active_branch_id, hidden=True))
shown = [m for m in s.messages if m.role == "system" and not m.hidden]
assert len(shown) == 1, f"one honest card per ask, got {len(shown)}"
def test_a_real_user_message_still_earns_a_fresh_card():
"""Negative control: the rule only ignores OUR sends. A human asking again deserves its own
answer, even if the answer is the same bad news."""
from backend.apps.agents.core.models import AgentSession, Message
from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card
s = AgentSession(name="t", model="gpt-5.6", dashboard_id="d")
card = "GPT subscription token is still refreshing."
absorb_repeat_card(s, Message(role="system", content=card, branch_id=s.active_branch_id))
s.messages.append(Message(role="user", content="try again please", branch_id=s.active_branch_id))
absorb_repeat_card(s, Message(role="system", content=card, branch_id=s.active_branch_id))
shown = [m for m in s.messages if m.role == "system"]
assert len(shown) == 2, "each real ask gets its own honest answer"
+126
View File
@@ -0,0 +1,126 @@
"""Never spend a turn on a lane the router has already given up on.
Measured cost of not doing this (live drill, 2026-08-20, real codex lane): a credential dead for 89
hours produced a "just rotated, every couple minutes, no action needed" card, a 75s wait, a doomed
retry and five identical follow-up cards, for zero files read. The router had published
testStatus="unavailable" and errorCode=401 the whole time.
"""
import asyncio
import pytest
import backend.apps.agents.manager.run.lane_preflight as lp
@pytest.fixture(autouse=True)
def p_clear_cooldown():
lp.LAST_BOUNCE.clear()
yield
lp.LAST_BOUNCE.clear()
def p_providers(monkeypatch, conns, bounce_result=None):
"""Stub the router's provider list; bounce_result, when given, is what the list becomes after a bounce."""
state = {"conns": conns, "bounced": 0}
async def fake_get_providers():
return state["conns"]
async def fake_bounce(provider):
state["bounced"] += 1
if bounce_result is not None:
state["conns"] = bounce_result
return True
import backend.apps.nine_router as nr
import backend.apps.nine_router.bounce_after_connect as ba
monkeypatch.setattr(nr, "get_providers", fake_get_providers, raising=True)
monkeypatch.setattr(ba, "bounce_router_after_connect", fake_bounce, raising=True)
return state
P_DEAD = [{"provider": "codex", "testStatus": "unavailable", "errorCode": 401}]
P_LIVE = [{"provider": "codex", "testStatus": "active", "errorCode": None}]
def test_a_healthy_lane_costs_nothing_and_says_nothing(monkeypatch):
st = p_providers(monkeypatch, P_LIVE)
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None
assert st["bounced"] == 0, "a working lane must never trigger a router restart"
def test_the_first_dead_encounter_bounces_and_lets_the_turn_decide(monkeypatch):
"""The bounce is an attempt, not a verdict. It must not block the turn, and it must not claim
a recovery it cannot see."""
st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD)
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None, "dispatch is the real test"
assert st["bounced"] == 1
def test_a_cleared_stamp_is_never_mistaken_for_a_working_credential(monkeypatch):
"""The bug this test exists for shipped for ten minutes on 2026-08-20. The first version
re-read health after the bounce and returned "recovered" because a fresh router has no
`unavailable` stamp yet. The credential was still dead and the turn 401'd seconds later.
A restart clears the accusation, not the cause."""
calls = {"health_reads": 0}
real = lp.dead_connection
async def counting(provider):
calls["health_reads"] += 1
return await real(provider)
monkeypatch.setattr(lp, "dead_connection", counting, raising=True)
p_providers(monkeypatch, P_DEAD, bounce_result=P_LIVE)
asyncio.run(lp.preflight_lane("cx/gpt-5.6"))
assert calls["health_reads"] == 1, (
"health is read once, BEFORE the bounce; a post-bounce read is the false-recovery bug"
)
def test_a_lane_still_dead_on_the_next_ask_gets_one_accurate_sentence(monkeypatch):
"""Second encounter inside the cooldown: we already spent a bounce and a turn, so stop
pretending and say the true thing."""
st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD)
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None
msg = asyncio.run(lp.preflight_lane("cx/gpt-5.6"))
assert msg and "ChatGPT" in msg and "Reconnect" in msg
assert "rotated" not in msg.lower(), "never claim a rotation that did not happen"
assert "no action needed" not in msg.lower(), "there IS action needed; saying otherwise is the bug"
assert st["bounced"] == 1, "the cooldown holds; one restart, not one per ask"
def test_the_bounce_is_rate_limited(monkeypatch):
"""A bounce restarts a process every other session shares, so it is once per lane per window, never once per turn."""
st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD)
for _ in range(4):
asyncio.run(lp.preflight_lane("cx/gpt-5.6"))
assert st["bounced"] == 1, f"expected a single bounce, got {st['bounced']}"
def test_direct_api_lanes_are_left_alone(monkeypatch):
"""Negative control: a direct API key never dispatches through the router, so the router's health says nothing about it and must not ground it."""
st = p_providers(monkeypatch, P_DEAD)
assert asyncio.run(lp.preflight_lane("claude-sonnet-4-6")) is None
assert st["bounced"] == 0
def test_unreadable_health_lets_the_turn_proceed(monkeypatch):
"""Negative control, and the important one: a preflight that cannot see must never guess 'dead'.
Grounding a working lane on a failed health read would be a worse bug than the one this fixes."""
async def boom():
raise RuntimeError("router unreachable")
import backend.apps.nine_router as nr
monkeypatch.setattr(nr, "get_providers", boom, raising=True)
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None
def test_only_terminal_states_count_as_dead():
assert lp.connection_is_dead({"testStatus": "unavailable"}) is True
assert lp.connection_is_dead({"errorCode": 401}) is True
assert lp.connection_is_dead({"errorCode": 403}) is True
# A slow, rate-limited or merely idle connection is NOT dead; grounding those would be the bug.
assert lp.connection_is_dead({"testStatus": "active", "errorCode": 429}) is False
assert lp.connection_is_dead({"testStatus": "active", "errorCode": 502}) is False
assert lp.connection_is_dead({}) is False