From 9a7570092f1e5210cd92f416b7febb38939af9ca Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 20 Aug 2026 09:20:46 -0700 Subject: [PATCH] [eric] agents: never spend a turn on a lane the router gave up on, and stop our own retries stacking cards Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_014wtspwSFzZmjCx9UNPAorQ --- backend/apps/agents/agent_manager.py | 20 +++ backend/apps/agents/core/models.py | 2 + .../agents/manager/run/handle_run_error.py | 52 +++++-- .../apps/agents/manager/run/lane_preflight.py | 135 ++++++++++++++++++ backend/tests/test_error_card_dedup.py | 54 +++++-- backend/tests/test_lane_preflight.py | 126 ++++++++++++++++ 6 files changed, 368 insertions(+), 21 deletions(-) create mode 100644 backend/apps/agents/manager/run/lane_preflight.py create mode 100644 backend/tests/test_lane_preflight.py diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index 3c2654db..131e2c33 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -207,6 +207,26 @@ class AgentManager(SessionLifecycle, SessionPersistence, Messaging, SessionContr p_router_model_id = p_resolve_model_id_early(session.model, load_settings()) p_api_type_for_session = p_get_api_type_early(session.model) + # Never spend a turn on a lane the router has already given up on: the live 2026-08-20 drill burned two minutes and six cards on a credential that had been dead for 89 hours. This heals it if it can, and tells the truth immediately if it cannot. + from backend.apps.agents.manager.run.lane_preflight import preflight_lane + p_lane_problem = await preflight_lane(p_router_model_id, session) + if p_lane_problem: + p_card = Message(role="system", content=p_lane_problem, branch_id=session.active_branch_id) + from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card + absorb_repeat_card(session, p_card) + session.status = "error" + await ws_manager.send_to_session(session_id, "agent:auth_error", { + "session_id": session_id, + "reason": "credential_expired", + "message": p_lane_problem, + "model": session.model, + }) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": p_card.model_dump(mode="json"), + }) + return + builtin_perms = load_builtin_permissions() # Builtins default to always_allow (frictionless); path_gate still force-prompts on catastrophic patterns (rm -rf), OS-scheduling, and sensitive paths, so poisoned-email -> destructive-command is still caught. Flip Bash to "ask" in the UI for a prompt on every command. Bind turn + stderr first: build_agent_options can raise early (no provider) and the except hands both to handle_run_error. diff --git a/backend/apps/agents/core/models.py b/backend/apps/agents/core/models.py index 0ad2ba6d..3907df9d 100644 --- a/backend/apps/agents/core/models.py +++ b/backend/apps/agents/core/models.py @@ -153,6 +153,8 @@ class AgentSession(BaseModel): # Consecutive dirty deaths this session was MID-TURN for; the crash auto-resume breaker (hermes #30719 pairing: auto-resume must never outrun its circuit breaker). crash_interrupt_count: int = 0 # Outage rounds spent on this ask: the in-turn ladder covers only 335s, and the work is checkpointed, so a longer drop is waited out rather than ending the task. + # True when the preflight found the router had already given up on this lane, so an auth failure this turn is a dead credential, not a rotation window worth waiting out. + lane_credential_dead: bool = False reconnect_attempts: int = 0 # True while a turn is parked waiting for the connection back; persisted so a quit DURING the wait is still an owed turn at next boot. awaiting_reconnect: bool = False diff --git a/backend/apps/agents/manager/run/handle_run_error.py b/backend/apps/agents/manager/run/handle_run_error.py index 852c128e..87c67a90 100644 --- a/backend/apps/agents/manager/run/handle_run_error.py +++ b/backend/apps/agents/manager/run/handle_run_error.py @@ -35,12 +35,19 @@ logger = logging.getLogger(__name__) @typechecked -def p_absorb_repeat_card(session: AgentSession, error_msg: Message) -> None: +def absorb_repeat_card(session: AgentSession, error_msg: Message) -> None: """Append the error card, unless the branch tail is already the IDENTICAL card with nothing after it: a retry ladder re-failing the same way then bumps the existing card instead of stacking a wall of clones (field screenshot 2026-08-19). A user message in between always - yields a fresh card, each ask deserves its own honest answer.""" - p_tail = [m for m in session.messages if getattr(m, "branch_id", None) in (None, session.active_branch_id)] + yields a fresh card, each ask deserves its own honest answer. + + HIDDEN messages do not count as that user message. Our own self-heal continuations are hidden + user-role sends, so counting them re-opened the exact clone wall this exists to stop: a live + codex drill on 2026-08-20 produced FIVE identical "still refreshing" cards, one per retry, + because each retry's hidden prompt had displaced the previous card from the tail.""" + p_tail = [m for m in session.messages + if getattr(m, "branch_id", None) in (None, session.active_branch_id) + and not getattr(m, "hidden", False)] if p_tail and p_tail[-1].role == "system" and p_tail[-1].content == error_msg.content: error_msg.id = p_tail[-1].id p_tail[-1].timestamp = error_msg.timestamp @@ -97,7 +104,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, "with a larger window." ) error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) - p_absorb_repeat_card(session, error_msg) + absorb_repeat_card(session, error_msg) p_ovf_payload = { "session_id": session_id, "reason": "long_context_required" if p_tier_gate else "context_overflow", @@ -144,7 +151,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, "api.anthropic.com; retrying won't help until one of those changes." ) error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) - p_absorb_repeat_card(session, error_msg) + absorb_repeat_card(session, error_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": error_msg.model_dump(mode="json"), @@ -160,7 +167,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, "are kept either way." ) error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) - p_absorb_repeat_card(session, error_msg) + absorb_repeat_card(session, error_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": error_msg.model_dump(mode="json"), @@ -216,7 +223,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, "OpenSwarm Pro." ) error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) - p_absorb_repeat_card(session, error_msg) + absorb_repeat_card(session, error_msg) await ws_manager.send_to_session(session_id, "agent:free_trial_exhausted", { "session_id": session_id, "message": friendly_msg, @@ -248,7 +255,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, "or starting a fresh chat about this topic, usually clears it." ) error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) - p_absorb_repeat_card(session, error_msg) + absorb_repeat_card(session, error_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": error_msg.model_dump(mode="json"), @@ -263,7 +270,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, "another option in Settings → Models." ) error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) - p_absorb_repeat_card(session, error_msg) + absorb_repeat_card(session, error_msg) await ws_manager.send_to_session(session_id, "agent:out_of_credits", { "session_id": session_id, "message": friendly_msg, @@ -283,6 +290,25 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or "401" in p_combined) ) # Every sub lane gets ONE silent self-heal before any card; only a missing credential (config problem, retry fails identically) goes straight to the card. Codex waits out its rotation window first. + # A lane the router had ALREADY given up on before this turn is a dead credential, so the + # rotation story is false and the wait is doomed; say the true thing straight away. + if getattr(session, "lane_credential_dead", False): + from backend.apps.agents.manager.run.lane_preflight import RECONNECT_COPY + p_prov = (session.provider or "").lower() + friendly_msg = RECONNECT_COPY.get( + p_prov, + "This model's sign-in expired and could not be renewed. Reconnect it in Settings, " + "then Models. Waiting will not clear this one.") + error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) + absorb_repeat_card(session, error_msg) + await ws_manager.send_to_session(session_id, "agent:auth_error", { + "session_id": session_id, "reason": "credential_expired", + "message": friendly_msg, "model": session.model, + }) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, "message": error_msg.model_dump(mode="json"), + }) + return if "no credentials for provider" not in p_combined: from backend.apps.agents.manager.streaming.auth_retry import try_auth_self_heal if try_auth_self_heal(session, delay_s=75 if p_codex_rotation else 5): @@ -341,7 +367,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, ) reason = "anthropic_auth_invalid" error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) - p_absorb_repeat_card(session, error_msg) + absorb_repeat_card(session, error_msg) try: from backend.apps.service.client import submit_diagnostic submit_diagnostic({ @@ -368,7 +394,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, # Upstream rejected the model code itself (e.g. Codex 1211 on a ChatGPT plan that lacks our GPT ids). Track it; the friendly "add an API key / pick another model" card is rendered frontend-side. p_report_model_error("unknown_model", session_id, session, turn, e, p_stderr_tail) error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id) - p_absorb_repeat_card(session, error_msg) + absorb_repeat_card(session, error_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": error_msg.model_dump(mode="json"), @@ -378,7 +404,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, # where the fix is entirely on our side of the wire. p_report_model_error("router_unavailable", session_id, session, turn, e, p_stderr_tail) error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id) - p_absorb_repeat_card(session, error_msg) + absorb_repeat_card(session, error_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": error_msg.model_dump(mode="json"), @@ -400,7 +426,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, if p_cause and "check stderr" in str(e).lower(): p_card_text += f"\n\nRuntime log tail:\n{p_cause}" error_msg = Message(role="system", content=p_card_text, branch_id=session.active_branch_id) - p_absorb_repeat_card(session, error_msg) + absorb_repeat_card(session, error_msg) await ws_manager.send_to_session(session_id, "agent:message", { "session_id": session_id, "message": error_msg.model_dump(mode="json"), diff --git a/backend/apps/agents/manager/run/lane_preflight.py b/backend/apps/agents/manager/run/lane_preflight.py new file mode 100644 index 00000000..4d4362da --- /dev/null +++ b/backend/apps/agents/manager/run/lane_preflight.py @@ -0,0 +1,135 @@ +"""Do not spend a user's turn on a lane we already know is dead. + +Live drill, 2026-08-20: a codex credential that expired 89 HOURS earlier produced "GPT subscription +token just rotated (automatic, every couple minutes), retrying automatically, no action needed", a +75 second wait, a doomed retry, and then five identical cards. Zero files read. Every word of that +was wrong, and the evidence to know better was already sitting in 9Router's own provider list: +`testStatus: "unavailable"` with `errorCode: 401`, published before we spend anything. + +So this looks first, and it tries to fix it before it complains: + + healthy -> say nothing, cost nothing, dispatch as normal + sticky-dead, first -> bounce the router ONCE (re-reads db.json, clears the in-process + `unavailable` stamp and modelLock cooldowns), then DISPATCH ANYWAY and let + the turn itself be the verdict. If it goes through, the user never learns + anything happened. + sticky-dead, again -> one accurate sentence, immediately. No invented rotation window, no wait + the user has already tried by hand, no promise of a self-heal that cannot + happen. + +The bounce is NOT allowed to declare success on its own, and that mistake is worth recording: the +first version re-read the health flag afterwards and called a cleared stamp a recovery. But a fresh +router starts with no stamp, so the check passed for a credential that was still dead, and the turn +hit the same 401 seconds later. A restart clears the accusation, not the cause. Only a real +dispatch can tell you whether a credential works, so that is what decides it now. + +The bounce is the same one ENG-315 already runs at the connect chokepoint, and is documented safe +mid-session; the in-flight kill drill on 2026-08-20 confirmed a live turn survives one. +""" + +import logging +import time +from typing import Dict, Optional, TYPE_CHECKING + +from typeguard import typechecked + +if TYPE_CHECKING: + from backend.apps.agents.core.models import AgentSession + +logger = logging.getLogger(__name__) + +# Router prefix -> the provider name its connection is filed under. +P_PREFIX_PROVIDER = {"cc/": "claude", "cx/": "codex", "gc/": "antigravity", "ag/": "antigravity"} + +# A bounce restarts a process every other session shares, so one per lane per window, never per turn. +BOUNCE_COOLDOWN_S = 300 + +LAST_BOUNCE: Dict[str, float] = {} + +RECONNECT_COPY = { + "codex": ("Your ChatGPT subscription needs reconnecting: the saved sign-in expired and could " + "not be renewed. Open Settings, then Models, and click Reconnect on the OpenAI / GPT " + "row. Waiting will not clear this one."), + "claude": ("Your Claude subscription needs reconnecting: the saved sign-in expired and could " + "not be renewed. Open Settings, then Models, and click Reconnect on the Claude " + "Pro / Max row. Waiting will not clear this one."), + "antigravity": ("Your Google sign-in needs reconnecting: the saved credential expired and " + "could not be renewed. Open Settings, then Models, and reconnect the Google " + "row. Waiting will not clear this one."), +} + + +@typechecked +def provider_for_model(resolved_model: str) -> Optional[str]: + """The router connection a resolved model id will dispatch through, or None when the call does not go through the router at all (direct API keys own their own errors).""" + for prefix, provider in P_PREFIX_PROVIDER.items(): + if resolved_model.startswith(prefix): + return provider + return None + + +@typechecked +def connection_is_dead(conn: Dict) -> bool: + """A connection the router has given up on. Deliberately narrow: only the states that mean a dispatch is guaranteed to fail, never a slow or merely idle one.""" + if conn.get("testStatus") == "unavailable": + return True + return conn.get("errorCode") in (401, 403) + + +async def dead_connection(provider: str) -> Optional[Dict]: + """The provider's connection if the router considers it dead, else None. Never raises: a preflight that cannot read health must let the turn proceed, because guessing "dead" would ground a working lane.""" + try: + from backend.apps.nine_router import get_providers + for conn in await get_providers(): + if conn.get("provider") != provider: + continue + return conn if connection_is_dead(conn) else None + except Exception: + logger.debug("lane preflight could not read provider health; proceeding", exc_info=True) + return None + + +async def preflight_lane(resolved_model: str, + session: Optional["AgentSession"] = None) -> Optional[str]: + """None when the turn should proceed, or the sentence to show the user when it should not. + + Returning a message here is a decision NOT to spend the turn, which is only correct because the + alternative was measured: a guaranteed 401, misleading copy, and a wait the user has already + tried themselves. + """ + provider = provider_for_model(resolved_model) + if provider is None: + return None + + dead = await dead_connection(provider) + if dead is None: + return None + + # Whatever happens next, an auth failure on THIS turn is a dead credential, not a rotation + # window: the router had already given up before we sent anything. + if session is not None: + try: + session.lane_credential_dead = True + except Exception: + pass + + now = time.time() + if now - LAST_BOUNCE.get(provider, 0.0) >= BOUNCE_COOLDOWN_S: + LAST_BOUNCE[provider] = now + logger.warning( + f"lane preflight: {provider} is {dead.get('testStatus')} (errorCode={dead.get('errorCode')}); " + "bouncing the router once, then letting the turn decide" + ) + try: + from backend.apps.nine_router.bounce_after_connect import bounce_router_after_connect + await bounce_router_after_connect(provider) + except Exception: + logger.debug("lane preflight bounce failed", exc_info=True) + # Deliberately no post-bounce health re-read: see the module docstring. Dispatch is the test. + return None + + return RECONNECT_COPY.get( + provider, + "This model's sign-in expired and could not be renewed. Reconnect it in Settings, then " + "Models. Waiting will not clear this one.", + ) diff --git a/backend/tests/test_error_card_dedup.py b/backend/tests/test_error_card_dedup.py index 00715537..e964af6a 100644 --- a/backend/tests/test_error_card_dedup.py +++ b/backend/tests/test_error_card_dedup.py @@ -3,7 +3,7 @@ identical "hit a snag" clones (field screenshot 2026-08-19); a user message in b earns a fresh card.""" from backend.apps.agents.core.models import AgentSession, Message -from backend.apps.agents.manager.run.handle_run_error import p_absorb_repeat_card +from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card def p_card(text: str) -> Message: @@ -13,30 +13,68 @@ def p_card(text: str) -> Message: def test_identical_consecutive_card_is_absorbed(): s = AgentSession(name="t", model="sonnet") first = p_card("That one failed.") - p_absorb_repeat_card(s, first) + absorb_repeat_card(s, first) repeat = p_card("That one failed.") - p_absorb_repeat_card(s, repeat) + absorb_repeat_card(s, repeat) assert len(s.messages) == 1 assert repeat.id == first.id, "the bump must reuse the id so the frontend updates in place" def test_a_user_message_in_between_earns_a_fresh_card(): s = AgentSession(name="t", model="sonnet") - p_absorb_repeat_card(s, p_card("That one failed.")) + absorb_repeat_card(s, p_card("That one failed.")) s.messages.append(Message(role="user", content="try again", branch_id="main")) - p_absorb_repeat_card(s, p_card("That one failed.")) + absorb_repeat_card(s, p_card("That one failed.")) assert len([m for m in s.messages if m.role == "system"]) == 2 def test_different_error_text_always_appends(): s = AgentSession(name="t", model="sonnet") - p_absorb_repeat_card(s, p_card("Error A")) - p_absorb_repeat_card(s, p_card("Error B")) + absorb_repeat_card(s, p_card("Error A")) + absorb_repeat_card(s, p_card("Error B")) assert len(s.messages) == 2 def test_other_branch_cards_do_not_mask(): s = AgentSession(name="t", model="sonnet") s.messages.append(Message(role="system", content="Same text", branch_id="side")) - p_absorb_repeat_card(s, p_card("Same text")) + absorb_repeat_card(s, p_card("Same text")) assert len(s.messages) == 2, "a card on another branch is invisible here and must not absorb" + + +def test_our_own_hidden_retries_do_not_earn_fresh_cards(): + """A live codex drill (2026-08-20) produced FIVE identical "still refreshing" cards on one ask. + Each self-heal retry sends a HIDDEN user-role continuation, which displaced the previous card + from the tail, so the dedup saw "a user message came in" and appended a clone. Our own + machinery was manufacturing the wall it was written to prevent.""" + from backend.apps.agents.core.models import AgentSession, Message + from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card + + s = AgentSession(name="t", model="gpt-5.6", dashboard_id="d") + s.messages.append(Message(role="user", content="do the thing", branch_id=s.active_branch_id)) + card = "GPT subscription token is still refreshing." + + for _ in range(5): + absorb_repeat_card(s, Message(role="system", content=card, branch_id=s.active_branch_id)) + # what every self-heal retry does next + s.messages.append(Message(role="user", content="[Automated message] retry", + branch_id=s.active_branch_id, hidden=True)) + + shown = [m for m in s.messages if m.role == "system" and not m.hidden] + assert len(shown) == 1, f"one honest card per ask, got {len(shown)}" + + +def test_a_real_user_message_still_earns_a_fresh_card(): + """Negative control: the rule only ignores OUR sends. A human asking again deserves its own + answer, even if the answer is the same bad news.""" + from backend.apps.agents.core.models import AgentSession, Message + from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card + + s = AgentSession(name="t", model="gpt-5.6", dashboard_id="d") + card = "GPT subscription token is still refreshing." + absorb_repeat_card(s, Message(role="system", content=card, branch_id=s.active_branch_id)) + s.messages.append(Message(role="user", content="try again please", branch_id=s.active_branch_id)) + absorb_repeat_card(s, Message(role="system", content=card, branch_id=s.active_branch_id)) + + shown = [m for m in s.messages if m.role == "system"] + assert len(shown) == 2, "each real ask gets its own honest answer" diff --git a/backend/tests/test_lane_preflight.py b/backend/tests/test_lane_preflight.py new file mode 100644 index 00000000..7c09a896 --- /dev/null +++ b/backend/tests/test_lane_preflight.py @@ -0,0 +1,126 @@ +"""Never spend a turn on a lane the router has already given up on. + +Measured cost of not doing this (live drill, 2026-08-20, real codex lane): a credential dead for 89 +hours produced a "just rotated, every couple minutes, no action needed" card, a 75s wait, a doomed +retry and five identical follow-up cards, for zero files read. The router had published +testStatus="unavailable" and errorCode=401 the whole time. +""" + +import asyncio + +import pytest + +import backend.apps.agents.manager.run.lane_preflight as lp + + +@pytest.fixture(autouse=True) +def p_clear_cooldown(): + lp.LAST_BOUNCE.clear() + yield + lp.LAST_BOUNCE.clear() + + +def p_providers(monkeypatch, conns, bounce_result=None): + """Stub the router's provider list; bounce_result, when given, is what the list becomes after a bounce.""" + state = {"conns": conns, "bounced": 0} + + async def fake_get_providers(): + return state["conns"] + + async def fake_bounce(provider): + state["bounced"] += 1 + if bounce_result is not None: + state["conns"] = bounce_result + return True + + import backend.apps.nine_router as nr + import backend.apps.nine_router.bounce_after_connect as ba + monkeypatch.setattr(nr, "get_providers", fake_get_providers, raising=True) + monkeypatch.setattr(ba, "bounce_router_after_connect", fake_bounce, raising=True) + return state + + +P_DEAD = [{"provider": "codex", "testStatus": "unavailable", "errorCode": 401}] +P_LIVE = [{"provider": "codex", "testStatus": "active", "errorCode": None}] + + +def test_a_healthy_lane_costs_nothing_and_says_nothing(monkeypatch): + st = p_providers(monkeypatch, P_LIVE) + assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None + assert st["bounced"] == 0, "a working lane must never trigger a router restart" + + +def test_the_first_dead_encounter_bounces_and_lets_the_turn_decide(monkeypatch): + """The bounce is an attempt, not a verdict. It must not block the turn, and it must not claim + a recovery it cannot see.""" + st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD) + assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None, "dispatch is the real test" + assert st["bounced"] == 1 + + +def test_a_cleared_stamp_is_never_mistaken_for_a_working_credential(monkeypatch): + """The bug this test exists for shipped for ten minutes on 2026-08-20. The first version + re-read health after the bounce and returned "recovered" because a fresh router has no + `unavailable` stamp yet. The credential was still dead and the turn 401'd seconds later. + A restart clears the accusation, not the cause.""" + calls = {"health_reads": 0} + real = lp.dead_connection + + async def counting(provider): + calls["health_reads"] += 1 + return await real(provider) + + monkeypatch.setattr(lp, "dead_connection", counting, raising=True) + p_providers(monkeypatch, P_DEAD, bounce_result=P_LIVE) + asyncio.run(lp.preflight_lane("cx/gpt-5.6")) + assert calls["health_reads"] == 1, ( + "health is read once, BEFORE the bounce; a post-bounce read is the false-recovery bug" + ) + + +def test_a_lane_still_dead_on_the_next_ask_gets_one_accurate_sentence(monkeypatch): + """Second encounter inside the cooldown: we already spent a bounce and a turn, so stop + pretending and say the true thing.""" + st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD) + assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None + msg = asyncio.run(lp.preflight_lane("cx/gpt-5.6")) + assert msg and "ChatGPT" in msg and "Reconnect" in msg + assert "rotated" not in msg.lower(), "never claim a rotation that did not happen" + assert "no action needed" not in msg.lower(), "there IS action needed; saying otherwise is the bug" + assert st["bounced"] == 1, "the cooldown holds; one restart, not one per ask" + + +def test_the_bounce_is_rate_limited(monkeypatch): + """A bounce restarts a process every other session shares, so it is once per lane per window, never once per turn.""" + st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD) + for _ in range(4): + asyncio.run(lp.preflight_lane("cx/gpt-5.6")) + assert st["bounced"] == 1, f"expected a single bounce, got {st['bounced']}" + + +def test_direct_api_lanes_are_left_alone(monkeypatch): + """Negative control: a direct API key never dispatches through the router, so the router's health says nothing about it and must not ground it.""" + st = p_providers(monkeypatch, P_DEAD) + assert asyncio.run(lp.preflight_lane("claude-sonnet-4-6")) is None + assert st["bounced"] == 0 + + +def test_unreadable_health_lets_the_turn_proceed(monkeypatch): + """Negative control, and the important one: a preflight that cannot see must never guess 'dead'. + Grounding a working lane on a failed health read would be a worse bug than the one this fixes.""" + async def boom(): + raise RuntimeError("router unreachable") + + import backend.apps.nine_router as nr + monkeypatch.setattr(nr, "get_providers", boom, raising=True) + assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None + + +def test_only_terminal_states_count_as_dead(): + assert lp.connection_is_dead({"testStatus": "unavailable"}) is True + assert lp.connection_is_dead({"errorCode": 401}) is True + assert lp.connection_is_dead({"errorCode": 403}) is True + # A slow, rate-limited or merely idle connection is NOT dead; grounding those would be the bug. + assert lp.connection_is_dead({"testStatus": "active", "errorCode": 429}) is False + assert lp.connection_is_dead({"testStatus": "active", "errorCode": 502}) is False + assert lp.connection_is_dead({}) is False