"""Never spend a turn on a lane the router has already given up on. Measured cost of not doing this (live drill, 2026-08-20, real codex lane): a credential dead for 89 hours produced a "just rotated, every couple minutes, no action needed" card, a 75s wait, a doomed retry and five identical follow-up cards, for zero files read. The router had published testStatus="unavailable" and errorCode=401 the whole time. """ import asyncio import backend.apps.agents.manager.run.lane_preflight as lp def p_providers(monkeypatch, conns, bounce_result=None): """Stub the router's provider list; bounce_result, when given, is what the list becomes after a bounce.""" state = {"conns": conns, "bounced": 0} async def fake_get_providers(): return state["conns"] async def fake_bounce(provider): state["bounced"] += 1 if bounce_result is not None: state["conns"] = bounce_result return True import backend.apps.nine_router as nr import backend.apps.nine_router.bounce_after_connect as ba monkeypatch.setattr(nr, "get_providers", fake_get_providers, raising=True) monkeypatch.setattr(ba, "bounce_router_after_connect", fake_bounce, raising=True) return state P_DEAD = [{"provider": "codex", "testStatus": "unavailable", "errorCode": 401}] P_LIVE = [{"provider": "codex", "testStatus": "active", "errorCode": None}] def test_a_healthy_lane_costs_nothing_and_says_nothing(monkeypatch): st = p_providers(monkeypatch, P_LIVE) assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None assert st["bounced"] == 0, "a working lane must never trigger a router restart" def test_a_cleared_stamp_is_never_mistaken_for_a_working_credential(monkeypatch): """The bug this test exists for shipped for ten minutes on 2026-08-20. The first version re-read health after the bounce and returned "recovered" because a fresh router has no `unavailable` stamp yet. The credential was still dead and the turn 401'd seconds later. A restart clears the accusation, not the cause.""" calls = {"health_reads": 0} real = lp.dead_connection async def counting(provider): calls["health_reads"] += 1 return await real(provider) monkeypatch.setattr(lp, "dead_connection", counting, raising=True) p_providers(monkeypatch, P_DEAD, bounce_result=P_LIVE) asyncio.run(lp.preflight_lane("cx/gpt-5.6")) assert calls["health_reads"] == 1, ( "health is read once, BEFORE the bounce; a post-bounce read is the false-recovery bug" ) def test_the_downstream_card_still_refuses_the_rotation_story(): """What the old assertion above was really protecting: when the card DOES fire, it must not invent a rotation window or claim no action is needed. That copy moved, it did not soften.""" for msg in lp.RECONNECT_COPY.values(): assert "rotated" not in msg.lower(), "never claim a rotation that did not happen" assert "no action needed" not in msg.lower(), "there IS action needed; saying otherwise is the bug" assert "reconnect" in msg.lower(), msg def test_direct_api_lanes_are_left_alone(monkeypatch): """Negative control: a direct API key never dispatches through the router, so the router's health says nothing about it and must not ground it.""" st = p_providers(monkeypatch, P_DEAD) assert asyncio.run(lp.preflight_lane("claude-sonnet-4-6")) is None assert st["bounced"] == 0 def test_unreadable_health_lets_the_turn_proceed(monkeypatch): """Negative control, and the important one: a preflight that cannot see must never guess 'dead'. Grounding a working lane on a failed health read would be a worse bug than the one this fixes.""" async def boom(): raise RuntimeError("router unreachable") import backend.apps.nine_router as nr monkeypatch.setattr(nr, "get_providers", boom, raising=True) assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None def test_only_terminal_states_count_as_dead(): # "unavailable" alone is NOT enough: the router stamps it for throttles and 5xx as well, so it # cannot distinguish a dead credential from a bad minute (corrected after a live false positive). assert lp.connection_is_dead({"testStatus": "unavailable"}) is False assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 401}) is True assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 403}) is True # A slow, rate-limited or merely idle connection is NOT dead; grounding those would be the bug. assert lp.connection_is_dead({"testStatus": "active", "errorCode": 429}) is False assert lp.connection_is_dead({"testStatus": "active", "errorCode": 502}) is False assert lp.connection_is_dead({}) is False def test_a_rate_limited_lane_is_not_a_dead_credential(): """Live 2026-08-20: antigravity sat at testStatus=unavailable with errorCode=429 and a credential valid for another 30 minutes. Telling that user to reconnect is the same lie as "just rotated" for a dead token, aimed the other way.""" assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 429}) is False assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 503}) is False assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": None}) is False def test_only_auth_shaped_failures_send_the_user_to_settings(): assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 401}) is True assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 403}) is True # The auth code has to be the router's CURRENT verdict: a row it calls active is serving traffic (2026-09-05, Eric's claude row). assert lp.connection_is_dead({"testStatus": "active", "errorCode": 403}) is False # The router's own row for a WORKING claude login (2026-09-05, db.json): the 401 is stale, the status is current. P_STALE_401_ACTIVE = [{"provider": "claude", "testStatus": "active", "errorCode": 401, "lastError": None, "lastErrorAt": None, "backoffLevel": 0}] def test_an_active_row_with_a_stale_401_is_not_dead_and_bounces_nothing(monkeypatch): st = p_providers(monkeypatch, P_STALE_401_ACTIVE) assert lp.connection_is_dead(P_STALE_401_ACTIVE[0]) is False assert asyncio.run(lp.preflight_lane("cc/claude-sonnet-5")) is None assert st["bounced"] == 0, "a lane the router calls active must never trigger a router restart" def test_a_throttle_is_still_not_death(): # testStatus unavailable with a 429 is the Google case the docstring cites; the tightened rule keeps it. assert lp.connection_is_dead({"provider": "gemini-cli", "testStatus": "unavailable", "errorCode": 429}) is False assert lp.connection_is_dead({"provider": "codex", "testStatus": "unavailable", "errorCode": 401}) is True def test_a_dead_lane_never_restarts_the_router(monkeypatch): """2026-09-06: the preflight bounce was a dead port for every chat on every lane, and a restart cannot revive a dead token. A dead lane dispatches (the real request is the test) and restarts nothing.""" st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD) for _ in range(4): assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None assert st["bounced"] == 0, f"preflight must never restart the router, got {st['bounced']}" def test_the_module_no_longer_owns_a_bounce_throttle(): assert not hasattr(lp, "LAST_BOUNCE") and not hasattr(lp, "BOUNCE_COOLDOWN_S")