mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-14 05:37:40 +02:00
151 lines
7.4 KiB
Python
151 lines
7.4 KiB
Python
"""Never spend a turn on a lane the router has already given up on.
|
|
|
|
Measured cost of not doing this (live drill, 2026-08-20, real codex lane): a credential dead for 89
|
|
hours produced a "just rotated, every couple minutes, no action needed" card, a 75s wait, a doomed
|
|
retry and five identical follow-up cards, for zero files read. The router had published
|
|
testStatus="unavailable" and errorCode=401 the whole time.
|
|
"""
|
|
|
|
import asyncio
|
|
|
|
|
|
import backend.apps.agents.manager.run.lane_preflight as lp
|
|
|
|
|
|
|
|
|
|
def p_providers(monkeypatch, conns, bounce_result=None):
|
|
"""Stub the router's provider list; bounce_result, when given, is what the list becomes after a bounce."""
|
|
state = {"conns": conns, "bounced": 0}
|
|
|
|
async def fake_get_providers():
|
|
return state["conns"]
|
|
|
|
async def fake_bounce(provider):
|
|
state["bounced"] += 1
|
|
if bounce_result is not None:
|
|
state["conns"] = bounce_result
|
|
return True
|
|
|
|
import backend.apps.nine_router as nr
|
|
import backend.apps.nine_router.bounce_after_connect as ba
|
|
monkeypatch.setattr(nr, "get_providers", fake_get_providers, raising=True)
|
|
monkeypatch.setattr(ba, "bounce_router_after_connect", fake_bounce, raising=True)
|
|
return state
|
|
|
|
|
|
P_DEAD = [{"provider": "codex", "testStatus": "unavailable", "errorCode": 401}]
|
|
P_LIVE = [{"provider": "codex", "testStatus": "active", "errorCode": None}]
|
|
|
|
|
|
def test_a_healthy_lane_costs_nothing_and_says_nothing(monkeypatch):
|
|
st = p_providers(monkeypatch, P_LIVE)
|
|
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None
|
|
assert st["bounced"] == 0, "a working lane must never trigger a router restart"
|
|
|
|
|
|
def test_a_cleared_stamp_is_never_mistaken_for_a_working_credential(monkeypatch):
|
|
"""The bug this test exists for shipped for ten minutes on 2026-08-20. The first version
|
|
re-read health after the bounce and returned "recovered" because a fresh router has no
|
|
`unavailable` stamp yet. The credential was still dead and the turn 401'd seconds later.
|
|
A restart clears the accusation, not the cause."""
|
|
calls = {"health_reads": 0}
|
|
real = lp.dead_connection
|
|
|
|
async def counting(provider):
|
|
calls["health_reads"] += 1
|
|
return await real(provider)
|
|
|
|
monkeypatch.setattr(lp, "dead_connection", counting, raising=True)
|
|
p_providers(monkeypatch, P_DEAD, bounce_result=P_LIVE)
|
|
asyncio.run(lp.preflight_lane("cx/gpt-5.6"))
|
|
assert calls["health_reads"] == 1, (
|
|
"health is read once, BEFORE the bounce; a post-bounce read is the false-recovery bug"
|
|
)
|
|
|
|
|
|
def test_the_downstream_card_still_refuses_the_rotation_story():
|
|
"""What the old assertion above was really protecting: when the card DOES fire, it must not
|
|
invent a rotation window or claim no action is needed. That copy moved, it did not soften."""
|
|
for msg in lp.RECONNECT_COPY.values():
|
|
assert "rotated" not in msg.lower(), "never claim a rotation that did not happen"
|
|
assert "no action needed" not in msg.lower(), "there IS action needed; saying otherwise is the bug"
|
|
assert "reconnect" in msg.lower(), msg
|
|
|
|
|
|
def test_direct_api_lanes_are_left_alone(monkeypatch):
|
|
"""Negative control: a direct API key never dispatches through the router, so the router's health says nothing about it and must not ground it."""
|
|
st = p_providers(monkeypatch, P_DEAD)
|
|
assert asyncio.run(lp.preflight_lane("claude-sonnet-4-6")) is None
|
|
assert st["bounced"] == 0
|
|
|
|
|
|
def test_unreadable_health_lets_the_turn_proceed(monkeypatch):
|
|
"""Negative control, and the important one: a preflight that cannot see must never guess 'dead'.
|
|
Grounding a working lane on a failed health read would be a worse bug than the one this fixes."""
|
|
async def boom():
|
|
raise RuntimeError("router unreachable")
|
|
|
|
import backend.apps.nine_router as nr
|
|
monkeypatch.setattr(nr, "get_providers", boom, raising=True)
|
|
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None
|
|
|
|
|
|
def test_only_terminal_states_count_as_dead():
|
|
# "unavailable" alone is NOT enough: the router stamps it for throttles and 5xx as well, so it
|
|
# cannot distinguish a dead credential from a bad minute (corrected after a live false positive).
|
|
assert lp.connection_is_dead({"testStatus": "unavailable"}) is False
|
|
assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 401}) is True
|
|
assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 403}) is True
|
|
# A slow, rate-limited or merely idle connection is NOT dead; grounding those would be the bug.
|
|
assert lp.connection_is_dead({"testStatus": "active", "errorCode": 429}) is False
|
|
assert lp.connection_is_dead({"testStatus": "active", "errorCode": 502}) is False
|
|
assert lp.connection_is_dead({}) is False
|
|
|
|
|
|
def test_a_rate_limited_lane_is_not_a_dead_credential():
|
|
"""Live 2026-08-20: antigravity sat at testStatus=unavailable with errorCode=429 and a
|
|
credential valid for another 30 minutes. Telling that user to reconnect is the same lie as
|
|
"just rotated" for a dead token, aimed the other way."""
|
|
assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 429}) is False
|
|
assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 503}) is False
|
|
assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": None}) is False
|
|
|
|
|
|
def test_only_auth_shaped_failures_send_the_user_to_settings():
|
|
assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 401}) is True
|
|
assert lp.connection_is_dead({"testStatus": "unavailable", "errorCode": 403}) is True
|
|
# The auth code has to be the router's CURRENT verdict: a row it calls active is serving traffic (2026-09-05, Eric's claude row).
|
|
assert lp.connection_is_dead({"testStatus": "active", "errorCode": 403}) is False
|
|
|
|
|
|
# The router's own row for a WORKING claude login (2026-09-05, db.json): the 401 is stale, the status is current.
|
|
P_STALE_401_ACTIVE = [{"provider": "claude", "testStatus": "active", "errorCode": 401, "lastError": None, "lastErrorAt": None, "backoffLevel": 0}]
|
|
|
|
|
|
def test_an_active_row_with_a_stale_401_is_not_dead_and_bounces_nothing(monkeypatch):
|
|
st = p_providers(monkeypatch, P_STALE_401_ACTIVE)
|
|
assert lp.connection_is_dead(P_STALE_401_ACTIVE[0]) is False
|
|
assert asyncio.run(lp.preflight_lane("cc/claude-sonnet-5")) is None
|
|
assert st["bounced"] == 0, "a lane the router calls active must never trigger a router restart"
|
|
|
|
|
|
def test_a_throttle_is_still_not_death():
|
|
# testStatus unavailable with a 429 is the Google case the docstring cites; the tightened rule keeps it.
|
|
assert lp.connection_is_dead({"provider": "gemini-cli", "testStatus": "unavailable", "errorCode": 429}) is False
|
|
assert lp.connection_is_dead({"provider": "codex", "testStatus": "unavailable", "errorCode": 401}) is True
|
|
|
|
|
|
def test_a_dead_lane_never_restarts_the_router(monkeypatch):
|
|
"""2026-09-06: the preflight bounce was a dead port for every chat on every lane, and a restart cannot revive
|
|
a dead token. A dead lane dispatches (the real request is the test) and restarts nothing."""
|
|
st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD)
|
|
for _ in range(4):
|
|
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None
|
|
assert st["bounced"] == 0, f"preflight must never restart the router, got {st['bounced']}"
|
|
|
|
|
|
def test_the_module_no_longer_owns_a_bounce_throttle():
|
|
assert not hasattr(lp, "LAST_BOUNCE") and not hasattr(lp, "BOUNCE_COOLDOWN_S")
|
|
|