mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-11 12:17:45 +02:00
[eric] agents: never spend a turn on a lane the router gave up on, and stop our own retries stacking cards
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014wtspwSFzZmjCx9UNPAorQ
This commit is contained in:
co-authored by
Claude Opus 5
parent
58e376496e
commit
9a7570092f
@@ -207,6 +207,26 @@ class AgentManager(SessionLifecycle, SessionPersistence, Messaging, SessionContr
|
||||
p_router_model_id = p_resolve_model_id_early(session.model, load_settings())
|
||||
p_api_type_for_session = p_get_api_type_early(session.model)
|
||||
|
||||
# Never spend a turn on a lane the router has already given up on: the live 2026-08-20 drill burned two minutes and six cards on a credential that had been dead for 89 hours. This heals it if it can, and tells the truth immediately if it cannot.
|
||||
from backend.apps.agents.manager.run.lane_preflight import preflight_lane
|
||||
p_lane_problem = await preflight_lane(p_router_model_id, session)
|
||||
if p_lane_problem:
|
||||
p_card = Message(role="system", content=p_lane_problem, branch_id=session.active_branch_id)
|
||||
from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card
|
||||
absorb_repeat_card(session, p_card)
|
||||
session.status = "error"
|
||||
await ws_manager.send_to_session(session_id, "agent:auth_error", {
|
||||
"session_id": session_id,
|
||||
"reason": "credential_expired",
|
||||
"message": p_lane_problem,
|
||||
"model": session.model,
|
||||
})
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": p_card.model_dump(mode="json"),
|
||||
})
|
||||
return
|
||||
|
||||
builtin_perms = load_builtin_permissions()
|
||||
|
||||
# Builtins default to always_allow (frictionless); path_gate still force-prompts on catastrophic patterns (rm -rf), OS-scheduling, and sensitive paths, so poisoned-email -> destructive-command is still caught. Flip Bash to "ask" in the UI for a prompt on every command. Bind turn + stderr first: build_agent_options can raise early (no provider) and the except hands both to handle_run_error.
|
||||
|
||||
@@ -153,6 +153,8 @@ class AgentSession(BaseModel):
|
||||
# Consecutive dirty deaths this session was MID-TURN for; the crash auto-resume breaker (hermes #30719 pairing: auto-resume must never outrun its circuit breaker).
|
||||
crash_interrupt_count: int = 0
|
||||
# Outage rounds spent on this ask: the in-turn ladder covers only 335s, and the work is checkpointed, so a longer drop is waited out rather than ending the task.
|
||||
# True when the preflight found the router had already given up on this lane, so an auth failure this turn is a dead credential, not a rotation window worth waiting out.
|
||||
lane_credential_dead: bool = False
|
||||
reconnect_attempts: int = 0
|
||||
# True while a turn is parked waiting for the connection back; persisted so a quit DURING the wait is still an owed turn at next boot.
|
||||
awaiting_reconnect: bool = False
|
||||
|
||||
@@ -35,12 +35,19 @@ logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_absorb_repeat_card(session: AgentSession, error_msg: Message) -> None:
|
||||
def absorb_repeat_card(session: AgentSession, error_msg: Message) -> None:
|
||||
"""Append the error card, unless the branch tail is already the IDENTICAL card with nothing
|
||||
after it: a retry ladder re-failing the same way then bumps the existing card instead of
|
||||
stacking a wall of clones (field screenshot 2026-08-19). A user message in between always
|
||||
yields a fresh card, each ask deserves its own honest answer."""
|
||||
p_tail = [m for m in session.messages if getattr(m, "branch_id", None) in (None, session.active_branch_id)]
|
||||
yields a fresh card, each ask deserves its own honest answer.
|
||||
|
||||
HIDDEN messages do not count as that user message. Our own self-heal continuations are hidden
|
||||
user-role sends, so counting them re-opened the exact clone wall this exists to stop: a live
|
||||
codex drill on 2026-08-20 produced FIVE identical "still refreshing" cards, one per retry,
|
||||
because each retry's hidden prompt had displaced the previous card from the tail."""
|
||||
p_tail = [m for m in session.messages
|
||||
if getattr(m, "branch_id", None) in (None, session.active_branch_id)
|
||||
and not getattr(m, "hidden", False)]
|
||||
if p_tail and p_tail[-1].role == "system" and p_tail[-1].content == error_msg.content:
|
||||
error_msg.id = p_tail[-1].id
|
||||
p_tail[-1].timestamp = error_msg.timestamp
|
||||
@@ -97,7 +104,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
"with a larger window."
|
||||
)
|
||||
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
||||
p_absorb_repeat_card(session, error_msg)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
p_ovf_payload = {
|
||||
"session_id": session_id,
|
||||
"reason": "long_context_required" if p_tier_gate else "context_overflow",
|
||||
@@ -144,7 +151,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
"api.anthropic.com; retrying won't help until one of those changes."
|
||||
)
|
||||
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
||||
p_absorb_repeat_card(session, error_msg)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": error_msg.model_dump(mode="json"),
|
||||
@@ -160,7 +167,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
"are kept either way."
|
||||
)
|
||||
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
||||
p_absorb_repeat_card(session, error_msg)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": error_msg.model_dump(mode="json"),
|
||||
@@ -216,7 +223,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
"OpenSwarm Pro."
|
||||
)
|
||||
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
||||
p_absorb_repeat_card(session, error_msg)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:free_trial_exhausted", {
|
||||
"session_id": session_id,
|
||||
"message": friendly_msg,
|
||||
@@ -248,7 +255,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
"or starting a fresh chat about this topic, usually clears it."
|
||||
)
|
||||
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
||||
p_absorb_repeat_card(session, error_msg)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": error_msg.model_dump(mode="json"),
|
||||
@@ -263,7 +270,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
"another option in Settings → Models."
|
||||
)
|
||||
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
||||
p_absorb_repeat_card(session, error_msg)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:out_of_credits", {
|
||||
"session_id": session_id,
|
||||
"message": friendly_msg,
|
||||
@@ -283,6 +290,25 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or "401" in p_combined)
|
||||
)
|
||||
# Every sub lane gets ONE silent self-heal before any card; only a missing credential (config problem, retry fails identically) goes straight to the card. Codex waits out its rotation window first.
|
||||
# A lane the router had ALREADY given up on before this turn is a dead credential, so the
|
||||
# rotation story is false and the wait is doomed; say the true thing straight away.
|
||||
if getattr(session, "lane_credential_dead", False):
|
||||
from backend.apps.agents.manager.run.lane_preflight import RECONNECT_COPY
|
||||
p_prov = (session.provider or "").lower()
|
||||
friendly_msg = RECONNECT_COPY.get(
|
||||
p_prov,
|
||||
"This model's sign-in expired and could not be renewed. Reconnect it in Settings, "
|
||||
"then Models. Waiting will not clear this one.")
|
||||
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:auth_error", {
|
||||
"session_id": session_id, "reason": "credential_expired",
|
||||
"message": friendly_msg, "model": session.model,
|
||||
})
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id, "message": error_msg.model_dump(mode="json"),
|
||||
})
|
||||
return
|
||||
if "no credentials for provider" not in p_combined:
|
||||
from backend.apps.agents.manager.streaming.auth_retry import try_auth_self_heal
|
||||
if try_auth_self_heal(session, delay_s=75 if p_codex_rotation else 5):
|
||||
@@ -341,7 +367,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
)
|
||||
reason = "anthropic_auth_invalid"
|
||||
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
||||
p_absorb_repeat_card(session, error_msg)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
try:
|
||||
from backend.apps.service.client import submit_diagnostic
|
||||
submit_diagnostic({
|
||||
@@ -368,7 +394,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
# Upstream rejected the model code itself (e.g. Codex 1211 on a ChatGPT plan that lacks our GPT ids). Track it; the friendly "add an API key / pick another model" card is rendered frontend-side.
|
||||
p_report_model_error("unknown_model", session_id, session, turn, e, p_stderr_tail)
|
||||
error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id)
|
||||
p_absorb_repeat_card(session, error_msg)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": error_msg.model_dump(mode="json"),
|
||||
@@ -378,7 +404,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
# where the fix is entirely on our side of the wire.
|
||||
p_report_model_error("router_unavailable", session_id, session, turn, e, p_stderr_tail)
|
||||
error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id)
|
||||
p_absorb_repeat_card(session, error_msg)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": error_msg.model_dump(mode="json"),
|
||||
@@ -400,7 +426,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
if p_cause and "check stderr" in str(e).lower():
|
||||
p_card_text += f"\n\nRuntime log tail:\n{p_cause}"
|
||||
error_msg = Message(role="system", content=p_card_text, branch_id=session.active_branch_id)
|
||||
p_absorb_repeat_card(session, error_msg)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": error_msg.model_dump(mode="json"),
|
||||
|
||||
@@ -0,0 +1,135 @@
|
||||
"""Do not spend a user's turn on a lane we already know is dead.
|
||||
|
||||
Live drill, 2026-08-20: a codex credential that expired 89 HOURS earlier produced "GPT subscription
|
||||
token just rotated (automatic, every couple minutes), retrying automatically, no action needed", a
|
||||
75 second wait, a doomed retry, and then five identical cards. Zero files read. Every word of that
|
||||
was wrong, and the evidence to know better was already sitting in 9Router's own provider list:
|
||||
`testStatus: "unavailable"` with `errorCode: 401`, published before we spend anything.
|
||||
|
||||
So this looks first, and it tries to fix it before it complains:
|
||||
|
||||
healthy -> say nothing, cost nothing, dispatch as normal
|
||||
sticky-dead, first -> bounce the router ONCE (re-reads db.json, clears the in-process
|
||||
`unavailable` stamp and modelLock cooldowns), then DISPATCH ANYWAY and let
|
||||
the turn itself be the verdict. If it goes through, the user never learns
|
||||
anything happened.
|
||||
sticky-dead, again -> one accurate sentence, immediately. No invented rotation window, no wait
|
||||
the user has already tried by hand, no promise of a self-heal that cannot
|
||||
happen.
|
||||
|
||||
The bounce is NOT allowed to declare success on its own, and that mistake is worth recording: the
|
||||
first version re-read the health flag afterwards and called a cleared stamp a recovery. But a fresh
|
||||
router starts with no stamp, so the check passed for a credential that was still dead, and the turn
|
||||
hit the same 401 seconds later. A restart clears the accusation, not the cause. Only a real
|
||||
dispatch can tell you whether a credential works, so that is what decides it now.
|
||||
|
||||
The bounce is the same one ENG-315 already runs at the connect chokepoint, and is documented safe
|
||||
mid-session; the in-flight kill drill on 2026-08-20 confirmed a live turn survives one.
|
||||
"""
|
||||
|
||||
import logging
|
||||
import time
|
||||
from typing import Dict, Optional, TYPE_CHECKING
|
||||
|
||||
from typeguard import typechecked
|
||||
|
||||
if TYPE_CHECKING:
|
||||
from backend.apps.agents.core.models import AgentSession
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Router prefix -> the provider name its connection is filed under.
|
||||
P_PREFIX_PROVIDER = {"cc/": "claude", "cx/": "codex", "gc/": "antigravity", "ag/": "antigravity"}
|
||||
|
||||
# A bounce restarts a process every other session shares, so one per lane per window, never per turn.
|
||||
BOUNCE_COOLDOWN_S = 300
|
||||
|
||||
LAST_BOUNCE: Dict[str, float] = {}
|
||||
|
||||
RECONNECT_COPY = {
|
||||
"codex": ("Your ChatGPT subscription needs reconnecting: the saved sign-in expired and could "
|
||||
"not be renewed. Open Settings, then Models, and click Reconnect on the OpenAI / GPT "
|
||||
"row. Waiting will not clear this one."),
|
||||
"claude": ("Your Claude subscription needs reconnecting: the saved sign-in expired and could "
|
||||
"not be renewed. Open Settings, then Models, and click Reconnect on the Claude "
|
||||
"Pro / Max row. Waiting will not clear this one."),
|
||||
"antigravity": ("Your Google sign-in needs reconnecting: the saved credential expired and "
|
||||
"could not be renewed. Open Settings, then Models, and reconnect the Google "
|
||||
"row. Waiting will not clear this one."),
|
||||
}
|
||||
|
||||
|
||||
@typechecked
|
||||
def provider_for_model(resolved_model: str) -> Optional[str]:
|
||||
"""The router connection a resolved model id will dispatch through, or None when the call does not go through the router at all (direct API keys own their own errors)."""
|
||||
for prefix, provider in P_PREFIX_PROVIDER.items():
|
||||
if resolved_model.startswith(prefix):
|
||||
return provider
|
||||
return None
|
||||
|
||||
|
||||
@typechecked
|
||||
def connection_is_dead(conn: Dict) -> bool:
|
||||
"""A connection the router has given up on. Deliberately narrow: only the states that mean a dispatch is guaranteed to fail, never a slow or merely idle one."""
|
||||
if conn.get("testStatus") == "unavailable":
|
||||
return True
|
||||
return conn.get("errorCode") in (401, 403)
|
||||
|
||||
|
||||
async def dead_connection(provider: str) -> Optional[Dict]:
|
||||
"""The provider's connection if the router considers it dead, else None. Never raises: a preflight that cannot read health must let the turn proceed, because guessing "dead" would ground a working lane."""
|
||||
try:
|
||||
from backend.apps.nine_router import get_providers
|
||||
for conn in await get_providers():
|
||||
if conn.get("provider") != provider:
|
||||
continue
|
||||
return conn if connection_is_dead(conn) else None
|
||||
except Exception:
|
||||
logger.debug("lane preflight could not read provider health; proceeding", exc_info=True)
|
||||
return None
|
||||
|
||||
|
||||
async def preflight_lane(resolved_model: str,
|
||||
session: Optional["AgentSession"] = None) -> Optional[str]:
|
||||
"""None when the turn should proceed, or the sentence to show the user when it should not.
|
||||
|
||||
Returning a message here is a decision NOT to spend the turn, which is only correct because the
|
||||
alternative was measured: a guaranteed 401, misleading copy, and a wait the user has already
|
||||
tried themselves.
|
||||
"""
|
||||
provider = provider_for_model(resolved_model)
|
||||
if provider is None:
|
||||
return None
|
||||
|
||||
dead = await dead_connection(provider)
|
||||
if dead is None:
|
||||
return None
|
||||
|
||||
# Whatever happens next, an auth failure on THIS turn is a dead credential, not a rotation
|
||||
# window: the router had already given up before we sent anything.
|
||||
if session is not None:
|
||||
try:
|
||||
session.lane_credential_dead = True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
now = time.time()
|
||||
if now - LAST_BOUNCE.get(provider, 0.0) >= BOUNCE_COOLDOWN_S:
|
||||
LAST_BOUNCE[provider] = now
|
||||
logger.warning(
|
||||
f"lane preflight: {provider} is {dead.get('testStatus')} (errorCode={dead.get('errorCode')}); "
|
||||
"bouncing the router once, then letting the turn decide"
|
||||
)
|
||||
try:
|
||||
from backend.apps.nine_router.bounce_after_connect import bounce_router_after_connect
|
||||
await bounce_router_after_connect(provider)
|
||||
except Exception:
|
||||
logger.debug("lane preflight bounce failed", exc_info=True)
|
||||
# Deliberately no post-bounce health re-read: see the module docstring. Dispatch is the test.
|
||||
return None
|
||||
|
||||
return RECONNECT_COPY.get(
|
||||
provider,
|
||||
"This model's sign-in expired and could not be renewed. Reconnect it in Settings, then "
|
||||
"Models. Waiting will not clear this one.",
|
||||
)
|
||||
@@ -3,7 +3,7 @@ identical "hit a snag" clones (field screenshot 2026-08-19); a user message in b
|
||||
earns a fresh card."""
|
||||
|
||||
from backend.apps.agents.core.models import AgentSession, Message
|
||||
from backend.apps.agents.manager.run.handle_run_error import p_absorb_repeat_card
|
||||
from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card
|
||||
|
||||
|
||||
def p_card(text: str) -> Message:
|
||||
@@ -13,30 +13,68 @@ def p_card(text: str) -> Message:
|
||||
def test_identical_consecutive_card_is_absorbed():
|
||||
s = AgentSession(name="t", model="sonnet")
|
||||
first = p_card("That one failed.")
|
||||
p_absorb_repeat_card(s, first)
|
||||
absorb_repeat_card(s, first)
|
||||
repeat = p_card("That one failed.")
|
||||
p_absorb_repeat_card(s, repeat)
|
||||
absorb_repeat_card(s, repeat)
|
||||
assert len(s.messages) == 1
|
||||
assert repeat.id == first.id, "the bump must reuse the id so the frontend updates in place"
|
||||
|
||||
|
||||
def test_a_user_message_in_between_earns_a_fresh_card():
|
||||
s = AgentSession(name="t", model="sonnet")
|
||||
p_absorb_repeat_card(s, p_card("That one failed."))
|
||||
absorb_repeat_card(s, p_card("That one failed."))
|
||||
s.messages.append(Message(role="user", content="try again", branch_id="main"))
|
||||
p_absorb_repeat_card(s, p_card("That one failed."))
|
||||
absorb_repeat_card(s, p_card("That one failed."))
|
||||
assert len([m for m in s.messages if m.role == "system"]) == 2
|
||||
|
||||
|
||||
def test_different_error_text_always_appends():
|
||||
s = AgentSession(name="t", model="sonnet")
|
||||
p_absorb_repeat_card(s, p_card("Error A"))
|
||||
p_absorb_repeat_card(s, p_card("Error B"))
|
||||
absorb_repeat_card(s, p_card("Error A"))
|
||||
absorb_repeat_card(s, p_card("Error B"))
|
||||
assert len(s.messages) == 2
|
||||
|
||||
|
||||
def test_other_branch_cards_do_not_mask():
|
||||
s = AgentSession(name="t", model="sonnet")
|
||||
s.messages.append(Message(role="system", content="Same text", branch_id="side"))
|
||||
p_absorb_repeat_card(s, p_card("Same text"))
|
||||
absorb_repeat_card(s, p_card("Same text"))
|
||||
assert len(s.messages) == 2, "a card on another branch is invisible here and must not absorb"
|
||||
|
||||
|
||||
def test_our_own_hidden_retries_do_not_earn_fresh_cards():
|
||||
"""A live codex drill (2026-08-20) produced FIVE identical "still refreshing" cards on one ask.
|
||||
Each self-heal retry sends a HIDDEN user-role continuation, which displaced the previous card
|
||||
from the tail, so the dedup saw "a user message came in" and appended a clone. Our own
|
||||
machinery was manufacturing the wall it was written to prevent."""
|
||||
from backend.apps.agents.core.models import AgentSession, Message
|
||||
from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card
|
||||
|
||||
s = AgentSession(name="t", model="gpt-5.6", dashboard_id="d")
|
||||
s.messages.append(Message(role="user", content="do the thing", branch_id=s.active_branch_id))
|
||||
card = "GPT subscription token is still refreshing."
|
||||
|
||||
for _ in range(5):
|
||||
absorb_repeat_card(s, Message(role="system", content=card, branch_id=s.active_branch_id))
|
||||
# what every self-heal retry does next
|
||||
s.messages.append(Message(role="user", content="[Automated message] retry",
|
||||
branch_id=s.active_branch_id, hidden=True))
|
||||
|
||||
shown = [m for m in s.messages if m.role == "system" and not m.hidden]
|
||||
assert len(shown) == 1, f"one honest card per ask, got {len(shown)}"
|
||||
|
||||
|
||||
def test_a_real_user_message_still_earns_a_fresh_card():
|
||||
"""Negative control: the rule only ignores OUR sends. A human asking again deserves its own
|
||||
answer, even if the answer is the same bad news."""
|
||||
from backend.apps.agents.core.models import AgentSession, Message
|
||||
from backend.apps.agents.manager.run.handle_run_error import absorb_repeat_card
|
||||
|
||||
s = AgentSession(name="t", model="gpt-5.6", dashboard_id="d")
|
||||
card = "GPT subscription token is still refreshing."
|
||||
absorb_repeat_card(s, Message(role="system", content=card, branch_id=s.active_branch_id))
|
||||
s.messages.append(Message(role="user", content="try again please", branch_id=s.active_branch_id))
|
||||
absorb_repeat_card(s, Message(role="system", content=card, branch_id=s.active_branch_id))
|
||||
|
||||
shown = [m for m in s.messages if m.role == "system"]
|
||||
assert len(shown) == 2, "each real ask gets its own honest answer"
|
||||
|
||||
@@ -0,0 +1,126 @@
|
||||
"""Never spend a turn on a lane the router has already given up on.
|
||||
|
||||
Measured cost of not doing this (live drill, 2026-08-20, real codex lane): a credential dead for 89
|
||||
hours produced a "just rotated, every couple minutes, no action needed" card, a 75s wait, a doomed
|
||||
retry and five identical follow-up cards, for zero files read. The router had published
|
||||
testStatus="unavailable" and errorCode=401 the whole time.
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
|
||||
import pytest
|
||||
|
||||
import backend.apps.agents.manager.run.lane_preflight as lp
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def p_clear_cooldown():
|
||||
lp.LAST_BOUNCE.clear()
|
||||
yield
|
||||
lp.LAST_BOUNCE.clear()
|
||||
|
||||
|
||||
def p_providers(monkeypatch, conns, bounce_result=None):
|
||||
"""Stub the router's provider list; bounce_result, when given, is what the list becomes after a bounce."""
|
||||
state = {"conns": conns, "bounced": 0}
|
||||
|
||||
async def fake_get_providers():
|
||||
return state["conns"]
|
||||
|
||||
async def fake_bounce(provider):
|
||||
state["bounced"] += 1
|
||||
if bounce_result is not None:
|
||||
state["conns"] = bounce_result
|
||||
return True
|
||||
|
||||
import backend.apps.nine_router as nr
|
||||
import backend.apps.nine_router.bounce_after_connect as ba
|
||||
monkeypatch.setattr(nr, "get_providers", fake_get_providers, raising=True)
|
||||
monkeypatch.setattr(ba, "bounce_router_after_connect", fake_bounce, raising=True)
|
||||
return state
|
||||
|
||||
|
||||
P_DEAD = [{"provider": "codex", "testStatus": "unavailable", "errorCode": 401}]
|
||||
P_LIVE = [{"provider": "codex", "testStatus": "active", "errorCode": None}]
|
||||
|
||||
|
||||
def test_a_healthy_lane_costs_nothing_and_says_nothing(monkeypatch):
|
||||
st = p_providers(monkeypatch, P_LIVE)
|
||||
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None
|
||||
assert st["bounced"] == 0, "a working lane must never trigger a router restart"
|
||||
|
||||
|
||||
def test_the_first_dead_encounter_bounces_and_lets_the_turn_decide(monkeypatch):
|
||||
"""The bounce is an attempt, not a verdict. It must not block the turn, and it must not claim
|
||||
a recovery it cannot see."""
|
||||
st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD)
|
||||
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None, "dispatch is the real test"
|
||||
assert st["bounced"] == 1
|
||||
|
||||
|
||||
def test_a_cleared_stamp_is_never_mistaken_for_a_working_credential(monkeypatch):
|
||||
"""The bug this test exists for shipped for ten minutes on 2026-08-20. The first version
|
||||
re-read health after the bounce and returned "recovered" because a fresh router has no
|
||||
`unavailable` stamp yet. The credential was still dead and the turn 401'd seconds later.
|
||||
A restart clears the accusation, not the cause."""
|
||||
calls = {"health_reads": 0}
|
||||
real = lp.dead_connection
|
||||
|
||||
async def counting(provider):
|
||||
calls["health_reads"] += 1
|
||||
return await real(provider)
|
||||
|
||||
monkeypatch.setattr(lp, "dead_connection", counting, raising=True)
|
||||
p_providers(monkeypatch, P_DEAD, bounce_result=P_LIVE)
|
||||
asyncio.run(lp.preflight_lane("cx/gpt-5.6"))
|
||||
assert calls["health_reads"] == 1, (
|
||||
"health is read once, BEFORE the bounce; a post-bounce read is the false-recovery bug"
|
||||
)
|
||||
|
||||
|
||||
def test_a_lane_still_dead_on_the_next_ask_gets_one_accurate_sentence(monkeypatch):
|
||||
"""Second encounter inside the cooldown: we already spent a bounce and a turn, so stop
|
||||
pretending and say the true thing."""
|
||||
st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD)
|
||||
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None
|
||||
msg = asyncio.run(lp.preflight_lane("cx/gpt-5.6"))
|
||||
assert msg and "ChatGPT" in msg and "Reconnect" in msg
|
||||
assert "rotated" not in msg.lower(), "never claim a rotation that did not happen"
|
||||
assert "no action needed" not in msg.lower(), "there IS action needed; saying otherwise is the bug"
|
||||
assert st["bounced"] == 1, "the cooldown holds; one restart, not one per ask"
|
||||
|
||||
|
||||
def test_the_bounce_is_rate_limited(monkeypatch):
|
||||
"""A bounce restarts a process every other session shares, so it is once per lane per window, never once per turn."""
|
||||
st = p_providers(monkeypatch, P_DEAD, bounce_result=P_DEAD)
|
||||
for _ in range(4):
|
||||
asyncio.run(lp.preflight_lane("cx/gpt-5.6"))
|
||||
assert st["bounced"] == 1, f"expected a single bounce, got {st['bounced']}"
|
||||
|
||||
|
||||
def test_direct_api_lanes_are_left_alone(monkeypatch):
|
||||
"""Negative control: a direct API key never dispatches through the router, so the router's health says nothing about it and must not ground it."""
|
||||
st = p_providers(monkeypatch, P_DEAD)
|
||||
assert asyncio.run(lp.preflight_lane("claude-sonnet-4-6")) is None
|
||||
assert st["bounced"] == 0
|
||||
|
||||
|
||||
def test_unreadable_health_lets_the_turn_proceed(monkeypatch):
|
||||
"""Negative control, and the important one: a preflight that cannot see must never guess 'dead'.
|
||||
Grounding a working lane on a failed health read would be a worse bug than the one this fixes."""
|
||||
async def boom():
|
||||
raise RuntimeError("router unreachable")
|
||||
|
||||
import backend.apps.nine_router as nr
|
||||
monkeypatch.setattr(nr, "get_providers", boom, raising=True)
|
||||
assert asyncio.run(lp.preflight_lane("cx/gpt-5.6")) is None
|
||||
|
||||
|
||||
def test_only_terminal_states_count_as_dead():
|
||||
assert lp.connection_is_dead({"testStatus": "unavailable"}) is True
|
||||
assert lp.connection_is_dead({"errorCode": 401}) is True
|
||||
assert lp.connection_is_dead({"errorCode": 403}) is True
|
||||
# A slow, rate-limited or merely idle connection is NOT dead; grounding those would be the bug.
|
||||
assert lp.connection_is_dead({"testStatus": "active", "errorCode": 429}) is False
|
||||
assert lp.connection_is_dead({"testStatus": "active", "errorCode": 502}) is False
|
||||
assert lp.connection_is_dead({}) is False
|
||||
Reference in New Issue
Block a user