From 156c6092a341d0ff6b1216fec9a989c8f8a686ec Mon Sep 17 00:00:00 2001 From: ciregenz Date: Fri, 14 Aug 2026 10:40:20 -0700 Subject: [PATCH] [eric] agents: an auth blip mid-run refreshes and resumes once instead of dying at the reconnect banner --- backend/apps/agents/core/error_classify.py | 47 ++++++++++ backend/apps/agents/manager/run/TurnRunner.py | 54 ++++++++++- backend/tests/test_auth_resume.py | 93 +++++++++++++++++++ 3 files changed, 190 insertions(+), 4 deletions(-) create mode 100644 backend/tests/test_auth_resume.py diff --git a/backend/apps/agents/core/error_classify.py b/backend/apps/agents/core/error_classify.py index 173ece10..75d82e0b 100644 --- a/backend/apps/agents/core/error_classify.py +++ b/backend/apps/agents/core/error_classify.py @@ -50,6 +50,53 @@ NON_TRANSIENT_PATTERNS = re.compile( ) +# Real account STATES a retry cannot fix: the subscription is gone, not the token. These must keep dying to the banner, or a canceled account silently burns a request per turn forever. +P_SUBSCRIPTION_STATE_PATTERNS = re.compile( + r"(?:no\s+active\s+subscription" + r"|subscription\s+(?:canceled|past_due)" + r"|free_trial_exhausted|used\s+your\s+free)", + re.IGNORECASE, +) + +AUTH_RESUME_WAIT_CAP = 120 + + +@typechecked +def auth_resume_wait(exc: BaseException, attempt: int, extra_text: str = "") -> Optional[int]: + """Seconds to wait before ONE refresh-and-resume of an auth-shaped turn failure (expired or + rotating token, 401/403), or None when the failure names a real account state (canceled + subscription, spent trial) that waiting cannot fix, or the single-attempt budget is spent. + Field incident (Alexander, 2026-08-14): a token expiring mid-long-task was classified + non-transient and killed the run at the banner; every big task died the same way. A misfire + here costs one bounded extra request; a miss is that death.""" + if attempt >= 1: + return None + combined = f"{exc!s}\n{extra_text}".strip() + if not combined: + return None + if P_SUBSCRIPTION_STATE_PATTERNS.search(combined): + return None + if is_translation_error(exc, extra_text): + return None + if not re.search( + r"\b(?:401|403)\b" + r"|unauthori[sz]ed" + r"|invalid\s+authentication" + r"|invalid.*api[_\s-]?key" + r"|invalid.*token" + r"|missing\s+bearer\s+token" + r"|authentication\s+token\s+(?:is|has)\s+expired" + r"|token\s+expired", + combined, + re.IGNORECASE, + ): + return None + hinted = parse_retry_after(exc, extra_text) + if hinted is not None: + return min(hinted + 5, AUTH_RESUME_WAIT_CAP) + return 20 + + @typechecked def is_router_unreachable_error(text: str) -> bool: """True when a turn-result error is the CLI failing to REACH its endpoint (our localhost diff --git a/backend/apps/agents/manager/run/TurnRunner.py b/backend/apps/agents/manager/run/TurnRunner.py index 633017e4..f8d770bd 100644 --- a/backend/apps/agents/manager/run/TurnRunner.py +++ b/backend/apps/agents/manager/run/TurnRunner.py @@ -11,7 +11,7 @@ from typeguard import typechecked from backend.apps.agents.core.models import AgentSession from backend.apps.agents.core.ws_manager import ws_manager -from backend.apps.agents.core.error_classify import CAPACITY_BACKOFFS, capacity_retry_wait, is_router_unreachable_error +from backend.apps.agents.core.error_classify import CAPACITY_BACKOFFS, auth_resume_wait, capacity_retry_wait, is_router_unreachable_error from backend.apps.agents.core import flight_recorder from backend.apps.agents.manager.streaming.state import ThinkingState, TurnState from backend.apps.agents.manager.streaming.handle_stream_event import handle_stream_event @@ -194,6 +194,7 @@ class TurnRunner(AgentManagerProtocol): p_use_persistent = persistent_client_enabled() capacity_retry_attempt = 0 p_router_retry_attempt = 0 + p_auth_retry_attempt = 0 # Baseline crumb so even a first-call failure's envelope names the turn it died in. flight_recorder.crumb(session_id, "turn-start", model=resolved_model, api=api_type) while True: @@ -204,12 +205,12 @@ class TurnRunner(AgentManagerProtocol): await p_run_streaming_turn() # The near-miss ledger: a turn that needed retries and still finished is a net that # FIRED, and "how often do the nets fire" needs a denominator in analytics. - if p_router_retry_attempt or capacity_retry_attempt: + if p_router_retry_attempt or capacity_retry_attempt or p_auth_retry_attempt: flight_recorder.record_recovery( session_id, - net="router-resume" if p_router_retry_attempt else "transient-backoff", + net="router-resume" if p_router_retry_attempt else ("auth-resume" if p_auth_retry_attempt else "transient-backoff"), model=resolved_model, - attempts=p_router_retry_attempt + capacity_retry_attempt, + attempts=p_router_retry_attempt + capacity_retry_attempt + p_auth_retry_attempt, sessions=self.sessions, ) settle_provider_retries(session_id, turn, resolved_model, self.sessions) @@ -239,6 +240,30 @@ class TurnRunner(AgentManagerProtocol): options_kwargs["resume"] = session.sdk_session_id options = ClaudeAgentOptions(**options_kwargs) continue + # A token that expired MID-RUN is the auth twin of the router blip: the account is + # fine, the credential rotated under a long task (Alexander, 2026-08-14: every big + # task died at the reconnect banner). One refresh-and-resume; a real subscription + # state (canceled, past_due, trial spent) never qualifies and still dies honestly. + p_auth_wait = auth_resume_wait(p_result_err, p_auth_retry_attempt, extra_text="\n".join(p_stderr_buffer[-50:])) + if p_auth_wait is not None: + p_auth_retry_attempt += 1 + flight_recorder.crumb(session_id, "auth-resume", wait_s=p_auth_wait, err=str(p_result_err)[:160]) + logger.warning( + f"Auth-shaped turn failure on session {session_id}; refreshing credentials and " + f"resuming once in {p_auth_wait}s. err={p_result_err!s}" + ) + try: + from backend.apps.nine_router.subscription_health import invalidate_health_cache + invalidate_health_cache() + except Exception: + logger.debug("health-cache invalidate before auth resume failed", exc_info=True) + await p_finalize_interrupted_stream() + await asyncio.sleep(p_auth_wait) + p_stderr_buffer.clear() + if session.sdk_session_id: + options_kwargs["resume"] = session.sdk_session_id + options = ClaudeAgentOptions(**options_kwargs) + continue # Any other error-shaped result: the CLI already ran the whole turn (tools executed) and then reported failure; a resume-retry would re-execute side effects, so this goes straight to the error card. raise except Exception as e: @@ -258,6 +283,27 @@ class TurnRunner(AgentManagerProtocol): if "CLIConnection" in p_name or "ProcessError" in p_name or "Transport" in p_name: logger.warning(f"[client-pool] {session_id}: dead client ({p_name}); one transparent respawn retry") wait = 0.0 + # Same auth twin as the TurnResultError branch: a 401 can also arrive as a raised + # exception (ProcessError with the cause only in stderr). One refresh-and-resume, + # on its own counter so it neither burns a capacity slot nor logs as transient. + if wait is None: + p_auth_wait2 = auth_resume_wait(e, p_auth_retry_attempt, extra_text=stderr_snapshot) + if p_auth_wait2 is not None: + p_auth_retry_attempt += 1 + flight_recorder.crumb(session_id, "auth-resume", wait_s=p_auth_wait2, err=str(e)[:160]) + logger.warning(f"Auth-shaped exception on session {session_id}; refreshing and resuming once in {p_auth_wait2}s. exc={e!r}") + try: + from backend.apps.nine_router.subscription_health import invalidate_health_cache + invalidate_health_cache() + except Exception: + logger.debug("health-cache invalidate before auth resume failed", exc_info=True) + await p_finalize_interrupted_stream() + await asyncio.sleep(p_auth_wait2) + p_stderr_buffer.clear() + if session.sdk_session_id: + options_kwargs["resume"] = session.sdk_session_id + options = ClaudeAgentOptions(**options_kwargs) + continue if wait is not None: capacity_retry_attempt += 1 flight_recorder.crumb(session_id, "transient-retry", attempt=capacity_retry_attempt, wait=wait, err=str(e)[:160]) diff --git a/backend/tests/test_auth_resume.py b/backend/tests/test_auth_resume.py new file mode 100644 index 00000000..77066c13 --- /dev/null +++ b/backend/tests/test_auth_resume.py @@ -0,0 +1,93 @@ +"""The auth refresh-and-resume seam (Alexander, 2026-08-14: every decently big task died at the +"Connection needs a refresh" banner). A token that expires or rotates MID-RUN is an auth-shaped +blip, not a dead account: the run must refresh and resume once before the terminal banner. A real +subscription STATE (canceled, past_due, trial spent) must keep dying to the banner, or a canceled +account silently burns a request per turn forever.""" + +import inspect + +from backend.apps.agents.core.error_classify import AUTH_RESUME_WAIT_CAP, auth_resume_wait +from backend.apps.agents.manager.run import TurnRunner + + +# --------------------------------------------------------------------------- the decision function + + +def test_an_expired_token_qualifies_for_one_resume(): + assert auth_resume_wait(Exception("API Error: 401 authentication token is expired"), 0) is not None + + +def test_a_bare_401_qualifies(): + assert auth_resume_wait(Exception("Request failed: 401 Unauthorized"), 0) is not None + + +def test_the_cause_can_live_only_in_stderr(): + # The SDK's ProcessError stringifies to a generic shell; the 401 arrives via the stderr tail. + exc = Exception("Command failed with exit code 1. Check stderr output for details.") + assert auth_resume_wait(exc, 0, extra_text="upstream says: invalid token (401)") is not None + + +def test_a_canceled_subscription_never_resumes(): + assert auth_resume_wait(Exception("401: No active subscription"), 0) is None + assert auth_resume_wait(Exception("Subscription canceled"), 0) is None + assert auth_resume_wait(Exception("Subscription past_due, 403"), 0) is None + + +def test_a_spent_free_trial_never_resumes(): + assert auth_resume_wait(Exception("402 free_trial_exhausted"), 0) is None + + +def test_the_budget_is_exactly_one_attempt(): + exc = Exception("401 token expired") + assert auth_resume_wait(exc, 0) is not None + assert auth_resume_wait(exc, 1) is None + + +def test_a_translation_400_is_not_auth(): + # A tool-schema 400 can carry wording that trips auth regexes; resuming re-sends the same broken schema. + assert auth_resume_wait(Exception("400 INVALID_ARGUMENT: tools[3].input_schema unknown name"), 0) is None + + +def test_a_non_auth_error_is_left_alone(): + assert auth_resume_wait(Exception("500 internal server error"), 0) is None + assert auth_resume_wait(Exception(""), 0) is None + + +def test_a_reset_hint_paces_the_wait_and_is_capped(): + w = auth_resume_wait(Exception("401 authentication token is expired, reset after 1m 30s"), 0) + assert w is not None and 90 < w <= AUTH_RESUME_WAIT_CAP + + +def test_the_exact_field_incident_shape_qualifies(): + # The banner Alexander hit is raised off these strings (MessageBubble auth matcher); the two + # blip-shaped ones must resume, the account-state ones above must not. + assert auth_resume_wait(Exception("Invalid bearer token"), 0) is not None + assert auth_resume_wait(Exception("Missing bearer token"), 0) is not None + + +# --------------------------------------------------------------------------- the TurnRunner wiring + + +def test_the_error_result_path_consults_auth_resume_before_raising(): + src = inspect.getsource(TurnRunner) + body = src.split("except TurnResultError", 1)[1].split("except Exception as e", 1)[0] + assert "auth_resume_wait" in body, "the auth check must live on the TurnResultError path" + assert body.index("auth_resume_wait") < body.index("raise"), "classify BEFORE the unconditional raise" + assert 'options_kwargs["resume"]' in body.split("auth_resume_wait", 1)[1], "the retry must resume the CLI conversation" + + +def test_the_exception_path_consults_auth_resume_too(): + src = inspect.getsource(TurnRunner) + body = src.split("except Exception as e", 1)[1] + assert "auth_resume_wait" in body, "a 401 raised as an exception must get the same one resume" + + +def test_the_resume_actively_refreshes_credentials(): + src = inspect.getsource(TurnRunner) + assert src.count("invalidate_health_cache") >= 2, "both paths must poke the credential health cache, not just wait" + + +def test_the_recovery_ledger_counts_auth_resumes(): + src = inspect.getsource(TurnRunner) + assert "auth-resume" in src.split("record_recovery", 1)[0] or "p_auth_retry_attempt" in src.split("record_recovery", 1)[1].split(")", 2)[1], \ + "a survived auth blip must land in the near-miss ledger"