[eric] agents: an auth blip mid-run refreshes and resumes once instead of dying at the reconnect banner

backend/apps/agents/core/error_classify.py

@@ -50,6 +50,53 @@ NON_TRANSIENT_PATTERNS = re.compile(
 )
 
 
+# Real account STATES a retry cannot fix: the subscription is gone, not the token. These must keep dying to the banner, or a canceled account silently burns a request per turn forever.
+P_SUBSCRIPTION_STATE_PATTERNS = re.compile(
+    r"(?:no\s+active\s+subscription"
+    r"|subscription\s+(?:canceled|past_due)"
+    r"|free_trial_exhausted|used\s+your\s+free)",
+    re.IGNORECASE,
+)
+
+AUTH_RESUME_WAIT_CAP = 120
+
+
+@typechecked
+def auth_resume_wait(exc: BaseException, attempt: int, extra_text: str = "") -> Optional[int]:
+    """Seconds to wait before ONE refresh-and-resume of an auth-shaped turn failure (expired or
+    rotating token, 401/403), or None when the failure names a real account state (canceled
+    subscription, spent trial) that waiting cannot fix, or the single-attempt budget is spent.
+    Field incident (Alexander, 2026-08-14): a token expiring mid-long-task was classified
+    non-transient and killed the run at the banner; every big task died the same way. A misfire
+    here costs one bounded extra request; a miss is that death."""
+    if attempt >= 1:
+        return None
+    combined = f"{exc!s}\n{extra_text}".strip()
+    if not combined:
+        return None
+    if P_SUBSCRIPTION_STATE_PATTERNS.search(combined):
+        return None
+    if is_translation_error(exc, extra_text):
+        return None
+    if not re.search(
+        r"\b(?:401|403)\b"
+        r"|unauthori[sz]ed"
+        r"|invalid\s+authentication"
+        r"|invalid.*api[_\s-]?key"
+        r"|invalid.*token"
+        r"|missing\s+bearer\s+token"
+        r"|authentication\s+token\s+(?:is|has)\s+expired"
+        r"|token\s+expired",
+        combined,
+        re.IGNORECASE,
+    ):
+        return None
+    hinted = parse_retry_after(exc, extra_text)
+    if hinted is not None:
+        return min(hinted + 5, AUTH_RESUME_WAIT_CAP)
+    return 20
+
+
 @typechecked
 def is_router_unreachable_error(text: str) -> bool:
     """True when a turn-result error is the CLI failing to REACH its endpoint (our localhost

backend/apps/agents/manager/run/TurnRunner.py

@@ -11,7 +11,7 @@ from typeguard import typechecked
 
 from backend.apps.agents.core.models import AgentSession
 from backend.apps.agents.core.ws_manager import ws_manager
-from backend.apps.agents.core.error_classify import CAPACITY_BACKOFFS, capacity_retry_wait, is_router_unreachable_error
+from backend.apps.agents.core.error_classify import CAPACITY_BACKOFFS, auth_resume_wait, capacity_retry_wait, is_router_unreachable_error
 from backend.apps.agents.core import flight_recorder
 from backend.apps.agents.manager.streaming.state import ThinkingState, TurnState
 from backend.apps.agents.manager.streaming.handle_stream_event import handle_stream_event
@@ -194,6 +194,7 @@ class TurnRunner(AgentManagerProtocol):
         p_use_persistent = persistent_client_enabled()
         capacity_retry_attempt = 0
         p_router_retry_attempt = 0
+        p_auth_retry_attempt = 0
         # Baseline crumb so even a first-call failure's envelope names the turn it died in.
         flight_recorder.crumb(session_id, "turn-start", model=resolved_model, api=api_type)
         while True:
@@ -204,12 +205,12 @@ class TurnRunner(AgentManagerProtocol):
                     await p_run_streaming_turn()
                 # The near-miss ledger: a turn that needed retries and still finished is a net that
                 # FIRED, and "how often do the nets fire" needs a denominator in analytics.
-                if p_router_retry_attempt or capacity_retry_attempt:
+                if p_router_retry_attempt or capacity_retry_attempt or p_auth_retry_attempt:
                     flight_recorder.record_recovery(
                         session_id,
-                        net="router-resume" if p_router_retry_attempt else "transient-backoff",
+                        net="router-resume" if p_router_retry_attempt else ("auth-resume" if p_auth_retry_attempt else "transient-backoff"),
                         model=resolved_model,
-                        attempts=p_router_retry_attempt + capacity_retry_attempt,
+                        attempts=p_router_retry_attempt + capacity_retry_attempt + p_auth_retry_attempt,
                         sessions=self.sessions,
                     )
                 settle_provider_retries(session_id, turn, resolved_model, self.sessions)
@@ -239,6 +240,30 @@ class TurnRunner(AgentManagerProtocol):
                         options_kwargs["resume"] = session.sdk_session_id
                         options = ClaudeAgentOptions(**options_kwargs)
                     continue
+                # A token that expired MID-RUN is the auth twin of the router blip: the account is
+                # fine, the credential rotated under a long task (Alexander, 2026-08-14: every big
+                # task died at the reconnect banner). One refresh-and-resume; a real subscription
+                # state (canceled, past_due, trial spent) never qualifies and still dies honestly.
+                p_auth_wait = auth_resume_wait(p_result_err, p_auth_retry_attempt, extra_text="\n".join(p_stderr_buffer[-50:]))
+                if p_auth_wait is not None:
+                    p_auth_retry_attempt += 1
+                    flight_recorder.crumb(session_id, "auth-resume", wait_s=p_auth_wait, err=str(p_result_err)[:160])
+                    logger.warning(
+                        f"Auth-shaped turn failure on session {session_id}; refreshing credentials and "
+                        f"resuming once in {p_auth_wait}s. err={p_result_err!s}"
+                    )
+                    try:
+                        from backend.apps.nine_router.subscription_health import invalidate_health_cache
+                        invalidate_health_cache()
+                    except Exception:
+                        logger.debug("health-cache invalidate before auth resume failed", exc_info=True)
+                    await p_finalize_interrupted_stream()
+                    await asyncio.sleep(p_auth_wait)
+                    p_stderr_buffer.clear()
+                    if session.sdk_session_id:
+                        options_kwargs["resume"] = session.sdk_session_id
+                        options = ClaudeAgentOptions(**options_kwargs)
+                    continue
                 # Any other error-shaped result: the CLI already ran the whole turn (tools executed) and then reported failure; a resume-retry would re-execute side effects, so this goes straight to the error card.
                 raise
             except Exception as e:
@@ -258,6 +283,27 @@ class TurnRunner(AgentManagerProtocol):
                     if "CLIConnection" in p_name or "ProcessError" in p_name or "Transport" in p_name:
                         logger.warning(f"[client-pool] {session_id}: dead client ({p_name}); one transparent respawn retry")
                         wait = 0.0
+                # Same auth twin as the TurnResultError branch: a 401 can also arrive as a raised
+                # exception (ProcessError with the cause only in stderr). One refresh-and-resume,
+                # on its own counter so it neither burns a capacity slot nor logs as transient.
+                if wait is None:
+                    p_auth_wait2 = auth_resume_wait(e, p_auth_retry_attempt, extra_text=stderr_snapshot)
+                    if p_auth_wait2 is not None:
+                        p_auth_retry_attempt += 1
+                        flight_recorder.crumb(session_id, "auth-resume", wait_s=p_auth_wait2, err=str(e)[:160])
+                        logger.warning(f"Auth-shaped exception on session {session_id}; refreshing and resuming once in {p_auth_wait2}s. exc={e!r}")
+                        try:
+                            from backend.apps.nine_router.subscription_health import invalidate_health_cache
+                            invalidate_health_cache()
+                        except Exception:
+                            logger.debug("health-cache invalidate before auth resume failed", exc_info=True)
+                        await p_finalize_interrupted_stream()
+                        await asyncio.sleep(p_auth_wait2)
+                        p_stderr_buffer.clear()
+                        if session.sdk_session_id:
+                            options_kwargs["resume"] = session.sdk_session_id
+                            options = ClaudeAgentOptions(**options_kwargs)
+                        continue
                 if wait is not None:
                     capacity_retry_attempt += 1
                     flight_recorder.crumb(session_id, "transient-retry", attempt=capacity_retry_attempt, wait=wait, err=str(e)[:160])

backend/tests/test_auth_resume.py

@@ -0,0 +1,93 @@
+"""The auth refresh-and-resume seam (Alexander, 2026-08-14: every decently big task died at the
+"Connection needs a refresh" banner). A token that expires or rotates MID-RUN is an auth-shaped
+blip, not a dead account: the run must refresh and resume once before the terminal banner. A real
+subscription STATE (canceled, past_due, trial spent) must keep dying to the banner, or a canceled
+account silently burns a request per turn forever."""
+
+import inspect
+
+from backend.apps.agents.core.error_classify import AUTH_RESUME_WAIT_CAP, auth_resume_wait
+from backend.apps.agents.manager.run import TurnRunner
+
+
+# --------------------------------------------------------------------------- the decision function
+
+
+def test_an_expired_token_qualifies_for_one_resume():
+    assert auth_resume_wait(Exception("API Error: 401 authentication token is expired"), 0) is not None
+
+
+def test_a_bare_401_qualifies():
+    assert auth_resume_wait(Exception("Request failed: 401 Unauthorized"), 0) is not None
+
+
+def test_the_cause_can_live_only_in_stderr():
+    # The SDK's ProcessError stringifies to a generic shell; the 401 arrives via the stderr tail.
+    exc = Exception("Command failed with exit code 1. Check stderr output for details.")
+    assert auth_resume_wait(exc, 0, extra_text="upstream says: invalid token (401)") is not None
+
+
+def test_a_canceled_subscription_never_resumes():
+    assert auth_resume_wait(Exception("401: No active subscription"), 0) is None
+    assert auth_resume_wait(Exception("Subscription canceled"), 0) is None
+    assert auth_resume_wait(Exception("Subscription past_due, 403"), 0) is None
+
+
+def test_a_spent_free_trial_never_resumes():
+    assert auth_resume_wait(Exception("402 free_trial_exhausted"), 0) is None
+
+
+def test_the_budget_is_exactly_one_attempt():
+    exc = Exception("401 token expired")
+    assert auth_resume_wait(exc, 0) is not None
+    assert auth_resume_wait(exc, 1) is None
+
+
+def test_a_translation_400_is_not_auth():
+    # A tool-schema 400 can carry wording that trips auth regexes; resuming re-sends the same broken schema.
+    assert auth_resume_wait(Exception("400 INVALID_ARGUMENT: tools[3].input_schema unknown name"), 0) is None
+
+
+def test_a_non_auth_error_is_left_alone():
+    assert auth_resume_wait(Exception("500 internal server error"), 0) is None
+    assert auth_resume_wait(Exception(""), 0) is None
+
+
+def test_a_reset_hint_paces_the_wait_and_is_capped():
+    w = auth_resume_wait(Exception("401 authentication token is expired, reset after 1m 30s"), 0)
+    assert w is not None and 90 < w <= AUTH_RESUME_WAIT_CAP
+
+
+def test_the_exact_field_incident_shape_qualifies():
+    # The banner Alexander hit is raised off these strings (MessageBubble auth matcher); the two
+    # blip-shaped ones must resume, the account-state ones above must not.
+    assert auth_resume_wait(Exception("Invalid bearer token"), 0) is not None
+    assert auth_resume_wait(Exception("Missing bearer token"), 0) is not None
+
+
+# --------------------------------------------------------------------------- the TurnRunner wiring
+
+
+def test_the_error_result_path_consults_auth_resume_before_raising():
+    src = inspect.getsource(TurnRunner)
+    body = src.split("except TurnResultError", 1)[1].split("except Exception as e", 1)[0]
+    assert "auth_resume_wait" in body, "the auth check must live on the TurnResultError path"
+    assert body.index("auth_resume_wait") < body.index("raise"), "classify BEFORE the unconditional raise"
+    assert 'options_kwargs["resume"]' in body.split("auth_resume_wait", 1)[1], "the retry must resume the CLI conversation"
+
+
+def test_the_exception_path_consults_auth_resume_too():
+    src = inspect.getsource(TurnRunner)
+    body = src.split("except Exception as e", 1)[1]
+    assert "auth_resume_wait" in body, "a 401 raised as an exception must get the same one resume"
+
+
+def test_the_resume_actively_refreshes_credentials():
+    src = inspect.getsource(TurnRunner)
+    assert src.count("invalidate_health_cache") >= 2, "both paths must poke the credential health cache, not just wait"
+
+
+def test_the_recovery_ledger_counts_auth_resumes():
+    src = inspect.getsource(TurnRunner)
+    assert "auth-resume" in src.split("record_recovery", 1)[0] or "p_auth_retry_attempt" in src.split("record_recovery", 1)[1].split(")", 2)[1], \
+        "a survived auth blip must land in the near-miss ledger"