From 93e2730c2fd444dfc67511ffb10275615b98350a Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 30 Jul 2026 20:00:42 -0700 Subject: [PATCH] [eric] agents: a bare connection error is transient, so a network blip stops killing runs --- backend/apps/agents/core/error_classify.py | 26 ++++++++++-- backend/tests/test_capacity_retry.py | 47 +++++++++++++++++++++- 2 files changed, 69 insertions(+), 4 deletions(-) diff --git a/backend/apps/agents/core/error_classify.py b/backend/apps/agents/core/error_classify.py index ba5417fc..d48cbf23 100644 --- a/backend/apps/agents/core/error_classify.py +++ b/backend/apps/agents/core/error_classify.py @@ -1,6 +1,8 @@ import re -from typing import Optional +from typing import Optional, Tuple +import anthropic +import httpx from typeguard import typechecked # Secret shapes that must never ride along when we ship a stderr tail or an error string to telemetry. own_key mode means the subprocess stderr can echo the user's OWN provider key, so this scrub is the wall between a diagnostic and a key leak; over-redacting is fine, leaking is not. @@ -199,13 +201,31 @@ def parse_retry_after(exc: BaseException, extra_text: str = "") -> int | None: return None +# Transports that fail without saying anything a pattern can read. anthropic.APIConnectionError +# stringifies to the bare "Connection error." (no code, no ECONNRESET, nothing), so the list above +# scores it NON-transient and the retry never fires: one network hiccup then throws away a run that +# was already several steps in, and the user is told "Error: Connection error." Measured live, twice +# in one sweep. A transport failure is transient by construction, so classify it by TYPE, which no +# rewording upstream can break. +P_TRANSIENT_EXC_TYPES: Tuple[type, ...] = ( + anthropic.APIConnectionError, # APITimeoutError subclasses this + anthropic.InternalServerError, + httpx.TransportError, # connect/read/write/pool timeouts, protocol errors + ConnectionError, + TimeoutError, +) + + @typechecked def is_transient_capacity_error(exc: BaseException, extra_text: str = "") -> bool: # The Claude CLI's underlying ProcessError stringifies to a generic "Command failed with exit code 1 / Check stderr output for details"; the real cause (rate_limit_error / No pool capacity available / 429 / overloaded) only surfaces in the subprocess's stderr stream, which we capture via the SDK's `stderr` callback and pass in as extra_text. Classify against both so we catch capacity errors regardless of which channel carried the message. combined = f"{exc!s}\n{extra_text}".strip() - if not combined: + if combined and NON_TRANSIENT_PATTERNS.search(combined): return False - if NON_TRANSIENT_PATTERNS.search(combined): + # Ahead of the empty-string bail on purpose: what the exception IS doesn't depend on whether it bothered to say anything. + if isinstance(exc, P_TRANSIENT_EXC_TYPES): + return True + if not combined: return False if TRANSIENT_CAPACITY_PATTERNS.search(combined): return True diff --git a/backend/tests/test_capacity_retry.py b/backend/tests/test_capacity_retry.py index 5909f3a9..4fac75c3 100644 --- a/backend/tests/test_capacity_retry.py +++ b/backend/tests/test_capacity_retry.py @@ -2,7 +2,14 @@ into error_classify.py next to the classifier it uses. It was previously inline + untestable in the agent loop's retry while-loop.""" -from backend.apps.agents.core.error_classify import CAPACITY_BACKOFFS, capacity_retry_wait +import anthropic +import httpx + +from backend.apps.agents.core.error_classify import ( + CAPACITY_BACKOFFS, + TRANSIENT_CAPACITY_PATTERNS, + capacity_retry_wait, +) # The classifier matches this proxy copy verbatim (a guaranteed-transient signal). TRANSIENT = "No pool capacity available. Try again shortly." @@ -32,3 +39,41 @@ def test_transient_signal_can_arrive_only_via_the_stderr_tail(): generic = Exception("upstream hiccup") assert capacity_retry_wait(generic, 0) is None # nothing transient yet assert capacity_retry_wait(generic, 0, extra_text=TRANSIENT) == 5 # stderr reveals it + + +# --- failures that say nothing a word list can read --------------------------------------------- +# Measured live: two browser runs died mid-task on anthropic.APIConnectionError, which stringifies +# to the bare "Connection error." The pattern list scored that NON-transient, so one network blip +# threw away work that was already several steps in. These pin the type-based classification. + +def p_req(): + return httpx.Request("POST", "https://api.anthropic.com/v1/messages") + + +def test_a_bare_connection_error_is_transient(): + exc = anthropic.APIConnectionError(request=p_req()) + assert str(exc) == "Connection error." # no code, no ECONNRESET, no wording + assert not TRANSIENT_CAPACITY_PATTERNS.search(str(exc)) # nothing for the list to match on + assert capacity_retry_wait(exc, 0) == 5 + + +def test_transport_and_timeout_failures_are_transient(): + for exc in ( + anthropic.APITimeoutError(request=p_req()), + httpx.ConnectError("nope"), + httpx.ReadTimeout("nope"), + httpx.RemoteProtocolError("server disconnected"), + ConnectionResetError(), + TimeoutError(), + ): + assert capacity_retry_wait(exc, 0) == 5, f"{type(exc).__name__} should retry" + + +def test_an_auth_failure_stays_non_transient_even_when_it_is_a_transport_type(): + # Retrying a 401 five times burns 335s of backoff and fails anyway, so wording still wins. + assert capacity_retry_wait(ConnectionError("401 invalid token"), 0) is None + + +def test_a_transport_error_that_says_nothing_at_all_still_retries(): + # An exception stringifying to "" used to bail out before it was ever classified. + assert capacity_retry_wait(httpx.ConnectError(""), 0) == 5