[eric] agents: a bare connection error is transient, so a network blip stops killing runs

This commit is contained in:
ciregenz
2026-07-30 20:00:42 -07:00
parent 4ba19a5b6e
commit 93e2730c2f
2 changed files with 69 additions and 4 deletions
+23 -3
View File
@@ -1,6 +1,8 @@
import re
from typing import Optional
from typing import Optional, Tuple
import anthropic
import httpx
from typeguard import typechecked
# Secret shapes that must never ride along when we ship a stderr tail or an error string to telemetry. own_key mode means the subprocess stderr can echo the user's OWN provider key, so this scrub is the wall between a diagnostic and a key leak; over-redacting is fine, leaking is not.
@@ -199,13 +201,31 @@ def parse_retry_after(exc: BaseException, extra_text: str = "") -> int | None:
return None
# Transports that fail without saying anything a pattern can read. anthropic.APIConnectionError
# stringifies to the bare "Connection error." (no code, no ECONNRESET, nothing), so the list above
# scores it NON-transient and the retry never fires: one network hiccup then throws away a run that
# was already several steps in, and the user is told "Error: Connection error." Measured live, twice
# in one sweep. A transport failure is transient by construction, so classify it by TYPE, which no
# rewording upstream can break.
P_TRANSIENT_EXC_TYPES: Tuple[type, ...] = (
anthropic.APIConnectionError, # APITimeoutError subclasses this
anthropic.InternalServerError,
httpx.TransportError, # connect/read/write/pool timeouts, protocol errors
ConnectionError,
TimeoutError,
)
@typechecked
def is_transient_capacity_error(exc: BaseException, extra_text: str = "") -> bool:
# The Claude CLI's underlying ProcessError stringifies to a generic "Command failed with exit code 1 / Check stderr output for details"; the real cause (rate_limit_error / No pool capacity available / 429 / overloaded) only surfaces in the subprocess's stderr stream, which we capture via the SDK's `stderr` callback and pass in as extra_text. Classify against both so we catch capacity errors regardless of which channel carried the message.
combined = f"{exc!s}\n{extra_text}".strip()
if not combined:
if combined and NON_TRANSIENT_PATTERNS.search(combined):
return False
if NON_TRANSIENT_PATTERNS.search(combined):
# Ahead of the empty-string bail on purpose: what the exception IS doesn't depend on whether it bothered to say anything.
if isinstance(exc, P_TRANSIENT_EXC_TYPES):
return True
if not combined:
return False
if TRANSIENT_CAPACITY_PATTERNS.search(combined):
return True
+46 -1
View File
@@ -2,7 +2,14 @@
into error_classify.py next to the classifier it uses. It was previously inline + untestable
in the agent loop's retry while-loop."""
from backend.apps.agents.core.error_classify import CAPACITY_BACKOFFS, capacity_retry_wait
import anthropic
import httpx
from backend.apps.agents.core.error_classify import (
CAPACITY_BACKOFFS,
TRANSIENT_CAPACITY_PATTERNS,
capacity_retry_wait,
)
# The classifier matches this proxy copy verbatim (a guaranteed-transient signal).
TRANSIENT = "No pool capacity available. Try again shortly."
@@ -32,3 +39,41 @@ def test_transient_signal_can_arrive_only_via_the_stderr_tail():
generic = Exception("upstream hiccup")
assert capacity_retry_wait(generic, 0) is None # nothing transient yet
assert capacity_retry_wait(generic, 0, extra_text=TRANSIENT) == 5 # stderr reveals it
# --- failures that say nothing a word list can read ---------------------------------------------
# Measured live: two browser runs died mid-task on anthropic.APIConnectionError, which stringifies
# to the bare "Connection error." The pattern list scored that NON-transient, so one network blip
# threw away work that was already several steps in. These pin the type-based classification.
def p_req():
return httpx.Request("POST", "https://api.anthropic.com/v1/messages")
def test_a_bare_connection_error_is_transient():
exc = anthropic.APIConnectionError(request=p_req())
assert str(exc) == "Connection error." # no code, no ECONNRESET, no wording
assert not TRANSIENT_CAPACITY_PATTERNS.search(str(exc)) # nothing for the list to match on
assert capacity_retry_wait(exc, 0) == 5
def test_transport_and_timeout_failures_are_transient():
for exc in (
anthropic.APITimeoutError(request=p_req()),
httpx.ConnectError("nope"),
httpx.ReadTimeout("nope"),
httpx.RemoteProtocolError("server disconnected"),
ConnectionResetError(),
TimeoutError(),
):
assert capacity_retry_wait(exc, 0) == 5, f"{type(exc).__name__} should retry"
def test_an_auth_failure_stays_non_transient_even_when_it_is_a_transport_type():
# Retrying a 401 five times burns 335s of backoff and fails anyway, so wording still wins.
assert capacity_retry_wait(ConnectionError("401 invalid token"), 0) is None
def test_a_transport_error_that_says_nothing_at_all_still_retries():
# An exception stringifying to "" used to bail out before it was ever classified.
assert capacity_retry_wait(httpx.ConnectError(""), 0) == 5