mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-28 12:34:50 +02:00
[eric] agents: a bare connection error is transient, so a network blip stops killing runs
This commit is contained in:
@@ -1,6 +1,8 @@
|
||||
import re
|
||||
from typing import Optional
|
||||
from typing import Optional, Tuple
|
||||
|
||||
import anthropic
|
||||
import httpx
|
||||
from typeguard import typechecked
|
||||
|
||||
# Secret shapes that must never ride along when we ship a stderr tail or an error string to telemetry. own_key mode means the subprocess stderr can echo the user's OWN provider key, so this scrub is the wall between a diagnostic and a key leak; over-redacting is fine, leaking is not.
|
||||
@@ -199,13 +201,31 @@ def parse_retry_after(exc: BaseException, extra_text: str = "") -> int | None:
|
||||
return None
|
||||
|
||||
|
||||
# Transports that fail without saying anything a pattern can read. anthropic.APIConnectionError
|
||||
# stringifies to the bare "Connection error." (no code, no ECONNRESET, nothing), so the list above
|
||||
# scores it NON-transient and the retry never fires: one network hiccup then throws away a run that
|
||||
# was already several steps in, and the user is told "Error: Connection error." Measured live, twice
|
||||
# in one sweep. A transport failure is transient by construction, so classify it by TYPE, which no
|
||||
# rewording upstream can break.
|
||||
P_TRANSIENT_EXC_TYPES: Tuple[type, ...] = (
|
||||
anthropic.APIConnectionError, # APITimeoutError subclasses this
|
||||
anthropic.InternalServerError,
|
||||
httpx.TransportError, # connect/read/write/pool timeouts, protocol errors
|
||||
ConnectionError,
|
||||
TimeoutError,
|
||||
)
|
||||
|
||||
|
||||
@typechecked
|
||||
def is_transient_capacity_error(exc: BaseException, extra_text: str = "") -> bool:
|
||||
# The Claude CLI's underlying ProcessError stringifies to a generic "Command failed with exit code 1 / Check stderr output for details"; the real cause (rate_limit_error / No pool capacity available / 429 / overloaded) only surfaces in the subprocess's stderr stream, which we capture via the SDK's `stderr` callback and pass in as extra_text. Classify against both so we catch capacity errors regardless of which channel carried the message.
|
||||
combined = f"{exc!s}\n{extra_text}".strip()
|
||||
if not combined:
|
||||
if combined and NON_TRANSIENT_PATTERNS.search(combined):
|
||||
return False
|
||||
if NON_TRANSIENT_PATTERNS.search(combined):
|
||||
# Ahead of the empty-string bail on purpose: what the exception IS doesn't depend on whether it bothered to say anything.
|
||||
if isinstance(exc, P_TRANSIENT_EXC_TYPES):
|
||||
return True
|
||||
if not combined:
|
||||
return False
|
||||
if TRANSIENT_CAPACITY_PATTERNS.search(combined):
|
||||
return True
|
||||
|
||||
@@ -2,7 +2,14 @@
|
||||
into error_classify.py next to the classifier it uses. It was previously inline + untestable
|
||||
in the agent loop's retry while-loop."""
|
||||
|
||||
from backend.apps.agents.core.error_classify import CAPACITY_BACKOFFS, capacity_retry_wait
|
||||
import anthropic
|
||||
import httpx
|
||||
|
||||
from backend.apps.agents.core.error_classify import (
|
||||
CAPACITY_BACKOFFS,
|
||||
TRANSIENT_CAPACITY_PATTERNS,
|
||||
capacity_retry_wait,
|
||||
)
|
||||
|
||||
# The classifier matches this proxy copy verbatim (a guaranteed-transient signal).
|
||||
TRANSIENT = "No pool capacity available. Try again shortly."
|
||||
@@ -32,3 +39,41 @@ def test_transient_signal_can_arrive_only_via_the_stderr_tail():
|
||||
generic = Exception("upstream hiccup")
|
||||
assert capacity_retry_wait(generic, 0) is None # nothing transient yet
|
||||
assert capacity_retry_wait(generic, 0, extra_text=TRANSIENT) == 5 # stderr reveals it
|
||||
|
||||
|
||||
# --- failures that say nothing a word list can read ---------------------------------------------
|
||||
# Measured live: two browser runs died mid-task on anthropic.APIConnectionError, which stringifies
|
||||
# to the bare "Connection error." The pattern list scored that NON-transient, so one network blip
|
||||
# threw away work that was already several steps in. These pin the type-based classification.
|
||||
|
||||
def p_req():
|
||||
return httpx.Request("POST", "https://api.anthropic.com/v1/messages")
|
||||
|
||||
|
||||
def test_a_bare_connection_error_is_transient():
|
||||
exc = anthropic.APIConnectionError(request=p_req())
|
||||
assert str(exc) == "Connection error." # no code, no ECONNRESET, no wording
|
||||
assert not TRANSIENT_CAPACITY_PATTERNS.search(str(exc)) # nothing for the list to match on
|
||||
assert capacity_retry_wait(exc, 0) == 5
|
||||
|
||||
|
||||
def test_transport_and_timeout_failures_are_transient():
|
||||
for exc in (
|
||||
anthropic.APITimeoutError(request=p_req()),
|
||||
httpx.ConnectError("nope"),
|
||||
httpx.ReadTimeout("nope"),
|
||||
httpx.RemoteProtocolError("server disconnected"),
|
||||
ConnectionResetError(),
|
||||
TimeoutError(),
|
||||
):
|
||||
assert capacity_retry_wait(exc, 0) == 5, f"{type(exc).__name__} should retry"
|
||||
|
||||
|
||||
def test_an_auth_failure_stays_non_transient_even_when_it_is_a_transport_type():
|
||||
# Retrying a 401 five times burns 335s of backoff and fails anyway, so wording still wins.
|
||||
assert capacity_retry_wait(ConnectionError("401 invalid token"), 0) is None
|
||||
|
||||
|
||||
def test_a_transport_error_that_says_nothing_at_all_still_retries():
|
||||
# An exception stringifying to "" used to bail out before it was ever classified.
|
||||
assert capacity_retry_wait(httpx.ConnectError(""), 0) == 5
|
||||
|
||||
Reference in New Issue
Block a user