mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-22 01:24:52 +02:00
[aidan] fix/agent-errors: surface provider rate limits
This commit is contained in:
@@ -1520,6 +1520,7 @@ class AgentManager:
|
||||
# exit code 1 / Check stderr output for details", which masks
|
||||
# transient capacity issues.
|
||||
_stderr_buffer: list[str] = []
|
||||
_system_event_buffer: list[str] = []
|
||||
|
||||
def _stderr_cb(line: str) -> None:
|
||||
_stderr_buffer.append(line)
|
||||
@@ -2376,6 +2377,12 @@ class AgentManager:
|
||||
# Log system messages (MCP server status, errors, etc.)
|
||||
if isinstance(message, SystemMessage):
|
||||
raw = message.__dict__ if hasattr(message, '__dict__') else str(message)
|
||||
try:
|
||||
_system_event_buffer.append(json.dumps(raw, default=str))
|
||||
if len(_system_event_buffer) > 200:
|
||||
del _system_event_buffer[:100]
|
||||
except Exception:
|
||||
pass
|
||||
logger.info(f"[MCP-DEBUG] SystemMessage: {raw}")
|
||||
|
||||
if isinstance(message, StreamEvent):
|
||||
@@ -2949,6 +2956,7 @@ class AgentManager:
|
||||
_current_turn_emitted = False
|
||||
await asyncio.sleep(wait)
|
||||
_stderr_buffer.clear()
|
||||
_system_event_buffer.clear()
|
||||
if session.sdk_session_id:
|
||||
options_kwargs["resume"] = session.sdk_session_id
|
||||
options = ClaudeAgentOptions(**options_kwargs)
|
||||
@@ -2991,7 +2999,10 @@ class AgentManager:
|
||||
# user can't recover by waiting, this is a tier-gate, not a rate
|
||||
# limit, so the UX matters.
|
||||
try:
|
||||
_stderr_tail = "\n".join(_stderr_buffer[-50:])
|
||||
_stderr_tail = "\n".join([
|
||||
"\n".join(_stderr_buffer[-50:]),
|
||||
"\n".join(_system_event_buffer[-50:]),
|
||||
])
|
||||
except Exception:
|
||||
_stderr_tail = ""
|
||||
# If we already streamed a substantive assistant response this
|
||||
@@ -3169,6 +3180,31 @@ class AgentManager:
|
||||
"session_id": session_id,
|
||||
"message": error_msg.model_dump(mode="json"),
|
||||
})
|
||||
elif _is_transient_capacity_error(e, extra_text=_stderr_tail):
|
||||
friendly_msg = (
|
||||
"provider_rate_limit: This model hit your account or "
|
||||
"session rate limit. Wait until the reset time shown by "
|
||||
"your provider, then send your message again, or switch "
|
||||
"to a different model."
|
||||
)
|
||||
try:
|
||||
from backend.apps.service.client import submit_diagnostic
|
||||
submit_diagnostic({
|
||||
"kind": "model_error",
|
||||
"subkind": "rate_limit",
|
||||
"model": session.model,
|
||||
"provider": session.provider,
|
||||
"connection_mode": getattr(load_settings(), "connection_mode", "own_key"),
|
||||
"error_preview": (f"{e!s}\n{_stderr_tail}")[:600],
|
||||
})
|
||||
except Exception:
|
||||
logger.debug("submit_diagnostic transient_capacity failed", exc_info=True)
|
||||
error_msg = Message(role="assistant", content=friendly_msg, branch_id=session.active_branch_id)
|
||||
session.messages.append(error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": error_msg.model_dump(mode="json"),
|
||||
})
|
||||
else:
|
||||
# Track unclassified agent failures too so we stop flying blind on them.
|
||||
try:
|
||||
|
||||
@@ -48,6 +48,17 @@ def test_exhaustion_is_classified_and_not_retried():
|
||||
assert not _is_transient_capacity_error(Exception("free_trial_exhausted"))
|
||||
|
||||
|
||||
def test_generic_cli_failure_uses_sdk_system_events_for_rate_limits():
|
||||
system_event_tail = (
|
||||
'{"subtype":"api_retry","data":{"error_status":429,'
|
||||
'"error":"rate_limit","max_retries":10}}'
|
||||
)
|
||||
assert _is_transient_capacity_error(
|
||||
Exception("Command failed with exit code 1"),
|
||||
extra_text=system_event_tail,
|
||||
)
|
||||
|
||||
|
||||
def test_has_own_model_never_shadows_a_real_provider():
|
||||
assert not _has_own_model(AppSettings(connection_mode="free-trial", free_trial_token="x"))
|
||||
assert not _has_own_model(AppSettings())
|
||||
|
||||
@@ -111,7 +111,17 @@ function parseOpenSwarmError(text: string, ctx?: OverflowContext): OpenSwarmErro
|
||||
ctaAction: 'upgrade',
|
||||
};
|
||||
}
|
||||
// Transient throttle: Anthropic's upstream 429/overload or our own pool-shed. Not the user's
|
||||
if (/provider_rate_limit|account'?s rate limit|session rate limit|This request would exceed your account'?s rate limit/i.test(text)) {
|
||||
const reset = text.match(/reset after ([^)\\.]+)/i)?.[1];
|
||||
return {
|
||||
kind: 'network',
|
||||
title: "You've hit this model's rate limit",
|
||||
detail: reset
|
||||
? `This model can send more requests after ${reset}. Wait for that reset window, or switch to another model.`
|
||||
: 'Wait for the reset window shown by your provider, or switch to another model.',
|
||||
};
|
||||
}
|
||||
// Transient throttle: Anthropic's upstream overload or our own pool-shed. Not the user's
|
||||
// fault and not a plan cap, so don't say "upgrade", just tell them it's busy. claude.ai-style.
|
||||
if (/rate_limit_error|free_pool_busy|overloaded_error|too many requests/i.test(text)) {
|
||||
return {
|
||||
|
||||
Reference in New Issue
Block a user