[aidan] fix/agent-errors: surface provider rate limits

This commit is contained in:
abccodes
2026-06-16 18:28:44 -07:00
parent b6252df401
commit 89ea813313
3 changed files with 59 additions and 2 deletions
+37 -1
View File
@@ -1520,6 +1520,7 @@ class AgentManager:
# exit code 1 / Check stderr output for details", which masks
# transient capacity issues.
_stderr_buffer: list[str] = []
_system_event_buffer: list[str] = []
def _stderr_cb(line: str) -> None:
_stderr_buffer.append(line)
@@ -2376,6 +2377,12 @@ class AgentManager:
# Log system messages (MCP server status, errors, etc.)
if isinstance(message, SystemMessage):
raw = message.__dict__ if hasattr(message, '__dict__') else str(message)
try:
_system_event_buffer.append(json.dumps(raw, default=str))
if len(_system_event_buffer) > 200:
del _system_event_buffer[:100]
except Exception:
pass
logger.info(f"[MCP-DEBUG] SystemMessage: {raw}")
if isinstance(message, StreamEvent):
@@ -2949,6 +2956,7 @@ class AgentManager:
_current_turn_emitted = False
await asyncio.sleep(wait)
_stderr_buffer.clear()
_system_event_buffer.clear()
if session.sdk_session_id:
options_kwargs["resume"] = session.sdk_session_id
options = ClaudeAgentOptions(**options_kwargs)
@@ -2991,7 +2999,10 @@ class AgentManager:
# user can't recover by waiting, this is a tier-gate, not a rate
# limit, so the UX matters.
try:
_stderr_tail = "\n".join(_stderr_buffer[-50:])
_stderr_tail = "\n".join([
"\n".join(_stderr_buffer[-50:]),
"\n".join(_system_event_buffer[-50:]),
])
except Exception:
_stderr_tail = ""
# If we already streamed a substantive assistant response this
@@ -3169,6 +3180,31 @@ class AgentManager:
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
})
elif _is_transient_capacity_error(e, extra_text=_stderr_tail):
friendly_msg = (
"provider_rate_limit: This model hit your account or "
"session rate limit. Wait until the reset time shown by "
"your provider, then send your message again, or switch "
"to a different model."
)
try:
from backend.apps.service.client import submit_diagnostic
submit_diagnostic({
"kind": "model_error",
"subkind": "rate_limit",
"model": session.model,
"provider": session.provider,
"connection_mode": getattr(load_settings(), "connection_mode", "own_key"),
"error_preview": (f"{e!s}\n{_stderr_tail}")[:600],
})
except Exception:
logger.debug("submit_diagnostic transient_capacity failed", exc_info=True)
error_msg = Message(role="assistant", content=friendly_msg, branch_id=session.active_branch_id)
session.messages.append(error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
})
else:
# Track unclassified agent failures too so we stop flying blind on them.
try:
+11
View File
@@ -48,6 +48,17 @@ def test_exhaustion_is_classified_and_not_retried():
assert not _is_transient_capacity_error(Exception("free_trial_exhausted"))
def test_generic_cli_failure_uses_sdk_system_events_for_rate_limits():
system_event_tail = (
'{"subtype":"api_retry","data":{"error_status":429,'
'"error":"rate_limit","max_retries":10}}'
)
assert _is_transient_capacity_error(
Exception("Command failed with exit code 1"),
extra_text=system_event_tail,
)
def test_has_own_model_never_shadows_a_real_provider():
assert not _has_own_model(AppSettings(connection_mode="free-trial", free_trial_token="x"))
assert not _has_own_model(AppSettings())
@@ -111,7 +111,17 @@ function parseOpenSwarmError(text: string, ctx?: OverflowContext): OpenSwarmErro
ctaAction: 'upgrade',
};
}
// Transient throttle: Anthropic's upstream 429/overload or our own pool-shed. Not the user's
if (/provider_rate_limit|account'?s rate limit|session rate limit|This request would exceed your account'?s rate limit/i.test(text)) {
const reset = text.match(/reset after ([^)\\.]+)/i)?.[1];
return {
kind: 'network',
title: "You've hit this model's rate limit",
detail: reset
? `This model can send more requests after ${reset}. Wait for that reset window, or switch to another model.`
: 'Wait for the reset window shown by your provider, or switch to another model.',
};
}
// Transient throttle: Anthropic's upstream overload or our own pool-shed. Not the user's
// fault and not a plan cap, so don't say "upgrade", just tell them it's busy. claude.ai-style.
if (/rate_limit_error|free_pool_busy|overloaded_error|too many requests/i.test(text)) {
return {