mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-02 14:28:59 +02:00
596 lines
36 KiB
Python
596 lines
36 KiB
Python
"""Friendly error cards for a failed agent run. The run_agent_loop except-handler classifies
|
|
the exception (long-context / capacity / free-trial / auth / unknown-model / unclassified) and
|
|
emits the matching system message + WS event. Pulled out of agent_manager so the loop stays under
|
|
the file ceiling; pure relocation, no self (operates on the passed run state)."""
|
|
|
|
import asyncio
|
|
import logging
|
|
from typing import List
|
|
from typeguard import typechecked
|
|
|
|
from backend.apps.agents.core.models import AgentSession, Message
|
|
from backend.apps.agents.core.ws_manager import ws_manager
|
|
from backend.apps.settings.settings import load_settings
|
|
from backend.apps.agents.manager.streaming.state import TurnState
|
|
from backend.apps.agents.core.error_classify import (
|
|
is_context_overflow_error,
|
|
is_long_context_error,
|
|
is_stale_tool_schema_error,
|
|
is_transient_capacity_error,
|
|
is_free_trial_exhausted,
|
|
is_out_of_tokens,
|
|
has_auth_status,
|
|
is_auth_error,
|
|
is_content_policy_block,
|
|
is_cert_failure,
|
|
is_cli_binary_missing,
|
|
is_connection_lost,
|
|
is_unknown_model_error,
|
|
parse_retry_after,
|
|
)
|
|
from backend.apps.agents.core.is_router_unavailable_error import is_router_unavailable_error
|
|
from backend.apps.agents.core.extract_reset_hint import extract_reset_hint
|
|
from backend.apps.agents.core.redact_for_telemetry import redact_for_telemetry
|
|
from backend.apps.agents.core import flight_recorder
|
|
from backend.apps.agents.manager.run.empty_finish import count_tool_calls
|
|
from backend.apps.agents.session_credential import api_key_twin_model
|
|
|
|
logger = logging.getLogger(__name__)
|
|
|
|
|
|
|
|
@typechecked
|
|
def p_renders_as_nothing(msg: Message) -> bool:
|
|
"""An empty thinking pill occupies the tail while showing the user nothing, which silently broke the dedup below (live drill 2026-08-20: three identical cards)."""
|
|
return msg.role == "thinking" and not str(msg.content or "").strip()
|
|
|
|
|
|
@typechecked
|
|
def absorb_repeat_card(session: AgentSession, error_msg: Message) -> None:
|
|
"""Append the error card, unless the branch tail is already the IDENTICAL card with nothing
|
|
after it: a retry ladder re-failing the same way then bumps the existing card instead of
|
|
stacking a wall of clones (field screenshot 2026-08-19). A user message in between always
|
|
yields a fresh card, each ask deserves its own honest answer.
|
|
|
|
HIDDEN messages do not count as that user message. Our own self-heal continuations are hidden
|
|
user-role sends, so counting them re-opened the exact clone wall this exists to stop: a live
|
|
codex drill on 2026-08-20 produced FIVE identical "still refreshing" cards, one per retry,
|
|
because each retry's hidden prompt had displaced the previous card from the tail."""
|
|
p_tail = [m for m in session.messages
|
|
if getattr(m, "branch_id", None) in (None, session.active_branch_id)
|
|
and not getattr(m, "hidden", False)
|
|
and not p_renders_as_nothing(m)]
|
|
if p_tail and p_tail[-1].role == "system" and p_tail[-1].content == error_msg.content:
|
|
error_msg.id = p_tail[-1].id
|
|
p_tail[-1].timestamp = error_msg.timestamp
|
|
return
|
|
session.messages.append(error_msg)
|
|
|
|
|
|
# Blocked chats were measured ~4.8x more likely to use InvokeAgent/SpawnAgent, and a delegated
|
|
# child's answer lands in the parent's context as another model's output, which is the literal
|
|
# wording of the clause being enforced. Recorded so that correlation can finally be tested.
|
|
P_DELEGATION_TOOLS = ("InvokeAgent", "SpawnAgent", "CreateBrowserAgent")
|
|
|
|
|
|
@typechecked
|
|
def p_used_delegation(session: AgentSession) -> bool:
|
|
for msg in session.messages:
|
|
content = msg.content
|
|
if isinstance(content, dict) and any(t in str(content.get("tool", "")) for t in P_DELEGATION_TOOLS):
|
|
return True
|
|
return False
|
|
|
|
|
|
@typechecked
|
|
def p_report_model_error(subkind: str, session_id: str, session: AgentSession, turn: TurnState,
|
|
e: BaseException, stderr_tail: str) -> None:
|
|
"""The three terminal model_error rungs differ only by subkind, so they share one submitter."""
|
|
try:
|
|
from backend.apps.service.client import submit_diagnostic
|
|
submit_diagnostic({
|
|
"kind": "model_error",
|
|
"subkind": subkind,
|
|
"flight": flight_recorder.build_envelope(session_id, "model_error", subkind, session.model, "stream" if turn.current_turn_emitted else "spawn", -1),
|
|
"model": session.model,
|
|
"provider": session.provider,
|
|
"connection_mode": getattr(load_settings(), "connection_mode", "own_key"),
|
|
"error_preview": redact_for_telemetry(str(e), limit=400),
|
|
"stderr_tail": redact_for_telemetry(stderr_tail),
|
|
# The SHAPE of what was refused. Without these, a content theory about the policy block
|
|
# is untestable: model_error was the ONE envelope kind carrying no session_id at all
|
|
# (1,028 of 1,028 null, while empty_finish_nudge was 0 of 4,001), so no block could ever
|
|
# be tied back to the conversation that produced it, and the class stayed unexplained
|
|
# while the theories were argued from inference (ENG-396).
|
|
"session_id": session_id,
|
|
"input_tokens": int((session.tokens or {}).get("input", 0) or 0),
|
|
"tool_calls": count_tool_calls(session),
|
|
"compacted": bool(session.needs_fresh_session),
|
|
# `compacted` is a pending-rebuild flag, not a history. These two are the history: how
|
|
# many times the CLI compacted this chat, and how many times we broke the turn first.
|
|
"cli_compactions": int(getattr(session, "cli_compactions", 0) or 0),
|
|
"midturn_breaks": int(getattr(session, "midturn_breaks", 0) or 0),
|
|
"proactive_prunes": int(getattr(session, "proactive_prunes", 0) or 0),
|
|
"history_prefix_sent": session.history_prefix_sent,
|
|
"delegated": p_used_delegation(session),
|
|
})
|
|
except Exception:
|
|
logger.debug(f"submit_diagnostic {subkind} failed", exc_info=True)
|
|
|
|
async def p_try_runtime_repair(session, session_id: str) -> bool:
|
|
"""Put the runtime back mid-turn. True only when it is back AND stayed back.
|
|
|
|
Returns False for every ambiguous outcome (no package, restore failed, antivirus took it again),
|
|
because the caller's fallback is an honest card and a half-repair must not suppress it."""
|
|
try:
|
|
from backend.apps.agents.core.bundled_cli_missing import bundled_cli_missing
|
|
from backend.apps.agents.core.cli_self_heal import repair_bundled_cli
|
|
p_gone = bundled_cli_missing()
|
|
if p_gone is None:
|
|
return False
|
|
p_result = await asyncio.to_thread(repair_bundled_cli, p_gone)
|
|
if not (p_result.repaired and not p_result.retaken):
|
|
logger.warning("runtime self-heal did not stick: %s", p_result.detail)
|
|
return False
|
|
logger.info("runtime self-heal succeeded mid-turn: %s", p_result.detail)
|
|
p_msg = Message(
|
|
role="system",
|
|
content="A core OpenSwarm component had been removed, most likely by antivirus. "
|
|
"It has been restored from your installer, so you can send that message again.",
|
|
branch_id=session.active_branch_id,
|
|
)
|
|
absorb_repeat_card(session, p_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": p_msg.model_dump(mode="json"),
|
|
})
|
|
return True
|
|
except Exception:
|
|
logger.exception("runtime self-heal raised; falling back to the card")
|
|
return False
|
|
|
|
|
|
async def handle_run_error(e: Exception, session: AgentSession, session_id: str, turn: TurnState, p_stderr_buffer: List[str]) -> None:
|
|
logger.exception(f"Agent {session_id} error: {e}")
|
|
session.status = "error"
|
|
|
|
# Long-context-required 429 fork: surface a friendly overflow event so the frontend can render an actionable card ("Switch to Chat mode" / "Start a fresh chat") instead of a raw error blob. The user can't recover by waiting, this is a tier-gate, not a rate limit, so the UX matters.
|
|
try:
|
|
p_stderr_tail = "\n".join(p_stderr_buffer[-50:])
|
|
except Exception:
|
|
p_stderr_tail = ""
|
|
# No completed-mask here anymore: current_turn_emitted stays True until a ResultMessage lands, so the old "already answered" early-return fired on every MID-TASK death (models narrate between tool calls) and converted a dead run into a fake "completed". Reaching this handler with an overflow means the valve's compact-and-retry already failed once; the user must see the card.
|
|
# Ahead of every card-emitting branch on purpose: this 400 is the CLI holding a stale tool
|
|
# registration, not the provider refusing. Below the generic error branch it would card out on
|
|
# top of a turn's real work (61 tool calls, live, 2026-08-31); a respawn just continues it.
|
|
if is_stale_tool_schema_error(e, extra_text=p_stderr_tail):
|
|
from backend.apps.agents.manager.streaming.auth_retry import try_stale_tool_schema_self_heal
|
|
if try_stale_tool_schema_self_heal(session):
|
|
logger.warning(
|
|
f"Agent {session_id}: CLI sent a deferred tool with cache_control (ENG-394); "
|
|
f"respawning it and redoing that step"
|
|
)
|
|
return
|
|
logger.warning(f"Agent {session_id}: deferred-tool 400 again after a respawn; carding it")
|
|
# The generic card below says "switch to another model", which for THIS failure blames the
|
|
# wrong thing: the model was fine, our router was restarting. Say what happened and what
|
|
# actually works (a plain resend once it settles; proven live on the same transcript).
|
|
friendly_msg = (
|
|
"The app's model router was restarting when this chat connected, and the automatic "
|
|
"retry hit the same window. It settles within a minute; send your message again and "
|
|
"the chat picks up where it left off."
|
|
)
|
|
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|
|
try:
|
|
from backend.apps.service.client import submit_diagnostic
|
|
submit_diagnostic({
|
|
"kind": "model_error",
|
|
"subkind": "stale_tool_schema_respawn_exhausted",
|
|
"flight": flight_recorder.build_envelope(session_id, "model_error", "stale_tool_schema", session.model, "stream" if turn.current_turn_emitted else "spawn", -1),
|
|
"session_id": session_id,
|
|
"model": session.model,
|
|
"error_preview": redact_for_telemetry(str(e), limit=400),
|
|
})
|
|
except Exception:
|
|
logger.debug("submit_diagnostic stale_tool_schema failed", exc_info=True)
|
|
return
|
|
|
|
if is_context_overflow_error(e, extra_text=p_stderr_tail):
|
|
p_tier_gate = is_long_context_error(e, extra_text=p_stderr_tail)
|
|
friendly_msg = (
|
|
"This conversation has grown too large for your account's "
|
|
"standard context window. Long-context requests require an "
|
|
"upgraded tier, switch to Chat mode or start a fresh chat "
|
|
"to continue."
|
|
) if p_tier_gate else (
|
|
"This conversation outgrew the model's context window, and "
|
|
"automatic compaction couldn't shrink it enough. Start a fresh "
|
|
"chat (your recent context carries over) or switch to a model "
|
|
"with a larger window."
|
|
)
|
|
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
p_ovf_payload = {
|
|
"session_id": session_id,
|
|
"reason": "long_context_required" if p_tier_gate else "context_overflow",
|
|
"message": friendly_msg,
|
|
"model": session.model,
|
|
"provider": session.provider,
|
|
"context_window": session.context_window,
|
|
"framework_overhead_tokens": session.framework_overhead_tokens,
|
|
"input_tokens": session.tokens.get("input", 0),
|
|
"active_mcps": list(session.active_mcps),
|
|
"compact_threshold_pct": session.compact_threshold_pct,
|
|
"context_soft_cap_pct": session.context_soft_cap_pct,
|
|
}
|
|
await ws_manager.send_to_session(session_id, "agent:context_overflow", p_ovf_payload)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|
|
try:
|
|
from backend.apps.service.client import submit_diagnostic
|
|
submit_diagnostic({
|
|
"kind": "context_overflow",
|
|
"where": "manager.run.handle_run_error",
|
|
"flight": flight_recorder.build_envelope(session_id, "context_overflow", "overflow", session.model, "stream" if turn.current_turn_emitted else "spawn", -1),
|
|
"session_id": session_id,
|
|
"model": session.model,
|
|
"provider": session.provider,
|
|
"context_window": session.context_window,
|
|
"input_tokens": session.tokens.get("input", 0),
|
|
"framework_overhead_tokens": session.framework_overhead_tokens,
|
|
"active_mcps_count": len(session.active_mcps),
|
|
"messages_count": len(session.messages),
|
|
"error_preview": redact_for_telemetry(str(e), limit=500),
|
|
})
|
|
except Exception:
|
|
logger.debug("submit_diagnostic for context_overflow failed", exc_info=True)
|
|
elif is_cert_failure(e, extra_text=p_stderr_tail):
|
|
# Deterministic per host: corporate TLS-inspection proxy, clock skew, or a stale CA bundle. Waiting can't fix any of them, so name the real remedies instead of a rate-limit-shaped pill (ENG-218).
|
|
friendly_msg = (
|
|
"OpenSwarm couldn't verify the AI provider's security certificate, so the "
|
|
"connection was refused. This usually means a corporate proxy or security "
|
|
"tool (Zscaler, Netskope) is inspecting your traffic, or your system clock "
|
|
"is wrong. Check the clock, try a different network, or ask IT to allow "
|
|
"api.anthropic.com; retrying won't help until one of those changes."
|
|
)
|
|
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|
|
p_report_model_error("cert_failure", session_id, session, turn, e, p_stderr_tail)
|
|
elif is_cli_binary_missing(e, extra_text=p_stderr_tail) and await p_try_runtime_repair(session, session_id):
|
|
# Repaired mid-turn: the file is back and stayed back, so the user gets a retry chip rather
|
|
# than a card about antivirus. Boot-time repair only covers a quarantine that happened while
|
|
# the app was closed; this is the one that happens while they are working.
|
|
pass
|
|
elif is_cli_binary_missing(e, extra_text=p_stderr_tail):
|
|
# The bundled CLI vanished from an installed app (Windows AV quarantine class; 22 of 25 field installs never recovered). The raw "not found at: C:\..." card is unactionable; name the likely cause and the two real fixes.
|
|
# "Restore it from your antivirus quarantine" is technically right and empirically useless:
|
|
# 22 of 25 affected field installs never recovered, and a real user replied "don't know how
|
|
# to take a file out of quarantine" (2026-08-29). A card that names a fix the reader cannot
|
|
# perform is a dead end, so spell the clicks out and put the fix that always works first.
|
|
friendly_msg = (
|
|
"A core OpenSwarm component (the bundled agent runtime) is missing from this "
|
|
"install. Antivirus software almost always caused this by quarantining it. "
|
|
"Your chats and settings are safe either way.\n\n"
|
|
"Easiest fix: reinstall from openswarm.com over the top of this install. "
|
|
"Nothing is lost.\n\n"
|
|
"Or restore it in Windows Security:\n"
|
|
"1. Open Windows Security, then Virus & threat protection.\n"
|
|
"2. Under Current threats choose Protection history.\n"
|
|
"3. Find the quarantined OpenSwarm or claude item and choose Actions, then Restore.\n"
|
|
"4. Back in Virus & threat protection, open Manage settings, scroll to Exclusions, "
|
|
"choose Add or remove exclusions, then Add an exclusion, then Folder, and pick your "
|
|
"OpenSwarm folder. Without this step it will be quarantined again.\n\n"
|
|
# Step 4 by hand is the step people give up on, and it is the one that makes the fix
|
|
# stick. OpenSwarm can do exactly that step itself now (ENG-422), so offer it here.
|
|
"OpenSwarm can do step 4 for you. Turn on Settings, then Advanced, then "
|
|
"Antivirus exclusion, and Windows will ask you to approve it. "
|
|
"Turning it back off undoes it."
|
|
)
|
|
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|
|
try:
|
|
from backend.apps.service.client import submit_diagnostic
|
|
submit_diagnostic({
|
|
"kind": "cli_binary_missing",
|
|
"where": "manager.run.handle_run_error",
|
|
"flight": flight_recorder.build_envelope(session_id, "cli_binary_missing", "missing", session.model, "stream" if turn.current_turn_emitted else "spawn", -1),
|
|
"session_id": session_id,
|
|
"model": session.model,
|
|
"error_preview": redact_for_telemetry(str(e), limit=400),
|
|
})
|
|
except Exception:
|
|
logger.debug("submit_diagnostic cli_binary_missing failed", exc_info=True)
|
|
elif is_transient_capacity_error(e, extra_text=p_stderr_tail):
|
|
# A genuine throttle (429/overload/capacity) that already burned the whole silent-backoff budget (the only way one reaches here). It's a limit, not a failure, so don't append a system-message card; emit a transient signal for the muted pill and mark the turn completed so it doesn't read as an error.
|
|
# 335s of ladder is a blip's worth of patience, and a closed lid or switched network outlasts it, so park and retry before conceding a turn the user never chose to end.
|
|
from backend.apps.agents.manager.run.reconnect_resume import arm_reconnect_resume
|
|
p_delay = arm_reconnect_resume(session, parse_retry_after(e, p_stderr_tail), is_connection_lost(e))
|
|
if p_delay is None:
|
|
# Budget spent: this turn is OVER, not parked. Leaving the flag set muzzles the
|
|
# terminal floor (which stays quiet for parked turns) and the ask ends in silence,
|
|
# which is the exact failure the floor exists to prevent. Caught live on a rate-limited
|
|
# Gemini run, 2026-08-20: status=completed, awaiting_reconnect=True, attempts=3, and
|
|
# not one word to the user.
|
|
from backend.apps.agents.manager.run.reconnect_resume import clear_reconnect_wait
|
|
clear_reconnect_wait(session)
|
|
if p_delay is not None:
|
|
p_why = "connection lost" if is_connection_lost(e) else "provider unavailable"
|
|
logger.info(f"Agent {session_id}: {p_why} past the in-turn budget; retrying in {p_delay}s")
|
|
await ws_manager.send_to_session(session_id, "agent:reconnect_wait", {
|
|
"session_id": session_id,
|
|
"retry_in_s": p_delay,
|
|
"attempt": session.reconnect_attempts,
|
|
})
|
|
return
|
|
session.status = "completed"
|
|
if turn.stream_text_msg_id:
|
|
try:
|
|
await ws_manager.send_to_session(session_id, "agent:stream_end", {
|
|
"session_id": session_id,
|
|
"message_id": turn.stream_text_msg_id,
|
|
})
|
|
except Exception:
|
|
pass
|
|
await ws_manager.send_to_session(session_id, "agent:rate_limited", {
|
|
"session_id": session_id,
|
|
"retry_after_s": parse_retry_after(e, p_stderr_tail),
|
|
})
|
|
elif is_free_trial_exhausted(e, extra_text=p_stderr_tail):
|
|
# Free runs spent. Flip back to own_key and show a friendly "connect a model" upsell instead of a raw 402.
|
|
try:
|
|
from backend.apps.subscription.free_trial import clear_free_trial
|
|
await clear_free_trial(load_settings())
|
|
except Exception:
|
|
logger.debug("clear_free_trial after exhaustion failed", exc_info=True)
|
|
friendly_msg = (
|
|
"You've used your free runs. Connect a model to keep going: "
|
|
"your own API key, an AI subscription you already pay for, or "
|
|
"OpenSwarm Pro."
|
|
)
|
|
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:free_trial_exhausted", {
|
|
"session_id": session_id,
|
|
"message": friendly_msg,
|
|
})
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|
|
elif is_content_policy_block(f"{e!s}\n{p_stderr_tail}"):
|
|
# The provider's abuse classifier declined the REQUEST, and on the subscription lane what it reads as "duplicating model outputs" is OUR recap of the chat on a fresh CLI session (192 blocks in 14 days, 0 on API keys); deterministic, so the one retry that can pass carries no history at all, the session stays that way, and every block is reported with the shape it sent.
|
|
p_sent = session.history_prefix_sent
|
|
p_report_model_error(f"policy_block:{p_sent}", session_id, session, turn, e, p_stderr_tail)
|
|
if p_sent != "none":
|
|
session.history_prefix_mode = "none"
|
|
session.needs_fresh_session = True
|
|
session.pending_continuation = True
|
|
session.pending_continuation_prompt = (
|
|
"Continue the task exactly where you left off; the session summary was reduced "
|
|
"this turn, rely on the visible conversation.")
|
|
logger.warning(f"Agent {session_id}: provider content-policy block on a turn carrying a {p_sent} history prefix; retrying with {session.history_prefix_mode}")
|
|
return
|
|
# The subscription lane declined and nothing is left to strip; fleet data says the same request passes on an API key (0 of 328 vs 4.4%), so a user who connected their own Anthropic key continues there, told in one line, instead of losing the ask (ENG-383).
|
|
p_twin = api_key_twin_model(session.model or "", load_settings())
|
|
if p_twin:
|
|
p_from = session.model
|
|
session.model = p_twin
|
|
session.lane_failover_from = p_from
|
|
session.needs_fresh_session = True
|
|
session.pending_continuation = True
|
|
session.pending_continuation_prompt = "Continue where you left off and finish the task, then answer in plain text."
|
|
p_notice = Message(
|
|
role="system",
|
|
content=("Claude declined this request on your subscription, so this answer is "
|
|
"finishing on your Anthropic API key (billed to that key). Your next "
|
|
"message goes back on the subscription."),
|
|
branch_id=session.active_branch_id,
|
|
)
|
|
absorb_repeat_card(session, p_notice)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id, "message": p_notice.model_dump(mode="json"),
|
|
})
|
|
flight_recorder.record_recovery(session_id, "lane_failover", session.model, 1)
|
|
logger.warning(f"Agent {session_id}: policy block on the subscription lane; failing over {p_from} -> {p_twin}")
|
|
return
|
|
p_retried = session.history_prefix_mode == "none"
|
|
friendly_msg = (
|
|
"The model provider declined this request (its automated policy filter flagged the "
|
|
"conversation's content)"
|
|
+ (", even after OpenSwarm retried without the session summary" if p_retried else "")
|
|
+ ". Retrying the same request won't change that. Rephrase your last message, or "
|
|
"start a fresh chat about this topic."
|
|
)
|
|
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|
|
elif is_out_of_tokens(e, extra_text=p_stderr_tail):
|
|
# The user's PROVIDER account is out of credits / over quota, distinct from OpenSwarm free-trial exhaustion above and from a 401 below ("credit balance too low", "insufficient_quota", "usage cap exceeded", OpenSwarm plan limit). Show a friendly card with the provider's reset hint when it gave one, instead of dropping to the raw-error blob in the else branch.
|
|
p_reset_hint = extract_reset_hint(f"{e!s}\n{p_stderr_tail}")
|
|
friendly_msg = (
|
|
"Your model provider reports you're out of credits or over your usage "
|
|
"limit" + (f" (resets {p_reset_hint})" if p_reset_hint else "") + ". Add "
|
|
"credits with your provider, switch to a different model, or connect "
|
|
"another option in Settings → Models."
|
|
)
|
|
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:out_of_credits", {
|
|
"session_id": session_id,
|
|
"message": friendly_msg,
|
|
"reset_hint": p_reset_hint,
|
|
"model": session.model,
|
|
})
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|
|
elif is_auth_error(e, extra_text=p_stderr_tail):
|
|
# Three sub-cases the user can hit, with distinct fixes: 1. "No credentials for provider: claude", user picked a -cc route but doesn't have Claude Pro/Max connected via 9Router. Tell them to either connect Claude Pro/Max OR pick a non--cc model. 2. OpenSwarm Pro 401, bearer expired. Reconnect. 3. Anthropic API key 401, wrong key. Re-enter.
|
|
p_model = (session.model or "").lower()
|
|
p_combined = f"{e!s}\n{p_stderr_tail}".lower()
|
|
p_codex_rotation = (
|
|
("codex/" in p_combined or "[codex/" in p_combined or p_model.startswith(("cx/", "gpt-")))
|
|
and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or has_auth_status(p_combined))
|
|
)
|
|
# Every sub lane gets ONE silent self-heal before any card; only a missing credential (config problem, retry fails identically) goes straight to the card. Codex waits out its rotation window first.
|
|
# A lane the router had ALREADY given up on before this turn is a dead credential, so the
|
|
# rotation story is false and the wait is doomed; say the true thing straight away.
|
|
if getattr(session, "lane_credential_dead", False):
|
|
from backend.apps.agents.manager.run.lane_preflight import RECONNECT_COPY
|
|
p_prov = (session.provider or "").lower()
|
|
friendly_msg = RECONNECT_COPY.get(
|
|
p_prov,
|
|
"This model's sign-in expired and could not be renewed. Reconnect it in Settings, "
|
|
"then Models. Waiting will not clear this one.")
|
|
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:auth_error", {
|
|
"session_id": session_id, "reason": "credential_expired",
|
|
"message": friendly_msg, "model": session.model,
|
|
})
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id, "message": error_msg.model_dump(mode="json"),
|
|
})
|
|
return
|
|
if "no credentials for provider" not in p_combined:
|
|
from backend.apps.agents.manager.streaming.auth_retry import try_auth_self_heal
|
|
if try_auth_self_heal(session, delay_s=75 if p_codex_rotation else 5):
|
|
if p_codex_rotation:
|
|
p_notice = Message(
|
|
role="system",
|
|
content="GPT subscription token just rotated (automatic, every couple minutes). Retrying your request automatically in about a minute, no action needed.",
|
|
branch_id=session.active_branch_id,
|
|
)
|
|
session.messages.append(p_notice)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": p_notice.model_dump(mode="json"),
|
|
})
|
|
logger.info(f"auth self-heal armed for {session_id} (codex_rotation={p_codex_rotation})")
|
|
return
|
|
# Codex/OpenAI subscription tokens rotate every ~2-3 minutes, the user sees the rotation window as a 401 with "reset after 1m 59s" or similar. Don't ask them to reconnect; just tell them to wait it out and retry.
|
|
if (
|
|
("codex/" in p_combined or "[codex/" in p_combined or p_model.startswith(("cx/", "gpt-")))
|
|
and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or has_auth_status(p_combined))
|
|
):
|
|
friendly_msg = (
|
|
"GPT subscription token just rotated, this is "
|
|
"automatic and resets every couple minutes. Send "
|
|
"your message again in ~1 minute and it'll go "
|
|
"through. (No need to reconnect anything.)"
|
|
)
|
|
reason = "codex_token_rotating"
|
|
elif "no credentials for provider" in p_combined:
|
|
friendly_msg = (
|
|
"Selected route requires Claude Pro / Max, but it's "
|
|
"not connected. Open Settings → Models and either "
|
|
"connect Claude Pro / Max, or switch the model to a "
|
|
"non-`-cc` variant (e.g. Claude Sonnet 4.6 instead "
|
|
"of Sonnet 4.6 -cc)."
|
|
)
|
|
reason = "claude_sub_not_connected"
|
|
elif (
|
|
"-cc" not in p_model
|
|
and getattr(load_settings(), "connection_mode", "own_key") == "openswarm-pro"
|
|
):
|
|
friendly_msg = (
|
|
"OpenSwarm Pro authentication failed. Your subscription "
|
|
"token may have expired even though the connection still "
|
|
"shows green. Open Settings → Models and click "
|
|
"Disconnect / Reconnect on Claude Pro / Max to refresh "
|
|
"the token."
|
|
)
|
|
reason = "openswarm_pro_auth_expired"
|
|
else:
|
|
friendly_msg = (
|
|
"Anthropic authentication failed. The API key or "
|
|
"subscription token for this model is invalid. Open "
|
|
"Settings → Models and re-enter the API key, or "
|
|
"reconnect Claude Pro / Max."
|
|
)
|
|
reason = "anthropic_auth_invalid"
|
|
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
try:
|
|
from backend.apps.service.client import submit_diagnostic
|
|
submit_diagnostic({
|
|
"kind": "model_error",
|
|
"subkind": "auth",
|
|
"model": session.model,
|
|
"provider": session.provider,
|
|
"error_preview": redact_for_telemetry(str(e), limit=400),
|
|
"flight": flight_recorder.build_envelope(session_id, "model_error", reason, session.model, "stream" if turn.current_turn_emitted else "spawn", -1),
|
|
})
|
|
except Exception:
|
|
logger.debug("submit_diagnostic auth failed", exc_info=True)
|
|
await ws_manager.send_to_session(session_id, "agent:auth_error", {
|
|
"session_id": session_id,
|
|
"reason": reason,
|
|
"message": friendly_msg,
|
|
"model": session.model,
|
|
})
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|
|
elif is_unknown_model_error(e, extra_text=p_stderr_tail):
|
|
# Upstream rejected the model code itself (e.g. Codex 1211 on a ChatGPT plan that lacks our GPT ids). Track it; the friendly "add an API key / pick another model" card is rendered frontend-side.
|
|
p_report_model_error("unknown_model", session_id, session, turn, e, p_stderr_tail)
|
|
error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|
|
elif is_router_unavailable_error(f"{e} {p_stderr_tail}"):
|
|
# Our own router is down. Naming it beats "unclassified": this is the one failure family
|
|
# where the fix is entirely on our side of the wire.
|
|
p_report_model_error("router_unavailable", session_id, session, turn, e, p_stderr_tail)
|
|
error_msg = Message(role="system", content=f"Error: {str(e)}", branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|
|
else:
|
|
# Track unclassified agent failures too so we stop flying blind on them.
|
|
p_report_model_error("unclassified", session_id, session, turn, e, p_stderr_tail)
|
|
# Every classified branch above is an EXTERNAL fact (auth, credits, capacity, certs) that a
|
|
# respawn cannot fix. Landing here instead means the failure may live in this session's own
|
|
# CLI state, and that state is replayed verbatim on every retry, which is how one chat bricks
|
|
# forever on "hit a snag" while its siblings are fine (ENG-258). Arm the proven fresh-session
|
|
# rebuild so the next ordinary send drops the resume transcript and respawns the client; the
|
|
# user's own retry becomes the cure instead of needing a manual branch.
|
|
session.needs_fresh_session = True
|
|
logger.info(f"Agent {session_id}: unclassified failure, next turn rebuilds on a fresh CLI session")
|
|
# The SDK's ProcessError masks the cause behind "Check stderr output for details"; append the scrubbed stderr tail so the card (and its analytics copy) names what actually broke instead of shipping a dead end.
|
|
p_card_text = f"Error: {str(e)}"
|
|
p_cause = redact_for_telemetry(p_stderr_tail, limit=400).strip()
|
|
if p_cause and "check stderr" in str(e).lower():
|
|
p_card_text += f"\n\nRuntime log tail:\n{p_cause}"
|
|
error_msg = Message(role="system", content=p_card_text, branch_id=session.active_branch_id)
|
|
absorb_repeat_card(session, error_msg)
|
|
await ws_manager.send_to_session(session_id, "agent:message", {
|
|
"session_id": session_id,
|
|
"message": error_msg.model_dump(mode="json"),
|
|
})
|