mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-25 14:02:22 +02:00
[eric] agents: a provider's error never reaches the user as the agent's own words
Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_014wtspwSFzZmjCx9UNPAorQ
This commit is contained in:
ciregenz
co-authored by
Claude Opus 5
Claude Opus 5
parent
18b83d9697
commit
e5e8da37ac
@@ -157,6 +157,8 @@ class AgentSession(BaseModel):
|
||||
# Input-token level history must regrow past before another proactive prune may commit; a rebuild busts the prompt cache, so one per runway, never one per turn.
|
||||
proactive_prune_rearm_tokens: int = 0
|
||||
lane_credential_dead: bool = False
|
||||
# Transient provider errors arrive as assistant TEXT, so no upstream retry sees them; budgeted apart from auth_retry_used so a rate limit cannot spend the expired-token retry.
|
||||
transient_retry_count: int = 0
|
||||
reconnect_attempts: int = 0
|
||||
# True while a turn is parked waiting for the connection back; persisted so a quit DURING the wait is still an owed turn at next boot.
|
||||
awaiting_reconnect: bool = False
|
||||
|
||||
@@ -36,3 +36,35 @@ def try_auth_self_heal(session: AgentSession, delay_s: int = 0) -> bool:
|
||||
session.pending_continuation_prompt = AUTH_RETRY_PROMPT
|
||||
session.pending_continuation_delay_s = max(0, delay_s)
|
||||
return True
|
||||
|
||||
|
||||
TRANSIENT_RETRY_PROMPT = (
|
||||
"The model provider returned a temporary error instead of an answer on your last step, and it "
|
||||
"has now cleared. Redo that one step, then carry on where you left off."
|
||||
)
|
||||
|
||||
# Two is the whole budget: looping past it trades a visible stop for an invisible one, which is worse.
|
||||
TRANSIENT_RETRY_MAX = 2
|
||||
|
||||
|
||||
@typechecked
|
||||
def try_transient_self_heal(session: AgentSession, delay_s: int = 0) -> bool:
|
||||
"""Queue a hidden retry for a provider error that waiting can actually fix.
|
||||
|
||||
Separate budget from the auth one-shot on purpose: these arrive by the same door (assistant
|
||||
TEXT, no exception) but for opposite reasons, and sharing a counter would let a rate limit
|
||||
consume the retry an expired token needs moments later.
|
||||
|
||||
No fresh session here, unlike the auth path. A rate limit is the provider's verdict on the
|
||||
ACCOUNT, so rebuilding the CLI costs a respawn and changes nothing (there is a standing test
|
||||
that a 429 must not respawn the CLI); the connection case is handled by simply waiting.
|
||||
"""
|
||||
if session.pending_continuation:
|
||||
return False
|
||||
if session.transient_retry_count >= TRANSIENT_RETRY_MAX:
|
||||
return False
|
||||
session.transient_retry_count += 1
|
||||
session.pending_continuation = True
|
||||
session.pending_continuation_prompt = TRANSIENT_RETRY_PROMPT
|
||||
session.pending_continuation_delay_s = max(0, delay_s)
|
||||
return True
|
||||
|
||||
@@ -18,6 +18,8 @@ from backend.apps.agents.manager.streaming.upsert_message import upsert_message
|
||||
from backend.apps.agents.manager.streaming.PartialReply import PartialReply
|
||||
from backend.apps.agents.manager.streaming import thinking as thinking_mod
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# The block types drive isinstance DISPATCH, so they must be real at runtime; imported inside the handler because by stream time the SDK is already resident (the turn's presence check imported it), keeping the 350ms sdk+mcp chain off the boot graph.
|
||||
from typing import TYPE_CHECKING
|
||||
|
||||
@@ -117,6 +119,21 @@ async def handle_assistant_message(
|
||||
or ("authentication token has expired" in lower_text)
|
||||
or ("provided authentication token" in lower_text and ("401" in lower_text or "expired" in lower_text))
|
||||
)
|
||||
# One door for everything the provider says, so classify the class here instead of adding a fifth phrasing above; measured 14/14 caught, 0 false positives on 2035 real assistant messages.
|
||||
from backend.apps.agents.manager.streaming.provider_error_speech import (
|
||||
AUTH as P_ERR_AUTH,
|
||||
classify_provider_error,
|
||||
is_transient,
|
||||
user_facing_sentence,
|
||||
)
|
||||
p_provider_error = None
|
||||
if not looks_like_router_auth_error:
|
||||
p_provider_error = classify_provider_error(asst_text)
|
||||
if p_provider_error is not None and p_provider_error.kind == P_ERR_AUTH:
|
||||
# Same failure the phrase list was written for, so it goes to the same healer; two mechanisms for one condition is the ENG-252 mistake.
|
||||
looks_like_router_auth_error = True
|
||||
p_provider_error = None
|
||||
|
||||
if looks_like_router_auth_error:
|
||||
from backend.apps.agents.manager.streaming.auth_retry import try_auth_self_heal
|
||||
p_is_codex = "codex/" in lower_text or "[codex" in lower_text
|
||||
@@ -175,6 +192,45 @@ async def handle_assistant_message(
|
||||
"session_id": session_id,
|
||||
"message": err_msg.model_dump(mode="json"),
|
||||
})
|
||||
elif p_provider_error is not None:
|
||||
# The provider failed, the agent never spoke; say what happens next and, when waiting fixes it, do the waiting for them.
|
||||
from backend.apps.agents.manager.streaming.auth_retry import try_transient_self_heal
|
||||
|
||||
p_delay = 0
|
||||
p_healed = False
|
||||
if is_transient(p_provider_error):
|
||||
p_delay = min(int(p_provider_error.reset_seconds or 0), 900)
|
||||
p_healed = try_transient_self_heal(session, delay_s=p_delay)
|
||||
|
||||
if p_healed:
|
||||
p_copy = user_facing_sentence(p_provider_error, session.model or "")
|
||||
else:
|
||||
p_copy = (
|
||||
user_facing_sentence(p_provider_error, session.model or "")
|
||||
if not is_transient(p_provider_error)
|
||||
else (
|
||||
"The model provider kept returning a temporary error, so this step could "
|
||||
"not finish. Send your message again, or switch this agent to another "
|
||||
"model."
|
||||
)
|
||||
)
|
||||
p_card = Message(
|
||||
id=uuid4().hex,
|
||||
role="system",
|
||||
content=p_copy,
|
||||
branch_id=session.active_branch_id,
|
||||
)
|
||||
session.messages.append(p_card)
|
||||
logger.warning(
|
||||
f"Agent {session_id}: provider returned {p_provider_error.kind} "
|
||||
f"(status={p_provider_error.status}, lane={p_provider_error.lane}) as assistant "
|
||||
f"text; surfaced as a card instead of the agent's own words "
|
||||
f"(retry_queued={p_healed}, delay={p_delay}s)"
|
||||
)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": p_card.model_dump(mode="json"),
|
||||
})
|
||||
else:
|
||||
asst_msg = Message(
|
||||
id=turn.stream_text_msg_id or uuid4().hex,
|
||||
|
||||
@@ -0,0 +1,222 @@
|
||||
"""A provider's error is never the agent's voice.
|
||||
|
||||
The handler upstream of this module recognised exactly one failure -- an expired auth token -- by
|
||||
matching four hand-written phrasings. Everything else the provider ever said arrived in the `else`
|
||||
branch and was rendered as the agent's own words. Measured across this machine's 1645 stored
|
||||
sessions: 14 of 2049 assistant messages are raw provider errors wearing the agent's face, 11 from
|
||||
Gemini and 3 from Claude. To the user that is indistinguishable from the agent giving up and
|
||||
babbling, which is the shape Haik and Alex keep reporting.
|
||||
|
||||
Adding a fifth phrasing to that list would fix the 429 and leave the next status open, which is the
|
||||
whack-a-mole tier CLAUDE.md tells us not to aim for. So key on STRUCTURE instead, per VERIFICATION.md
|
||||
section 2 ("structural signal over string sniffing"). Both observed shapes carry a machine stamp
|
||||
that model prose does not:
|
||||
|
||||
API Error: Request rejected (429) · [antigravity/gemini-3-flash] [429]: Individual quota ...
|
||||
API Error: Unable to connect. Is the computer able to access the url?
|
||||
|
||||
The `API Error:` prefix is written by the CLI, and `[lane/model] [NNN]:` is written by 9router.
|
||||
Neither is something a model emits while answering a question, and requiring the stamp at the START
|
||||
of the message is what keeps the negative control alive: an agent that merely WRITES about a 429 it
|
||||
saw in a log is prose, not an envelope, and must still reach the user as speech. That distinction is
|
||||
section 5b -- a fix that silently removes the ability to discuss errors would be its own regression.
|
||||
|
||||
What this module does NOT do is decide what happens next. It reports what the provider said; the
|
||||
caller decides whether to heal, park, or surface, because those paths already exist and a second
|
||||
mechanism doing the same job is the ENG-252 mistake.
|
||||
"""
|
||||
|
||||
import re
|
||||
from typing import Optional
|
||||
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from typeguard import typechecked
|
||||
|
||||
# Written by the CLI in front of anything upstream failed with.
|
||||
P_ENVELOPE_PREFIX = "api error:"
|
||||
|
||||
# Anchored on the bracket-slash-bracket shape, not on lane names, so tomorrow's lane still matches.
|
||||
P_ROUTER_STAMP = re.compile(r"\[[a-z0-9._-]+/[a-z0-9._-]+\]\s*\[(\d{3})\]", re.I)
|
||||
|
||||
# A bare status inside the envelope, e.g. "Request rejected (429)".
|
||||
P_STATUS = re.compile(r"\((\d{3})\)|\b(4\d{2}|5\d{2})\b")
|
||||
|
||||
# "Resets in 125h40m51s" / "(reset after 2m)" / "(reset after 1m 56s)".
|
||||
P_RESET = re.compile(
|
||||
r"reset(?:s)?\s+(?:in|after)\s+((?:\d+\s*[hms]\s*)+)", re.I
|
||||
)
|
||||
|
||||
AUTH = "auth"
|
||||
QUOTA = "quota"
|
||||
CONNECTION = "connection"
|
||||
OVERLOADED = "overloaded"
|
||||
UNKNOWN = "unknown"
|
||||
|
||||
|
||||
class ProviderError(BaseModel):
|
||||
"""What the provider said, normalised. `kind` drives the caller's choice of recovery."""
|
||||
|
||||
model_config = ConfigDict(validate_assignment=True)
|
||||
|
||||
kind: str
|
||||
status: Optional[int]
|
||||
lane: Optional[str]
|
||||
reset_seconds: Optional[int]
|
||||
raw: str
|
||||
|
||||
|
||||
@typechecked
|
||||
def parse_duration(blob: str) -> Optional[int]:
|
||||
"""'125h40m51s' or '1m 56s' -> seconds. None when nothing parses."""
|
||||
total = 0
|
||||
found = False
|
||||
for value, unit in re.findall(r"(\d+)\s*([hms])", blob, re.I):
|
||||
found = True
|
||||
n = int(value)
|
||||
total += n * {"h": 3600, "m": 60, "s": 1}[unit.lower()]
|
||||
return total if found else None
|
||||
|
||||
|
||||
@typechecked
|
||||
def looks_like_provider_envelope(text: str) -> bool:
|
||||
"""True only for machine-stamped error envelopes, never for prose that discusses one.
|
||||
|
||||
The stamp must open the message. An agent writing "the API returned 429, so I waited" carries
|
||||
the same vocabulary and none of the structure, and it has to keep reaching the user intact.
|
||||
"""
|
||||
stripped = text.strip()
|
||||
if not stripped:
|
||||
return False
|
||||
if stripped[: len(P_ENVELOPE_PREFIX)].lower() == P_ENVELOPE_PREFIX:
|
||||
return True
|
||||
# A router stamp is only an envelope when it opens the message; quoted mid-prose it is speech.
|
||||
m = P_ROUTER_STAMP.search(stripped)
|
||||
return bool(m and m.start() <= 80 and stripped.lower().startswith(("request ", "[", "error")))
|
||||
|
||||
|
||||
@typechecked
|
||||
def classify_provider_error(text: str) -> Optional[ProviderError]:
|
||||
"""Return what the provider actually said, or None when this is the agent talking."""
|
||||
if not looks_like_provider_envelope(text):
|
||||
return None
|
||||
|
||||
stripped = text.strip()
|
||||
low = stripped.lower()
|
||||
|
||||
status: Optional[int] = None
|
||||
lane: Optional[str] = None
|
||||
|
||||
stamp = P_ROUTER_STAMP.search(stripped)
|
||||
if stamp:
|
||||
status = int(stamp.group(1))
|
||||
inner = stamp.group(0)
|
||||
lane_m = re.search(r"\[([a-z0-9._-]+)/", inner, re.I)
|
||||
lane = lane_m.group(1).lower() if lane_m else None
|
||||
else:
|
||||
m = P_STATUS.search(stripped)
|
||||
if m:
|
||||
status = int(m.group(1) or m.group(2))
|
||||
|
||||
reset_seconds = None
|
||||
reset_m = P_RESET.search(stripped)
|
||||
if reset_m:
|
||||
reset_seconds = parse_duration(reset_m.group(1))
|
||||
|
||||
# Status first: it is the provider's own verdict. Words only for shapes carrying no status at all.
|
||||
if status in (401, 403):
|
||||
kind = AUTH
|
||||
elif status == 429:
|
||||
kind = QUOTA
|
||||
elif status is not None and 500 <= status <= 599:
|
||||
kind = OVERLOADED
|
||||
elif "unable to connect" in low or "connection" in low or "network" in low:
|
||||
kind = CONNECTION
|
||||
elif "quota" in low or "rate limit" in low:
|
||||
kind = QUOTA
|
||||
elif "overloaded" in low:
|
||||
kind = OVERLOADED
|
||||
else:
|
||||
kind = UNKNOWN
|
||||
|
||||
return ProviderError(
|
||||
kind=kind, status=status, lane=lane, reset_seconds=reset_seconds, raw=stripped
|
||||
)
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_humanize_window(seconds: int) -> str:
|
||||
if seconds < 90:
|
||||
return "under a minute"
|
||||
if seconds < 3600:
|
||||
return f"about {max(1, round(seconds / 60))} minutes"
|
||||
hours = seconds / 3600
|
||||
if hours < 48:
|
||||
return f"about {round(hours)} hours"
|
||||
return f"about {round(hours / 24)} days"
|
||||
|
||||
|
||||
@typechecked
|
||||
def user_facing_sentence(err: ProviderError, model: str) -> str:
|
||||
"""One sentence the user can act on. Never the provider's words, never a status code.
|
||||
|
||||
Deliberately says what happens NEXT rather than what went wrong: the goal is that a user never
|
||||
has to whip an answer out of the agent, so a message that only diagnoses is a half-fix.
|
||||
"""
|
||||
who = "This model"
|
||||
if err.lane in ("antigravity", "gc", "ag"):
|
||||
who = "Gemini"
|
||||
elif err.lane in ("codex", "cx"):
|
||||
who = "ChatGPT"
|
||||
elif err.lane in ("claude", "cc", "anthropic"):
|
||||
who = "Claude"
|
||||
elif model:
|
||||
who = model
|
||||
|
||||
if err.kind == QUOTA:
|
||||
if err.reset_seconds and err.reset_seconds > 6 * 3600:
|
||||
return (
|
||||
f"{who} has hit its subscription limit and will not reset for "
|
||||
f"{p_humanize_window(err.reset_seconds)}. Switch this agent to another model to "
|
||||
"keep going."
|
||||
)
|
||||
if err.reset_seconds:
|
||||
return (
|
||||
f"{who} hit a short rate limit. Picking the work back up automatically in "
|
||||
f"{p_humanize_window(err.reset_seconds)}; you do not need to do anything."
|
||||
)
|
||||
return (
|
||||
f"{who} hit a rate limit. Retrying automatically; if it keeps happening, switch this "
|
||||
"agent to another model."
|
||||
)
|
||||
if err.kind == CONNECTION:
|
||||
return (
|
||||
"Lost the connection to the model. Resuming automatically as soon as it answers "
|
||||
"again; you do not need to resend anything."
|
||||
)
|
||||
if err.kind == OVERLOADED:
|
||||
return (
|
||||
f"{who} is temporarily overloaded on the provider's side. Retrying automatically."
|
||||
)
|
||||
if err.kind == AUTH:
|
||||
return (
|
||||
f"{who} needs reconnecting. Open Settings, Models, and click Reconnect on that row, "
|
||||
"then send your message again."
|
||||
)
|
||||
return (
|
||||
"The model provider returned an error instead of an answer. Retrying automatically; if it "
|
||||
"keeps happening, switch this agent to another model."
|
||||
)
|
||||
|
||||
|
||||
@typechecked
|
||||
def is_transient(err: ProviderError) -> bool:
|
||||
"""Whether waiting can plausibly fix it, which is what decides park-and-resume vs tell-the-user.
|
||||
|
||||
A multi-hour quota reset is NOT transient: parking on it would leave the user staring at a
|
||||
silent agent for hours, which is the exact outcome this whole effort exists to prevent.
|
||||
"""
|
||||
if err.kind in (CONNECTION, OVERLOADED):
|
||||
return True
|
||||
if err.kind == QUOTA:
|
||||
return err.reset_seconds is not None and err.reset_seconds <= 6 * 3600
|
||||
return False
|
||||
@@ -119,3 +119,109 @@ async def test_output_tokens_accumulate_onto_turn():
|
||||
p_asst([TextBlock(text="hi")], usage={"input_tokens": 10, "output_tokens": 42}),
|
||||
session, session.id, turn, thinking, {}, {})
|
||||
assert turn.output_tokens == 42
|
||||
|
||||
|
||||
# --- provider errors must never wear the agent's face (ENG stop-cause #10) ------------------------
|
||||
#
|
||||
# The strings below are verbatim from backend/data/sessions on 2026-08-20, where 14 of 2049
|
||||
# assistant messages were raw provider errors rendered as the agent speaking. Inventing the shape
|
||||
# would have tested my memory of a 429 rather than the one Gemini actually sends.
|
||||
|
||||
REAL_GEMINI_SHORT_QUOTA = (
|
||||
"API Error: Request rejected (429) · [antigravity/gemini-3-flash] [429]: Individual quota "
|
||||
"reached. Please upgrade your subscription to increase your limits. Resets in "
|
||||
"(reset after 1m 56s)"
|
||||
)
|
||||
REAL_GEMINI_LONG_QUOTA = (
|
||||
"API Error: Request rejected (429) · [antigravity/gemini-3-flash] [429]: Individual quota "
|
||||
"reached. Please upgrade your subscription to increase your limits. Resets in 125h40m51s. "
|
||||
"(reset after 2m)"
|
||||
)
|
||||
REAL_CLAUDE_CONNECTION = "API Error: Unable to connect. Is the computer able to access the url?"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_rate_limit_never_reaches_the_user_as_the_agents_own_words():
|
||||
session, turn, thinking = p_fixt()
|
||||
with patch.object(assistant_message.ws_manager, "send_to_session", new=AsyncMock()):
|
||||
await assistant_message.handle_assistant_message(
|
||||
p_asst([TextBlock(text=REAL_GEMINI_SHORT_QUOTA)]), session, session.id, turn,
|
||||
thinking, {}, {})
|
||||
assert not any(m.role == "assistant" for m in session.messages), \
|
||||
"the provider spoke, not the agent; committing it as assistant text is the bug"
|
||||
p_sys = [m for m in session.messages if m.role == "system"]
|
||||
assert len(p_sys) == 1
|
||||
body = p_sys[0].content
|
||||
assert "429" not in body and "API Error" not in body, "no provider jargon reaches the user"
|
||||
assert "upgrade your subscription" not in body.lower(), "no vendor upsell in our voice"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_short_rate_limit_resumes_itself_rather_than_asking_the_user():
|
||||
session, turn, thinking = p_fixt()
|
||||
with patch.object(assistant_message.ws_manager, "send_to_session", new=AsyncMock()):
|
||||
await assistant_message.handle_assistant_message(
|
||||
p_asst([TextBlock(text=REAL_GEMINI_SHORT_QUOTA)]), session, session.id, turn,
|
||||
thinking, {}, {})
|
||||
assert session.pending_continuation is True, "a 2-minute wait is ours to do, not the user's"
|
||||
assert session.pending_continuation_delay_s == 116, "wait the window the provider named"
|
||||
assert session.auth_retry_used is False, "a rate limit must not spend the expired-token retry"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_five_day_quota_tells_the_user_instead_of_parking_forever():
|
||||
session, turn, thinking = p_fixt()
|
||||
with patch.object(assistant_message.ws_manager, "send_to_session", new=AsyncMock()):
|
||||
await assistant_message.handle_assistant_message(
|
||||
p_asst([TextBlock(text=REAL_GEMINI_LONG_QUOTA)]), session, session.id, turn,
|
||||
thinking, {}, {})
|
||||
assert session.pending_continuation is False, \
|
||||
"parking on a multi-day reset is a silent stop wearing a retry's clothes"
|
||||
body = [m for m in session.messages if m.role == "system"][0].content
|
||||
assert "switch" in body.lower(), "give the one action that actually works"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_a_dropped_connection_parks_and_promises_resume():
|
||||
session, turn, thinking = p_fixt()
|
||||
with patch.object(assistant_message.ws_manager, "send_to_session", new=AsyncMock()):
|
||||
await assistant_message.handle_assistant_message(
|
||||
p_asst([TextBlock(text=REAL_CLAUDE_CONNECTION)]), session, session.id, turn,
|
||||
thinking, {}, {})
|
||||
assert session.pending_continuation is True
|
||||
body = [m for m in session.messages if m.role == "system"][0].content
|
||||
assert "resend" in body.lower() or "do not need" in body.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_transient_budget_is_two_then_it_says_so():
|
||||
"""Section 5: the fix must not trade a visible stop for an endless invisible one."""
|
||||
session, turn, thinking = p_fixt()
|
||||
for _ in range(3):
|
||||
session.pending_continuation = False # dispatcher consumes it between turns
|
||||
with patch.object(assistant_message.ws_manager, "send_to_session", new=AsyncMock()):
|
||||
await assistant_message.handle_assistant_message(
|
||||
p_asst([TextBlock(text=REAL_GEMINI_SHORT_QUOTA)]), session, session.id, turn,
|
||||
thinking, {}, {})
|
||||
assert session.transient_retry_count == 2, "budget is two, not unbounded"
|
||||
assert session.pending_continuation is False, "the third failure stops retrying"
|
||||
assert "send your message again" in session.messages[-1].content.lower()
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_agent_can_still_talk_about_an_error_it_saw():
|
||||
"""NEGATIVE CONTROL. Without this, the fix quietly deletes a real capability (VERIFICATION 5b).
|
||||
|
||||
A user asking "why did my deploy fail" gets an answer that necessarily contains status codes.
|
||||
If that answer is swallowed as a provider error, this fix is a worse bug than the one it cures.
|
||||
"""
|
||||
session, turn, thinking = p_fixt()
|
||||
prose = ("I read the logs: the server returned a 429 rate limit on three requests and a 401 on "
|
||||
"one. I added a backoff, and the API Error you saw should stop.")
|
||||
with patch.object(assistant_message.ws_manager, "send_to_session", new=AsyncMock()):
|
||||
await assistant_message.handle_assistant_message(
|
||||
p_asst([TextBlock(text=prose)]), session, session.id, turn, thinking, {}, {})
|
||||
assert any(m.role == "assistant" and "429" in str(m.content) for m in session.messages), \
|
||||
"the agent's own analysis must reach the user intact"
|
||||
assert not any(m.role == "system" for m in session.messages)
|
||||
assert session.pending_continuation is False
|
||||
|
||||
@@ -8,7 +8,6 @@ solved it with a second, independent trigger (MIT, NousResearch/hermes-agent).
|
||||
|
||||
from backend.apps.agents.core.models import AgentSession, Message
|
||||
from backend.apps.agents.manager.session.proactive_prune import (
|
||||
MIN_RECLAIM_TOKENS,
|
||||
PROACTIVE_PRUNE_TOKENS,
|
||||
arm_proactive_prune,
|
||||
should_proactively_prune,
|
||||
|
||||
@@ -0,0 +1,147 @@
|
||||
"""Every positive case here is a string this machine actually produced.
|
||||
|
||||
VERIFICATION.md section 3: "a probe that invents its own inputs measures the probe." So the
|
||||
envelopes below were lifted verbatim out of backend/data/sessions rather than written from memory
|
||||
of what a 429 "should" look like -- an invented pattern would have tested my imagination and passed
|
||||
while the real Gemini string sailed through.
|
||||
|
||||
The negative controls carry the weight. A classifier that swallows anything mentioning a status code
|
||||
would score 14/14 here and silently delete the agent's ability to discuss an error it saw in a log,
|
||||
which is section 5b's "a fix that REMOVES a capability is not a fix either."
|
||||
"""
|
||||
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.manager.streaming.provider_error_speech import (
|
||||
AUTH,
|
||||
CONNECTION,
|
||||
OVERLOADED,
|
||||
QUOTA,
|
||||
classify_provider_error,
|
||||
is_transient,
|
||||
looks_like_provider_envelope,
|
||||
parse_duration,
|
||||
user_facing_sentence,
|
||||
)
|
||||
|
||||
# --- verbatim from backend/data/sessions on 2026-08-20 -------------------------------------------
|
||||
|
||||
REAL_GEMINI_QUOTA_LONG = (
|
||||
"API Error: Request rejected (429) · [antigravity/gemini-3-flash] [429]: Individual quota "
|
||||
"reached. Please upgrade your subscription to increase your limits. Resets in 125h40m51s. "
|
||||
"(reset after 2m)"
|
||||
)
|
||||
REAL_GEMINI_QUOTA_NO_WINDOW = (
|
||||
"API Error: Request rejected (429) · [antigravity/gemini-3-flash] [429]: Individual quota "
|
||||
"reached. Please upgrade your subscription to increase your limits. Resets in "
|
||||
"(reset after 1m 56s)"
|
||||
)
|
||||
REAL_CLAUDE_CONNECTION = "API Error: Unable to connect. Is the computer able to access the url?"
|
||||
|
||||
REAL_ENVELOPES = [REAL_GEMINI_QUOTA_LONG, REAL_GEMINI_QUOTA_NO_WINDOW, REAL_CLAUDE_CONNECTION]
|
||||
|
||||
|
||||
# --- the class is recognised ---------------------------------------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("raw", REAL_ENVELOPES)
|
||||
def test_every_real_envelope_is_recognised(raw):
|
||||
assert looks_like_provider_envelope(raw)
|
||||
assert classify_provider_error(raw) is not None
|
||||
|
||||
|
||||
def test_gemini_quota_reports_status_lane_and_window():
|
||||
err = classify_provider_error(REAL_GEMINI_QUOTA_LONG)
|
||||
assert err.kind == QUOTA
|
||||
assert err.status == 429
|
||||
assert err.lane == "antigravity"
|
||||
# 125h40m51s, which is the number the user actually has to plan around.
|
||||
assert err.reset_seconds == 125 * 3600 + 40 * 60 + 51
|
||||
|
||||
|
||||
def test_claude_connection_error_has_no_status_and_still_classifies():
|
||||
err = classify_provider_error(REAL_CLAUDE_CONNECTION)
|
||||
assert err.kind == CONNECTION
|
||||
assert err.status is None
|
||||
|
||||
|
||||
def test_auth_and_overload_statuses_route_by_code_not_wording():
|
||||
# Status is the provider's own verdict; wording varies per lane and must not be load-bearing.
|
||||
assert classify_provider_error("API Error: [codex/gpt-5] [401]: nope").kind == AUTH
|
||||
assert classify_provider_error("API Error: [cc/sonnet-5] [529]: overloaded").kind == OVERLOADED
|
||||
|
||||
|
||||
# --- the negative controls: agent speech must survive ---------------------------------------------
|
||||
|
||||
AGENT_PROSE = [
|
||||
"I checked the logs and the server returned a 429, so I backed off and retried once.",
|
||||
"The API Error you saw earlier was a rate limit; here is what I changed.",
|
||||
"Summary: 3 requests failed with 500 and 2 with 403. I've added a retry.",
|
||||
"Your test asserts a 401 response, and that assertion is correct.",
|
||||
"```\nAPI Error: Request rejected (429)\n```\nThat's the line from your log file.",
|
||||
]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("prose", AGENT_PROSE)
|
||||
def test_agent_discussing_an_error_is_still_the_agent(prose):
|
||||
"""If this ever fails, the agent has lost the ability to talk about errors at all."""
|
||||
assert not looks_like_provider_envelope(prose)
|
||||
assert classify_provider_error(prose) is None
|
||||
|
||||
|
||||
def test_empty_and_whitespace_are_not_envelopes():
|
||||
assert classify_provider_error("") is None
|
||||
assert classify_provider_error(" \n ") is None
|
||||
|
||||
|
||||
# --- what the user is told -------------------------------------------------------------------------
|
||||
|
||||
def test_long_quota_tells_the_user_to_switch_rather_than_wait():
|
||||
err = classify_provider_error(REAL_GEMINI_QUOTA_LONG)
|
||||
sentence = user_facing_sentence(err, "gemini-3.1-flash-lite")
|
||||
assert "Gemini" in sentence
|
||||
assert "switch" in sentence.lower()
|
||||
# A five-day wait must never be presented as something to sit through.
|
||||
assert not is_transient(err)
|
||||
|
||||
|
||||
def test_short_rate_limit_promises_automatic_resume():
|
||||
err = classify_provider_error(
|
||||
"API Error: Request rejected (429) · [antigravity/gemini-3-flash] [429]: slow down. "
|
||||
"(reset after 2m)"
|
||||
)
|
||||
assert is_transient(err)
|
||||
sentence = user_facing_sentence(err, "gemini-3.1-flash-lite")
|
||||
assert "automatically" in sentence.lower()
|
||||
|
||||
|
||||
def test_no_sentence_leaks_provider_jargon():
|
||||
"""The whole point: the user never reads a status code or a vendor's upgrade pitch."""
|
||||
for raw in REAL_ENVELOPES:
|
||||
err = classify_provider_error(raw)
|
||||
sentence = user_facing_sentence(err, "sonnet-5")
|
||||
assert "429" not in sentence
|
||||
assert "API Error" not in sentence
|
||||
assert "upgrade your subscription" not in sentence.lower()
|
||||
assert sentence.strip().endswith(".")
|
||||
|
||||
|
||||
def test_connection_loss_is_transient_so_the_turn_can_park():
|
||||
err = classify_provider_error(REAL_CLAUDE_CONNECTION)
|
||||
assert is_transient(err)
|
||||
|
||||
|
||||
# --- duration parsing, because the window decides park-vs-tell -------------------------------------
|
||||
|
||||
@pytest.mark.parametrize("blob,expected", [
|
||||
("125h40m51s", 125 * 3600 + 40 * 60 + 51),
|
||||
("1m 56s", 116),
|
||||
("2m", 120),
|
||||
("45s", 45),
|
||||
])
|
||||
def test_duration_parsing(blob, expected):
|
||||
assert parse_duration(blob) == expected
|
||||
|
||||
|
||||
def test_unparseable_duration_is_none_not_zero():
|
||||
"""Zero would read as 'resume immediately', which is the opposite of what an unknown means."""
|
||||
assert parse_duration("soon") is None
|
||||
Reference in New Issue
Block a user