diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index 409efe46..4e85e4ec 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -244,6 +244,7 @@ class AgentManager(SessionLifecycle, SessionHistory, SessionPersistence, Messagi return session.status = "running" + session.last_failure_kind = None # Resolve the model id now so every closure (approval hook, tool executed handler, etc.) has both the short name and the 9Router-prefixed id available without re-resolving. The short name is what the user sees; the router id is what 9Router reports its per-model counters under. from backend.apps.agents.providers.registry import ( diff --git a/backend/apps/agents/core/models.py b/backend/apps/agents/core/models.py index 99758bc2..b436f835 100644 --- a/backend/apps/agents/core/models.py +++ b/backend/apps/agents/core/models.py @@ -169,6 +169,8 @@ class AgentSession(BaseModel): # moment they send again. Failing over is for finishing THE CURRENT ASK; leaving the chat on a # metered key forever would bill them per token with one line said about it, once (ENG-383). lane_failover_from: Optional[str] = None + # How the last turn ended when it ended on a card; a parent reads it off a child so a declined task is not re-dispatched. + last_failure_kind: Optional[Literal["policy_block"]] = None # Consecutive dirty deaths this session was MID-TURN for; the crash auto-resume breaker (hermes #30719 pairing: auto-resume must never outrun its circuit breaker). crash_interrupt_count: int = 0 # Outage rounds spent on this ask: the in-turn ladder covers only 335s, and the work is checkpointed, so a longer drop is waited out rather than ending the task. diff --git a/backend/apps/agents/manager/run/TurnRunner.py b/backend/apps/agents/manager/run/TurnRunner.py index 5cd39465..63b9a680 100644 --- a/backend/apps/agents/manager/run/TurnRunner.py +++ b/backend/apps/agents/manager/run/TurnRunner.py @@ -50,7 +50,8 @@ class TurnRunner(AgentManagerProtocol): # Deliberate faults, so the guards below get drilled instead of waited for. Inert unless # OSW_FAULT names them; a shipped build never sets it. Raised HERE because this is the same # door a real provider failure comes through, so the drill exercises the real recovery. - if p_fault_armed("policy_block"): + # Budgeted like auth_401 (OSW_FAULT_FIRES=N), so a drill can block ONE turn and watch the failover finish the ask. + if p_fault_fire("policy_block"): # The provider's real wording, so the drill hits the same classifier a field block does. raise RuntimeError( "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":" diff --git a/backend/apps/agents/manager/run/handle_run_error.py b/backend/apps/agents/manager/run/handle_run_error.py index e722e9c1..fa3bb842 100644 --- a/backend/apps/agents/manager/run/handle_run_error.py +++ b/backend/apps/agents/manager/run/handle_run_error.py @@ -47,7 +47,7 @@ from backend.apps.agents.core.extract_reset_hint import extract_reset_hint from backend.apps.agents.core.redact_for_telemetry import redact_for_telemetry from backend.apps.agents.core import flight_recorder from backend.apps.agents.manager.run.empty_finish import count_tool_calls -from backend.apps.agents.session_credential import api_key_twin_model +from backend.apps.agents.session_credential import api_key_twin_model, policy_block_sibling logger = logging.getLogger(__name__) @@ -502,8 +502,35 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, "this turn, rely on the visible conversation.") logger.warning(f"Agent {session_id}: provider content-policy block on a turn carrying a {p_sent} history prefix; retrying with {session.history_prefix_mode}") return + # Nothing left to strip. The classifier declines Opus 5 where it passes Opus 4.8 (fleet, 2026-09-07: + # 78 blocks vs 0 across a comparable error volume), so the step finishes on the same lane's sibling + # first: same subscription, same wallet, nothing billed. One ask, announced, and the next real + # message puts the chat back on the model the user chose (Messaging restores lane_failover_from). + p_sibling = policy_block_sibling(session.model or "") if not session.lane_failover_from else None + if p_sibling: + p_from = session.model + session.model = p_sibling + session.lane_failover_from = p_from + session.needs_fresh_session = True + session.pending_continuation = True + session.pending_continuation_prompt = "Continue where you left off and finish the task, then answer in plain text." + p_notice = Message( + role="system", + content=("Anthropic's filter declined this request on Claude Opus 5, which it does far more often " + "than on other models. This step is finishing on Claude Opus 4.8 on the same subscription; " + "your next message goes back to Opus 5. If it keeps happening, switch this chat to Opus 4.8 or Sonnet 5."), + branch_id=session.active_branch_id, + ) + absorb_repeat_card(session, p_notice) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, "message": p_notice.model_dump(mode="json"), + }) + flight_recorder.record_recovery(session_id, "policy_sibling", session.model, 1) + logger.warning(f"Agent {session_id}: policy block on {p_from}; finishing this ask on the same-lane sibling {p_sibling}") + return # The subscription lane declined and nothing is left to strip; fleet data says the same request passes on an API key (0 of 328 vs 4.4%), so a user who connected their own Anthropic key continues there, told in one line, instead of losing the ask (ENG-383). - p_twin = api_key_twin_model(session.model or "", load_settings()) + # One failover per ask, whichever door it took: a borrowed sibling that is blocked again ends on the card, never on a second borrow that bills the key. + p_twin = api_key_twin_model(session.model or "", load_settings()) if not session.lane_failover_from else None if p_twin: p_from = session.model session.model = p_twin @@ -526,6 +553,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, logger.warning(f"Agent {session_id}: policy block on the subscription lane; failing over {p_from} -> {p_twin}") return p_retried = session.history_prefix_mode == "none" + session.last_failure_kind = "policy_block" friendly_msg = ( "The model provider declined this request (its automated policy filter flagged the " "conversation's content)" diff --git a/backend/apps/agents/session_credential.py b/backend/apps/agents/session_credential.py index e482f43c..2e7b8a06 100644 --- a/backend/apps/agents/session_credential.py +++ b/backend/apps/agents/session_credential.py @@ -189,6 +189,22 @@ def write_would_suicide(field: str, new_value: Any, powering: PoweringCredential return False +# Anthropic's abuse classifier declines Opus 5 requests it passes on every other model: 78 of 84 fleet +# blocks in the fortnight to 2026-09-07 were on opus-5 against 0 on opus-4-8 across a comparable error +# volume, and Anthropic's own block text says to switch models. The sibling is the same vendor on the +# SAME lane and wallet, so a step can finish there without spending anything the user did not connect. +POLICY_BLOCK_SIBLING: dict[str, str] = { + "opus-5": "opus-4-8", + "opus-5-cc": "opus-4-8-cc", + "opus-5-api": "opus-4-8-api", +} + + +def policy_block_sibling(model_value: str) -> Optional[str]: + """The same-lane model a policy-blocked step may finish on, or None when there is no such sibling.""" + return POLICY_BLOCK_SIBLING.get(model_value) + + def api_key_twin_model(model_value: str, settings: AppSettings) -> Optional[str]: """The same Claude model on the user's OWN Anthropic API key, when this run was on a subscription lane and such a key is configured; None otherwise. Never the OpenSwarm Pro pool (not their money to diff --git a/backend/tests/test_content_policy_block.py b/backend/tests/test_content_policy_block.py index a398122e..70af6a89 100644 --- a/backend/tests/test_content_policy_block.py +++ b/backend/tests/test_content_policy_block.py @@ -82,6 +82,7 @@ def test_block_on_a_recap_bearing_turn_drops_to_none_and_retries_silently(monkey def test_block_with_no_recap_renders_the_honest_terminal_card(monkeypatch): captured: list = [] s = p_session_with_history() + s.model = "sonnet-5" s.history_prefix_mode = "none" s.history_prefix_sent = "none" p_block(s, captured, monkeypatch, text="The agent runtime reported this turn failed (error_during_execution). API Error: 400 https://www.anthrop" + FIELD_TAIL) @@ -134,11 +135,11 @@ def test_api_key_twin_only_for_a_subscription_lane_with_the_users_own_key(): def test_block_with_no_recap_fails_over_to_the_users_own_api_key(monkeypatch): captured: list = [] s = p_session_with_history() - s.model = "opus-5-cc" + s.model = "sonnet-5-cc" s.history_prefix_mode = "none" s.history_prefix_sent = "none" p_block(s, captured, monkeypatch, settings=p_settings(anthropic_api_key="sk-ant-x")) - assert s.model == "opus-5-api" + assert s.model == "sonnet-5-api" assert s.pending_continuation is True and s.needs_fresh_session is True cards = [m for m in s.messages if m.role == "system"] assert len(cards) == 1 and "API key" in str(cards[0].content) and "declined" not in str(cards[0].content).lower().replace("declined this request on your subscription", "") @@ -150,9 +151,58 @@ def test_block_with_no_recap_fails_over_to_the_users_own_api_key(monkeypatch): def test_block_with_no_recap_and_no_key_still_ends_with_the_card(monkeypatch): captured: list = [] s = p_session_with_history() - s.model = "opus-5-cc" + s.model = "sonnet-5-cc" s.history_prefix_mode = "none" s.history_prefix_sent = "none" p_block(s, captured, monkeypatch, settings=p_settings()) - assert s.model == "opus-5-cc" and s.pending_continuation is False + assert s.model == "sonnet-5-cc" and s.pending_continuation is False assert [m for m in s.messages if m.role == "system"][0].content.startswith("The model provider declined") + + +# Fleet, 14 days to 2026-09-07: 78 policy blocks on opus-5 against 0 on opus-4-8 (87 errors of every other +# kind there), every one at spawn on the subscription lane. The same lane's sibling is free and passes. +def test_policy_block_sibling_is_opus_5_to_opus_4_8_on_the_same_lane_only(): + from backend.apps.agents.session_credential import policy_block_sibling + assert policy_block_sibling("opus-5") == "opus-4-8" + assert policy_block_sibling("opus-5-cc") == "opus-4-8-cc" + assert policy_block_sibling("opus-5-api") == "opus-4-8-api" + assert policy_block_sibling("opus-4-8") is None, "the sibling has no sibling, so a block there cards" + assert policy_block_sibling("sonnet-5") is None + assert policy_block_sibling("gpt-5.6") is None + + +def test_opus_5_block_with_nothing_to_strip_finishes_on_opus_4_8_and_says_so(monkeypatch): + captured: list = [] + s = p_session_with_history() + s.model = "opus-5-cc" + s.history_prefix_mode = "none" + s.history_prefix_sent = "none" + p_block(s, captured, monkeypatch, settings=p_settings(anthropic_api_key="sk-ant-x")) + assert s.model == "opus-4-8-cc", "the free same-lane sibling comes before the billed API-key twin" + assert s.lane_failover_from == "opus-5-cc" + assert s.pending_continuation is True and s.needs_fresh_session is True + cards = [m for m in s.messages if m.role == "system"] + assert len(cards) == 1 and "Opus 4.8" in str(cards[0].content) and "next message goes back" in str(cards[0].content) + kinds = [(d.get("kind"), d.get("subkind")) for d in captured] + assert ("model_error", "policy_block:none") in kinds, "the block itself is still reported" + assert ("recovered", "policy_sibling") in kinds + + +def test_a_block_on_the_sibling_never_chains_to_a_third_model(monkeypatch): + """A borrowed ask that is blocked again ends on the card; without this the failover could walk twin -> sibling -> twin.""" + captured: list = [] + s = p_session_with_history() + s.model = "opus-4-8-cc" + s.lane_failover_from = "opus-5-cc" + s.history_prefix_mode = "none" + s.history_prefix_sent = "none" + p_block(s, captured, monkeypatch, settings=p_settings(anthropic_api_key="sk-ant-x")) + assert s.model == "opus-4-8-cc" and s.pending_continuation is False + assert [m for m in s.messages if m.role == "system"][0].content.startswith("The model provider declined") + + +def test_the_sibling_door_sits_above_the_api_key_twin_door(): + """Ordering is the contract: the free lane sibling is tried before the user's key is spent.""" + import inspect + src = inspect.getsource(hre.handle_run_error) + assert src.index("policy_block_sibling(") < src.index("api_key_twin_model(")