[eric] policy: a block on Opus 5 finishes the ask on Opus 4.8 on the same lane before touching the user's key, one failover per ask; fleet 14 days: 78 blocks on opus-5, 0 on opus-4-8

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
This commit is contained in:
ciregenz
2026-09-07 18:32:56 -07:00
co-authored by Claude Fable 5.1
parent 9a56b2058b
commit 9bdd45f791
6 changed files with 105 additions and 7 deletions
+1
View File
@@ -244,6 +244,7 @@ class AgentManager(SessionLifecycle, SessionHistory, SessionPersistence, Messagi
return
session.status = "running"
session.last_failure_kind = None
# Resolve the model id now so every closure (approval hook, tool executed handler, etc.) has both the short name and the 9Router-prefixed id available without re-resolving. The short name is what the user sees; the router id is what 9Router reports its per-model counters under.
from backend.apps.agents.providers.registry import (
+2
View File
@@ -169,6 +169,8 @@ class AgentSession(BaseModel):
# moment they send again. Failing over is for finishing THE CURRENT ASK; leaving the chat on a
# metered key forever would bill them per token with one line said about it, once (ENG-383).
lane_failover_from: Optional[str] = None
# How the last turn ended when it ended on a card; a parent reads it off a child so a declined task is not re-dispatched.
last_failure_kind: Optional[Literal["policy_block"]] = None
# Consecutive dirty deaths this session was MID-TURN for; the crash auto-resume breaker (hermes #30719 pairing: auto-resume must never outrun its circuit breaker).
crash_interrupt_count: int = 0
# Outage rounds spent on this ask: the in-turn ladder covers only 335s, and the work is checkpointed, so a longer drop is waited out rather than ending the task.
@@ -50,7 +50,8 @@ class TurnRunner(AgentManagerProtocol):
# Deliberate faults, so the guards below get drilled instead of waited for. Inert unless
# OSW_FAULT names them; a shipped build never sets it. Raised HERE because this is the same
# door a real provider failure comes through, so the drill exercises the real recovery.
if p_fault_armed("policy_block"):
# Budgeted like auth_401 (OSW_FAULT_FIRES=N), so a drill can block ONE turn and watch the failover finish the ask.
if p_fault_fire("policy_block"):
# The provider's real wording, so the drill hits the same classifier a field block does.
raise RuntimeError(
"API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":"
@@ -47,7 +47,7 @@ from backend.apps.agents.core.extract_reset_hint import extract_reset_hint
from backend.apps.agents.core.redact_for_telemetry import redact_for_telemetry
from backend.apps.agents.core import flight_recorder
from backend.apps.agents.manager.run.empty_finish import count_tool_calls
from backend.apps.agents.session_credential import api_key_twin_model
from backend.apps.agents.session_credential import api_key_twin_model, policy_block_sibling
logger = logging.getLogger(__name__)
@@ -502,8 +502,35 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
"this turn, rely on the visible conversation.")
logger.warning(f"Agent {session_id}: provider content-policy block on a turn carrying a {p_sent} history prefix; retrying with {session.history_prefix_mode}")
return
# Nothing left to strip. The classifier declines Opus 5 where it passes Opus 4.8 (fleet, 2026-09-07:
# 78 blocks vs 0 across a comparable error volume), so the step finishes on the same lane's sibling
# first: same subscription, same wallet, nothing billed. One ask, announced, and the next real
# message puts the chat back on the model the user chose (Messaging restores lane_failover_from).
p_sibling = policy_block_sibling(session.model or "") if not session.lane_failover_from else None
if p_sibling:
p_from = session.model
session.model = p_sibling
session.lane_failover_from = p_from
session.needs_fresh_session = True
session.pending_continuation = True
session.pending_continuation_prompt = "Continue where you left off and finish the task, then answer in plain text."
p_notice = Message(
role="system",
content=("Anthropic's filter declined this request on Claude Opus 5, which it does far more often "
"than on other models. This step is finishing on Claude Opus 4.8 on the same subscription; "
"your next message goes back to Opus 5. If it keeps happening, switch this chat to Opus 4.8 or Sonnet 5."),
branch_id=session.active_branch_id,
)
absorb_repeat_card(session, p_notice)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id, "message": p_notice.model_dump(mode="json"),
})
flight_recorder.record_recovery(session_id, "policy_sibling", session.model, 1)
logger.warning(f"Agent {session_id}: policy block on {p_from}; finishing this ask on the same-lane sibling {p_sibling}")
return
# The subscription lane declined and nothing is left to strip; fleet data says the same request passes on an API key (0 of 328 vs 4.4%), so a user who connected their own Anthropic key continues there, told in one line, instead of losing the ask (ENG-383).
p_twin = api_key_twin_model(session.model or "", load_settings())
# One failover per ask, whichever door it took: a borrowed sibling that is blocked again ends on the card, never on a second borrow that bills the key.
p_twin = api_key_twin_model(session.model or "", load_settings()) if not session.lane_failover_from else None
if p_twin:
p_from = session.model
session.model = p_twin
@@ -526,6 +553,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
logger.warning(f"Agent {session_id}: policy block on the subscription lane; failing over {p_from} -> {p_twin}")
return
p_retried = session.history_prefix_mode == "none"
session.last_failure_kind = "policy_block"
friendly_msg = (
"The model provider declined this request (its automated policy filter flagged the "
"conversation's content)"
+16
View File
@@ -189,6 +189,22 @@ def write_would_suicide(field: str, new_value: Any, powering: PoweringCredential
return False
# Anthropic's abuse classifier declines Opus 5 requests it passes on every other model: 78 of 84 fleet
# blocks in the fortnight to 2026-09-07 were on opus-5 against 0 on opus-4-8 across a comparable error
# volume, and Anthropic's own block text says to switch models. The sibling is the same vendor on the
# SAME lane and wallet, so a step can finish there without spending anything the user did not connect.
POLICY_BLOCK_SIBLING: dict[str, str] = {
"opus-5": "opus-4-8",
"opus-5-cc": "opus-4-8-cc",
"opus-5-api": "opus-4-8-api",
}
def policy_block_sibling(model_value: str) -> Optional[str]:
"""The same-lane model a policy-blocked step may finish on, or None when there is no such sibling."""
return POLICY_BLOCK_SIBLING.get(model_value)
def api_key_twin_model(model_value: str, settings: AppSettings) -> Optional[str]:
"""The same Claude model on the user's OWN Anthropic API key, when this run was on a subscription
lane and such a key is configured; None otherwise. Never the OpenSwarm Pro pool (not their money to
+54 -4
View File
@@ -82,6 +82,7 @@ def test_block_on_a_recap_bearing_turn_drops_to_none_and_retries_silently(monkey
def test_block_with_no_recap_renders_the_honest_terminal_card(monkeypatch):
captured: list = []
s = p_session_with_history()
s.model = "sonnet-5"
s.history_prefix_mode = "none"
s.history_prefix_sent = "none"
p_block(s, captured, monkeypatch, text="The agent runtime reported this turn failed (error_during_execution). API Error: 400 https://www.anthrop" + FIELD_TAIL)
@@ -134,11 +135,11 @@ def test_api_key_twin_only_for_a_subscription_lane_with_the_users_own_key():
def test_block_with_no_recap_fails_over_to_the_users_own_api_key(monkeypatch):
captured: list = []
s = p_session_with_history()
s.model = "opus-5-cc"
s.model = "sonnet-5-cc"
s.history_prefix_mode = "none"
s.history_prefix_sent = "none"
p_block(s, captured, monkeypatch, settings=p_settings(anthropic_api_key="sk-ant-x"))
assert s.model == "opus-5-api"
assert s.model == "sonnet-5-api"
assert s.pending_continuation is True and s.needs_fresh_session is True
cards = [m for m in s.messages if m.role == "system"]
assert len(cards) == 1 and "API key" in str(cards[0].content) and "declined" not in str(cards[0].content).lower().replace("declined this request on your subscription", "")
@@ -150,9 +151,58 @@ def test_block_with_no_recap_fails_over_to_the_users_own_api_key(monkeypatch):
def test_block_with_no_recap_and_no_key_still_ends_with_the_card(monkeypatch):
captured: list = []
s = p_session_with_history()
s.model = "opus-5-cc"
s.model = "sonnet-5-cc"
s.history_prefix_mode = "none"
s.history_prefix_sent = "none"
p_block(s, captured, monkeypatch, settings=p_settings())
assert s.model == "opus-5-cc" and s.pending_continuation is False
assert s.model == "sonnet-5-cc" and s.pending_continuation is False
assert [m for m in s.messages if m.role == "system"][0].content.startswith("The model provider declined")
# Fleet, 14 days to 2026-09-07: 78 policy blocks on opus-5 against 0 on opus-4-8 (87 errors of every other
# kind there), every one at spawn on the subscription lane. The same lane's sibling is free and passes.
def test_policy_block_sibling_is_opus_5_to_opus_4_8_on_the_same_lane_only():
from backend.apps.agents.session_credential import policy_block_sibling
assert policy_block_sibling("opus-5") == "opus-4-8"
assert policy_block_sibling("opus-5-cc") == "opus-4-8-cc"
assert policy_block_sibling("opus-5-api") == "opus-4-8-api"
assert policy_block_sibling("opus-4-8") is None, "the sibling has no sibling, so a block there cards"
assert policy_block_sibling("sonnet-5") is None
assert policy_block_sibling("gpt-5.6") is None
def test_opus_5_block_with_nothing_to_strip_finishes_on_opus_4_8_and_says_so(monkeypatch):
captured: list = []
s = p_session_with_history()
s.model = "opus-5-cc"
s.history_prefix_mode = "none"
s.history_prefix_sent = "none"
p_block(s, captured, monkeypatch, settings=p_settings(anthropic_api_key="sk-ant-x"))
assert s.model == "opus-4-8-cc", "the free same-lane sibling comes before the billed API-key twin"
assert s.lane_failover_from == "opus-5-cc"
assert s.pending_continuation is True and s.needs_fresh_session is True
cards = [m for m in s.messages if m.role == "system"]
assert len(cards) == 1 and "Opus 4.8" in str(cards[0].content) and "next message goes back" in str(cards[0].content)
kinds = [(d.get("kind"), d.get("subkind")) for d in captured]
assert ("model_error", "policy_block:none") in kinds, "the block itself is still reported"
assert ("recovered", "policy_sibling") in kinds
def test_a_block_on_the_sibling_never_chains_to_a_third_model(monkeypatch):
"""A borrowed ask that is blocked again ends on the card; without this the failover could walk twin -> sibling -> twin."""
captured: list = []
s = p_session_with_history()
s.model = "opus-4-8-cc"
s.lane_failover_from = "opus-5-cc"
s.history_prefix_mode = "none"
s.history_prefix_sent = "none"
p_block(s, captured, monkeypatch, settings=p_settings(anthropic_api_key="sk-ant-x"))
assert s.model == "opus-4-8-cc" and s.pending_continuation is False
assert [m for m in s.messages if m.role == "system"][0].content.startswith("The model provider declined")
def test_the_sibling_door_sits_above_the_api_key_twin_door():
"""Ordering is the contract: the free lane sibling is tried before the user's key is spent."""
import inspect
src = inspect.getsource(hre.handle_run_error)
assert src.index("policy_block_sibling(") < src.index("api_key_twin_model(")