diff --git a/backend/apps/agents/core/error_classify.py b/backend/apps/agents/core/error_classify.py index 228760e7..bf7825df 100644 --- a/backend/apps/agents/core/error_classify.py +++ b/backend/apps/agents/core/error_classify.py @@ -66,6 +66,13 @@ NON_TRANSIENT_PATTERNS = re.compile( # The provider's abuse classifier declined the REQUEST itself (Anthropic: "blocked as it seems to violate ... reverse engineering or duplicating model outputs"); retrying the same bytes is guaranteed futile. +# The shape a provider's own refusal arrives in ("API Error: 400 {...}"), as opposed to prose that +# merely discusses policy. Used to make sure a refusal guard can never eat a genuine answer. +P_PROVIDER_ENVELOPE = re.compile( + r"API\s+Error\b|\"type\"\s*:\s*\"error\"|\"error\"\s*:\s*\{|status\s*(?:code)?\s*[:=]\s*4\d\d", + re.IGNORECASE, +) + P_CONTENT_POLICY_BLOCK = re.compile(r"blocked\s+as\s+it\s+seems\s+to\s+violate|legal/aup|acceptable\s+use\s+policy", re.IGNORECASE) @@ -103,6 +110,13 @@ def neutralize_provider_refusal(text: str) -> str: instead, and leaves any real answer untouched.""" if not text: return text + # Refusal WORDING alone is not enough to destroy a delegated answer. Ask an agent to summarise a + # site's Acceptable Use Policy and its real, correct answer contains the very phrases this + # matches; replacing it would delete the user's work and tell nobody, which is a worse bug than + # the one this guard exists for. A relayed refusal is always wrapped in a provider ENVELOPE, and + # prose about policy never is, so require both before anything is thrown away. + if not P_PROVIDER_ENVELOPE.search(text): + return text if is_content_policy_block(text) or "unable to respond to this request" in text.lower(): return ("That agent could not answer this request and returned no usable result. " "Do not repeat or quote its response; continue with what you already have, " diff --git a/backend/tests/test_delegation_refusal_hygiene.py b/backend/tests/test_delegation_refusal_hygiene.py index 933c2d77..163c1e86 100644 --- a/backend/tests/test_delegation_refusal_hygiene.py +++ b/backend/tests/test_delegation_refusal_hygiene.py @@ -69,3 +69,29 @@ def test_both_delegation_doors_defuse_not_just_one(): p_body = p_src[p_at:p_at + 2000] assert "defuse_extraction_ask" in p_body, f"{p_route} dispatches an un-defused handoff prompt" assert re.search(r"=\s*defuse_extraction_ask\(", p_body), f"{p_route} calls the defuse but drops its result" + + +# ---- The guard must not become the bug (severity ladder, row 1: silent work loss) ---- +# Neutralising on refusal WORDING alone deletes a genuine delegated answer and tells nobody. The +# realistic case is mundane: ask an agent to summarise a site's Acceptable Use Policy and its +# correct answer contains the exact phrases the classifier looks for. + +def test_a_real_answer_about_policy_is_never_destroyed(): + real = ("Summary of their terms: content blocked as it seems to violate the Acceptable Use " + "Policy is removed within 24h, and repeat offenders lose API access.") + assert neutralize_provider_refusal(real) == real, \ + "refusal wording in a genuine answer must never be replaced; that is silent work loss" + + +def test_a_relayed_refusal_still_gets_neutralised(): + relay = ('API Error: 400 {"type":"error","error":{"message":"Output blocked as it seems to ' + 'violate our Acceptable Use Policy (legal/aup): duplicating model outputs"}}') + out = neutralize_provider_refusal(relay) + assert out != relay and "could not answer" in out + + +def test_the_envelope_is_what_separates_them(): + """Same policy wording, with and without the provider envelope: only the envelope is discarded.""" + wording = "Output blocked as it seems to violate our Acceptable Use Policy (legal/aup)" + assert neutralize_provider_refusal(wording) == wording # prose: kept + assert neutralize_provider_refusal(f"API Error: 400 {wording}") != wording # envelope: neutralised