[eric] agents: OSW_FAULT fires real provider failures on demand so the guards get drilled, not hoped for

backend/apps/agents/core/fault_injection.py

@@ -0,0 +1,42 @@
+"""Deliberate faults, so a guard is proven rather than hoped for.
+
+Half our safety code only runs when something rare goes wrong: a wedged sidecar, a frozen loop,
+a provider refusal, a mislabelled 401. Waiting for those in the wild means they are never drilled,
+and a guard that never executes is indistinguishable from one that never needed to (the 50KB cap
+measured 0.0%, ENG-385; the mid-turn breaker never fires on codex at all, ENG-391).
+
+`OSW_FAULT` is a comma-separated list of faults to arm. Unset in every shipped build, and an
+unknown name is ignored rather than guessed at, so a typo can never silently arm nothing while
+the drill reports a pass.
+
+    OSW_FAULT=policy_block,auth_401 bash run.sh
+"""
+
+import os
+from typing import Set
+
+# Every fault the harness knows. A name outside this set is a typo, not a feature.
+KNOWN_FAULTS: Set[str] = {
+    "policy_block",      # the provider declines the request (ENG-383 failover, ENG-387 doors)
+    "auth_401",          # a 401 mid-turn (ENG-361 self-heal, ENG-365 must not misread "line 401,")
+    "sidecar_wedge",     # a builtin tool never returns (ENG-368 heartbeat ceiling)
+    "transport_death",   # the CLI's pipe dies, not the provider (ENG-382 respawn-not-rebuild)
+    "empty_finish",      # a turn ends with no answer after tool work (ENG-354, ENG-390)
+}
+
+
+def armed(name: str) -> bool:
+    """True when this fault was deliberately armed. Never true in a shipped build."""
+    raw = os.environ.get("OSW_FAULT", "")
+    if not raw:
+        return False
+    wanted = {p.strip() for p in raw.split(",") if p.strip()}
+    return name in (wanted & KNOWN_FAULTS)
+
+
+def unknown_faults() -> Set[str]:
+    """Names asked for that this build does not know: surfaced loudly so a typo cannot read as a pass."""
+    raw = os.environ.get("OSW_FAULT", "")
+    if not raw:
+        return set()
+    return {p.strip() for p in raw.split(",") if p.strip()} - KNOWN_FAULTS

backend/apps/agents/manager/run/TurnRunner.py

@@ -45,6 +45,23 @@ class TurnRunner(AgentManagerProtocol):
                                     global_settings: AppSettings, force_respawn: bool = False) -> None:
         from claude_agent_sdk import query, ClaudeAgentOptions, AssistantMessage, ResultMessage
         from claude_agent_sdk.types import StreamEvent, SystemMessage
+        from backend.apps.agents.core.fault_injection import armed as p_fault_armed
+
+        # Deliberate faults, so the guards below get drilled instead of waited for. Inert unless
+        # OSW_FAULT names them; a shipped build never sets it. Raised HERE because this is the same
+        # door a real provider failure comes through, so the drill exercises the real recovery.
+        if p_fault_armed("policy_block"):
+            # The provider's real wording, so the drill hits the same classifier a field block does.
+            raise RuntimeError(
+                "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":"
+                "\"Output blocked as it seems to violate our Acceptable Use Policy (legal/aup): "
+                "reverse engineering or duplicating model outputs\"}}"
+            )
+        if p_fault_armed("auth_401"):
+            raise RuntimeError("API Error: 401 {\"error\":{\"type\":\"authentication_error\",\"message\":\"invalid x-api-key\"}}")
+        if p_fault_armed("transport_death"):
+            import anyio
+            raise anyio.BrokenResourceError()
 
         async def prompt_stream():
             yield {

backend/tests/test_fault_injection.py

@@ -0,0 +1,59 @@
+"""A fault harness is only useful if the fault it fires is the one the guard actually looks for.
+
+Half the safety code only runs on rare failures, so it was never drilled: waiting for a wedged
+sidecar or a provider refusal in the wild means the guard is hoped for, not proven (ENG-385's cap
+measured 0.0%; ENG-391's breaker never runs on codex). These pin that the injected text is
+classified the same way a REAL failure is, and that the harness is inert unless armed.
+"""
+
+import re
+import pytest
+
+from backend.apps.agents.core.fault_injection import KNOWN_FAULTS, armed, unknown_faults
+from backend.apps.agents.core.error_classify import has_auth_status, is_content_policy_block
+
+TURN_RUNNER = "backend/apps/agents/manager/run/TurnRunner.py"
+
+
+@pytest.fixture(autouse=True)
+def p_clean(monkeypatch):
+    monkeypatch.delenv("OSW_FAULT", raising=False)
+
+
+def p_injected(kind: str) -> str:
+    """The literal the harness raises, read out of the source so the test cannot drift from it."""
+    src = open(TURN_RUNNER).read()
+    block = src.split(f'p_fault_armed("{kind}")')[1].split("if p_fault_armed")[0]
+    return "".join(re.findall(r'"((?:[^"\\]|\\.)*)"', block)).replace('\\"', '"')
+
+
+def test_inert_unless_armed():
+    assert armed("policy_block") is False
+    assert armed("auth_401") is False
+
+
+def test_arming_is_explicit_and_typos_are_surfaced(monkeypatch):
+    monkeypatch.setenv("OSW_FAULT", "policy_block,auth_401")
+    assert armed("policy_block") and armed("auth_401")
+    assert not armed("sidecar_wedge"), "only what was named may arm"
+    monkeypatch.setenv("OSW_FAULT", "policyblock")
+    assert armed("policy_block") is False
+    assert unknown_faults() == {"policyblock"}, "a typo must be reported, never silently arm nothing"
+
+
+def test_the_policy_fault_is_what_the_real_classifier_catches():
+    assert is_content_policy_block(p_injected("policy_block")), \
+        "if this drifts, the drill fires a fault the guard ignores and still reports a pass"
+    assert not is_content_policy_block("API Error: 500 internal server error")
+
+
+def test_the_auth_fault_is_what_the_real_classifier_catches():
+    assert has_auth_status(p_injected("auth_401"))
+    # ENG-365's control: a traceback line number must never read as an auth failure.
+    assert not has_auth_status("File runner.py, line 401, in execute")
+
+
+def test_every_known_fault_is_wired_or_declared():
+    src = open(TURN_RUNNER).read()
+    wired = {m for m in KNOWN_FAULTS if f'p_fault_armed("{m}")' in src}
+    assert {"policy_block", "auth_401", "transport_death"} <= wired