diff --git a/backend/apps/agents/core/fault_injection.py b/backend/apps/agents/core/fault_injection.py new file mode 100644 index 00000000..dfde3e50 --- /dev/null +++ b/backend/apps/agents/core/fault_injection.py @@ -0,0 +1,42 @@ +"""Deliberate faults, so a guard is proven rather than hoped for. + +Half our safety code only runs when something rare goes wrong: a wedged sidecar, a frozen loop, +a provider refusal, a mislabelled 401. Waiting for those in the wild means they are never drilled, +and a guard that never executes is indistinguishable from one that never needed to (the 50KB cap +measured 0.0%, ENG-385; the mid-turn breaker never fires on codex at all, ENG-391). + +`OSW_FAULT` is a comma-separated list of faults to arm. Unset in every shipped build, and an +unknown name is ignored rather than guessed at, so a typo can never silently arm nothing while +the drill reports a pass. + + OSW_FAULT=policy_block,auth_401 bash run.sh +""" + +import os +from typing import Set + +# Every fault the harness knows. A name outside this set is a typo, not a feature. +KNOWN_FAULTS: Set[str] = { + "policy_block", # the provider declines the request (ENG-383 failover, ENG-387 doors) + "auth_401", # a 401 mid-turn (ENG-361 self-heal, ENG-365 must not misread "line 401,") + "sidecar_wedge", # a builtin tool never returns (ENG-368 heartbeat ceiling) + "transport_death", # the CLI's pipe dies, not the provider (ENG-382 respawn-not-rebuild) + "empty_finish", # a turn ends with no answer after tool work (ENG-354, ENG-390) +} + + +def armed(name: str) -> bool: + """True when this fault was deliberately armed. Never true in a shipped build.""" + raw = os.environ.get("OSW_FAULT", "") + if not raw: + return False + wanted = {p.strip() for p in raw.split(",") if p.strip()} + return name in (wanted & KNOWN_FAULTS) + + +def unknown_faults() -> Set[str]: + """Names asked for that this build does not know: surfaced loudly so a typo cannot read as a pass.""" + raw = os.environ.get("OSW_FAULT", "") + if not raw: + return set() + return {p.strip() for p in raw.split(",") if p.strip()} - KNOWN_FAULTS diff --git a/backend/apps/agents/manager/run/TurnRunner.py b/backend/apps/agents/manager/run/TurnRunner.py index 8f63d212..b913fff0 100644 --- a/backend/apps/agents/manager/run/TurnRunner.py +++ b/backend/apps/agents/manager/run/TurnRunner.py @@ -45,6 +45,23 @@ class TurnRunner(AgentManagerProtocol): global_settings: AppSettings, force_respawn: bool = False) -> None: from claude_agent_sdk import query, ClaudeAgentOptions, AssistantMessage, ResultMessage from claude_agent_sdk.types import StreamEvent, SystemMessage + from backend.apps.agents.core.fault_injection import armed as p_fault_armed + + # Deliberate faults, so the guards below get drilled instead of waited for. Inert unless + # OSW_FAULT names them; a shipped build never sets it. Raised HERE because this is the same + # door a real provider failure comes through, so the drill exercises the real recovery. + if p_fault_armed("policy_block"): + # The provider's real wording, so the drill hits the same classifier a field block does. + raise RuntimeError( + "API Error: 400 {\"type\":\"error\",\"error\":{\"type\":\"invalid_request_error\",\"message\":" + "\"Output blocked as it seems to violate our Acceptable Use Policy (legal/aup): " + "reverse engineering or duplicating model outputs\"}}" + ) + if p_fault_armed("auth_401"): + raise RuntimeError("API Error: 401 {\"error\":{\"type\":\"authentication_error\",\"message\":\"invalid x-api-key\"}}") + if p_fault_armed("transport_death"): + import anyio + raise anyio.BrokenResourceError() async def prompt_stream(): yield { diff --git a/backend/tests/test_fault_injection.py b/backend/tests/test_fault_injection.py new file mode 100644 index 00000000..ebd0126e --- /dev/null +++ b/backend/tests/test_fault_injection.py @@ -0,0 +1,59 @@ +"""A fault harness is only useful if the fault it fires is the one the guard actually looks for. + +Half the safety code only runs on rare failures, so it was never drilled: waiting for a wedged +sidecar or a provider refusal in the wild means the guard is hoped for, not proven (ENG-385's cap +measured 0.0%; ENG-391's breaker never runs on codex). These pin that the injected text is +classified the same way a REAL failure is, and that the harness is inert unless armed. +""" + +import re +import pytest + +from backend.apps.agents.core.fault_injection import KNOWN_FAULTS, armed, unknown_faults +from backend.apps.agents.core.error_classify import has_auth_status, is_content_policy_block + +TURN_RUNNER = "backend/apps/agents/manager/run/TurnRunner.py" + + +@pytest.fixture(autouse=True) +def p_clean(monkeypatch): + monkeypatch.delenv("OSW_FAULT", raising=False) + + +def p_injected(kind: str) -> str: + """The literal the harness raises, read out of the source so the test cannot drift from it.""" + src = open(TURN_RUNNER).read() + block = src.split(f'p_fault_armed("{kind}")')[1].split("if p_fault_armed")[0] + return "".join(re.findall(r'"((?:[^"\\]|\\.)*)"', block)).replace('\\"', '"') + + +def test_inert_unless_armed(): + assert armed("policy_block") is False + assert armed("auth_401") is False + + +def test_arming_is_explicit_and_typos_are_surfaced(monkeypatch): + monkeypatch.setenv("OSW_FAULT", "policy_block,auth_401") + assert armed("policy_block") and armed("auth_401") + assert not armed("sidecar_wedge"), "only what was named may arm" + monkeypatch.setenv("OSW_FAULT", "policyblock") + assert armed("policy_block") is False + assert unknown_faults() == {"policyblock"}, "a typo must be reported, never silently arm nothing" + + +def test_the_policy_fault_is_what_the_real_classifier_catches(): + assert is_content_policy_block(p_injected("policy_block")), \ + "if this drifts, the drill fires a fault the guard ignores and still reports a pass" + assert not is_content_policy_block("API Error: 500 internal server error") + + +def test_the_auth_fault_is_what_the_real_classifier_catches(): + assert has_auth_status(p_injected("auth_401")) + # ENG-365's control: a traceback line number must never read as an auth failure. + assert not has_auth_status("File runner.py, line 401, in execute") + + +def test_every_known_fault_is_wired_or_declared(): + src = open(TURN_RUNNER).read() + wired = {m for m in KNOWN_FAULTS if f'p_fault_armed("{m}")' in src} + assert {"policy_block", "auth_401", "transport_death"} <= wired