openswarm/backend/tests/test_stale_tool_schema_respawn.py

"""ENG-394: the deferred-tool 400 is cured by a respawn, so it must not card out on top of the work.

Reproduced live on packaged 1.7.10-exp.2, 2026-08-31, deterministically: kill 9router, launch a
session, and the CLI defers all 44 MCP tools. The first ToolSearch that loads one sends it carrying
both defer_loading and cache_control, the API 400s, and the turn died after 61 real tool calls.

Control pair from that run: router UP -> 0 ToolSearch, both arms completed; router DOWN -> 1
ToolSearch, both arms 400.
"""
import pytest

from backend.apps.agents.core.error_classify import is_stale_tool_schema_error, is_transient_capacity_error
from backend.apps.agents.core.models import AgentSession
from backend.apps.agents.manager.streaming.auth_retry import (
    try_auth_self_heal,
    try_stale_tool_schema_self_heal,
)

REAL = (
    "The agent runtime reported this turn failed (stop_sequence). API Error: 400 "
    '{"error":{"message":"[claude/claude-sonnet-4-6] [400]: Tool '
    "'mcp__openswarm-core__CreateBrowserAgent' cannot have both defer_loading=true and cache_control "
    'set. Tools with defer_loading cannot use prompt caching. (reset after 30s)"}}'
)
REAL_SHOWUI = REAL.replace("CreateBrowserAgent", "ShowUI")
# The router truncates the upstream message mid-word; this is the byte-real card text from the
# packaged exp.3 drill (2026-08-31), and the first classifier missed it by anchoring on the full
# word "cache_control".
REAL_TRUNCATED = (
    "The agent runtime reported this turn failed (stop_sequence). API Error: 400 "
    '{"error":{"message":"[claude/claude-sonnet-4-6] [400]: Tool '
    "'mcp__openswarm-core__CreateBrowserAgent' cannot have both defer_loading=true and cache_ "
    '(reset after 15s)"}}'
)


def p_session() -> AgentSession:
    return AgentSession(id="s1", name="t", model="sonnet")


def test_the_real_400_from_the_live_drill_is_recognised():
    assert is_stale_tool_schema_error(RuntimeError(REAL))
    assert is_stale_tool_schema_error(RuntimeError(REAL_SHOWUI))


def test_the_router_truncated_form_is_recognised():
    assert is_stale_tool_schema_error(RuntimeError(REAL_TRUNCATED))


def test_the_respawn_waits_out_the_router_warm_up():
    """A 0s retry reconnects into the same dead window the first CLI hit (live: second 400 in <1s)."""
    s = p_session()
    assert try_stale_tool_schema_self_heal(s) is True
    assert s.pending_continuation_delay_s >= 15


def test_it_is_also_found_when_the_text_arrives_on_stderr():
    assert is_stale_tool_schema_error(RuntimeError("Command failed with exit code 1"), extra_text=REAL)


@pytest.mark.parametrize("innocent", [
    # The question this list answers: what legitimate case looks like the bad case?
    "API Error: 400 {\"error\":{\"message\":\"max_tokens is too large\"}}",
    "API Error: 429 rate_limit_error (reset after 21s)",
    "prompt caching is enabled for this request",
    "defer_loading is supported on this model",
    "File \"x.py\", line 400, in run  # cache_control mentioned in a traceback",
    "The model provider returned an expired-credential error",
])
def test_ordinary_failures_never_claim_a_respawn(innocent):
    assert not is_stale_tool_schema_error(RuntimeError(innocent))


def test_the_400_still_refuses_to_be_waited_on():
    """The ENG-395 rule must survive: this is a 400, so no backoff ladder may adopt it."""
    assert not is_transient_capacity_error(RuntimeError(REAL))


def test_one_respawn_is_armed_then_the_budget_is_spent():
    s = p_session()
    assert try_stale_tool_schema_self_heal(s) is True
    assert s.needs_respawn is True
    assert s.pending_continuation is True

    s.pending_continuation = False  # the dispatcher consumed it
    assert try_stale_tool_schema_self_heal(s) is False, "a second identical 400 must card, not loop"


def test_it_does_not_eat_the_auth_one_shot():
    """Separate budgets: these arrive by different doors moments apart."""
    s = p_session()
    assert try_stale_tool_schema_self_heal(s) is True
    s.pending_continuation = False
    assert try_auth_self_heal(s) is True, "the auth retry lost its budget to the schema retry"


def test_it_never_stomps_a_continuation_that_is_already_armed():
    s = p_session()
    s.pending_continuation = True
    s.pending_continuation_prompt = "something else already owns this"
    assert try_stale_tool_schema_self_heal(s) is False
    assert s.pending_continuation_prompt == "something else already owns this"


def test_the_veto_sits_above_every_card_emitting_branch():
    """Ordering, not just behaviour (the recurring defect in this codebase is a guard placed inside
    one branch). Below the generic handler this would card on top of the turn's work."""
    import inspect
    from backend.apps.agents.manager.run import handle_run_error as mod
    src = inspect.getsource(mod.handle_run_error)
    mine = src.index("is_stale_tool_schema_error(e")
    for later in ("is_context_overflow_error(e", "is_transient_capacity_error(e", "is_auth_error(e"):
        assert mine < src.index(later), f"the respawn veto must precede {later}"


@pytest.mark.asyncio
async def test_handle_run_error_arms_the_respawn_and_emits_no_card(monkeypatch):
    """The wiring test. The unit tests above all pass with the branch deleted from the handler,
    which is exactly how a fix ships broken; this one drives the real door."""
    from backend.apps.agents.manager.run import handle_run_error as mod
    from backend.apps.agents.manager.streaming.state import TurnState

    sent: list[tuple[str, dict]] = []

    async def p_send(session_id, event, payload):
        sent.append((event, payload))

    monkeypatch.setattr(mod.ws_manager, "send_to_session", p_send)

    s = p_session()
    s.dashboard_id = "d"
    turn = TurnState()
    await mod.handle_run_error(RuntimeError(REAL), s, s.id, turn, [])

    assert s.needs_respawn is True, "the CLI must be respawned; a fresh one re-registers its tools"
    assert s.pending_continuation is True, "the turn must continue, not die on top of its work"
    cards = [m for m in s.messages if m.role == "system"]
    assert cards == [], f"a curable 400 must not card out; got {[c.content[:60] for c in cards]}"
    assert not any(e == "agent:message" for e, _ in sent), "no error message may reach the user"


@pytest.mark.asyncio
async def test_the_second_identical_400_does_card(monkeypatch):
    """The reverse obligation: when respawning is provably not the cure, the user is owed the card
    rather than a silent loop."""
    from backend.apps.agents.manager.run import handle_run_error as mod
    from backend.apps.agents.manager.streaming.state import TurnState

    async def p_send(session_id, event, payload):
        return None

    monkeypatch.setattr(mod.ws_manager, "send_to_session", p_send)

    s = p_session()
    s.dashboard_id = "d"
    s.stale_tool_schema_retry_used = True  # the one-shot is already spent
    await mod.handle_run_error(RuntimeError(REAL), s, s.id, TurnState(), [])

    cards = [m for m in s.messages if m.role == "system"]
    assert cards, "a spent budget must produce an honest card"
    # The card must blame the actual culprit (our router restarting), never the model, and must
    # promise the thing that is proven to work (a plain resend on the same chat).
    text = cards[-1].content
    assert "router" in text.lower(), text
    assert "send your message again" in text.lower(), text
    assert "switch" not in text.lower(), "the generic switch-models advice blames the wrong thing"