openswarm/backend/tests/test_provider_error_speech.py

"""Every positive case here is a string this machine actually produced.

VERIFICATION.md section 3: "a probe that invents its own inputs measures the probe." So the
envelopes below were lifted verbatim out of backend/data/sessions rather than written from memory
of what a 429 "should" look like -- an invented pattern would have tested my imagination and passed
while the real Gemini string sailed through.

The negative controls carry the weight. A classifier that swallows anything mentioning a status code
would score 14/14 here and silently delete the agent's ability to discuss an error it saw in a log,
which is section 5b's "a fix that REMOVES a capability is not a fix either."
"""

import pytest

from backend.apps.agents.manager.streaming.provider_error_speech import (
    AUTH,
    CONNECTION,
    OVERLOADED,
    QUOTA,
    classify_provider_error,
    is_transient,
    looks_like_provider_envelope,
    parse_duration,
    user_facing_sentence,
)

# --- verbatim from backend/data/sessions on 2026-08-20 -------------------------------------------

REAL_GEMINI_QUOTA_LONG = (
    "API Error: Request rejected (429) · [antigravity/gemini-3-flash] [429]: Individual quota "
    "reached. Please upgrade your subscription to increase your limits. Resets in 125h40m51s. "
    "(reset after 2m)"
)
REAL_GEMINI_QUOTA_NO_WINDOW = (
    "API Error: Request rejected (429) · [antigravity/gemini-3-flash] [429]: Individual quota "
    "reached. Please upgrade your subscription to increase your limits. Resets in "
    "(reset after 1m 56s)"
)
REAL_CLAUDE_CONNECTION = "API Error: Unable to connect. Is the computer able to access the url?"

REAL_ENVELOPES = [REAL_GEMINI_QUOTA_LONG, REAL_GEMINI_QUOTA_NO_WINDOW, REAL_CLAUDE_CONNECTION]


# --- the class is recognised ---------------------------------------------------------------------

@pytest.mark.parametrize("raw", REAL_ENVELOPES)
def test_every_real_envelope_is_recognised(raw):
    assert looks_like_provider_envelope(raw)
    assert classify_provider_error(raw) is not None


def test_gemini_quota_reports_status_lane_and_window():
    err = classify_provider_error(REAL_GEMINI_QUOTA_LONG)
    assert err.kind == QUOTA
    assert err.status == 429
    assert err.lane == "antigravity"
    # 125h40m51s, which is the number the user actually has to plan around.
    assert err.reset_seconds == 125 * 3600 + 40 * 60 + 51


def test_claude_connection_error_has_no_status_and_still_classifies():
    err = classify_provider_error(REAL_CLAUDE_CONNECTION)
    assert err.kind == CONNECTION
    assert err.status is None


def test_auth_and_overload_statuses_route_by_code_not_wording():
    # Status is the provider's own verdict; wording varies per lane and must not be load-bearing.
    assert classify_provider_error("API Error: [codex/gpt-5] [401]: nope").kind == AUTH
    assert classify_provider_error("API Error: [cc/sonnet-5] [529]: overloaded").kind == OVERLOADED


# --- the negative controls: agent speech must survive ---------------------------------------------

AGENT_PROSE = [
    "I checked the logs and the server returned a 429, so I backed off and retried once.",
    "The API Error you saw earlier was a rate limit; here is what I changed.",
    "Summary: 3 requests failed with 500 and 2 with 403. I've added a retry.",
    "Your test asserts a 401 response, and that assertion is correct.",
    "```\nAPI Error: Request rejected (429)\n```\nThat's the line from your log file.",
]


@pytest.mark.parametrize("prose", AGENT_PROSE)
def test_agent_discussing_an_error_is_still_the_agent(prose):
    """If this ever fails, the agent has lost the ability to talk about errors at all."""
    assert not looks_like_provider_envelope(prose)
    assert classify_provider_error(prose) is None


def test_empty_and_whitespace_are_not_envelopes():
    assert classify_provider_error("") is None
    assert classify_provider_error("   \n  ") is None


# --- what the user is told -------------------------------------------------------------------------

def test_long_quota_tells_the_user_to_switch_rather_than_wait():
    err = classify_provider_error(REAL_GEMINI_QUOTA_LONG)
    sentence = user_facing_sentence(err, "gemini-3.1-flash-lite")
    assert "Gemini" in sentence
    assert "switch" in sentence.lower()
    # A five-day wait must never be presented as something to sit through.
    assert not is_transient(err)


def test_short_rate_limit_promises_automatic_resume():
    err = classify_provider_error(
        "API Error: Request rejected (429) · [antigravity/gemini-3-flash] [429]: slow down. "
        "(reset after 2m)"
    )
    assert is_transient(err)
    sentence = user_facing_sentence(err, "gemini-3.1-flash-lite")
    assert "automatically" in sentence.lower()


def test_no_sentence_leaks_provider_jargon():
    """The whole point: the user never reads a status code or a vendor's upgrade pitch."""
    for raw in REAL_ENVELOPES:
        err = classify_provider_error(raw)
        sentence = user_facing_sentence(err, "sonnet-5")
        assert "429" not in sentence
        assert "API Error" not in sentence
        assert "upgrade your subscription" not in sentence.lower()
        assert sentence.strip().endswith(".")


def test_connection_loss_is_transient_so_the_turn_can_park():
    err = classify_provider_error(REAL_CLAUDE_CONNECTION)
    assert is_transient(err)


# --- duration parsing, because the window decides park-vs-tell -------------------------------------

@pytest.mark.parametrize("blob,expected", [
    ("125h40m51s", 125 * 3600 + 40 * 60 + 51),
    ("1m 56s", 116),
    ("2m", 120),
    ("45s", 45),
])
def test_duration_parsing(blob, expected):
    assert parse_duration(blob) == expected


def test_unparseable_duration_is_none_not_zero():
    """Zero would read as 'resume immediately', which is the opposite of what an unknown means."""
    assert parse_duration("soon") is None


def test_policy_refusal_is_its_own_kind_and_never_promises_a_retry():
    """Alex read 'Retrying automatically' before every bricked turn; nothing retried. The 400 with
    the policy verdict in words must classify as POLICY ahead of its status, and the unknown-error
    sentence must stop promising a retry it does not queue."""
    from backend.apps.agents.manager.streaming.provider_error_speech import POLICY, UNKNOWN
    raw = ("API Error: 400 https://www.anthropic.com/legal/aup). This request was blocked as it seems "
           "to violate Anthropic's Terms of Service restrictions on reverse engineering or duplicating model outputs.")
    err = classify_provider_error(raw)
    assert err is not None and err.kind == POLICY
    assert is_transient(err) is False
    assert "policy filter" in user_facing_sentence(err, "opus-5")
    unknown = classify_provider_error("API Error: 400 something nobody classified")
    assert unknown is not None and unknown.kind == UNKNOWN
    assert "Retrying automatically" not in user_facing_sentence(unknown, "opus-5")