import json
import urllib.request
from io import BytesIO
from types import SimpleNamespace
from typing import Any, cast
import pytest
from llm.core.interface import LLMProvider
from llm.core.types import LLMInput, LLMOutput, Message, Role
from llm.providers.astraflow import AstraflowProvider
from llm.providers.atlas import AtlasProvider
from llm.providers.ollama import OllamaProvider
from llm.providers.openai import OpenAIProvider
from llm.providers.reasoning import strip_reasoning
# Answers that mention the tags without carrying reasoning. None of them may be
# changed, whatever the deployment's reasoning format.
LITERAL_ANSWERS = [
"Wrap it in and tags.",
"Close it with and nothing else.",
" comes before here.",
'{"closing":"","opening":""}',
'{"note": "x"}',
]
@pytest.mark.parametrize(
("content", "expected"),
[
("17 * 3 = 51.51", "51"),
("\n\nplan\n\n\nanswer", "answer"),
('build the object{"answer": 51}', '{"answer": 51}'),
('plan{"closing":""}', '{"closing":""}'),
("plain answer", "plain answer"),
("", ""),
("cut off by max_tokens", "cut off by max_tokens"),
# Close-only output is ambiguous without a declared format, so it stays.
("17 * 3 = 51.51", "17 * 3 = 51.51"),
],
)
def test_strip_reasoning_declared_leading(content: str, expected: str) -> None:
assert strip_reasoning(content, leading=True) == expected
@pytest.mark.parametrize("content", LITERAL_ANSWERS)
def test_strip_reasoning_default_preserves_literal_tags(content: str) -> None:
assert strip_reasoning(content) == content
@pytest.mark.parametrize(
("content", "expected"),
[
("17 * 3 = 51.51", "51"),
('build the object\n\n{"answer": 51}', '{"answer": 51}'),
("17 * 3 = 51.51", "51"),
("plain answer", "plain answer"),
(
"Wrap it in and tags.",
"Wrap it in and tags.",
),
("17 * 3 = 51.", ""),
# A complete JSON answer is never reasoning plus a closing tag.
('{"closing":""}', '{"closing":""}'),
(
' {"closing":"","opening":""}\n',
' {"closing":"","opening":""}\n',
),
('["", 1]', '["", 1]'),
],
)
def test_strip_reasoning_prefilled(content: str, expected: str) -> None:
assert strip_reasoning(content, prefilled=True) == expected
def test_strip_reasoning_prefilled_survives_deeply_nested_json() -> None:
# Deep enough for json.loads to raise RecursionError on every supported Python.
depth = 100_000
content = "[" * depth + '""' + "]" * depth
assert strip_reasoning(content, prefilled=True) == content
def _openai_provider(content: str, **kwargs: Any) -> OpenAIProvider:
provider = OpenAIProvider(api_key="test", **kwargs)
message = SimpleNamespace(content=content, tool_calls=None)
response = SimpleNamespace(
choices=[SimpleNamespace(message=message, finish_reason="stop")],
model="spark-x2.5",
usage=None,
)
client: Any = SimpleNamespace(
chat=SimpleNamespace(
completions=SimpleNamespace(create=lambda **params: response)
)
)
provider.client = client
return provider
def _ask(provider: LLMProvider, **kwargs: Any) -> LLMOutput:
return provider.generate(
LLMInput(messages=[Message(role=Role.USER, content="17*3?")], **kwargs)
)
def test_openai_provider_strips_balanced_reasoning() -> None:
assert (
_ask(
_openai_provider("17 * 3 = 51.51", leading_reasoning=True)
).content
== "51"
)
@pytest.mark.parametrize("content", LITERAL_ANSWERS)
def test_openai_provider_keeps_literal_tags(content: str) -> None:
assert _ask(_openai_provider(content)).content == content
def test_openai_provider_close_only_needs_opt_in(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("OPENAI_PREFILLED_REASONING", raising=False)
assert (
_ask(_openai_provider("17 * 3 = 51.51")).content
== "17 * 3 = 51.51"
)
assert (
_ask(
_openai_provider("17 * 3 = 51.51", prefilled_reasoning=True)
).content
== "51"
)
monkeypatch.setenv("OPENAI_PREFILLED_REASONING", "true")
assert _ask(_openai_provider("17 * 3 = 51.51")).content == "51"
def test_hosted_provider_never_strips_close_only(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.setenv("OPENAI_PREFILLED_REASONING", "true")
provider = AstraflowProvider(api_key="test")
message = SimpleNamespace(content='{"closing":""}', tool_calls=None)
response = SimpleNamespace(
choices=[SimpleNamespace(message=message, finish_reason="stop")],
model="deepseek-v3.2",
usage=None,
)
client: Any = SimpleNamespace(
chat=SimpleNamespace(
completions=SimpleNamespace(create=lambda **params: response)
)
)
provider.client = client
assert _ask(provider).content == '{"closing":""}'
def _fake_ollama(monkeypatch: pytest.MonkeyPatch, content: str) -> list[dict[str, Any]]:
sent: list[dict[str, Any]] = []
def fake_urlopen(request: urllib.request.Request, timeout: float) -> BytesIO:
sent.append(json.loads(cast(bytes, request.data)))
return BytesIO(
json.dumps(
{"message": {"content": content}, "done_reason": "stop"}
).encode()
)
monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen)
return sent
@pytest.mark.parametrize("content", LITERAL_ANSWERS)
def test_ollama_provider_keeps_literal_tags(
monkeypatch: pytest.MonkeyPatch, content: str
) -> None:
monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False)
_fake_ollama(monkeypatch, content)
assert _ask(OllamaProvider()).content == content
def test_ollama_provider_strips_prefilled_reasoning_when_declared(
monkeypatch: pytest.MonkeyPatch,
) -> None:
monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False)
_fake_ollama(monkeypatch, "17 * 3 = 51.51")
assert _ask(OllamaProvider()).content == "17 * 3 = 51.51"
assert _ask(OllamaProvider(prefilled_reasoning=True)).content == "51"
monkeypatch.setenv("OLLAMA_PREFILLED_REASONING", "1")
assert _ask(OllamaProvider()).content == "51"
def test_ollama_stream_flag_still_strips_complete_message(
monkeypatch: pytest.MonkeyPatch,
) -> None:
# The provider always requests a single complete message, even when the
# caller asks for streaming, so stripping sees the whole reply. If chunked
# streaming is added, this needs an incremental stripper instead.
sent = _fake_ollama(monkeypatch, 'plan{"closing":""}')
output = _ask(OllamaProvider(leading_reasoning=True), stream=True)
assert sent[0]["stream"] is False
assert output.content == '{"closing":""}'
@pytest.mark.parametrize(
"content",
[
"display this XML literally",
"display this XML literallykeep both",
],
)
def test_default_provider_preserves_complete_leading_xml(
monkeypatch: pytest.MonkeyPatch, content: str
) -> None:
monkeypatch.delenv("OPENAI_LEADING_REASONING", raising=False)
monkeypatch.delenv("OPENAI_PREFILLED_REASONING", raising=False)
monkeypatch.delenv("OLLAMA_LEADING_REASONING", raising=False)
monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False)
assert _ask(_openai_provider(content)).content == content
_fake_ollama(monkeypatch, content)
assert _ask(OllamaProvider()).content == content
def test_balanced_leading_reasoning_requires_explicit_format(
monkeypatch: pytest.MonkeyPatch,
) -> None:
content = "plananswer"
assert _ask(_openai_provider(content, leading_reasoning=True)).content == "answer"
monkeypatch.setenv("OPENAI_LEADING_REASONING", "true")
assert _ask(_openai_provider(content)).content == "answer"
assert _ask(_openai_provider(content, leading_reasoning=False)).content == content
_fake_ollama(monkeypatch, content)
assert _ask(OllamaProvider(leading_reasoning=True)).content == "answer"
monkeypatch.setenv("OLLAMA_LEADING_REASONING", "true")
assert _ask(OllamaProvider()).content == "answer"
assert _ask(OllamaProvider(leading_reasoning=False)).content == content
@pytest.mark.parametrize("provider_type", [AtlasProvider, AstraflowProvider])
@pytest.mark.parametrize(
"content",
["literal XML", "literal XMLwhole"],
)
def test_hosted_providers_preserve_leading_xml(
monkeypatch: pytest.MonkeyPatch, provider_type: Any, content: str
) -> None:
monkeypatch.setenv("OPENAI_LEADING_REASONING", "true")
monkeypatch.setenv("OPENAI_PREFILLED_REASONING", "true")
provider = provider_type(api_key="test")
response = SimpleNamespace(
choices=[
SimpleNamespace(
message=SimpleNamespace(content=content, tool_calls=None),
finish_reason="stop",
)
],
model="fixture",
usage=None,
)
provider.client = SimpleNamespace(
chat=SimpleNamespace(
completions=SimpleNamespace(create=lambda **params: response)
)
)
assert _ask(provider).content == content
@pytest.mark.parametrize(
"content",
["literal XML", "literal XMLwhole"],
)
def test_helper_defaults_preserve_leading_xml(content: str) -> None:
assert strip_reasoning(content) == content