From d6f20191def6ea7304a012c484b5401c88a4421c Mon Sep 17 00:00:00 2001 From: FenjuFu Date: Wed, 30 Sep 2026 18:41:24 +0800 Subject: [PATCH 1/5] fix(llm): strip leaked reasoning from self-hosted model output Self-hosted reasoning models served through vLLM or llama.cpp without a reasoning parser, or through Ollama without a thinking template, return their chain of thought inside message content. The OpenAI-compatible providers and the Ollama provider passed it through verbatim, so LLMOutput.content and ReActAgent final answers began with the reasoning. Models whose chat template prefills in the generation prompt (Spark-X2.5, Qwen3 thinking variants) only emit the closing tag, so the content arrives as "...answer" with no opening tag. Add strip_reasoning() and apply it in the OpenAI, Astraflow, Atlas and Ollama providers. A that does not open the output is kept as a literal, and unterminated reasoning is left unchanged. Signed-off-by: FenjuFu --- src/llm/providers/astraflow.py | 3 +- src/llm/providers/atlas.py | 3 +- src/llm/providers/ollama.py | 3 +- src/llm/providers/openai.py | 3 +- src/llm/providers/reasoning.py | 33 +++++++++++++++++++++ tests/test_reasoning.py | 54 ++++++++++++++++++++++++++++++++++ 6 files changed, 95 insertions(+), 4 deletions(-) create mode 100644 src/llm/providers/reasoning.py create mode 100644 tests/test_reasoning.py diff --git a/src/llm/providers/astraflow.py b/src/llm/providers/astraflow.py index 1f7cceab9..5bf2433f3 100644 --- a/src/llm/providers/astraflow.py +++ b/src/llm/providers/astraflow.py @@ -16,6 +16,7 @@ from llm.core.interface import ( ) from llm.core.types import LLMInput, LLMOutput, ModelInfo, ProviderType, ToolCall from llm.providers.constants import EMPTY_FILTERED_RESPONSE_ERROR +from llm.providers.reasoning import strip_reasoning ASTRAFLOW_BASE_URL = "https://api.umodelverse.ai/v1" ASTRAFLOW_CN_BASE_URL = "https://api.modelverse.cn/v1" @@ -104,7 +105,7 @@ class _AstraflowBaseProvider(LLMProvider): } return LLMOutput( - content=choice.message.content or "", + content=strip_reasoning(choice.message.content or ""), tool_calls=tool_calls, model=response.model, usage=usage, diff --git a/src/llm/providers/atlas.py b/src/llm/providers/atlas.py index b266bb614..ccb042055 100644 --- a/src/llm/providers/atlas.py +++ b/src/llm/providers/atlas.py @@ -16,6 +16,7 @@ from llm.core.interface import ( ) from llm.core.types import LLMInput, LLMOutput, ModelInfo, ProviderType, ToolCall from llm.providers.constants import EMPTY_FILTERED_RESPONSE_ERROR +from llm.providers.reasoning import strip_reasoning ATLAS_BASE_URL = "https://api.atlascloud.ai/v1" DEFAULT_ATLAS_MODEL = "deepseek-ai/deepseek-v4-pro" @@ -121,7 +122,7 @@ class AtlasProvider(LLMProvider): } return LLMOutput( - content=choice.message.content or "", + content=strip_reasoning(choice.message.content or ""), tool_calls=tool_calls, model=response.model, usage=usage, diff --git a/src/llm/providers/ollama.py b/src/llm/providers/ollama.py index 58505ac7b..1cfc5120a 100644 --- a/src/llm/providers/ollama.py +++ b/src/llm/providers/ollama.py @@ -19,6 +19,7 @@ from llm.core.types import ( ProviderType, ToolCall, ) +from llm.providers.reasoning import strip_reasoning class OllamaProvider(LLMProvider): @@ -85,7 +86,7 @@ class OllamaProvider(LLMProvider): with urllib.request.urlopen(req, timeout=60) as response: result = json.loads(response.read().decode("utf-8")) - content = result.get("message", {}).get("content", "") + content = strip_reasoning(result.get("message", {}).get("content", "")) tool_calls = None if result.get("message", {}).get("tool_calls"): diff --git a/src/llm/providers/openai.py b/src/llm/providers/openai.py index 0bf84a33d..22ab043ea 100644 --- a/src/llm/providers/openai.py +++ b/src/llm/providers/openai.py @@ -22,6 +22,7 @@ from llm.core.types import ( ToolCall, ) from llm.providers.constants import EMPTY_FILTERED_RESPONSE_ERROR +from llm.providers.reasoning import strip_reasoning class OpenAIProvider(LLMProvider): @@ -105,7 +106,7 @@ class OpenAIProvider(LLMProvider): } return LLMOutput( - content=choice.message.content or "", + content=strip_reasoning(choice.message.content or ""), tool_calls=tool_calls, model=response.model, usage=usage, diff --git a/src/llm/providers/reasoning.py b/src/llm/providers/reasoning.py new file mode 100644 index 000000000..497ab8cf4 --- /dev/null +++ b/src/llm/providers/reasoning.py @@ -0,0 +1,33 @@ +"""Helpers for reasoning text that self-hosted models leave in message content.""" + +from __future__ import annotations + +THINK_OPEN = "" +THINK_CLOSE = "" + + +def strip_reasoning(content: str) -> str: + """Drop a leading reasoning block from model output. + + Hosted APIs return reasoning in a separate field, but self-hosted reasoning + models (vLLM or llama.cpp without a reasoning parser, Ollama models without + a thinking template) put it in ``content``. It arrives in two shapes: + + - ``...answer`` + - ``...answer`` when the chat template prefills the opening tag in + the generation prompt, so the model only ever emits the closing tag. + + A ```` that does not open the output is treated as a literal and + left alone, as is unterminated reasoning (for example when ``max_tokens`` + cut the model off before it reached an answer). + """ + close = content.find(THINK_CLOSE) + if close == -1: + return content + + head = content[:close] + opening = head.find(THINK_OPEN) + if opening != -1 and head[:opening].strip(): + return content + + return content[close + len(THINK_CLOSE) :].lstrip() diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py new file mode 100644 index 000000000..8ff274908 --- /dev/null +++ b/tests/test_reasoning.py @@ -0,0 +1,54 @@ +import urllib.request +from io import BytesIO +from types import SimpleNamespace + +import pytest + +from llm.core.types import LLMInput, Message, Role +from llm.providers.ollama import OllamaProvider +from llm.providers.openai import OpenAIProvider +from llm.providers.reasoning import strip_reasoning + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + ("17 * 3 = 51.51", "51"), + ("\n\nplan\n\n\nanswer", "answer"), + ("17 * 3 = 51.51", "51"), + ("plain answer", "plain answer"), + ("", ""), + ("Wrap it in and tags.", "Wrap it in and tags."), + ("cut off by max_tokens", "cut off by max_tokens"), + ], +) +def test_strip_reasoning(content, expected): + assert strip_reasoning(content) == expected + + +def test_openai_provider_strips_inline_reasoning(): + provider = OpenAIProvider(api_key="test") + message = SimpleNamespace(content="17 * 3 = 51.51", tool_calls=None) + response = SimpleNamespace( + choices=[SimpleNamespace(message=message, finish_reason="stop")], + model="spark-x2.5", + usage=None, + ) + provider.client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **params: response)) + ) + + output = provider.generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")])) + + assert output.content == "51" + + +def test_ollama_provider_strips_prefilled_reasoning(monkeypatch): + def fake_urlopen(request, timeout): + return BytesIO(b'{"message": {"content": "17 * 3 = 51.51"}, "done_reason": "stop"}') + + monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + + output = OllamaProvider().generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")])) + + assert output.content == "51" From 2861dee17672727e201268140f9e82022abda15e Mon Sep 17 00:00:00 2001 From: FenjuFu <92919259+FenjuFu@users.noreply.github.com> Date: Fri, 2 Oct 2026 11:03:17 +0800 Subject: [PATCH 2/5] fix(llm): only strip close-only reasoning when the deployment declares it Strip a balanced ... block that opens the reply by default and leave every other reply untouched, so answers that mention the tags (a lone or reversed closing tag, JSON string values) are never truncated. The close-only form produced by chat templates that prefill is now handled only when the OpenAI or Ollama provider is configured for it, via prefilled_reasoning=True or OPENAI_PREFILLED_REASONING / OLLAMA_PREFILLED_REASONING. Hosted providers keep the default. Signed-off-by: FenjuFu <92919259+FenjuFu@users.noreply.github.com> --- src/llm/providers/ollama.py | 14 +++- src/llm/providers/openai.py | 18 ++++- src/llm/providers/reasoning.py | 46 ++++++++---- tests/test_reasoning.py | 127 +++++++++++++++++++++++++++++---- 4 files changed, 171 insertions(+), 34 deletions(-) diff --git a/src/llm/providers/ollama.py b/src/llm/providers/ollama.py index 1cfc5120a..6ee4aa8d4 100644 --- a/src/llm/providers/ollama.py +++ b/src/llm/providers/ollama.py @@ -19,7 +19,7 @@ from llm.core.types import ( ProviderType, ToolCall, ) -from llm.providers.reasoning import strip_reasoning +from llm.providers.reasoning import prefilled_reasoning_from_env, strip_reasoning class OllamaProvider(LLMProvider): @@ -29,7 +29,15 @@ class OllamaProvider(LLMProvider): self, base_url: str | None = None, default_model: str | None = None, + prefilled_reasoning: bool | None = None, ) -> None: + # Set for models whose chat template prefills , so replies arrive + # as "...answer" without the opening tag. + self.prefilled_reasoning = ( + prefilled_reasoning + if prefilled_reasoning is not None + else prefilled_reasoning_from_env("OLLAMA_PREFILLED_REASONING") + ) self.base_url = base_url or os.environ.get("OLLAMA_BASE_URL", "http://localhost:11434") self.default_model = default_model or os.environ.get("OLLAMA_MODEL", "llama3.2") self._models = [ @@ -86,7 +94,9 @@ class OllamaProvider(LLMProvider): with urllib.request.urlopen(req, timeout=60) as response: result = json.loads(response.read().decode("utf-8")) - content = strip_reasoning(result.get("message", {}).get("content", "")) + content = strip_reasoning( + result.get("message", {}).get("content", ""), prefilled=self.prefilled_reasoning + ) tool_calls = None if result.get("message", {}).get("tool_calls"): diff --git a/src/llm/providers/openai.py b/src/llm/providers/openai.py index 22ab043ea..011c71fcf 100644 --- a/src/llm/providers/openai.py +++ b/src/llm/providers/openai.py @@ -22,13 +22,25 @@ from llm.core.types import ( ToolCall, ) from llm.providers.constants import EMPTY_FILTERED_RESPONSE_ERROR -from llm.providers.reasoning import strip_reasoning +from llm.providers.reasoning import prefilled_reasoning_from_env, strip_reasoning class OpenAIProvider(LLMProvider): provider_type = ProviderType.OPENAI - def __init__(self, api_key: str | None = None, base_url: str | None = None) -> None: + def __init__( + self, + api_key: str | None = None, + base_url: str | None = None, + prefilled_reasoning: bool | None = None, + ) -> None: + # Set for self-hosted servers whose chat template prefills , so + # replies arrive as "...answer" without the opening tag. + self.prefilled_reasoning = ( + prefilled_reasoning + if prefilled_reasoning is not None + else prefilled_reasoning_from_env("OPENAI_PREFILLED_REASONING") + ) self.client = OpenAI( api_key=api_key or os.environ.get("OPENAI_API_KEY"), base_url=base_url, @@ -106,7 +118,7 @@ class OpenAIProvider(LLMProvider): } return LLMOutput( - content=strip_reasoning(choice.message.content or ""), + content=strip_reasoning(choice.message.content or "", prefilled=self.prefilled_reasoning), tool_calls=tool_calls, model=response.model, usage=usage, diff --git a/src/llm/providers/reasoning.py b/src/llm/providers/reasoning.py index 497ab8cf4..4f2add4ef 100644 --- a/src/llm/providers/reasoning.py +++ b/src/llm/providers/reasoning.py @@ -2,32 +2,48 @@ from __future__ import annotations +import os + THINK_OPEN = "" THINK_CLOSE = "" +_TRUTHY = {"1", "true", "yes", "on"} -def strip_reasoning(content: str) -> str: + +def prefilled_reasoning_from_env(name: str) -> bool: + """Read a ``*_PREFILLED_REASONING`` flag from the environment.""" + return os.environ.get(name, "").strip().lower() in _TRUTHY + + +def strip_reasoning(content: str, *, prefilled: bool = False) -> str: """Drop a leading reasoning block from model output. Hosted APIs return reasoning in a separate field, but self-hosted reasoning models (vLLM or llama.cpp without a reasoning parser, Ollama models without - a thinking template) put it in ``content``. It arrives in two shapes: + a thinking template) put it in ``content``. - - ``...answer`` - - ``...answer`` when the chat template prefills the opening tag in - the generation prompt, so the model only ever emits the closing tag. + By default only a balanced block that opens the output is removed: + ``...answer``. Anything else is returned unchanged, so an + answer that merely mentions the tags (including a lone or reversed closing + tag) is never truncated, and neither is unterminated reasoning (for example + when ``max_tokens`` cut the model off before it reached an answer). - A ```` that does not open the output is treated as a literal and - left alone, as is unterminated reasoning (for example when ``max_tokens`` - cut the model off before it reached an answer). + ``prefilled=True`` is for deployments whose chat template prefills the + opening tag in the generation prompt, so the model only ever emits + ``...answer``. Only enable it when the model is known to behave this + way: everything up to the first closing tag is then treated as reasoning. """ + stripped = content.lstrip() + if stripped.startswith(THINK_OPEN): + close = stripped.find(THINK_CLOSE, len(THINK_OPEN)) + if close == -1: + return content + return stripped[close + len(THINK_CLOSE) :].lstrip() + + if not prefilled: + return content + close = content.find(THINK_CLOSE) - if close == -1: + if close == -1 or THINK_OPEN in content[:close]: return content - - head = content[:close] - opening = head.find(THINK_OPEN) - if opening != -1 and head[:opening].strip(): - return content - return content[close + len(THINK_CLOSE) :].lstrip() diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py index 8ff274908..5ac45563d 100644 --- a/tests/test_reasoning.py +++ b/tests/test_reasoning.py @@ -1,3 +1,4 @@ +import json import urllib.request from io import BytesIO from types import SimpleNamespace @@ -5,30 +6,63 @@ from types import SimpleNamespace import pytest from llm.core.types import LLMInput, Message, Role +from llm.providers.astraflow import AstraflowProvider from llm.providers.ollama import OllamaProvider from llm.providers.openai import OpenAIProvider from llm.providers.reasoning import strip_reasoning +# Answers that mention the tags without carrying reasoning. None of them may be +# changed, whatever the deployment's reasoning format. +LITERAL_ANSWERS = [ + "Wrap it in and tags.", + "Close it with and nothing else.", + " comes before here.", + '{"closing":"","opening":""}', + '{"note": "x"}', +] + @pytest.mark.parametrize( ("content", "expected"), [ ("17 * 3 = 51.51", "51"), ("\n\nplan\n\n\nanswer", "answer"), - ("17 * 3 = 51.51", "51"), + ('build the object{"answer": 51}', '{"answer": 51}'), + ('plan{"closing":""}', '{"closing":""}'), ("plain answer", "plain answer"), ("", ""), - ("Wrap it in and tags.", "Wrap it in and tags."), ("cut off by max_tokens", "cut off by max_tokens"), + # Close-only output is ambiguous without a declared format, so it stays. + ("17 * 3 = 51.51", "17 * 3 = 51.51"), ], ) -def test_strip_reasoning(content, expected): +def test_strip_reasoning_default(content, expected): assert strip_reasoning(content) == expected -def test_openai_provider_strips_inline_reasoning(): - provider = OpenAIProvider(api_key="test") - message = SimpleNamespace(content="17 * 3 = 51.51", tool_calls=None) +@pytest.mark.parametrize("content", LITERAL_ANSWERS) +def test_strip_reasoning_default_preserves_literal_tags(content): + assert strip_reasoning(content) == content + + +@pytest.mark.parametrize( + ("content", "expected"), + [ + ("17 * 3 = 51.51", "51"), + ('build the object\n\n{"answer": 51}', '{"answer": 51}'), + ("17 * 3 = 51.51", "51"), + ("plain answer", "plain answer"), + ("Wrap it in and tags.", "Wrap it in and tags."), + ("17 * 3 = 51.", ""), + ], +) +def test_strip_reasoning_prefilled(content, expected): + assert strip_reasoning(content, prefilled=True) == expected + + +def _openai_provider(content, **kwargs): + provider = OpenAIProvider(api_key="test", **kwargs) + message = SimpleNamespace(content=content, tool_calls=None) response = SimpleNamespace( choices=[SimpleNamespace(message=message, finish_reason="stop")], model="spark-x2.5", @@ -37,18 +71,83 @@ def test_openai_provider_strips_inline_reasoning(): provider.client = SimpleNamespace( chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **params: response)) ) - - output = provider.generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")])) - - assert output.content == "51" + return provider -def test_ollama_provider_strips_prefilled_reasoning(monkeypatch): +def _ask(provider, **kwargs): + return provider.generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")], **kwargs)) + + +def test_openai_provider_strips_balanced_reasoning(): + assert _ask(_openai_provider("17 * 3 = 51.51")).content == "51" + + +@pytest.mark.parametrize("content", LITERAL_ANSWERS) +def test_openai_provider_keeps_literal_tags(content): + assert _ask(_openai_provider(content)).content == content + + +def test_openai_provider_close_only_needs_opt_in(monkeypatch): + monkeypatch.delenv("OPENAI_PREFILLED_REASONING", raising=False) + assert _ask(_openai_provider("17 * 3 = 51.51")).content == "17 * 3 = 51.51" + assert _ask(_openai_provider("17 * 3 = 51.51", prefilled_reasoning=True)).content == "51" + + monkeypatch.setenv("OPENAI_PREFILLED_REASONING", "true") + assert _ask(_openai_provider("17 * 3 = 51.51")).content == "51" + + +def test_hosted_provider_never_strips_close_only(monkeypatch): + monkeypatch.setenv("OPENAI_PREFILLED_REASONING", "true") + provider = AstraflowProvider(api_key="test") + message = SimpleNamespace(content='{"closing":""}', tool_calls=None) + response = SimpleNamespace( + choices=[SimpleNamespace(message=message, finish_reason="stop")], + model="deepseek-v3.2", + usage=None, + ) + provider.client = SimpleNamespace( + chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **params: response)) + ) + + assert _ask(provider).content == '{"closing":""}' + + +def _fake_ollama(monkeypatch, content): + sent = [] + def fake_urlopen(request, timeout): - return BytesIO(b'{"message": {"content": "17 * 3 = 51.51"}, "done_reason": "stop"}') + sent.append(json.loads(request.data)) + return BytesIO(json.dumps({"message": {"content": content}, "done_reason": "stop"}).encode()) monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) + return sent - output = OllamaProvider().generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")])) - assert output.content == "51" +@pytest.mark.parametrize("content", LITERAL_ANSWERS) +def test_ollama_provider_keeps_literal_tags(monkeypatch, content): + monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False) + _fake_ollama(monkeypatch, content) + + assert _ask(OllamaProvider()).content == content + + +def test_ollama_provider_strips_prefilled_reasoning_when_declared(monkeypatch): + monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False) + _fake_ollama(monkeypatch, "17 * 3 = 51.51") + assert _ask(OllamaProvider()).content == "17 * 3 = 51.51" + assert _ask(OllamaProvider(prefilled_reasoning=True)).content == "51" + + monkeypatch.setenv("OLLAMA_PREFILLED_REASONING", "1") + assert _ask(OllamaProvider()).content == "51" + + +def test_ollama_stream_flag_still_strips_complete_message(monkeypatch): + # The provider always requests a single complete message, even when the + # caller asks for streaming, so stripping sees the whole reply. If chunked + # streaming is added, this needs an incremental stripper instead. + sent = _fake_ollama(monkeypatch, 'plan{"closing":""}') + + output = _ask(OllamaProvider(), stream=True) + + assert sent[0]["stream"] is False + assert output.content == '{"closing":""}' From af325fbb85086d4b5173affcde37df4d9342f237 Mon Sep 17 00:00:00 2001 From: FenjuFu Date: Sat, 3 Oct 2026 01:41:21 +0800 Subject: [PATCH 3/5] fix(llm): keep complete JSON answers whole in prefilled reasoning mode With prefilled reasoning enabled, a structured answer such as {"closing":""} was cut at the tag inside its string value. Reasoning followed by a closing tag is never a valid JSON document, so a reply that parses as one complete object or array is now returned unchanged. Also annotate the tests and helpers this PR adds. Signed-off-by: FenjuFu --- src/llm/providers/reasoning.py | 19 +++++++++++++- tests/test_reasoning.py | 46 ++++++++++++++++++++-------------- 2 files changed, 45 insertions(+), 20 deletions(-) diff --git a/src/llm/providers/reasoning.py b/src/llm/providers/reasoning.py index 4f2add4ef..cbb8b1270 100644 --- a/src/llm/providers/reasoning.py +++ b/src/llm/providers/reasoning.py @@ -2,6 +2,7 @@ from __future__ import annotations +import json import os THINK_OPEN = "" @@ -44,6 +45,22 @@ def strip_reasoning(content: str, *, prefilled: bool = False) -> str: return content close = content.find(THINK_CLOSE) - if close == -1 or THINK_OPEN in content[:close]: + if close == -1 or THINK_OPEN in content[:close] or _is_json_document(content): return content return content[close + len(THINK_CLOSE) :].lstrip() + + +def _is_json_document(content: str) -> bool: + """Whether ``content`` is one complete JSON object or array. + + Reasoning followed by a closing tag is never valid JSON, so a structured + answer that carries the tag inside a string value is left whole. + """ + stripped = content.strip() + if not stripped.startswith(("{", "[")): + return False + try: + json.loads(stripped) + except ValueError: + return False + return True diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py index 5ac45563d..b2ade433b 100644 --- a/tests/test_reasoning.py +++ b/tests/test_reasoning.py @@ -2,10 +2,12 @@ import json import urllib.request from io import BytesIO from types import SimpleNamespace +from typing import Any, cast import pytest -from llm.core.types import LLMInput, Message, Role +from llm.core.interface import LLMProvider +from llm.core.types import LLMInput, LLMOutput, Message, Role from llm.providers.astraflow import AstraflowProvider from llm.providers.ollama import OllamaProvider from llm.providers.openai import OpenAIProvider @@ -36,12 +38,12 @@ LITERAL_ANSWERS = [ ("17 * 3 = 51.51", "17 * 3 = 51.51"), ], ) -def test_strip_reasoning_default(content, expected): +def test_strip_reasoning_default(content: str, expected: str) -> None: assert strip_reasoning(content) == expected @pytest.mark.parametrize("content", LITERAL_ANSWERS) -def test_strip_reasoning_default_preserves_literal_tags(content): +def test_strip_reasoning_default_preserves_literal_tags(content: str) -> None: assert strip_reasoning(content) == content @@ -54,13 +56,17 @@ def test_strip_reasoning_default_preserves_literal_tags(content): ("plain answer", "plain answer"), ("Wrap it in and tags.", "Wrap it in and tags."), ("17 * 3 = 51.", ""), + # A complete JSON answer is never reasoning plus a closing tag. + ('{"closing":""}', '{"closing":""}'), + (' {"closing":"","opening":""}\n', ' {"closing":"","opening":""}\n'), + ('["", 1]', '["", 1]'), ], ) -def test_strip_reasoning_prefilled(content, expected): +def test_strip_reasoning_prefilled(content: str, expected: str) -> None: assert strip_reasoning(content, prefilled=True) == expected -def _openai_provider(content, **kwargs): +def _openai_provider(content: str, **kwargs: Any) -> OpenAIProvider: provider = OpenAIProvider(api_key="test", **kwargs) message = SimpleNamespace(content=content, tool_calls=None) response = SimpleNamespace( @@ -68,26 +74,27 @@ def _openai_provider(content, **kwargs): model="spark-x2.5", usage=None, ) - provider.client = SimpleNamespace( + client: Any = SimpleNamespace( chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **params: response)) ) + provider.client = client return provider -def _ask(provider, **kwargs): +def _ask(provider: LLMProvider, **kwargs: Any) -> LLMOutput: return provider.generate(LLMInput(messages=[Message(role=Role.USER, content="17*3?")], **kwargs)) -def test_openai_provider_strips_balanced_reasoning(): +def test_openai_provider_strips_balanced_reasoning() -> None: assert _ask(_openai_provider("17 * 3 = 51.51")).content == "51" @pytest.mark.parametrize("content", LITERAL_ANSWERS) -def test_openai_provider_keeps_literal_tags(content): +def test_openai_provider_keeps_literal_tags(content: str) -> None: assert _ask(_openai_provider(content)).content == content -def test_openai_provider_close_only_needs_opt_in(monkeypatch): +def test_openai_provider_close_only_needs_opt_in(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("OPENAI_PREFILLED_REASONING", raising=False) assert _ask(_openai_provider("17 * 3 = 51.51")).content == "17 * 3 = 51.51" assert _ask(_openai_provider("17 * 3 = 51.51", prefilled_reasoning=True)).content == "51" @@ -96,7 +103,7 @@ def test_openai_provider_close_only_needs_opt_in(monkeypatch): assert _ask(_openai_provider("17 * 3 = 51.51")).content == "51" -def test_hosted_provider_never_strips_close_only(monkeypatch): +def test_hosted_provider_never_strips_close_only(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setenv("OPENAI_PREFILLED_REASONING", "true") provider = AstraflowProvider(api_key="test") message = SimpleNamespace(content='{"closing":""}', tool_calls=None) @@ -105,18 +112,19 @@ def test_hosted_provider_never_strips_close_only(monkeypatch): model="deepseek-v3.2", usage=None, ) - provider.client = SimpleNamespace( + client: Any = SimpleNamespace( chat=SimpleNamespace(completions=SimpleNamespace(create=lambda **params: response)) ) + provider.client = client assert _ask(provider).content == '{"closing":""}' -def _fake_ollama(monkeypatch, content): - sent = [] +def _fake_ollama(monkeypatch: pytest.MonkeyPatch, content: str) -> list[dict[str, Any]]: + sent: list[dict[str, Any]] = [] - def fake_urlopen(request, timeout): - sent.append(json.loads(request.data)) + def fake_urlopen(request: urllib.request.Request, timeout: float) -> BytesIO: + sent.append(json.loads(cast(bytes, request.data))) return BytesIO(json.dumps({"message": {"content": content}, "done_reason": "stop"}).encode()) monkeypatch.setattr(urllib.request, "urlopen", fake_urlopen) @@ -124,14 +132,14 @@ def _fake_ollama(monkeypatch, content): @pytest.mark.parametrize("content", LITERAL_ANSWERS) -def test_ollama_provider_keeps_literal_tags(monkeypatch, content): +def test_ollama_provider_keeps_literal_tags(monkeypatch: pytest.MonkeyPatch, content: str) -> None: monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False) _fake_ollama(monkeypatch, content) assert _ask(OllamaProvider()).content == content -def test_ollama_provider_strips_prefilled_reasoning_when_declared(monkeypatch): +def test_ollama_provider_strips_prefilled_reasoning_when_declared(monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.delenv("OLLAMA_PREFILLED_REASONING", raising=False) _fake_ollama(monkeypatch, "17 * 3 = 51.51") assert _ask(OllamaProvider()).content == "17 * 3 = 51.51" @@ -141,7 +149,7 @@ def test_ollama_provider_strips_prefilled_reasoning_when_declared(monkeypatch): assert _ask(OllamaProvider()).content == "51" -def test_ollama_stream_flag_still_strips_complete_message(monkeypatch): +def test_ollama_stream_flag_still_strips_complete_message(monkeypatch: pytest.MonkeyPatch) -> None: # The provider always requests a single complete message, even when the # caller asks for streaming, so stripping sees the whole reply. If chunked # streaming is added, this needs an incremental stripper instead. From d351fc550208c0a8db5b9f640be63cfcf4b11238 Mon Sep 17 00:00:00 2001 From: FenjuFu Date: Sun, 4 Oct 2026 16:31:47 +0800 Subject: [PATCH 4/5] fix(llm): fall back when a prefilled answer is too deeply nested to parse json.loads raises RecursionError on deeply nested input, which escaped the complete-JSON check and failed the request. Treat it like any other input that is not a complete JSON document. Signed-off-by: FenjuFu --- src/llm/providers/reasoning.py | 6 ++++-- tests/test_reasoning.py | 7 +++++++ 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/llm/providers/reasoning.py b/src/llm/providers/reasoning.py index cbb8b1270..752132292 100644 --- a/src/llm/providers/reasoning.py +++ b/src/llm/providers/reasoning.py @@ -54,13 +54,15 @@ def _is_json_document(content: str) -> bool: """Whether ``content`` is one complete JSON object or array. Reasoning followed by a closing tag is never valid JSON, so a structured - answer that carries the tag inside a string value is left whole. + answer that carries the tag inside a string value is left whole. Input + nested too deeply to parse is not treated as JSON rather than failing the + request. """ stripped = content.strip() if not stripped.startswith(("{", "[")): return False try: json.loads(stripped) - except ValueError: + except (ValueError, RecursionError): return False return True diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py index b2ade433b..3cbee1ef7 100644 --- a/tests/test_reasoning.py +++ b/tests/test_reasoning.py @@ -66,6 +66,13 @@ def test_strip_reasoning_prefilled(content: str, expected: str) -> None: assert strip_reasoning(content, prefilled=True) == expected +def test_strip_reasoning_prefilled_survives_deeply_nested_json() -> None: + # Deep enough for json.loads to raise RecursionError on every supported Python. + depth = 100_000 + content = "[" * depth + '""' + "]" * depth + assert strip_reasoning(content, prefilled=True) == '"' + "]" * depth + + def _openai_provider(content: str, **kwargs: Any) -> OpenAIProvider: provider = OpenAIProvider(api_key="test", **kwargs) message = SimpleNamespace(content=content, tool_calls=None) From 586d4b91cc4c1afa04e261f28f3547feb3fbeebf Mon Sep 17 00:00:00 2001 From: FenjuFu Date: Mon, 5 Oct 2026 01:19:14 +0800 Subject: [PATCH 5/5] fix(llm): keep answers too deeply nested to parse whole in prefilled mode Signed-off-by: FenjuFu --- src/llm/providers/reasoning.py | 9 ++++++--- tests/test_reasoning.py | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/src/llm/providers/reasoning.py b/src/llm/providers/reasoning.py index 752132292..4a45c1fc1 100644 --- a/src/llm/providers/reasoning.py +++ b/src/llm/providers/reasoning.py @@ -55,14 +55,17 @@ def _is_json_document(content: str) -> bool: Reasoning followed by a closing tag is never valid JSON, so a structured answer that carries the tag inside a string value is left whole. Input - nested too deeply to parse is not treated as JSON rather than failing the - request. + nested too deeply for the parser to finish is kept whole as well: reasoning + does not open with that many brackets, so cutting it would only risk + truncating a real answer. """ stripped = content.strip() if not stripped.startswith(("{", "[")): return False try: json.loads(stripped) - except (ValueError, RecursionError): + except RecursionError: + return True + except ValueError: return False return True diff --git a/tests/test_reasoning.py b/tests/test_reasoning.py index 3cbee1ef7..c8d923c17 100644 --- a/tests/test_reasoning.py +++ b/tests/test_reasoning.py @@ -70,7 +70,7 @@ def test_strip_reasoning_prefilled_survives_deeply_nested_json() -> None: # Deep enough for json.loads to raise RecursionError on every supported Python. depth = 100_000 content = "[" * depth + '""' + "]" * depth - assert strip_reasoning(content, prefilled=True) == '"' + "]" * depth + assert strip_reasoning(content, prefilled=True) == content def _openai_provider(content: str, **kwargs: Any) -> OpenAIProvider: