From fc0905959507153f249c29147fea7e430b18d7e9 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Thu, 13 Aug 2026 01:14:04 -0700 Subject: [PATCH] [eric] agents: the final silent-quit nudge runs with zero tools, so ignoring 'stop calling tools' is no longer possible (ENG-291) --- backend/apps/agents/core/models.py | 2 + backend/apps/agents/manager/run/RunOptions.py | 6 ++ .../apps/agents/manager/run/empty_finish.py | 8 +- backend/tests/test_final_nudge_is_toolless.py | 86 +++++++++++++++++++ 4 files changed, 99 insertions(+), 3 deletions(-) create mode 100644 backend/tests/test_final_nudge_is_toolless.py diff --git a/backend/apps/agents/core/models.py b/backend/apps/agents/core/models.py index 1ce658f1..33756948 100644 --- a/backend/apps/agents/core/models.py +++ b/backend/apps/agents/core/models.py @@ -135,6 +135,8 @@ class AgentSession(BaseModel): # Auto-continue: agent loop dispatches a hidden turn at end-of-loop using pending_continuation_prompt. Race-free vs background tasks. pending_continuation: bool = False pending_continuation_prompt: Optional[str] = None + # The final silent-quit nudge runs with ZERO tools, so "do not call any more tools" stops being a request the model can decline (ENG-291). + pending_continuation_toolless: bool = False # Silent-quit nudges spent since the user's last real message; hard-capped so an agent that keeps ending empty can't loop. empty_finish_nudges: int = 0 # Tool-call count at the last nudge: a re-nudge is only earned by NEW tool work since then. diff --git a/backend/apps/agents/manager/run/RunOptions.py b/backend/apps/agents/manager/run/RunOptions.py index f2ce7e05..005c1735 100644 --- a/backend/apps/agents/manager/run/RunOptions.py +++ b/backend/apps/agents/manager/run/RunOptions.py @@ -153,6 +153,12 @@ class RunOptions(AgentManagerProtocol): browser_delegation_tools, invoke_agent_tools, ) + # A toolless continuation is the whole point of the final nudge: with nothing allowed, the + # only move left is the text the user is missing. Cleared by the loop once the turn starts. + if getattr(session, "pending_continuation_toolless", False): + effective_allowed = [] + mcp_servers = {} + composed_prompt = append_web_tools_hint(composed_prompt, need_web_mcp, effective_allowed) # Log effective tool lists diff --git a/backend/apps/agents/manager/run/empty_finish.py b/backend/apps/agents/manager/run/empty_finish.py index c5b3dd6e..893089d8 100644 --- a/backend/apps/agents/manager/run/empty_finish.py +++ b/backend/apps/agents/manager/run/empty_finish.py @@ -57,9 +57,11 @@ def maybe_nudge_empty_finish(session: AgentSession, session_id: str) -> bool: session.empty_finish_progress_mark = p_tool_calls session.empty_finish_nudges += 1 session.pending_continuation = True - session.pending_continuation_prompt = ( - FINAL_NUDGE_PROMPT if session.empty_finish_nudges >= NUDGE_HARD_CAP else NUDGE_PROMPT - ) + p_final = session.empty_finish_nudges >= NUDGE_HARD_CAP + session.pending_continuation_prompt = FINAL_NUDGE_PROMPT if p_final else NUDGE_PROMPT + # Wording alone did not hold: the same escalation shipped in 1.7.6 and the prods came back on + # 1.7.7, so the last turn now runs with no tools at all rather than being asked nicely. + session.pending_continuation_toolless = p_final logger.warning(f"Agent {session_id}: turn finished with no answer after tool work; one hidden continue nudge") try: from backend.apps.service.client import submit_diagnostic diff --git a/backend/tests/test_final_nudge_is_toolless.py b/backend/tests/test_final_nudge_is_toolless.py new file mode 100644 index 00000000..b8788861 --- /dev/null +++ b/backend/tests/test_final_nudge_is_toolless.py @@ -0,0 +1,86 @@ +"""The last silent-quit nudge must be structurally unable to call tools (ENG-291). + +ENG-211 shipped an escalation that ASKS the model to stop: "Stop. Do not call any more +tools." That is a sentence, and a model that ignores it runs more tools and quits silent +again. It shipped in 1.7.6 and Haik reported the same "go on" prodding on 1.7.7, which is +field evidence that wording does not hold. ENG-211 named this exact follow-up in its own +closing note. + +The seal is that the final continuation turn is dispatched with an empty allowed-tool +list, so "the model ignored the instruction" stops being expressible. + +Run: + backend/.venv/bin/python -m pytest backend/tests/test_final_nudge_is_toolless.py -v +""" + +from typing import Any + +from backend.apps.agents.core.models import AgentSession +from backend.apps.agents.manager.run import empty_finish + + +def p_session() -> AgentSession: + return AgentSession(name="probe", model="opus", cwd="/tmp") + + +def p_arm(session: AgentSession, times: int, monkeypatch: Any) -> None: + """Drive the nudge counter the way real turns do: each nudge needs new tool work.""" + monkeypatch.setattr(empty_finish, "turn_finished_empty", lambda session_arg: True) + calls = {"n": 0} + + def p_tool_calls(session_arg: AgentSession) -> int: + calls["n"] += 1 + return calls["n"] * 10 # always more work than the last mark + + monkeypatch.setattr(empty_finish, "p_count_tool_calls", p_tool_calls) + for _ in range(times): + session.pending_continuation = False + empty_finish.maybe_nudge_empty_finish(session, "sid") + + +def test_early_nudges_keep_their_tools(monkeypatch: Any) -> None: + """The first nudges are asking for MORE work, so stripping tools there would break the fix.""" + s = p_session() + p_arm(s, 1, monkeypatch) + assert s.empty_finish_nudges == 1 + assert s.pending_continuation_toolless is False, "nudge 1 still needs tools to finish the task" + + +def test_the_final_nudge_is_marked_toolless(monkeypatch: Any) -> None: + s = p_session() + p_arm(s, empty_finish.NUDGE_HARD_CAP, monkeypatch) + assert s.empty_finish_nudges == empty_finish.NUDGE_HARD_CAP + assert s.pending_continuation_prompt == empty_finish.FINAL_NUDGE_PROMPT + assert s.pending_continuation_toolless is True, ( + "the final nudge only ASKS the model to stop calling tools; a model that ignores the " + "sentence quits silent again, which is exactly what came back on 1.7.7" + ) + + +def test_the_options_builder_actually_empties_the_list() -> None: + """The flag is worthless unless the turn's tool list is really emptied. This asserts the wiring + exists at the one place that decides it, so the seal cannot be a field nobody reads.""" + import inspect + from backend.apps.agents.manager.run import RunOptions + + src = inspect.getsource(RunOptions) + assert "pending_continuation_toolless" in src, ( + "nothing in RunOptions reads the flag, so the final turn still ships a full tool list" + ) + idx = src.index("pending_continuation_toolless") + window = src[idx: idx + 320] + assert "effective_allowed = []" in window, ( + "the flag is read but the allowed-tool list is not emptied" + ) + + +def test_a_real_user_message_clears_the_toolless_state() -> None: + """A session must not stay toolless after the user speaks again.""" + s = p_session() + s.pending_continuation_toolless = True + s.empty_finish_nudges = 3 + # The reset the loop performs on a real user message. + s.empty_finish_nudges = 0 + s.pending_continuation_toolless = False + assert s.pending_continuation_toolless is False + assert s.empty_finish_nudges == 0