diff --git a/backend/apps/agents/agent_manager.py b/backend/apps/agents/agent_manager.py index 8ae07529..5270b927 100644 --- a/backend/apps/agents/agent_manager.py +++ b/backend/apps/agents/agent_manager.py @@ -156,6 +156,13 @@ class AgentManager(SessionLifecycle, SessionPersistence, Messaging, SessionContr ) session.status = "completed" + # Silent-quit seal: a turn that ran tools and ended with no visible answer gets ONE hidden continue nudge (dispatched by the auto-continuation block below); a second silent quit in the same ask surfaces as-is rather than looping. + try: + from backend.apps.agents.manager.run.empty_finish import maybe_nudge_empty_finish + maybe_nudge_empty_finish(session, session_id) + except Exception: + logger.exception("empty-finish detection failed; continuing") + # Auto-continuation hook (Phase 3). If MCPActivate (or any analogous flow) flagged pending_continuation during this turn, kick off a follow-up turn immediately with the captured prompt. We dispatch as a fire-and-forget task so the current run_agent_loop frame can unwind cleanly before the next turn's options + history rebuild kicks in. The follow-up is `hidden=True` so it doesn't add a user bubble to the visible chat; the model sees it as a synthetic prompt to keep working. try: if getattr(session, "pending_continuation", False): diff --git a/backend/apps/agents/core/models.py b/backend/apps/agents/core/models.py index 41964275..e2415437 100644 --- a/backend/apps/agents/core/models.py +++ b/backend/apps/agents/core/models.py @@ -133,6 +133,8 @@ class AgentSession(BaseModel): # Auto-continue: agent loop dispatches a hidden turn at end-of-loop using pending_continuation_prompt. Race-free vs background tasks. pending_continuation: bool = False pending_continuation_prompt: Optional[str] = None + # Silent-quit nudges spent since the user's last real message; capped at 1 so an agent that keeps ending empty can't loop. + empty_finish_nudges: int = 0 # Sanitized server names model has explicitly activated this session; _build_mcp_servers intersects connected MCPs with this. Non-bypassable; dispatch-layer gate. active_mcps: list[str] = Field(default_factory=list) # Heuristic preamble tokens (preset + tool defs + MCP descs + composed prompt); subtracted from displayed input. diff --git a/backend/apps/agents/manager/Messaging.py b/backend/apps/agents/manager/Messaging.py index 169f3e3f..daf5ba96 100644 --- a/backend/apps/agents/manager/Messaging.py +++ b/backend/apps/agents/manager/Messaging.py @@ -133,6 +133,9 @@ class Messaging(AgentManagerProtocol): "message": user_msg.model_dump(mode="json"), }) + # A real user message opens a fresh silent-quit budget; the cap only guards within one ask. + if not hidden: + session.empty_finish_nudges = 0 # Fire a background aux LLM call to generate a 3-6 word verb-phrase describing this turn ("Auditing the pull request", "Drafting your email"). The narrator pill swaps from its heuristic verb to this label as soon as it lands, usually ~500ms-1s into the turn, which is exactly when "Thinking…" starts feeling generic. Provider-agnostic via resolve_aux_model. Non-blocking; failure is silent and the heuristic stays. if not hidden and prompt: try: diff --git a/backend/apps/agents/manager/context_budget.py b/backend/apps/agents/manager/context_budget.py index 0d4cacfe..9310f1a9 100644 --- a/backend/apps/agents/manager/context_budget.py +++ b/backend/apps/agents/manager/context_budget.py @@ -52,6 +52,7 @@ def maybe_break_midturn(session: AgentSession, turn: TurnState, msg_usage: Dict) return False # Keep the session's counter honest mid-turn: a broken turn never gets its ResultMessage accounting, and the next pre-send guard reads this. session.tokens["input"] = total + turn.last_step_input = total if total < compact_trigger_tokens(session): turn.saw_input_below_trigger = True return False diff --git a/backend/apps/agents/manager/run/empty_finish.py b/backend/apps/agents/manager/run/empty_finish.py new file mode 100644 index 00000000..1389ae1c --- /dev/null +++ b/backend/apps/agents/manager/run/empty_finish.py @@ -0,0 +1,73 @@ +"""Detect a turn that ended without an answer: the model ran tools and then quit with a +thinking-only/empty end_turn, so the chat's last visible event is a tool result and the user +gets a Done pill with no response. Live incident (2026-08-03, opus-5-cc lint audit): the final +inference was a 2-char thinking block + end_turn at 70K/1M context, scored as a clean success. +The loop nudges such a turn ONCE with a hidden continuation; twice in a row surfaces honestly.""" + +import logging +from typing import List + +from typeguard import typechecked + +from backend.apps.agents.core.models import AgentSession +from backend.apps.agents.manager.session.history_compaction import get_branch_messages + +NUDGE_PROMPT = ( + "You ended your turn without reporting anything. Continue exactly where you left off and " + "finish the task; when done, always end with your findings or answer as normal text." +) + +logger = logging.getLogger(__name__) + + +@typechecked +def maybe_nudge_empty_finish(session: AgentSession, session_id: str) -> bool: + """Arm one hidden continue nudge when the finished turn quit silently; the loop's existing + auto-continuation block dispatches it. Capped per user ask; never loops.""" + if getattr(session, "pending_continuation", False) or session.empty_finish_nudges >= 1: + return False + if not turn_finished_empty(session): + return False + session.empty_finish_nudges += 1 + session.pending_continuation = True + session.pending_continuation_prompt = NUDGE_PROMPT + logger.warning(f"Agent {session_id}: turn finished with no answer after tool work; one hidden continue nudge") + try: + from backend.apps.service.client import submit_diagnostic + submit_diagnostic({"kind": "empty_finish_nudge", "session_id": session_id, "model": session.model}) + except Exception: + pass + return True + +# A turn legitimately ENDS on these tools: the rendered widget or delegation IS the answer. +P_ANSWER_TOOL_MARKERS = ("openswarm-ui", "ShowUI", "AskUI", "AskUserQuestion") + + +def p_tool_name_of(msg: object) -> str: + content = getattr(msg, "content", None) + if isinstance(content, dict): + return str(content.get("tool") or content.get("tool_name") or "") + return "" + + +@typechecked +def turn_finished_empty(session: AgentSession) -> bool: + """True when the branch's last visible message is a tool result whose call was ordinary work + (not a UI/answer tool): the model did things and then said nothing.""" + msgs: List = get_branch_messages(session) + p_last_call_name = "" + for m in reversed(msgs): + if getattr(m, "hidden", False): + continue + role = getattr(m, "role", "") + if role == "assistant": + text = m.content if isinstance(m.content, str) else "" + return not text.strip() + if role == "tool_result": + continue + if role == "tool_call": + p_last_call_name = p_tool_name_of(m) + return not any(marker in p_last_call_name for marker in P_ANSWER_TOOL_MARKERS) + if role in ("user", "system"): + return False + return False diff --git a/backend/apps/agents/manager/streaming/handle_result_message.py b/backend/apps/agents/manager/streaming/handle_result_message.py index e69b5d43..fae05a87 100644 --- a/backend/apps/agents/manager/streaming/handle_result_message.py +++ b/backend/apps/agents/manager/streaming/handle_result_message.py @@ -140,7 +140,9 @@ async def handle_result_message( cache_create = usage.get("cache_creation_input_tokens", 0) or 0 cache_read = usage.get("cache_read_input_tokens", 0) or 0 total_input = inp + cache_create + cache_read - session.tokens["input"] = total_input + # The result's input usage is summed across every inference step of the turn, which is BILLING; live context is the last step's request size. On a 9-step audit turn the sum read 589K while the real context was 70K, and the meter (plus the compaction trigger) believed it. + p_ctx_input = turn.last_step_input if turn.last_step_input > 0 else total_input + session.tokens["input"] = p_ctx_input session.tokens["input_fresh"] = inp session.tokens["output"] = out @@ -202,12 +204,12 @@ async def handle_result_message( if isinstance(usage, dict): # Per-turn context-usage broadcast. Drives the UI status pill and the auto-compact threshold. The denominator is the session's real model cap, populated from registry.get_context_window at session creation, restore, and model-switch (see apply_context_window). max(1, ...) is a belt-and-braces guard against zero/None drift from any future restore-from-disk corner case. ctx_window = max(1, getattr(session, "context_window", 0) or 200_000) - ctx_used_pct = round(total_input / ctx_window, 4) if total_input else 0.0 + ctx_used_pct = round(p_ctx_input / ctx_window, 4) if p_ctx_input else 0.0 cache_read_pct = round(cache_read / total_input, 4) if total_input else 0.0 try: await ws_manager.send_to_session(session_id, "agent:context_update", { "session_id": session_id, - "input_tokens": total_input, + "input_tokens": p_ctx_input, "output_tokens": out, "cache_read_tokens": cache_read, "cache_read_pct": cache_read_pct, diff --git a/backend/apps/agents/manager/streaming/state.py b/backend/apps/agents/manager/streaming/state.py index 6bb33398..96ca7c72 100644 --- a/backend/apps/agents/manager/streaming/state.py +++ b/backend/apps/agents/manager/streaming/state.py @@ -59,3 +59,5 @@ class TurnState(BaseModel): # Mid-turn context breaker: fires once per turn, and only after a below-trigger reading (a turn that STARTS over the trigger must run, or a failed shrink would break-loop forever). context_break_fired: bool = False saw_input_below_trigger: bool = False + # The LAST inference step's request size (input + cache read + cache creation): the true live context. The ResultMessage's usage sums these across every step of the turn, which is billing, not context. + last_step_input: int = 0 diff --git a/backend/tests/test_empty_finish.py b/backend/tests/test_empty_finish.py new file mode 100644 index 00000000..7b5d60ca --- /dev/null +++ b/backend/tests/test_empty_finish.py @@ -0,0 +1,99 @@ +"""The silent-quit seal: a turn that runs tools and ends with no visible answer gets one hidden +continue nudge, never a loop. Detector shapes pinned here; the loop wiring is pinned in +test_context_pressure_valve-style fashion against run_agent_loop.""" + +import asyncio + +from backend.apps.agents.agent_manager import agent_manager +import backend.apps.agents.agent_manager as agent_manager_module +from backend.apps.agents.core.models import AgentSession, Message +from backend.apps.agents.manager.run.empty_finish import NUDGE_PROMPT, turn_finished_empty + + +def p_session(*msgs) -> AgentSession: + s = AgentSession(name="t", model="sonnet") + for role, content in msgs: + s.messages.append(Message(role=role, content=content, branch_id="main")) + return s + + +def test_tool_result_tail_is_an_empty_finish(): + s = p_session(("user", "audit the repo"), + ("tool_call", {"tool": "Bash", "input": {"command": "ls"}}), + ("tool_result", {"text": "ok"})) + assert turn_finished_empty(s) is True + + +def test_final_answer_text_is_not_empty(): + s = p_session(("user", "audit"), + ("tool_call", {"tool": "Bash", "input": {}}), + ("tool_result", {"text": "ok"}), + ("assistant", "Here is the audit report.")) + assert turn_finished_empty(s) is False + + +def test_empty_assistant_text_is_an_empty_finish(): + s = p_session(("user", "audit"), ("assistant", "")) + assert turn_finished_empty(s) is True + + +def test_ui_answer_tools_are_a_legit_finish(): + s = p_session(("user", "show me"), + ("tool_call", {"tool": "mcp__openswarm-ui__ShowUI", "input": {}}), + ("tool_result", {"text": "rendered"})) + assert turn_finished_empty(s) is False + + +def test_plain_chat_answer_is_not_empty(): + s = p_session(("user", "hi"), ("assistant", "Hey! What can I do for you?")) + assert turn_finished_empty(s) is False + + +def test_bare_user_prompt_is_not_claimed(): + s = p_session(("user", "hi")) + assert turn_finished_empty(s) is False + + +def p_install_run_fakes(monkeypatch, run_turn_fake) -> None: + async def fake_build(session, session_id, prompt, prompt_content, builtin_perms, + selected_browser_ids, selected_app_output_ids, selected_setting_ids, + fork_session, router_model_id, api_type): + from backend.apps.settings.settings import load_settings + return object(), {}, prompt_content, [], load_settings() + + monkeypatch.setattr(agent_manager, "build_agent_options", fake_build) + monkeypatch.setattr(agent_manager, "run_turn_with_retry", run_turn_fake) + monkeypatch.setattr(agent_manager_module, "save_session", lambda sid, data: None) + + +def test_loop_nudges_a_silent_quit_once(monkeypatch) -> None: + session = AgentSession(name="t", model="sonnet", dashboard_id="d") + agent_manager.sessions[session.id] = session + continues: list = [] + + async def fake_run_turn(sess, session_id, prompt_content, options, options_kwargs, + turn, thinking, stderr, resolved_model, api_type, + global_settings, force_respawn=False): + # Every turn ends as a silent quit: tools ran, no answer text. + sess.messages.append(Message(role="tool_call", content={"tool": "Bash", "input": {}}, branch_id="main")) + sess.messages.append(Message(role="tool_result", content={"text": "out"}, branch_id="main")) + + async def fake_send_message(session_id, prompt, hidden=False, **kwargs): + continues.append({"prompt": prompt, "hidden": hidden}) + + p_install_run_fakes(monkeypatch, fake_run_turn) + monkeypatch.setattr(agent_manager, "send_message", fake_send_message) + + async def main(): + await agent_manager.run_agent_loop(session.id, "audit everything") + await asyncio.sleep(0) + + asyncio.run(main()) + assert continues == [{"prompt": NUDGE_PROMPT, "hidden": True}] + assert session.empty_finish_nudges == 1 + + # The nudged turn ALSO quits silently: the cap must hold, no second nudge, no loop. + continues.clear() + asyncio.run(main()) + assert continues == [] + assert session.empty_finish_nudges == 1 diff --git a/backend/tests/test_result_message.py b/backend/tests/test_result_message.py index 44ecad4d..da0ee0c3 100644 --- a/backend/tests/test_result_message.py +++ b/backend/tests/test_result_message.py @@ -114,3 +114,34 @@ async def test_resets_per_turn_state_at_completion(): assert turn.tool_count == 0 assert thinking.total_ms == 0 assert thinking.block_starts == {} + + +@pytest.mark.asyncio +async def test_context_meter_prefers_last_step_over_cumulative_billing(): + # A 9-step turn's result usage sums input across steps (billing); the meter must show the last step's request size (real context). The 925K/1M incident read the sum. + session, turn, thinking = p_fixt() + turn.last_step_input = 70_454 + payloads = [] + + async def fake_send(sid, ev, data): + if ev == "agent:context_update": + payloads.append(data) + + with patch.object(result_message.ws_manager, "send_to_session", AsyncMock(side_effect=fake_send)): + await result_message.handle_result_message( + p_result(usage={"input_tokens": 2_023, "cache_read_input_tokens": 500_000, "cache_creation_input_tokens": 86_972, "output_tokens": 1_210}), + session, "sid", turn, thinking, {}, "cc/claude-opus-5", "anthropic", load_settings(), + ) + assert session.tokens["input"] == 70_454 + assert payloads and payloads[0]["input_tokens"] == 70_454 + + +@pytest.mark.asyncio +async def test_context_meter_falls_back_to_result_usage_without_step_readings(): + session, turn, thinking = p_fixt() + with patch.object(result_message.ws_manager, "send_to_session", AsyncMock()): + await result_message.handle_result_message( + p_result(usage={"input_tokens": 1_000, "cache_read_input_tokens": 2_000, "output_tokens": 10}), + session, "sid", turn, thinking, {}, "sonnet", "anthropic", load_settings(), + ) + assert session.tokens["input"] == 3_000