mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] agents: a silent-quit turn gets one hidden continue nudge, and the context meter reads real context not summed billing
This commit is contained in:
@@ -156,6 +156,13 @@ class AgentManager(SessionLifecycle, SessionPersistence, Messaging, SessionContr
|
||||
)
|
||||
session.status = "completed"
|
||||
|
||||
# Silent-quit seal: a turn that ran tools and ended with no visible answer gets ONE hidden continue nudge (dispatched by the auto-continuation block below); a second silent quit in the same ask surfaces as-is rather than looping.
|
||||
try:
|
||||
from backend.apps.agents.manager.run.empty_finish import maybe_nudge_empty_finish
|
||||
maybe_nudge_empty_finish(session, session_id)
|
||||
except Exception:
|
||||
logger.exception("empty-finish detection failed; continuing")
|
||||
|
||||
# Auto-continuation hook (Phase 3). If MCPActivate (or any analogous flow) flagged pending_continuation during this turn, kick off a follow-up turn immediately with the captured prompt. We dispatch as a fire-and-forget task so the current run_agent_loop frame can unwind cleanly before the next turn's options + history rebuild kicks in. The follow-up is `hidden=True` so it doesn't add a user bubble to the visible chat; the model sees it as a synthetic prompt to keep working.
|
||||
try:
|
||||
if getattr(session, "pending_continuation", False):
|
||||
|
||||
@@ -133,6 +133,8 @@ class AgentSession(BaseModel):
|
||||
# Auto-continue: agent loop dispatches a hidden turn at end-of-loop using pending_continuation_prompt. Race-free vs background tasks.
|
||||
pending_continuation: bool = False
|
||||
pending_continuation_prompt: Optional[str] = None
|
||||
# Silent-quit nudges spent since the user's last real message; capped at 1 so an agent that keeps ending empty can't loop.
|
||||
empty_finish_nudges: int = 0
|
||||
# Sanitized server names model has explicitly activated this session; _build_mcp_servers intersects connected MCPs with this. Non-bypassable; dispatch-layer gate.
|
||||
active_mcps: list[str] = Field(default_factory=list)
|
||||
# Heuristic preamble tokens (preset + tool defs + MCP descs + composed prompt); subtracted from displayed input.
|
||||
|
||||
@@ -133,6 +133,9 @@ class Messaging(AgentManagerProtocol):
|
||||
"message": user_msg.model_dump(mode="json"),
|
||||
})
|
||||
|
||||
# A real user message opens a fresh silent-quit budget; the cap only guards within one ask.
|
||||
if not hidden:
|
||||
session.empty_finish_nudges = 0
|
||||
# Fire a background aux LLM call to generate a 3-6 word verb-phrase describing this turn ("Auditing the pull request", "Drafting your email"). The narrator pill swaps from its heuristic verb to this label as soon as it lands, usually ~500ms-1s into the turn, which is exactly when "Thinking…" starts feeling generic. Provider-agnostic via resolve_aux_model. Non-blocking; failure is silent and the heuristic stays.
|
||||
if not hidden and prompt:
|
||||
try:
|
||||
|
||||
@@ -52,6 +52,7 @@ def maybe_break_midturn(session: AgentSession, turn: TurnState, msg_usage: Dict)
|
||||
return False
|
||||
# Keep the session's counter honest mid-turn: a broken turn never gets its ResultMessage accounting, and the next pre-send guard reads this.
|
||||
session.tokens["input"] = total
|
||||
turn.last_step_input = total
|
||||
if total < compact_trigger_tokens(session):
|
||||
turn.saw_input_below_trigger = True
|
||||
return False
|
||||
|
||||
@@ -0,0 +1,73 @@
|
||||
"""Detect a turn that ended without an answer: the model ran tools and then quit with a
|
||||
thinking-only/empty end_turn, so the chat's last visible event is a tool result and the user
|
||||
gets a Done pill with no response. Live incident (2026-08-03, opus-5-cc lint audit): the final
|
||||
inference was a 2-char thinking block + end_turn at 70K/1M context, scored as a clean success.
|
||||
The loop nudges such a turn ONCE with a hidden continuation; twice in a row surfaces honestly."""
|
||||
|
||||
import logging
|
||||
from typing import List
|
||||
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.agents.core.models import AgentSession
|
||||
from backend.apps.agents.manager.session.history_compaction import get_branch_messages
|
||||
|
||||
NUDGE_PROMPT = (
|
||||
"You ended your turn without reporting anything. Continue exactly where you left off and "
|
||||
"finish the task; when done, always end with your findings or answer as normal text."
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@typechecked
|
||||
def maybe_nudge_empty_finish(session: AgentSession, session_id: str) -> bool:
|
||||
"""Arm one hidden continue nudge when the finished turn quit silently; the loop's existing
|
||||
auto-continuation block dispatches it. Capped per user ask; never loops."""
|
||||
if getattr(session, "pending_continuation", False) or session.empty_finish_nudges >= 1:
|
||||
return False
|
||||
if not turn_finished_empty(session):
|
||||
return False
|
||||
session.empty_finish_nudges += 1
|
||||
session.pending_continuation = True
|
||||
session.pending_continuation_prompt = NUDGE_PROMPT
|
||||
logger.warning(f"Agent {session_id}: turn finished with no answer after tool work; one hidden continue nudge")
|
||||
try:
|
||||
from backend.apps.service.client import submit_diagnostic
|
||||
submit_diagnostic({"kind": "empty_finish_nudge", "session_id": session_id, "model": session.model})
|
||||
except Exception:
|
||||
pass
|
||||
return True
|
||||
|
||||
# A turn legitimately ENDS on these tools: the rendered widget or delegation IS the answer.
|
||||
P_ANSWER_TOOL_MARKERS = ("openswarm-ui", "ShowUI", "AskUI", "AskUserQuestion")
|
||||
|
||||
|
||||
def p_tool_name_of(msg: object) -> str:
|
||||
content = getattr(msg, "content", None)
|
||||
if isinstance(content, dict):
|
||||
return str(content.get("tool") or content.get("tool_name") or "")
|
||||
return ""
|
||||
|
||||
|
||||
@typechecked
|
||||
def turn_finished_empty(session: AgentSession) -> bool:
|
||||
"""True when the branch's last visible message is a tool result whose call was ordinary work
|
||||
(not a UI/answer tool): the model did things and then said nothing."""
|
||||
msgs: List = get_branch_messages(session)
|
||||
p_last_call_name = ""
|
||||
for m in reversed(msgs):
|
||||
if getattr(m, "hidden", False):
|
||||
continue
|
||||
role = getattr(m, "role", "")
|
||||
if role == "assistant":
|
||||
text = m.content if isinstance(m.content, str) else ""
|
||||
return not text.strip()
|
||||
if role == "tool_result":
|
||||
continue
|
||||
if role == "tool_call":
|
||||
p_last_call_name = p_tool_name_of(m)
|
||||
return not any(marker in p_last_call_name for marker in P_ANSWER_TOOL_MARKERS)
|
||||
if role in ("user", "system"):
|
||||
return False
|
||||
return False
|
||||
@@ -140,7 +140,9 @@ async def handle_result_message(
|
||||
cache_create = usage.get("cache_creation_input_tokens", 0) or 0
|
||||
cache_read = usage.get("cache_read_input_tokens", 0) or 0
|
||||
total_input = inp + cache_create + cache_read
|
||||
session.tokens["input"] = total_input
|
||||
# The result's input usage is summed across every inference step of the turn, which is BILLING; live context is the last step's request size. On a 9-step audit turn the sum read 589K while the real context was 70K, and the meter (plus the compaction trigger) believed it.
|
||||
p_ctx_input = turn.last_step_input if turn.last_step_input > 0 else total_input
|
||||
session.tokens["input"] = p_ctx_input
|
||||
session.tokens["input_fresh"] = inp
|
||||
session.tokens["output"] = out
|
||||
|
||||
@@ -202,12 +204,12 @@ async def handle_result_message(
|
||||
if isinstance(usage, dict):
|
||||
# Per-turn context-usage broadcast. Drives the UI status pill and the auto-compact threshold. The denominator is the session's real model cap, populated from registry.get_context_window at session creation, restore, and model-switch (see apply_context_window). max(1, ...) is a belt-and-braces guard against zero/None drift from any future restore-from-disk corner case.
|
||||
ctx_window = max(1, getattr(session, "context_window", 0) or 200_000)
|
||||
ctx_used_pct = round(total_input / ctx_window, 4) if total_input else 0.0
|
||||
ctx_used_pct = round(p_ctx_input / ctx_window, 4) if p_ctx_input else 0.0
|
||||
cache_read_pct = round(cache_read / total_input, 4) if total_input else 0.0
|
||||
try:
|
||||
await ws_manager.send_to_session(session_id, "agent:context_update", {
|
||||
"session_id": session_id,
|
||||
"input_tokens": total_input,
|
||||
"input_tokens": p_ctx_input,
|
||||
"output_tokens": out,
|
||||
"cache_read_tokens": cache_read,
|
||||
"cache_read_pct": cache_read_pct,
|
||||
|
||||
@@ -59,3 +59,5 @@ class TurnState(BaseModel):
|
||||
# Mid-turn context breaker: fires once per turn, and only after a below-trigger reading (a turn that STARTS over the trigger must run, or a failed shrink would break-loop forever).
|
||||
context_break_fired: bool = False
|
||||
saw_input_below_trigger: bool = False
|
||||
# The LAST inference step's request size (input + cache read + cache creation): the true live context. The ResultMessage's usage sums these across every step of the turn, which is billing, not context.
|
||||
last_step_input: int = 0
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""The silent-quit seal: a turn that runs tools and ends with no visible answer gets one hidden
|
||||
continue nudge, never a loop. Detector shapes pinned here; the loop wiring is pinned in
|
||||
test_context_pressure_valve-style fashion against run_agent_loop."""
|
||||
|
||||
import asyncio
|
||||
|
||||
from backend.apps.agents.agent_manager import agent_manager
|
||||
import backend.apps.agents.agent_manager as agent_manager_module
|
||||
from backend.apps.agents.core.models import AgentSession, Message
|
||||
from backend.apps.agents.manager.run.empty_finish import NUDGE_PROMPT, turn_finished_empty
|
||||
|
||||
|
||||
def p_session(*msgs) -> AgentSession:
|
||||
s = AgentSession(name="t", model="sonnet")
|
||||
for role, content in msgs:
|
||||
s.messages.append(Message(role=role, content=content, branch_id="main"))
|
||||
return s
|
||||
|
||||
|
||||
def test_tool_result_tail_is_an_empty_finish():
|
||||
s = p_session(("user", "audit the repo"),
|
||||
("tool_call", {"tool": "Bash", "input": {"command": "ls"}}),
|
||||
("tool_result", {"text": "ok"}))
|
||||
assert turn_finished_empty(s) is True
|
||||
|
||||
|
||||
def test_final_answer_text_is_not_empty():
|
||||
s = p_session(("user", "audit"),
|
||||
("tool_call", {"tool": "Bash", "input": {}}),
|
||||
("tool_result", {"text": "ok"}),
|
||||
("assistant", "Here is the audit report."))
|
||||
assert turn_finished_empty(s) is False
|
||||
|
||||
|
||||
def test_empty_assistant_text_is_an_empty_finish():
|
||||
s = p_session(("user", "audit"), ("assistant", ""))
|
||||
assert turn_finished_empty(s) is True
|
||||
|
||||
|
||||
def test_ui_answer_tools_are_a_legit_finish():
|
||||
s = p_session(("user", "show me"),
|
||||
("tool_call", {"tool": "mcp__openswarm-ui__ShowUI", "input": {}}),
|
||||
("tool_result", {"text": "rendered"}))
|
||||
assert turn_finished_empty(s) is False
|
||||
|
||||
|
||||
def test_plain_chat_answer_is_not_empty():
|
||||
s = p_session(("user", "hi"), ("assistant", "Hey! What can I do for you?"))
|
||||
assert turn_finished_empty(s) is False
|
||||
|
||||
|
||||
def test_bare_user_prompt_is_not_claimed():
|
||||
s = p_session(("user", "hi"))
|
||||
assert turn_finished_empty(s) is False
|
||||
|
||||
|
||||
def p_install_run_fakes(monkeypatch, run_turn_fake) -> None:
|
||||
async def fake_build(session, session_id, prompt, prompt_content, builtin_perms,
|
||||
selected_browser_ids, selected_app_output_ids, selected_setting_ids,
|
||||
fork_session, router_model_id, api_type):
|
||||
from backend.apps.settings.settings import load_settings
|
||||
return object(), {}, prompt_content, [], load_settings()
|
||||
|
||||
monkeypatch.setattr(agent_manager, "build_agent_options", fake_build)
|
||||
monkeypatch.setattr(agent_manager, "run_turn_with_retry", run_turn_fake)
|
||||
monkeypatch.setattr(agent_manager_module, "save_session", lambda sid, data: None)
|
||||
|
||||
|
||||
def test_loop_nudges_a_silent_quit_once(monkeypatch) -> None:
|
||||
session = AgentSession(name="t", model="sonnet", dashboard_id="d")
|
||||
agent_manager.sessions[session.id] = session
|
||||
continues: list = []
|
||||
|
||||
async def fake_run_turn(sess, session_id, prompt_content, options, options_kwargs,
|
||||
turn, thinking, stderr, resolved_model, api_type,
|
||||
global_settings, force_respawn=False):
|
||||
# Every turn ends as a silent quit: tools ran, no answer text.
|
||||
sess.messages.append(Message(role="tool_call", content={"tool": "Bash", "input": {}}, branch_id="main"))
|
||||
sess.messages.append(Message(role="tool_result", content={"text": "out"}, branch_id="main"))
|
||||
|
||||
async def fake_send_message(session_id, prompt, hidden=False, **kwargs):
|
||||
continues.append({"prompt": prompt, "hidden": hidden})
|
||||
|
||||
p_install_run_fakes(monkeypatch, fake_run_turn)
|
||||
monkeypatch.setattr(agent_manager, "send_message", fake_send_message)
|
||||
|
||||
async def main():
|
||||
await agent_manager.run_agent_loop(session.id, "audit everything")
|
||||
await asyncio.sleep(0)
|
||||
|
||||
asyncio.run(main())
|
||||
assert continues == [{"prompt": NUDGE_PROMPT, "hidden": True}]
|
||||
assert session.empty_finish_nudges == 1
|
||||
|
||||
# The nudged turn ALSO quits silently: the cap must hold, no second nudge, no loop.
|
||||
continues.clear()
|
||||
asyncio.run(main())
|
||||
assert continues == []
|
||||
assert session.empty_finish_nudges == 1
|
||||
@@ -114,3 +114,34 @@ async def test_resets_per_turn_state_at_completion():
|
||||
assert turn.tool_count == 0
|
||||
assert thinking.total_ms == 0
|
||||
assert thinking.block_starts == {}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_context_meter_prefers_last_step_over_cumulative_billing():
|
||||
# A 9-step turn's result usage sums input across steps (billing); the meter must show the last step's request size (real context). The 925K/1M incident read the sum.
|
||||
session, turn, thinking = p_fixt()
|
||||
turn.last_step_input = 70_454
|
||||
payloads = []
|
||||
|
||||
async def fake_send(sid, ev, data):
|
||||
if ev == "agent:context_update":
|
||||
payloads.append(data)
|
||||
|
||||
with patch.object(result_message.ws_manager, "send_to_session", AsyncMock(side_effect=fake_send)):
|
||||
await result_message.handle_result_message(
|
||||
p_result(usage={"input_tokens": 2_023, "cache_read_input_tokens": 500_000, "cache_creation_input_tokens": 86_972, "output_tokens": 1_210}),
|
||||
session, "sid", turn, thinking, {}, "cc/claude-opus-5", "anthropic", load_settings(),
|
||||
)
|
||||
assert session.tokens["input"] == 70_454
|
||||
assert payloads and payloads[0]["input_tokens"] == 70_454
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_context_meter_falls_back_to_result_usage_without_step_readings():
|
||||
session, turn, thinking = p_fixt()
|
||||
with patch.object(result_message.ws_manager, "send_to_session", AsyncMock()):
|
||||
await result_message.handle_result_message(
|
||||
p_result(usage={"input_tokens": 1_000, "cache_read_input_tokens": 2_000, "output_tokens": 10}),
|
||||
session, "sid", turn, thinking, {}, "sonnet", "anthropic", load_settings(),
|
||||
)
|
||||
assert session.tokens["input"] == 3_000
|
||||
|
||||
Reference in New Issue
Block a user