mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-13 13:17:40 +02:00
[eric] agents: autocompact thrash and a stalled long chat stop telling the user to switch models
This commit is contained in:
@@ -35,6 +35,17 @@ EXHAUSTED_NOTE_NO_PROGRESS = (
|
||||
"another model."
|
||||
)
|
||||
|
||||
# Same "no work this turn", but in a chat that has ALREADY done a lot of it. Telling that user the
|
||||
# agent "could not get started" is plainly false to anyone reading their own transcript, and telling
|
||||
# them to switch models sends them after the wrong thing: a new model inherits the same long
|
||||
# conversation and stalls the same way. Seen live 2026-09-01 (Ken, 1.7.9): one chat, 322 tool calls,
|
||||
# input 111K-164K, six stalls in an hour, and this was the note it ended on.
|
||||
EXHAUSTED_NOTE_LONG_CHAT = (
|
||||
"The agent stopped without replying, and this conversation has gotten long enough that it is "
|
||||
"the likely cause. Send your message again to retry, or start a fresh chat for the next part; "
|
||||
"switching models will not help, since a new one carries the same conversation."
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@@ -154,6 +165,18 @@ def p_recovery_retry_pending(session: AgentSession) -> bool:
|
||||
)
|
||||
|
||||
|
||||
@typechecked
|
||||
def session_showed_work(session: AgentSession) -> bool:
|
||||
"""Whether this CHAT has real work behind it, regardless of what this one turn managed.
|
||||
|
||||
`turn_showed_work` answers "is there anything to point at since the user last spoke", which is
|
||||
the right question for the note's wording but the wrong one for its ADVICE: a chat with hundreds
|
||||
of tool calls that stalls is a depth problem, not a model problem."""
|
||||
from backend.apps.agents.manager.session.history_compaction import get_branch_messages
|
||||
return any(getattr(m, "role", "") in ("tool_call", "tool_result")
|
||||
for m in get_branch_messages(session))
|
||||
|
||||
|
||||
@typechecked
|
||||
def turn_showed_work(session: AgentSession) -> bool:
|
||||
"""Whether anything the note could point at actually exists since the user last spoke."""
|
||||
@@ -186,7 +209,12 @@ def surface_exhausted(session: AgentSession, session_id: str) -> None:
|
||||
import asyncio
|
||||
from backend.apps.agents.core.models import Message
|
||||
from backend.apps.agents.core.ws_manager import ws_manager
|
||||
p_note = EXHAUSTED_NOTE if turn_showed_work(session) else EXHAUSTED_NOTE_NO_PROGRESS
|
||||
if turn_showed_work(session):
|
||||
p_note = EXHAUSTED_NOTE
|
||||
elif session_showed_work(session):
|
||||
p_note = EXHAUSTED_NOTE_LONG_CHAT
|
||||
else:
|
||||
p_note = EXHAUSTED_NOTE_NO_PROGRESS
|
||||
p_msg = Message(role="system", content=p_note, branch_id=session.active_branch_id)
|
||||
session.messages.append(p_msg)
|
||||
asyncio.get_running_loop().create_task(ws_manager.send_to_session(session_id, "agent:message", {
|
||||
|
||||
@@ -15,6 +15,7 @@ from backend.apps.agents.manager.streaming.state import TurnState
|
||||
from backend.apps.agents.core.error_classify import (
|
||||
is_context_overflow_error,
|
||||
is_long_context_error,
|
||||
is_context_pressure_death,
|
||||
is_stale_tool_schema_error,
|
||||
is_transient_capacity_error,
|
||||
is_free_trial_exhausted,
|
||||
@@ -561,6 +562,38 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
|
||||
"session_id": session_id,
|
||||
"message": error_msg.model_dump(mode="json"),
|
||||
})
|
||||
elif is_context_pressure_death(e, turn.compact_boundaries, extra_text=p_stderr_tail):
|
||||
# Autocompact thrash that OUTLIVED the pressure valve. The valve gets one fresh-session
|
||||
# rebuild (agent_manager); a conversation that refills the window within three turns simply
|
||||
# does it again, and the second death used to fall through to the generic card, which says
|
||||
# "switch this agent to another model". That is wrong twice: it blames the provider for our
|
||||
# context problem, and a different model inherits the same oversized conversation and
|
||||
# thrashes identically. Seen live 2026-09-01 on a real user's LinkedIn run (sonnet-5).
|
||||
friendly_msg = (
|
||||
"This chat has grown too large to keep compacting: it refills the model's context "
|
||||
"faster than it can be summarized, which is also why it started forgetting earlier "
|
||||
"steps. Start a fresh chat to carry on (your recent context comes with it). Switching "
|
||||
"models will not help, because a new one inherits the same conversation."
|
||||
)
|
||||
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
|
||||
absorb_repeat_card(session, error_msg)
|
||||
await ws_manager.send_to_session(session_id, "agent:message", {
|
||||
"session_id": session_id,
|
||||
"message": error_msg.model_dump(mode="json"),
|
||||
})
|
||||
try:
|
||||
from backend.apps.service.client import submit_diagnostic
|
||||
submit_diagnostic({
|
||||
"kind": "model_error",
|
||||
"subkind": "autocompact_thrash",
|
||||
"flight": flight_recorder.build_envelope(session_id, "model_error", "autocompact_thrash", session.model, "stream" if turn.current_turn_emitted else "spawn", -1),
|
||||
"session_id": session_id,
|
||||
"model": session.model,
|
||||
"compact_boundaries": turn.compact_boundaries,
|
||||
"error_preview": redact_for_telemetry(str(e), limit=400),
|
||||
})
|
||||
except Exception:
|
||||
logger.debug("submit_diagnostic autocompact_thrash failed", exc_info=True)
|
||||
elif is_router_unavailable_error(f"{e} {p_stderr_tail}"):
|
||||
# Our own router is down. Naming it beats "unclassified": this is the one failure family
|
||||
# where the fix is entirely on our side of the wire.
|
||||
|
||||
@@ -0,0 +1,78 @@
|
||||
"""Autocompact thrash that outlives the pressure valve must not be blamed on the model.
|
||||
|
||||
Field, 2026-09-01 (Ken, 1.7.9, sonnet-5, a LinkedIn outreach chat): the turn died with the CLI's own
|
||||
verdict, "Autocompact is thrashing: the context refilled to the limit within 3 turns of the previous
|
||||
compact, 3 times in a row." It was recorded as `unclassified` and the user was told:
|
||||
|
||||
"The model provider returned an error instead of an answer. Send your message again; if it
|
||||
keeps happening, switch this agent to another model."
|
||||
|
||||
Wrong twice. It blames the provider for our own context problem, and switching models cannot help,
|
||||
because a new model inherits the same oversized conversation and thrashes identically. The same chat
|
||||
was visibly losing earlier steps, which is the same cause wearing a different face.
|
||||
|
||||
The valve (agent_manager) gets ONE fresh-session rebuild. A conversation that refills the window
|
||||
within three turns does it again, and that second death is what reaches this handler.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
from backend.apps.agents.core.models import AgentSession
|
||||
from backend.apps.agents.core.error_classify import is_context_pressure_death
|
||||
|
||||
KEN = ("The agent runtime reported this turn failed (stop_sequence). Autocompact is thrashing: "
|
||||
"the context refilled to the limit within 3 turns of the previous compact, 3 times in a row.")
|
||||
|
||||
|
||||
def test_the_field_error_is_recognised_at_zero_boundaries():
|
||||
"""The CLI naming its own thrash is self-identifying; it must not need a boundary count."""
|
||||
assert is_context_pressure_death(RuntimeError(KEN), 0)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_the_card_names_the_conversation_not_the_model(monkeypatch):
|
||||
from backend.apps.agents.manager.run import handle_run_error as mod
|
||||
from backend.apps.agents.manager.streaming.state import TurnState
|
||||
|
||||
sent = []
|
||||
|
||||
async def p_send(session_id, event, payload):
|
||||
sent.append(event)
|
||||
|
||||
monkeypatch.setattr(mod.ws_manager, "send_to_session", p_send)
|
||||
s = AgentSession(name="t", model="sonnet-5", dashboard_id="d")
|
||||
await mod.handle_run_error(RuntimeError(KEN), s, s.id, TurnState(), [])
|
||||
|
||||
cards = [m for m in s.messages if m.role == "system"]
|
||||
assert len(cards) == 1
|
||||
text = cards[0].content.lower()
|
||||
assert "switching models will not help" in text, "the wrong remedy must be ruled OUT explicitly"
|
||||
assert "fresh chat" in text, "and the one that works must be named"
|
||||
assert "forgetting" in text, "the memory loss the user is also seeing has the same cause; say so"
|
||||
assert "provider returned an error" not in text, "the generic model-blaming card must not fire"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_it_sits_above_the_generic_fallback(monkeypatch):
|
||||
"""Ordering, not just presence: the generic `else` is what produced the bad advice, so a branch
|
||||
added below it would change nothing."""
|
||||
import inspect
|
||||
import re
|
||||
from backend.apps.agents.manager.run import handle_run_error as mod
|
||||
src = inspect.getsource(mod.handle_run_error)
|
||||
# Line-anchored: a bare `src.index(" else:")` also matches the deeper ` else:` of a
|
||||
# nested block and reported a false failure while the real ordering was fine.
|
||||
p_fallback = re.search(r"^ else:", src, re.M)
|
||||
assert p_fallback, "the unclassified fallback should still exist"
|
||||
assert src.index("is_context_pressure_death(e") < p_fallback.start(), \
|
||||
"the thrash branch must precede the unclassified fallback"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("innocent", [
|
||||
"API Error: 429 rate_limit_error (reset after 21s)",
|
||||
"API Error: 401 authentication_error",
|
||||
"This conversation has grown too large for your account's standard context window",
|
||||
])
|
||||
async def test_errors_that_belong_to_other_branches_are_not_stolen(innocent):
|
||||
"""is_context_pressure_death only claims deaths no other classifier owns."""
|
||||
assert not is_context_pressure_death(RuntimeError(innocent), 1)
|
||||
@@ -0,0 +1,55 @@
|
||||
"""The nudge ladder's give-up note must not send a working chat after the wrong remedy.
|
||||
|
||||
There were two notes: "work is above, carry on" when the turn produced something, and "could not get
|
||||
started... switch this agent to another model" when it did not. The second is right for a chat that
|
||||
truly never started, and wrong for a chat that has done a pile of work and stalled on one turn: the
|
||||
user can see their own transcript, and switching models drags the same conversation into the same
|
||||
stall. Third case added 2026-09-01 after a real LinkedIn run ended on the model-blaming note.
|
||||
"""
|
||||
from backend.apps.agents.core.models import AgentSession, Message
|
||||
from backend.apps.agents.manager.run.empty_finish import (
|
||||
EXHAUSTED_NOTE, EXHAUSTED_NOTE_LONG_CHAT, EXHAUSTED_NOTE_NO_PROGRESS,
|
||||
session_showed_work, turn_showed_work,
|
||||
)
|
||||
|
||||
|
||||
def p_session():
|
||||
s = AgentSession(name="t", model="sonnet-5", dashboard_id="d")
|
||||
return s, s.active_branch_id
|
||||
|
||||
|
||||
def test_a_chat_that_never_started_still_gets_the_original_advice():
|
||||
s, b = p_session()
|
||||
s.messages.append(Message(role="user", content="do the thing", branch_id=b))
|
||||
assert turn_showed_work(s) is False
|
||||
assert session_showed_work(s) is False, "nothing has ever run; switching models is fair advice"
|
||||
assert "switch this agent to another model" in EXHAUSTED_NOTE_NO_PROGRESS
|
||||
|
||||
|
||||
def test_a_chat_with_work_behind_it_is_not_told_it_never_started():
|
||||
s, b = p_session()
|
||||
s.messages.append(Message(role="user", content="find people to connect with", branch_id=b))
|
||||
s.messages.append(Message(role="tool_call", content={"id": "t1", "tool": "BrowserAgent", "input": {}}, branch_id=b))
|
||||
s.messages.append(Message(role="tool_result", content={"tool_use_id": "t1", "text": "ok"}, branch_id=b))
|
||||
s.messages.append(Message(role="user", content="keep going", branch_id=b))
|
||||
assert turn_showed_work(s) is False, "nothing since the LAST user message"
|
||||
assert session_showed_work(s) is True, "but the chat plainly did work"
|
||||
|
||||
|
||||
def test_the_long_chat_note_rules_out_the_wrong_remedy_and_names_the_right_one():
|
||||
low = EXHAUSTED_NOTE_LONG_CHAT.lower()
|
||||
assert "switching models will not help" in low
|
||||
assert "fresh chat" in low
|
||||
assert "could not get started" not in low, "false to anyone reading their own transcript"
|
||||
|
||||
|
||||
def test_the_three_notes_are_distinct():
|
||||
assert len({EXHAUSTED_NOTE, EXHAUSTED_NOTE_LONG_CHAT, EXHAUSTED_NOTE_NO_PROGRESS}) == 3
|
||||
|
||||
|
||||
def test_the_selector_prefers_turn_work_then_session_work():
|
||||
"""Ordering: a turn that produced work must still get the 'work is above' note."""
|
||||
import inspect
|
||||
from backend.apps.agents.manager.run import empty_finish as mod
|
||||
src = inspect.getsource(mod.surface_exhausted)
|
||||
assert src.index("turn_showed_work(session)") < src.index("session_showed_work(session)")
|
||||
Reference in New Issue
Block a user