[eric] agents: autocompact thrash and a stalled long chat stop telling the user to switch models

This commit is contained in:
ciregenz
2026-08-31 23:00:30 -07:00
parent ae44920cfe
commit e5b3749f90
4 changed files with 195 additions and 1 deletions
@@ -35,6 +35,17 @@ EXHAUSTED_NOTE_NO_PROGRESS = (
"another model."
)
# Same "no work this turn", but in a chat that has ALREADY done a lot of it. Telling that user the
# agent "could not get started" is plainly false to anyone reading their own transcript, and telling
# them to switch models sends them after the wrong thing: a new model inherits the same long
# conversation and stalls the same way. Seen live 2026-09-01 (Ken, 1.7.9): one chat, 322 tool calls,
# input 111K-164K, six stalls in an hour, and this was the note it ended on.
EXHAUSTED_NOTE_LONG_CHAT = (
"The agent stopped without replying, and this conversation has gotten long enough that it is "
"the likely cause. Send your message again to retry, or start a fresh chat for the next part; "
"switching models will not help, since a new one carries the same conversation."
)
logger = logging.getLogger(__name__)
@@ -154,6 +165,18 @@ def p_recovery_retry_pending(session: AgentSession) -> bool:
)
@typechecked
def session_showed_work(session: AgentSession) -> bool:
"""Whether this CHAT has real work behind it, regardless of what this one turn managed.
`turn_showed_work` answers "is there anything to point at since the user last spoke", which is
the right question for the note's wording but the wrong one for its ADVICE: a chat with hundreds
of tool calls that stalls is a depth problem, not a model problem."""
from backend.apps.agents.manager.session.history_compaction import get_branch_messages
return any(getattr(m, "role", "") in ("tool_call", "tool_result")
for m in get_branch_messages(session))
@typechecked
def turn_showed_work(session: AgentSession) -> bool:
"""Whether anything the note could point at actually exists since the user last spoke."""
@@ -186,7 +209,12 @@ def surface_exhausted(session: AgentSession, session_id: str) -> None:
import asyncio
from backend.apps.agents.core.models import Message
from backend.apps.agents.core.ws_manager import ws_manager
p_note = EXHAUSTED_NOTE if turn_showed_work(session) else EXHAUSTED_NOTE_NO_PROGRESS
if turn_showed_work(session):
p_note = EXHAUSTED_NOTE
elif session_showed_work(session):
p_note = EXHAUSTED_NOTE_LONG_CHAT
else:
p_note = EXHAUSTED_NOTE_NO_PROGRESS
p_msg = Message(role="system", content=p_note, branch_id=session.active_branch_id)
session.messages.append(p_msg)
asyncio.get_running_loop().create_task(ws_manager.send_to_session(session_id, "agent:message", {
@@ -15,6 +15,7 @@ from backend.apps.agents.manager.streaming.state import TurnState
from backend.apps.agents.core.error_classify import (
is_context_overflow_error,
is_long_context_error,
is_context_pressure_death,
is_stale_tool_schema_error,
is_transient_capacity_error,
is_free_trial_exhausted,
@@ -561,6 +562,38 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
})
elif is_context_pressure_death(e, turn.compact_boundaries, extra_text=p_stderr_tail):
# Autocompact thrash that OUTLIVED the pressure valve. The valve gets one fresh-session
# rebuild (agent_manager); a conversation that refills the window within three turns simply
# does it again, and the second death used to fall through to the generic card, which says
# "switch this agent to another model". That is wrong twice: it blames the provider for our
# context problem, and a different model inherits the same oversized conversation and
# thrashes identically. Seen live 2026-09-01 on a real user's LinkedIn run (sonnet-5).
friendly_msg = (
"This chat has grown too large to keep compacting: it refills the model's context "
"faster than it can be summarized, which is also why it started forgetting earlier "
"steps. Start a fresh chat to carry on (your recent context comes with it). Switching "
"models will not help, because a new one inherits the same conversation."
)
error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
absorb_repeat_card(session, error_msg)
await ws_manager.send_to_session(session_id, "agent:message", {
"session_id": session_id,
"message": error_msg.model_dump(mode="json"),
})
try:
from backend.apps.service.client import submit_diagnostic
submit_diagnostic({
"kind": "model_error",
"subkind": "autocompact_thrash",
"flight": flight_recorder.build_envelope(session_id, "model_error", "autocompact_thrash", session.model, "stream" if turn.current_turn_emitted else "spawn", -1),
"session_id": session_id,
"model": session.model,
"compact_boundaries": turn.compact_boundaries,
"error_preview": redact_for_telemetry(str(e), limit=400),
})
except Exception:
logger.debug("submit_diagnostic autocompact_thrash failed", exc_info=True)
elif is_router_unavailable_error(f"{e} {p_stderr_tail}"):
# Our own router is down. Naming it beats "unclassified": this is the one failure family
# where the fix is entirely on our side of the wire.
@@ -0,0 +1,78 @@
"""Autocompact thrash that outlives the pressure valve must not be blamed on the model.
Field, 2026-09-01 (Ken, 1.7.9, sonnet-5, a LinkedIn outreach chat): the turn died with the CLI's own
verdict, "Autocompact is thrashing: the context refilled to the limit within 3 turns of the previous
compact, 3 times in a row." It was recorded as `unclassified` and the user was told:
"The model provider returned an error instead of an answer. Send your message again; if it
keeps happening, switch this agent to another model."
Wrong twice. It blames the provider for our own context problem, and switching models cannot help,
because a new model inherits the same oversized conversation and thrashes identically. The same chat
was visibly losing earlier steps, which is the same cause wearing a different face.
The valve (agent_manager) gets ONE fresh-session rebuild. A conversation that refills the window
within three turns does it again, and that second death is what reaches this handler.
"""
import pytest
from backend.apps.agents.core.models import AgentSession
from backend.apps.agents.core.error_classify import is_context_pressure_death
KEN = ("The agent runtime reported this turn failed (stop_sequence). Autocompact is thrashing: "
"the context refilled to the limit within 3 turns of the previous compact, 3 times in a row.")
def test_the_field_error_is_recognised_at_zero_boundaries():
"""The CLI naming its own thrash is self-identifying; it must not need a boundary count."""
assert is_context_pressure_death(RuntimeError(KEN), 0)
@pytest.mark.asyncio
async def test_the_card_names_the_conversation_not_the_model(monkeypatch):
from backend.apps.agents.manager.run import handle_run_error as mod
from backend.apps.agents.manager.streaming.state import TurnState
sent = []
async def p_send(session_id, event, payload):
sent.append(event)
monkeypatch.setattr(mod.ws_manager, "send_to_session", p_send)
s = AgentSession(name="t", model="sonnet-5", dashboard_id="d")
await mod.handle_run_error(RuntimeError(KEN), s, s.id, TurnState(), [])
cards = [m for m in s.messages if m.role == "system"]
assert len(cards) == 1
text = cards[0].content.lower()
assert "switching models will not help" in text, "the wrong remedy must be ruled OUT explicitly"
assert "fresh chat" in text, "and the one that works must be named"
assert "forgetting" in text, "the memory loss the user is also seeing has the same cause; say so"
assert "provider returned an error" not in text, "the generic model-blaming card must not fire"
@pytest.mark.asyncio
async def test_it_sits_above_the_generic_fallback(monkeypatch):
"""Ordering, not just presence: the generic `else` is what produced the bad advice, so a branch
added below it would change nothing."""
import inspect
import re
from backend.apps.agents.manager.run import handle_run_error as mod
src = inspect.getsource(mod.handle_run_error)
# Line-anchored: a bare `src.index(" else:")` also matches the deeper ` else:` of a
# nested block and reported a false failure while the real ordering was fine.
p_fallback = re.search(r"^ else:", src, re.M)
assert p_fallback, "the unclassified fallback should still exist"
assert src.index("is_context_pressure_death(e") < p_fallback.start(), \
"the thrash branch must precede the unclassified fallback"
@pytest.mark.asyncio
@pytest.mark.parametrize("innocent", [
"API Error: 429 rate_limit_error (reset after 21s)",
"API Error: 401 authentication_error",
"This conversation has grown too large for your account's standard context window",
])
async def test_errors_that_belong_to_other_branches_are_not_stolen(innocent):
"""is_context_pressure_death only claims deaths no other classifier owns."""
assert not is_context_pressure_death(RuntimeError(innocent), 1)
@@ -0,0 +1,55 @@
"""The nudge ladder's give-up note must not send a working chat after the wrong remedy.
There were two notes: "work is above, carry on" when the turn produced something, and "could not get
started... switch this agent to another model" when it did not. The second is right for a chat that
truly never started, and wrong for a chat that has done a pile of work and stalled on one turn: the
user can see their own transcript, and switching models drags the same conversation into the same
stall. Third case added 2026-09-01 after a real LinkedIn run ended on the model-blaming note.
"""
from backend.apps.agents.core.models import AgentSession, Message
from backend.apps.agents.manager.run.empty_finish import (
EXHAUSTED_NOTE, EXHAUSTED_NOTE_LONG_CHAT, EXHAUSTED_NOTE_NO_PROGRESS,
session_showed_work, turn_showed_work,
)
def p_session():
s = AgentSession(name="t", model="sonnet-5", dashboard_id="d")
return s, s.active_branch_id
def test_a_chat_that_never_started_still_gets_the_original_advice():
s, b = p_session()
s.messages.append(Message(role="user", content="do the thing", branch_id=b))
assert turn_showed_work(s) is False
assert session_showed_work(s) is False, "nothing has ever run; switching models is fair advice"
assert "switch this agent to another model" in EXHAUSTED_NOTE_NO_PROGRESS
def test_a_chat_with_work_behind_it_is_not_told_it_never_started():
s, b = p_session()
s.messages.append(Message(role="user", content="find people to connect with", branch_id=b))
s.messages.append(Message(role="tool_call", content={"id": "t1", "tool": "BrowserAgent", "input": {}}, branch_id=b))
s.messages.append(Message(role="tool_result", content={"tool_use_id": "t1", "text": "ok"}, branch_id=b))
s.messages.append(Message(role="user", content="keep going", branch_id=b))
assert turn_showed_work(s) is False, "nothing since the LAST user message"
assert session_showed_work(s) is True, "but the chat plainly did work"
def test_the_long_chat_note_rules_out_the_wrong_remedy_and_names_the_right_one():
low = EXHAUSTED_NOTE_LONG_CHAT.lower()
assert "switching models will not help" in low
assert "fresh chat" in low
assert "could not get started" not in low, "false to anyone reading their own transcript"
def test_the_three_notes_are_distinct():
assert len({EXHAUSTED_NOTE, EXHAUSTED_NOTE_LONG_CHAT, EXHAUSTED_NOTE_NO_PROGRESS}) == 3
def test_the_selector_prefers_turn_work_then_session_work():
"""Ordering: a turn that produced work must still get the 'work is above' note."""
import inspect
from backend.apps.agents.manager.run import empty_finish as mod
src = inspect.getsource(mod.surface_exhausted)
assert src.index("turn_showed_work(session)") < src.index("session_showed_work(session)")