From e5b3749f90a37d83f071a71751f9ce5cd009a1f7 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Mon, 31 Aug 2026 23:00:30 -0700 Subject: [PATCH] [eric] agents: autocompact thrash and a stalled long chat stop telling the user to switch models --- .../apps/agents/manager/run/empty_finish.py | 30 ++++++- .../agents/manager/run/handle_run_error.py | 33 ++++++++ backend/tests/test_autocompact_thrash_card.py | 78 +++++++++++++++++++ backend/tests/test_exhausted_note_advice.py | 55 +++++++++++++ 4 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 backend/tests/test_autocompact_thrash_card.py create mode 100644 backend/tests/test_exhausted_note_advice.py diff --git a/backend/apps/agents/manager/run/empty_finish.py b/backend/apps/agents/manager/run/empty_finish.py index c6d771ce..21f1f2fa 100644 --- a/backend/apps/agents/manager/run/empty_finish.py +++ b/backend/apps/agents/manager/run/empty_finish.py @@ -35,6 +35,17 @@ EXHAUSTED_NOTE_NO_PROGRESS = ( "another model." ) +# Same "no work this turn", but in a chat that has ALREADY done a lot of it. Telling that user the +# agent "could not get started" is plainly false to anyone reading their own transcript, and telling +# them to switch models sends them after the wrong thing: a new model inherits the same long +# conversation and stalls the same way. Seen live 2026-09-01 (Ken, 1.7.9): one chat, 322 tool calls, +# input 111K-164K, six stalls in an hour, and this was the note it ended on. +EXHAUSTED_NOTE_LONG_CHAT = ( + "The agent stopped without replying, and this conversation has gotten long enough that it is " + "the likely cause. Send your message again to retry, or start a fresh chat for the next part; " + "switching models will not help, since a new one carries the same conversation." +) + logger = logging.getLogger(__name__) @@ -154,6 +165,18 @@ def p_recovery_retry_pending(session: AgentSession) -> bool: ) +@typechecked +def session_showed_work(session: AgentSession) -> bool: + """Whether this CHAT has real work behind it, regardless of what this one turn managed. + + `turn_showed_work` answers "is there anything to point at since the user last spoke", which is + the right question for the note's wording but the wrong one for its ADVICE: a chat with hundreds + of tool calls that stalls is a depth problem, not a model problem.""" + from backend.apps.agents.manager.session.history_compaction import get_branch_messages + return any(getattr(m, "role", "") in ("tool_call", "tool_result") + for m in get_branch_messages(session)) + + @typechecked def turn_showed_work(session: AgentSession) -> bool: """Whether anything the note could point at actually exists since the user last spoke.""" @@ -186,7 +209,12 @@ def surface_exhausted(session: AgentSession, session_id: str) -> None: import asyncio from backend.apps.agents.core.models import Message from backend.apps.agents.core.ws_manager import ws_manager - p_note = EXHAUSTED_NOTE if turn_showed_work(session) else EXHAUSTED_NOTE_NO_PROGRESS + if turn_showed_work(session): + p_note = EXHAUSTED_NOTE + elif session_showed_work(session): + p_note = EXHAUSTED_NOTE_LONG_CHAT + else: + p_note = EXHAUSTED_NOTE_NO_PROGRESS p_msg = Message(role="system", content=p_note, branch_id=session.active_branch_id) session.messages.append(p_msg) asyncio.get_running_loop().create_task(ws_manager.send_to_session(session_id, "agent:message", { diff --git a/backend/apps/agents/manager/run/handle_run_error.py b/backend/apps/agents/manager/run/handle_run_error.py index 32255892..65a63b69 100644 --- a/backend/apps/agents/manager/run/handle_run_error.py +++ b/backend/apps/agents/manager/run/handle_run_error.py @@ -15,6 +15,7 @@ from backend.apps.agents.manager.streaming.state import TurnState from backend.apps.agents.core.error_classify import ( is_context_overflow_error, is_long_context_error, + is_context_pressure_death, is_stale_tool_schema_error, is_transient_capacity_error, is_free_trial_exhausted, @@ -561,6 +562,38 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, "session_id": session_id, "message": error_msg.model_dump(mode="json"), }) + elif is_context_pressure_death(e, turn.compact_boundaries, extra_text=p_stderr_tail): + # Autocompact thrash that OUTLIVED the pressure valve. The valve gets one fresh-session + # rebuild (agent_manager); a conversation that refills the window within three turns simply + # does it again, and the second death used to fall through to the generic card, which says + # "switch this agent to another model". That is wrong twice: it blames the provider for our + # context problem, and a different model inherits the same oversized conversation and + # thrashes identically. Seen live 2026-09-01 on a real user's LinkedIn run (sonnet-5). + friendly_msg = ( + "This chat has grown too large to keep compacting: it refills the model's context " + "faster than it can be summarized, which is also why it started forgetting earlier " + "steps. Start a fresh chat to carry on (your recent context comes with it). Switching " + "models will not help, because a new one inherits the same conversation." + ) + error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) + absorb_repeat_card(session, error_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": error_msg.model_dump(mode="json"), + }) + try: + from backend.apps.service.client import submit_diagnostic + submit_diagnostic({ + "kind": "model_error", + "subkind": "autocompact_thrash", + "flight": flight_recorder.build_envelope(session_id, "model_error", "autocompact_thrash", session.model, "stream" if turn.current_turn_emitted else "spawn", -1), + "session_id": session_id, + "model": session.model, + "compact_boundaries": turn.compact_boundaries, + "error_preview": redact_for_telemetry(str(e), limit=400), + }) + except Exception: + logger.debug("submit_diagnostic autocompact_thrash failed", exc_info=True) elif is_router_unavailable_error(f"{e} {p_stderr_tail}"): # Our own router is down. Naming it beats "unclassified": this is the one failure family # where the fix is entirely on our side of the wire. diff --git a/backend/tests/test_autocompact_thrash_card.py b/backend/tests/test_autocompact_thrash_card.py new file mode 100644 index 00000000..aa7d55c3 --- /dev/null +++ b/backend/tests/test_autocompact_thrash_card.py @@ -0,0 +1,78 @@ +"""Autocompact thrash that outlives the pressure valve must not be blamed on the model. + +Field, 2026-09-01 (Ken, 1.7.9, sonnet-5, a LinkedIn outreach chat): the turn died with the CLI's own +verdict, "Autocompact is thrashing: the context refilled to the limit within 3 turns of the previous +compact, 3 times in a row." It was recorded as `unclassified` and the user was told: + + "The model provider returned an error instead of an answer. Send your message again; if it + keeps happening, switch this agent to another model." + +Wrong twice. It blames the provider for our own context problem, and switching models cannot help, +because a new model inherits the same oversized conversation and thrashes identically. The same chat +was visibly losing earlier steps, which is the same cause wearing a different face. + +The valve (agent_manager) gets ONE fresh-session rebuild. A conversation that refills the window +within three turns does it again, and that second death is what reaches this handler. +""" +import pytest + +from backend.apps.agents.core.models import AgentSession +from backend.apps.agents.core.error_classify import is_context_pressure_death + +KEN = ("The agent runtime reported this turn failed (stop_sequence). Autocompact is thrashing: " + "the context refilled to the limit within 3 turns of the previous compact, 3 times in a row.") + + +def test_the_field_error_is_recognised_at_zero_boundaries(): + """The CLI naming its own thrash is self-identifying; it must not need a boundary count.""" + assert is_context_pressure_death(RuntimeError(KEN), 0) + + +@pytest.mark.asyncio +async def test_the_card_names_the_conversation_not_the_model(monkeypatch): + from backend.apps.agents.manager.run import handle_run_error as mod + from backend.apps.agents.manager.streaming.state import TurnState + + sent = [] + + async def p_send(session_id, event, payload): + sent.append(event) + + monkeypatch.setattr(mod.ws_manager, "send_to_session", p_send) + s = AgentSession(name="t", model="sonnet-5", dashboard_id="d") + await mod.handle_run_error(RuntimeError(KEN), s, s.id, TurnState(), []) + + cards = [m for m in s.messages if m.role == "system"] + assert len(cards) == 1 + text = cards[0].content.lower() + assert "switching models will not help" in text, "the wrong remedy must be ruled OUT explicitly" + assert "fresh chat" in text, "and the one that works must be named" + assert "forgetting" in text, "the memory loss the user is also seeing has the same cause; say so" + assert "provider returned an error" not in text, "the generic model-blaming card must not fire" + + +@pytest.mark.asyncio +async def test_it_sits_above_the_generic_fallback(monkeypatch): + """Ordering, not just presence: the generic `else` is what produced the bad advice, so a branch + added below it would change nothing.""" + import inspect + import re + from backend.apps.agents.manager.run import handle_run_error as mod + src = inspect.getsource(mod.handle_run_error) + # Line-anchored: a bare `src.index(" else:")` also matches the deeper ` else:` of a + # nested block and reported a false failure while the real ordering was fine. + p_fallback = re.search(r"^ else:", src, re.M) + assert p_fallback, "the unclassified fallback should still exist" + assert src.index("is_context_pressure_death(e") < p_fallback.start(), \ + "the thrash branch must precede the unclassified fallback" + + +@pytest.mark.asyncio +@pytest.mark.parametrize("innocent", [ + "API Error: 429 rate_limit_error (reset after 21s)", + "API Error: 401 authentication_error", + "This conversation has grown too large for your account's standard context window", +]) +async def test_errors_that_belong_to_other_branches_are_not_stolen(innocent): + """is_context_pressure_death only claims deaths no other classifier owns.""" + assert not is_context_pressure_death(RuntimeError(innocent), 1) diff --git a/backend/tests/test_exhausted_note_advice.py b/backend/tests/test_exhausted_note_advice.py new file mode 100644 index 00000000..fda01f13 --- /dev/null +++ b/backend/tests/test_exhausted_note_advice.py @@ -0,0 +1,55 @@ +"""The nudge ladder's give-up note must not send a working chat after the wrong remedy. + +There were two notes: "work is above, carry on" when the turn produced something, and "could not get +started... switch this agent to another model" when it did not. The second is right for a chat that +truly never started, and wrong for a chat that has done a pile of work and stalled on one turn: the +user can see their own transcript, and switching models drags the same conversation into the same +stall. Third case added 2026-09-01 after a real LinkedIn run ended on the model-blaming note. +""" +from backend.apps.agents.core.models import AgentSession, Message +from backend.apps.agents.manager.run.empty_finish import ( + EXHAUSTED_NOTE, EXHAUSTED_NOTE_LONG_CHAT, EXHAUSTED_NOTE_NO_PROGRESS, + session_showed_work, turn_showed_work, +) + + +def p_session(): + s = AgentSession(name="t", model="sonnet-5", dashboard_id="d") + return s, s.active_branch_id + + +def test_a_chat_that_never_started_still_gets_the_original_advice(): + s, b = p_session() + s.messages.append(Message(role="user", content="do the thing", branch_id=b)) + assert turn_showed_work(s) is False + assert session_showed_work(s) is False, "nothing has ever run; switching models is fair advice" + assert "switch this agent to another model" in EXHAUSTED_NOTE_NO_PROGRESS + + +def test_a_chat_with_work_behind_it_is_not_told_it_never_started(): + s, b = p_session() + s.messages.append(Message(role="user", content="find people to connect with", branch_id=b)) + s.messages.append(Message(role="tool_call", content={"id": "t1", "tool": "BrowserAgent", "input": {}}, branch_id=b)) + s.messages.append(Message(role="tool_result", content={"tool_use_id": "t1", "text": "ok"}, branch_id=b)) + s.messages.append(Message(role="user", content="keep going", branch_id=b)) + assert turn_showed_work(s) is False, "nothing since the LAST user message" + assert session_showed_work(s) is True, "but the chat plainly did work" + + +def test_the_long_chat_note_rules_out_the_wrong_remedy_and_names_the_right_one(): + low = EXHAUSTED_NOTE_LONG_CHAT.lower() + assert "switching models will not help" in low + assert "fresh chat" in low + assert "could not get started" not in low, "false to anyone reading their own transcript" + + +def test_the_three_notes_are_distinct(): + assert len({EXHAUSTED_NOTE, EXHAUSTED_NOTE_LONG_CHAT, EXHAUSTED_NOTE_NO_PROGRESS}) == 3 + + +def test_the_selector_prefers_turn_work_then_session_work(): + """Ordering: a turn that produced work must still get the 'work is above' note.""" + import inspect + from backend.apps.agents.manager.run import empty_finish as mod + src = inspect.getsource(mod.surface_exhausted) + assert src.index("turn_showed_work(session)") < src.index("session_showed_work(session)")