[eric] agents: self-heals stop being invisible: a stuck tool's restart and the CLI's own compaction reach the card, any signal death resumes, previews never quote the harness, spent retries are not "Done", one copy of the token-rotation notice

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012G8kyALnPjsA7aJFmMBq3R

backend/apps/agents/agents.py

@@ -11,6 +11,7 @@ from backend.apps.agents.agent_manager import agent_manager
 from backend.apps.agents.core.fault_injection import announce as p_announce_armed_faults
 from backend.apps.agents.core.models import AgentConfig, AgentSession, ApprovalResponse
 from backend.apps.agents.core.seq_log import seq_log
+from backend.apps.agents.core.ws_manager import preview_text
 from backend.apps.agents.manager.session.history_compaction import estimate_post_compact_input
 from backend.config.Apps import SubApp
 
@@ -52,14 +53,13 @@ def p_session_list_item(session: AgentSession) -> Dict[str, Any]:
     """Serialize dashboard metadata without retaining the full chat history."""
     data = session.model_dump(mode="json", exclude={"messages"})
     messages = session.messages
-    last_content = messages[-1].content if messages else ""
     first_user_content = next(
         (message.content for message in messages if message.role == "user"),
         "",
     )
     data.update(
         messages=[],
-        last_message_preview=last_content[:120] if isinstance(last_content, str) else "",
+        last_message_preview=preview_text(messages),
         first_user_message=(
             first_user_content[:200] if isinstance(first_user_content, str) else ""
         ),

backend/apps/agents/browser/effect_disposition.py

@@ -70,5 +70,5 @@ def unverified_reads_line(action_log: List[Dict]) -> str:
         return ""
     if any(a.get("ok") and str(a.get("result_summary") or "").strip() for a in p_reads):
         return ""
-    return ("No page content was read back successfully during this run, so any specific values "
-            "above are unverified and must not be treated as confirmed.")
+    return ("**Heads up: no page content was read back successfully during this run, so any specific "
+            "values above are unverified and must not be treated as confirmed.**")

backend/apps/agents/core/error_classify.py

@@ -464,8 +464,8 @@ def is_stale_tool_schema_error(exc: BaseException, extra_text: str = "") -> bool
 # narrow. 401 stays out (a rotating token really does heal, which is why the reset-hint rule exists),
 # and so do 408/429. Matched only in status POSITION, so a "400" in a line number or a byte count
 # cannot promote itself into a verdict (ENG-365 learned that the hard way with "line 401,").
-# 143/137 when the CLI re-raises the signal it caught; -15/-9 when it could not (SIGKILL is uncatchable, so a SIGKILL always arrives as -9).
-P_KILLED_EXIT = re.compile(r"Command failed with exit code (143|137|-15|-9)\b")
+# A negative code is the signal that killed the process (-9 SIGKILL, -15 SIGTERM, -2 SIGINT, -1 excluded: the SDK's own wait() sentinel); 129-159 is the same signal re-raised by a handler (128+n).
+P_KILLED_EXIT = re.compile(r"Command failed with exit code (-(?:[2-9]|[12]\d|3[01])|1(?:29|[3-4]\d|5\d))\b")
 
 
 @typechecked

backend/apps/agents/core/ws_manager.py

@@ -25,6 +25,19 @@ BROWSER_CMD_REBROADCAST_S = 3.0
 P_WS_RECONNECT_WAIT_S = 8.0
 
 
+def preview_text(messages: list) -> str:
+    """The last thing the USER or the model said, never a hidden harness prompt or a system note:
+    a collapsed card used to preview "Finish the task, then answer in plain text." as if the user
+    had typed it. Accepts message dicts or objects."""
+    for m in reversed(messages or []):
+        get = (lambda k, d=None: m.get(k, d)) if isinstance(m, dict) else (lambda k, d=None: getattr(m, k, d))
+        if get("hidden") or get("role") not in ("user", "assistant"):
+            continue
+        c = get("content", "")
+        return c[:120] if isinstance(c, str) else ""
+    return ""
+
+
 def slim_status_data(event: str, data: dict) -> dict:
     """agent:status frames carry session METADATA, never the transcript: every message already
     reaches clients as its own agent:message event (and the stream), so re-shipping full history
@@ -36,11 +49,10 @@ def slim_status_data(event: str, data: dict) -> dict:
     if not isinstance(sess, dict) or not sess.get("messages"):
         return data
     messages = sess["messages"]
-    last = messages[-1].get("content", "")
     first_user = next((m.get("content") for m in messages if m.get("role") == "user"), "")
     slim = dict(sess)
     slim["messages"] = []
-    slim["last_message_preview"] = last[:120] if isinstance(last, str) else ""
+    slim["last_message_preview"] = preview_text(messages)
     slim["first_user_message"] = first_user[:200] if isinstance(first_user, str) else ""
     slim["message_count"] = len(messages)
     out = dict(data)

backend/apps/agents/manager/run/TurnRunner.py

@@ -157,6 +157,10 @@ class TurnRunner(AgentManagerProtocol):
                     if p_subtype == "compact_boundary":
                         turn.compact_boundaries += 1
                         session.cli_compactions += 1
+                        await ws_manager.send_to_session(session_id, "agent:context_status", {
+                            "session_id": session_id,
+                            "reason": "cli_compacted",
+                        })
                     elif p_subtype == "api_retry":
                         note_provider_retry(session_id, raw, turn)
 

backend/apps/agents/manager/run/handle_run_error.py

@@ -8,6 +8,8 @@ import logging
 from typing import List
 from typeguard import typechecked
 
+from backend.apps.agents.manager.streaming.provider_error_speech import CODEX_ROTATION_RESEND_NOTICE, CODEX_ROTATION_RETRY_NOTICE
+
 from backend.apps.agents.core.models import AgentSession, Message
 from backend.apps.agents.core.ws_manager import ws_manager
 from backend.apps.settings.settings import load_settings
@@ -383,7 +385,8 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
                 "attempt": session.reconnect_attempts,
             })
             return
-        session.status = "completed"
+        # Nothing reached the user this turn and the retries are spent: that is a failure the card must show, not a green Done next to "stopped before reporting back".
+        session.status = "completed" if turn.current_turn_emitted else "error"
         if turn.stream_text_msg_id:
             try:
                 await ws_manager.send_to_session(session_id, "agent:stream_end", {
@@ -523,7 +526,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
                 if p_codex_rotation:
                     p_notice = Message(
                         role="system",
-                        content="GPT subscription token just rotated (automatic, every couple minutes). Retrying your request automatically in about a minute, no action needed.",
+                        content=CODEX_ROTATION_RETRY_NOTICE,
                         branch_id=session.active_branch_id,
                     )
                     session.messages.append(p_notice)
@@ -538,12 +541,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
             ("codex/" in p_combined or "[codex/" in p_combined or p_model.startswith(("cx/", "gpt-")))
             and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or has_auth_status(p_combined))
         ):
-            friendly_msg = (
-                "GPT subscription token just rotated, this is "
-                "automatic and resets every couple minutes. Send "
-                "your message again in ~1 minute and it'll go "
-                "through. (No need to reconnect anything.)"
-            )
+            friendly_msg = CODEX_ROTATION_RESEND_NOTICE
             reason = "codex_token_rotating"
         elif "no credentials for provider" in p_combined:
             friendly_msg = (

backend/apps/agents/manager/streaming/handle_assistant_message.py

@@ -122,6 +122,8 @@ async def handle_assistant_message(
         # One door for everything the provider says, so classify the class here instead of adding a fifth phrasing above; measured 14/14 caught, 0 false positives on 2035 real assistant messages.
         from backend.apps.agents.manager.streaming.provider_error_speech import (
             AUTH as P_ERR_AUTH,
+            CODEX_ROTATION_RESEND_NOTICE,
+            CODEX_ROTATION_RETRY_NOTICE,
             POLICY as P_ERR_POLICY,
             classify_provider_error,
             is_transient,
@@ -151,7 +153,7 @@ async def handle_assistant_message(
                 p_notice = Message(
                     id=uuid4().hex,
                     role="system",
-                    content="GPT subscription token just rotated (automatic, every couple minutes). Retrying your request automatically in about a minute, no action needed.",
+                    content=CODEX_ROTATION_RETRY_NOTICE,
                     branch_id=session.active_branch_id,
                 )
                 session.messages.append(p_notice)
@@ -161,12 +163,7 @@ async def handle_assistant_message(
                 })
             if not p_healed:
                 if p_is_codex:
-                    friendly = (
-                        "GPT subscription token is still refreshing. This usually clears on "
-                        "its own; wait a minute and send your message again. If it keeps "
-                        "happening, open Settings → Models and click Reconnect on the "
-                        "OpenAI / GPT row."
-                    )
+                    friendly = CODEX_ROTATION_RESEND_NOTICE
                     reason = "codex_token_expired"
                 elif "gemini-cli/" in lower_text or "[gemini" in lower_text:
                     friendly = (

backend/apps/agents/manager/streaming/provider_error_speech.py

@@ -258,3 +258,16 @@ def is_transient(err: ProviderError) -> bool:
             return False
         return err.reset_seconds is not None and err.reset_seconds <= 6 * 3600
     return False
+
+
+# The GPT subscription lane rotates its token every couple of minutes and the 401 that lands inside
+# the window used to be worded four different ways depending on the door it came through.
+CODEX_ROTATION_RETRY_NOTICE = (
+    "GPT subscription token just rotated (automatic, every couple minutes). Retrying your request "
+    "automatically in about a minute, no action needed."
+)
+CODEX_ROTATION_RESEND_NOTICE = (
+    "GPT subscription token just rotated (automatic, every couple minutes). Send your message again "
+    "in about a minute and it will go through; nothing to reconnect. If it keeps happening, open "
+    "Settings > Models and click Reconnect on the OpenAI / GPT row."
+)

backend/apps/agents/manager/streaming/unwedge_sidecar.py

@@ -131,6 +131,22 @@ RETRY_PROMPT = (
 )
 
 
+@typechecked
+def announce_tool_recovery(session_id: str, tool_name: str, outstanding_s: float) -> None:
+    """The one self-heal a user could watch for five minutes and never be told about: the card kept
+    its working dot while the sidecar was shot and the step redone. One transient pill, sent from
+    the loop thread the watchdog already runs on."""
+    try:
+        from backend.apps.agents.core.ws_manager import ws_manager
+        asyncio.ensure_future(ws_manager.send_to_session(session_id, "agent:tool_recovered", {
+            "session_id": session_id,
+            "tool": tool_name.replace(CORE_PREFIX, ""),
+            "outstanding_s": round(outstanding_s),
+        }))
+    except Exception:
+        logger.debug("tool_recovered announce failed", exc_info=True)
+
+
 @typechecked
 def arm_retry(session: object) -> bool:
     """Queue one hidden continuation so the agent redoes the lost step. Reuses the seam the
@@ -247,5 +263,6 @@ def arm_wedge_watchdog(ctx: object, tool_use_id: str, tool_name: str) -> None:
         # shutdown leaks them parked forever (the suite's flaky hang at interpreter exit).
         threading.Thread(target=unwedge, args=(session_id, tool_name, outstanding), daemon=True, name="unwedge").start()
         arm_retry(getattr(ctx, "session", None))
+        announce_tool_recovery(session_id, tool_name, outstanding)
 
     loop.call_later(WEDGE_SECONDS, p_check)

backend/tests/test_external_kill_respawn.py

@@ -22,6 +22,9 @@ def test_the_real_143_and_every_signal_spelling_are_recognised():
     # A SIGKILL cannot be caught, so the SDK reports it as a negative signal number, never 137 (live, dev kill matrix A3).
     assert is_external_kill_error(RuntimeError("Command failed with exit code -9 (exit code: -9)\nError output: Check stderr output for details"))
     assert is_external_kill_error(RuntimeError("Command failed with exit code -15 (exit code: -15)"))
+    # Any signal is the same story (dev kill matrix, 2026-09-01): SIGINT, SIGHUP, SIGABRT, raw or re-raised as 128+n.
+    for code in ("-2", "-1 " if False else "-6", "130", "134", "129", "159"):
+        assert is_external_kill_error(RuntimeError(f"Command failed with exit code {code} (exit code: {code})")), code
 
 
 @pytest.mark.parametrize("innocent", [
@@ -29,7 +32,9 @@ def test_the_real_143_and_every_signal_spelling_are_recognised():
     ("Command failed with exit code 143 (exit code: 143)", "API Error: 401 authentication_error"),
     ("Command failed with exit code 143 (exit code: 143)", "Error: request blocked by Usage Policy"),
     ("Command failed with exit code 1430", ""),
-    ("Command failed with exit code -1 (exit code: -1)", ""),
+    ("Command failed with exit code -1 (exit code: -1)", ""),  # the SDK's own wait() sentinel, not a signal
+    ("Command failed with exit code 128", ""),
+    ("Command failed with exit code 160", ""),
     ("Command failed with exit code -90", ""),
     ("Error code: 429 - rate limit", ""),
 ])

backend/tests/test_preview_never_quotes_the_harness.py

@@ -0,0 +1,15 @@
+"""A collapsed card's preview must never read a hidden harness prompt or a system note as if the user typed it."""
+from backend.apps.agents.core.models import Message
+from backend.apps.agents.core.ws_manager import preview_text
+
+
+def test_hidden_and_system_tails_are_skipped_for_dicts_and_objects() -> None:
+    dicts = [{"role": "user", "content": "build the thing"}, {"role": "user", "content": "Finish the task, then answer in plain text.", "hidden": True}, {"role": "system", "content": "This chat was still running when..."}]
+    assert preview_text(dicts) == "build the thing"
+    objs = [Message(role="user", content="build the thing", branch_id="main"), Message(role="assistant", content="Done: 3 files.", branch_id="main"), Message(role="user", content="The engine process running you was stopped from outside...", branch_id="main", hidden=True)]
+    assert preview_text(objs) == "Done: 3 files."
+
+
+def test_no_visible_turn_means_no_preview() -> None:
+    assert preview_text([{"role": "system", "content": "x"}]) == ""
+    assert preview_text([]) == ""

backend/tests/test_reconnect_resume.py

@@ -27,7 +27,7 @@ def p_session() -> AgentSession:
     return s
 
 
-def p_drive(monkeypatch, exc, session=None, stderr=None):
+def p_drive(monkeypatch, exc, session=None, stderr=None, emitted=False):
     events = []
 
     async def fake_send(session_id, event, data):
@@ -37,7 +37,9 @@ def p_drive(monkeypatch, exc, session=None, stderr=None):
     import backend.apps.service.client as service_client
     monkeypatch.setattr(service_client, "submit_diagnostic", lambda payload: None, raising=True)
     session = session or p_session()
-    asyncio.run(handle_run_error(exc, session, session.id, TurnState(), stderr or []))
+    turn = TurnState()
+    turn.current_turn_emitted = emitted
+    asyncio.run(handle_run_error(exc, session, session.id, turn, stderr or []))
     return session, events
 
 
@@ -64,6 +66,18 @@ def test_the_wait_widens_and_then_concedes(monkeypatch):
     session, events = p_drive(monkeypatch, ConnectionError("network is unreachable"), session=session)
     assert session.pending_continuation is False
     assert "agent:rate_limited" in [e for e, _ in events]
+    # Nothing reached the user and the retries are spent: "completed" put a green Done next to the
+    # exhausted note (self-heal audit, 2026-09-01). A turn that got some text out still completes.
+    assert session.status == "error"
+
+
+def test_a_turn_that_already_spoke_ends_completed_when_the_budget_is_spent(monkeypatch):
+    session = p_session()
+    for _ in RECONNECT_BACKOFFS:
+        session.pending_continuation = False
+        p_drive(monkeypatch, ConnectionError("network is unreachable"), session=session)
+    session.pending_continuation = False
+    session, _events = p_drive(monkeypatch, ConnectionError("network is unreachable"), session=session, emitted=True)
     assert session.status == "completed"