[eric] agents: a CLI that died between turns is respawned and resumed instead of carded on every send; a crash signal is named on the exhausted card

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_01C9zwUaHucUgrdxvK8FvjYT

backend/apps/agents/core/error_classify.py

@@ -480,6 +480,31 @@ def is_stale_tool_schema_error(exc: BaseException, extra_text: str = "") -> bool
 # cannot promote itself into a verdict (ENG-365 learned that the hard way with "line 401,").
 # A negative code is the signal that killed the process (-9 SIGKILL, -15 SIGTERM, -2 SIGINT, -1 excluded: the SDK's own wait() sentinel); 129-159 is the same signal re-raised by a handler (128+n).
 P_KILLED_EXIT = re.compile(r"Command failed with exit code (-(?:[2-9]|[12]\d|3[01])|1(?:29|[3-4]\d|5\d))\b")
+# The persistent client's own words when the CLI is already dead by the time we write to it: the process
+# died of a signal between turns (Alex, 2026-09-03: 33 turns in 72 h on two lanes, `exit code: -11`, a
+# segfault at spawn), and the generic branch carded every one without ever replacing the corpse.
+P_DEAD_PROCESS_WRITE = re.compile(r"Cannot write to terminated process \(exit code: (-?\d+)\)")
+P_CRASH_SIGNALS = {-4: "illegal instruction", -6: "abort", -7: "bus error", -10: "bus error", -11: "segmentation fault"}
+
+
+@typechecked
+def process_exit_code(exc: BaseException) -> Optional[int]:
+    """The exit code a dead-CLI error carries, whichever of the two shapes it wears."""
+    text = f"{exc!s}"
+    m = P_DEAD_PROCESS_WRITE.search(text) or P_KILLED_EXIT.search(text)
+    if not m:
+        return None
+    try:
+        return int(m.group(1))
+    except ValueError:
+        return None
+
+
+@typechecked
+def crash_signal_name(exc: BaseException) -> Optional[str]:
+    """A name when the CLI died of a crash signal (not SIGTERM/SIGKILL, which something else sent)."""
+    code = process_exit_code(exc)
+    return P_CRASH_SIGNALS.get(code) if code is not None else None
 
 
 @typechecked
@@ -490,7 +515,8 @@ def is_external_kill_error(exc: BaseException, extra_text: str = "") -> bool:
     conversation is intact in the CLI's own transcript, so a respawn that RESUMES it is the cure; the
     generic branch used to card it and force the expensive rebuild instead. A tail that carries an
     error of its own is some other failure wearing the exit code, and is left to the other branches."""
-    if not P_KILLED_EXIT.search(f"{exc!s}"):
+    text = f"{exc!s}"
+    if not (P_KILLED_EXIT.search(text) or P_DEAD_PROCESS_WRITE.search(text)):
         return False
     tail = (extra_text or "").strip()
     return not re.search(r"error", tail, re.IGNORECASE)

backend/apps/agents/manager/run/handle_run_error.py

@@ -24,6 +24,8 @@ from backend.apps.agents.core.error_classify import (
     is_long_context_error,
     is_context_pressure_death,
     is_external_kill_error,
+    crash_signal_name,
+    process_exit_code,
     is_stale_tool_schema_error,
     is_transient_capacity_error,
     is_free_trial_exhausted,
@@ -230,10 +232,19 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
                 logger.debug("submit_diagnostic external_kill_respawned failed", exc_info=True)
             return
         logger.warning(f"Agent {session_id}: the CLI was killed from outside again after a respawn; carding it")
-        friendly_msg = (
-            "Something on this computer, not OpenSwarm, stopped the agent's engine process twice in a "
-            "row while it was working. Its work so far is above; send your message again to continue."
-        )
+        p_crash = crash_signal_name(e)
+        if p_crash:
+            # A crash signal is the engine falling over, not a hand on the kill switch; the fix is on the machine.
+            friendly_msg = (
+                f"The agent's engine process crashed twice in a row ({p_crash}). Its work so far is above. "
+                "Quit and reopen OpenSwarm, then send your message again; if it keeps happening, send us "
+                "the newest claude crash report from ~/Library/Logs/DiagnosticReports."
+            )
+        else:
+            friendly_msg = (
+                "Something on this computer, not OpenSwarm, stopped the agent's engine process twice in a "
+                "row while it was working. Its work so far is above; send your message again to continue."
+            )
         error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
         absorb_repeat_card(session, error_msg)
         await ws_manager.send_to_session(session_id, "agent:message", {
@@ -244,7 +255,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
             from backend.apps.service.client import submit_diagnostic
             submit_diagnostic({
                 "kind": "model_error",
-                "subkind": "external_kill_respawn_exhausted",
+                "subkind": f"external_kill_respawn_exhausted:{process_exit_code(e)}",
                 "flight": flight_recorder.build_envelope(session_id, "model_error", "external_kill", session.model, "stream" if turn.current_turn_emitted else "spawn", -1),
                 "session_id": session_id,
                 "model": session.model,

backend/tests/test_dead_process_write.py

@@ -0,0 +1,37 @@
+"""Alex, 2026-09-03: 33 turns in 72 hours on two lanes died with `Cannot write to terminated process
+(exit code: -11)`, every one carded as a generic provider error, none respawned. The CLI had
+segfaulted between turns and the persistent client kept writing to the corpse. That shape is a dead
+process like any SIGTERM/SIGKILL exit and takes the same respawn-and-resume, and when the code is a
+crash signal the honest card names it and says to reopen the app."""
+
+import os
+
+from backend.apps.agents.core.error_classify import crash_signal_name, is_external_kill_error, process_exit_code
+
+P_REAL = "Cannot write to terminated process (exit code: -11)"
+
+
+def test_the_persistent_clients_dead_process_error_is_a_dead_process():
+    assert is_external_kill_error(RuntimeError(P_REAL))
+    assert process_exit_code(RuntimeError(P_REAL)) == -11
+    assert crash_signal_name(RuntimeError(P_REAL)) == "segmentation fault"
+
+
+def test_sigterm_and_sigkill_are_kills_not_crashes():
+    e = RuntimeError("Command failed with exit code 143 (exit code: 143)")
+    assert is_external_kill_error(e) and crash_signal_name(e) is None
+    e = RuntimeError("Cannot write to terminated process (exit code: -9)")
+    assert is_external_kill_error(e) and crash_signal_name(e) is None
+
+
+def test_a_dead_process_with_a_real_error_on_stderr_is_left_to_the_other_branches():
+    assert not is_external_kill_error(RuntimeError(P_REAL), extra_text="Error: API Error: 401 authentication_error")
+    assert not is_external_kill_error(RuntimeError("some other failure"))
+    assert process_exit_code(RuntimeError("line 11, in run")) is None
+
+
+def test_the_exhausted_card_names_a_crash_and_the_envelope_carries_the_code():
+    src = open(os.path.join(os.path.dirname(__file__), "..", "apps", "agents", "manager", "run", "handle_run_error.py")).read()
+    assert "crash_signal_name(e)" in src
+    assert "crashed twice in a row" in src and "DiagnosticReports" in src
+    assert 'f"external_kill_respawn_exhausted:{process_exit_code(e)}"' in src