[eric] agents: a CLI killed from outside resumes on a fresh process once, instead of carding the turn and forcing the recap rebuild

Co-Authored-By: Claude Fable 5.1 <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_012G8kyALnPjsA7aJFmMBq3R

backend/apps/agents/core/error_classify.py

@@ -464,6 +464,23 @@ def is_stale_tool_schema_error(exc: BaseException, extra_text: str = "") -> bool
 # narrow. 401 stays out (a rotating token really does heal, which is why the reset-hint rule exists),
 # and so do 408/429. Matched only in status POSITION, so a "400" in a line number or a byte count
 # cannot promote itself into a verdict (ENG-365 learned that the hard way with "line 401,").
+P_KILLED_EXIT = re.compile(r"Command failed with exit code (143|137)\b")
+
+
+@typechecked
+def is_external_kill_error(exc: BaseException, extra_text: str = "") -> bool:
+    """The CLI process died of SIGTERM (143) or SIGKILL (137) with nothing of its own to say. Seen
+    2026-09-01 on a forked real chat (2 of 3 runs, 11-23 s after connect, empty stderr, no assistant
+    output yet) and across the fleet (one install 30x in an hour); nothing in our logs sends it. The
+    conversation is intact in the CLI's own transcript, so a respawn that RESUMES it is the cure; the
+    generic branch used to card it and force the expensive rebuild instead. A tail that carries an
+    error of its own is some other failure wearing the exit code, and is left to the other branches."""
+    if not P_KILLED_EXIT.search(f"{exc!s}"):
+        return False
+    tail = (extra_text or "").strip()
+    return not re.search(r"error", tail, re.IGNORECASE)
+
+
 P_PERMANENT_STATUS = re.compile(
     r"(?:API\s+Error:\s*|HTTP\s+|status(?:\s*code)?\s*[:=]\s*|\[)\s*(?:400|422)\b",
     re.IGNORECASE,

backend/apps/agents/core/models.py

@@ -140,6 +140,7 @@ class AgentSession(BaseModel):
     # A new CLI process that RESUMES the same transcript (dead transport, stale token, core sidecar never connected); unlike needs_fresh_session nothing is rebuilt, so no history is ever re-authored as text (ENG-382).
     needs_respawn: bool = False
     stale_tool_schema_retry_used: bool = False
+    external_kill_retry_used: bool = False
     # Auto-continue: agent loop dispatches a hidden turn at end-of-loop using pending_continuation_prompt. Race-free vs background tasks.
     pending_continuation: bool = False
     pending_continuation_prompt: Optional[str] = None

backend/apps/agents/manager/Messaging.py

@@ -173,6 +173,7 @@ class Messaging(AgentManagerProtocol):
             session.empty_finish_surfaced = False
             session.auth_retry_used = False
             session.stale_tool_schema_retry_used = False
+            session.external_kill_retry_used = False
             # The repeat-quit floor and the vanishing-quit rule key on this; one false positive used to arm both for the session's life (ENG-364).
             session.empty_finish_total = 0
             # The borrowed API key was for one ask, and this is a new one; back to the lane they chose.

backend/apps/agents/manager/run/handle_run_error.py

@@ -16,6 +16,7 @@ from backend.apps.agents.core.error_classify import (
     is_context_overflow_error,
     is_long_context_error,
     is_context_pressure_death,
+    is_external_kill_error,
     is_stale_tool_schema_error,
     is_transient_capacity_error,
     is_free_trial_exhausted,
@@ -201,6 +202,50 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
             logger.debug("submit_diagnostic stale_tool_schema failed", exc_info=True)
         return
 
+    # Same shape, different killer: the CLI process died of a signal with nothing to say for itself.
+    # The transcript is intact on disk, so a respawn that resumes it is the cure; the generic branch
+    # below would card it and force the recap rebuild, which on a long chat is the cliff itself.
+    if is_external_kill_error(e, extra_text=p_stderr_tail):
+        from backend.apps.agents.manager.streaming.auth_retry import try_external_kill_self_heal
+        if try_external_kill_self_heal(session):
+            logger.warning(f"Agent {session_id}: the CLI process was killed from outside ({str(e).splitlines()[0]}); respawning it and resuming the same transcript")
+            try:
+                from backend.apps.service.client import submit_diagnostic
+                submit_diagnostic({
+                    "kind": "recovered",
+                    "subkind": "external_kill_respawned",
+                    "session_id": session_id,
+                    "model": session.model,
+                    "error_preview": redact_for_telemetry(str(e), limit=200),
+                })
+            except Exception:
+                logger.debug("submit_diagnostic external_kill_respawned failed", exc_info=True)
+            return
+        logger.warning(f"Agent {session_id}: the CLI was killed from outside again after a respawn; carding it")
+        friendly_msg = (
+            "Something on this computer, not OpenSwarm, stopped the agent's engine process twice in a "
+            "row while it was working. Its work so far is above; send your message again to continue."
+        )
+        error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id)
+        absorb_repeat_card(session, error_msg)
+        await ws_manager.send_to_session(session_id, "agent:message", {
+            "session_id": session_id,
+            "message": error_msg.model_dump(mode="json"),
+        })
+        try:
+            from backend.apps.service.client import submit_diagnostic
+            submit_diagnostic({
+                "kind": "model_error",
+                "subkind": "external_kill_respawn_exhausted",
+                "flight": flight_recorder.build_envelope(session_id, "model_error", "external_kill", session.model, "stream" if turn.current_turn_emitted else "spawn", -1),
+                "session_id": session_id,
+                "model": session.model,
+                "error_preview": redact_for_telemetry(str(e), limit=400),
+            })
+        except Exception:
+            logger.debug("submit_diagnostic external_kill_respawn_exhausted failed", exc_info=True)
+        return
+
     if is_context_overflow_error(e, extra_text=p_stderr_tail):
         p_tier_gate = is_long_context_error(e, extra_text=p_stderr_tail)
         friendly_msg = (

backend/apps/agents/manager/streaming/auth_retry.py

@@ -99,3 +99,26 @@ def try_stale_tool_schema_self_heal(session: AgentSession) -> bool:
     # resend once the router was warm, which is what dates the failure to timing, not the transcript.
     session.pending_continuation_delay_s = 20
     return True
+
+
+EXTERNAL_KILL_RETRY_PROMPT = (
+    "The engine process running you was stopped from outside and has been restarted with this same "
+    "conversation. Carry on from exactly where you left off; if you were in the middle of a tool "
+    "step, redo that one step."
+)
+
+
+@typechecked
+def try_external_kill_self_heal(session: AgentSession) -> bool:
+    """One respawn for a CLI killed from outside (SIGTERM/SIGKILL with nothing on stderr): the new
+    process resumes the same transcript, so the turn continues on top of its work instead of being
+    carded and rebuilt. Its own budget, like the other one-shots; a second kill inside the same ask
+    means something on the machine is hunting the process and the user is owed the honest card."""
+    if session.external_kill_retry_used or session.pending_continuation:
+        return False
+    session.external_kill_retry_used = True
+    session.needs_respawn = True
+    session.pending_continuation = True
+    session.pending_continuation_prompt = EXTERNAL_KILL_RETRY_PROMPT
+    session.pending_continuation_delay_s = 2
+    return True

backend/tests/test_external_kill_respawn.py

@@ -0,0 +1,114 @@
+"""A CLI killed from outside (exit 143/137, nothing on stderr) resumes on a fresh process instead of
+carding and forcing the rebuild. Seen 2026-09-01 on a forked real chat, 2 of 3 runs; the fleet carries
+the same string (one install 30x in an hour, Haik 17x in 14 days)."""
+import inspect
+import pytest
+from backend.apps.agents.core.error_classify import is_external_kill_error, is_transient_capacity_error
+from backend.apps.agents.core.models import AgentSession
+from backend.apps.agents.manager.streaming.auth_retry import (
+    EXTERNAL_KILL_RETRY_PROMPT, try_auth_self_heal, try_external_kill_self_heal, try_stale_tool_schema_self_heal,
+)
+
+REAL = "Command failed with exit code 143 (exit code: 143)\nError output: Check stderr output for details"
+
+
+def p_session() -> AgentSession:
+    return AgentSession(name="t", model="sonnet")
+
+
+def test_the_real_143_and_a_sigkill_137_are_recognised():
+    assert is_external_kill_error(RuntimeError(REAL))
+    assert is_external_kill_error(RuntimeError("Command failed with exit code 137 (exit code: 137)"))
+
+
+@pytest.mark.parametrize("innocent", [
+    ("Command failed with exit code 1 (exit code: 1)", ""),
+    ("Command failed with exit code 143 (exit code: 143)", "API Error: 401 authentication_error"),
+    ("Command failed with exit code 143 (exit code: 143)", "Error: request blocked by Usage Policy"),
+    ("Command failed with exit code 1430", ""),
+    ("Error code: 429 - rate limit", ""),
+])
+def test_other_failures_never_claim_it(innocent):
+    text, tail = innocent
+    assert not is_external_kill_error(RuntimeError(text), extra_text=tail)
+
+
+def test_it_is_not_a_transient_capacity_error_either():
+    assert not is_transient_capacity_error(RuntimeError(REAL))
+
+
+def test_one_respawn_then_the_budget_is_spent():
+    s = p_session()
+    assert try_external_kill_self_heal(s)
+    assert s.needs_respawn and s.pending_continuation and s.pending_continuation_prompt == EXTERNAL_KILL_RETRY_PROMPT
+    assert "verbatim" not in EXTERNAL_KILL_RETRY_PROMPT.lower()
+    s.pending_continuation = False
+    assert not try_external_kill_self_heal(s), "one is the whole budget"
+
+
+def test_it_never_stomps_a_continuation_already_armed():
+    s = p_session()
+    s.pending_continuation = True
+    assert not try_external_kill_self_heal(s)
+
+
+def test_it_does_not_eat_the_other_one_shots():
+    s = p_session()
+    assert try_external_kill_self_heal(s)
+    s.pending_continuation = False
+    assert try_stale_tool_schema_self_heal(s), "separate budgets"
+    s.pending_continuation = False
+    assert try_auth_self_heal(s), "separate budgets"
+
+
+def test_a_real_user_message_refills_the_budget():
+    from backend.apps.agents.manager import Messaging
+    src = inspect.getsource(Messaging)
+    assert "session.external_kill_retry_used = False" in src
+
+
+def test_the_branch_sits_above_every_card_emitting_branch():
+    from backend.apps.agents.manager.run import handle_run_error as mod
+    src = inspect.getsource(mod.handle_run_error)
+    mine = src.index("is_external_kill_error(e")
+    assert src.index("is_stale_tool_schema_error(e") < mine
+    for later in ("is_context_overflow_error(e", "is_transient_capacity_error(e", "is_auth_error(e", "unclassified failure"):
+        assert mine < src.index(later), f"the respawn must precede {later}"
+
+
+@pytest.mark.asyncio
+async def test_handle_run_error_resumes_on_a_fresh_process_and_emits_no_card(monkeypatch):
+    from backend.apps.agents.manager.run import handle_run_error as mod
+    from backend.apps.agents.manager.streaming.state import TurnState
+    sent: list = []
+
+    async def p_send(session_id, event, payload):
+        sent.append((event, payload))
+
+    monkeypatch.setattr(mod.ws_manager, "send_to_session", p_send)
+    s = p_session(); s.dashboard_id = "d"
+    await mod.handle_run_error(RuntimeError(REAL), s, s.id, TurnState(), [])
+    assert s.needs_respawn is True, "resume the same transcript on a new process"
+    assert s.needs_fresh_session is False, "never the rebuild: on a long chat the recap IS the cliff"
+    assert s.pending_continuation is True
+    assert [m for m in s.messages if m.role == "system"] == []
+    assert not any(e == "agent:message" for e, _ in sent)
+
+
+@pytest.mark.asyncio
+async def test_the_second_kill_cards_honestly(monkeypatch):
+    from backend.apps.agents.manager.run import handle_run_error as mod
+    from backend.apps.agents.manager.streaming.state import TurnState
+
+    async def p_send(session_id, event, payload):
+        return None
+
+    monkeypatch.setattr(mod.ws_manager, "send_to_session", p_send)
+    s = p_session(); s.dashboard_id = "d"; s.external_kill_retry_used = True
+    await mod.handle_run_error(RuntimeError(REAL), s, s.id, TurnState(), [])
+    cards = [m for m in s.messages if m.role == "system"]
+    assert len(cards) == 1
+    low = str(cards[0].content).lower()
+    assert "not openswarm" in low and "send your message again" in low
+    assert "switch" not in low and "model" not in low, "the model did nothing wrong"
+    assert s.needs_fresh_session is False