From 1ca4d0c5a38015f710c6805750c307729be2a305 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Tue, 1 Sep 2026 19:49:29 -0700 Subject: [PATCH] [eric] agents: a CLI killed from outside resumes on a fresh process once, instead of carding the turn and forcing the recap rebuild Co-Authored-By: Claude Fable 5.1 Claude-Session: https://claude.ai/code/session_012G8kyALnPjsA7aJFmMBq3R --- backend/apps/agents/core/error_classify.py | 17 +++ backend/apps/agents/core/models.py | 1 + backend/apps/agents/manager/Messaging.py | 1 + .../agents/manager/run/handle_run_error.py | 45 +++++++ .../agents/manager/streaming/auth_retry.py | 23 ++++ backend/tests/test_external_kill_respawn.py | 114 ++++++++++++++++++ 6 files changed, 201 insertions(+) create mode 100644 backend/tests/test_external_kill_respawn.py diff --git a/backend/apps/agents/core/error_classify.py b/backend/apps/agents/core/error_classify.py index 7bb57ea0..641e7383 100644 --- a/backend/apps/agents/core/error_classify.py +++ b/backend/apps/agents/core/error_classify.py @@ -464,6 +464,23 @@ def is_stale_tool_schema_error(exc: BaseException, extra_text: str = "") -> bool # narrow. 401 stays out (a rotating token really does heal, which is why the reset-hint rule exists), # and so do 408/429. Matched only in status POSITION, so a "400" in a line number or a byte count # cannot promote itself into a verdict (ENG-365 learned that the hard way with "line 401,"). +P_KILLED_EXIT = re.compile(r"Command failed with exit code (143|137)\b") + + +@typechecked +def is_external_kill_error(exc: BaseException, extra_text: str = "") -> bool: + """The CLI process died of SIGTERM (143) or SIGKILL (137) with nothing of its own to say. Seen + 2026-09-01 on a forked real chat (2 of 3 runs, 11-23 s after connect, empty stderr, no assistant + output yet) and across the fleet (one install 30x in an hour); nothing in our logs sends it. The + conversation is intact in the CLI's own transcript, so a respawn that RESUMES it is the cure; the + generic branch used to card it and force the expensive rebuild instead. A tail that carries an + error of its own is some other failure wearing the exit code, and is left to the other branches.""" + if not P_KILLED_EXIT.search(f"{exc!s}"): + return False + tail = (extra_text or "").strip() + return not re.search(r"error", tail, re.IGNORECASE) + + P_PERMANENT_STATUS = re.compile( r"(?:API\s+Error:\s*|HTTP\s+|status(?:\s*code)?\s*[:=]\s*|\[)\s*(?:400|422)\b", re.IGNORECASE, diff --git a/backend/apps/agents/core/models.py b/backend/apps/agents/core/models.py index e9bce0e0..8eeaa38e 100644 --- a/backend/apps/agents/core/models.py +++ b/backend/apps/agents/core/models.py @@ -140,6 +140,7 @@ class AgentSession(BaseModel): # A new CLI process that RESUMES the same transcript (dead transport, stale token, core sidecar never connected); unlike needs_fresh_session nothing is rebuilt, so no history is ever re-authored as text (ENG-382). needs_respawn: bool = False stale_tool_schema_retry_used: bool = False + external_kill_retry_used: bool = False # Auto-continue: agent loop dispatches a hidden turn at end-of-loop using pending_continuation_prompt. Race-free vs background tasks. pending_continuation: bool = False pending_continuation_prompt: Optional[str] = None diff --git a/backend/apps/agents/manager/Messaging.py b/backend/apps/agents/manager/Messaging.py index 73e7d4a8..a06dd605 100644 --- a/backend/apps/agents/manager/Messaging.py +++ b/backend/apps/agents/manager/Messaging.py @@ -173,6 +173,7 @@ class Messaging(AgentManagerProtocol): session.empty_finish_surfaced = False session.auth_retry_used = False session.stale_tool_schema_retry_used = False + session.external_kill_retry_used = False # The repeat-quit floor and the vanishing-quit rule key on this; one false positive used to arm both for the session's life (ENG-364). session.empty_finish_total = 0 # The borrowed API key was for one ask, and this is a new one; back to the lane they chose. diff --git a/backend/apps/agents/manager/run/handle_run_error.py b/backend/apps/agents/manager/run/handle_run_error.py index 65a63b69..d203aa55 100644 --- a/backend/apps/agents/manager/run/handle_run_error.py +++ b/backend/apps/agents/manager/run/handle_run_error.py @@ -16,6 +16,7 @@ from backend.apps.agents.core.error_classify import ( is_context_overflow_error, is_long_context_error, is_context_pressure_death, + is_external_kill_error, is_stale_tool_schema_error, is_transient_capacity_error, is_free_trial_exhausted, @@ -201,6 +202,50 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, logger.debug("submit_diagnostic stale_tool_schema failed", exc_info=True) return + # Same shape, different killer: the CLI process died of a signal with nothing to say for itself. + # The transcript is intact on disk, so a respawn that resumes it is the cure; the generic branch + # below would card it and force the recap rebuild, which on a long chat is the cliff itself. + if is_external_kill_error(e, extra_text=p_stderr_tail): + from backend.apps.agents.manager.streaming.auth_retry import try_external_kill_self_heal + if try_external_kill_self_heal(session): + logger.warning(f"Agent {session_id}: the CLI process was killed from outside ({str(e).splitlines()[0]}); respawning it and resuming the same transcript") + try: + from backend.apps.service.client import submit_diagnostic + submit_diagnostic({ + "kind": "recovered", + "subkind": "external_kill_respawned", + "session_id": session_id, + "model": session.model, + "error_preview": redact_for_telemetry(str(e), limit=200), + }) + except Exception: + logger.debug("submit_diagnostic external_kill_respawned failed", exc_info=True) + return + logger.warning(f"Agent {session_id}: the CLI was killed from outside again after a respawn; carding it") + friendly_msg = ( + "Something on this computer, not OpenSwarm, stopped the agent's engine process twice in a " + "row while it was working. Its work so far is above; send your message again to continue." + ) + error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) + absorb_repeat_card(session, error_msg) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": error_msg.model_dump(mode="json"), + }) + try: + from backend.apps.service.client import submit_diagnostic + submit_diagnostic({ + "kind": "model_error", + "subkind": "external_kill_respawn_exhausted", + "flight": flight_recorder.build_envelope(session_id, "model_error", "external_kill", session.model, "stream" if turn.current_turn_emitted else "spawn", -1), + "session_id": session_id, + "model": session.model, + "error_preview": redact_for_telemetry(str(e), limit=400), + }) + except Exception: + logger.debug("submit_diagnostic external_kill_respawn_exhausted failed", exc_info=True) + return + if is_context_overflow_error(e, extra_text=p_stderr_tail): p_tier_gate = is_long_context_error(e, extra_text=p_stderr_tail) friendly_msg = ( diff --git a/backend/apps/agents/manager/streaming/auth_retry.py b/backend/apps/agents/manager/streaming/auth_retry.py index ab652550..96868720 100644 --- a/backend/apps/agents/manager/streaming/auth_retry.py +++ b/backend/apps/agents/manager/streaming/auth_retry.py @@ -99,3 +99,26 @@ def try_stale_tool_schema_self_heal(session: AgentSession) -> bool: # resend once the router was warm, which is what dates the failure to timing, not the transcript. session.pending_continuation_delay_s = 20 return True + + +EXTERNAL_KILL_RETRY_PROMPT = ( + "The engine process running you was stopped from outside and has been restarted with this same " + "conversation. Carry on from exactly where you left off; if you were in the middle of a tool " + "step, redo that one step." +) + + +@typechecked +def try_external_kill_self_heal(session: AgentSession) -> bool: + """One respawn for a CLI killed from outside (SIGTERM/SIGKILL with nothing on stderr): the new + process resumes the same transcript, so the turn continues on top of its work instead of being + carded and rebuilt. Its own budget, like the other one-shots; a second kill inside the same ask + means something on the machine is hunting the process and the user is owed the honest card.""" + if session.external_kill_retry_used or session.pending_continuation: + return False + session.external_kill_retry_used = True + session.needs_respawn = True + session.pending_continuation = True + session.pending_continuation_prompt = EXTERNAL_KILL_RETRY_PROMPT + session.pending_continuation_delay_s = 2 + return True diff --git a/backend/tests/test_external_kill_respawn.py b/backend/tests/test_external_kill_respawn.py new file mode 100644 index 00000000..d4f06900 --- /dev/null +++ b/backend/tests/test_external_kill_respawn.py @@ -0,0 +1,114 @@ +"""A CLI killed from outside (exit 143/137, nothing on stderr) resumes on a fresh process instead of +carding and forcing the rebuild. Seen 2026-09-01 on a forked real chat, 2 of 3 runs; the fleet carries +the same string (one install 30x in an hour, Haik 17x in 14 days).""" +import inspect +import pytest +from backend.apps.agents.core.error_classify import is_external_kill_error, is_transient_capacity_error +from backend.apps.agents.core.models import AgentSession +from backend.apps.agents.manager.streaming.auth_retry import ( + EXTERNAL_KILL_RETRY_PROMPT, try_auth_self_heal, try_external_kill_self_heal, try_stale_tool_schema_self_heal, +) + +REAL = "Command failed with exit code 143 (exit code: 143)\nError output: Check stderr output for details" + + +def p_session() -> AgentSession: + return AgentSession(name="t", model="sonnet") + + +def test_the_real_143_and_a_sigkill_137_are_recognised(): + assert is_external_kill_error(RuntimeError(REAL)) + assert is_external_kill_error(RuntimeError("Command failed with exit code 137 (exit code: 137)")) + + +@pytest.mark.parametrize("innocent", [ + ("Command failed with exit code 1 (exit code: 1)", ""), + ("Command failed with exit code 143 (exit code: 143)", "API Error: 401 authentication_error"), + ("Command failed with exit code 143 (exit code: 143)", "Error: request blocked by Usage Policy"), + ("Command failed with exit code 1430", ""), + ("Error code: 429 - rate limit", ""), +]) +def test_other_failures_never_claim_it(innocent): + text, tail = innocent + assert not is_external_kill_error(RuntimeError(text), extra_text=tail) + + +def test_it_is_not_a_transient_capacity_error_either(): + assert not is_transient_capacity_error(RuntimeError(REAL)) + + +def test_one_respawn_then_the_budget_is_spent(): + s = p_session() + assert try_external_kill_self_heal(s) + assert s.needs_respawn and s.pending_continuation and s.pending_continuation_prompt == EXTERNAL_KILL_RETRY_PROMPT + assert "verbatim" not in EXTERNAL_KILL_RETRY_PROMPT.lower() + s.pending_continuation = False + assert not try_external_kill_self_heal(s), "one is the whole budget" + + +def test_it_never_stomps_a_continuation_already_armed(): + s = p_session() + s.pending_continuation = True + assert not try_external_kill_self_heal(s) + + +def test_it_does_not_eat_the_other_one_shots(): + s = p_session() + assert try_external_kill_self_heal(s) + s.pending_continuation = False + assert try_stale_tool_schema_self_heal(s), "separate budgets" + s.pending_continuation = False + assert try_auth_self_heal(s), "separate budgets" + + +def test_a_real_user_message_refills_the_budget(): + from backend.apps.agents.manager import Messaging + src = inspect.getsource(Messaging) + assert "session.external_kill_retry_used = False" in src + + +def test_the_branch_sits_above_every_card_emitting_branch(): + from backend.apps.agents.manager.run import handle_run_error as mod + src = inspect.getsource(mod.handle_run_error) + mine = src.index("is_external_kill_error(e") + assert src.index("is_stale_tool_schema_error(e") < mine + for later in ("is_context_overflow_error(e", "is_transient_capacity_error(e", "is_auth_error(e", "unclassified failure"): + assert mine < src.index(later), f"the respawn must precede {later}" + + +@pytest.mark.asyncio +async def test_handle_run_error_resumes_on_a_fresh_process_and_emits_no_card(monkeypatch): + from backend.apps.agents.manager.run import handle_run_error as mod + from backend.apps.agents.manager.streaming.state import TurnState + sent: list = [] + + async def p_send(session_id, event, payload): + sent.append((event, payload)) + + monkeypatch.setattr(mod.ws_manager, "send_to_session", p_send) + s = p_session(); s.dashboard_id = "d" + await mod.handle_run_error(RuntimeError(REAL), s, s.id, TurnState(), []) + assert s.needs_respawn is True, "resume the same transcript on a new process" + assert s.needs_fresh_session is False, "never the rebuild: on a long chat the recap IS the cliff" + assert s.pending_continuation is True + assert [m for m in s.messages if m.role == "system"] == [] + assert not any(e == "agent:message" for e, _ in sent) + + +@pytest.mark.asyncio +async def test_the_second_kill_cards_honestly(monkeypatch): + from backend.apps.agents.manager.run import handle_run_error as mod + from backend.apps.agents.manager.streaming.state import TurnState + + async def p_send(session_id, event, payload): + return None + + monkeypatch.setattr(mod.ws_manager, "send_to_session", p_send) + s = p_session(); s.dashboard_id = "d"; s.external_kill_retry_used = True + await mod.handle_run_error(RuntimeError(REAL), s, s.id, TurnState(), []) + cards = [m for m in s.messages if m.role == "system"] + assert len(cards) == 1 + low = str(cards[0].content).lower() + assert "not openswarm" in low and "send your message again" in low + assert "switch" not in low and "model" not in low, "the model did nothing wrong" + assert s.needs_fresh_session is False