diff --git a/backend/apps/agents/core/error_classify.py b/backend/apps/agents/core/error_classify.py index 71aae888..34bf9acf 100644 --- a/backend/apps/agents/core/error_classify.py +++ b/backend/apps/agents/core/error_classify.py @@ -480,6 +480,31 @@ def is_stale_tool_schema_error(exc: BaseException, extra_text: str = "") -> bool # cannot promote itself into a verdict (ENG-365 learned that the hard way with "line 401,"). # A negative code is the signal that killed the process (-9 SIGKILL, -15 SIGTERM, -2 SIGINT, -1 excluded: the SDK's own wait() sentinel); 129-159 is the same signal re-raised by a handler (128+n). P_KILLED_EXIT = re.compile(r"Command failed with exit code (-(?:[2-9]|[12]\d|3[01])|1(?:29|[3-4]\d|5\d))\b") +# The persistent client's own words when the CLI is already dead by the time we write to it: the process +# died of a signal between turns (Alex, 2026-09-03: 33 turns in 72 h on two lanes, `exit code: -11`, a +# segfault at spawn), and the generic branch carded every one without ever replacing the corpse. +P_DEAD_PROCESS_WRITE = re.compile(r"Cannot write to terminated process \(exit code: (-?\d+)\)") +P_CRASH_SIGNALS = {-4: "illegal instruction", -6: "abort", -7: "bus error", -10: "bus error", -11: "segmentation fault"} + + +@typechecked +def process_exit_code(exc: BaseException) -> Optional[int]: + """The exit code a dead-CLI error carries, whichever of the two shapes it wears.""" + text = f"{exc!s}" + m = P_DEAD_PROCESS_WRITE.search(text) or P_KILLED_EXIT.search(text) + if not m: + return None + try: + return int(m.group(1)) + except ValueError: + return None + + +@typechecked +def crash_signal_name(exc: BaseException) -> Optional[str]: + """A name when the CLI died of a crash signal (not SIGTERM/SIGKILL, which something else sent).""" + code = process_exit_code(exc) + return P_CRASH_SIGNALS.get(code) if code is not None else None @typechecked @@ -490,7 +515,8 @@ def is_external_kill_error(exc: BaseException, extra_text: str = "") -> bool: conversation is intact in the CLI's own transcript, so a respawn that RESUMES it is the cure; the generic branch used to card it and force the expensive rebuild instead. A tail that carries an error of its own is some other failure wearing the exit code, and is left to the other branches.""" - if not P_KILLED_EXIT.search(f"{exc!s}"): + text = f"{exc!s}" + if not (P_KILLED_EXIT.search(text) or P_DEAD_PROCESS_WRITE.search(text)): return False tail = (extra_text or "").strip() return not re.search(r"error", tail, re.IGNORECASE) diff --git a/backend/apps/agents/manager/run/handle_run_error.py b/backend/apps/agents/manager/run/handle_run_error.py index c8ed278e..fa91baab 100644 --- a/backend/apps/agents/manager/run/handle_run_error.py +++ b/backend/apps/agents/manager/run/handle_run_error.py @@ -24,6 +24,8 @@ from backend.apps.agents.core.error_classify import ( is_long_context_error, is_context_pressure_death, is_external_kill_error, + crash_signal_name, + process_exit_code, is_stale_tool_schema_error, is_transient_capacity_error, is_free_trial_exhausted, @@ -230,10 +232,19 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, logger.debug("submit_diagnostic external_kill_respawned failed", exc_info=True) return logger.warning(f"Agent {session_id}: the CLI was killed from outside again after a respawn; carding it") - friendly_msg = ( - "Something on this computer, not OpenSwarm, stopped the agent's engine process twice in a " - "row while it was working. Its work so far is above; send your message again to continue." - ) + p_crash = crash_signal_name(e) + if p_crash: + # A crash signal is the engine falling over, not a hand on the kill switch; the fix is on the machine. + friendly_msg = ( + f"The agent's engine process crashed twice in a row ({p_crash}). Its work so far is above. " + "Quit and reopen OpenSwarm, then send your message again; if it keeps happening, send us " + "the newest claude crash report from ~/Library/Logs/DiagnosticReports." + ) + else: + friendly_msg = ( + "Something on this computer, not OpenSwarm, stopped the agent's engine process twice in a " + "row while it was working. Its work so far is above; send your message again to continue." + ) error_msg = Message(role="system", content=friendly_msg, branch_id=session.active_branch_id) absorb_repeat_card(session, error_msg) await ws_manager.send_to_session(session_id, "agent:message", { @@ -244,7 +255,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, from backend.apps.service.client import submit_diagnostic submit_diagnostic({ "kind": "model_error", - "subkind": "external_kill_respawn_exhausted", + "subkind": f"external_kill_respawn_exhausted:{process_exit_code(e)}", "flight": flight_recorder.build_envelope(session_id, "model_error", "external_kill", session.model, "stream" if turn.current_turn_emitted else "spawn", -1), "session_id": session_id, "model": session.model, diff --git a/backend/tests/test_dead_process_write.py b/backend/tests/test_dead_process_write.py new file mode 100644 index 00000000..5351fb32 --- /dev/null +++ b/backend/tests/test_dead_process_write.py @@ -0,0 +1,37 @@ +"""Alex, 2026-09-03: 33 turns in 72 hours on two lanes died with `Cannot write to terminated process +(exit code: -11)`, every one carded as a generic provider error, none respawned. The CLI had +segfaulted between turns and the persistent client kept writing to the corpse. That shape is a dead +process like any SIGTERM/SIGKILL exit and takes the same respawn-and-resume, and when the code is a +crash signal the honest card names it and says to reopen the app.""" + +import os + +from backend.apps.agents.core.error_classify import crash_signal_name, is_external_kill_error, process_exit_code + +P_REAL = "Cannot write to terminated process (exit code: -11)" + + +def test_the_persistent_clients_dead_process_error_is_a_dead_process(): + assert is_external_kill_error(RuntimeError(P_REAL)) + assert process_exit_code(RuntimeError(P_REAL)) == -11 + assert crash_signal_name(RuntimeError(P_REAL)) == "segmentation fault" + + +def test_sigterm_and_sigkill_are_kills_not_crashes(): + e = RuntimeError("Command failed with exit code 143 (exit code: 143)") + assert is_external_kill_error(e) and crash_signal_name(e) is None + e = RuntimeError("Cannot write to terminated process (exit code: -9)") + assert is_external_kill_error(e) and crash_signal_name(e) is None + + +def test_a_dead_process_with_a_real_error_on_stderr_is_left_to_the_other_branches(): + assert not is_external_kill_error(RuntimeError(P_REAL), extra_text="Error: API Error: 401 authentication_error") + assert not is_external_kill_error(RuntimeError("some other failure")) + assert process_exit_code(RuntimeError("line 11, in run")) is None + + +def test_the_exhausted_card_names_a_crash_and_the_envelope_carries_the_code(): + src = open(os.path.join(os.path.dirname(__file__), "..", "apps", "agents", "manager", "run", "handle_run_error.py")).read() + assert "crash_signal_name(e)" in src + assert "crashed twice in a row" in src and "DiagnosticReports" in src + assert 'f"external_kill_respawn_exhausted:{process_exit_code(e)}"' in src