diff --git a/backend/apps/agents/agents.py b/backend/apps/agents/agents.py index 55a88c4c..63a3a1b3 100644 --- a/backend/apps/agents/agents.py +++ b/backend/apps/agents/agents.py @@ -11,6 +11,7 @@ from backend.apps.agents.agent_manager import agent_manager from backend.apps.agents.core.fault_injection import announce as p_announce_armed_faults from backend.apps.agents.core.models import AgentConfig, AgentSession, ApprovalResponse from backend.apps.agents.core.seq_log import seq_log +from backend.apps.agents.core.ws_manager import preview_text from backend.apps.agents.manager.session.history_compaction import estimate_post_compact_input from backend.config.Apps import SubApp @@ -52,14 +53,13 @@ def p_session_list_item(session: AgentSession) -> Dict[str, Any]: """Serialize dashboard metadata without retaining the full chat history.""" data = session.model_dump(mode="json", exclude={"messages"}) messages = session.messages - last_content = messages[-1].content if messages else "" first_user_content = next( (message.content for message in messages if message.role == "user"), "", ) data.update( messages=[], - last_message_preview=last_content[:120] if isinstance(last_content, str) else "", + last_message_preview=preview_text(messages), first_user_message=( first_user_content[:200] if isinstance(first_user_content, str) else "" ), diff --git a/backend/apps/agents/browser/effect_disposition.py b/backend/apps/agents/browser/effect_disposition.py index 9567f1d6..bd05be2c 100644 --- a/backend/apps/agents/browser/effect_disposition.py +++ b/backend/apps/agents/browser/effect_disposition.py @@ -70,5 +70,5 @@ def unverified_reads_line(action_log: List[Dict]) -> str: return "" if any(a.get("ok") and str(a.get("result_summary") or "").strip() for a in p_reads): return "" - return ("No page content was read back successfully during this run, so any specific values " - "above are unverified and must not be treated as confirmed.") + return ("**Heads up: no page content was read back successfully during this run, so any specific " + "values above are unverified and must not be treated as confirmed.**") diff --git a/backend/apps/agents/core/error_classify.py b/backend/apps/agents/core/error_classify.py index 18d1e1c0..56c7a0ee 100644 --- a/backend/apps/agents/core/error_classify.py +++ b/backend/apps/agents/core/error_classify.py @@ -464,8 +464,8 @@ def is_stale_tool_schema_error(exc: BaseException, extra_text: str = "") -> bool # narrow. 401 stays out (a rotating token really does heal, which is why the reset-hint rule exists), # and so do 408/429. Matched only in status POSITION, so a "400" in a line number or a byte count # cannot promote itself into a verdict (ENG-365 learned that the hard way with "line 401,"). -# 143/137 when the CLI re-raises the signal it caught; -15/-9 when it could not (SIGKILL is uncatchable, so a SIGKILL always arrives as -9). -P_KILLED_EXIT = re.compile(r"Command failed with exit code (143|137|-15|-9)\b") +# A negative code is the signal that killed the process (-9 SIGKILL, -15 SIGTERM, -2 SIGINT, -1 excluded: the SDK's own wait() sentinel); 129-159 is the same signal re-raised by a handler (128+n). +P_KILLED_EXIT = re.compile(r"Command failed with exit code (-(?:[2-9]|[12]\d|3[01])|1(?:29|[3-4]\d|5\d))\b") @typechecked diff --git a/backend/apps/agents/core/ws_manager.py b/backend/apps/agents/core/ws_manager.py index 0db3e8e1..3578c7fa 100644 --- a/backend/apps/agents/core/ws_manager.py +++ b/backend/apps/agents/core/ws_manager.py @@ -25,6 +25,19 @@ BROWSER_CMD_REBROADCAST_S = 3.0 P_WS_RECONNECT_WAIT_S = 8.0 +def preview_text(messages: list) -> str: + """The last thing the USER or the model said, never a hidden harness prompt or a system note: + a collapsed card used to preview "Finish the task, then answer in plain text." as if the user + had typed it. Accepts message dicts or objects.""" + for m in reversed(messages or []): + get = (lambda k, d=None: m.get(k, d)) if isinstance(m, dict) else (lambda k, d=None: getattr(m, k, d)) + if get("hidden") or get("role") not in ("user", "assistant"): + continue + c = get("content", "") + return c[:120] if isinstance(c, str) else "" + return "" + + def slim_status_data(event: str, data: dict) -> dict: """agent:status frames carry session METADATA, never the transcript: every message already reaches clients as its own agent:message event (and the stream), so re-shipping full history @@ -36,11 +49,10 @@ def slim_status_data(event: str, data: dict) -> dict: if not isinstance(sess, dict) or not sess.get("messages"): return data messages = sess["messages"] - last = messages[-1].get("content", "") first_user = next((m.get("content") for m in messages if m.get("role") == "user"), "") slim = dict(sess) slim["messages"] = [] - slim["last_message_preview"] = last[:120] if isinstance(last, str) else "" + slim["last_message_preview"] = preview_text(messages) slim["first_user_message"] = first_user[:200] if isinstance(first_user, str) else "" slim["message_count"] = len(messages) out = dict(data) diff --git a/backend/apps/agents/manager/run/TurnRunner.py b/backend/apps/agents/manager/run/TurnRunner.py index 85558553..2d146182 100644 --- a/backend/apps/agents/manager/run/TurnRunner.py +++ b/backend/apps/agents/manager/run/TurnRunner.py @@ -157,6 +157,10 @@ class TurnRunner(AgentManagerProtocol): if p_subtype == "compact_boundary": turn.compact_boundaries += 1 session.cli_compactions += 1 + await ws_manager.send_to_session(session_id, "agent:context_status", { + "session_id": session_id, + "reason": "cli_compacted", + }) elif p_subtype == "api_retry": note_provider_retry(session_id, raw, turn) diff --git a/backend/apps/agents/manager/run/handle_run_error.py b/backend/apps/agents/manager/run/handle_run_error.py index d203aa55..699a4b2f 100644 --- a/backend/apps/agents/manager/run/handle_run_error.py +++ b/backend/apps/agents/manager/run/handle_run_error.py @@ -8,6 +8,8 @@ import logging from typing import List from typeguard import typechecked +from backend.apps.agents.manager.streaming.provider_error_speech import CODEX_ROTATION_RESEND_NOTICE, CODEX_ROTATION_RETRY_NOTICE + from backend.apps.agents.core.models import AgentSession, Message from backend.apps.agents.core.ws_manager import ws_manager from backend.apps.settings.settings import load_settings @@ -383,7 +385,8 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, "attempt": session.reconnect_attempts, }) return - session.status = "completed" + # Nothing reached the user this turn and the retries are spent: that is a failure the card must show, not a green Done next to "stopped before reporting back". + session.status = "completed" if turn.current_turn_emitted else "error" if turn.stream_text_msg_id: try: await ws_manager.send_to_session(session_id, "agent:stream_end", { @@ -523,7 +526,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, if p_codex_rotation: p_notice = Message( role="system", - content="GPT subscription token just rotated (automatic, every couple minutes). Retrying your request automatically in about a minute, no action needed.", + content=CODEX_ROTATION_RETRY_NOTICE, branch_id=session.active_branch_id, ) session.messages.append(p_notice) @@ -538,12 +541,7 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str, ("codex/" in p_combined or "[codex/" in p_combined or p_model.startswith(("cx/", "gpt-"))) and ("authentication token is expired" in p_combined or "authentication token has expired" in p_combined or has_auth_status(p_combined)) ): - friendly_msg = ( - "GPT subscription token just rotated, this is " - "automatic and resets every couple minutes. Send " - "your message again in ~1 minute and it'll go " - "through. (No need to reconnect anything.)" - ) + friendly_msg = CODEX_ROTATION_RESEND_NOTICE reason = "codex_token_rotating" elif "no credentials for provider" in p_combined: friendly_msg = ( diff --git a/backend/apps/agents/manager/streaming/handle_assistant_message.py b/backend/apps/agents/manager/streaming/handle_assistant_message.py index c319b1db..9e4ddd89 100644 --- a/backend/apps/agents/manager/streaming/handle_assistant_message.py +++ b/backend/apps/agents/manager/streaming/handle_assistant_message.py @@ -122,6 +122,8 @@ async def handle_assistant_message( # One door for everything the provider says, so classify the class here instead of adding a fifth phrasing above; measured 14/14 caught, 0 false positives on 2035 real assistant messages. from backend.apps.agents.manager.streaming.provider_error_speech import ( AUTH as P_ERR_AUTH, + CODEX_ROTATION_RESEND_NOTICE, + CODEX_ROTATION_RETRY_NOTICE, POLICY as P_ERR_POLICY, classify_provider_error, is_transient, @@ -151,7 +153,7 @@ async def handle_assistant_message( p_notice = Message( id=uuid4().hex, role="system", - content="GPT subscription token just rotated (automatic, every couple minutes). Retrying your request automatically in about a minute, no action needed.", + content=CODEX_ROTATION_RETRY_NOTICE, branch_id=session.active_branch_id, ) session.messages.append(p_notice) @@ -161,12 +163,7 @@ async def handle_assistant_message( }) if not p_healed: if p_is_codex: - friendly = ( - "GPT subscription token is still refreshing. This usually clears on " - "its own; wait a minute and send your message again. If it keeps " - "happening, open Settings → Models and click Reconnect on the " - "OpenAI / GPT row." - ) + friendly = CODEX_ROTATION_RESEND_NOTICE reason = "codex_token_expired" elif "gemini-cli/" in lower_text or "[gemini" in lower_text: friendly = ( diff --git a/backend/apps/agents/manager/streaming/provider_error_speech.py b/backend/apps/agents/manager/streaming/provider_error_speech.py index 929c75fc..5b7d0f78 100644 --- a/backend/apps/agents/manager/streaming/provider_error_speech.py +++ b/backend/apps/agents/manager/streaming/provider_error_speech.py @@ -258,3 +258,16 @@ def is_transient(err: ProviderError) -> bool: return False return err.reset_seconds is not None and err.reset_seconds <= 6 * 3600 return False + + +# The GPT subscription lane rotates its token every couple of minutes and the 401 that lands inside +# the window used to be worded four different ways depending on the door it came through. +CODEX_ROTATION_RETRY_NOTICE = ( + "GPT subscription token just rotated (automatic, every couple minutes). Retrying your request " + "automatically in about a minute, no action needed." +) +CODEX_ROTATION_RESEND_NOTICE = ( + "GPT subscription token just rotated (automatic, every couple minutes). Send your message again " + "in about a minute and it will go through; nothing to reconnect. If it keeps happening, open " + "Settings > Models and click Reconnect on the OpenAI / GPT row." +) diff --git a/backend/apps/agents/manager/streaming/unwedge_sidecar.py b/backend/apps/agents/manager/streaming/unwedge_sidecar.py index decb1a32..e0a1c0fa 100644 --- a/backend/apps/agents/manager/streaming/unwedge_sidecar.py +++ b/backend/apps/agents/manager/streaming/unwedge_sidecar.py @@ -131,6 +131,22 @@ RETRY_PROMPT = ( ) +@typechecked +def announce_tool_recovery(session_id: str, tool_name: str, outstanding_s: float) -> None: + """The one self-heal a user could watch for five minutes and never be told about: the card kept + its working dot while the sidecar was shot and the step redone. One transient pill, sent from + the loop thread the watchdog already runs on.""" + try: + from backend.apps.agents.core.ws_manager import ws_manager + asyncio.ensure_future(ws_manager.send_to_session(session_id, "agent:tool_recovered", { + "session_id": session_id, + "tool": tool_name.replace(CORE_PREFIX, ""), + "outstanding_s": round(outstanding_s), + })) + except Exception: + logger.debug("tool_recovered announce failed", exc_info=True) + + @typechecked def arm_retry(session: object) -> bool: """Queue one hidden continuation so the agent redoes the lost step. Reuses the seam the @@ -247,5 +263,6 @@ def arm_wedge_watchdog(ctx: object, tool_use_id: str, tool_name: str) -> None: # shutdown leaks them parked forever (the suite's flaky hang at interpreter exit). threading.Thread(target=unwedge, args=(session_id, tool_name, outstanding), daemon=True, name="unwedge").start() arm_retry(getattr(ctx, "session", None)) + announce_tool_recovery(session_id, tool_name, outstanding) loop.call_later(WEDGE_SECONDS, p_check) diff --git a/backend/tests/test_external_kill_respawn.py b/backend/tests/test_external_kill_respawn.py index b2f03f68..ceabb179 100644 --- a/backend/tests/test_external_kill_respawn.py +++ b/backend/tests/test_external_kill_respawn.py @@ -22,6 +22,9 @@ def test_the_real_143_and_every_signal_spelling_are_recognised(): # A SIGKILL cannot be caught, so the SDK reports it as a negative signal number, never 137 (live, dev kill matrix A3). assert is_external_kill_error(RuntimeError("Command failed with exit code -9 (exit code: -9)\nError output: Check stderr output for details")) assert is_external_kill_error(RuntimeError("Command failed with exit code -15 (exit code: -15)")) + # Any signal is the same story (dev kill matrix, 2026-09-01): SIGINT, SIGHUP, SIGABRT, raw or re-raised as 128+n. + for code in ("-2", "-1 " if False else "-6", "130", "134", "129", "159"): + assert is_external_kill_error(RuntimeError(f"Command failed with exit code {code} (exit code: {code})")), code @pytest.mark.parametrize("innocent", [ @@ -29,7 +32,9 @@ def test_the_real_143_and_every_signal_spelling_are_recognised(): ("Command failed with exit code 143 (exit code: 143)", "API Error: 401 authentication_error"), ("Command failed with exit code 143 (exit code: 143)", "Error: request blocked by Usage Policy"), ("Command failed with exit code 1430", ""), - ("Command failed with exit code -1 (exit code: -1)", ""), + ("Command failed with exit code -1 (exit code: -1)", ""), # the SDK's own wait() sentinel, not a signal + ("Command failed with exit code 128", ""), + ("Command failed with exit code 160", ""), ("Command failed with exit code -90", ""), ("Error code: 429 - rate limit", ""), ]) diff --git a/backend/tests/test_preview_never_quotes_the_harness.py b/backend/tests/test_preview_never_quotes_the_harness.py new file mode 100644 index 00000000..537df547 --- /dev/null +++ b/backend/tests/test_preview_never_quotes_the_harness.py @@ -0,0 +1,15 @@ +"""A collapsed card's preview must never read a hidden harness prompt or a system note as if the user typed it.""" +from backend.apps.agents.core.models import Message +from backend.apps.agents.core.ws_manager import preview_text + + +def test_hidden_and_system_tails_are_skipped_for_dicts_and_objects() -> None: + dicts = [{"role": "user", "content": "build the thing"}, {"role": "user", "content": "Finish the task, then answer in plain text.", "hidden": True}, {"role": "system", "content": "This chat was still running when..."}] + assert preview_text(dicts) == "build the thing" + objs = [Message(role="user", content="build the thing", branch_id="main"), Message(role="assistant", content="Done: 3 files.", branch_id="main"), Message(role="user", content="The engine process running you was stopped from outside...", branch_id="main", hidden=True)] + assert preview_text(objs) == "Done: 3 files." + + +def test_no_visible_turn_means_no_preview() -> None: + assert preview_text([{"role": "system", "content": "x"}]) == "" + assert preview_text([]) == "" diff --git a/backend/tests/test_reconnect_resume.py b/backend/tests/test_reconnect_resume.py index 384371ba..bd3ed130 100644 --- a/backend/tests/test_reconnect_resume.py +++ b/backend/tests/test_reconnect_resume.py @@ -27,7 +27,7 @@ def p_session() -> AgentSession: return s -def p_drive(monkeypatch, exc, session=None, stderr=None): +def p_drive(monkeypatch, exc, session=None, stderr=None, emitted=False): events = [] async def fake_send(session_id, event, data): @@ -37,7 +37,9 @@ def p_drive(monkeypatch, exc, session=None, stderr=None): import backend.apps.service.client as service_client monkeypatch.setattr(service_client, "submit_diagnostic", lambda payload: None, raising=True) session = session or p_session() - asyncio.run(handle_run_error(exc, session, session.id, TurnState(), stderr or [])) + turn = TurnState() + turn.current_turn_emitted = emitted + asyncio.run(handle_run_error(exc, session, session.id, turn, stderr or [])) return session, events @@ -64,6 +66,18 @@ def test_the_wait_widens_and_then_concedes(monkeypatch): session, events = p_drive(monkeypatch, ConnectionError("network is unreachable"), session=session) assert session.pending_continuation is False assert "agent:rate_limited" in [e for e, _ in events] + # Nothing reached the user and the retries are spent: "completed" put a green Done next to the + # exhausted note (self-heal audit, 2026-09-01). A turn that got some text out still completes. + assert session.status == "error" + + +def test_a_turn_that_already_spoke_ends_completed_when_the_budget_is_spent(monkeypatch): + session = p_session() + for _ in RECONNECT_BACKOFFS: + session.pending_continuation = False + p_drive(monkeypatch, ConnectionError("network is unreachable"), session=session) + session.pending_continuation = False + session, _events = p_drive(monkeypatch, ConnectionError("network is unreachable"), session=session, emitted=True) assert session.status == "completed"