From a81958acb980097b6e02b2c8289d5cd49e0e7ab9 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 16 Aug 2026 02:06:33 -0700 Subject: [PATCH] [eric] agents: first expired-token failure heals itself (fresh CLI + one hidden retry); the reconnect banner is the second rung (ENG-294) --- backend/apps/agents/core/models.py | 2 + backend/apps/agents/manager/Messaging.py | 1 + .../agents/manager/streaming/auth_retry.py | 32 ++++++++ .../streaming/handle_assistant_message.py | 74 ++++++++++--------- backend/apps/help/changelog.py | 1 + backend/tests/test_assistant_message.py | 47 +++++++++++- 6 files changed, 121 insertions(+), 36 deletions(-) create mode 100644 backend/apps/agents/manager/streaming/auth_retry.py diff --git a/backend/apps/agents/core/models.py b/backend/apps/agents/core/models.py index 33756948..7b7a74e3 100644 --- a/backend/apps/agents/core/models.py +++ b/backend/apps/agents/core/models.py @@ -139,6 +139,8 @@ class AgentSession(BaseModel): pending_continuation_toolless: bool = False # Silent-quit nudges spent since the user's last real message; hard-capped so an agent that keeps ending empty can't loop. empty_finish_nudges: int = 0 + # One transparent expired-token retry per user ask; the second failure earns the honest banner (ENG-294). + auth_retry_used: bool = False # Tool-call count at the last nudge: a re-nudge is only earned by NEW tool work since then. empty_finish_progress_mark: int = 0 # One honest "stopped without a report" line per exhausted nudge budget; resets with the budget. diff --git a/backend/apps/agents/manager/Messaging.py b/backend/apps/agents/manager/Messaging.py index d7695b61..5fc67516 100644 --- a/backend/apps/agents/manager/Messaging.py +++ b/backend/apps/agents/manager/Messaging.py @@ -139,6 +139,7 @@ class Messaging(AgentManagerProtocol): session.empty_finish_nudges = 0 session.empty_finish_progress_mark = 0 session.empty_finish_surfaced = False + session.auth_retry_used = False # Fire a background aux LLM call to generate a 3-6 word verb-phrase describing this turn ("Auditing the pull request", "Drafting your email"). The narrator pill swaps from its heuristic verb to this label as soon as it lands, usually ~500ms-1s into the turn, which is exactly when "Thinking…" starts feeling generic. Provider-agnostic via resolve_aux_model. Non-blocking; failure is silent and the heuristic stays. if not hidden and prompt: try: diff --git a/backend/apps/agents/manager/streaming/auth_retry.py b/backend/apps/agents/manager/streaming/auth_retry.py new file mode 100644 index 00000000..db6a3462 --- /dev/null +++ b/backend/apps/agents/manager/streaming/auth_retry.py @@ -0,0 +1,32 @@ +"""One transparent retry when a subscription token expires mid-session (ENG-294). + +The router surfaces an upstream 401 as assistant TEXT, and the old handling was a banner telling +the user to open Settings, find Models, click Reconnect, wait, and re-send: six actions to recover +from a token doing the one thing tokens always do. The router's own dispatcher usually refreshes +the credential within moments; what stays stale is OUR side, a pooled CLI still carrying the old +env. So the first expiry in an ask now rebuilds the session (fresh CLI, fresh router token) and +queues one hidden continuation to redo the failed step. A second expiry in the same ask means the +credential is genuinely dead, and the honest banner still fires; swallowing every 401 forever is +the failure mode this deliberately refuses. +""" +from typeguard import typechecked + +from backend.apps.agents.core.models import AgentSession + +AUTH_RETRY_PROMPT = ( + "The model provider returned an expired-credential error on your last step; the connection " + "has been rebuilt with a refreshed token. Redo that one step, then carry on where you left off." +) + + +@typechecked +def try_auth_self_heal(session: AgentSession) -> bool: + """Queue the one hidden retry on a fresh CLI. False = budget spent or a continuation is + already pending, and the caller should show the honest banner instead.""" + if session.auth_retry_used or session.pending_continuation: + return False + session.auth_retry_used = True + session.needs_fresh_session = True + session.pending_continuation = True + session.pending_continuation_prompt = AUTH_RETRY_PROMPT + return True diff --git a/backend/apps/agents/manager/streaming/handle_assistant_message.py b/backend/apps/agents/manager/streaming/handle_assistant_message.py index fb9d0c7b..3ab26084 100644 --- a/backend/apps/agents/manager/streaming/handle_assistant_message.py +++ b/backend/apps/agents/manager/streaming/handle_assistant_message.py @@ -118,42 +118,46 @@ async def handle_assistant_message( or ("provided authentication token" in lower_text and ("401" in lower_text or "expired" in lower_text)) ) if looks_like_router_auth_error: - if "codex/" in lower_text or "[codex" in lower_text: - friendly = ( - "GPT subscription token expired. Open Settings → Models and click " - "Reconnect on the OpenAI / GPT row to refresh, should take ~10s, " - "then send your message again." + from backend.apps.agents.manager.streaming.auth_retry import try_auth_self_heal + # First expiry in this ask heals silently (fresh CLI + hidden retry); the banner is + # reserved for the second failure, when the credential is genuinely dead (ENG-294). + if not try_auth_self_heal(session): + if "codex/" in lower_text or "[codex" in lower_text: + friendly = ( + "GPT subscription token expired. Open Settings → Models and click " + "Reconnect on the OpenAI / GPT row to refresh, should take ~10s, " + "then send your message again." + ) + reason = "codex_token_expired" + elif "gemini-cli/" in lower_text or "[gemini" in lower_text: + friendly = ( + "Gemini subscription token expired. Open Settings → Models and click " + "Reconnect on the Google / Gemini row, then send your message again." + ) + reason = "gemini_token_expired" + else: + friendly = ( + "Provider authentication expired. Open Settings → Models and " + "reconnect, then send your message again." + ) + reason = "router_auth_expired" + err_msg = Message( + id=uuid4().hex, + role="system", + content=friendly, + branch_id=session.active_branch_id, ) - reason = "codex_token_expired" - elif "gemini-cli/" in lower_text or "[gemini" in lower_text: - friendly = ( - "Gemini subscription token expired. Open Settings → Models and click " - "Reconnect on the Google / Gemini row, then send your message again." - ) - reason = "gemini_token_expired" - else: - friendly = ( - "Provider authentication expired. Open Settings → Models and " - "reconnect, then send your message again." - ) - reason = "router_auth_expired" - err_msg = Message( - id=uuid4().hex, - role="system", - content=friendly, - branch_id=session.active_branch_id, - ) - session.messages.append(err_msg) - await ws_manager.send_to_session(session_id, "agent:auth_error", { - "session_id": session_id, - "reason": reason, - "message": friendly, - "model": session.model, - }) - await ws_manager.send_to_session(session_id, "agent:message", { - "session_id": session_id, - "message": err_msg.model_dump(mode="json"), - }) + session.messages.append(err_msg) + await ws_manager.send_to_session(session_id, "agent:auth_error", { + "session_id": session_id, + "reason": reason, + "message": friendly, + "model": session.model, + }) + await ws_manager.send_to_session(session_id, "agent:message", { + "session_id": session_id, + "message": err_msg.model_dump(mode="json"), + }) else: asst_msg = Message( id=turn.stream_text_msg_id or uuid4().hex, diff --git a/backend/apps/help/changelog.py b/backend/apps/help/changelog.py index 04ddbf2a..cc9baef5 100644 --- a/backend/apps/help/changelog.py +++ b/backend/apps/help/changelog.py @@ -90,6 +90,7 @@ P_RELEASES: List[ReleaseNote] = [ "Clicking a chat in the sidebar or history now frames the whole card. The camera used to aim at the chat's collapsed footprint, so an opened chat could land with its bottom half off-screen and need a manual pan after every autofocus.", "Starting a new chat no longer makes another agent's revealed subagents disappear from the canvas. Their cards were being cleaned up as strays by the same pass that places the new one.", "Workflow run history shows each run's workflow name instead of the word \"Workflow\" on every row. The name now travels with the run, so it survives renames and deleted workflows.", + "An expired provider login heals itself mid-chat: the first failure rebuilds the connection and retries your message with zero clicks, and only a second failure asks you to reconnect. It used to take six manual steps every time a token aged out.", "Heavy sessions no longer vanish without a trace. When memory climbs past the safe line the app now sheds weight itself: preview thumbnails pause and refetchable caches drop, instead of growing until the operating system kills it mid-task.", ], ), diff --git a/backend/tests/test_assistant_message.py b/backend/tests/test_assistant_message.py index 163a0d5a..d0cac41f 100644 --- a/backend/tests/test_assistant_message.py +++ b/backend/tests/test_assistant_message.py @@ -33,11 +33,38 @@ async def test_plain_text_commits_assistant_message(): @pytest.mark.asyncio -async def test_router_auth_error_surfaces_card_not_assistant_text(): +async def test_first_token_expiry_heals_silently(monkeypatch): + # ENG-294: the first expiry in an ask must cost the user ZERO actions: no banner, no committed + # reply, just a fresh-CLI rebuild and one hidden retry queued on the continuation seam. session, turn, thinking = p_fixt() txt = "[codex/gpt-5] Failed to authenticate: 401 provided authentication token is expired" events = [] + async def fake_send(sid, event, data): + events.append(event) + + with patch.object(assistant_message.ws_manager, "send_to_session", new=fake_send): + await assistant_message.handle_assistant_message( + p_asst([TextBlock(text=txt)]), session, session.id, turn, thinking, {}, {}) + assert not any(m.role == "system" for m in session.messages), "no banner on the first expiry" + assert not any(m.role == "assistant" for m in session.messages) + assert "agent:auth_error" not in events + assert session.auth_retry_used is True + assert session.needs_fresh_session is True, "the fresh CLI is what drops the stale token" + assert session.pending_continuation is True and session.pending_continuation_prompt + assert turn.number == 1, "healing must not skip the turn bookkeeping" + + +@pytest.mark.asyncio +async def test_second_token_expiry_surfaces_card_not_assistant_text(): + # The banner is the SECOND rung: a credential that fails right after a rebuilt session is + # genuinely dead, and swallowing every 401 forever is the failure mode this refuses. + session, turn, thinking = p_fixt() + session.auth_retry_used = True + session.pending_continuation = False + txt = "[codex/gpt-5] Failed to authenticate: 401 provided authentication token is expired" + events = [] + async def fake_send(sid, event, data): events.append(event) @@ -49,6 +76,24 @@ async def test_router_auth_error_surfaces_card_not_assistant_text(): assert "agent:auth_error" in events +def test_heal_never_stacks_on_a_pending_continuation(): + from backend.apps.agents.manager.streaming.auth_retry import try_auth_self_heal + session = AgentSession(name="t", model="sonnet", dashboard_id="d") + session.pending_continuation = True + assert try_auth_self_heal(session) is False, "stacking would double-fire the continuation seam" + assert session.auth_retry_used is False, "a refused heal must not burn the budget" + + +def test_a_real_user_message_reopens_the_heal_budget(): + # Wire-check both directions: the flag is set by the heal AND cleared with the other per-ask + # budgets on a real (non-hidden) user message. + import inspect + from backend.apps.agents.manager import Messaging + src = inspect.getsource(Messaging) + block = src[src.index("if not hidden:"):src.index("if not hidden:") + 400] + assert "auth_retry_used = False" in block + + @pytest.mark.asyncio async def test_tool_use_block_commits_tool_call(): session, turn, thinking = p_fixt()