[eric] agents: an outage parks the turn and resumes when the provider answers; OAuth connects survive a backend restart

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014wtspwSFzZmjCx9UNPAorQ

backend/apps/agents/agent_manager.py

@@ -50,7 +50,13 @@ class AgentManager(SessionLifecycle, SessionPersistence, Messaging, SessionContr
         already resumes the work, so the stale continuation quietly stands down."""
         if delay_s > 0:
             p_before = len(getattr(self.sessions.get(session_id), "messages", []) or [])
-            await asyncio.sleep(delay_s)
+            p_parked = self.sessions.get(session_id)
+            if p_parked is not None and getattr(p_parked, "awaiting_reconnect", False):
+                # An outage wait is a CEILING, not a sentence: a blind sleep strands the user long after their wifi returned. Rotation waits keep the flat sleep, where the window IS the point.
+                from backend.apps.agents.manager.run.reconnect_resume import wait_for_reconnect
+                await wait_for_reconnect(p_parked, delay_s)
+            else:
+                await asyncio.sleep(delay_s)
             p_session = self.sessions.get(session_id)
             if p_session is None:
                 return
@@ -229,6 +235,9 @@ class AgentManager(SessionLifecycle, SessionPersistence, Messaging, SessionContr
                     force_respawn=p_force_respawn,
                 )
             session.status = "completed"
+            # The turn got through, so the outage is over: the next unrelated blip starts from a full budget.
+            from backend.apps.agents.manager.run.reconnect_resume import clear_reconnect_wait
+            clear_reconnect_wait(session)
 
             # Silent-quit seal: a turn that ran tools and ended with no visible answer gets ONE hidden continue nudge (dispatched by the auto-continuation block below); a second silent quit in the same ask surfaces as-is rather than looping.
             try:

backend/apps/agents/core/error_classify.py

@@ -328,6 +328,17 @@ def is_cert_failure(exc: BaseException, extra_text: str = "") -> bool:
     return bool(CERT_FAILURE_PATTERNS.search(f"{exc!s}\n{extra_text}"))
 
 
+@typechecked
+def is_connection_lost(exc: BaseException) -> bool:
+    """True when the transport itself died, as opposed to the provider answering with a refusal.
+
+    Both arrive as "transient", but they want different recoveries: a dead socket leaves the CLI
+    holding a corpse and must respawn, while a 429 is a healthy connection carrying a NO, where
+    respawning just spends a process to be told the same thing.
+    """
+    return isinstance(exc, p_get_transient_exc_types())
+
+
 @typechecked
 def is_transient_capacity_error(exc: BaseException, extra_text: str = "") -> bool:
     # The Claude CLI's underlying ProcessError stringifies to a generic "Command failed with exit code 1 / Check stderr output for details"; the real cause (rate_limit_error / No pool capacity available / 429 / overloaded) only surfaces in the subprocess's stderr stream, which we capture via the SDK's `stderr` callback and pass in as extra_text. Classify against both so we catch capacity errors regardless of which channel carried the message.

backend/apps/agents/core/models.py

@@ -152,6 +152,10 @@ class AgentSession(BaseModel):
     suppress_recap_once: bool = False
     # Consecutive dirty deaths this session was MID-TURN for; the crash auto-resume breaker (hermes #30719 pairing: auto-resume must never outrun its circuit breaker).
     crash_interrupt_count: int = 0
+    # Outage rounds spent on this ask: the in-turn ladder covers only 335s, and the work is checkpointed, so a longer drop is waited out rather than ending the task.
+    reconnect_attempts: int = 0
+    # True while a turn is parked waiting for the connection back; persisted so a quit DURING the wait is still an owed turn at next boot.
+    awaiting_reconnect: bool = False
     # Seconds the auto-continuation dispatcher sleeps before sending (codex rotation windows last 1-2 min; an instant retry lands inside the same window and burns the one-shot budget).
     pending_continuation_delay_s: int = 0
     # Memory prompt block frozen at first compose (prefix-cache discipline: mid-chat fact writes must

backend/apps/agents/manager/Messaging.py

@@ -152,6 +152,9 @@ class Messaging(AgentManagerProtocol):
             session.empty_finish_progress_mark = 0
             session.empty_finish_surfaced = False
             session.auth_retry_used = False
+            # A human is here and driving, so an earlier outage stops counting against the next one.
+            session.reconnect_attempts = 0
+            session.awaiting_reconnect = False
         # Fire a background aux LLM call to generate a 3-6 word verb-phrase describing this turn ("Auditing the pull request", "Drafting your email"). The narrator pill swaps from its heuristic verb to this label as soon as it lands, usually ~500ms-1s into the turn, which is exactly when "Thinking…" starts feeling generic. Provider-agnostic via resolve_aux_model. Non-blocking; failure is silent and the heuristic stays.
         if not hidden and prompt:
             try:

backend/apps/agents/manager/run/handle_run_error.py

@@ -21,6 +21,7 @@ from backend.apps.agents.core.error_classify import (
     is_auth_error,
     is_cert_failure,
     is_cli_binary_missing,
+    is_connection_lost,
     is_unknown_model_error,
     parse_retry_after,
 )
@@ -178,6 +179,17 @@ async def handle_run_error(e: Exception, session: AgentSession, session_id: str,
             logger.debug("submit_diagnostic cli_binary_missing failed", exc_info=True)
     elif is_transient_capacity_error(e, extra_text=p_stderr_tail):
         # A genuine throttle (429/overload/capacity) that already burned the whole silent-backoff budget (the only way one reaches here). It's a limit, not a failure, so don't append a system-message card; emit a transient signal for the muted pill and mark the turn completed so it doesn't read as an error.
+        # 335s of ladder is a blip's worth of patience, and a closed lid or switched network outlasts it, so park and retry before conceding a turn the user never chose to end.
+        from backend.apps.agents.manager.run.reconnect_resume import arm_reconnect_resume
+        p_delay = arm_reconnect_resume(session, parse_retry_after(e, p_stderr_tail), is_connection_lost(e))
+        if p_delay is not None:
+            logger.info(f"Agent {session_id}: connection lost past the in-turn budget; retrying in {p_delay}s")
+            await ws_manager.send_to_session(session_id, "agent:reconnect_wait", {
+                "session_id": session_id,
+                "retry_in_s": p_delay,
+                "attempt": session.reconnect_attempts,
+            })
+            return
         session.status = "completed"
         if turn.stream_text_msg_id:
             try:

backend/apps/agents/manager/run/reconnect_resume.py

@@ -0,0 +1,136 @@
+"""Keep a turn alive across an outage that outlasts the turn's own retry budget.
+
+The in-turn ladder (CAPACITY_BACKOFFS) spends 335s and then gives up, which is the right call for
+a blip. It is the wrong call for a closed lid, a switched network, a hotel captive portal or a
+provider having a bad ten minutes: the user comes back to a task that stopped for a reason that was
+never theirs, and has to retype it.
+
+Nothing about that is unrecoverable. The transcript is already checkpointed, the tools that ran are
+still recorded, and the continuation seam that the auth self-heal uses will pick the work back up
+mid-task. So an outage becomes a wait on a widening schedule rather than an ending.
+
+Two properties this deliberately keeps:
+  - it is bounded (three rounds, then the honest pill), because a retry loop with no end is how you
+    burn a user's quota on a provider that is genuinely gone;
+  - the wait is PERSISTED, so quitting mid-wait leaves an owed turn that boot-restore resumes,
+    instead of a task that evaporated while nobody was looking.
+"""
+
+import asyncio
+import logging
+from typing import Optional
+
+from typeguard import typechecked
+
+from backend.apps.agents.core.models import AgentSession
+
+logger = logging.getLogger(__name__)
+
+# Widening but short at the start: most outages are seconds, and a user should see it heal itself rather than learn to press a button.
+RECONNECT_BACKOFFS = (60, 300, 900)
+
+# A provider's own "reset after" hint outranks our schedule, but capped, because a bad hint must not park a turn for an hour.
+RECONNECT_MAX_DELAY_S = 1800
+
+RECONNECT_PROMPT = (
+    "The connection to the model dropped and has just come back. Continue exactly where you left "
+    "off; do not redo completed steps."
+)
+
+
+@typechecked
+def arm_reconnect_resume(session: AgentSession, retry_after_s: Optional[int] = None,
+                         connection_lost: bool = False) -> Optional[int]:
+    """Park the turn and queue one more attempt. Returns the delay armed, or None when the budget
+    is spent and the caller should surface the honest pill instead.
+
+    Retrying the work IS the connectivity test, so there is no separate reachability oracle to get
+    wrong: either the next attempt goes through, or it fails and buys the next (longer) round.
+    """
+    if session.pending_continuation:
+        return None
+    attempts = int(getattr(session, "reconnect_attempts", 0) or 0)
+    if attempts >= len(RECONNECT_BACKOFFS):
+        return None
+
+    delay = RECONNECT_BACKOFFS[attempts]
+    if retry_after_s and retry_after_s > 0:
+        delay = max(delay, min(int(retry_after_s) + 5, RECONNECT_MAX_DELAY_S))
+
+    session.reconnect_attempts = attempts + 1
+    session.awaiting_reconnect = True
+    # Only a dead transport leaves the CLI holding a corpse; a 429 is a healthy pipe carrying a NO, and respawning for that spends a process to be told the same thing.
+    if connection_lost:
+        session.needs_fresh_session = True
+    session.pending_continuation = True
+    session.pending_continuation_prompt = RECONNECT_PROMPT
+    session.pending_continuation_delay_s = delay
+    return delay
+
+
+@typechecked
+def clear_reconnect_wait(session: AgentSession) -> None:
+    """A turn that got through ends the outage: drop the parked flag so a later, unrelated blip
+    starts from a full budget rather than inheriting this one's."""
+    session.awaiting_reconnect = False
+
+
+# How often to look while parked: short enough that a wifi blip costs seconds, long enough to stay cheap over a 15 minute outage.
+RECONNECT_POLL_S = 3
+
+# The probe must be fast: a hung connect would turn "check every 3s" into "check whenever the socket gives up".
+RECONNECT_PROBE_TIMEOUT_S = 1.5
+
+
+@typechecked
+def provider_probe_host(session: AgentSession) -> str:
+    """The host whose reachability actually decides whether a retry can succeed.
+
+    Router-backed lanes point the CLI at localhost, so probing THAT would come back healthy while
+    the machine is offline, which is the wrong answer at the only moment it matters. Probe the
+    provider the router is proxying to instead.
+    """
+    p_model = (getattr(session, "model", "") or "").lower()
+    if p_model.startswith(("cx/", "gpt-")) or "openai" in p_model:
+        return "api.openai.com"
+    if p_model.startswith(("gc/", "ag/", "gemini")) or "gemini" in p_model:
+        return "generativelanguage.googleapis.com"
+    return "api.anthropic.com"
+
+
+@typechecked
+async def provider_reachable(host: str) -> bool:
+    """True when a TCP connection to the provider completes. Deliberately not an HTTP request: no
+    auth, no cost, no quota, and nothing that a retry would have spent anyway."""
+    try:
+        p_fut = asyncio.open_connection(host, 443)
+        reader, writer = await asyncio.wait_for(p_fut, timeout=RECONNECT_PROBE_TIMEOUT_S)
+        writer.close()
+        try:
+            await writer.wait_closed()
+        except Exception:
+            pass
+        return True
+    except Exception:
+        return False
+
+
+@typechecked
+async def wait_for_reconnect(session: AgentSession, ceiling_s: int) -> None:
+    """Sleep until the provider answers again, or until the ceiling, whichever comes FIRST.
+
+    The backoff is a bound on patience, never a fixed sentence: a blind sleep would leave a user
+    watching a spinner for fourteen more minutes after their wifi already came back, which is worse
+    than the button it replaced. A captive portal can still answer TCP and fail the real request;
+    that costs one round and lands us exactly where a blind wait would have been anyway.
+    """
+    p_host = provider_probe_host(session)
+    p_waited = 0
+    while p_waited < ceiling_s:
+        p_step = min(RECONNECT_POLL_S, ceiling_s - p_waited)
+        await asyncio.sleep(p_step)
+        p_waited += p_step
+        if await provider_reachable(p_host):
+            logger.info(f"reconnect: {p_host} answered after {p_waited}s (ceiling was {ceiling_s}s)")
+            return
+    logger.info(f"reconnect: ceiling {ceiling_s}s reached without {p_host} answering; trying anyway")

backend/apps/agents/manager/session/SessionPersistence.py

@@ -49,6 +49,19 @@ class SessionPersistence(AgentManagerProtocol):
                         self.crash_resume_queue.append(sid)
                     else:
                         logger.warning(f"crash-resume breaker: session {sid} was mid-turn at {count} consecutive dirty deaths; leaving it for the manual chip")
+            elif data.get("awaiting_reconnect") and data.get("closed_at") is None:
+                # Parked mid-outage when the app went down. The file says "completed" only because
+                # the wait was dispatched as a continuation, so the status check above cannot see
+                # it; without this the task the user never chose to end just evaporates.
+                data["awaiting_reconnect"] = False
+                dirty = True
+                count = int(data.get("crash_interrupt_count", 0) or 0) + 1
+                data["crash_interrupt_count"] = count
+                if count <= 1:
+                    self.crash_resume_queue.append(sid)
+                else:
+                    logger.warning(f"crash-resume breaker: session {sid} was parked mid-outage at {count} consecutive dirty deaths; leaving it for the manual chip")
+
             # Mode migration: Chat was merged into Ask. Rewrite mode="chat" so old sessions keep loading after the chat.json file is gone.
             if data.get("mode") == "chat":
                 data["mode"] = "ask"

backend/apps/oauth_state.py

@@ -1,10 +1,108 @@
-# In-memory store for pending OAuth flows (state -> {provider, code_verifier, redirect_uri})
-pending_oauth: dict[str, dict] = {}
+"""Pending and recently-completed OAuth flows.
+
+The pending map used to live only in memory, and the gap that opens is the whole of ENG-363: the
+user clicks Connect, the browser leaves for the provider, the backend restarts for ANY reason
+(uvicorn reload in dev, the ENG-357 frozen-loop exit, a watchdog respawn, a crash), and the state
+that proves the returning callback is ours is simply gone. The callback then lands on the
+unknown-state branch and renders "Session expired", the Settings row spins forever, and the user
+concludes the product cannot connect to Anthropic. Haik reported exactly that.
+
+Retrying is not the fix, because the user is not the one who failed. Making the state outlive a
+restart is: the callback then completes on its own and there is nothing to click.
+
+The verifier is a short-lived, single-use secret, so it is written 0600, expires on a TTL, and is
+deleted the moment it is consumed. It never becomes a durable credential lying around on disk.
+"""
+
+import json
+import os
+import time
+from typing import Dict, Optional
+
+from typeguard import typechecked
+
+from backend.config.paths import DATA_ROOT
+
+# One OAuth round trip is a browser hop and a login; a quarter hour is generous for a human doing that, and short enough that an abandoned flow's verifier does not linger.
+PENDING_TTL_S = 15 * 60
+
+PENDING_PATH = os.path.join(DATA_ROOT, "pending_oauth.json")
+
+
+@typechecked
+def p_load() -> Dict[str, dict]:
+    """Read the durable map, dropping anything past its TTL. Unreadable state is treated as empty: a corrupt file must not make Connect permanently impossible."""
+    try:
+        with open(PENDING_PATH, encoding="utf-8") as fh:
+            raw = json.load(fh)
+    except Exception:
+        return {}
+    if not isinstance(raw, dict):
+        return {}
+    now = time.time()
+    return {
+        k: v for k, v in raw.items()
+        if isinstance(v, dict) and float(v.get("stored_at", 0) or 0) + PENDING_TTL_S > now
+    }
+
+
+@typechecked
+def p_store(entries: Dict[str, dict]) -> None:
+    """Write 0600 and replace atomically, so a crash mid-write cannot leave a half-parsed file that strands every later Connect."""
+    try:
+        os.makedirs(DATA_ROOT, exist_ok=True)
+        tmp = f"{PENDING_PATH}.tmp"
+        with open(tmp, "w", encoding="utf-8") as fh:
+            json.dump(entries, fh)
+        os.chmod(tmp, 0o600)
+        os.replace(tmp, PENDING_PATH)
+    except OSError:
+        pass
+
+
+class PendingOAuth:
+    """Dict-shaped so every existing call site keeps working, but backed by disk.
+
+    Deliberately not a plain dict subclass: the whole point is that reads come from the file, so an
+    entry written before a restart is still found by the process that comes back.
+    """
+
+    @typechecked
+    def __setitem__(self, state: str, value: dict) -> None:
+        entries = p_load()
+        entries[state] = {**value, "stored_at": time.time()}
+        p_store(entries)
+
+    @typechecked
+    def get(self, state: str, default: Optional[dict] = None) -> Optional[dict]:
+        return p_load().get(state, default)
+
+    @typechecked
+    def pop(self, state: str, default: Optional[dict] = None) -> Optional[dict]:
+        entries = p_load()
+        found = entries.pop(state, None)
+        if found is None:
+            return default
+        # Consumed: the verifier is single-use, so it stops existing here rather than aging out later.
+        p_store(entries)
+        return found
+
+    @typechecked
+    def __contains__(self, state: str) -> bool:
+        return state in p_load()
+
+    @typechecked
+    def __len__(self) -> int:
+        return len(p_load())
+
+
+pending_oauth = PendingOAuth()
 # Recently-completed OAuth states so the /api/subscriptions/callback handler can distinguish a legitimate duplicate callback (browser prefetch, refresh, or Google redirect retry after a slow first response) from a truly stale request. Bounded FIFO, drops the oldest entries once it grows past MAX_COMPLETED_OAUTH so it can't leak memory.
 completed_oauth: list[str] = []
 MAX_COMPLETED_OAUTH = 64
 
 
+@typechecked
 def mark_oauth_completed(state: str) -> None:
     if state in completed_oauth:
         return