[eric] agents: an outage parks the turn and resumes when the provider answers; OAuth connects survive a backend restart

Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
Claude-Session: https://claude.ai/code/session_014wtspwSFzZmjCx9UNPAorQ

backend/tests/test_oauth_state_durable.py

@@ -0,0 +1,68 @@
+"""ENG-363: a Connect that survives a backend restart.
+
+The user clicks Connect, the browser leaves for Anthropic, the backend restarts for any reason, and
+the returning callback used to find nothing and render "Session expired" while Settings spun
+forever. Haik reported that as "Model connection for anthropic does not work". The state is ours to
+keep, so keeping it is the fix; there is nothing for the user to retry.
+"""
+
+import json
+import os
+import time
+
+import pytest
+
+
+@pytest.fixture()
+def p_store(monkeypatch, tmp_path):
+    import backend.apps.oauth_state as st
+    monkeypatch.setattr(st, "PENDING_PATH", str(tmp_path / "pending_oauth.json"), raising=True)
+    monkeypatch.setattr(st, "DATA_ROOT", str(tmp_path), raising=True)
+    return st
+
+
+def test_a_pending_flow_survives_a_restart(p_store):
+    p_store.pending_oauth["state-abc"] = {
+        "provider": "claude", "code_verifier": "v", "redirect_uri": "http://localhost:20128/cb",
+    }
+    # A restart is a brand new object over the same file; the old process kept nothing.
+    fresh = p_store.PendingOAuth()
+    found = fresh.pop("state-abc")
+    assert found is not None, "the callback must still recognise its own flow"
+    assert found["code_verifier"] == "v"
+
+
+def test_consuming_a_flow_deletes_the_verifier(p_store):
+    p_store.pending_oauth["state-abc"] = {"provider": "claude", "code_verifier": "secret"}
+    p_store.pending_oauth.pop("state-abc")
+    on_disk = json.load(open(p_store.PENDING_PATH))
+    assert on_disk == {}, "a single-use secret must not outlive its use"
+
+
+def test_an_abandoned_flow_ages_out(p_store):
+    p_store.pending_oauth["stale"] = {"provider": "claude", "code_verifier": "x"}
+    raw = json.load(open(p_store.PENDING_PATH))
+    raw["stale"]["stored_at"] = time.time() - (p_store.PENDING_TTL_S + 60)
+    open(p_store.PENDING_PATH, "w").write(json.dumps(raw))
+    assert p_store.pending_oauth.get("stale") is None
+    assert "stale" not in p_store.pending_oauth
+
+
+def test_the_verifier_is_not_world_readable(p_store):
+    p_store.pending_oauth["state-abc"] = {"provider": "claude", "code_verifier": "secret"}
+    mode = os.stat(p_store.PENDING_PATH).st_mode & 0o777
+    assert mode == 0o600, f"pending verifiers must be owner-only, got {oct(mode)}"
+
+
+def test_a_corrupt_file_never_blocks_connecting(p_store):
+    """Negative control: unreadable state must degrade to 'no pending flow', never to an exception
+    that makes Connect impossible forever."""
+    open(p_store.PENDING_PATH, "w").write("{not json")
+    assert p_store.pending_oauth.get("anything") is None
+    p_store.pending_oauth["fresh"] = {"provider": "claude", "code_verifier": "v"}
+    assert p_store.pending_oauth.get("fresh") is not None, "a new flow still works"
+
+
+def test_an_unknown_state_is_still_unknown(p_store):
+    """Negative control: durability must not make the handler accept a state nobody issued."""
+    assert p_store.pending_oauth.pop("never-issued") is None

backend/tests/test_reconnect_resume.py

@@ -0,0 +1,212 @@
+"""An outage longer than the in-turn ladder must park the turn, not end it.
+
+The 335s of CAPACITY_BACKOFFS is a blip's worth of patience. A closed lid, a switched network or a
+provider's bad ten minutes outlasts it, and the user then comes back to a task that stopped for a
+reason that was never theirs. These pin the widening retry, its bound, and the persistence that
+makes quitting mid-wait survivable. Cross-platform by construction: file state and asyncio only,
+no signals and no platform paths.
+"""
+
+import asyncio
+
+import backend.apps.agents.core.ws_manager as ws_mod
+from backend.apps.agents.core.models import AgentSession, Message
+from backend.apps.agents.manager.run.handle_run_error import handle_run_error
+from backend.apps.agents.manager.run.reconnect_resume import (
+    RECONNECT_BACKOFFS,
+    RECONNECT_MAX_DELAY_S,
+    arm_reconnect_resume,
+    clear_reconnect_wait,
+)
+from backend.apps.agents.manager.streaming.state import TurnState
+
+
+def p_session() -> AgentSession:
+    s = AgentSession(name="t", model="sonnet", dashboard_id="d")
+    s.messages.append(Message(role="user", content="long task", branch_id=s.active_branch_id))
+    return s
+
+
+def p_drive(monkeypatch, exc, session=None, stderr=None):
+    events = []
+
+    async def fake_send(session_id, event, data):
+        events.append((event, data))
+
+    monkeypatch.setattr(ws_mod.ws_manager, "send_to_session", fake_send, raising=True)
+    import backend.apps.service.client as service_client
+    monkeypatch.setattr(service_client, "submit_diagnostic", lambda payload: None, raising=True)
+    session = session or p_session()
+    asyncio.run(handle_run_error(exc, session, session.id, TurnState(), stderr or []))
+    return session, events
+
+
+def test_an_outage_parks_the_turn_instead_of_ending_it(monkeypatch):
+    session, events = p_drive(monkeypatch, ConnectionError("Connection reset by peer"))
+    assert session.pending_continuation is True, "the work is queued to continue"
+    assert session.pending_continuation_delay_s == RECONNECT_BACKOFFS[0]
+    assert session.awaiting_reconnect is True
+    assert session.needs_fresh_session is True, "the CLI died with the outage; resume on a fresh one"
+    assert not [m for m in session.messages if m.role == "system"], "no card: nothing is over yet"
+    assert "agent:reconnect_wait" in [e for e, _ in events]
+    assert "agent:rate_limited" not in [e for e, _ in events]
+
+
+def test_the_wait_widens_and_then_concedes(monkeypatch):
+    session = p_session()
+    for expected in RECONNECT_BACKOFFS:
+        session.pending_continuation = False
+        p_drive(monkeypatch, ConnectionError("network is unreachable"), session=session)
+        assert session.pending_continuation_delay_s == expected
+
+    # Budget spent: the honest pill fires rather than a fourth, longer silence.
+    session.pending_continuation = False
+    session, events = p_drive(monkeypatch, ConnectionError("network is unreachable"), session=session)
+    assert session.pending_continuation is False
+    assert "agent:rate_limited" in [e for e, _ in events]
+    assert session.status == "completed"
+
+
+def test_a_provider_reset_hint_outranks_our_schedule_but_is_capped():
+    s = p_session()
+    assert arm_reconnect_resume(s, retry_after_s=600) == 605
+    s2 = p_session()
+    assert arm_reconnect_resume(s2, retry_after_s=99999) == RECONNECT_MAX_DELAY_S
+
+
+def test_a_shorter_hint_never_shrinks_the_wait():
+    """Negative control: a 1s hint during a real outage must not become a 1s hot loop."""
+    s = p_session()
+    assert arm_reconnect_resume(s, retry_after_s=1) == RECONNECT_BACKOFFS[0]
+
+
+def test_an_armed_continuation_is_never_stomped():
+    """Negative control: something else already queued the next turn, so this must stand down."""
+    s = p_session()
+    s.pending_continuation = True
+    s.pending_continuation_prompt = "someone else's continuation"
+    assert arm_reconnect_resume(s) is None
+    assert s.pending_continuation_prompt == "someone else's continuation"
+
+
+def test_a_turn_that_got_through_returns_the_budget():
+    s = p_session()
+    arm_reconnect_resume(s)
+    assert s.awaiting_reconnect is True
+    clear_reconnect_wait(s)
+    assert s.awaiting_reconnect is False
+
+
+def test_quitting_mid_wait_leaves_an_owed_turn(monkeypatch, tmp_path):
+    """The whole point of persisting the flag: the app going down DURING the wait must not be how
+    a task quietly evaporates. Boot-restore has to see it, even though the status says completed."""
+    from backend.apps.agents.agent_manager import AgentManager
+    import backend.apps.agents.manager.session.SessionPersistence as sp
+
+    parked = {
+        "id": "sess-parked", "status": "completed", "awaiting_reconnect": True,
+        "closed_at": None, "active_branch_id": "main",
+        "messages": [{"role": "user", "content": "go", "branch_id": "main"}],
+    }
+    saved = {}
+    monkeypatch.setattr(sp, "load_all_session_data", lambda: [("sess-parked", parked)], raising=True)
+    monkeypatch.setattr(sp, "save_session", lambda sid, data: saved.update({sid: data}), raising=True)
+
+    mgr = AgentManager()
+    asyncio.run(mgr.reconcile_on_startup())
+    assert "sess-parked" in mgr.crash_resume_queue, "a parked turn is an owed turn"
+    assert saved["sess-parked"]["awaiting_reconnect"] is False, "the flag is consumed, not left to re-fire"
+
+
+def test_the_breaker_stops_a_task_that_keeps_killing_the_app(monkeypatch):
+    """If the work itself is what takes the process down, a second boot must hand it to the manual
+    chip rather than launching it again."""
+    import backend.apps.agents.manager.session.SessionPersistence as sp
+    from backend.apps.agents.agent_manager import AgentManager
+
+    parked = {
+        "id": "sess-bad", "status": "completed", "awaiting_reconnect": True,
+        "closed_at": None, "active_branch_id": "main", "crash_interrupt_count": 1,
+        "messages": [{"role": "user", "content": "go", "branch_id": "main"}],
+    }
+    monkeypatch.setattr(sp, "load_all_session_data", lambda: [("sess-bad", parked)], raising=True)
+    monkeypatch.setattr(sp, "save_session", lambda sid, data: None, raising=True)
+
+    mgr = AgentManager()
+    asyncio.run(mgr.reconcile_on_startup())
+    assert mgr.crash_resume_queue == [], "second consecutive death: no third automatic run"
+
+
+def test_the_wait_ends_the_moment_the_provider_answers(monkeypatch):
+    """The backoff is a CEILING, not a sentence. A blind sleep would leave someone watching a
+    spinner for fourteen more minutes after their wifi came back, which is worse than the button it
+    replaced (Eric, 2026-08-20)."""
+    import backend.apps.agents.manager.run.reconnect_resume as rr
+
+    calls = {"probes": 0, "slept": 0.0}
+
+    async def fake_sleep(secs, *a, **k):
+        calls["slept"] += secs
+
+    async def reachable_on_third_look(host):
+        calls["probes"] += 1
+        return calls["probes"] >= 3
+
+    monkeypatch.setattr(rr.asyncio, "sleep", fake_sleep, raising=False)
+    monkeypatch.setattr(rr, "provider_reachable", reachable_on_third_look, raising=True)
+
+    s = p_session()
+    asyncio.run(rr.wait_for_reconnect(s, 900))
+    assert calls["probes"] == 3, "it stops looking as soon as the answer is yes"
+    assert calls["slept"] == 3 * rr.RECONNECT_POLL_S, "it waited 9s of a 900s ceiling"
+
+
+def test_an_outage_that_never_heals_still_honours_the_ceiling(monkeypatch):
+    """Negative control: if nothing ever answers, the wait must END at the ceiling and try anyway,
+    not poll forever."""
+    import backend.apps.agents.manager.run.reconnect_resume as rr
+
+    slept = {"total": 0.0}
+
+    async def fake_sleep(secs, *a, **k):
+        slept["total"] += secs
+
+    async def never(host):
+        return False
+
+    monkeypatch.setattr(rr.asyncio, "sleep", fake_sleep, raising=False)
+    monkeypatch.setattr(rr, "provider_reachable", never, raising=True)
+
+    asyncio.run(rr.wait_for_reconnect(p_session(), 60))
+    assert slept["total"] == 60, "the ceiling is honoured exactly, not overshot"
+
+
+def test_the_probe_targets_the_provider_not_our_own_localhost_router():
+    """A router lane points the CLI at localhost, which answers happily while the machine is
+    offline: probing it would return the one wrong answer at the only moment it matters."""
+    import backend.apps.agents.manager.run.reconnect_resume as rr
+
+    s = p_session()
+    for model, expected in (
+        ("sonnet-5", "api.anthropic.com"),
+        ("gpt-5.6-terra", "api.openai.com"),
+        ("gemini-3-pro", "generativelanguage.googleapis.com"),
+    ):
+        s.model = model
+        host = rr.provider_probe_host(s)
+        assert host == expected, f"{model} -> {host}"
+        assert "localhost" not in host and "127.0.0.1" not in host
+
+
+def test_a_dead_socket_respawns_the_cli_but_a_429_does_not(monkeypatch):
+    """Both arrive as 'transient', and they want different recoveries. A dead transport leaves the
+    CLI holding a corpse, so the retry needs a fresh one. A 429 is a healthy pipe carrying a NO, and
+    respawning for that spends a whole process to be told the same thing (caught by the existing
+    test_rate_limit_does_not_respawn_the_cli when this shipped ungated)."""
+    dead, _ = p_drive(monkeypatch, ConnectionError("Connection reset by peer"))
+    assert dead.needs_fresh_session is True
+    assert dead.awaiting_reconnect is True
+
+    throttled, _ = p_drive(monkeypatch, Exception("429 rate_limit_error: overloaded"))
+    assert throttled.needs_fresh_session is False, "a refusal is not a broken pipe"
+    assert throttled.awaiting_reconnect is True, "but it is still worth waiting out"