mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] WS resilience: per-session seq + ring buffer + resume protocol so agent runs survive transient disconnects (wifi flap, sleep, NAT
drop) instead of flipping to completed. Adds heartbeat (25s ping/10s pong), reconnect with infinite jittered backoff, outbound queue gated on resume_ack, gap_detected fallback for long offlines, on-disk persistence of terminal events for post-restart recovery, and a reconnecting connection state decoupled from session.status. 1089 backend tests covering 500 randomized disconnect scenarios + concurrent broadcast races. Backend/WS handler does not cancel agent task on disconnect
This commit is contained in:
@@ -0,0 +1,232 @@
|
||||
"""Per-session WS event sequencing, ring buffer, and terminal-event persistence.
|
||||
|
||||
Why this exists
|
||||
---------------
|
||||
WS sockets die for a thousand reasons that have nothing to do with the
|
||||
agent task: laptop sleep, captive portals, NAT idle timeout, VPN
|
||||
renegotiation. Without this module, a transient drop is fatal —
|
||||
mid-stream events are lost forever and the UI can't tell whether the
|
||||
run finished or merely went quiet.
|
||||
|
||||
Contract
|
||||
--------
|
||||
Every WS event for a session goes through `stamp(...)`, which is an
|
||||
async context manager that:
|
||||
1. Acquires the per-session lock.
|
||||
2. Bumps a monotonic `seq` integer.
|
||||
3. Appends the JSON payload to a bounded ring buffer.
|
||||
4. Yields (seq, payload_str) to the caller.
|
||||
5. Holds the lock until the caller exits the `async with` — meaning
|
||||
the caller's `ws.send_text(...)` happens *under the same lock*,
|
||||
guaranteeing wire order == seq order even when many coroutines
|
||||
broadcast concurrently.
|
||||
|
||||
Without (5), two coroutines can each get a unique seq under separate
|
||||
lock acquisitions, yet the higher-seq event can reach the wire first
|
||||
because asyncio scheduled its `send_text` earlier. That corrupts both
|
||||
wire order and the ring buffer on resume.
|
||||
|
||||
Resume protocol
|
||||
---------------
|
||||
On reconnect, the client sends `client:resume {connection_uuid,
|
||||
last_seq}`. The server:
|
||||
- Returns ring-buffer events with `seq > last_seq` if available.
|
||||
- Returns `agent:gap_detected` if `last_seq` is older than the
|
||||
oldest buffered seq — the client falls back to a REST refresh.
|
||||
- Returns the persisted terminal event (if any) when the session
|
||||
is no longer in memory at all (e.g. after a process restart).
|
||||
|
||||
Persistence
|
||||
-----------
|
||||
Terminal events (status: completed/stopped/error) are written
|
||||
atomically to disk so a client that comes back hours later — long
|
||||
after the in-memory ring buffer has been GC'd — still sees the right
|
||||
outcome instead of a spinner that never resolves. Persistence is
|
||||
opportunistic: an I/O error never blocks the broadcast path.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
from collections import deque
|
||||
from contextlib import asynccontextmanager
|
||||
from typing import AsyncIterator, Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# Ring buffer size per session. ~500 events comfortably covers a 30s
|
||||
# transient drop even in the busiest streams (thinking deltas at
|
||||
# ~20Hz). Memory is bounded: ~50KB per active session.
|
||||
BUFFER_LIMIT = 500
|
||||
|
||||
TERMINAL_STATUSES = {"completed", "stopped", "error"}
|
||||
|
||||
|
||||
class _SessionSeqLog:
|
||||
"""Per-session lock + monotonic seq + recent-event ring buffer."""
|
||||
|
||||
__slots__ = ("lock", "seq", "buffer")
|
||||
|
||||
def __init__(self) -> None:
|
||||
self.lock: asyncio.Lock = asyncio.Lock()
|
||||
self.seq: int = 0
|
||||
# Each entry: (seq, json_payload_str). Pre-serialized so a
|
||||
# replay doesn't redo json.dumps for every reconnect.
|
||||
self.buffer: deque[tuple[int, str]] = deque(maxlen=BUFFER_LIMIT)
|
||||
|
||||
|
||||
class SeqLogStore:
|
||||
"""Process-wide store. Per-session locks live inside `_SessionSeqLog`."""
|
||||
|
||||
def __init__(self, persist_dir: Optional[str] = None) -> None:
|
||||
self._per_session: dict[str, _SessionSeqLog] = {}
|
||||
# Coarse lock guarding only the dict's setdefault path. Held
|
||||
# for nanoseconds; never crosses an `await` past the `_get`.
|
||||
self._dict_lock = asyncio.Lock()
|
||||
self._persist_dir = persist_dir
|
||||
if persist_dir:
|
||||
try:
|
||||
os.makedirs(persist_dir, exist_ok=True)
|
||||
except Exception:
|
||||
logger.warning("seq_log: failed to create persist dir %s", persist_dir)
|
||||
|
||||
async def _get_or_create(self, session_id: str) -> _SessionSeqLog:
|
||||
log = self._per_session.get(session_id)
|
||||
if log is not None:
|
||||
return log
|
||||
async with self._dict_lock:
|
||||
log = self._per_session.get(session_id)
|
||||
if log is None:
|
||||
log = _SessionSeqLog()
|
||||
self._per_session[session_id] = log
|
||||
return log
|
||||
|
||||
def _peek(self, session_id: str) -> Optional[_SessionSeqLog]:
|
||||
return self._per_session.get(session_id)
|
||||
|
||||
@asynccontextmanager
|
||||
async def stamp(
|
||||
self, session_id: str, event: str, data: dict
|
||||
) -> AsyncIterator[tuple[int, str]]:
|
||||
"""Atomically assign a seq, buffer it, and yield (seq, payload).
|
||||
|
||||
Caller is expected to perform the actual `send_text` *inside*
|
||||
the `async with` block. The per-session lock is held for the
|
||||
entire body, so wire order is guaranteed equal to seq order
|
||||
no matter how many tasks broadcast concurrently.
|
||||
"""
|
||||
log = await self._get_or_create(session_id)
|
||||
async with log.lock:
|
||||
log.seq += 1
|
||||
seq = log.seq
|
||||
payload = {
|
||||
"event": event,
|
||||
"session_id": session_id,
|
||||
"data": data,
|
||||
"seq": seq,
|
||||
}
|
||||
payload_str = json.dumps(payload)
|
||||
log.buffer.append((seq, payload_str))
|
||||
yield seq, payload_str
|
||||
|
||||
def replay(
|
||||
self, session_id: str, last_seq: int
|
||||
) -> tuple[Optional[int], Optional[int], list[str]]:
|
||||
"""Return (oldest_buffered_seq, newest_buffered_seq, events).
|
||||
|
||||
Caller decides what to do with the result:
|
||||
- `events` empty AND newest_buffered_seq is None: no buffer
|
||||
for this session in memory. Fall back to persisted
|
||||
terminal event.
|
||||
- `last_seq` < `oldest_buffered_seq`: there's a gap. Send
|
||||
`agent:gap_detected`; the client REST-refreshes.
|
||||
- Otherwise `events` are the missed payloads in seq order.
|
||||
"""
|
||||
log = self._peek(session_id)
|
||||
if log is None:
|
||||
return (None, None, [])
|
||||
# Snapshot the deque under the lock-free fast path. asyncio is
|
||||
# single-threaded so a list() of a deque mutated by append is
|
||||
# safe; eviction (via maxlen) is also a single-step op. We
|
||||
# don't need to hold the per-session lock for a read.
|
||||
snapshot = list(log.buffer)
|
||||
if not snapshot:
|
||||
return (None, log.seq, [])
|
||||
oldest = snapshot[0][0]
|
||||
newest = snapshot[-1][0]
|
||||
events = [s for (i, s) in snapshot if i > last_seq]
|
||||
return (oldest, newest, events)
|
||||
|
||||
def current_seq(self, session_id: str) -> int:
|
||||
"""Last assigned seq, or 0 if no log exists for the session."""
|
||||
log = self._peek(session_id)
|
||||
return log.seq if log else 0
|
||||
|
||||
# ----- Terminal-event persistence -----
|
||||
|
||||
def _terminal_path(self, session_id: str) -> Optional[str]:
|
||||
if not self._persist_dir:
|
||||
return None
|
||||
# session ids are uuid4 hex in this codebase, but sanitize
|
||||
# against path traversal anyway.
|
||||
safe = "".join(c for c in session_id if c.isalnum() or c in ("-", "_"))
|
||||
if not safe:
|
||||
return None
|
||||
return os.path.join(self._persist_dir, f"{safe}.json")
|
||||
|
||||
def persist_terminal(self, session_id: str, payload_str: str) -> None:
|
||||
"""Atomic write of a terminal event for post-restart clients.
|
||||
|
||||
Best-effort: an I/O failure must never block the broadcast.
|
||||
"""
|
||||
path = self._terminal_path(session_id)
|
||||
if not path:
|
||||
return
|
||||
try:
|
||||
tmp = path + ".tmp"
|
||||
with open(tmp, "w", encoding="utf-8") as f:
|
||||
f.write(payload_str)
|
||||
os.replace(tmp, path)
|
||||
except Exception:
|
||||
logger.debug(
|
||||
"seq_log: failed to persist terminal event for %s", session_id, exc_info=True
|
||||
)
|
||||
|
||||
def load_terminal(self, session_id: str) -> Optional[str]:
|
||||
path = self._terminal_path(session_id)
|
||||
if not path or not os.path.exists(path):
|
||||
return None
|
||||
try:
|
||||
with open(path, "r", encoding="utf-8") as f:
|
||||
return f.read()
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def clear(self, session_id: str) -> None:
|
||||
"""Drop in-memory log + persisted terminal event.
|
||||
|
||||
Use on full session deletion. Closed-but-retained sessions
|
||||
keep their terminal file so late reconnects still resolve.
|
||||
"""
|
||||
self._per_session.pop(session_id, None)
|
||||
path = self._terminal_path(session_id)
|
||||
if path and os.path.exists(path):
|
||||
try:
|
||||
os.remove(path)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
|
||||
def _default_persist_dir() -> Optional[str]:
|
||||
try:
|
||||
from backend.config.paths import DATA_ROOT
|
||||
return os.path.join(DATA_ROOT, "agents", "terminal_events")
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
|
||||
# Process-wide singleton wired to the agents data dir.
|
||||
seq_log = SeqLogStore(persist_dir=_default_persist_dir())
|
||||
@@ -3,11 +3,26 @@ import json
|
||||
import logging
|
||||
from fastapi import WebSocket
|
||||
|
||||
from backend.apps.agents.seq_log import TERMINAL_STATUSES, seq_log
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
class ConnectionManager:
|
||||
"""Manages WebSocket connections and bridges HITL approval requests."""
|
||||
|
||||
"""Manages WebSocket connections and bridges HITL approval requests.
|
||||
|
||||
Every outbound event flows through the seq log so reconnecting
|
||||
clients can replay missed events. The send happens *under* the
|
||||
per-session lock yielded by `seq_log.stamp(...)`, which guarantees
|
||||
wire order matches seq order even under concurrent broadcasts.
|
||||
|
||||
A WS disconnect (`disconnect_session`) ONLY removes the socket
|
||||
from the connection registry. It does NOT cancel the underlying
|
||||
agent task. The task lives on `agent_manager.tasks`; only an
|
||||
explicit `agent:stop`, REST `/close`, natural completion, or
|
||||
process shutdown ends a run.
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
self.connections: dict[str, list[WebSocket]] = {}
|
||||
self.global_connections: list[WebSocket] = []
|
||||
@@ -38,23 +53,123 @@ class ConnectionManager:
|
||||
]
|
||||
|
||||
async def send_to_session(self, session_id: str, event: str, data: dict):
|
||||
"""Send a message to all connections watching a specific session."""
|
||||
payload = json.dumps({"event": event, "session_id": session_id, "data": data})
|
||||
for ws in self.connections.get(session_id, []):
|
||||
"""Broadcast a session event with monotonic sequencing.
|
||||
|
||||
The send to every socket happens inside the seq_log lock so a
|
||||
slow/dead WS doesn't reorder events on the fast ones. If a
|
||||
single send raises (broken pipe, half-open socket), we log and
|
||||
continue — the ring buffer still has the event so the client
|
||||
will replay it on reconnect.
|
||||
|
||||
For terminal status events (completed/stopped/error) we also
|
||||
atomically persist the payload to disk; a client that returns
|
||||
after a process restart can then resolve the spinner via
|
||||
`seq_log.load_terminal(...)` instead of being stuck.
|
||||
"""
|
||||
async with seq_log.stamp(session_id, event, data) as (seq, payload_str):
|
||||
for ws in list(self.connections.get(session_id, [])):
|
||||
try:
|
||||
await ws.send_text(payload_str)
|
||||
except Exception:
|
||||
logger.debug("send_to_session: send failed (will retry on reconnect)", exc_info=True)
|
||||
for ws in list(self.global_connections):
|
||||
try:
|
||||
await ws.send_text(payload_str)
|
||||
except Exception:
|
||||
logger.debug("send_to_session: global send failed", exc_info=True)
|
||||
# Persist terminal events under the lock so a concurrent
|
||||
# `agent:status: running` can't race past and overwrite
|
||||
# the disk file with a stale state.
|
||||
if event == "agent:status" and data.get("status") in TERMINAL_STATUSES:
|
||||
seq_log.persist_terminal(session_id, payload_str)
|
||||
|
||||
async def replay_to(
|
||||
self, session_id: str, websocket: WebSocket, last_seq: int
|
||||
) -> dict:
|
||||
"""Replay buffered events with seq > last_seq to one socket.
|
||||
|
||||
Returns a small ack envelope describing what happened so the
|
||||
caller (the WS handler) can send a `server:resume_ack` frame.
|
||||
|
||||
Three cases:
|
||||
1. `events` non-empty: replay them in order; ack carries
|
||||
`from_seq`, `to_seq`.
|
||||
2. No buffer at all (process restarted, session evicted)
|
||||
but a persisted terminal exists: send it; ack signals
|
||||
`terminal_only=True`.
|
||||
3. `last_seq` predates the oldest buffered seq: emit
|
||||
`agent:gap_detected`; client REST-refreshes the session.
|
||||
"""
|
||||
oldest, newest, events = seq_log.replay(session_id, last_seq)
|
||||
|
||||
# Check for gap FIRST. If the client's last_seq is below the
|
||||
# buffer's oldest seq, we can't deliver everything they
|
||||
# missed — silently replaying only the in-buffer tail would
|
||||
# leave a hole in their state. Tell them to REST-refresh
|
||||
# instead, even if the tail looks safe to send.
|
||||
# Treat last_seq=0 as "fresh client" — they want a full
|
||||
# replay of whatever's in the buffer, not a gap signal.
|
||||
if last_seq > 0 and oldest is not None and last_seq < oldest - 1:
|
||||
gap_payload = json.dumps({
|
||||
"event": "agent:gap_detected",
|
||||
"session_id": session_id,
|
||||
"data": {
|
||||
"session_id": session_id,
|
||||
"oldest_seq": oldest,
|
||||
"newest_seq": newest,
|
||||
"client_seq": last_seq,
|
||||
},
|
||||
})
|
||||
try:
|
||||
await ws.send_text(payload)
|
||||
await websocket.send_text(gap_payload)
|
||||
except Exception:
|
||||
pass
|
||||
for ws in self.global_connections:
|
||||
return {
|
||||
"ok": False,
|
||||
"reason": "gap",
|
||||
"oldest_seq": oldest,
|
||||
"newest_seq": newest,
|
||||
}
|
||||
|
||||
if events:
|
||||
for s in events:
|
||||
try:
|
||||
await websocket.send_text(s)
|
||||
except Exception:
|
||||
logger.debug("replay_to: send failed", exc_info=True)
|
||||
break
|
||||
return {
|
||||
"ok": True,
|
||||
"replayed": len(events),
|
||||
"from_seq": last_seq,
|
||||
"to_seq": newest,
|
||||
}
|
||||
|
||||
# Nothing in memory. Try a persisted terminal event.
|
||||
terminal = seq_log.load_terminal(session_id)
|
||||
if terminal is not None:
|
||||
try:
|
||||
await ws.send_text(payload)
|
||||
await websocket.send_text(terminal)
|
||||
except Exception:
|
||||
pass
|
||||
return {"ok": True, "replayed": 1, "terminal_only": True}
|
||||
|
||||
# Nothing missed, nothing to replay. Caller's caught up.
|
||||
return {
|
||||
"ok": True,
|
||||
"replayed": 0,
|
||||
"current_seq": newest if newest is not None else 0,
|
||||
}
|
||||
|
||||
async def broadcast_global(self, event: str, data: dict):
|
||||
"""Send a message to all global (dashboard) connections."""
|
||||
"""Send a message to all global (dashboard) connections.
|
||||
|
||||
Dashboard-scoped events don't go through the per-session seq
|
||||
log — they're not session-bound and the dashboard WS has its
|
||||
own resume story (full state refetch on reconnect).
|
||||
"""
|
||||
payload = json.dumps({"event": event, "data": data})
|
||||
for ws in self.global_connections:
|
||||
for ws in list(self.global_connections):
|
||||
try:
|
||||
await ws.send_text(payload)
|
||||
except Exception:
|
||||
@@ -65,17 +180,20 @@ class ConnectionManager:
|
||||
timeout: float = 600.0,
|
||||
) -> dict:
|
||||
"""Send an approval request and wait for the user's response.
|
||||
Returns the approval decision dict. Times out after *timeout* seconds
|
||||
(default 10 minutes) to prevent permanently stuck agents."""
|
||||
|
||||
Returns the approval decision dict. Times out after `timeout`
|
||||
seconds (default 10 minutes) so a forgotten request doesn't
|
||||
permanently park the agent.
|
||||
"""
|
||||
future = asyncio.get_event_loop().create_future()
|
||||
self.pending_futures[request_id] = future
|
||||
|
||||
|
||||
await self.send_to_session(session_id, "agent:approval_request", {
|
||||
"request_id": request_id,
|
||||
"tool_name": tool_name,
|
||||
"tool_input": tool_input,
|
||||
})
|
||||
|
||||
|
||||
try:
|
||||
result = await asyncio.wait_for(future, timeout=timeout)
|
||||
return result
|
||||
@@ -123,4 +241,5 @@ class ConnectionManager:
|
||||
if future and not future.done():
|
||||
future.set_result(result)
|
||||
|
||||
|
||||
ws_manager = ConnectionManager()
|
||||
|
||||
+50
-2
@@ -133,6 +133,22 @@ async def _auth_middleware(request: Request, call_next):
|
||||
|
||||
@app.websocket("/ws/agents/{session_id}")
|
||||
async def websocket_session(websocket: WebSocket, session_id: str):
|
||||
"""Per-session WS endpoint with resume + heartbeat.
|
||||
|
||||
Resilience contract (see backend/apps/agents/seq_log.py):
|
||||
- Every server→client event carries a monotonic `seq` per session.
|
||||
- On (re)connect the client sends `client:hello` with its
|
||||
last-seen seq; the server replays missed events (or emits
|
||||
`agent:gap_detected` if the gap is too large) and answers
|
||||
with `server:hello` carrying the current high-water seq.
|
||||
- `client:ping` → `server:pong` heartbeat (default 25s) so
|
||||
silent socket deaths (NAT idle drop, laptop sleep) are
|
||||
detected without waiting for the next outbound frame.
|
||||
- `WebSocketDisconnect` only removes the socket from the
|
||||
connection registry. The agent task keeps running. The only
|
||||
things that end a run are: natural completion, explicit
|
||||
`agent:stop`, REST `/close`, or process shutdown.
|
||||
"""
|
||||
if not _ws_auth_ok(websocket):
|
||||
return
|
||||
await ws_manager.connect_session(session_id, websocket)
|
||||
@@ -142,8 +158,38 @@ async def websocket_session(websocket: WebSocket, session_id: str):
|
||||
msg = json.loads(data)
|
||||
event = msg.get("event")
|
||||
payload = msg.get("data", {})
|
||||
|
||||
if event == "agent:send_message":
|
||||
|
||||
if event == "client:hello":
|
||||
# Resume handshake. The client sends this immediately
|
||||
# after the WS opens, with `last_seq` = the highest
|
||||
# seq it has applied. We replay anything newer; on
|
||||
# first connect last_seq=0 and replay() correctly
|
||||
# returns nothing (empty buffer) or the persisted
|
||||
# terminal event for already-finished sessions.
|
||||
last_seq = int(payload.get("last_seq") or 0)
|
||||
connection_uuid = payload.get("connection_uuid") or ""
|
||||
ack = await ws_manager.replay_to(session_id, websocket, last_seq)
|
||||
from backend.apps.agents.seq_log import seq_log as _sl
|
||||
await websocket.send_text(json.dumps({
|
||||
"event": "server:hello",
|
||||
"session_id": session_id,
|
||||
"data": {
|
||||
"connection_uuid": connection_uuid,
|
||||
"current_seq": _sl.current_seq(session_id),
|
||||
"ack": ack,
|
||||
},
|
||||
}))
|
||||
elif event == "client:ping":
|
||||
# Heartbeat. Cheap, keeps NATs/firewalls from
|
||||
# silently dropping the connection. Carry the
|
||||
# client's nonce back so it can match pong→ping for
|
||||
# round-trip latency tracking if it wants.
|
||||
await websocket.send_text(json.dumps({
|
||||
"event": "server:pong",
|
||||
"session_id": session_id,
|
||||
"data": {"nonce": payload.get("nonce")},
|
||||
}))
|
||||
elif event == "agent:send_message":
|
||||
from backend.apps.agents.agent_manager import agent_manager
|
||||
await agent_manager.send_message(
|
||||
session_id,
|
||||
@@ -171,6 +217,8 @@ async def websocket_session(websocket: WebSocket, session_id: str):
|
||||
from backend.apps.agents.agent_manager import agent_manager
|
||||
await agent_manager.stop_agent(session_id)
|
||||
except WebSocketDisconnect:
|
||||
# Drops the socket from the connection list. Does NOT cancel
|
||||
# the agent task — that's intentional. See module docstring.
|
||||
ws_manager.disconnect_session(session_id, websocket)
|
||||
|
||||
def _ws_auth_ok(websocket: WebSocket) -> bool:
|
||||
|
||||
@@ -0,0 +1,543 @@
|
||||
"""Stress test for the WS resilience layer.
|
||||
|
||||
Goal: actively try to break the agent run with disconnects, drops,
|
||||
reconnects, and concurrent broadcasts. The visible bug we're chasing
|
||||
is "Network issue" toasts that flip a still-running task to a
|
||||
terminal state. After these fixes the contract should be:
|
||||
|
||||
1. The agent task NEVER dies because of a WS drop.
|
||||
2. Every event the server emits is replayable, in order, with no
|
||||
duplicates and no gaps, after any number of disconnects.
|
||||
3. Terminal events (completed/stopped/error) are always observable
|
||||
by a client that reconnects later — even if the only persistence
|
||||
of the event is the on-disk terminal log.
|
||||
4. Concurrent broadcasts (thinking deltas + tool calls + status
|
||||
changes from many tasks) preserve seq order == wire order.
|
||||
5. A client that's been gone too long for the ring buffer gets a
|
||||
`agent:gap_detected` instead of silent loss.
|
||||
|
||||
We don't run real Claude Code; we install a stub `agent_loop` that
|
||||
emits the same WS event shapes a real run would (status, stream,
|
||||
message, completed). That keeps the test fast (>200 iterations in
|
||||
seconds) and self-contained.
|
||||
|
||||
Run:
|
||||
cd backend && .venv/bin/python -m pytest tests/test_disconnect_resilience.py -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import random
|
||||
import sys
|
||||
import tempfile
|
||||
from typing import Any
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Boot env: route data to a tempdir BEFORE importing backend modules so
|
||||
# the persistence dir for terminal events lives under our control.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
_TMPROOT = tempfile.mkdtemp(prefix="openswarm-disconnect-test-")
|
||||
os.environ.setdefault("OPENSWARM_DATA_DIR", _TMPROOT)
|
||||
|
||||
# Push the seq_log persist dir to a deterministic location too.
|
||||
_SEQ_DIR = os.path.join(_TMPROOT, "seq_terminals")
|
||||
os.makedirs(_SEQ_DIR, exist_ok=True)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _patch_persist_dir():
|
||||
"""Force the seq_log to use our tmp dir so we can assert on disk state."""
|
||||
from backend.apps.agents import seq_log as sl_mod
|
||||
|
||||
# Rebuild the singleton with our test dir.
|
||||
new_store = sl_mod.SeqLogStore(persist_dir=_SEQ_DIR)
|
||||
monkey = patch.object(sl_mod, "seq_log", new_store)
|
||||
monkey.start()
|
||||
# Also patch the symbol re-exported into ws_manager's import scope.
|
||||
from backend.apps.agents import ws_manager as wm_mod
|
||||
wm_monkey = patch.object(wm_mod, "seq_log", new_store)
|
||||
wm_monkey.start()
|
||||
yield new_store
|
||||
monkey.stop()
|
||||
wm_monkey.stop()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Minimal FastAPI app with the real WS endpoint logic. We import
|
||||
# ws_manager directly and replicate the handler from backend/main.py
|
||||
# without any of its auth middleware so the TestClient can connect
|
||||
# without a token.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def _build_app(seq_log):
|
||||
"""Replicates main.py's WS handler + adds a /test/emit endpoint
|
||||
so the test thread can drive event emission through the same
|
||||
event loop as the WS handler — avoiding the cross-loop hazards
|
||||
of `asyncio.run()` mid-test."""
|
||||
from backend.apps.agents.ws_manager import ws_manager
|
||||
|
||||
app = FastAPI()
|
||||
|
||||
@app.websocket("/ws/agents/{session_id}")
|
||||
async def ws_session(websocket: WebSocket, session_id: str):
|
||||
await ws_manager.connect_session(session_id, websocket)
|
||||
try:
|
||||
while True:
|
||||
data = await websocket.receive_text()
|
||||
msg = json.loads(data)
|
||||
event = msg.get("event")
|
||||
payload = msg.get("data", {})
|
||||
if event == "client:hello":
|
||||
last_seq = int(payload.get("last_seq") or 0)
|
||||
ack = await ws_manager.replay_to(session_id, websocket, last_seq)
|
||||
await websocket.send_text(json.dumps({
|
||||
"event": "server:hello",
|
||||
"session_id": session_id,
|
||||
"data": {
|
||||
"connection_uuid": payload.get("connection_uuid", ""),
|
||||
"current_seq": seq_log.current_seq(session_id),
|
||||
"ack": ack,
|
||||
},
|
||||
}))
|
||||
elif event == "client:ping":
|
||||
await websocket.send_text(json.dumps({
|
||||
"event": "server:pong",
|
||||
"session_id": session_id,
|
||||
"data": {"nonce": payload.get("nonce")},
|
||||
}))
|
||||
except WebSocketDisconnect:
|
||||
ws_manager.disconnect_session(session_id, websocket)
|
||||
|
||||
@app.post("/test/emit/{session_id}")
|
||||
async def emit_events(session_id: str, body: dict):
|
||||
n = int(body.get("n", 0))
|
||||
terminate = body.get("terminate") # str or None
|
||||
concurrent = int(body.get("concurrent", 1))
|
||||
await _emit_run(session_id, n, terminate=terminate, concurrent_tasks=concurrent)
|
||||
return {"ok": True, "current_seq": seq_log.current_seq(session_id)}
|
||||
|
||||
return app
|
||||
|
||||
|
||||
def _emit(client, session_id: str, n: int, terminate: str | None = None, concurrent: int = 1):
|
||||
"""Drive event emission via the test-only HTTP endpoint."""
|
||||
r = client.post(f"/test/emit/{session_id}", json={
|
||||
"n": n, "terminate": terminate, "concurrent": concurrent,
|
||||
})
|
||||
assert r.status_code == 200, r.text
|
||||
return r.json()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
async def _emit_run(session_id: str, n_events: int, terminate: str | None = "completed", concurrent_tasks: int = 1):
|
||||
"""Emit a synthetic agent run.
|
||||
|
||||
`concurrent_tasks` lets the test stress the per-session lock by
|
||||
fanning out the broadcast across multiple coroutines. The seq
|
||||
log must still order them strictly.
|
||||
"""
|
||||
from backend.apps.agents.ws_manager import ws_manager
|
||||
|
||||
async def emit_chunk(start: int, count: int):
|
||||
for i in range(count):
|
||||
await ws_manager.send_to_session(session_id, "agent:stream_delta", {
|
||||
"session_id": session_id,
|
||||
"message_id": "m1",
|
||||
"delta": f"chunk-{start + i}",
|
||||
})
|
||||
# Yield to the scheduler so other coroutines interleave —
|
||||
# this is what surfaces the seq race if locking is wrong.
|
||||
await asyncio.sleep(0)
|
||||
|
||||
if concurrent_tasks <= 1:
|
||||
await emit_chunk(0, n_events)
|
||||
else:
|
||||
per = n_events // concurrent_tasks
|
||||
tasks = [
|
||||
asyncio.create_task(emit_chunk(i * per, per))
|
||||
for i in range(concurrent_tasks)
|
||||
]
|
||||
await asyncio.gather(*tasks)
|
||||
# Mop up the remainder so total event count is exact.
|
||||
rem = n_events - per * concurrent_tasks
|
||||
if rem > 0:
|
||||
await emit_chunk(per * concurrent_tasks, rem)
|
||||
|
||||
if terminate is not None:
|
||||
await ws_manager.send_to_session(session_id, "agent:status", {
|
||||
"session_id": session_id,
|
||||
"status": terminate,
|
||||
})
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Unit-level: seq log fundamentals
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_seq_monotonic_under_concurrency(_patch_persist_dir):
|
||||
"""200 concurrent broadcasts must yield strictly monotonic seq."""
|
||||
app = _build_app(_patch_persist_dir)
|
||||
sid = "session-conc-1"
|
||||
with TestClient(app) as client:
|
||||
_emit(client, sid, n=200, terminate=None, concurrent=4)
|
||||
_, newest, events = _patch_persist_dir.replay(sid, 0)
|
||||
assert newest == 200
|
||||
seqs = [json.loads(s)["seq"] for s in events]
|
||||
assert seqs == sorted(seqs)
|
||||
assert len(set(seqs)) == len(seqs)
|
||||
|
||||
|
||||
def test_terminal_event_persisted(_patch_persist_dir):
|
||||
app = _build_app(_patch_persist_dir)
|
||||
sid = "session-term-1"
|
||||
with TestClient(app) as client:
|
||||
_emit(client, sid, n=0, terminate="completed")
|
||||
raw = _patch_persist_dir.load_terminal(sid)
|
||||
assert raw is not None
|
||||
obj = json.loads(raw)
|
||||
assert obj["event"] == "agent:status"
|
||||
assert obj["data"]["status"] == "completed"
|
||||
|
||||
|
||||
def test_replay_after_eviction_reports_gap(_patch_persist_dir):
|
||||
app = _build_app(_patch_persist_dir)
|
||||
sid = "session-evict-1"
|
||||
with TestClient(app) as client:
|
||||
_emit(client, sid, n=700, terminate=None)
|
||||
oldest, newest, events = _patch_persist_dir.replay(sid, last_seq=10)
|
||||
assert newest == 700
|
||||
assert oldest is not None and oldest > 10
|
||||
# Replay only includes seqs > 10 that survived eviction.
|
||||
assert all(json.loads(s)["seq"] > 10 for s in events)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Integration: full WS connect / disconnect / resume cycle
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_resume_after_disconnect_recovers_all_events(_patch_persist_dir):
|
||||
"""Simulate a single disconnect mid-run, then a clean resume."""
|
||||
app = _build_app(_patch_persist_dir)
|
||||
|
||||
sid = "session-res-1"
|
||||
received: list[dict] = []
|
||||
|
||||
with TestClient(app) as client:
|
||||
# Phase 1: connect, hello, see N events, then close abruptly.
|
||||
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
|
||||
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 0, "connection_uuid": "c1"}}))
|
||||
hello = json.loads(ws.receive_text())
|
||||
assert hello["event"] == "server:hello"
|
||||
# Inject a few events via the /test/emit endpoint.
|
||||
_emit(client, sid, n=10, terminate=None)
|
||||
for _ in range(10):
|
||||
received.append(json.loads(ws.receive_text()))
|
||||
assert len(received) == 10
|
||||
assert received[-1]["seq"] == 10
|
||||
|
||||
# Phase 2: between connections, the server keeps emitting. The
|
||||
# agent task is alive; only the WS is gone.
|
||||
_emit(client, sid, n=10, terminate="completed")
|
||||
|
||||
# Phase 3: reconnect with last_seq=10, expect replay of seq 11..21
|
||||
# (10 deltas + 1 status), then the server:hello ack.
|
||||
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
|
||||
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 10, "connection_uuid": "c2"}}))
|
||||
replay: list[dict] = []
|
||||
while True:
|
||||
msg = json.loads(ws.receive_text())
|
||||
if msg["event"] == "server:hello":
|
||||
break
|
||||
replay.append(msg)
|
||||
seqs = [m["seq"] for m in replay]
|
||||
assert seqs == list(range(11, 22)), f"unexpected replay seqs: {seqs}"
|
||||
statuses = [m for m in replay if m["event"] == "agent:status"]
|
||||
assert len(statuses) == 1
|
||||
assert statuses[0]["data"]["status"] == "completed"
|
||||
|
||||
|
||||
def test_terminal_event_visible_after_full_eviction(_patch_persist_dir):
|
||||
"""If the in-memory log is wiped (process restart simulation),
|
||||
a reconnecting client should still see the terminal event from
|
||||
disk — never a phantom 'running' spinner."""
|
||||
app = _build_app(_patch_persist_dir)
|
||||
sid = "session-evict-term-1"
|
||||
|
||||
seq_log = _patch_persist_dir
|
||||
|
||||
with TestClient(app) as client:
|
||||
_emit(client, sid, n=5, terminate="completed")
|
||||
|
||||
# Simulate a process restart: clear the in-memory ring buffer
|
||||
# but keep the persisted terminal file.
|
||||
seq_log._per_session.pop(sid, None)
|
||||
|
||||
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
|
||||
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 0, "connection_uuid": "c1"}}))
|
||||
received = []
|
||||
while True:
|
||||
msg = json.loads(ws.receive_text())
|
||||
if msg["event"] == "server:hello":
|
||||
received.append(msg)
|
||||
break
|
||||
received.append(msg)
|
||||
terminals = [m for m in received if m["event"] == "agent:status"]
|
||||
assert len(terminals) == 1
|
||||
assert terminals[0]["data"]["status"] == "completed"
|
||||
|
||||
|
||||
def test_gap_detected_when_buffer_evicted(_patch_persist_dir):
|
||||
"""A client whose lastSeq is older than the oldest buffered seq
|
||||
should receive `agent:gap_detected` so it can REST-refresh,
|
||||
rather than silently miss events."""
|
||||
app = _build_app(_patch_persist_dir)
|
||||
sid = "session-gap-1"
|
||||
|
||||
with TestClient(app) as client:
|
||||
# Fill the buffer past its limit so seq 1..200 are evicted.
|
||||
_emit(client, sid, n=700, terminate=None)
|
||||
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
|
||||
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 5, "connection_uuid": "c1"}}))
|
||||
saw_gap = False
|
||||
saw_hello = False
|
||||
while not saw_hello:
|
||||
msg = json.loads(ws.receive_text())
|
||||
if msg["event"] == "agent:gap_detected":
|
||||
saw_gap = True
|
||||
elif msg["event"] == "server:hello":
|
||||
saw_hello = True
|
||||
assert msg["data"]["ack"]["ok"] is False
|
||||
assert msg["data"]["ack"]["reason"] == "gap"
|
||||
assert saw_gap
|
||||
|
||||
|
||||
def test_ping_pong_round_trip(_patch_persist_dir):
|
||||
app = _build_app(_patch_persist_dir)
|
||||
sid = "session-ping-1"
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
|
||||
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 0, "connection_uuid": "c1"}}))
|
||||
assert json.loads(ws.receive_text())["event"] == "server:hello"
|
||||
ws.send_text(json.dumps({"event": "client:ping", "data": {"nonce": "abc"}}))
|
||||
pong = json.loads(ws.receive_text())
|
||||
assert pong["event"] == "server:pong"
|
||||
assert pong["data"]["nonce"] == "abc"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# The big one: hundreds of randomized disconnect scenarios.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
N_STRESS_ITERATIONS = int(os.environ.get("DISCONNECT_STRESS_N", "500"))
|
||||
|
||||
|
||||
@pytest.mark.parametrize("iteration", range(N_STRESS_ITERATIONS))
|
||||
def test_stress_random_disconnect(iteration, _patch_persist_dir):
|
||||
"""Each iteration: a random number of events, a random number of
|
||||
disconnects at random points, optionally ending in a terminal
|
||||
status. After all reconnects, the client must have observed
|
||||
every event exactly once, in seq order, and the terminal event
|
||||
if one was emitted."""
|
||||
rng = random.Random(iteration) # deterministic per iteration
|
||||
app = _build_app(_patch_persist_dir)
|
||||
|
||||
sid = f"session-stress-{iteration}"
|
||||
total_events = rng.randint(5, 80)
|
||||
n_disconnects = rng.randint(1, min(5, total_events // 2 or 1))
|
||||
will_terminate = rng.random() < 0.7 # 70% of runs reach a terminal
|
||||
|
||||
# Disconnect points: each is a count of events emitted *before*
|
||||
# the WS drops. We deliberately exclude `total_events` itself
|
||||
# so the breakpoint list never collides with the appended final
|
||||
# iteration (which is when the optional terminal status fires).
|
||||
if total_events > 1:
|
||||
breakpoints = sorted(rng.sample(range(1, total_events), min(n_disconnects, total_events - 1)))
|
||||
else:
|
||||
breakpoints = []
|
||||
seen: dict[int, dict] = {} # seq -> event payload
|
||||
last_seq = 0
|
||||
|
||||
with TestClient(app) as client:
|
||||
emitted_so_far = 0
|
||||
for bp in breakpoints + [total_events]:
|
||||
# Open a fresh socket, hello with our last_seq.
|
||||
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
|
||||
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": last_seq, "connection_uuid": f"c-{rng.random()}"}}))
|
||||
|
||||
# Drain until server:hello, recording any replayed events.
|
||||
while True:
|
||||
msg = json.loads(ws.receive_text())
|
||||
if msg["event"] == "server:hello":
|
||||
break
|
||||
if "seq" in msg:
|
||||
seen[msg["seq"]] = msg
|
||||
last_seq = max(last_seq, msg["seq"])
|
||||
|
||||
to_emit = bp - emitted_so_far
|
||||
emitted_so_far = bp
|
||||
terminate = "completed" if (bp == total_events and will_terminate) else None
|
||||
|
||||
# Drive the emit through the test app's HTTP endpoint so
|
||||
# the broadcast happens on the same event loop as the WS
|
||||
# handler. Using asyncio.run() here would create an
|
||||
# isolated loop and re-bind the per-session asyncio.Lock
|
||||
# to a different loop, which is hostile to anyio's
|
||||
# blocking-portal pattern.
|
||||
_emit(client, sid, n=to_emit, terminate=terminate)
|
||||
|
||||
expected = to_emit + (1 if terminate else 0)
|
||||
for _ in range(expected):
|
||||
msg = json.loads(ws.receive_text())
|
||||
seen[msg["seq"]] = msg
|
||||
last_seq = max(last_seq, msg["seq"])
|
||||
|
||||
# Closing the with-block disconnects the WS. The loop
|
||||
# opens a fresh socket on the next iteration.
|
||||
|
||||
# ----- Assertions: completeness, ordering, no dups, terminal -----
|
||||
expected_total = total_events + (1 if will_terminate else 0)
|
||||
assert len(seen) == expected_total, f"missing events: expected {expected_total}, got {len(seen)}"
|
||||
seqs = sorted(seen.keys())
|
||||
assert seqs == list(range(1, expected_total + 1)), f"non-contiguous seqs: {seqs[:5]}...{seqs[-5:]}"
|
||||
if will_terminate:
|
||||
last = seen[expected_total]
|
||||
assert last["event"] == "agent:status"
|
||||
assert last["data"]["status"] == "completed"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Concurrent broadcast: many fan-out coroutines must preserve seq order
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("trial", range(30))
|
||||
def test_concurrent_broadcast_preserves_order(trial, _patch_persist_dir):
|
||||
"""8 coroutines fanning out 400 events under the per-session lock.
|
||||
Drives the emit through the TestClient's portal so we use the
|
||||
real event loop the rest of the WS layer runs on."""
|
||||
app = _build_app(_patch_persist_dir)
|
||||
sid = f"session-conc-{trial}"
|
||||
|
||||
with TestClient(app) as client:
|
||||
_emit(client, sid, n=400, terminate="completed", concurrent=8)
|
||||
|
||||
oldest, newest, events = _patch_persist_dir.replay(sid, last_seq=0)
|
||||
assert newest == 401 # 400 deltas + 1 status
|
||||
seqs = [json.loads(s)["seq"] for s in events]
|
||||
assert seqs == sorted(seqs)
|
||||
# Each seq appears exactly once in the buffer.
|
||||
assert len(seqs) == len(set(seqs))
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Auth/security smoke: the WS endpoint here is unauth'd by design (test
|
||||
# scaffolding) — but main.py's _ws_auth_ok must remain in place. This
|
||||
# test pins that contract so a future refactor can't accidentally
|
||||
# strip it.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Extra stress: terminate happens INSIDE a disconnect window. The
|
||||
# client must see the terminal event on its next reconnect (whether
|
||||
# from ring buffer or persisted disk record).
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.parametrize("trial", range(50))
|
||||
def test_terminate_during_disconnect_is_observable(trial, _patch_persist_dir):
|
||||
rng = random.Random(1000 + trial)
|
||||
app = _build_app(_patch_persist_dir)
|
||||
sid = f"session-mid-term-{trial}"
|
||||
n_pre = rng.randint(0, 40)
|
||||
n_post = rng.randint(0, 40)
|
||||
seen: dict[int, dict] = {}
|
||||
last_seq = 0
|
||||
|
||||
with TestClient(app) as client:
|
||||
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
|
||||
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 0, "connection_uuid": "c1"}}))
|
||||
assert json.loads(ws.receive_text())["event"] == "server:hello"
|
||||
if n_pre:
|
||||
_emit(client, sid, n=n_pre, terminate=None)
|
||||
for _ in range(n_pre):
|
||||
msg = json.loads(ws.receive_text())
|
||||
seen[msg["seq"]] = msg
|
||||
last_seq = max(last_seq, msg["seq"])
|
||||
# Disconnected. Emit the rest + terminate while WS is gone.
|
||||
_emit(client, sid, n=n_post, terminate="completed")
|
||||
# Reconnect. We expect to receive everything from last_seq+1
|
||||
# through to the terminal — possibly via disk if the buffer
|
||||
# rolled (it won't here; numbers are small).
|
||||
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
|
||||
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": last_seq, "connection_uuid": "c2"}}))
|
||||
while True:
|
||||
msg = json.loads(ws.receive_text())
|
||||
if msg["event"] == "server:hello":
|
||||
break
|
||||
if "seq" in msg:
|
||||
seen[msg["seq"]] = msg
|
||||
|
||||
expected = n_pre + n_post + 1
|
||||
assert len(seen) == expected
|
||||
seqs = sorted(seen.keys())
|
||||
assert seqs == list(range(1, expected + 1))
|
||||
last = seen[expected]
|
||||
assert last["event"] == "agent:status"
|
||||
assert last["data"]["status"] == "completed"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Sanity: an explicit `WebSocketDisconnect` MUST NOT cancel the
|
||||
# underlying agent task. We don't have a real agent here, but we can
|
||||
# at least assert that the ws_manager's disconnect path doesn't touch
|
||||
# any task registry.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_disconnect_does_not_touch_agent_task(_patch_persist_dir):
|
||||
"""If a future refactor adds task cancellation to disconnect_session,
|
||||
this test will catch it. We import agent_manager lazily so the
|
||||
`tasks` dict starts empty; we register a sentinel task and confirm
|
||||
disconnect_session doesn't poke it."""
|
||||
from backend.apps.agents.ws_manager import ws_manager
|
||||
# Insert a real Future into a parallel registry to mimic
|
||||
# `agent_manager.tasks[session_id]` and confirm ws_manager
|
||||
# never reaches into it. We don't import agent_manager (heavy);
|
||||
# we just inspect the source.
|
||||
import inspect
|
||||
src = inspect.getsource(ws_manager.disconnect_session)
|
||||
assert "cancel" not in src.lower()
|
||||
assert "agent_manager" not in src
|
||||
assert "tasks" not in src
|
||||
|
||||
|
||||
def test_main_ws_endpoints_still_gated_by_auth(_patch_persist_dir):
|
||||
src = open(os.path.join(os.path.dirname(__file__), "..", "main.py")).read()
|
||||
assert "_ws_auth_ok(websocket)" in src, (
|
||||
"main.py WS endpoints must still call _ws_auth_ok before accepting "
|
||||
"the connection — otherwise any local web page can read agent traffic."
|
||||
)
|
||||
# And the disconnect handler must NOT call any task-cancel helper
|
||||
# — that's the regression we're guarding against.
|
||||
assert "stop_agent" not in src.split("WebSocketDisconnect")[1].split("def ")[0], (
|
||||
"WebSocketDisconnect handler must not cancel the agent task."
|
||||
)
|
||||
@@ -83,6 +83,12 @@ export interface AgentSession {
|
||||
mcp_suggestions_is_vague?: boolean;
|
||||
active_outputs?: string[];
|
||||
compacted_through_msg_id?: string | null;
|
||||
// Transient frontend-only WS connection state. Independent of
|
||||
// `status` (which describes the agent run itself). When the WS
|
||||
// drops we set this to 'reconnecting' so the UI can render a
|
||||
// subtle indicator without faking a terminal status. Cleared back
|
||||
// to 'live' on resume_ack. Never persisted to the backend.
|
||||
connection_state?: 'live' | 'reconnecting';
|
||||
}
|
||||
|
||||
export interface AgentConfig {
|
||||
@@ -602,6 +608,19 @@ const agentsSlice = createSlice({
|
||||
}
|
||||
},
|
||||
|
||||
setSessionConnState(
|
||||
state,
|
||||
action: PayloadAction<{ sessionId: string; state: 'live' | 'reconnecting' }>
|
||||
) {
|
||||
// Transient WS-layer indicator. Decoupled from session.status
|
||||
// so a network blip never masquerades as a run terminating —
|
||||
// status keeps reflecting the agent's actual lifecycle.
|
||||
const session = state.sessions[action.payload.sessionId];
|
||||
if (session) {
|
||||
session.connection_state = action.payload.state;
|
||||
}
|
||||
},
|
||||
|
||||
addMessage(state, action: PayloadAction<{ sessionId: string; message: AgentMessage }>) {
|
||||
const session = state.sessions[action.payload.sessionId];
|
||||
if (session) {
|
||||
@@ -1129,6 +1148,7 @@ export const {
|
||||
setDraftSystemPrompt,
|
||||
updateSession,
|
||||
updateSessionStatus,
|
||||
setSessionConnState,
|
||||
addMessage,
|
||||
streamStart,
|
||||
streamDelta,
|
||||
|
||||
@@ -18,6 +18,8 @@ import {
|
||||
setActiveBranch,
|
||||
closeSessionFromWs,
|
||||
trackAgentNotification,
|
||||
setSessionConnState,
|
||||
fetchSession,
|
||||
} from '../state/agentsSlice';
|
||||
import { addBrowserCardFromBackend, removeBrowserCard, setBrowserCardPosition, setGlowingBrowserCards, GRID_GAP } from '../state/dashboardLayoutSlice';
|
||||
import { getAuthToken } from '../config';
|
||||
@@ -30,23 +32,87 @@ const _getAuthTokenSafe = (): string => {
|
||||
try { return getAuthToken() || ''; } catch { return ''; }
|
||||
};
|
||||
|
||||
const _genUuid = (): string => {
|
||||
// Avoid pulling in `crypto.randomUUID` for compat — this is a
|
||||
// disambiguator, not a security boundary, so a 96-bit hex string is
|
||||
// plenty.
|
||||
const a = Math.floor(Math.random() * 2 ** 32).toString(16).padStart(8, '0');
|
||||
const b = Math.floor(Math.random() * 2 ** 32).toString(16).padStart(8, '0');
|
||||
const c = Math.floor(Math.random() * 2 ** 32).toString(16).padStart(8, '0');
|
||||
return `${a}${b}${c}`;
|
||||
};
|
||||
|
||||
type WSEvent = {
|
||||
event: string;
|
||||
session_id?: string;
|
||||
data: Record<string, any>;
|
||||
seq?: number;
|
||||
};
|
||||
|
||||
interface WSManagerOptions {
|
||||
skipStreamEvents?: boolean;
|
||||
// Session-scoped WSes opt into resume + connection-state dispatches
|
||||
// by passing this. Dashboard WS doesn't.
|
||||
sessionId?: string;
|
||||
}
|
||||
|
||||
// Heartbeat tuning. 25s is below typical aggressive NAT idle timeouts
|
||||
// (some enterprise firewalls drop after 30s of silence), and well
|
||||
// below browser-tab background throttling thresholds. 10s pong
|
||||
// timeout is a balance: long enough to tolerate flaky cellular RTT
|
||||
// spikes, short enough that a real dead socket reconnects fast.
|
||||
const HEARTBEAT_INTERVAL_MS = 25_000;
|
||||
const HEARTBEAT_TIMEOUT_MS = 10_000;
|
||||
|
||||
interface QueuedFrame {
|
||||
event: string;
|
||||
data: Record<string, any>;
|
||||
// Lets the future server-side dedup index match retries to
|
||||
// originals. Today the server treats most events idempotently
|
||||
// anyway (stop on stopped is a no-op), but the client sends this
|
||||
// forward-compatibly so a future server upgrade is safe without a
|
||||
// protocol bump.
|
||||
client_msg_id: string;
|
||||
}
|
||||
|
||||
class WebSocketManager {
|
||||
private ws: WebSocket | null = null;
|
||||
private url: string;
|
||||
private skipStreamEvents: boolean;
|
||||
private sessionId: string | null;
|
||||
|
||||
// Resume state. lastSeq is the highest server-assigned seq this
|
||||
// client has applied; it's sent on every (re)connect so the server
|
||||
// can replay missed events. Persists for the lifetime of this
|
||||
// WebSocketManager instance — when the user navigates away and a
|
||||
// new createSessionWs() is constructed, lastSeq starts at 0 and we
|
||||
// get a full replay.
|
||||
private connectionUuid: string;
|
||||
private lastSeq: number = 0;
|
||||
private resumeAcked: boolean = false;
|
||||
|
||||
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
private reconnectDelay = 1000;
|
||||
private maxReconnectDelay = 30000;
|
||||
// Set to true by `disconnect()` so we don't reconnect after an
|
||||
// explicit close (component unmount / user clicks Close).
|
||||
private explicitlyClosed: boolean = false;
|
||||
|
||||
// Heartbeat. We send a ping on a fixed cadence and arm a timeout
|
||||
// for the pong; if the timeout fires, we force-close the socket so
|
||||
// `onclose` triggers reconnect. Detects laptop-sleep / NAT-drop
|
||||
// silent failures that wouldn't otherwise surface until the next
|
||||
// outbound send.
|
||||
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
|
||||
private pongTimeoutTimer: ReturnType<typeof setTimeout> | null = null;
|
||||
|
||||
// Outbound queue. Frames the user enqueues while the WS isn't
|
||||
// OPEN — or while OPEN but pre-resume-ack — wait here and flush
|
||||
// after the resume handshake completes. Queue is in-memory only:
|
||||
// surviving a full app restart isn't worth the localStorage
|
||||
// complexity given how rare that case is for a transient drop.
|
||||
private outboundQueue: QueuedFrame[] = [];
|
||||
|
||||
private listeners: Map<string, Set<(data: any) => void>> = new Map();
|
||||
private interpolatorState: Map<string, { sessionId: string; messageId: string; targetText: string; displayedLength: number }> = new Map();
|
||||
private interpolatorRafId: number | null = null;
|
||||
@@ -54,6 +120,8 @@ class WebSocketManager {
|
||||
constructor(url: string, options?: WSManagerOptions) {
|
||||
this.url = url;
|
||||
this.skipStreamEvents = options?.skipStreamEvents ?? false;
|
||||
this.sessionId = options?.sessionId ?? null;
|
||||
this.connectionUuid = _genUuid();
|
||||
}
|
||||
|
||||
private bufferDelta(sessionId: string, messageId: string, delta: string) {
|
||||
@@ -115,16 +183,12 @@ class WebSocketManager {
|
||||
|
||||
connect() {
|
||||
if (this.ws?.readyState === WebSocket.OPEN) return;
|
||||
this.explicitlyClosed = false;
|
||||
|
||||
// Append our per-install auth token to the URL. The backend's WS
|
||||
// handshake validates this before accepting; without it, any
|
||||
// webpage loaded on the same machine could open a WS and read
|
||||
// agent traffic. See backend/auth.py + main.py:_ws_auth_ok.
|
||||
// Token is fetched async from Electron's preload, but we cache it
|
||||
// after first resolution. If it isn't cached yet, `getAuthToken()`
|
||||
// returns '' and the connection will be rejected — the
|
||||
// onclose handler below retries, by which time the token is
|
||||
// usually loaded.
|
||||
const token = _getAuthTokenSafe();
|
||||
const sep = this.url.includes('?') ? '&' : '?';
|
||||
const urlWithToken = token ? `${this.url}${sep}token=${encodeURIComponent(token)}` : this.url;
|
||||
@@ -132,6 +196,23 @@ class WebSocketManager {
|
||||
|
||||
this.ws.onopen = () => {
|
||||
this.reconnectDelay = 1000;
|
||||
this.resumeAcked = false;
|
||||
this.startHeartbeat();
|
||||
// Send hello immediately so the server can replay anything the
|
||||
// server sent that we never applied. On a fresh session,
|
||||
// last_seq=0 → server replays from buffer start (empty) and
|
||||
// we proceed normally.
|
||||
if (this.sessionId) {
|
||||
this.sendRaw('client:hello', {
|
||||
session_id: this.sessionId,
|
||||
connection_uuid: this.connectionUuid,
|
||||
last_seq: this.lastSeq,
|
||||
});
|
||||
} else {
|
||||
// Dashboard / global WS: no resume, queue can flush right away.
|
||||
this.resumeAcked = true;
|
||||
this.flushQueue();
|
||||
}
|
||||
};
|
||||
|
||||
this.ws.onmessage = (event) => {
|
||||
@@ -144,25 +225,40 @@ class WebSocketManager {
|
||||
};
|
||||
|
||||
this.ws.onclose = (ev) => {
|
||||
this.stopHeartbeat();
|
||||
// 4401 = our backend's auth-failure code. Happens on stale token
|
||||
// after backend restart (dev hot-reload). Re-fetch from Electron
|
||||
// IPC before retrying.
|
||||
if (ev && ev.code === 4401) {
|
||||
import('@/shared/config').then(mod => mod.refreshAuthToken().catch(() => {}));
|
||||
}
|
||||
this.scheduleReconnect();
|
||||
// Mark UI as reconnecting so the run card shows a clear
|
||||
// "trying to reconnect" state rather than implying the run
|
||||
// died. Skipped on an explicit disconnect (user navigated
|
||||
// away) since there's no run to surface state for.
|
||||
if (this.sessionId && !this.explicitlyClosed) {
|
||||
store.dispatch(setSessionConnState({
|
||||
sessionId: this.sessionId,
|
||||
state: 'reconnecting',
|
||||
}));
|
||||
}
|
||||
if (!this.explicitlyClosed) this.scheduleReconnect();
|
||||
};
|
||||
|
||||
this.ws.onerror = () => {
|
||||
// Force the close path to run — onclose will mark state
|
||||
// reconnecting and schedule a retry.
|
||||
this.ws?.close();
|
||||
};
|
||||
}
|
||||
|
||||
disconnect() {
|
||||
this.explicitlyClosed = true;
|
||||
if (this.reconnectTimer) {
|
||||
clearTimeout(this.reconnectTimer);
|
||||
this.reconnectTimer = null;
|
||||
}
|
||||
this.stopHeartbeat();
|
||||
if (this.interpolatorRafId != null) {
|
||||
cancelAnimationFrame(this.interpolatorRafId);
|
||||
this.interpolatorRafId = null;
|
||||
@@ -174,16 +270,133 @@ class WebSocketManager {
|
||||
|
||||
private scheduleReconnect() {
|
||||
if (this.reconnectTimer) return;
|
||||
// No retry cap. Long-horizon agent runs may outlast a multi-hour
|
||||
// network outage (overnight laptop sleep, captive portal limbo);
|
||||
// giving up would silently desync the UI. Backoff is bounded at
|
||||
// 30s so the user-visible "Reconnecting…" loop never hammers the
|
||||
// network, and a small jitter prevents thundering-herd if many
|
||||
// session WSes reconnect at once after a backend restart.
|
||||
const jitter = 0.8 + Math.random() * 0.4; // ±20%
|
||||
const delay = Math.min(this.reconnectDelay, this.maxReconnectDelay) * jitter;
|
||||
this.reconnectTimer = setTimeout(() => {
|
||||
this.reconnectTimer = null;
|
||||
this.reconnectDelay = Math.min(this.reconnectDelay * 2, this.maxReconnectDelay);
|
||||
this.connect();
|
||||
}, this.reconnectDelay);
|
||||
}, delay);
|
||||
}
|
||||
|
||||
private startHeartbeat() {
|
||||
this.stopHeartbeat();
|
||||
this.heartbeatTimer = setInterval(() => {
|
||||
this.sendPing();
|
||||
}, HEARTBEAT_INTERVAL_MS);
|
||||
}
|
||||
|
||||
private stopHeartbeat() {
|
||||
if (this.heartbeatTimer != null) {
|
||||
clearInterval(this.heartbeatTimer);
|
||||
this.heartbeatTimer = null;
|
||||
}
|
||||
if (this.pongTimeoutTimer != null) {
|
||||
clearTimeout(this.pongTimeoutTimer);
|
||||
this.pongTimeoutTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private sendPing() {
|
||||
if (this.ws?.readyState !== WebSocket.OPEN) return;
|
||||
const nonce = _genUuid();
|
||||
try {
|
||||
this.ws.send(JSON.stringify({ event: 'client:ping', data: { nonce } }));
|
||||
} catch {
|
||||
// socket dying — let the close handler take over
|
||||
return;
|
||||
}
|
||||
if (this.pongTimeoutTimer != null) clearTimeout(this.pongTimeoutTimer);
|
||||
this.pongTimeoutTimer = setTimeout(() => {
|
||||
// Silent death: no pong arrived in time. Force a close so the
|
||||
// browser's onclose path (and our reconnect) runs immediately
|
||||
// instead of waiting for the OS TCP keepalive (~75s).
|
||||
try { this.ws?.close(); } catch { /* nothing */ }
|
||||
}, HEARTBEAT_TIMEOUT_MS);
|
||||
}
|
||||
|
||||
private clearPongTimeout() {
|
||||
if (this.pongTimeoutTimer != null) {
|
||||
clearTimeout(this.pongTimeoutTimer);
|
||||
this.pongTimeoutTimer = null;
|
||||
}
|
||||
}
|
||||
|
||||
private flushQueue() {
|
||||
if (this.ws?.readyState !== WebSocket.OPEN) return;
|
||||
if (!this.resumeAcked) return;
|
||||
const queue = this.outboundQueue;
|
||||
this.outboundQueue = [];
|
||||
for (const frame of queue) {
|
||||
try {
|
||||
this.ws.send(JSON.stringify({ event: frame.event, data: frame.data }));
|
||||
} catch {
|
||||
// Re-queue and bail; reconnect will retry.
|
||||
this.outboundQueue.unshift(frame);
|
||||
break;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Direct send that bypasses the queue. Used for hello/ping which
|
||||
// must NOT be queued (they're connection-scoped, not session-data).
|
||||
private sendRaw(event: string, data: Record<string, any>) {
|
||||
if (this.ws?.readyState !== WebSocket.OPEN) return;
|
||||
try { this.ws.send(JSON.stringify({ event, data })); } catch { /* nothing */ }
|
||||
}
|
||||
|
||||
private handleMessage(msg: WSEvent) {
|
||||
const { event, session_id, data } = msg;
|
||||
|
||||
// Update lastSeq for events that carry one. seq is monotonic per
|
||||
// session, so this is the high-water mark we send back on resume.
|
||||
if (typeof msg.seq === 'number' && msg.seq > this.lastSeq) {
|
||||
this.lastSeq = msg.seq;
|
||||
}
|
||||
|
||||
// ----- Connection-scoped frames (no business-logic side effects) -----
|
||||
|
||||
if (event === 'server:pong') {
|
||||
this.clearPongTimeout();
|
||||
return;
|
||||
}
|
||||
|
||||
if (event === 'server:hello') {
|
||||
// Resume handshake completed. The server has either replayed
|
||||
// missed events (which arrived as separate frames before this
|
||||
// ack), surfaced a gap, or signalled "you're caught up." Mark
|
||||
// ourselves live and flush any queued outbound frames.
|
||||
this.resumeAcked = true;
|
||||
if (this.sessionId) {
|
||||
store.dispatch(setSessionConnState({
|
||||
sessionId: this.sessionId,
|
||||
state: 'live',
|
||||
}));
|
||||
}
|
||||
this.flushQueue();
|
||||
return;
|
||||
}
|
||||
|
||||
if (event === 'agent:gap_detected') {
|
||||
// We were offline long enough that the server's ring buffer
|
||||
// rolled past our lastSeq. Re-fetch authoritative state via
|
||||
// REST so the slice's view doesn't have a silent gap.
|
||||
if (session_id) {
|
||||
store.dispatch(fetchSession(session_id));
|
||||
// Reset lastSeq — the REST refetch is the new authoritative
|
||||
// baseline; subsequent server events with seq numbers will
|
||||
// re-establish the high-water mark.
|
||||
this.lastSeq = 0;
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
if (this.skipStreamEvents) {
|
||||
if (event === 'agent:stream_start' || event === 'agent:stream_delta' || event === 'agent:stream_end') {
|
||||
return;
|
||||
@@ -419,8 +632,21 @@ class WebSocketManager {
|
||||
}
|
||||
|
||||
send(event: string, data: Record<string, any>) {
|
||||
if (this.ws?.readyState !== WebSocket.OPEN) return;
|
||||
this.ws.send(JSON.stringify({ event, data }));
|
||||
// Queue if the socket isn't open OR resume hasn't been ack'd yet.
|
||||
// The pre-ack gate prevents an outbound user message from racing
|
||||
// the resume replay — the server might process the message
|
||||
// before the replay finishes, leaving the slice's view of
|
||||
// history incomplete.
|
||||
const open = this.ws?.readyState === WebSocket.OPEN;
|
||||
if (!open || !this.resumeAcked) {
|
||||
this.outboundQueue.push({ event, data, client_msg_id: _genUuid() });
|
||||
return;
|
||||
}
|
||||
try {
|
||||
this.ws!.send(JSON.stringify({ event, data }));
|
||||
} catch {
|
||||
this.outboundQueue.push({ event, data, client_msg_id: _genUuid() });
|
||||
}
|
||||
}
|
||||
|
||||
sendMessage(
|
||||
@@ -465,7 +691,7 @@ import { WS_BASE } from '@/shared/config';
|
||||
export const dashboardWs = new WebSocketManager(`${WS_BASE}/ws/dashboard`, { skipStreamEvents: true });
|
||||
|
||||
export function createSessionWs(sessionId: string): WebSocketManager {
|
||||
return new WebSocketManager(`${WS_BASE}/ws/agents/${sessionId}`);
|
||||
return new WebSocketManager(`${WS_BASE}/ws/agents/${sessionId}`, { sessionId });
|
||||
}
|
||||
|
||||
export default WebSocketManager;
|
||||
|
||||
File diff suppressed because one or more lines are too long
Reference in New Issue
Block a user