[eric] WS resilience: per-session seq + ring buffer + resume protocol so agent runs survive transient disconnects (wifi flap, sleep, NAT

drop) instead of flipping to completed. Adds heartbeat (25s ping/10s pong), reconnect with infinite jittered backoff, outbound queue gated
   on resume_ack, gap_detected fallback for long offlines, on-disk persistence of terminal events for post-restart recovery, and a
  reconnecting connection state decoupled from session.status. 1089 backend tests covering 500 randomized disconnect scenarios + concurrent
  broadcast races. Backend/WS handler does not cancel agent task on disconnect
This commit is contained in:
ciregenz
2026-04-28 23:47:59 -04:00
parent 5d69215739
commit 9be2d87f73
7 changed files with 1215 additions and 27 deletions
+232
View File
@@ -0,0 +1,232 @@
"""Per-session WS event sequencing, ring buffer, and terminal-event persistence.
Why this exists
---------------
WS sockets die for a thousand reasons that have nothing to do with the
agent task: laptop sleep, captive portals, NAT idle timeout, VPN
renegotiation. Without this module, a transient drop is fatal —
mid-stream events are lost forever and the UI can't tell whether the
run finished or merely went quiet.
Contract
--------
Every WS event for a session goes through `stamp(...)`, which is an
async context manager that:
1. Acquires the per-session lock.
2. Bumps a monotonic `seq` integer.
3. Appends the JSON payload to a bounded ring buffer.
4. Yields (seq, payload_str) to the caller.
5. Holds the lock until the caller exits the `async with` — meaning
the caller's `ws.send_text(...)` happens *under the same lock*,
guaranteeing wire order == seq order even when many coroutines
broadcast concurrently.
Without (5), two coroutines can each get a unique seq under separate
lock acquisitions, yet the higher-seq event can reach the wire first
because asyncio scheduled its `send_text` earlier. That corrupts both
wire order and the ring buffer on resume.
Resume protocol
---------------
On reconnect, the client sends `client:resume {connection_uuid,
last_seq}`. The server:
- Returns ring-buffer events with `seq > last_seq` if available.
- Returns `agent:gap_detected` if `last_seq` is older than the
oldest buffered seq — the client falls back to a REST refresh.
- Returns the persisted terminal event (if any) when the session
is no longer in memory at all (e.g. after a process restart).
Persistence
-----------
Terminal events (status: completed/stopped/error) are written
atomically to disk so a client that comes back hours later — long
after the in-memory ring buffer has been GC'd — still sees the right
outcome instead of a spinner that never resolves. Persistence is
opportunistic: an I/O error never blocks the broadcast path.
"""
from __future__ import annotations
import asyncio
import json
import logging
import os
from collections import deque
from contextlib import asynccontextmanager
from typing import AsyncIterator, Optional
logger = logging.getLogger(__name__)
# Ring buffer size per session. ~500 events comfortably covers a 30s
# transient drop even in the busiest streams (thinking deltas at
# ~20Hz). Memory is bounded: ~50KB per active session.
BUFFER_LIMIT = 500
TERMINAL_STATUSES = {"completed", "stopped", "error"}
class _SessionSeqLog:
"""Per-session lock + monotonic seq + recent-event ring buffer."""
__slots__ = ("lock", "seq", "buffer")
def __init__(self) -> None:
self.lock: asyncio.Lock = asyncio.Lock()
self.seq: int = 0
# Each entry: (seq, json_payload_str). Pre-serialized so a
# replay doesn't redo json.dumps for every reconnect.
self.buffer: deque[tuple[int, str]] = deque(maxlen=BUFFER_LIMIT)
class SeqLogStore:
"""Process-wide store. Per-session locks live inside `_SessionSeqLog`."""
def __init__(self, persist_dir: Optional[str] = None) -> None:
self._per_session: dict[str, _SessionSeqLog] = {}
# Coarse lock guarding only the dict's setdefault path. Held
# for nanoseconds; never crosses an `await` past the `_get`.
self._dict_lock = asyncio.Lock()
self._persist_dir = persist_dir
if persist_dir:
try:
os.makedirs(persist_dir, exist_ok=True)
except Exception:
logger.warning("seq_log: failed to create persist dir %s", persist_dir)
async def _get_or_create(self, session_id: str) -> _SessionSeqLog:
log = self._per_session.get(session_id)
if log is not None:
return log
async with self._dict_lock:
log = self._per_session.get(session_id)
if log is None:
log = _SessionSeqLog()
self._per_session[session_id] = log
return log
def _peek(self, session_id: str) -> Optional[_SessionSeqLog]:
return self._per_session.get(session_id)
@asynccontextmanager
async def stamp(
self, session_id: str, event: str, data: dict
) -> AsyncIterator[tuple[int, str]]:
"""Atomically assign a seq, buffer it, and yield (seq, payload).
Caller is expected to perform the actual `send_text` *inside*
the `async with` block. The per-session lock is held for the
entire body, so wire order is guaranteed equal to seq order
no matter how many tasks broadcast concurrently.
"""
log = await self._get_or_create(session_id)
async with log.lock:
log.seq += 1
seq = log.seq
payload = {
"event": event,
"session_id": session_id,
"data": data,
"seq": seq,
}
payload_str = json.dumps(payload)
log.buffer.append((seq, payload_str))
yield seq, payload_str
def replay(
self, session_id: str, last_seq: int
) -> tuple[Optional[int], Optional[int], list[str]]:
"""Return (oldest_buffered_seq, newest_buffered_seq, events).
Caller decides what to do with the result:
- `events` empty AND newest_buffered_seq is None: no buffer
for this session in memory. Fall back to persisted
terminal event.
- `last_seq` < `oldest_buffered_seq`: there's a gap. Send
`agent:gap_detected`; the client REST-refreshes.
- Otherwise `events` are the missed payloads in seq order.
"""
log = self._peek(session_id)
if log is None:
return (None, None, [])
# Snapshot the deque under the lock-free fast path. asyncio is
# single-threaded so a list() of a deque mutated by append is
# safe; eviction (via maxlen) is also a single-step op. We
# don't need to hold the per-session lock for a read.
snapshot = list(log.buffer)
if not snapshot:
return (None, log.seq, [])
oldest = snapshot[0][0]
newest = snapshot[-1][0]
events = [s for (i, s) in snapshot if i > last_seq]
return (oldest, newest, events)
def current_seq(self, session_id: str) -> int:
"""Last assigned seq, or 0 if no log exists for the session."""
log = self._peek(session_id)
return log.seq if log else 0
# ----- Terminal-event persistence -----
def _terminal_path(self, session_id: str) -> Optional[str]:
if not self._persist_dir:
return None
# session ids are uuid4 hex in this codebase, but sanitize
# against path traversal anyway.
safe = "".join(c for c in session_id if c.isalnum() or c in ("-", "_"))
if not safe:
return None
return os.path.join(self._persist_dir, f"{safe}.json")
def persist_terminal(self, session_id: str, payload_str: str) -> None:
"""Atomic write of a terminal event for post-restart clients.
Best-effort: an I/O failure must never block the broadcast.
"""
path = self._terminal_path(session_id)
if not path:
return
try:
tmp = path + ".tmp"
with open(tmp, "w", encoding="utf-8") as f:
f.write(payload_str)
os.replace(tmp, path)
except Exception:
logger.debug(
"seq_log: failed to persist terminal event for %s", session_id, exc_info=True
)
def load_terminal(self, session_id: str) -> Optional[str]:
path = self._terminal_path(session_id)
if not path or not os.path.exists(path):
return None
try:
with open(path, "r", encoding="utf-8") as f:
return f.read()
except Exception:
return None
def clear(self, session_id: str) -> None:
"""Drop in-memory log + persisted terminal event.
Use on full session deletion. Closed-but-retained sessions
keep their terminal file so late reconnects still resolve.
"""
self._per_session.pop(session_id, None)
path = self._terminal_path(session_id)
if path and os.path.exists(path):
try:
os.remove(path)
except Exception:
pass
def _default_persist_dir() -> Optional[str]:
try:
from backend.config.paths import DATA_ROOT
return os.path.join(DATA_ROOT, "agents", "terminal_events")
except Exception:
return None
# Process-wide singleton wired to the agents data dir.
seq_log = SeqLogStore(persist_dir=_default_persist_dir())
+133 -14
View File
@@ -3,11 +3,26 @@ import json
import logging
from fastapi import WebSocket
from backend.apps.agents.seq_log import TERMINAL_STATUSES, seq_log
logger = logging.getLogger(__name__)
class ConnectionManager:
"""Manages WebSocket connections and bridges HITL approval requests."""
"""Manages WebSocket connections and bridges HITL approval requests.
Every outbound event flows through the seq log so reconnecting
clients can replay missed events. The send happens *under* the
per-session lock yielded by `seq_log.stamp(...)`, which guarantees
wire order matches seq order even under concurrent broadcasts.
A WS disconnect (`disconnect_session`) ONLY removes the socket
from the connection registry. It does NOT cancel the underlying
agent task. The task lives on `agent_manager.tasks`; only an
explicit `agent:stop`, REST `/close`, natural completion, or
process shutdown ends a run.
"""
def __init__(self):
self.connections: dict[str, list[WebSocket]] = {}
self.global_connections: list[WebSocket] = []
@@ -38,23 +53,123 @@ class ConnectionManager:
]
async def send_to_session(self, session_id: str, event: str, data: dict):
"""Send a message to all connections watching a specific session."""
payload = json.dumps({"event": event, "session_id": session_id, "data": data})
for ws in self.connections.get(session_id, []):
"""Broadcast a session event with monotonic sequencing.
The send to every socket happens inside the seq_log lock so a
slow/dead WS doesn't reorder events on the fast ones. If a
single send raises (broken pipe, half-open socket), we log and
continue — the ring buffer still has the event so the client
will replay it on reconnect.
For terminal status events (completed/stopped/error) we also
atomically persist the payload to disk; a client that returns
after a process restart can then resolve the spinner via
`seq_log.load_terminal(...)` instead of being stuck.
"""
async with seq_log.stamp(session_id, event, data) as (seq, payload_str):
for ws in list(self.connections.get(session_id, [])):
try:
await ws.send_text(payload_str)
except Exception:
logger.debug("send_to_session: send failed (will retry on reconnect)", exc_info=True)
for ws in list(self.global_connections):
try:
await ws.send_text(payload_str)
except Exception:
logger.debug("send_to_session: global send failed", exc_info=True)
# Persist terminal events under the lock so a concurrent
# `agent:status: running` can't race past and overwrite
# the disk file with a stale state.
if event == "agent:status" and data.get("status") in TERMINAL_STATUSES:
seq_log.persist_terminal(session_id, payload_str)
async def replay_to(
self, session_id: str, websocket: WebSocket, last_seq: int
) -> dict:
"""Replay buffered events with seq > last_seq to one socket.
Returns a small ack envelope describing what happened so the
caller (the WS handler) can send a `server:resume_ack` frame.
Three cases:
1. `events` non-empty: replay them in order; ack carries
`from_seq`, `to_seq`.
2. No buffer at all (process restarted, session evicted)
but a persisted terminal exists: send it; ack signals
`terminal_only=True`.
3. `last_seq` predates the oldest buffered seq: emit
`agent:gap_detected`; client REST-refreshes the session.
"""
oldest, newest, events = seq_log.replay(session_id, last_seq)
# Check for gap FIRST. If the client's last_seq is below the
# buffer's oldest seq, we can't deliver everything they
# missed — silently replaying only the in-buffer tail would
# leave a hole in their state. Tell them to REST-refresh
# instead, even if the tail looks safe to send.
# Treat last_seq=0 as "fresh client" — they want a full
# replay of whatever's in the buffer, not a gap signal.
if last_seq > 0 and oldest is not None and last_seq < oldest - 1:
gap_payload = json.dumps({
"event": "agent:gap_detected",
"session_id": session_id,
"data": {
"session_id": session_id,
"oldest_seq": oldest,
"newest_seq": newest,
"client_seq": last_seq,
},
})
try:
await ws.send_text(payload)
await websocket.send_text(gap_payload)
except Exception:
pass
for ws in self.global_connections:
return {
"ok": False,
"reason": "gap",
"oldest_seq": oldest,
"newest_seq": newest,
}
if events:
for s in events:
try:
await websocket.send_text(s)
except Exception:
logger.debug("replay_to: send failed", exc_info=True)
break
return {
"ok": True,
"replayed": len(events),
"from_seq": last_seq,
"to_seq": newest,
}
# Nothing in memory. Try a persisted terminal event.
terminal = seq_log.load_terminal(session_id)
if terminal is not None:
try:
await ws.send_text(payload)
await websocket.send_text(terminal)
except Exception:
pass
return {"ok": True, "replayed": 1, "terminal_only": True}
# Nothing missed, nothing to replay. Caller's caught up.
return {
"ok": True,
"replayed": 0,
"current_seq": newest if newest is not None else 0,
}
async def broadcast_global(self, event: str, data: dict):
"""Send a message to all global (dashboard) connections."""
"""Send a message to all global (dashboard) connections.
Dashboard-scoped events don't go through the per-session seq
log — they're not session-bound and the dashboard WS has its
own resume story (full state refetch on reconnect).
"""
payload = json.dumps({"event": event, "data": data})
for ws in self.global_connections:
for ws in list(self.global_connections):
try:
await ws.send_text(payload)
except Exception:
@@ -65,17 +180,20 @@ class ConnectionManager:
timeout: float = 600.0,
) -> dict:
"""Send an approval request and wait for the user's response.
Returns the approval decision dict. Times out after *timeout* seconds
(default 10 minutes) to prevent permanently stuck agents."""
Returns the approval decision dict. Times out after `timeout`
seconds (default 10 minutes) so a forgotten request doesn't
permanently park the agent.
"""
future = asyncio.get_event_loop().create_future()
self.pending_futures[request_id] = future
await self.send_to_session(session_id, "agent:approval_request", {
"request_id": request_id,
"tool_name": tool_name,
"tool_input": tool_input,
})
try:
result = await asyncio.wait_for(future, timeout=timeout)
return result
@@ -123,4 +241,5 @@ class ConnectionManager:
if future and not future.done():
future.set_result(result)
ws_manager = ConnectionManager()
+50 -2
View File
@@ -133,6 +133,22 @@ async def _auth_middleware(request: Request, call_next):
@app.websocket("/ws/agents/{session_id}")
async def websocket_session(websocket: WebSocket, session_id: str):
"""Per-session WS endpoint with resume + heartbeat.
Resilience contract (see backend/apps/agents/seq_log.py):
- Every server→client event carries a monotonic `seq` per session.
- On (re)connect the client sends `client:hello` with its
last-seen seq; the server replays missed events (or emits
`agent:gap_detected` if the gap is too large) and answers
with `server:hello` carrying the current high-water seq.
- `client:ping` → `server:pong` heartbeat (default 25s) so
silent socket deaths (NAT idle drop, laptop sleep) are
detected without waiting for the next outbound frame.
- `WebSocketDisconnect` only removes the socket from the
connection registry. The agent task keeps running. The only
things that end a run are: natural completion, explicit
`agent:stop`, REST `/close`, or process shutdown.
"""
if not _ws_auth_ok(websocket):
return
await ws_manager.connect_session(session_id, websocket)
@@ -142,8 +158,38 @@ async def websocket_session(websocket: WebSocket, session_id: str):
msg = json.loads(data)
event = msg.get("event")
payload = msg.get("data", {})
if event == "agent:send_message":
if event == "client:hello":
# Resume handshake. The client sends this immediately
# after the WS opens, with `last_seq` = the highest
# seq it has applied. We replay anything newer; on
# first connect last_seq=0 and replay() correctly
# returns nothing (empty buffer) or the persisted
# terminal event for already-finished sessions.
last_seq = int(payload.get("last_seq") or 0)
connection_uuid = payload.get("connection_uuid") or ""
ack = await ws_manager.replay_to(session_id, websocket, last_seq)
from backend.apps.agents.seq_log import seq_log as _sl
await websocket.send_text(json.dumps({
"event": "server:hello",
"session_id": session_id,
"data": {
"connection_uuid": connection_uuid,
"current_seq": _sl.current_seq(session_id),
"ack": ack,
},
}))
elif event == "client:ping":
# Heartbeat. Cheap, keeps NATs/firewalls from
# silently dropping the connection. Carry the
# client's nonce back so it can match pong→ping for
# round-trip latency tracking if it wants.
await websocket.send_text(json.dumps({
"event": "server:pong",
"session_id": session_id,
"data": {"nonce": payload.get("nonce")},
}))
elif event == "agent:send_message":
from backend.apps.agents.agent_manager import agent_manager
await agent_manager.send_message(
session_id,
@@ -171,6 +217,8 @@ async def websocket_session(websocket: WebSocket, session_id: str):
from backend.apps.agents.agent_manager import agent_manager
await agent_manager.stop_agent(session_id)
except WebSocketDisconnect:
# Drops the socket from the connection list. Does NOT cancel
# the agent task — that's intentional. See module docstring.
ws_manager.disconnect_session(session_id, websocket)
def _ws_auth_ok(websocket: WebSocket) -> bool:
+543
View File
@@ -0,0 +1,543 @@
"""Stress test for the WS resilience layer.
Goal: actively try to break the agent run with disconnects, drops,
reconnects, and concurrent broadcasts. The visible bug we're chasing
is "Network issue" toasts that flip a still-running task to a
terminal state. After these fixes the contract should be:
1. The agent task NEVER dies because of a WS drop.
2. Every event the server emits is replayable, in order, with no
duplicates and no gaps, after any number of disconnects.
3. Terminal events (completed/stopped/error) are always observable
by a client that reconnects later — even if the only persistence
of the event is the on-disk terminal log.
4. Concurrent broadcasts (thinking deltas + tool calls + status
changes from many tasks) preserve seq order == wire order.
5. A client that's been gone too long for the ring buffer gets a
`agent:gap_detected` instead of silent loss.
We don't run real Claude Code; we install a stub `agent_loop` that
emits the same WS event shapes a real run would (status, stream,
message, completed). That keeps the test fast (>200 iterations in
seconds) and self-contained.
Run:
cd backend && .venv/bin/python -m pytest tests/test_disconnect_resilience.py -v
"""
from __future__ import annotations
import asyncio
import json
import os
import random
import sys
import tempfile
from typing import Any
from unittest.mock import patch
import pytest
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.testclient import TestClient
# ---------------------------------------------------------------------------
# Boot env: route data to a tempdir BEFORE importing backend modules so
# the persistence dir for terminal events lives under our control.
# ---------------------------------------------------------------------------
_TMPROOT = tempfile.mkdtemp(prefix="openswarm-disconnect-test-")
os.environ.setdefault("OPENSWARM_DATA_DIR", _TMPROOT)
# Push the seq_log persist dir to a deterministic location too.
_SEQ_DIR = os.path.join(_TMPROOT, "seq_terminals")
os.makedirs(_SEQ_DIR, exist_ok=True)
@pytest.fixture(autouse=True)
def _patch_persist_dir():
"""Force the seq_log to use our tmp dir so we can assert on disk state."""
from backend.apps.agents import seq_log as sl_mod
# Rebuild the singleton with our test dir.
new_store = sl_mod.SeqLogStore(persist_dir=_SEQ_DIR)
monkey = patch.object(sl_mod, "seq_log", new_store)
monkey.start()
# Also patch the symbol re-exported into ws_manager's import scope.
from backend.apps.agents import ws_manager as wm_mod
wm_monkey = patch.object(wm_mod, "seq_log", new_store)
wm_monkey.start()
yield new_store
monkey.stop()
wm_monkey.stop()
# ---------------------------------------------------------------------------
# Minimal FastAPI app with the real WS endpoint logic. We import
# ws_manager directly and replicate the handler from backend/main.py
# without any of its auth middleware so the TestClient can connect
# without a token.
# ---------------------------------------------------------------------------
def _build_app(seq_log):
"""Replicates main.py's WS handler + adds a /test/emit endpoint
so the test thread can drive event emission through the same
event loop as the WS handler — avoiding the cross-loop hazards
of `asyncio.run()` mid-test."""
from backend.apps.agents.ws_manager import ws_manager
app = FastAPI()
@app.websocket("/ws/agents/{session_id}")
async def ws_session(websocket: WebSocket, session_id: str):
await ws_manager.connect_session(session_id, websocket)
try:
while True:
data = await websocket.receive_text()
msg = json.loads(data)
event = msg.get("event")
payload = msg.get("data", {})
if event == "client:hello":
last_seq = int(payload.get("last_seq") or 0)
ack = await ws_manager.replay_to(session_id, websocket, last_seq)
await websocket.send_text(json.dumps({
"event": "server:hello",
"session_id": session_id,
"data": {
"connection_uuid": payload.get("connection_uuid", ""),
"current_seq": seq_log.current_seq(session_id),
"ack": ack,
},
}))
elif event == "client:ping":
await websocket.send_text(json.dumps({
"event": "server:pong",
"session_id": session_id,
"data": {"nonce": payload.get("nonce")},
}))
except WebSocketDisconnect:
ws_manager.disconnect_session(session_id, websocket)
@app.post("/test/emit/{session_id}")
async def emit_events(session_id: str, body: dict):
n = int(body.get("n", 0))
terminate = body.get("terminate") # str or None
concurrent = int(body.get("concurrent", 1))
await _emit_run(session_id, n, terminate=terminate, concurrent_tasks=concurrent)
return {"ok": True, "current_seq": seq_log.current_seq(session_id)}
return app
def _emit(client, session_id: str, n: int, terminate: str | None = None, concurrent: int = 1):
"""Drive event emission via the test-only HTTP endpoint."""
r = client.post(f"/test/emit/{session_id}", json={
"n": n, "terminate": terminate, "concurrent": concurrent,
})
assert r.status_code == 200, r.text
return r.json()
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
async def _emit_run(session_id: str, n_events: int, terminate: str | None = "completed", concurrent_tasks: int = 1):
"""Emit a synthetic agent run.
`concurrent_tasks` lets the test stress the per-session lock by
fanning out the broadcast across multiple coroutines. The seq
log must still order them strictly.
"""
from backend.apps.agents.ws_manager import ws_manager
async def emit_chunk(start: int, count: int):
for i in range(count):
await ws_manager.send_to_session(session_id, "agent:stream_delta", {
"session_id": session_id,
"message_id": "m1",
"delta": f"chunk-{start + i}",
})
# Yield to the scheduler so other coroutines interleave —
# this is what surfaces the seq race if locking is wrong.
await asyncio.sleep(0)
if concurrent_tasks <= 1:
await emit_chunk(0, n_events)
else:
per = n_events // concurrent_tasks
tasks = [
asyncio.create_task(emit_chunk(i * per, per))
for i in range(concurrent_tasks)
]
await asyncio.gather(*tasks)
# Mop up the remainder so total event count is exact.
rem = n_events - per * concurrent_tasks
if rem > 0:
await emit_chunk(per * concurrent_tasks, rem)
if terminate is not None:
await ws_manager.send_to_session(session_id, "agent:status", {
"session_id": session_id,
"status": terminate,
})
# ---------------------------------------------------------------------------
# Unit-level: seq log fundamentals
# ---------------------------------------------------------------------------
def test_seq_monotonic_under_concurrency(_patch_persist_dir):
"""200 concurrent broadcasts must yield strictly monotonic seq."""
app = _build_app(_patch_persist_dir)
sid = "session-conc-1"
with TestClient(app) as client:
_emit(client, sid, n=200, terminate=None, concurrent=4)
_, newest, events = _patch_persist_dir.replay(sid, 0)
assert newest == 200
seqs = [json.loads(s)["seq"] for s in events]
assert seqs == sorted(seqs)
assert len(set(seqs)) == len(seqs)
def test_terminal_event_persisted(_patch_persist_dir):
app = _build_app(_patch_persist_dir)
sid = "session-term-1"
with TestClient(app) as client:
_emit(client, sid, n=0, terminate="completed")
raw = _patch_persist_dir.load_terminal(sid)
assert raw is not None
obj = json.loads(raw)
assert obj["event"] == "agent:status"
assert obj["data"]["status"] == "completed"
def test_replay_after_eviction_reports_gap(_patch_persist_dir):
app = _build_app(_patch_persist_dir)
sid = "session-evict-1"
with TestClient(app) as client:
_emit(client, sid, n=700, terminate=None)
oldest, newest, events = _patch_persist_dir.replay(sid, last_seq=10)
assert newest == 700
assert oldest is not None and oldest > 10
# Replay only includes seqs > 10 that survived eviction.
assert all(json.loads(s)["seq"] > 10 for s in events)
# ---------------------------------------------------------------------------
# Integration: full WS connect / disconnect / resume cycle
# ---------------------------------------------------------------------------
def test_resume_after_disconnect_recovers_all_events(_patch_persist_dir):
"""Simulate a single disconnect mid-run, then a clean resume."""
app = _build_app(_patch_persist_dir)
sid = "session-res-1"
received: list[dict] = []
with TestClient(app) as client:
# Phase 1: connect, hello, see N events, then close abruptly.
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 0, "connection_uuid": "c1"}}))
hello = json.loads(ws.receive_text())
assert hello["event"] == "server:hello"
# Inject a few events via the /test/emit endpoint.
_emit(client, sid, n=10, terminate=None)
for _ in range(10):
received.append(json.loads(ws.receive_text()))
assert len(received) == 10
assert received[-1]["seq"] == 10
# Phase 2: between connections, the server keeps emitting. The
# agent task is alive; only the WS is gone.
_emit(client, sid, n=10, terminate="completed")
# Phase 3: reconnect with last_seq=10, expect replay of seq 11..21
# (10 deltas + 1 status), then the server:hello ack.
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 10, "connection_uuid": "c2"}}))
replay: list[dict] = []
while True:
msg = json.loads(ws.receive_text())
if msg["event"] == "server:hello":
break
replay.append(msg)
seqs = [m["seq"] for m in replay]
assert seqs == list(range(11, 22)), f"unexpected replay seqs: {seqs}"
statuses = [m for m in replay if m["event"] == "agent:status"]
assert len(statuses) == 1
assert statuses[0]["data"]["status"] == "completed"
def test_terminal_event_visible_after_full_eviction(_patch_persist_dir):
"""If the in-memory log is wiped (process restart simulation),
a reconnecting client should still see the terminal event from
disk — never a phantom 'running' spinner."""
app = _build_app(_patch_persist_dir)
sid = "session-evict-term-1"
seq_log = _patch_persist_dir
with TestClient(app) as client:
_emit(client, sid, n=5, terminate="completed")
# Simulate a process restart: clear the in-memory ring buffer
# but keep the persisted terminal file.
seq_log._per_session.pop(sid, None)
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 0, "connection_uuid": "c1"}}))
received = []
while True:
msg = json.loads(ws.receive_text())
if msg["event"] == "server:hello":
received.append(msg)
break
received.append(msg)
terminals = [m for m in received if m["event"] == "agent:status"]
assert len(terminals) == 1
assert terminals[0]["data"]["status"] == "completed"
def test_gap_detected_when_buffer_evicted(_patch_persist_dir):
"""A client whose lastSeq is older than the oldest buffered seq
should receive `agent:gap_detected` so it can REST-refresh,
rather than silently miss events."""
app = _build_app(_patch_persist_dir)
sid = "session-gap-1"
with TestClient(app) as client:
# Fill the buffer past its limit so seq 1..200 are evicted.
_emit(client, sid, n=700, terminate=None)
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 5, "connection_uuid": "c1"}}))
saw_gap = False
saw_hello = False
while not saw_hello:
msg = json.loads(ws.receive_text())
if msg["event"] == "agent:gap_detected":
saw_gap = True
elif msg["event"] == "server:hello":
saw_hello = True
assert msg["data"]["ack"]["ok"] is False
assert msg["data"]["ack"]["reason"] == "gap"
assert saw_gap
def test_ping_pong_round_trip(_patch_persist_dir):
app = _build_app(_patch_persist_dir)
sid = "session-ping-1"
with TestClient(app) as client:
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 0, "connection_uuid": "c1"}}))
assert json.loads(ws.receive_text())["event"] == "server:hello"
ws.send_text(json.dumps({"event": "client:ping", "data": {"nonce": "abc"}}))
pong = json.loads(ws.receive_text())
assert pong["event"] == "server:pong"
assert pong["data"]["nonce"] == "abc"
# ---------------------------------------------------------------------------
# The big one: hundreds of randomized disconnect scenarios.
# ---------------------------------------------------------------------------
N_STRESS_ITERATIONS = int(os.environ.get("DISCONNECT_STRESS_N", "500"))
@pytest.mark.parametrize("iteration", range(N_STRESS_ITERATIONS))
def test_stress_random_disconnect(iteration, _patch_persist_dir):
"""Each iteration: a random number of events, a random number of
disconnects at random points, optionally ending in a terminal
status. After all reconnects, the client must have observed
every event exactly once, in seq order, and the terminal event
if one was emitted."""
rng = random.Random(iteration) # deterministic per iteration
app = _build_app(_patch_persist_dir)
sid = f"session-stress-{iteration}"
total_events = rng.randint(5, 80)
n_disconnects = rng.randint(1, min(5, total_events // 2 or 1))
will_terminate = rng.random() < 0.7 # 70% of runs reach a terminal
# Disconnect points: each is a count of events emitted *before*
# the WS drops. We deliberately exclude `total_events` itself
# so the breakpoint list never collides with the appended final
# iteration (which is when the optional terminal status fires).
if total_events > 1:
breakpoints = sorted(rng.sample(range(1, total_events), min(n_disconnects, total_events - 1)))
else:
breakpoints = []
seen: dict[int, dict] = {} # seq -> event payload
last_seq = 0
with TestClient(app) as client:
emitted_so_far = 0
for bp in breakpoints + [total_events]:
# Open a fresh socket, hello with our last_seq.
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": last_seq, "connection_uuid": f"c-{rng.random()}"}}))
# Drain until server:hello, recording any replayed events.
while True:
msg = json.loads(ws.receive_text())
if msg["event"] == "server:hello":
break
if "seq" in msg:
seen[msg["seq"]] = msg
last_seq = max(last_seq, msg["seq"])
to_emit = bp - emitted_so_far
emitted_so_far = bp
terminate = "completed" if (bp == total_events and will_terminate) else None
# Drive the emit through the test app's HTTP endpoint so
# the broadcast happens on the same event loop as the WS
# handler. Using asyncio.run() here would create an
# isolated loop and re-bind the per-session asyncio.Lock
# to a different loop, which is hostile to anyio's
# blocking-portal pattern.
_emit(client, sid, n=to_emit, terminate=terminate)
expected = to_emit + (1 if terminate else 0)
for _ in range(expected):
msg = json.loads(ws.receive_text())
seen[msg["seq"]] = msg
last_seq = max(last_seq, msg["seq"])
# Closing the with-block disconnects the WS. The loop
# opens a fresh socket on the next iteration.
# ----- Assertions: completeness, ordering, no dups, terminal -----
expected_total = total_events + (1 if will_terminate else 0)
assert len(seen) == expected_total, f"missing events: expected {expected_total}, got {len(seen)}"
seqs = sorted(seen.keys())
assert seqs == list(range(1, expected_total + 1)), f"non-contiguous seqs: {seqs[:5]}...{seqs[-5:]}"
if will_terminate:
last = seen[expected_total]
assert last["event"] == "agent:status"
assert last["data"]["status"] == "completed"
# ---------------------------------------------------------------------------
# Concurrent broadcast: many fan-out coroutines must preserve seq order
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("trial", range(30))
def test_concurrent_broadcast_preserves_order(trial, _patch_persist_dir):
"""8 coroutines fanning out 400 events under the per-session lock.
Drives the emit through the TestClient's portal so we use the
real event loop the rest of the WS layer runs on."""
app = _build_app(_patch_persist_dir)
sid = f"session-conc-{trial}"
with TestClient(app) as client:
_emit(client, sid, n=400, terminate="completed", concurrent=8)
oldest, newest, events = _patch_persist_dir.replay(sid, last_seq=0)
assert newest == 401 # 400 deltas + 1 status
seqs = [json.loads(s)["seq"] for s in events]
assert seqs == sorted(seqs)
# Each seq appears exactly once in the buffer.
assert len(seqs) == len(set(seqs))
# ---------------------------------------------------------------------------
# Auth/security smoke: the WS endpoint here is unauth'd by design (test
# scaffolding) — but main.py's _ws_auth_ok must remain in place. This
# test pins that contract so a future refactor can't accidentally
# strip it.
# ---------------------------------------------------------------------------
# ---------------------------------------------------------------------------
# Extra stress: terminate happens INSIDE a disconnect window. The
# client must see the terminal event on its next reconnect (whether
# from ring buffer or persisted disk record).
# ---------------------------------------------------------------------------
@pytest.mark.parametrize("trial", range(50))
def test_terminate_during_disconnect_is_observable(trial, _patch_persist_dir):
rng = random.Random(1000 + trial)
app = _build_app(_patch_persist_dir)
sid = f"session-mid-term-{trial}"
n_pre = rng.randint(0, 40)
n_post = rng.randint(0, 40)
seen: dict[int, dict] = {}
last_seq = 0
with TestClient(app) as client:
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": 0, "connection_uuid": "c1"}}))
assert json.loads(ws.receive_text())["event"] == "server:hello"
if n_pre:
_emit(client, sid, n=n_pre, terminate=None)
for _ in range(n_pre):
msg = json.loads(ws.receive_text())
seen[msg["seq"]] = msg
last_seq = max(last_seq, msg["seq"])
# Disconnected. Emit the rest + terminate while WS is gone.
_emit(client, sid, n=n_post, terminate="completed")
# Reconnect. We expect to receive everything from last_seq+1
# through to the terminal — possibly via disk if the buffer
# rolled (it won't here; numbers are small).
with client.websocket_connect(f"/ws/agents/{sid}") as ws:
ws.send_text(json.dumps({"event": "client:hello", "data": {"last_seq": last_seq, "connection_uuid": "c2"}}))
while True:
msg = json.loads(ws.receive_text())
if msg["event"] == "server:hello":
break
if "seq" in msg:
seen[msg["seq"]] = msg
expected = n_pre + n_post + 1
assert len(seen) == expected
seqs = sorted(seen.keys())
assert seqs == list(range(1, expected + 1))
last = seen[expected]
assert last["event"] == "agent:status"
assert last["data"]["status"] == "completed"
# ---------------------------------------------------------------------------
# Sanity: an explicit `WebSocketDisconnect` MUST NOT cancel the
# underlying agent task. We don't have a real agent here, but we can
# at least assert that the ws_manager's disconnect path doesn't touch
# any task registry.
# ---------------------------------------------------------------------------
def test_disconnect_does_not_touch_agent_task(_patch_persist_dir):
"""If a future refactor adds task cancellation to disconnect_session,
this test will catch it. We import agent_manager lazily so the
`tasks` dict starts empty; we register a sentinel task and confirm
disconnect_session doesn't poke it."""
from backend.apps.agents.ws_manager import ws_manager
# Insert a real Future into a parallel registry to mimic
# `agent_manager.tasks[session_id]` and confirm ws_manager
# never reaches into it. We don't import agent_manager (heavy);
# we just inspect the source.
import inspect
src = inspect.getsource(ws_manager.disconnect_session)
assert "cancel" not in src.lower()
assert "agent_manager" not in src
assert "tasks" not in src
def test_main_ws_endpoints_still_gated_by_auth(_patch_persist_dir):
src = open(os.path.join(os.path.dirname(__file__), "..", "main.py")).read()
assert "_ws_auth_ok(websocket)" in src, (
"main.py WS endpoints must still call _ws_auth_ok before accepting "
"the connection — otherwise any local web page can read agent traffic."
)
# And the disconnect handler must NOT call any task-cancel helper
# — that's the regression we're guarding against.
assert "stop_agent" not in src.split("WebSocketDisconnect")[1].split("def ")[0], (
"WebSocketDisconnect handler must not cancel the agent task."
)
+20
View File
@@ -83,6 +83,12 @@ export interface AgentSession {
mcp_suggestions_is_vague?: boolean;
active_outputs?: string[];
compacted_through_msg_id?: string | null;
// Transient frontend-only WS connection state. Independent of
// `status` (which describes the agent run itself). When the WS
// drops we set this to 'reconnecting' so the UI can render a
// subtle indicator without faking a terminal status. Cleared back
// to 'live' on resume_ack. Never persisted to the backend.
connection_state?: 'live' | 'reconnecting';
}
export interface AgentConfig {
@@ -602,6 +608,19 @@ const agentsSlice = createSlice({
}
},
setSessionConnState(
state,
action: PayloadAction<{ sessionId: string; state: 'live' | 'reconnecting' }>
) {
// Transient WS-layer indicator. Decoupled from session.status
// so a network blip never masquerades as a run terminating —
// status keeps reflecting the agent's actual lifecycle.
const session = state.sessions[action.payload.sessionId];
if (session) {
session.connection_state = action.payload.state;
}
},
addMessage(state, action: PayloadAction<{ sessionId: string; message: AgentMessage }>) {
const session = state.sessions[action.payload.sessionId];
if (session) {
@@ -1129,6 +1148,7 @@ export const {
setDraftSystemPrompt,
updateSession,
updateSessionStatus,
setSessionConnState,
addMessage,
streamStart,
streamDelta,
+236 -10
View File
@@ -18,6 +18,8 @@ import {
setActiveBranch,
closeSessionFromWs,
trackAgentNotification,
setSessionConnState,
fetchSession,
} from '../state/agentsSlice';
import { addBrowserCardFromBackend, removeBrowserCard, setBrowserCardPosition, setGlowingBrowserCards, GRID_GAP } from '../state/dashboardLayoutSlice';
import { getAuthToken } from '../config';
@@ -30,23 +32,87 @@ const _getAuthTokenSafe = (): string => {
try { return getAuthToken() || ''; } catch { return ''; }
};
const _genUuid = (): string => {
// Avoid pulling in `crypto.randomUUID` for compat — this is a
// disambiguator, not a security boundary, so a 96-bit hex string is
// plenty.
const a = Math.floor(Math.random() * 2 ** 32).toString(16).padStart(8, '0');
const b = Math.floor(Math.random() * 2 ** 32).toString(16).padStart(8, '0');
const c = Math.floor(Math.random() * 2 ** 32).toString(16).padStart(8, '0');
return `${a}${b}${c}`;
};
type WSEvent = {
event: string;
session_id?: string;
data: Record<string, any>;
seq?: number;
};
interface WSManagerOptions {
skipStreamEvents?: boolean;
// Session-scoped WSes opt into resume + connection-state dispatches
// by passing this. Dashboard WS doesn't.
sessionId?: string;
}
// Heartbeat tuning. 25s is below typical aggressive NAT idle timeouts
// (some enterprise firewalls drop after 30s of silence), and well
// below browser-tab background throttling thresholds. 10s pong
// timeout is a balance: long enough to tolerate flaky cellular RTT
// spikes, short enough that a real dead socket reconnects fast.
const HEARTBEAT_INTERVAL_MS = 25_000;
const HEARTBEAT_TIMEOUT_MS = 10_000;
interface QueuedFrame {
event: string;
data: Record<string, any>;
// Lets the future server-side dedup index match retries to
// originals. Today the server treats most events idempotently
// anyway (stop on stopped is a no-op), but the client sends this
// forward-compatibly so a future server upgrade is safe without a
// protocol bump.
client_msg_id: string;
}
class WebSocketManager {
private ws: WebSocket | null = null;
private url: string;
private skipStreamEvents: boolean;
private sessionId: string | null;
// Resume state. lastSeq is the highest server-assigned seq this
// client has applied; it's sent on every (re)connect so the server
// can replay missed events. Persists for the lifetime of this
// WebSocketManager instance — when the user navigates away and a
// new createSessionWs() is constructed, lastSeq starts at 0 and we
// get a full replay.
private connectionUuid: string;
private lastSeq: number = 0;
private resumeAcked: boolean = false;
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
private reconnectDelay = 1000;
private maxReconnectDelay = 30000;
// Set to true by `disconnect()` so we don't reconnect after an
// explicit close (component unmount / user clicks Close).
private explicitlyClosed: boolean = false;
// Heartbeat. We send a ping on a fixed cadence and arm a timeout
// for the pong; if the timeout fires, we force-close the socket so
// `onclose` triggers reconnect. Detects laptop-sleep / NAT-drop
// silent failures that wouldn't otherwise surface until the next
// outbound send.
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
private pongTimeoutTimer: ReturnType<typeof setTimeout> | null = null;
// Outbound queue. Frames the user enqueues while the WS isn't
// OPEN — or while OPEN but pre-resume-ack — wait here and flush
// after the resume handshake completes. Queue is in-memory only:
// surviving a full app restart isn't worth the localStorage
// complexity given how rare that case is for a transient drop.
private outboundQueue: QueuedFrame[] = [];
private listeners: Map<string, Set<(data: any) => void>> = new Map();
private interpolatorState: Map<string, { sessionId: string; messageId: string; targetText: string; displayedLength: number }> = new Map();
private interpolatorRafId: number | null = null;
@@ -54,6 +120,8 @@ class WebSocketManager {
constructor(url: string, options?: WSManagerOptions) {
this.url = url;
this.skipStreamEvents = options?.skipStreamEvents ?? false;
this.sessionId = options?.sessionId ?? null;
this.connectionUuid = _genUuid();
}
private bufferDelta(sessionId: string, messageId: string, delta: string) {
@@ -115,16 +183,12 @@ class WebSocketManager {
connect() {
if (this.ws?.readyState === WebSocket.OPEN) return;
this.explicitlyClosed = false;
// Append our per-install auth token to the URL. The backend's WS
// handshake validates this before accepting; without it, any
// webpage loaded on the same machine could open a WS and read
// agent traffic. See backend/auth.py + main.py:_ws_auth_ok.
// Token is fetched async from Electron's preload, but we cache it
// after first resolution. If it isn't cached yet, `getAuthToken()`
// returns '' and the connection will be rejected — the
// onclose handler below retries, by which time the token is
// usually loaded.
const token = _getAuthTokenSafe();
const sep = this.url.includes('?') ? '&' : '?';
const urlWithToken = token ? `${this.url}${sep}token=${encodeURIComponent(token)}` : this.url;
@@ -132,6 +196,23 @@ class WebSocketManager {
this.ws.onopen = () => {
this.reconnectDelay = 1000;
this.resumeAcked = false;
this.startHeartbeat();
// Send hello immediately so the server can replay anything the
// server sent that we never applied. On a fresh session,
// last_seq=0 → server replays from buffer start (empty) and
// we proceed normally.
if (this.sessionId) {
this.sendRaw('client:hello', {
session_id: this.sessionId,
connection_uuid: this.connectionUuid,
last_seq: this.lastSeq,
});
} else {
// Dashboard / global WS: no resume, queue can flush right away.
this.resumeAcked = true;
this.flushQueue();
}
};
this.ws.onmessage = (event) => {
@@ -144,25 +225,40 @@ class WebSocketManager {
};
this.ws.onclose = (ev) => {
this.stopHeartbeat();
// 4401 = our backend's auth-failure code. Happens on stale token
// after backend restart (dev hot-reload). Re-fetch from Electron
// IPC before retrying.
if (ev && ev.code === 4401) {
import('@/shared/config').then(mod => mod.refreshAuthToken().catch(() => {}));
}
this.scheduleReconnect();
// Mark UI as reconnecting so the run card shows a clear
// "trying to reconnect" state rather than implying the run
// died. Skipped on an explicit disconnect (user navigated
// away) since there's no run to surface state for.
if (this.sessionId && !this.explicitlyClosed) {
store.dispatch(setSessionConnState({
sessionId: this.sessionId,
state: 'reconnecting',
}));
}
if (!this.explicitlyClosed) this.scheduleReconnect();
};
this.ws.onerror = () => {
// Force the close path to run — onclose will mark state
// reconnecting and schedule a retry.
this.ws?.close();
};
}
disconnect() {
this.explicitlyClosed = true;
if (this.reconnectTimer) {
clearTimeout(this.reconnectTimer);
this.reconnectTimer = null;
}
this.stopHeartbeat();
if (this.interpolatorRafId != null) {
cancelAnimationFrame(this.interpolatorRafId);
this.interpolatorRafId = null;
@@ -174,16 +270,133 @@ class WebSocketManager {
private scheduleReconnect() {
if (this.reconnectTimer) return;
// No retry cap. Long-horizon agent runs may outlast a multi-hour
// network outage (overnight laptop sleep, captive portal limbo);
// giving up would silently desync the UI. Backoff is bounded at
// 30s so the user-visible "Reconnecting…" loop never hammers the
// network, and a small jitter prevents thundering-herd if many
// session WSes reconnect at once after a backend restart.
const jitter = 0.8 + Math.random() * 0.4; // ±20%
const delay = Math.min(this.reconnectDelay, this.maxReconnectDelay) * jitter;
this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = null;
this.reconnectDelay = Math.min(this.reconnectDelay * 2, this.maxReconnectDelay);
this.connect();
}, this.reconnectDelay);
}, delay);
}
private startHeartbeat() {
this.stopHeartbeat();
this.heartbeatTimer = setInterval(() => {
this.sendPing();
}, HEARTBEAT_INTERVAL_MS);
}
private stopHeartbeat() {
if (this.heartbeatTimer != null) {
clearInterval(this.heartbeatTimer);
this.heartbeatTimer = null;
}
if (this.pongTimeoutTimer != null) {
clearTimeout(this.pongTimeoutTimer);
this.pongTimeoutTimer = null;
}
}
private sendPing() {
if (this.ws?.readyState !== WebSocket.OPEN) return;
const nonce = _genUuid();
try {
this.ws.send(JSON.stringify({ event: 'client:ping', data: { nonce } }));
} catch {
// socket dying — let the close handler take over
return;
}
if (this.pongTimeoutTimer != null) clearTimeout(this.pongTimeoutTimer);
this.pongTimeoutTimer = setTimeout(() => {
// Silent death: no pong arrived in time. Force a close so the
// browser's onclose path (and our reconnect) runs immediately
// instead of waiting for the OS TCP keepalive (~75s).
try { this.ws?.close(); } catch { /* nothing */ }
}, HEARTBEAT_TIMEOUT_MS);
}
private clearPongTimeout() {
if (this.pongTimeoutTimer != null) {
clearTimeout(this.pongTimeoutTimer);
this.pongTimeoutTimer = null;
}
}
private flushQueue() {
if (this.ws?.readyState !== WebSocket.OPEN) return;
if (!this.resumeAcked) return;
const queue = this.outboundQueue;
this.outboundQueue = [];
for (const frame of queue) {
try {
this.ws.send(JSON.stringify({ event: frame.event, data: frame.data }));
} catch {
// Re-queue and bail; reconnect will retry.
this.outboundQueue.unshift(frame);
break;
}
}
}
// Direct send that bypasses the queue. Used for hello/ping which
// must NOT be queued (they're connection-scoped, not session-data).
private sendRaw(event: string, data: Record<string, any>) {
if (this.ws?.readyState !== WebSocket.OPEN) return;
try { this.ws.send(JSON.stringify({ event, data })); } catch { /* nothing */ }
}
private handleMessage(msg: WSEvent) {
const { event, session_id, data } = msg;
// Update lastSeq for events that carry one. seq is monotonic per
// session, so this is the high-water mark we send back on resume.
if (typeof msg.seq === 'number' && msg.seq > this.lastSeq) {
this.lastSeq = msg.seq;
}
// ----- Connection-scoped frames (no business-logic side effects) -----
if (event === 'server:pong') {
this.clearPongTimeout();
return;
}
if (event === 'server:hello') {
// Resume handshake completed. The server has either replayed
// missed events (which arrived as separate frames before this
// ack), surfaced a gap, or signalled "you're caught up." Mark
// ourselves live and flush any queued outbound frames.
this.resumeAcked = true;
if (this.sessionId) {
store.dispatch(setSessionConnState({
sessionId: this.sessionId,
state: 'live',
}));
}
this.flushQueue();
return;
}
if (event === 'agent:gap_detected') {
// We were offline long enough that the server's ring buffer
// rolled past our lastSeq. Re-fetch authoritative state via
// REST so the slice's view doesn't have a silent gap.
if (session_id) {
store.dispatch(fetchSession(session_id));
// Reset lastSeq — the REST refetch is the new authoritative
// baseline; subsequent server events with seq numbers will
// re-establish the high-water mark.
this.lastSeq = 0;
}
return;
}
if (this.skipStreamEvents) {
if (event === 'agent:stream_start' || event === 'agent:stream_delta' || event === 'agent:stream_end') {
return;
@@ -419,8 +632,21 @@ class WebSocketManager {
}
send(event: string, data: Record<string, any>) {
if (this.ws?.readyState !== WebSocket.OPEN) return;
this.ws.send(JSON.stringify({ event, data }));
// Queue if the socket isn't open OR resume hasn't been ack'd yet.
// The pre-ack gate prevents an outbound user message from racing
// the resume replay — the server might process the message
// before the replay finishes, leaving the slice's view of
// history incomplete.
const open = this.ws?.readyState === WebSocket.OPEN;
if (!open || !this.resumeAcked) {
this.outboundQueue.push({ event, data, client_msg_id: _genUuid() });
return;
}
try {
this.ws!.send(JSON.stringify({ event, data }));
} catch {
this.outboundQueue.push({ event, data, client_msg_id: _genUuid() });
}
}
sendMessage(
@@ -465,7 +691,7 @@ import { WS_BASE } from '@/shared/config';
export const dashboardWs = new WebSocketManager(`${WS_BASE}/ws/dashboard`, { skipStreamEvents: true });
export function createSessionWs(sessionId: string): WebSocketManager {
return new WebSocketManager(`${WS_BASE}/ws/agents/${sessionId}`);
return new WebSocketManager(`${WS_BASE}/ws/agents/${sessionId}`, { sessionId });
}
export default WebSocketManager;
File diff suppressed because one or more lines are too long