WEBDIENSTE/openswarm
1
0
Fork 0
mirror of https://github.com/openswarm-ai/openswarm.git synced 2026-09-07 18:27:45 +02:00
Code Issues Packages Projects Releases Wiki Activity

[eric] WS resilience: per-session seq + ring buffer + resume protocol so agent runs survive transient disconnects (wifi flap, sleep, NAT

Browse Source 
drop) instead of flipping to completed. Adds heartbeat (25s ping/10s pong), reconnect with infinite jittered backoff, outbound queue gated
   on resume_ack, gap_detected fallback for long offlines, on-disk persistence of terminal events for post-restart recovery, and a
  reconnecting connection state decoupled from session.status. 1089 backend tests covering 500 randomized disconnect scenarios + concurrent
  broadcast races. Backend/WS handler does not cancel agent task on disconnect
This commit is contained in:
ciregenz
2026-04-28 23:47:59 -04:00
parent 5d69215739
commit 9be2d87f73
7 changed files with 1215 additions and 27 deletions
Download Patch File Download Diff File Expand all files Collapse all files
frontend/src/shared/ws/WebSocketManager.ts
+236 -10
View File
@@ -18,6 +18,8 @@ import {
setActiveBranch,
closeSessionFromWs,
trackAgentNotification,
setSessionConnState,
fetchSession,
} from '../state/agentsSlice';
import { addBrowserCardFromBackend, removeBrowserCard, setBrowserCardPosition, setGlowingBrowserCards, GRID_GAP } from '../state/dashboardLayoutSlice';
import { getAuthToken } from '../config';
@@ -30,23 +32,87 @@ const _getAuthTokenSafe = (): string => {
try { return getAuthToken() || ''; } catch { return ''; }
};
const _genUuid = (): string => {
// Avoid pulling in `crypto.randomUUID` for compat — this is a
// disambiguator, not a security boundary, so a 96-bit hex string is
// plenty.
const a = Math.floor(Math.random() * 2 ** 32).toString(16).padStart(8, '0');
const b = Math.floor(Math.random() * 2 ** 32).toString(16).padStart(8, '0');
const c = Math.floor(Math.random() * 2 ** 32).toString(16).padStart(8, '0');
return `${a}${b}${c}`;
};
type WSEvent = {
event: string;
session_id?: string;
data: Record<string, any>;
seq?: number;
};
interface WSManagerOptions {
skipStreamEvents?: boolean;
// Session-scoped WSes opt into resume + connection-state dispatches
// by passing this. Dashboard WS doesn't.
sessionId?: string;
}
// Heartbeat tuning. 25s is below typical aggressive NAT idle timeouts
// (some enterprise firewalls drop after 30s of silence), and well
// below browser-tab background throttling thresholds. 10s pong
// timeout is a balance: long enough to tolerate flaky cellular RTT
// spikes, short enough that a real dead socket reconnects fast.
const HEARTBEAT_INTERVAL_MS = 25_000;
const HEARTBEAT_TIMEOUT_MS = 10_000;
interface QueuedFrame {
event: string;
data: Record<string, any>;
// Lets the future server-side dedup index match retries to
// originals. Today the server treats most events idempotently
// anyway (stop on stopped is a no-op), but the client sends this
// forward-compatibly so a future server upgrade is safe without a
// protocol bump.
client_msg_id: string;
}
class WebSocketManager {
private ws: WebSocket | null = null;
private url: string;
private skipStreamEvents: boolean;
private sessionId: string | null;
// Resume state. lastSeq is the highest server-assigned seq this
// client has applied; it's sent on every (re)connect so the server
// can replay missed events. Persists for the lifetime of this
// WebSocketManager instance — when the user navigates away and a
// new createSessionWs() is constructed, lastSeq starts at 0 and we
// get a full replay.
private connectionUuid: string;
private lastSeq: number = 0;
private resumeAcked: boolean = false;
private reconnectTimer: ReturnType<typeof setTimeout> | null = null;
private reconnectDelay = 1000;
private maxReconnectDelay = 30000;
// Set to true by `disconnect()` so we don't reconnect after an
// explicit close (component unmount / user clicks Close).
private explicitlyClosed: boolean = false;
// Heartbeat. We send a ping on a fixed cadence and arm a timeout
// for the pong; if the timeout fires, we force-close the socket so
// `onclose` triggers reconnect. Detects laptop-sleep / NAT-drop
// silent failures that wouldn't otherwise surface until the next
// outbound send.
private heartbeatTimer: ReturnType<typeof setInterval> | null = null;
private pongTimeoutTimer: ReturnType<typeof setTimeout> | null = null;
// Outbound queue. Frames the user enqueues while the WS isn't
// OPEN — or while OPEN but pre-resume-ack — wait here and flush
// after the resume handshake completes. Queue is in-memory only:
// surviving a full app restart isn't worth the localStorage
// complexity given how rare that case is for a transient drop.
private outboundQueue: QueuedFrame[] = [];
private listeners: Map<string, Set<(data: any) => void>> = new Map();
private interpolatorState: Map<string, { sessionId: string; messageId: string; targetText: string; displayedLength: number }> = new Map();
private interpolatorRafId: number | null = null;
@@ -54,6 +120,8 @@ class WebSocketManager {
constructor(url: string, options?: WSManagerOptions) {
this.url = url;
this.skipStreamEvents = options?.skipStreamEvents ?? false;
this.sessionId = options?.sessionId ?? null;
this.connectionUuid = _genUuid();
}
private bufferDelta(sessionId: string, messageId: string, delta: string) {
@@ -115,16 +183,12 @@ class WebSocketManager {
connect() {
if (this.ws?.readyState === WebSocket.OPEN) return;
this.explicitlyClosed = false;
// Append our per-install auth token to the URL. The backend's WS
// handshake validates this before accepting; without it, any
// webpage loaded on the same machine could open a WS and read
// agent traffic. See backend/auth.py + main.py:_ws_auth_ok.
// Token is fetched async from Electron's preload, but we cache it
// after first resolution. If it isn't cached yet, `getAuthToken()`
// returns '' and the connection will be rejected — the
// onclose handler below retries, by which time the token is
// usually loaded.
const token = _getAuthTokenSafe();
const sep = this.url.includes('?') ? '&' : '?';
const urlWithToken = token ? `${this.url}${sep}token=${encodeURIComponent(token)}` : this.url;
@@ -132,6 +196,23 @@ class WebSocketManager {
this.ws.onopen = () => {
this.reconnectDelay = 1000;
this.resumeAcked = false;
this.startHeartbeat();
// Send hello immediately so the server can replay anything the
// server sent that we never applied. On a fresh session,
// last_seq=0 → server replays from buffer start (empty) and
// we proceed normally.
if (this.sessionId) {
this.sendRaw('client:hello', {
session_id: this.sessionId,
connection_uuid: this.connectionUuid,
last_seq: this.lastSeq,
});
} else {
// Dashboard / global WS: no resume, queue can flush right away.
this.resumeAcked = true;
this.flushQueue();
}
};
this.ws.onmessage = (event) => {
@@ -144,25 +225,40 @@ class WebSocketManager {
};
this.ws.onclose = (ev) => {
this.stopHeartbeat();
// 4401 = our backend's auth-failure code. Happens on stale token
// after backend restart (dev hot-reload). Re-fetch from Electron
// IPC before retrying.
if (ev && ev.code === 4401) {
import('@/shared/config').then(mod => mod.refreshAuthToken().catch(() => {}));
}
this.scheduleReconnect();
// Mark UI as reconnecting so the run card shows a clear
// "trying to reconnect" state rather than implying the run
// died. Skipped on an explicit disconnect (user navigated
// away) since there's no run to surface state for.
if (this.sessionId && !this.explicitlyClosed) {
store.dispatch(setSessionConnState({
sessionId: this.sessionId,
state: 'reconnecting',
}));
}
if (!this.explicitlyClosed) this.scheduleReconnect();
};
this.ws.onerror = () => {
// Force the close path to run — onclose will mark state
// reconnecting and schedule a retry.
this.ws?.close();
};
}
disconnect() {
this.explicitlyClosed = true;
if (this.reconnectTimer) {
clearTimeout(this.reconnectTimer);
this.reconnectTimer = null;
}
this.stopHeartbeat();
if (this.interpolatorRafId != null) {
cancelAnimationFrame(this.interpolatorRafId);
this.interpolatorRafId = null;
@@ -174,16 +270,133 @@ class WebSocketManager {
private scheduleReconnect() {
if (this.reconnectTimer) return;
// No retry cap. Long-horizon agent runs may outlast a multi-hour
// network outage (overnight laptop sleep, captive portal limbo);
// giving up would silently desync the UI. Backoff is bounded at
// 30s so the user-visible "Reconnecting…" loop never hammers the
// network, and a small jitter prevents thundering-herd if many
// session WSes reconnect at once after a backend restart.
const jitter = 0.8 + Math.random() * 0.4; // ±20%
const delay = Math.min(this.reconnectDelay, this.maxReconnectDelay) * jitter;
this.reconnectTimer = setTimeout(() => {
this.reconnectTimer = null;
this.reconnectDelay = Math.min(this.reconnectDelay * 2, this.maxReconnectDelay);
this.connect();
}, this.reconnectDelay);
}, delay);
}
private startHeartbeat() {
this.stopHeartbeat();
this.heartbeatTimer = setInterval(() => {
this.sendPing();
}, HEARTBEAT_INTERVAL_MS);
}
private stopHeartbeat() {
if (this.heartbeatTimer != null) {
clearInterval(this.heartbeatTimer);
this.heartbeatTimer = null;
}
if (this.pongTimeoutTimer != null) {
clearTimeout(this.pongTimeoutTimer);
this.pongTimeoutTimer = null;
}
}
private sendPing() {
if (this.ws?.readyState !== WebSocket.OPEN) return;
const nonce = _genUuid();
try {
this.ws.send(JSON.stringify({ event: 'client:ping', data: { nonce } }));
} catch {
// socket dying — let the close handler take over
return;
}
if (this.pongTimeoutTimer != null) clearTimeout(this.pongTimeoutTimer);
this.pongTimeoutTimer = setTimeout(() => {
// Silent death: no pong arrived in time. Force a close so the
// browser's onclose path (and our reconnect) runs immediately
// instead of waiting for the OS TCP keepalive (~75s).
try { this.ws?.close(); } catch { /* nothing */ }
}, HEARTBEAT_TIMEOUT_MS);
}
private clearPongTimeout() {
if (this.pongTimeoutTimer != null) {
clearTimeout(this.pongTimeoutTimer);
this.pongTimeoutTimer = null;
}
}
private flushQueue() {
if (this.ws?.readyState !== WebSocket.OPEN) return;
if (!this.resumeAcked) return;
const queue = this.outboundQueue;
this.outboundQueue = [];
for (const frame of queue) {
try {
this.ws.send(JSON.stringify({ event: frame.event, data: frame.data }));
} catch {
// Re-queue and bail; reconnect will retry.
this.outboundQueue.unshift(frame);
break;
}
}
}
// Direct send that bypasses the queue. Used for hello/ping which
// must NOT be queued (they're connection-scoped, not session-data).
private sendRaw(event: string, data: Record<string, any>) {
if (this.ws?.readyState !== WebSocket.OPEN) return;
try { this.ws.send(JSON.stringify({ event, data })); } catch { /* nothing */ }
}
private handleMessage(msg: WSEvent) {
const { event, session_id, data } = msg;
// Update lastSeq for events that carry one. seq is monotonic per
// session, so this is the high-water mark we send back on resume.
if (typeof msg.seq === 'number' && msg.seq > this.lastSeq) {
this.lastSeq = msg.seq;
}
// ----- Connection-scoped frames (no business-logic side effects) -----
if (event === 'server:pong') {
this.clearPongTimeout();
return;
}
if (event === 'server:hello') {
// Resume handshake completed. The server has either replayed
// missed events (which arrived as separate frames before this
// ack), surfaced a gap, or signalled "you're caught up." Mark
// ourselves live and flush any queued outbound frames.
this.resumeAcked = true;
if (this.sessionId) {
store.dispatch(setSessionConnState({
sessionId: this.sessionId,
state: 'live',
}));
}
this.flushQueue();
return;
}
if (event === 'agent:gap_detected') {
// We were offline long enough that the server's ring buffer
// rolled past our lastSeq. Re-fetch authoritative state via
// REST so the slice's view doesn't have a silent gap.
if (session_id) {
store.dispatch(fetchSession(session_id));
// Reset lastSeq — the REST refetch is the new authoritative
// baseline; subsequent server events with seq numbers will
// re-establish the high-water mark.
this.lastSeq = 0;
}
return;
}
if (this.skipStreamEvents) {
if (event === 'agent:stream_start' || event === 'agent:stream_delta' || event === 'agent:stream_end') {
return;
@@ -419,8 +632,21 @@ class WebSocketManager {
}
send(event: string, data: Record<string, any>) {
if (this.ws?.readyState !== WebSocket.OPEN) return;
this.ws.send(JSON.stringify({ event, data }));
// Queue if the socket isn't open OR resume hasn't been ack'd yet.
// The pre-ack gate prevents an outbound user message from racing
// the resume replay — the server might process the message
// before the replay finishes, leaving the slice's view of
// history incomplete.
const open = this.ws?.readyState === WebSocket.OPEN;
if (!open || !this.resumeAcked) {
this.outboundQueue.push({ event, data, client_msg_id: _genUuid() });
return;
}
try {
this.ws!.send(JSON.stringify({ event, data }));
} catch {
this.outboundQueue.push({ event, data, client_msg_id: _genUuid() });
}
}
sendMessage(
@@ -465,7 +691,7 @@ import { WS_BASE } from '@/shared/config';
export const dashboardWs = new WebSocketManager(`${WS_BASE}/ws/dashboard`, { skipStreamEvents: true });
export function createSessionWs(sessionId: string): WebSocketManager {
return new WebSocketManager(`${WS_BASE}/ws/agents/${sessionId}`);
return new WebSocketManager(`${WS_BASE}/ws/agents/${sessionId}`, { sessionId });
}
export default WebSocketManager;