openswarm/backend/apps/agents/core/ws_manager.py

import asyncio
import json
import logging
from typing import Optional
from fastapi import WebSocket

from backend.apps.agents.core.seq_log import TERMINAL_STATUSES, seq_log

logger = logging.getLogger(__name__)

# Per-action browser-command timeouts (seconds). A hung tab makes EVERY command block to its timeout, so these bound how fast a freeze surfaces. Reads/clicks operate on an already-loaded page and should be quick; navigation legitimately loads the network so it gets a longer leash. Was a flat 30s, which let one wedged page spin for ~20 minutes across retries.
BROWSER_CMD_TIMEOUT_DEFAULT = 15.0   # modest load headroom; still "short" so a wedged tab fails fast
BROWSER_CMD_TIMEOUTS = {
    "navigate": 25.0,     # a real page load can be slow (more leash under load)
    "replay_route": 20.0, # an API fetch can be slow
    "wait": 12.0,         # smart-wait already caps itself well under this
}
BROWSER_CMD_REBROADCAST_S = 3.0
# A CPU-starved renderer can briefly drop its WS (a missed heartbeat) and the frontend auto-reconnects a beat later; bridge that gap instead of hard-failing a live run into it. Short enough that a genuinely-closed window still fails quickly (and no LLM turns are ever burned waiting); long enough to ride out a reconnect even on a loaded machine.
P_WS_RECONNECT_WAIT_S = 8.0


async def await_reconnect(has_conn) -> bool:
    """Poll up to P_WS_RECONNECT_WAIT_S for a dashboard socket to (re)appear.
    `has_conn` is a 0-arg callable returning truthy when connected."""
    if has_conn():
        return True
    waited = 0.0
    while waited < P_WS_RECONNECT_WAIT_S:
        await asyncio.sleep(0.5)
        waited += 0.5
        if has_conn():
            return True
    return bool(has_conn())


class ConnectionManager:
    """Manages WebSocket connections and HITL approval bridging; events flow through seq_log so reconnects can replay."""

    def __init__(self):
        self.connections: dict[str, list[WebSocket]] = {}
        self.global_connections: list[WebSocket] = []
        # Which dashboard each global socket is currently showing, keyed by id(websocket). active_dashboard_id is the last one activated (the window the user is looking at most recently); a scheduled run targets it so its browser card spawns where the renderer can render it.
        self.global_dashboard_ids: dict[int, str] = {}
        self.active_dashboard_id: Optional[str] = None
        self.pending_futures: dict[str, asyncio.Future] = {}
        self.browser_futures: dict[str, asyncio.Future] = {}

    async def connect_session(self, session_id: str, websocket: WebSocket):
        await websocket.accept()
        if session_id not in self.connections:
            self.connections[session_id] = []
        self.connections[session_id].append(websocket)

    async def connect_global(self, websocket: WebSocket):
        await websocket.accept()
        self.global_connections.append(websocket)

    def disconnect_session(self, session_id: str, websocket: WebSocket):
        if session_id in self.connections:
            self.connections[session_id] = [
                ws for ws in self.connections[session_id] if ws != websocket
            ]
            if not self.connections[session_id]:
                del self.connections[session_id]

    def set_active_dashboard(self, websocket: WebSocket, dashboard_id: str):
        """Record which dashboard a renderer is showing; last activation wins."""
        self.global_dashboard_ids[id(websocket)] = dashboard_id
        self.active_dashboard_id = dashboard_id

    def disconnect_global(self, websocket: WebSocket):
        self.global_connections = [
            ws for ws in self.global_connections if ws != websocket
        ]
        # Drop this socket's active-dashboard pointer; if it owned the global one, fall back to any window still connected so a closed tab doesn't leave a stale target.
        self.global_dashboard_ids.pop(id(websocket), None)
        if self.active_dashboard_id not in self.global_dashboard_ids.values():
            self.active_dashboard_id = next(iter(self.global_dashboard_ids.values()), None)

    async def send_to_session(self, session_id: str, event: str, data: dict):
        """Broadcast a session event with monotonic sequencing; terminal statuses also persist to disk."""
        async with seq_log.stamp(session_id, event, data) as (seq, payload_str):
            for ws in list(self.connections.get(session_id, [])):
                try:
                    await ws.send_text(payload_str)
                except Exception:
                    logger.debug("send_to_session: send failed (will retry on reconnect)", exc_info=True)
            for ws in list(self.global_connections):
                try:
                    await ws.send_text(payload_str)
                except Exception:
                    logger.debug("send_to_session: global send failed", exc_info=True)
            # Persist under the lock so a concurrent running status can't race past and overwrite with stale state.
            if event == "agent:status" and data.get("status") in TERMINAL_STATUSES:
                seq_log.persist_terminal(session_id, payload_str)

        # Outside the stamp lock so analytics can't gate the broadcast; replays go via ws.send_text, so reconnects don't double-count.
        if event == "agent:message":
            try:
                from backend.apps.service.analytics.agent_bridge import bridge_agent_message, BroadcastMessage
                bridge_agent_message(session_id, BroadcastMessage.model_validate(data.get("message") or {}))
            except Exception:
                logger.debug("agent:message analytics bridge failed", exc_info=True)

    async def replay_to(
        self, session_id: str, websocket: WebSocket, last_seq: int
    ) -> dict:
        """Replay buffered events with seq > last_seq; returns ack envelope for the resume handshake."""
        oldest, newest, events = seq_log.replay(session_id, last_seq)

        # Gap-check first: if last_seq predates the buffer, signal REST-refresh; last_seq=0 means fresh client (full replay).
        if last_seq > 0 and oldest is not None and last_seq < oldest - 1:
            gap_payload = json.dumps({
                "event": "agent:gap_detected",
                "session_id": session_id,
                "data": {
                    "session_id": session_id,
                    "oldest_seq": oldest,
                    "newest_seq": newest,
                    "client_seq": last_seq,
                },
            })
            try:
                await websocket.send_text(gap_payload)
            except Exception:
                pass
            return {
                "ok": False,
                "reason": "gap",
                "oldest_seq": oldest,
                "newest_seq": newest,
            }

        if events:
            # Drop already-resolved approval requests from the replay. The ring buffer holds every event we ever stamped, including the original `agent:approval_request`. Without this filter, a client that reconnects (e.g. after navigating away and back, which re-mounts AgentChat with last_seq=0) re-fires every past approval as if it were live, but the backing future was popped from pending_futures the moment the user answered, so the resurrected card is a dead no-op. Lifecycle is simple: send_approval_request() inserts into pending_futures BEFORE the event is stamped, and resolve_approval()/timeout/cancel all pop it; so "in pending_futures" is the authoritative is-still-live signal for the request_id. A process restart wipes pending_futures, which is correct because reconcile_on_startup also marks waiting_approval sessions as stopped so there's nothing to answer anyway.
            events = self.p_filter_stale_approvals(events)
            events = self.p_strip_replayed_closes(events)
            for s in events:
                try:
                    await websocket.send_text(s)
                except Exception:
                    logger.debug("replay_to: send failed", exc_info=True)
                    break
            return {
                "ok": True,
                "replayed": len(events),
                "from_seq": last_seq,
                "to_seq": newest,
            }

        terminal = seq_log.load_terminal(session_id)
        if terminal is not None:
            try:
                await websocket.send_text(terminal)
            except Exception:
                pass
            return {"ok": True, "replayed": 1, "terminal_only": True}

        return {
            "ok": True,
            "replayed": 0,
            "current_seq": newest if newest is not None else 0,
        }

    def p_strip_replayed_closes(self, events: list[str]) -> list[str]:
        """Drop `agent:closed` events from a replay buffer.

        agent:closed is a transition event ("session JUST closed") whose
        frontend reducer (closeSessionFromWs) destructively deletes the
        session from state.sessions. Replaying it on a fresh client (e.g.
        a user who just clicked the closed chat in history) deletes the
        session they're trying to open. The current closed state is
        already conveyed by the REST hydrate (status=stopped, closed_at
        set) and by the latest agent:status event in the replay, so
        suppressing the transition replay is non-lossy.
        """
        out: list[str] = []
        for payload_str in events:
            try:
                parsed = json.loads(payload_str)
            except (ValueError, TypeError):
                out.append(payload_str)
                continue
            if parsed.get("event") == "agent:closed":
                continue
            out.append(payload_str)
        return out

    def p_filter_stale_approvals(self, events: list[str]) -> list[str]:
        """Return events minus any `agent:approval_request` whose request_id
        is no longer in pending_futures. JSON parse is per-event but replay
        only runs on (re)connect, so it isn't a hot path.
        """
        alive = self.pending_futures
        out: list[str] = []
        for payload_str in events:
            try:
                parsed = json.loads(payload_str)
            except (ValueError, TypeError):
                out.append(payload_str)
                continue
            if parsed.get("event") != "agent:approval_request":
                out.append(payload_str)
                continue
            data = parsed.get("data") or {}
            request_id = data.get("request_id")
            if request_id and request_id in alive:
                out.append(payload_str)
        return out

    async def broadcast_global(self, event: str, data: dict):
        """Send to all dashboard connections; bypasses seq_log (dashboard resumes via full state refetch)."""
        payload = json.dumps({"event": event, "data": data})
        for ws in list(self.global_connections):
            try:
                await ws.send_text(payload)
            except Exception:
                pass

    async def send_approval_request(
        self, session_id: str, request_id: str, tool_name: str, tool_input: dict,
        timeout: float = 600.0,
        sensitive_pattern: str | None = None,
        sensitive_label: str | None = None,
        sensitive_why: str | None = None,
    ) -> dict:
        """Send an approval request and wait for the user's decision; 10-minute timeout prevents permanent park."""
        future = asyncio.get_event_loop().create_future()
        self.pending_futures[request_id] = future

        payload: dict = {
            "request_id": request_id,
            "tool_name": tool_name,
            "tool_input": tool_input,
        }
        if sensitive_pattern:
            payload["sensitive_pattern"] = sensitive_pattern
            payload["sensitive_label"] = sensitive_label
            payload["sensitive_why"] = sensitive_why
        await self.send_to_session(session_id, "agent:approval_request", payload)

        try:
            result = await asyncio.wait_for(future, timeout=timeout)
            return result
        except asyncio.TimeoutError:
            logger.warning("Approval %s for session %s timed out after %ss", request_id, session_id, timeout)
            return {"behavior": "deny", "message": "Approval timed out"}
        finally:
            self.pending_futures.pop(request_id, None)

    def resolve_approval(self, request_id: str, decision: dict):
        """Resolve a pending approval Future with the user's decision."""
        future = self.pending_futures.get(request_id)
        if future and not future.done():
            future.set_result(decision)

    async def send_browser_command(
        self, request_id: str, action: str, browser_id: str, params: dict, tab_id: str = ""
    ) -> dict:
        """Send a browser command to the frontend and wait for the result."""
        if not self.global_connections and not await await_reconnect(lambda: bool(self.global_connections)):
            return {"error": "No dashboard is connected. Open the dashboard to use browser tools."}

        loop = asyncio.get_event_loop()
        future = loop.create_future()
        self.browser_futures[request_id] = future

        payload = {
            "request_id": request_id,
            "action": action,
            "browser_id": browser_id,
            "tab_id": tab_id,
            "params": params,
        }

        try:
            # Bound each command so a wedged tab can't block for 30s (the cost that turned one hung LinkedIn page into a 20-minute spin). Navigation legitimately takes longer than reads/clicks on an already-loaded page, so it gets a longer leash; everything else fails fast. A one-off slow command just times out and the next success resets the agent's streak, so only a SUSTAINED hang trips the fast-fail abort.
            timeout = BROWSER_CMD_TIMEOUTS.get(action, BROWSER_CMD_TIMEOUT_DEFAULT)
            deadline = loop.time() + timeout
            # Re-broadcast until a client answers: a silently-dead dashboard socket takes up to ~35s of heartbeat to notice, and a command sent into that gap is lost forever (broadcast skips seq_log). The renderer dedupes by request_id so re-sends can't double-act.
            while True:
                await self.broadcast_global("browser:command", payload)
                remaining = deadline - loop.time()
                if remaining <= 0:
                    return {"error": "Browser command timed out"}
                done, _ = await asyncio.wait(
                    {future}, timeout=min(BROWSER_CMD_REBROADCAST_S, remaining)
                )
                if done:
                    return future.result()
        finally:
            self.browser_futures.pop(request_id, None)

    def resolve_browser_command(self, request_id: str, result: dict):
        """Resolve a pending browser command Future with the frontend's result."""
        future = self.browser_futures.get(request_id)
        if future and not future.done():
            future.set_result(result)


ws_manager = ConnectionManager()