From a588645a965e2750ccaa3c38dfb14f81bc0bd9d4 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 7 Jun 2026 17:37:19 -0700 Subject: [PATCH] [eric] browser: ride out a momentary dashboard WS drop instead of failing the run under load --- backend/apps/agents/browser/browser_agent.py | 10 +++++--- backend/apps/agents/core/ws_manager.py | 26 +++++++++++++++++--- backend/tests/test_browser_fast_path.py | 10 +++++--- 3 files changed, 36 insertions(+), 10 deletions(-) diff --git a/backend/apps/agents/browser/browser_agent.py b/backend/apps/agents/browser/browser_agent.py index 7b3b73d4..c8309e80 100644 --- a/backend/apps/agents/browser/browser_agent.py +++ b/backend/apps/agents/browser/browser_agent.py @@ -76,7 +76,7 @@ from backend.apps.agents.browser.browser_schema import ( SYSTEM_PROMPT, ) from backend.apps.agents.core.models import AgentSession, ApprovalRequest, Message -from backend.apps.agents.core.ws_manager import ws_manager +from backend.apps.agents.core.ws_manager import ws_manager, _await_reconnect from backend.apps.tools_lib.tools_lib import load_builtin_permissions logger = logging.getLogger(__name__) @@ -2032,9 +2032,11 @@ async def run_browser_agents( # No dashboard renderer means every browser command is dead on arrival; # failing here saves the 2-5 LLM turns a sub burns narrating timeouts at a - # corpse before card-gone detection trips. - if not ws_manager.global_connections: - logger.warning("[browser-agent] dispatch refused: no dashboard connected") + # corpse before card-gone detection trips. But a CPU-starved renderer can + # briefly drop its WS then auto-reconnect, so wait (capped) for it to come + # back before refusing, turning a load blip into a pause, not a failed run. + if not ws_manager.global_connections and not await _await_reconnect(lambda: bool(ws_manager.global_connections)): + logger.warning("[browser-agent] dispatch refused: no dashboard after reconnect wait") return [{ "summary": ( "Error: no dashboard window is connected, so browser tools cannot run. " diff --git a/backend/apps/agents/core/ws_manager.py b/backend/apps/agents/core/ws_manager.py index 0f043ca8..0a92e2dd 100644 --- a/backend/apps/agents/core/ws_manager.py +++ b/backend/apps/agents/core/ws_manager.py @@ -12,13 +12,33 @@ logger = logging.getLogger(__name__) # operate on an already-loaded page and should be quick; navigation legitimately # loads the network so it gets a longer leash. Was a flat 30s, which let one # wedged page spin for ~20 minutes across retries. -_BROWSER_CMD_TIMEOUT_DEFAULT = 12.0 +_BROWSER_CMD_TIMEOUT_DEFAULT = 15.0 # modest load headroom; still "short" so a wedged tab fails fast _BROWSER_CMD_TIMEOUTS = { - "navigate": 20.0, # a real page load can be slow + "navigate": 25.0, # a real page load can be slow (more leash under load) "replay_route": 20.0, # an API fetch can be slow "wait": 12.0, # smart-wait already caps itself well under this } _BROWSER_CMD_REBROADCAST_S = 3.0 +# A CPU-starved renderer can briefly drop its WS (a missed heartbeat) and the +# frontend auto-reconnects a beat later; bridge that gap instead of hard-failing +# a live run into it. Short enough that a genuinely-closed window still fails +# quickly (and no LLM turns are ever burned waiting); long enough to ride out a +# reconnect even on a loaded machine. +_WS_RECONNECT_WAIT_S = 8.0 + + +async def _await_reconnect(has_conn) -> bool: + """Poll up to _WS_RECONNECT_WAIT_S for a dashboard socket to (re)appear. + `has_conn` is a 0-arg callable returning truthy when connected.""" + if has_conn(): + return True + waited = 0.0 + while waited < _WS_RECONNECT_WAIT_S: + await asyncio.sleep(0.5) + waited += 0.5 + if has_conn(): + return True + return bool(has_conn()) class ConnectionManager: @@ -240,7 +260,7 @@ class ConnectionManager: self, request_id: str, action: str, browser_id: str, params: dict, tab_id: str = "" ) -> dict: """Send a browser command to the frontend and wait for the result.""" - if not self.global_connections: + if not self.global_connections and not await _await_reconnect(lambda: bool(self.global_connections)): return {"error": "No dashboard is connected. Open the dashboard to use browser tools."} loop = asyncio.get_event_loop() diff --git a/backend/tests/test_browser_fast_path.py b/backend/tests/test_browser_fast_path.py index bdbdf5fa..bb0952db 100644 --- a/backend/tests/test_browser_fast_path.py +++ b/backend/tests/test_browser_fast_path.py @@ -94,12 +94,16 @@ def test_text_normalizes_to_message_without_phone_number(): assert "message r10-os" in _normalize_for_classifier(count) -def test_dispatch_refused_instantly_when_no_dashboard_connected(): +def test_dispatch_refused_when_no_dashboard_connected(monkeypatch): import asyncio from backend.apps.agents.browser.browser_agent import run_browser_agents - from backend.apps.agents.core.ws_manager import ws_manager + from backend.apps.agents.core import ws_manager as wsm - assert not ws_manager.global_connections + # Dispatch now waits briefly for a momentary WS drop to reconnect; with a + # genuinely-closed window that wait just elapses and it still refuses without + # dispatching an agent or burning a turn. Zero the wait so the test is instant. + monkeypatch.setattr(wsm, "_WS_RECONNECT_WAIT_S", 0.0) + assert not wsm.ws_manager.global_connections results = asyncio.run(run_browser_agents(tasks=[{"task": "go to example.com"}], model="sonnet")) assert len(results) == 1 assert results[0]["summary"].startswith("Error: no dashboard window is connected")