diff --git a/backend/apps/agents/core/ws_manager.py b/backend/apps/agents/core/ws_manager.py index f2719b42..04a63400 100644 --- a/backend/apps/agents/core/ws_manager.py +++ b/backend/apps/agents/core/ws_manager.py @@ -7,6 +7,18 @@ from backend.apps.agents.core.seq_log import TERMINAL_STATUSES, seq_log logger = logging.getLogger(__name__) +# Per-action browser-command timeouts (seconds). A hung tab makes EVERY command +# block to its timeout, so these bound how fast a freeze surfaces. Reads/clicks +# operate on an already-loaded page and should be quick; navigation legitimately +# loads the network so it gets a longer leash. Was a flat 30s, which let one +# wedged page spin for ~20 minutes across retries. +_BROWSER_CMD_TIMEOUT_DEFAULT = 12.0 +_BROWSER_CMD_TIMEOUTS = { + "navigate": 20.0, # a real page load can be slow + "replay_route": 20.0, # an API fetch can be slow + "wait": 12.0, # smart-wait already caps itself well under this +} + class ConnectionManager: """Manages WebSocket connections and HITL approval bridging; events flow through seq_log so reconnects can replay.""" @@ -242,7 +254,14 @@ class ConnectionManager: }) try: - result = await asyncio.wait_for(future, timeout=30.0) + # Bound each command so a wedged tab can't block for 30s (the cost + # that turned one hung LinkedIn page into a 20-minute spin). Navigation + # legitimately takes longer than reads/clicks on an already-loaded page, + # so it gets a longer leash; everything else fails fast. A one-off slow + # command just times out and the next success resets the agent's streak, + # so only a SUSTAINED hang trips the fast-fail abort. + timeout = _BROWSER_CMD_TIMEOUTS.get(action, _BROWSER_CMD_TIMEOUT_DEFAULT) + result = await asyncio.wait_for(future, timeout=timeout) return result except asyncio.TimeoutError: return {"error": "Browser command timed out"} diff --git a/backend/tests/test_browser_command_timeout.py b/backend/tests/test_browser_command_timeout.py new file mode 100644 index 00000000..19dce3ae --- /dev/null +++ b/backend/tests/test_browser_command_timeout.py @@ -0,0 +1,78 @@ +"""Per-action browser-command timeouts. + +A hung tab makes every command block to its timeout; a flat 30s let one wedged +page spin ~20 minutes across retries. These pin that the bound is now short and +per-action, so a freeze surfaces in seconds. +""" + +import asyncio +import time + +import pytest + +from backend.apps.agents.core import ws_manager as wsm + + +class _FakeSock: + async def send_text(self, _): + return None + + +def _mgr(): + m = wsm.ConnectionManager() + m.global_connections = [_FakeSock()] # get past the 'no dashboard' guard + return m + + +def test_timeout_map_reads_are_short_navigation_longer(): + # reads/clicks act on a loaded page -> short; navigation loads network -> longer + assert wsm._BROWSER_CMD_TIMEOUT_DEFAULT <= 15 + assert wsm._BROWSER_CMD_TIMEOUTS["navigate"] <= 25 + assert wsm._BROWSER_CMD_TIMEOUTS["navigate"] > wsm._BROWSER_CMD_TIMEOUT_DEFAULT + # the old flat 30s is gone for the common path + assert wsm._BROWSER_CMD_TIMEOUT_DEFAULT < 30 + + +@pytest.mark.asyncio +async def test_hung_command_returns_fast_at_the_bound(monkeypatch): + # shrink the bounds so the test is quick, then never resolve the future: + # the command must return a timeout error at ~the (default) bound, not hang. + monkeypatch.setattr(wsm, "_BROWSER_CMD_TIMEOUT_DEFAULT", 0.3) + monkeypatch.setattr(wsm, "_BROWSER_CMD_TIMEOUTS", {"navigate": 0.6}) + m = _mgr() + t0 = time.monotonic() + res = await m.send_browser_command("rid1", "get_text", "b1", {}) # never resolved + elapsed = time.monotonic() - t0 + assert res == {"error": "Browser command timed out"} + assert 0.25 < elapsed < 1.0, f"a read should time out near its 0.3s bound, took {elapsed:.2f}s" + + +@pytest.mark.asyncio +async def test_navigate_gets_the_longer_leash(monkeypatch): + monkeypatch.setattr(wsm, "_BROWSER_CMD_TIMEOUT_DEFAULT", 0.3) + monkeypatch.setattr(wsm, "_BROWSER_CMD_TIMEOUTS", {"navigate": 0.7}) + m = _mgr() + t0 = time.monotonic() + await m.send_browser_command("rid2", "navigate", "b1", {"url": "x"}) + elapsed = time.monotonic() - t0 + assert elapsed > 0.5, "navigate should use its longer bound, not the default" + + +@pytest.mark.asyncio +async def test_a_resolved_command_returns_immediately(monkeypatch): + # a healthy command returns the moment the renderer resolves it, not at the bound + monkeypatch.setattr(wsm, "_BROWSER_CMD_TIMEOUT_DEFAULT", 5.0) + m = _mgr() + + async def _resolve_soon(): + await asyncio.sleep(0.05) + # find the pending future and resolve it like the renderer would + rid = next(iter(m.browser_futures)) + m.resolve_browser_command(rid, {"text": "ok", "url": "u"}) + + asyncio.create_task(_resolve_soon()) + t0 = time.monotonic() + res = await m.send_browser_command("rid3", "get_text", "b1", {}) + elapsed = time.monotonic() - t0 + assert res == {"text": "ok", "url": "u"} + assert elapsed < 1.0, "healthy command returns on resolve, not at the timeout"