diff --git a/backend/apps/agents/manager/permissions/gate_hooks.py b/backend/apps/agents/manager/permissions/gate_hooks.py index 478c08e5..5e4e40c0 100644 --- a/backend/apps/agents/manager/permissions/gate_hooks.py +++ b/backend/apps/agents/manager/permissions/gate_hooks.py @@ -23,7 +23,7 @@ from backend.apps.agents.core.ws_manager import ws_manager from backend.apps.settings.settings import load_settings from backend.apps.agents.manager.permissions import path_gate from backend.apps.agents.manager.permissions.decision import effective_policy -from backend.apps.agents.manager.streaming.unwedge_sidecar import arm_wedge_watchdog +from backend.apps.agents.manager.streaming.unwedge_sidecar import arm_delegation_watchdog, arm_wedge_watchdog from backend.apps.agents.manager.prompt.tool_catalog import gated_mcp_server_names from backend.apps.agents.manager.prompt.prompt_context import ( TOOLSEARCH_LOOP_THRESHOLD, @@ -198,4 +198,5 @@ async def pre_tool_hook(ctx: HookContext, input_data: dict, tool_use_id: Optiona ctx.tool_start_times[tool_use_id] = time.time() # A frozen sidecar hangs a quick core tool forever; the watchdog turns that into the self-healing death (ENG-303). arm_wedge_watchdog(ctx, tool_use_id, tool_name) + arm_delegation_watchdog(ctx, tool_use_id, tool_name) return {} diff --git a/backend/apps/agents/manager/streaming/unwedge_sidecar.py b/backend/apps/agents/manager/streaming/unwedge_sidecar.py index ece8944e..23298fea 100644 --- a/backend/apps/agents/manager/streaming/unwedge_sidecar.py +++ b/backend/apps/agents/manager/streaming/unwedge_sidecar.py @@ -148,3 +148,71 @@ def arm_wedge_watchdog(ctx: object, tool_use_id: str, tool_name: str) -> None: arm_retry(getattr(ctx, "session", None)) loop.call_later(WEDGE_SECONDS, p_check) + + +# Delegation tools legitimately run for minutes, so they are exempt from the 25s deadline above. +# The exemption assumed the child's result always comes home; measured 2026-08-15 on a packaged +# build, a CreateBrowserAgent child COMPLETED (backend returned its HTTP 200, sidecar went back to +# readline) while the parent hung on the outstanding tool call for 20+ minutes: the result died on +# the sidecar->CLI stdio hop and nothing above it has a deadline. When every child is terminal and +# stays terminal across two consecutive checks, the wait is provably pointless; recover the same +# way the quick class does. +P_DELEGATION_TOOLS: Set[str] = {"CreateBrowserAgent", "BrowserAgent", "BrowserAgents", "AppAgent"} +DELEGATION_CHECK_SECONDS = 75.0 + + +@typechecked +def is_delegation_core_tool(tool_name: str) -> bool: + return tool_name.startswith(P_CORE_PREFIX) and tool_name[len(P_CORE_PREFIX):] in P_DELEGATION_TOOLS + + +@typechecked +def delegation_children_settled(session_id: str) -> bool: + """True when this session HAS delegated children and every one of them is terminal. No children + yet is NOT settled: a run queued behind the admission cap can wait minutes legitimately.""" + from backend.apps.agents.agent_manager import agent_manager + kids = [s for s in agent_manager.sessions.values() + if getattr(s, "parent_session_id", None) == session_id and getattr(s, "mode", "") == "browser-agent"] + if not kids: + return False + return all(getattr(s, "status", "") in ("completed", "error", "failed", "stopped") for s in kids) + + +@typechecked +def arm_delegation_watchdog(ctx: object, tool_use_id: str, tool_name: str) -> None: + """Recurring, slow-cadence sibling of arm_wedge_watchdog for delegation tools. Fires the same + unwedge+retry only after TWO consecutive checks (>=75s apart) see every child terminal while + the tool call is still outstanding, so a child that is merely slow can never trip it.""" + if not is_delegation_core_tool(tool_name): + return + try: + loop = asyncio.get_running_loop() + except RuntimeError: + return + + started = time.time() + settled_streak = {"n": 0} + + def p_check() -> None: + times = getattr(ctx, "tool_start_times", None) + if not isinstance(times, dict) or tool_use_id not in times: + return + session_id = getattr(ctx, "session_id", "") + if not session_id: + return + try: + settled = delegation_children_settled(session_id) + except Exception: + settled = False + settled_streak["n"] = settled_streak["n"] + 1 if settled else 0 + if settled_streak["n"] >= 2: + logger.warning( + "delegation result lost: %s outstanding %.0fs on session %s with every child terminal; recovering", + tool_name, time.time() - started, session_id[:8], + ) + loop.run_in_executor(None, unwedge, session_id, tool_name, time.time() - started) + arm_retry(getattr(ctx, "session", None)) + return + loop.call_later(DELEGATION_CHECK_SECONDS, p_check) + + loop.call_later(DELEGATION_CHECK_SECONDS, p_check) diff --git a/backend/tests/test_delegation_result_backstop.py b/backend/tests/test_delegation_result_backstop.py new file mode 100644 index 00000000..13e8417f --- /dev/null +++ b/backend/tests/test_delegation_result_backstop.py @@ -0,0 +1,92 @@ +"""A delegated run whose result is lost cannot hang the parent forever (live specimen 2026-08-15). + +CreateBrowserAgent is exempt from the 25s unwedge because a browser run legitimately takes minutes. +The exemption assumed results always come home; a packaged-build stress run produced the miss: the +child COMPLETED (backend logged the run summary and answered the sidecar's HTTP call with 200, the +sidecar returned to readline), and the parent sat 'running' on the outstanding tool call for 20+ +minutes. These pin the backstop: two consecutive all-children-terminal observations while the call +is still outstanding -> the same unwedge+retry recovery the quick class already has. +""" +import asyncio +from typing import Dict + +import pytest + +from backend.apps.agents.core.models import AgentSession +from backend.apps.agents.manager.streaming import unwedge_sidecar as u + + +def test_delegation_classifier_is_exact(): + assert u.is_delegation_core_tool("mcp__openswarm-core__CreateBrowserAgent") + assert u.is_delegation_core_tool("mcp__openswarm-core__AppAgent") + assert not u.is_delegation_core_tool("mcp__openswarm-core__MemoryWrite"), "quick class stays on the 25s path" + assert not u.is_delegation_core_tool("mcp__openswarm-core__AskUI"), "a human answering is not a lost result" + assert not u.is_delegation_core_tool("CreateBrowserAgent"), "only core-prefixed names" + + +def test_settled_requires_children_and_all_terminal(monkeypatch): + from backend.apps.agents import agent_manager as am + + parent = AgentSession(id="par-1", name="p", model="sonnet") + running_child = AgentSession(id="kid-1", name="k1", model="sonnet", mode="browser-agent", parent_session_id="par-1") + running_child.status = "running" + done_child = AgentSession(id="kid-2", name="k2", model="sonnet", mode="browser-agent", parent_session_id="par-1") + done_child.status = "completed" + + monkeypatch.setattr(am.agent_manager, "sessions", {"par-1": parent}, raising=False) + assert u.delegation_children_settled("par-1") is False, "no children = maybe queued behind admission, NOT settled" + + monkeypatch.setattr(am.agent_manager, "sessions", {"par-1": parent, "kid-1": running_child, "kid-2": done_child}, raising=False) + assert u.delegation_children_settled("par-1") is False, "one live child means the wait is legitimate" + + running_child.status = "completed" + assert u.delegation_children_settled("par-1") is True + + +class P_Ctx: + def __init__(self, session: AgentSession, times: Dict[str, float]): + self.session = session + self.session_id = session.id + self.tool_start_times = times + + +@pytest.mark.asyncio +async def test_two_settled_checks_fire_the_recovery(monkeypatch): + fired = {} + monkeypatch.setattr(u, "unwedge", lambda sid, tool, age: fired.setdefault("unwedge", (sid, tool))) + monkeypatch.setattr(u, "arm_retry", lambda s: fired.setdefault("retry", s.id if s else None)) + monkeypatch.setattr(u, "delegation_children_settled", lambda sid: True) + monkeypatch.setattr(u, "DELEGATION_CHECK_SECONDS", 0.05) + + sess = AgentSession(id="par-2", name="p", model="sonnet") + ctx = P_Ctx(sess, {"tu-1": 0.0}) + u.arm_delegation_watchdog(ctx, "tu-1", "mcp__openswarm-core__CreateBrowserAgent") + await asyncio.sleep(0.3) + assert fired.get("unwedge", (None, None))[0] == "par-2", "two settled checks must recover the parent" + assert fired.get("retry") == "par-2", "the retry is what redoes the lost step" + + +@pytest.mark.asyncio +async def test_a_finished_call_disarms_and_a_live_child_resets_the_streak(monkeypatch): + fired = {} + monkeypatch.setattr(u, "unwedge", lambda *a: fired.setdefault("unwedge", a)) + monkeypatch.setattr(u, "arm_retry", lambda s: fired.setdefault("retry", True)) + monkeypatch.setattr(u, "DELEGATION_CHECK_SECONDS", 0.05) + + # Finished call: the post hook popped the id, so the first check returns silently. + sess = AgentSession(id="par-3", name="p", model="sonnet") + u.arm_delegation_watchdog(P_Ctx(sess, {}), "tu-gone", "mcp__openswarm-core__CreateBrowserAgent") + + # Oscillating child (settles once, then a new child appears): the streak must reset, never fire. + seq = iter([True, False, True, False, True, False]) + monkeypatch.setattr(u, "delegation_children_settled", lambda sid: next(seq, False)) + u.arm_delegation_watchdog(P_Ctx(sess, {"tu-2": 0.0}), "tu-2", "mcp__openswarm-core__CreateBrowserAgent") + await asyncio.sleep(0.4) + assert "unwedge" not in fired, "a merely-slow delegation must never be recovered out from under itself" + + +def test_quick_tools_never_arm_the_delegation_watchdog(): + # Belt and suspenders: arming for a quick tool would double-recover with the 25s path. + sess = AgentSession(id="par-4", name="p", model="sonnet") + u.arm_delegation_watchdog(P_Ctx(sess, {"tu-3": 0.0}), "tu-3", "mcp__openswarm-core__MemoryRead") + # No loop assertions needed: is_delegation_core_tool returned False, nothing scheduled.