From 69569c8079843660ec299d9bc256a4eb84dec4d8 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sun, 16 Aug 2026 19:23:23 -0700 Subject: [PATCH] [eric] agents: delegation watchdog only counts children born after its own tool call, ending the mid-run sidecar shootings (ENG-327) --- .../manager/streaming/unwedge_sidecar.py | 25 ++++++-- backend/apps/help/changelog.py | 1 + backend/tests/test_unwedge_sidecar.py | 62 +++++++++++++++++++ 3 files changed, 82 insertions(+), 6 deletions(-) diff --git a/backend/apps/agents/manager/streaming/unwedge_sidecar.py b/backend/apps/agents/manager/streaming/unwedge_sidecar.py index c04eda05..7adaf3bf 100644 --- a/backend/apps/agents/manager/streaming/unwedge_sidecar.py +++ b/backend/apps/agents/manager/streaming/unwedge_sidecar.py @@ -167,12 +167,25 @@ def is_delegation_core_tool(tool_name: str) -> bool: @typechecked -def delegation_children_settled(session_id: str) -> bool: - """True when this session HAS delegated children and every one of them is terminal. No children - yet is NOT settled: a run queued behind the admission cap can wait minutes legitimately.""" +def delegation_children_settled(session_id: str, since: float) -> bool: + """True when this session has delegated children born AFTER this tool call started and every one + of them is terminal. No children yet is NOT settled: a run queued behind the admission cap can + wait minutes legitimately. The `since` scope is load-bearing: a parent's SECOND delegation used + to read its first run's terminal children as 'settled' while the new run was still queued, and + the watchdog shot a healthy sidecar mid-run (39 kills + 40 force-ended turns in one afternoon + of concurrent load, measured 2026-08-16 on the packaged build).""" from backend.apps.agents.agent_manager import agent_manager - kids = [s for s in agent_manager.sessions.values() - if getattr(s, "parent_session_id", None) == session_id and getattr(s, "mode", "") == "browser-agent"] + kids = [] + for s in agent_manager.sessions.values(): + if getattr(s, "parent_session_id", None) != session_id or getattr(s, "mode", "") != "browser-agent": + continue + born = getattr(s, "created_at", None) + try: + if born is None or born.timestamp() < since: + continue + except Exception: + continue + kids.append(s) if not kids: return False return all(getattr(s, "status", "") in ("completed", "error", "failed", "stopped") for s in kids) @@ -201,7 +214,7 @@ def arm_delegation_watchdog(ctx: object, tool_use_id: str, tool_name: str) -> No if not session_id: return try: - settled = delegation_children_settled(session_id) + settled = delegation_children_settled(session_id, started) except Exception: settled = False settled_streak["n"] = settled_streak["n"] + 1 if settled else 0 diff --git a/backend/apps/help/changelog.py b/backend/apps/help/changelog.py index 4e21277c..c42721ec 100644 --- a/backend/apps/help/changelog.py +++ b/backend/apps/help/changelog.py @@ -98,6 +98,7 @@ P_RELEASES: List[ReleaseNote] = [ "The mouse-wheel Zoom/Scroll setting works on real mice now. Accelerated wheels (Magic Mouse, Logitech smooth scrolling) report fractional scroll amounts that were being mistaken for a trackpad, so the wheel always panned no matter what the setting said.", "A browser helper that talks itself into refusing (\"I\u2019m a text-based AI\") no longer poisons its browser for every later task. Refused and fabricated runs are forgotten instead of remembered, and agents can ask for a completely fresh browser when one misbehaves.", "Agents no longer claim \"the user told me to stop\" when it was OpenSwarm\u2019s own housekeeping talking. Internal wrap-up and retry messages now identify themselves, so an agent\u2019s explanation of why it stopped reflects what actually happened.", + "Agents no longer lose their tools partway through a run. Under heavy multi-agent load, a safety timer could mistake a queued browser or app task for a lost one and restart the agent's tool connection mid-task; it now checks the right task before acting.", "Heavy sessions no longer vanish without a trace. When memory climbs past the safe line the app now sheds weight itself: preview thumbnails pause and refetchable caches drop, instead of growing until the operating system kills it mid-task.", ], ), diff --git a/backend/tests/test_unwedge_sidecar.py b/backend/tests/test_unwedge_sidecar.py index ced5590d..f65a2cd9 100644 --- a/backend/tests/test_unwedge_sidecar.py +++ b/backend/tests/test_unwedge_sidecar.py @@ -132,3 +132,65 @@ def test_a_blocking_tool_arms_nothing_at_all(): arm_wedge_watchdog(ctx, "tu-2", "mcp__openswarm-core__AskUI") assert len(loop._scheduled) == before, "an exempt tool must not even schedule a timer" asyncio.run(run()) + + +# ------------------------------------------------- stale children must never read as settled + + +def p_fake_kid(parent: str, status: str, born_ts: float): + from datetime import datetime + + class Kid: + parent_session_id = parent + mode = "browser-agent" + + k = Kid() + k.status = status + k.created_at = datetime.fromtimestamp(born_ts) + return k + + +def test_stale_terminal_children_do_not_settle_a_new_delegation(monkeypatch): + # The packaged-log failure (2026-08-16): a parent's second AppAgent call queued behind the + # admission cap while its FIRST run's children sat terminal; the old check read that as + # settled and shot a healthy sidecar. Children born before `since` must be invisible. + import time as t + from backend.apps.agents import agent_manager as am_mod + now = t.time() + kids = {"a": p_fake_kid("parent1", "completed", now - 600), + "b": p_fake_kid("parent1", "stopped", now - 300)} + monkeypatch.setattr(am_mod.agent_manager, "sessions", kids) + assert unwedge_sidecar.delegation_children_settled("parent1", since=now - 10) is False + # Negative control: with the scope disabled (since=0 sees every child), the old verdict comes + # back, proving the filter is the thing doing the work. + assert unwedge_sidecar.delegation_children_settled("parent1", since=0.0) is True + + +def test_fresh_terminal_child_settles_and_fresh_running_child_does_not(monkeypatch): + import time as t + from backend.apps.agents import agent_manager as am_mod + now = t.time() + call_started = now - 120 + monkeypatch.setattr(am_mod.agent_manager, "sessions", + {"a": p_fake_kid("parent1", "completed", now - 60)}) + assert unwedge_sidecar.delegation_children_settled("parent1", since=call_started) is True + monkeypatch.setattr(am_mod.agent_manager, "sessions", + {"a": p_fake_kid("parent1", "running", now - 60)}) + assert unwedge_sidecar.delegation_children_settled("parent1", since=call_started) is False + + +def test_no_children_is_not_settled(monkeypatch): + # A run queued behind the admission cap has no child yet; waiting is legitimate. + from backend.apps.agents import agent_manager as am_mod + monkeypatch.setattr(am_mod.agent_manager, "sessions", {}) + assert unwedge_sidecar.delegation_children_settled("parent1", since=0.0) is False + + +def test_mixed_stale_terminal_and_fresh_running_is_not_settled(monkeypatch): + import time as t + from backend.apps.agents import agent_manager as am_mod + now = t.time() + monkeypatch.setattr(am_mod.agent_manager, "sessions", + {"old": p_fake_kid("parent1", "completed", now - 900), + "new": p_fake_kid("parent1", "running", now - 30)}) + assert unwedge_sidecar.delegation_children_settled("parent1", since=now - 60) is False