openswarm/backend/tests/test_frozen_sidecar_bound.py

"""A delegation call must be bounded by something, even when no child is ever born.

Reported by Haik Decie (ENG-402): during a Spotify task the MCP server "froze entirely and had to be
restarted by the harness before the call would return". The cause: `CreateBrowserAgent` is in
P_BLOCKING_TOOLS, which exempts it from the 25s wedge deadline AND therefore from the 300s hard
ceiling ENG-368 added so "a hung per-call thread cannot hang the session forever". Its replacement,
the delegation watchdog, keys on `delegation_children_settled`, which returns False when there are
no children -- correct for a run queued behind the admission cap, and fatal for a sidecar that died
before spawning one. Those two states were indistinguishable, so the second waited forever.

The discriminator is LIVENESS, not time.
"""

from backend.apps.agents.manager.streaming import delegation_watchdog as uw
from backend.apps.agents.manager.streaming import unwedge_sidecar as quick


def test_a_blocking_tool_is_exempt_from_the_quick_deadline():
    # Pinning the premise: if this ever stops being true the bug below cannot happen.
    assert quick.is_quick_core_tool("mcp__openswarm-core__CreateBrowserAgent") is False
    assert uw.is_delegation_core_tool("mcp__openswarm-core__CreateBrowserAgent") is True


def test_a_stale_heartbeat_reads_as_dead(monkeypatch):
    monkeypatch.setattr(uw, "heartbeat_age", lambda sid: uw.HEARTBEAT_FRESH_S + 1, raising=True)
    assert uw.sidecar_is_dead("s1") is True


def test_a_beating_sidecar_is_never_called_dead(monkeypatch):
    # The control that keeps this from killing a run legitimately queued behind the admission cap:
    # that run's sidecar is alive and heartbeating, so it must survive indefinitely.
    monkeypatch.setattr(uw, "heartbeat_age", lambda sid: 0.0, raising=True)
    assert uw.sidecar_is_dead("s1") is False
    monkeypatch.setattr(uw, "heartbeat_age", lambda sid: uw.HEARTBEAT_FRESH_S - 0.1, raising=True)
    assert uw.sidecar_is_dead("s1") is False


def test_the_watchdog_consults_liveness_not_only_settledness():
    src = open("backend/apps/agents/manager/streaming/delegation_watchdog.py").read()
    i_settled = src.index("settled = delegation_children_settled(session_id, started)")
    i_dead = src.index("sidecar_is_dead(session_id)")
    i_streak = src.index('settled_streak["n"] = settled_streak["n"] + 1 if settled else 0')
    assert i_settled < i_dead < i_streak, \
        "the liveness check has to run between the settled test and the streak, or it changes nothing"


def test_no_children_still_means_not_settled_on_its_own():
    # The admission-cap protection this must not break (ENG-327: scoping this wrong shot 39 healthy
    # sidecars in one afternoon). Liveness is an ADDITIONAL door, never a replacement.
    src = open("backend/apps/agents/manager/streaming/delegation_watchdog.py").read()
    assert "if not kids:\n        return False" in src


def test_a_live_child_keeps_the_call_alive_however_quiet_the_sidecar_looks(monkeypatch):
    """The control this fix must not break, scoped the way the watchdog scopes everything else."""
    import time
    from datetime import datetime, timezone
    from backend.apps.agents.core.models import AgentSession
    from backend.apps.agents import agent_manager as am

    started = time.time()
    kid = AgentSession(id="kid-live", name="k", model="sonnet")
    kid.parent_session_id = "par-live"
    kid.mode = "browser-agent"
    kid.created_at = datetime.now(timezone.utc)
    am.agent_manager.sessions["kid-live"] = kid
    try:
        assert uw.no_child_ever_born("par-live", started) is False, \
            "a child born after the call started must block the liveness door entirely"
    finally:
        am.agent_manager.sessions.pop("kid-live", None)


def test_a_child_from_an_EARLIER_delegation_does_not_count(monkeypatch):
    # Same `since` scoping ENG-327 paid 39 dead sidecars to learn: a parent's second delegation must
    # not read its first run's children as this run's.
    import time
    from datetime import datetime, timezone, timedelta
    from backend.apps.agents.core.models import AgentSession
    from backend.apps.agents import agent_manager as am

    old_kid = AgentSession(id="kid-old", name="k", model="sonnet")
    old_kid.parent_session_id = "par-old"
    old_kid.mode = "browser-agent"
    old_kid.created_at = datetime.now(timezone.utc) - timedelta(hours=1)
    am.agent_manager.sessions["kid-old"] = old_kid
    try:
        assert uw.no_child_ever_born("par-old", time.time()) is True
    finally:
        am.agent_manager.sessions.pop("kid-old", None)


def test_a_stopped_agent_is_never_resurrected_by_the_liveness_door(monkeypatch):
    """The regression this fix nearly shipped, caught by Eric asking the right question.

    A browser agent the user STOPS looks identical to a frozen sidecar: no child was born and the
    heartbeat is stale. The human-Stop guard used to live only inside `delegation_children_settled`,
    where it returns False meaning "do not recover" -- and the liveness door then overrode that
    False to True. That is ENG-369 ("stage 3 resent RETRY_PROMPT into the session the user had just
    stopped, every ~150s") coming back through a side door.
    """
    import asyncio
    from backend.apps.agents.core.models import AgentSession
    from backend.apps.agents import agent_manager as am
    from backend.apps.agents.manager.streaming import delegation_watchdog as dw

    fired = {}
    monkeypatch.setattr(dw, "unwedge", lambda *a: fired.setdefault("unwedge", a), raising=True)
    monkeypatch.setattr(dw, "arm_retry", lambda s: fired.setdefault("retry", True), raising=True)
    monkeypatch.setattr(dw, "DELEGATION_CHECK_SECONDS", 0.05, raising=True)
    # The frozen-sidecar condition: dead heartbeat, and no child ever born.
    monkeypatch.setattr(dw, "heartbeat_age", lambda sid: dw.HEARTBEAT_FRESH_S + 99, raising=True)

    sess = AgentSession(id="par-stopped", name="p", model="sonnet")
    sess.ended_by_user = True
    am.agent_manager.sessions["par-stopped"] = sess

    class P_Ctx:
        def __init__(self):
            self.session = sess
            self.session_id = "par-stopped"
            self.tool_start_times = {"tu-1": 0.0}

    async def main():
        dw.arm_delegation_watchdog(P_Ctx(), "tu-1", "mcp__openswarm-core__CreateBrowserAgent")
        await asyncio.sleep(0.4)

    try:
        asyncio.run(main())
    finally:
        am.agent_manager.sessions.pop("par-stopped", None)

    assert "unwedge" not in fired, "a session the user stopped must never be unwedged back to life"
    assert "retry" not in fired, "and must never be handed a retry prompt"


def test_the_human_stop_check_runs_BEFORE_any_settled_predicate():
    # Placement is the fix. Inside a predicate it can be overridden; at the top it cannot.
    src = open("backend/apps/agents/manager/streaming/delegation_watchdog.py").read()
    i_stop = src.index("if p_ended_by_user(session_id):")
    i_settled = src.index("settled = delegation_children_settled(session_id, started)")
    i_door = src.index("sidecar_is_dead(session_id)", i_settled)
    assert i_stop < i_settled < i_door, \
        "a human's Stop has to short-circuit before anything can set settled"