mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-22 01:24:52 +02:00
238 lines
9.1 KiB
Python
238 lines
9.1 KiB
Python
"""9Router resilience: the watchdog revives a dead router (backing off on repeated failure and
|
|
dying with stop()), and provider DETECTION revives before concluding "no provider" — gated on
|
|
evidence so a zero-config user never boots a router with nothing to route."""
|
|
|
|
import asyncio
|
|
import json
|
|
from unittest.mock import patch
|
|
|
|
|
|
import backend.apps.nine_router.process as proc
|
|
from backend.apps.settings.models import AppSettings
|
|
|
|
|
|
def test_watchdog_revives_then_backs_off():
|
|
async def run():
|
|
sleeps: list = []
|
|
ensures: list = []
|
|
|
|
async def fake_sleep(d):
|
|
sleeps.append(d)
|
|
await real_sleep(0)
|
|
|
|
async def fake_ensure():
|
|
ensures.append(1)
|
|
|
|
real_sleep = asyncio.sleep
|
|
with patch.object(proc, "is_running", return_value=False), \
|
|
patch.object(proc, "ensure_running", fake_ensure), \
|
|
patch.object(proc.asyncio, "sleep", fake_sleep):
|
|
task = asyncio.get_running_loop().create_task(proc.watchdog_loop())
|
|
while len(sleeps) < 9:
|
|
await real_sleep(0)
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
assert len(ensures) >= 2, "a confirmed-down router must be revived"
|
|
assert sleeps[0] == proc.WATCHDOG_INTERVAL_SECONDS
|
|
assert sleeps[1] == 2, "two-strike: a single failed probe must be re-confirmed before reviving"
|
|
assert proc.WATCHDOG_BACKOFF_SECONDS in sleeps, "3 straight failures must back off"
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_watchdog_single_false_negative_never_revives():
|
|
async def run():
|
|
sleeps: list = []
|
|
ensures: list = []
|
|
probes: list = []
|
|
|
|
async def fake_sleep(d):
|
|
sleeps.append(d)
|
|
await real_sleep(0)
|
|
|
|
async def fake_ensure():
|
|
ensures.append(1)
|
|
|
|
def flaky_is_running():
|
|
# First probe of each pulse fails (busy-router false negative); the confirm succeeds.
|
|
probes.append(1)
|
|
return len(probes) % 2 == 0
|
|
|
|
real_sleep = asyncio.sleep
|
|
with patch.object(proc, "is_running", flaky_is_running), \
|
|
patch.object(proc, "ensure_running", fake_ensure), \
|
|
patch.object(proc.asyncio, "sleep", fake_sleep):
|
|
task = asyncio.get_running_loop().create_task(proc.watchdog_loop())
|
|
while len(probes) < 8:
|
|
await real_sleep(0)
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
assert not ensures, "a transient probe failure must never trigger a revive"
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_death_watcher_revives_instantly_and_guards_loops():
|
|
async def run():
|
|
ensures: list = []
|
|
|
|
async def fake_ensure():
|
|
ensures.append(1)
|
|
|
|
class FakeProc:
|
|
def __init__(self):
|
|
self.dead = False
|
|
def wait(self):
|
|
while not self.dead:
|
|
pass
|
|
def poll(self):
|
|
return 1 if self.dead else None
|
|
|
|
fp = FakeProc()
|
|
proc.recent_death_monos.clear()
|
|
with patch.object(proc, "ensure_running", fake_ensure), \
|
|
patch.object(proc, "p_process", fp):
|
|
task = asyncio.get_running_loop().create_task(proc.death_watch(fp))
|
|
await asyncio.sleep(0.05)
|
|
assert not ensures, "no revive while the process lives"
|
|
fp.dead = True
|
|
for _ in range(200):
|
|
if ensures:
|
|
break
|
|
await asyncio.sleep(0.01)
|
|
assert ensures, "process death must trigger an instant revive"
|
|
await task
|
|
# Crash-loop guard: a 3rd death inside 60s defers to the watchdog.
|
|
ensures.clear()
|
|
proc.recent_death_monos[:] = [proc.time.monotonic() - 5, proc.time.monotonic() - 3]
|
|
fp2 = FakeProc(); fp2.dead = True
|
|
with patch.object(proc, "ensure_running", fake_ensure), \
|
|
patch.object(proc, "p_process", fp2):
|
|
await proc.death_watch(fp2)
|
|
assert not ensures, "3 deaths in 60s must defer to the backed-off watchdog"
|
|
# A superseded/stopped handle never revives.
|
|
ensures.clear()
|
|
proc.recent_death_monos.clear()
|
|
fp3 = FakeProc(); fp3.dead = True
|
|
with patch.object(proc, "ensure_running", fake_ensure), \
|
|
patch.object(proc, "p_process", None):
|
|
await proc.death_watch(fp3)
|
|
assert not ensures, "a deliberately stopped router must stay down"
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_watchdog_healthy_router_never_spawns():
|
|
async def run():
|
|
sleeps: list = []
|
|
ensures: list = []
|
|
|
|
async def fake_sleep(d):
|
|
sleeps.append(d)
|
|
await real_sleep(0)
|
|
|
|
async def fake_ensure():
|
|
ensures.append(1)
|
|
|
|
real_sleep = asyncio.sleep
|
|
with patch.object(proc, "is_running", return_value=True), \
|
|
patch.object(proc, "ensure_running", fake_ensure), \
|
|
patch.object(proc.asyncio, "sleep", fake_sleep):
|
|
task = asyncio.get_running_loop().create_task(proc.watchdog_loop())
|
|
while len(sleeps) < 4:
|
|
await real_sleep(0)
|
|
task.cancel()
|
|
try:
|
|
await task
|
|
except asyncio.CancelledError:
|
|
pass
|
|
assert not ensures
|
|
assert all(d == proc.WATCHDOG_INTERVAL_SECONDS for d in sleeps)
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_stop_cancels_watchdog():
|
|
async def run():
|
|
async def forever():
|
|
while True:
|
|
await asyncio.sleep(3600)
|
|
|
|
proc.watchdog_task = asyncio.get_running_loop().create_task(forever())
|
|
proc.stop()
|
|
assert proc.watchdog_task is None
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_has_persisted_connections(tmp_path, monkeypatch):
|
|
monkeypatch.setenv("DATA_DIR", str(tmp_path))
|
|
assert proc.has_persisted_connections() is False # no db at all
|
|
(tmp_path / "db.json").write_text(json.dumps({"providerConnections": [{"provider": "claude", "isActive": False}]}))
|
|
assert proc.has_persisted_connections() is False # inactive only
|
|
(tmp_path / "db.json").write_text(json.dumps({"providerConnections": [{"provider": "claude", "isActive": True}]}))
|
|
assert proc.has_persisted_connections() is True
|
|
(tmp_path / "db.json").write_text("{corrupt")
|
|
assert proc.has_persisted_connections() is False # fail-closed
|
|
|
|
|
|
def test_detection_revival_gated_on_evidence():
|
|
from backend.apps.agents.manager import configure_provider_env as cpe
|
|
|
|
async def run():
|
|
ensures: list = []
|
|
|
|
async def fake_ensure():
|
|
ensures.append(1)
|
|
|
|
import backend.apps.nine_router as nr_pkg
|
|
with patch.object(nr_pkg, "is_running", return_value=False), \
|
|
patch.object(nr_pkg, "ensure_running", fake_ensure), \
|
|
patch.object(proc, "has_persisted_connections", return_value=False):
|
|
# Zero-config: no keys, no proxy mode, no persisted connections -> no revival attempt.
|
|
assert await cpe.router_available(AppSettings()) is False
|
|
assert not ensures
|
|
# A persisted subscription connection alone IS evidence -> revival attempted.
|
|
with patch.object(proc, "has_persisted_connections", return_value=True):
|
|
assert await cpe.router_available(AppSettings()) is False # ensure failed (router stays down)
|
|
assert ensures, "sub-only users must get a revival attempt"
|
|
|
|
asyncio.run(run())
|
|
|
|
|
|
def test_a_healthy_router_is_adopted_not_evicted():
|
|
"""The dev-machine thrash engine, removed.
|
|
|
|
Every dev backend used to kill any 9Router it had not personally spawned, on a "stale build"
|
|
rationale that does not apply to a component we pin by npm version instead of building from
|
|
source. On a shared box that is a loop: each boot evicts the router the others are using, their
|
|
watchdogs revive it, the next boot evicts it again. Measured with 8 backends on one machine, a
|
|
coverage sweep scored 1/9 with zero code change, wall times went 30s -> 202s, and one backend
|
|
logged 0 kills of its own against 4 revives.
|
|
|
|
is_running() has already confirmed HTTP 200 on /v1/models before this point, so the router is
|
|
serving by definition; killing it settles ownership and nothing else.
|
|
"""
|
|
killed = []
|
|
with patch.object(proc, "is_running", return_value=True), \
|
|
patch.object(proc, "p_process", None), \
|
|
patch("subprocess.run", side_effect=lambda *a, **k: killed.append(a)), \
|
|
patch.dict("os.environ", {"OPENSWARM_PACKAGED": "0"}, clear=False):
|
|
asyncio.run(proc.p_ensure_running_impl())
|
|
assert killed == [], "a healthy router another process owns must be adopted, never pkill'd"
|
|
|
|
|
|
def test_takeover_escape_hatch_still_exists():
|
|
"""Adoption must not become a trap: someone genuinely needing a fresh spawn keeps a way to get
|
|
one, and it is explicit rather than the silent default."""
|
|
import inspect
|
|
src = inspect.getsource(proc.p_ensure_running_impl)
|
|
assert "OPENSWARM_ROUTER_TAKEOVER" in src
|
|
assert 'os.environ.get("OPENSWARM_ROUTER_TAKEOVER") != "1"' in src
|