mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-02 14:28:59 +02:00
ciregenzandClaude Fable 5
e90abb7bfc
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_01En8dRGsJPLrJCQBEkTH4Mp
104 lines
3.4 KiB
Python
104 lines
3.4 KiB
Python
"""Pins the out-of-loop watchdog (hermes #66892 lift): a frozen loop is detected and killed with
|
|
forensics; a healthy or merely-slow loop is never touched; arming failures fail open."""
|
|
|
|
import subprocess
|
|
import sys
|
|
import textwrap
|
|
|
|
from backend.apps.system import loop_liveness_watchdog as w
|
|
|
|
|
|
def p_run_child(code: str, timeout: int = 60) -> subprocess.CompletedProcess:
|
|
return subprocess.run([sys.executable, "-c", textwrap.dedent(code)], capture_output=True, text=True, timeout=timeout)
|
|
|
|
|
|
P_PRELUDE = """
|
|
import asyncio, sys, time
|
|
sys.path.insert(0, ".")
|
|
from backend.apps.system import loop_liveness_watchdog as w
|
|
w.PROBE_INTERVAL_S = 0.3
|
|
w.PROBE_TIMEOUT_S = 0.3
|
|
w.DUMP_PATH = "/tmp/loop_watchdog_test_dump.log"
|
|
"""
|
|
|
|
|
|
def test_wedged_loop_is_killed_with_forensics(tmp_path):
|
|
code = P_PRELUDE + """
|
|
async def main():
|
|
loop = asyncio.get_running_loop()
|
|
assert w.start_loop_liveness_watchdog(loop) is not None
|
|
time.sleep(30) # wedge the loop with a sync sleep: probes can never run
|
|
|
|
asyncio.run(main())
|
|
print("SURVIVED")
|
|
"""
|
|
r = p_run_child(code)
|
|
assert r.returncode == w.RESTART_EXIT_CODE, f"expected exit {w.RESTART_EXIT_CODE}, got {r.returncode}: {r.stderr[:300]}"
|
|
assert "SURVIVED" not in r.stdout
|
|
dump = open("/tmp/loop_watchdog_test_dump.log").read()
|
|
assert "loop watchdog fired" in dump
|
|
assert "Thread" in dump, "faulthandler stack dump missing"
|
|
|
|
|
|
def test_healthy_loop_never_killed():
|
|
code = P_PRELUDE + """
|
|
async def main():
|
|
loop = asyncio.get_running_loop()
|
|
stop = w.start_loop_liveness_watchdog(loop)
|
|
await asyncio.sleep(2.5) # many probe intervals, loop responsive throughout
|
|
stop.set()
|
|
|
|
asyncio.run(main())
|
|
print("SURVIVED")
|
|
"""
|
|
r = p_run_child(code)
|
|
assert r.returncode == 0 and "SURVIVED" in r.stdout
|
|
|
|
|
|
def test_slow_but_alive_loop_survives_single_strikes():
|
|
"""Blocks shorter than MAX_STRIKES consecutive misses must never kill (sync httpx on the loop is a known 2s block)."""
|
|
code = P_PRELUDE + """
|
|
async def main():
|
|
loop = asyncio.get_running_loop()
|
|
stop = w.start_loop_liveness_watchdog(loop)
|
|
for _ in range(3):
|
|
time.sleep(0.5) # one missed probe worth of block
|
|
await asyncio.sleep(0.7) # then responsive again: strikes reset
|
|
stop.set()
|
|
|
|
asyncio.run(main())
|
|
print("SURVIVED")
|
|
"""
|
|
r = p_run_child(code)
|
|
assert r.returncode == 0 and "SURVIVED" in r.stdout
|
|
|
|
|
|
def test_closed_loop_ends_watchdog_quietly():
|
|
code = P_PRELUDE + """
|
|
async def main():
|
|
loop = asyncio.get_running_loop()
|
|
w.start_loop_liveness_watchdog(loop)
|
|
|
|
asyncio.run(main())
|
|
time.sleep(1.2) # loop closed; watchdog must not fire on a normally exiting process
|
|
print("SURVIVED")
|
|
"""
|
|
r = p_run_child(code)
|
|
assert r.returncode == 0 and "SURVIVED" in r.stdout
|
|
|
|
|
|
def test_the_exit_leaves_a_marker_the_next_boot_can_consume(tmp_path, monkeypatch):
|
|
"""ENG-366: record_boot reads this marker to tell a watchdog restart from a workflow-caused death."""
|
|
import os
|
|
marker = str(tmp_path / "loop-watchdog-exit")
|
|
monkeypatch.setattr(w, "WATCHDOG_EXIT_MARKER", marker)
|
|
monkeypatch.setattr(w, "DUMP_PATH", str(tmp_path / "dump.log"))
|
|
exits = []
|
|
monkeypatch.setattr(os, "_exit", lambda code: exits.append(code))
|
|
w.dump_and_exit(3)
|
|
assert exits == [w.RESTART_EXIT_CODE]
|
|
assert os.path.exists(marker)
|
|
assert w.consume_watchdog_exit_marker() is True
|
|
assert not os.path.exists(marker)
|
|
assert w.consume_watchdog_exit_marker() is False
|