mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-24 05:22:22 +02:00
[eric] backend: fast-fail is_running() to kill the ~18s cold-start event-loop freeze
- faulthandler on the signed cold build caught the asyncio loop frozen in socket.create_connection inside is_running() (process.py): a synchronous httpx.get to "localhost:20128" called ~5x on the boot path before 9Router is up - on Windows a dead-port connect to "localhost" stalls ~7s each (tries ::1 first, loopback refusal is slow), freezing the loop ~18s so uvicorn could not answer the health probe -> cold backend-http-ready was ~23s - fix: probe 127.0.0.1 with a 0.3s TCP timeout first (measured 306ms vs ~7s), only HTTP-confirm when the port is open; 9Router binds 0.0.0.0 so reachability is unchanged. drop the faulthandler diagnostic from main.py
This commit is contained in:
@@ -16,6 +16,7 @@ import logging
|
||||
import os
|
||||
import secrets
|
||||
import shutil
|
||||
import socket
|
||||
import subprocess
|
||||
import tempfile
|
||||
import time
|
||||
@@ -74,13 +75,29 @@ _is_running_last_ok: float = 0.0
|
||||
|
||||
|
||||
def is_running() -> bool:
|
||||
"""Check if 9Router is running."""
|
||||
"""Check if 9Router is running.
|
||||
|
||||
Fast-fail when down. is_running() is called ~5x on the cold boot path (the
|
||||
settings key-sync sequence + ensure_running) BEFORE 9Router is up. The old
|
||||
body did a synchronous httpx.get to "localhost:20128"; on Windows a dead-port
|
||||
connect to "localhost" stalls multiple seconds (it tries ::1 first and the
|
||||
loopback refusal is slow), so those probes froze the asyncio event loop ~18s
|
||||
and dominated cold startup (faulthandler caught the loop stuck in
|
||||
socket.create_connection here). Fix: probe 127.0.0.1 with a 0.3s TCP timeout
|
||||
first; a down 9Router is detected in <~0.3s instead of ~7s. Only when the
|
||||
port is open do we do the HTTP confirm. 9Router binds 0.0.0.0 (the warm app
|
||||
reaches it via 127.0.0.1 today), so this changes timing, not reachability."""
|
||||
global _is_running_last_ok
|
||||
now = time.monotonic()
|
||||
if now - _is_running_last_ok < _IS_RUNNING_TTL:
|
||||
return True
|
||||
try:
|
||||
r = httpx.get(f"{NINE_ROUTER_V1}/models", timeout=2.0)
|
||||
with socket.create_connection(("127.0.0.1", NINE_ROUTER_PORT), timeout=0.3):
|
||||
pass
|
||||
except OSError:
|
||||
return False
|
||||
try:
|
||||
r = httpx.get(f"http://127.0.0.1:{NINE_ROUTER_PORT}/v1/models", timeout=2.0)
|
||||
if r.status_code == 200:
|
||||
_is_running_last_ok = now
|
||||
return True
|
||||
|
||||
@@ -18,18 +18,6 @@ if not _backend_logger.handlers:
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# [perf][diagnostic] Cold-start stall hunt: dump every thread's stack every 7s to
|
||||
# a temp file. During the cold ~13s event-loop stall a dump lands inside the
|
||||
# frozen window and names the exact synchronous call the loop is stuck in. Temp
|
||||
# file only, best-effort; remove once the stall is diagnosed.
|
||||
try:
|
||||
import faulthandler as _faulthandler
|
||||
import tempfile as _tempfile
|
||||
_fh_diag = open(os.path.join(_tempfile.gettempdir(), "openswarm-faulthandler.log"), "w")
|
||||
_faulthandler.dump_traceback_later(7, repeat=True, file=_fh_diag)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
from fastapi.responses import JSONResponse, HTMLResponse
|
||||
from fastapi import Request
|
||||
|
||||
|
||||
Reference in New Issue
Block a user