[eric] backend: fast-fail is_running() to kill the ~18s cold-start event-loop freeze

- faulthandler on the signed cold build caught the asyncio loop frozen in
  socket.create_connection inside is_running() (process.py): a synchronous
  httpx.get to "localhost:20128" called ~5x on the boot path before 9Router is up
- on Windows a dead-port connect to "localhost" stalls ~7s each (tries ::1 first,
  loopback refusal is slow), freezing the loop ~18s so uvicorn could not answer
  the health probe -> cold backend-http-ready was ~23s
- fix: probe 127.0.0.1 with a 0.3s TCP timeout first (measured 306ms vs ~7s), only
  HTTP-confirm when the port is open; 9Router binds 0.0.0.0 so reachability is
  unchanged. drop the faulthandler diagnostic from main.py
This commit is contained in:
Eric
2026-06-17 19:06:10 -07:00
parent 36f54aa534
commit 24df3e8fac
2 changed files with 19 additions and 14 deletions
+19 -2
View File
@@ -16,6 +16,7 @@ import logging
import os
import secrets
import shutil
import socket
import subprocess
import tempfile
import time
@@ -74,13 +75,29 @@ _is_running_last_ok: float = 0.0
def is_running() -> bool:
"""Check if 9Router is running."""
"""Check if 9Router is running.
Fast-fail when down. is_running() is called ~5x on the cold boot path (the
settings key-sync sequence + ensure_running) BEFORE 9Router is up. The old
body did a synchronous httpx.get to "localhost:20128"; on Windows a dead-port
connect to "localhost" stalls multiple seconds (it tries ::1 first and the
loopback refusal is slow), so those probes froze the asyncio event loop ~18s
and dominated cold startup (faulthandler caught the loop stuck in
socket.create_connection here). Fix: probe 127.0.0.1 with a 0.3s TCP timeout
first; a down 9Router is detected in <~0.3s instead of ~7s. Only when the
port is open do we do the HTTP confirm. 9Router binds 0.0.0.0 (the warm app
reaches it via 127.0.0.1 today), so this changes timing, not reachability."""
global _is_running_last_ok
now = time.monotonic()
if now - _is_running_last_ok < _IS_RUNNING_TTL:
return True
try:
r = httpx.get(f"{NINE_ROUTER_V1}/models", timeout=2.0)
with socket.create_connection(("127.0.0.1", NINE_ROUTER_PORT), timeout=0.3):
pass
except OSError:
return False
try:
r = httpx.get(f"http://127.0.0.1:{NINE_ROUTER_PORT}/v1/models", timeout=2.0)
if r.status_code == 200:
_is_running_last_ok = now
return True
-12
View File
@@ -18,18 +18,6 @@ if not _backend_logger.handlers:
logger = logging.getLogger(__name__)
# [perf][diagnostic] Cold-start stall hunt: dump every thread's stack every 7s to
# a temp file. During the cold ~13s event-loop stall a dump lands inside the
# frozen window and names the exact synchronous call the loop is stuck in. Temp
# file only, best-effort; remove once the stall is diagnosed.
try:
import faulthandler as _faulthandler
import tempfile as _tempfile
_fh_diag = open(os.path.join(_tempfile.gettempdir(), "openswarm-faulthandler.log"), "w")
_faulthandler.dump_traceback_later(7, repeat=True, file=_fh_diag)
except Exception:
pass
from fastapi.responses import JSONResponse, HTMLResponse
from fastapi import Request