diff --git a/backend/apps/outputs/runtime.py b/backend/apps/outputs/runtime.py index f7f08990..3cf11a16 100644 --- a/backend/apps/outputs/runtime.py +++ b/backend/apps/outputs/runtime.py @@ -46,12 +46,13 @@ from backend.apps.outputs.runtime_proc import ( suspend_process_tree, write_env_value, ) +from backend.config.loop_local import loop_local from backend.config.paths import AUTH_TOKEN_FILE logger = logging.getLogger(__name__) # Module-level lock so only ONE vite optimizeDeps runs at a time; must be acquired before manager.p_lock to avoid deadlock with manager.attach. -p_vite_boot_lock = asyncio.Lock() +get_vite_boot_lock = loop_local(asyncio.Lock) # How often the manager stats each attached workspace for a restart sentinel; one stat/sec/runtime is noise. RESTART_SENTINEL_POLL_SECONDS = 1.0 @@ -226,15 +227,16 @@ class AppRuntime: self.p_reset_terminal_log() if self.is_new_mode: # Acquire the module-level boot lock BEFORE the spawn so only one new-mode workspace is mid-bundle at a time. The lock is released by the bind-poll task the moment vite emits "frontend ready" (or its 180s timeout fires), which is the moment the next workspace can start its own vite without competing for the same CPU. See `p_await_frontend_bind` for the release. - await p_vite_boot_lock.acquire() + p_boot_lock = get_vite_boot_lock() + await p_boot_lock.acquire() try: ok = await self.p_start_new_mode() if not ok: # Spawn failed before the bind-poll task was created; release synchronously so we don't wedge the next workspace. - p_vite_boot_lock.release() + p_boot_lock.release() return ok except Exception: - p_vite_boot_lock.release() + p_boot_lock.release() raise return await self.p_start_old_mode() @@ -378,7 +380,7 @@ class AppRuntime: return lock_released = True try: - p_vite_boot_lock.release() + get_vite_boot_lock().release() except RuntimeError: # Lock already released (e.g. start() failure path released synchronously before spawning the poll task). pass diff --git a/backend/config/loop_local.py b/backend/config/loop_local.py new file mode 100644 index 00000000..081e0366 --- /dev/null +++ b/backend/config/loop_local.py @@ -0,0 +1,39 @@ +"""One asyncio primitive per event loop, so module-level state can never outlive the loop it used.""" + +import asyncio +from typing import Callable, Optional, TypeVar + +T = TypeVar("T") + + +def loop_local(factory: Callable[[], T]) -> Callable[[], T]: + """Wrap an asyncio primitive so it is rebuilt whenever the running event loop changes. + + A module-level ``asyncio.Lock()`` outlives the loop that used it. If that loop dies while the + lock is HELD, the flag stays set forever and the next loop waits on a release that can never + come: no error, no log line, just a process that stops. That is what wedged the entire backend + test suite (ENG-219), and after a ``uvicorn --reload`` it is the same silent hang in the app. + A Semaphore loses its count the same way; an Event raises "bound to a different event loop" + and kills whatever loop was driving it. + + Pass the CLASS, not an instance, and call the result:: + + p_boot_lock = loop_local(asyncio.Lock) + + async with p_boot_lock(): + ... + + Needs a running loop, which is the whole point: nothing else can say which loop to build for. + """ + held: Optional[T] = None + held_loop: Optional[asyncio.AbstractEventLoop] = None + + def get() -> T: + nonlocal held, held_loop + running = asyncio.get_running_loop() + if held is None or held_loop is not running: + held = factory() + held_loop = running + return held + + return get diff --git a/backend/tests/test_serve_static_mode.py b/backend/tests/test_serve_static_mode.py index 1a7bd1df..fe0b7f0e 100644 --- a/backend/tests/test_serve_static_mode.py +++ b/backend/tests/test_serve_static_mode.py @@ -50,7 +50,9 @@ def test_start_skips_serve_when_edited(tmp_path, monkeypatch): rt = AppRuntime("ws-t2", ws) spawned = {"n": 0} - async def p_no_spawn(env): + # Sync on purpose: p_resolve_launch became a plain method, and an async mock here returns a + # never-awaited coroutine whose body (the counter) never runs, failing the assert at 0. + def p_no_spawn(env): spawned["n"] += 1 return None, ws, "stub" monkeypatch.setattr(rt, "p_resolve_launch", p_no_spawn)