From 913f5976568a64892aa479311401d6dede2bab8e Mon Sep 17 00:00:00 2001 From: ciregenz Date: Fri, 7 Aug 2026 19:47:16 -0700 Subject: [PATCH] [eric] apps: ghosts get swept every 10 minutes not just at boot, and a frozen idle runtime dies after 15 minutes instead of squatting RAM forever --- backend/apps/outputs/outputs.py | 18 ++++++++++++++ backend/apps/outputs/runtime.py | 30 ++++++++++++++++++++++- backend/apps/outputs/runtime_proc.py | 4 +++ backend/tests/test_reap_ghost_runtimes.py | 30 +++++++++++++++++++++++ 4 files changed, 81 insertions(+), 1 deletion(-) diff --git a/backend/apps/outputs/outputs.py b/backend/apps/outputs/outputs.py index ef2b5920..f18654de 100644 --- a/backend/apps/outputs/outputs.py +++ b/backend/apps/outputs/outputs.py @@ -1,3 +1,4 @@ +import asyncio import json import os import logging @@ -74,9 +75,26 @@ async def outputs_lifespan(): logger.warning("outputs lifespan: reaped %d ghost runtime(s) from a previous session", ghosts) except Exception: logger.exception("ghost-runtime reap failed; stale processes stay but boot continues") + # The boot reap catches ghosts from a PREVIOUS session, but a session can live for days: this + # sweep keeps catching them while we run (another backend dying leaves orphans mid-session) and + # retires idle runtimes past their TTL, so "quit but still around" has a bounded lifetime. + async def p_periodic_sweep() -> None: + from backend.apps.outputs.reap_ghost_runtimes import reap_ghost_runtimes + from backend.apps.outputs.runtime import manager as p_sweep_manager + while True: + await asyncio.sleep(600) + try: + ghosts = await asyncio.to_thread(reap_ghost_runtimes) + stale = await p_sweep_manager.reap_stale_idle() + if ghosts or stale: + logger.info("periodic sweep: %d ghost(s) reaped, %d stale idle runtime(s) stopped", ghosts, stale) + except Exception: + logger.exception("periodic sweep failed; will retry next interval") + p_sweep_task = asyncio.create_task(p_periodic_sweep()) try: yield finally: + p_sweep_task.cancel() # Reap every per-app subprocess. Without this each `bash run.sh` (and its vite/uvicorn descendants) reparents to PID 1 when the main backend dies, leaving ghost listeners on the .env-pinned ports that block the next OpenSwarm launch's reload preview. try: from backend.apps.outputs.runtime import manager as runtime_manager diff --git a/backend/apps/outputs/runtime.py b/backend/apps/outputs/runtime.py index 84192ee1..0f69e874 100644 --- a/backend/apps/outputs/runtime.py +++ b/backend/apps/outputs/runtime.py @@ -5,6 +5,7 @@ import logging import os import shutil import sys +import time from collections import deque, OrderedDict from dataclasses import dataclass from typing import Callable, Optional @@ -30,6 +31,7 @@ from backend.apps.outputs.runtime_proc import ( FRONTEND_BIND_POLL_INTERVAL, FRONTEND_BIND_TIMEOUT_SECONDS, LOG_BUFFER_LINES, + IDLE_RUNTIME_TTL_S, MAX_IDLE_RUNTIMES, RECENT_ERRORS_MAX, TERMINATE_GRACE_SECONDS, @@ -582,6 +584,8 @@ class AppRuntimeManager: self.p_attached: dict[str, int] = {} # workspace_id → AppRuntime with no subscribers but still alive. OrderedDict gives O(1) move_to_end + popitem(last=False) for LRU semantics. self.idle_lru: "OrderedDict[str, AppRuntime]" = OrderedDict() + # workspace key -> monotonic seconds when it was parked; drives the idle TTL sweep. + self.p_idle_since: dict[str, float] = {} self.p_lock = asyncio.Lock() # Public: tests cancel it during teardown. self.restart_watch_task: Optional[asyncio.Task] = None @@ -631,6 +635,7 @@ class AppRuntimeManager: if rt is None: # Maybe the runtime is sitting idle in the LRU; revive it without paying the spawn cost again. idle_rt = self.idle_lru.pop(key, None) + self.p_idle_since.pop(key, None) if idle_rt is not None and idle_rt.running: rt = idle_rt rt.workspace_path = workspace_path @@ -678,10 +683,12 @@ class AppRuntimeManager: else: self.idle_lru[key] = rt self.idle_lru.move_to_end(key) + self.p_idle_since[key] = time.monotonic() suspend_process_tree(rt.process) rt.p_suspended = True while len(self.idle_lru) > MAX_IDLE_RUNTIMES: - _, old_rt = self.idle_lru.popitem(last=False) + old_key, old_rt = self.idle_lru.popitem(last=False) + self.p_idle_since.pop(old_key, None) # Reaping a stopped process: SIGCONT first so the SIGTERM in stop() can be delivered cleanly (a SIGSTOP'd process can't run its own shutdown). resume_process_tree(old_rt.process) to_reap.append(old_rt) @@ -696,6 +703,27 @@ class AppRuntimeManager: if to_idle is not None: logger.debug("workspace %s idled (LRU size now %d)", key, len(self.idle_lru)) + async def reap_stale_idle(self, ttl_s: float = IDLE_RUNTIME_TTL_S) -> int: + """Fully stop idle runtimes parked longer than ttl_s. Frozen is 0% CPU but never 0 cost: + each one holds its memory and its port for as long as it sits there, which is the "app is + quit but something of it is still around" complaint. Returns how many were stopped.""" + now = time.monotonic() + stale: list[AppRuntime] = [] + async with self.p_lock: + for key in [k for k, t in self.p_idle_since.items() if now - t >= ttl_s]: + rt = self.idle_lru.pop(key, None) + self.p_idle_since.pop(key, None) + if rt is None: + continue + resume_process_tree(rt.process) + stale.append(rt) + for rt in stale: + try: + await rt.stop() + except Exception: + logger.exception("failed to stop stale idle runtime") + return len(stale) + def get(self, workspace_id: str, instance: int = 1) -> Optional[AppRuntime]: key = runtime_key(workspace_id, instance) # Active subscribers see the live runtime; idle-pool members are also accessible so a status probe between detach and the next attach still works. diff --git a/backend/apps/outputs/runtime_proc.py b/backend/apps/outputs/runtime_proc.py index e5c56a48..d10c2f38 100644 --- a/backend/apps/outputs/runtime_proc.py +++ b/backend/apps/outputs/runtime_proc.py @@ -26,6 +26,10 @@ LOG_BUFFER_LINES = 2000 # Idle runtimes kept in LRU; trades memory for instant switch-back, beyond 1 because typical users ping-pong 2-3 apps. MAX_IDLE_RUNTIMES = 3 +# How long a detached runtime may sit frozen in the idle pool before it is fully stopped. Frozen +# costs 0% CPU but keeps holding memory and its port; past this nobody is coming back for it soon +# and a fresh spawn on the next open is a fair trade for not squatting RAM indefinitely. +IDLE_RUNTIME_TTL_S = 15 * 60.0 # Cap on recent error lines the agent gets; 50 is enough for babel error + stack + a few warnings. RECENT_ERRORS_MAX = 50 diff --git a/backend/tests/test_reap_ghost_runtimes.py b/backend/tests/test_reap_ghost_runtimes.py index 0ab4ae63..d9574a87 100644 --- a/backend/tests/test_reap_ghost_runtimes.py +++ b/backend/tests/test_reap_ghost_runtimes.py @@ -58,3 +58,33 @@ def test_a_broken_ps_reaps_nothing_rather_than_guessing(): with patch.object(mod.subprocess, "run", side_effect=boom): assert mod.find_ghost_runtime_pids() == [] assert mod.reap_ghost_runtimes() == 0 + + +def test_stale_idle_runtimes_are_stopped_after_the_ttl(monkeypatch): + """Frozen-idle is 0% CPU but holds memory and a port forever; past the TTL it must actually die.""" + import asyncio + from backend.apps.outputs import runtime as rt_mod + + class P_FakeRuntime: + def __init__(self) -> None: + self.process = None + self.running = True + self.stopped = False + + async def stop(self) -> None: + self.stopped = True + + monkeypatch.setattr(rt_mod, "resume_process_tree", lambda proc: None) + m = rt_mod.AppRuntimeManager() + old, fresh = P_FakeRuntime(), P_FakeRuntime() + m.idle_lru["ws-old:1"] = old + m.idle_lru["ws-new:1"] = fresh + import time as p_time + m.p_idle_since["ws-old:1"] = p_time.monotonic() - 3600 + m.p_idle_since["ws-new:1"] = p_time.monotonic() + + reaped = asyncio.run(m.reap_stale_idle(ttl_s=900)) + assert reaped == 1 + assert old.stopped and not fresh.stopped + assert "ws-old:1" not in m.idle_lru and "ws-new:1" in m.idle_lru + assert "ws-old:1" not in m.p_idle_since