[eric] apps: ghosts get swept every 10 minutes not just at boot, and a frozen idle runtime dies after 15 minutes instead of squatting RAM forever

backend/apps/outputs/outputs.py

@@ -1,3 +1,4 @@
+import asyncio
 import json
 import os
 import logging
@@ -74,9 +75,26 @@ async def outputs_lifespan():
             logger.warning("outputs lifespan: reaped %d ghost runtime(s) from a previous session", ghosts)
     except Exception:
         logger.exception("ghost-runtime reap failed; stale processes stay but boot continues")
+    # The boot reap catches ghosts from a PREVIOUS session, but a session can live for days: this
+    # sweep keeps catching them while we run (another backend dying leaves orphans mid-session) and
+    # retires idle runtimes past their TTL, so "quit but still around" has a bounded lifetime.
+    async def p_periodic_sweep() -> None:
+        from backend.apps.outputs.reap_ghost_runtimes import reap_ghost_runtimes
+        from backend.apps.outputs.runtime import manager as p_sweep_manager
+        while True:
+            await asyncio.sleep(600)
+            try:
+                ghosts = await asyncio.to_thread(reap_ghost_runtimes)
+                stale = await p_sweep_manager.reap_stale_idle()
+                if ghosts or stale:
+                    logger.info("periodic sweep: %d ghost(s) reaped, %d stale idle runtime(s) stopped", ghosts, stale)
+            except Exception:
+                logger.exception("periodic sweep failed; will retry next interval")
+    p_sweep_task = asyncio.create_task(p_periodic_sweep())
     try:
         yield
     finally:
+        p_sweep_task.cancel()
         # Reap every per-app subprocess. Without this each `bash run.sh` (and its vite/uvicorn descendants) reparents to PID 1 when the main backend dies, leaving ghost listeners on the .env-pinned ports that block the next OpenSwarm launch's reload preview.
         try:
             from backend.apps.outputs.runtime import manager as runtime_manager

backend/apps/outputs/runtime.py

@@ -5,6 +5,7 @@ import logging
 import os
 import shutil
 import sys
+import time
 from collections import deque, OrderedDict
 from dataclasses import dataclass
 from typing import Callable, Optional
@@ -30,6 +31,7 @@ from backend.apps.outputs.runtime_proc import (
     FRONTEND_BIND_POLL_INTERVAL,
     FRONTEND_BIND_TIMEOUT_SECONDS,
     LOG_BUFFER_LINES,
+    IDLE_RUNTIME_TTL_S,
     MAX_IDLE_RUNTIMES,
     RECENT_ERRORS_MAX,
     TERMINATE_GRACE_SECONDS,
@@ -582,6 +584,8 @@ class AppRuntimeManager:
         self.p_attached: dict[str, int] = {}
         # workspace_id → AppRuntime with no subscribers but still alive. OrderedDict gives O(1) move_to_end + popitem(last=False) for LRU semantics.
         self.idle_lru: "OrderedDict[str, AppRuntime]" = OrderedDict()
+        # workspace key -> monotonic seconds when it was parked; drives the idle TTL sweep.
+        self.p_idle_since: dict[str, float] = {}
         self.p_lock = asyncio.Lock()
         # Public: tests cancel it during teardown.
         self.restart_watch_task: Optional[asyncio.Task] = None
@@ -631,6 +635,7 @@ class AppRuntimeManager:
             if rt is None:
                 # Maybe the runtime is sitting idle in the LRU; revive it without paying the spawn cost again.
                 idle_rt = self.idle_lru.pop(key, None)
+                self.p_idle_since.pop(key, None)
                 if idle_rt is not None and idle_rt.running:
                     rt = idle_rt
                     rt.workspace_path = workspace_path
@@ -678,10 +683,12 @@ class AppRuntimeManager:
             else:
                 self.idle_lru[key] = rt
                 self.idle_lru.move_to_end(key)
+                self.p_idle_since[key] = time.monotonic()
                 suspend_process_tree(rt.process)
                 rt.p_suspended = True
                 while len(self.idle_lru) > MAX_IDLE_RUNTIMES:
-                    _, old_rt = self.idle_lru.popitem(last=False)
+                    old_key, old_rt = self.idle_lru.popitem(last=False)
+                    self.p_idle_since.pop(old_key, None)
                     # Reaping a stopped process: SIGCONT first so the SIGTERM in stop() can be delivered cleanly (a SIGSTOP'd process can't run its own shutdown).
                     resume_process_tree(old_rt.process)
                     to_reap.append(old_rt)
@@ -696,6 +703,27 @@ class AppRuntimeManager:
         if to_idle is not None:
             logger.debug("workspace %s idled (LRU size now %d)", key, len(self.idle_lru))
 
+    async def reap_stale_idle(self, ttl_s: float = IDLE_RUNTIME_TTL_S) -> int:
+        """Fully stop idle runtimes parked longer than ttl_s. Frozen is 0% CPU but never 0 cost:
+        each one holds its memory and its port for as long as it sits there, which is the "app is
+        quit but something of it is still around" complaint. Returns how many were stopped."""
+        now = time.monotonic()
+        stale: list[AppRuntime] = []
+        async with self.p_lock:
+            for key in [k for k, t in self.p_idle_since.items() if now - t >= ttl_s]:
+                rt = self.idle_lru.pop(key, None)
+                self.p_idle_since.pop(key, None)
+                if rt is None:
+                    continue
+                resume_process_tree(rt.process)
+                stale.append(rt)
+        for rt in stale:
+            try:
+                await rt.stop()
+            except Exception:
+                logger.exception("failed to stop stale idle runtime")
+        return len(stale)
+
     def get(self, workspace_id: str, instance: int = 1) -> Optional[AppRuntime]:
         key = runtime_key(workspace_id, instance)
         # Active subscribers see the live runtime; idle-pool members are also accessible so a status probe between detach and the next attach still works.

backend/apps/outputs/runtime_proc.py

@@ -26,6 +26,10 @@ LOG_BUFFER_LINES = 2000
 
 # Idle runtimes kept in LRU; trades memory for instant switch-back, beyond 1 because typical users ping-pong 2-3 apps.
 MAX_IDLE_RUNTIMES = 3
+# How long a detached runtime may sit frozen in the idle pool before it is fully stopped. Frozen
+# costs 0% CPU but keeps holding memory and its port; past this nobody is coming back for it soon
+# and a fresh spawn on the next open is a fair trade for not squatting RAM indefinitely.
+IDLE_RUNTIME_TTL_S = 15 * 60.0
 
 # Cap on recent error lines the agent gets; 50 is enough for babel error + stack + a few warnings.
 RECENT_ERRORS_MAX = 50

backend/tests/test_reap_ghost_runtimes.py

@@ -58,3 +58,33 @@ def test_a_broken_ps_reaps_nothing_rather_than_guessing():
     with patch.object(mod.subprocess, "run", side_effect=boom):
         assert mod.find_ghost_runtime_pids() == []
         assert mod.reap_ghost_runtimes() == 0
+
+
+def test_stale_idle_runtimes_are_stopped_after_the_ttl(monkeypatch):
+    """Frozen-idle is 0% CPU but holds memory and a port forever; past the TTL it must actually die."""
+    import asyncio
+    from backend.apps.outputs import runtime as rt_mod
+
+    class P_FakeRuntime:
+        def __init__(self) -> None:
+            self.process = None
+            self.running = True
+            self.stopped = False
+
+        async def stop(self) -> None:
+            self.stopped = True
+
+    monkeypatch.setattr(rt_mod, "resume_process_tree", lambda proc: None)
+    m = rt_mod.AppRuntimeManager()
+    old, fresh = P_FakeRuntime(), P_FakeRuntime()
+    m.idle_lru["ws-old:1"] = old
+    m.idle_lru["ws-new:1"] = fresh
+    import time as p_time
+    m.p_idle_since["ws-old:1"] = p_time.monotonic() - 3600
+    m.p_idle_since["ws-new:1"] = p_time.monotonic()
+
+    reaped = asyncio.run(m.reap_stale_idle(ttl_s=900))
+    assert reaped == 1
+    assert old.stopped and not fresh.stopped
+    assert "ws-old:1" not in m.idle_lru and "ws-new:1" in m.idle_lru
+    assert "ws-old:1" not in m.p_idle_since