[eric] apps: the bundled python keeps ensurepip so app-backend venvs are real, and a hollow venv can never reach an app

backend/apps/outputs/runtime.py

@@ -227,6 +227,8 @@ class AppRuntime:
                 return True
 
             self.p_reset_terminal_log()
+            # Every boot re-decides serve mode from scratch; a stale True from the previous boot would make ready/frontend_url claim a processless app is fine after a restart that exists to spawn one.
+            self.serve_static = False
             if self.is_new_mode:
                 # Acquire the module-level boot lock BEFORE the spawn so only one new-mode workspace is mid-bundle at a time. The lock is released by the bind-poll task the moment vite emits "frontend ready" (or its 180s timeout fires), which is the moment the next workspace can start its own vite without competing for the same CPU. See `p_await_frontend_bind` for the release.
                 p_boot_lock = get_vite_boot_lock()
@@ -247,7 +249,11 @@ class AppRuntime:
     async def p_start_new_mode(self) -> bool:
         # Serve-mode (ENG-209): a fresh built bundle + nobody editing = no process at all. Primary
         # instance only (secondaries are explicitly "another independent window", keep them live).
-        if self.instance == 1:
+        # An app with a backend never qualifies: serve mode spawns NOTHING, so the bundle would be
+        # served against an API that was never started, and start() would return True anyway. The
+        # freshness check only stats frontend files, so it cannot see a backend at all.
+        p_declares_backend = (read_env_value(os.path.join(self.workspace_path, ".env"), "BACKEND_PORT") or "NONE") != "NONE"
+        if self.instance == 1 and not p_declares_backend:
             from backend.apps.outputs.static_serve import static_fresh, workspace_being_edited
             if static_fresh(self.workspace_path) and not workspace_being_edited(self.workspace_path):
                 self.serve_static = True
@@ -465,7 +471,9 @@ class AppRuntime:
         """Inherited env minus the install token. Backend.py can hit our
         REST API back via its own creds if it really needs to, but it
         shouldn't inherit the host process's token by default."""
-        env = {k: v for k, v in os.environ.items() if k != "OPENSWARM_AUTH_TOKEN"}
+        # PYTHONPATH goes too: the host's points at the app bundle's own site-packages, which shadows the workspace venv so `pip install` reports success while installing nothing and `import fastapi` resolves to OUR copy. The app then works until it is run cleanly, and its venv was never real (Haik, 2026-08-14).
+        p_stripped = ("OPENSWARM_AUTH_TOKEN", "PYTHONPATH", "PYTHONHOME")
+        env = {k: v for k, v in os.environ.items() if k not in p_stripped}
         # Where the token lives, not the token itself: an app that legitimately calls our REST API reads it from disk and so picks up rotations, without the value sitting in its env for any child to inherit. Dev and packaged builds keep their data roots in different places, so an app hardcoding one of them is silently wrong in the other.
         env["OPENSWARM_HOST_TOKEN_FILE"] = AUTH_TOKEN_FILE
         env["OPENSWARM_OUTPUT_ID"] = self.workspace_id
@@ -661,7 +669,11 @@ class AppRuntimeManager:
                     except OSError:
                         continue
                     for peer in list(self.runtimes.values()):
-                        if peer.workspace_path == ws_path and peer.running and not peer.p_suspended:
+                        # serve_static counts even though `running` is False: it has no process BY
+                        # DESIGN, so gating on running made restart.sh time out on exactly the
+                        # runtimes that most need a restart, then blame a runtime that was fine.
+                        p_restartable = peer.running or peer.serve_static
+                        if peer.workspace_path == ws_path and p_restartable and not peer.p_suspended:
                             peer.announce("[runtime] restart requested from the workspace (restart.sh); restarting...")
                             asyncio.create_task(peer.restart())
             except Exception:

backend/apps/outputs/view_builder_templates.py

@@ -489,6 +489,7 @@ def p_ensure_warm_python_venv() -> str | None:
             )
             if r.returncode != 0:
                 logger.warning("warm-venv create failed: %s", r.stderr[-1500:])
+                shutil.rmtree(venv_dir, ignore_errors=True)
                 return None
 
             # Install the template's dependencies (fastapi[standard], typeguard, swarm-debug, transitives); keep this list in sync with webapp_template/backend/pyproject.toml. NOT the workspace's own backend, which gets editable-installed per-workspace by run.sh after the cache copy. The venv layout differs by platform: POSIX puts executables in `bin/`, Windows in `Scripts/`, and the executable name itself gets `.exe`.
@@ -496,6 +497,15 @@ def p_ensure_warm_python_venv() -> str | None:
                 pip = os.path.join(venv_dir, "Scripts", "pip.exe")
             else:
                 pip = os.path.join(venv_dir, "bin", "pip")
+            # An interpreter with no ensurepip yields a venv with no pip: bin/ holds three symlinks and nothing else, and every app built from it dies on `pip install -e .`. Catch it HERE, where one cache entry is wrong, not four layers down in a user's app (Haik, 2026-08-14).
+            if not os.path.exists(pip):
+                logger.error(
+                    "warm-venv has no pip (interpreter %s cannot bootstrap one); "
+                    "discarding so no app inherits a hollow venv", py,
+                )
+                shutil.rmtree(venv_dir, ignore_errors=True)
+                return None
+
             deps = ["fastapi[standard]", "typeguard==4.4.2", "swarm-debug"]
             r = subprocess.run(
                 [pip, "install", "--disable-pip-version-check", *deps],
@@ -503,6 +513,18 @@ def p_ensure_warm_python_venv() -> str | None:
             )
             if r.returncode != 0:
                 logger.warning("warm-venv pip install failed: %s", r.stderr[-1500:])
+                shutil.rmtree(venv_dir, ignore_errors=True)
+                return None
+
+            # The sentinel means USABLE, not merely attempted, so it is written only after the venv is proven to import what the template needs.
+            probe = subprocess.run(
+                [os.path.join(venv_dir, "Scripts" if os.name == "nt" else "bin", "python"),
+                 "-c", "import fastapi, typeguard, httpx, uvicorn"],
+                capture_output=True, text=True, timeout=120,
+            )
+            if probe.returncode != 0:
+                logger.warning("warm-venv import probe failed: %s", probe.stderr[-800:])
+                shutil.rmtree(venv_dir, ignore_errors=True)
                 return None
 
             with open(sentinel, "w", encoding="utf-8") as fh:
@@ -511,6 +533,7 @@ def p_ensure_warm_python_venv() -> str | None:
             return venv_dir
         except Exception as exc:
             logger.warning("warm python venv failed: %s", exc)
+            shutil.rmtree(venv_dir, ignore_errors=True)
             return None
 
 

backend/apps/outputs/webapp_template/backend_init.sh

@@ -64,7 +64,14 @@ chmod +x ./backend/run.sh
 # the activate script's VIRTUAL_ENV path is rewritten so `source
 # .venv/bin/activate` resolves to the correct workspace path.
 CACHE_VENV="${OPENSWARM_BACKEND_VENV_CACHE:-}/.venv"
-if [[ -d "$CACHE_VENV" ]]; then
+# Reuse only a venv the builder MARKED usable. Gating on "the directory exists"
+# copied hollow venvs (no pip, no site-packages) into every new app and printed
+# success anyway; the .populated sentinel was sitting right there unread.
+CACHE_SENTINEL="${OPENSWARM_BACKEND_VENV_CACHE:-}/.populated"
+if [[ -d "$CACHE_VENV" && ! -f "$CACHE_SENTINEL" ]]; then
+    echo "Warm backend venv at $CACHE_VENV is unpopulated; building a fresh one instead."
+fi
+if [[ -d "$CACHE_VENV" && -f "$CACHE_SENTINEL" ]]; then
     echo "Reusing warm backend venv from $CACHE_VENV..."
     cp -aR "$CACHE_VENV" ./backend/.venv
     NEW_VENV_ABS="$HERE/backend/.venv"

backend/tests/test_app_backend_venv_chain.py

@@ -0,0 +1,116 @@
+"""The fresh-app-cannot-boot-a-backend chain (Haik, 2026-08-14, stable 1.7.7).
+
+Root cause, verified against the shipped bundle: the packaged interpreter has no `ensurepip` and
+no `pip`, so `python -m venv` yields a venv holding three symlinks and nothing else. The warm-cache
+builder left that hollow venv on disk on every failure path, and `backend_init.sh` reused it on a
+directory-exists check, so a known-broken venv was copied into every new app while three layers
+printed success. These pin each link so the chain cannot silently reform.
+"""
+
+import inspect
+import os
+import re
+
+from backend.apps.outputs import view_builder_templates
+
+
+TEMPLATE_DIR = os.path.join(os.path.dirname(inspect.getfile(view_builder_templates)), "webapp_template")
+
+
+def p_builder_src() -> str:
+    return inspect.getsource(view_builder_templates.p_ensure_warm_python_venv)
+
+
+# --------------------------------------------------------------- never leave known-bad output
+
+
+def test_every_failure_path_deletes_the_half_built_venv():
+    src = p_builder_src()
+    # Each `return None` in the body must be preceded by a cleanup; count them as pairs.
+    returns = src.count("return None")
+    cleanups = src.count("shutil.rmtree(venv_dir, ignore_errors=True)")
+    assert returns >= 3, "the builder should still have its failure paths"
+    assert cleanups >= returns, (
+        f"{returns} failure paths but only {cleanups} cleanups; a hollow venv left on disk "
+        "gets copied into the next app forever"
+    )
+
+
+def test_the_builder_refuses_a_venv_with_no_pip():
+    src = p_builder_src()
+    assert "os.path.exists(pip)" in src, "a venv without pip must be discarded, not populated"
+
+
+def test_the_sentinel_means_usable_not_merely_attempted():
+    src = p_builder_src()
+    i_probe = src.find("import fastapi, typeguard, httpx, uvicorn")
+    i_sentinel = src.find('open(sentinel, "w"')
+    assert i_probe != -1, "the builder must prove the venv can import what the template needs"
+    assert i_probe < i_sentinel, "the import probe must run BEFORE the sentinel is written"
+
+
+# --------------------------------------------------------------- the consumer honors the sentinel
+
+
+def test_backend_init_gates_reuse_on_the_sentinel_not_directory_existence():
+    init = open(os.path.join(TEMPLATE_DIR, "backend_init.sh"), encoding="utf-8").read()
+    assert "CACHE_SENTINEL" in init, "the reuse gate must consult .populated"
+    reuse = re.search(r'if \[\[ -d "\$CACHE_VENV".*?\]\]; then\n\s*echo "Reusing', init, re.S)
+    assert reuse and "CACHE_SENTINEL" in reuse.group(0), (
+        "the branch that copies the cache must require the sentinel; -d alone copies hollow venvs"
+    )
+
+
+# --------------------------------------------------------------- the interpreter must be capable
+
+
+def test_the_packaged_build_keeps_ensurepip():
+    build = open(os.path.join(os.path.dirname(view_builder_templates.__file__), "..", "..", "..",
+                              "scripts", "build-python-env.sh"), encoding="utf-8").read()
+    assert 'rm -rf "$PYTHON_ENV_DIR/lib/python3.13/ensurepip"' not in build, (
+        "stripping ensurepip makes every app-backend venv hollow; that is the root cause"
+    )
+    assert "cannot create a venv with pip" in build, (
+        "the build must PROVE the bundled interpreter can make a working venv, not assume it"
+    )
+
+
+# --------------------------------------------------------------- host env must not leak
+
+
+def test_pythonpath_is_stripped_from_workspace_subprocesses():
+    from backend.apps.outputs.runtime import AppRuntime
+    src = inspect.getsource(AppRuntime.p_spawn_env_base)
+    assert "PYTHONPATH" in src, (
+        "the host's PYTHONPATH points at the app bundle's site-packages and shadows the "
+        "workspace venv, so pip reports success while installing nothing"
+    )
+
+
+# --------------------------------------------------------------- serve mode vs a real backend
+
+
+def test_serve_mode_never_engages_for_an_app_that_declares_a_backend():
+    from backend.apps.outputs.runtime import AppRuntime
+    src = inspect.getsource(AppRuntime.p_start_new_mode)
+    assert "p_declares_backend" in src, (
+        "serve mode spawns nothing; engaging it for an app with a backend serves a bundle "
+        "against an API that was never started, and returns True"
+    )
+    assert src.index("p_declares_backend") < src.index("self.serve_static = True")
+
+
+def test_a_restart_re_decides_serve_mode():
+    from backend.apps.outputs.runtime import AppRuntime
+    src = inspect.getsource(AppRuntime.start)
+    assert "self.serve_static = False" in src, "a stale serve_static survives the restart meant to clear it"
+
+
+def test_the_restart_sentinel_watcher_can_see_serve_mode_runtimes():
+    from backend.apps.outputs.runtime import AppRuntimeManager
+    src = inspect.getsource(AppRuntimeManager)
+    watcher = src[src.find("RESTART_SENTINEL_NAME"):]
+    assert "peer.serve_static" in watcher, (
+        "a serve-mode runtime has no process, so gating on `running` made restart.sh dead on "
+        "exactly the runtimes that needed restarting"
+    )

scripts/build-python-env.sh

@@ -140,8 +140,13 @@ rm -rf "$PYTHON_ENV_DIR/include"
 # IDLE editor + Tk GUI toolkit — embedded headless backend has no UI.
 rm -rf "$PYTHON_ENV_DIR/lib/python3.13/idlelib"
 rm -rf "$PYTHON_ENV_DIR/lib/python3.13/tkinter"
-# Pip bootstrap module — backend never installs packages at runtime.
-rm -rf "$PYTHON_ENV_DIR/lib/python3.13/ensurepip"
+# ensurepip STAYS. It is what `python -m venv` uses to put pip inside a new
+# venv, and the App Builder builds every app's backend venv from THIS
+# interpreter (view_builder_templates.py p_resolve_python -> sys.executable).
+# Stripping it made every generated venv hollow: bin/ with three symlinks, no
+# pip, no site-packages, so `pip install -e .` died with "No module named pip"
+# and no app could ever boot a backend on a packaged build (Haik, 2026-08-14).
+# It costs ~10MB and it is the difference between apps working and not.
 # Educational drawing examples that ship with stdlib — never imported.
 rm -rf "$PYTHON_ENV_DIR/lib/python3.13/turtledemo"
 # turtle itself: a Tk-based graphics module. It imports tkinter (stripped
@@ -149,17 +154,30 @@ rm -rf "$PYTHON_ENV_DIR/lib/python3.13/turtledemo"
 rm -rf "$PYTHON_ENV_DIR/lib/python3.13/turtle.py"
 # Man pages / desktop-integration files — embedded Python doesn't read these.
 rm -rf "$PYTHON_ENV_DIR/share"
-# pip itself + launcher shims. Verified the packaged backend never invokes
-# pip: uvx (used by MCPs) is a self-contained installer; the App Builder's
-# view_builder_templates.py:382 picks SYSTEM python via shutil.which, never
-# this bundled one; backend code only mentions "pip install" in error-message
-# strings. `python -m venv` from this bundled env is also dead (ensurepip
-# already stripped above) but nothing calls it.
+# Top-level pip and the editor shims go; ensurepip above carries its own pip
+# wheel, so `python -m venv` still produces a venv WITH pip. The old comment
+# here justified stripping ensurepip too by claiming the App Builder used a
+# system python, which stopped being true when p_resolve_python switched to
+# sys.executable, and nothing re-read this file. That is why the gate below
+# verifies the interpreter instead of trusting a comment.
 rm -rf "$PYTHON_ENV_DIR/lib/python3.13/site-packages/pip" \
        "$PYTHON_ENV_DIR/lib/python3.13/site-packages"/pip-*.dist-info
 rm -f "$PYTHON_ENV_DIR/bin/pip" "$PYTHON_ENV_DIR/bin/pip3" "$PYTHON_ENV_DIR/bin/pip3.13" \
       "$PYTHON_ENV_DIR/bin/idle3" "$PYTHON_ENV_DIR/bin/idle3.13" \
       "$PYTHON_ENV_DIR/bin/pydoc3" "$PYTHON_ENV_DIR/bin/pydoc3.13"
+
+# The bundled interpreter MUST be able to create a working venv, or every app
+# backend is dead on arrival. Prove it here, at build time, where the failure
+# is a red build instead of a user's broken app four layers downstream.
+VENV_PROBE="$(mktemp -d)/probe"
+if ! "$PYTHON_BIN" -m venv "$VENV_PROBE" >/dev/null 2>&1 || [ ! -x "$VENV_PROBE/bin/pip" ]; then
+    echo "FATAL: the bundled interpreter cannot create a venv with pip." >&2
+    echo "App backends would all fail with 'No module named pip'. Check the prune block above." >&2
+    rm -rf "$VENV_PROBE"
+    exit 1
+fi
+rm -rf "$VENV_PROBE"
+echo "Verified: bundled python can create a venv with pip."
 # pydoc_data: keyword/topic tables consumed only by stdlib `pydoc` / `help()`.
 # Backend never starts a REPL or calls help().
 rm -rf "$PYTHON_ENV_DIR/lib/python3.13/pydoc_data"