From 60414566ac46c09aabae2b6ce9f357dc8a1252f6 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sat, 23 May 2026 02:47:51 -0700 Subject: [PATCH] [eric] split: extract outputs html/workspace + runtime process helpers --- backend/apps/outputs/html_inject.py | 162 +++++++++++++++ backend/apps/outputs/outputs.py | 295 ++------------------------- backend/apps/outputs/prompts.py | 17 ++ backend/apps/outputs/runtime.py | 262 ++---------------------- backend/apps/outputs/runtime_proc.py | 252 +++++++++++++++++++++++ backend/apps/outputs/workspace_io.py | 117 +++++++++++ 6 files changed, 587 insertions(+), 518 deletions(-) create mode 100644 backend/apps/outputs/html_inject.py create mode 100644 backend/apps/outputs/prompts.py create mode 100644 backend/apps/outputs/runtime_proc.py create mode 100644 backend/apps/outputs/workspace_io.py diff --git a/backend/apps/outputs/html_inject.py b/backend/apps/outputs/html_inject.py new file mode 100644 index 00000000..c9fd8d1b --- /dev/null +++ b/backend/apps/outputs/html_inject.py @@ -0,0 +1,162 @@ +"""HTML data-injection + relative-URL token rewriting for served outputs. + +The token rewrite (`_inject_token_into_relative_urls`) is a security boundary: +iframe sub-resource fetches drop the parent's ?token= query, so the serve +routes re-stamp it onto every relative href/src or they 401. Keep it wired to +the serve routes.""" + +import base64 +import json +import logging +import re + +from jsonschema import validate as schema_validate, ValidationError as SchemaValidationError + +logger = logging.getLogger(__name__) + +MODEL_MAP = { + "sonnet": "claude-sonnet-4-20250514", + "opus": "claude-opus-4-20250514", + "haiku": "claude-haiku-4-5-20251001", +} + + +def _resolve_model(short_name: str) -> str: + return MODEL_MAP.get(short_name, short_name) + + +def _get_anthropic_client(api_model: str | None = None): + """Create an AsyncAnthropic client using the API key from app settings. + + When `api_model` is provided and carries a 9Router prefix (cc/, cx/, gc/), + the client is pointed at 9Router so non-Anthropic aux calls don't 400 on + api.anthropic.com. Without an api_model we fall back to the default + connection-mode-driven client. + """ + from backend.apps.settings.credentials import ( + get_anthropic_client, + get_anthropic_client_for_model, + ) + from backend.apps.settings.settings import load_settings + settings = load_settings() + if api_model: + return get_anthropic_client_for_model(settings, api_model) + return get_anthropic_client(settings) + + +def _validate_against_schema(data: dict, schema: dict) -> str | None: + """Validate *data* against *schema*. Return an error string or None.""" + try: + schema_validate(instance=data, schema=schema) + return None + except SchemaValidationError as exc: + path = " -> ".join(str(p) for p in exc.absolute_path) if exc.absolute_path else "(root)" + return f"Schema validation failed at {path}: {exc.message}" + + +def _build_data_injection(input_json: str, result_json: str, backend_url_json: str = "null") -> str: + """Build a " + ) + + +def _inject_data_into_html(html: str, input_json: str = "{}", result_json: str = "null", backend_url_json: str = "null") -> str: + injection = _build_data_injection(input_json, result_json, backend_url_json) + if "" in html: + return html.replace("", f"{injection}\n", 1) + if " str: + """Return the JSON-encoded backend URL for the given workspace, or + "null" if no runtime is active. Cheap inline lookup so serve_workspace_file + doesn't have to think about it.""" + try: + from backend.apps.outputs.runtime import manager as runtime_manager + rt = runtime_manager.get(workspace_id) + if rt and rt.running and rt.port: + return json.dumps(f"http://127.0.0.1:{rt.port}") + except Exception: + logger.exception("backend url lookup failed for %s", workspace_id) + return "null" + + +# URL schemes / prefixes that must NOT have ?token= appended. These are either +# external (CDNs, mailto) or non-network references that the auth middleware +# never sees. Anything else is treated as a same-origin relative URL pointing +# at our /api/outputs/.../serve/ subtree, which DOES need the token. +_ABSOLUTE_URL_PREFIXES = ( + "http://", "https://", "//", "data:", "blob:", + "mailto:", "tel:", "javascript:", "about:", "#", +) + +_HREF_SRC_ATTR_RE = re.compile( + r"""(\s(?:href|src))\s*=\s*(["'])([^"']+)\2""", + re.IGNORECASE, +) + + +def _inject_token_into_relative_urls(html: str, token: str) -> str: + """Append `?token=` to every relative href/src in the served HTML. + + Browsers strip the parent iframe URL's query string before resolving + relative `` / `" - ) - - -def _inject_data_into_html(html: str, input_json: str = "{}", result_json: str = "null", backend_url_json: str = "null") -> str: - injection = _build_data_injection(input_json, result_json, backend_url_json) - if "" in html: - return html.replace("", f"{injection}\n", 1) - if " str: - """Return the JSON-encoded backend URL for the given workspace, or - "null" if no runtime is active. Cheap inline lookup so serve_workspace_file - doesn't have to think about it.""" - try: - from backend.apps.outputs.runtime import manager as runtime_manager - rt = runtime_manager.get(workspace_id) - if rt and rt.running and rt.port: - return json.dumps(f"http://127.0.0.1:{rt.port}") - except Exception: - logger.exception("backend url lookup failed for %s", workspace_id) - return "null" - - -# URL schemes / prefixes that must NOT have ?token= appended. These are either -# external (CDNs, mailto) or non-network references that the auth middleware -# never sees. Anything else is treated as a same-origin relative URL pointing -# at our /api/outputs/.../serve/ subtree, which DOES need the token. -_ABSOLUTE_URL_PREFIXES = ( - "http://", "https://", "//", "data:", "blob:", - "mailto:", "tel:", "javascript:", "about:", "#", -) - -_HREF_SRC_ATTR_RE = re.compile( - r"""(\s(?:href|src))\s*=\s*(["'])([^"']+)\2""", - re.IGNORECASE, -) - - -def _inject_token_into_relative_urls(html: str, token: str) -> str: - """Append `?token=` to every relative href/src in the served HTML. - - Browsers strip the parent iframe URL's query string before resolving - relative `` / ` - - Input data is at window.OUTPUT_INPUT (object), backend result at window.OUTPUT_BACKEND_RESULT. -2. **input_schema**: A JSON Schema object defining the structured input. -3. **backend_code** (optional): Python code where input_data is a global dict and result is a global dict to assign to. -4. **name**: A short name for the view. -5. **description**: A one-sentence description. -6. **message**: A brief explanation of what you did/changed. - -Return ONLY valid JSON with these keys. No markdown fences, no extra text.\ -""" - - @outputs.router.post("/vibe-code") async def vibe_code(body: VibeCodeRequest): """Use an LLM to generate or iterate on Output code from a natural language prompt.""" diff --git a/backend/apps/outputs/prompts.py b/backend/apps/outputs/prompts.py new file mode 100644 index 00000000..508caf06 --- /dev/null +++ b/backend/apps/outputs/prompts.py @@ -0,0 +1,17 @@ +"""System prompts for the outputs SubApp's LLM-driven generators.""" + +VIBE_CODE_SYSTEM_PROMPT = """\ +You are an expert at building self-contained HTML/JS/CSS applications that run in an iframe. + +The user will describe what they want, and you will generate: +1. **frontend_code**: A complete HTML document. React 18 is available via esm.sh CDN. + - Use: + - Input data is at window.OUTPUT_INPUT (object), backend result at window.OUTPUT_BACKEND_RESULT. +2. **input_schema**: A JSON Schema object defining the structured input. +3. **backend_code** (optional): Python code where input_data is a global dict and result is a global dict to assign to. +4. **name**: A short name for the view. +5. **description**: A one-sentence description. +6. **message**: A brief explanation of what you did/changed. + +Return ONLY valid JSON with these keys. No markdown fences, no extra text.\ +""" diff --git a/backend/apps/outputs/runtime.py b/backend/apps/outputs/runtime.py index f8cf32f7..39c2e284 100644 --- a/backend/apps/outputs/runtime.py +++ b/backend/apps/outputs/runtime.py @@ -3,259 +3,35 @@ import asyncio import logging import os -import signal -import socket -import subprocess import sys from collections import deque, OrderedDict from dataclasses import dataclass from typing import Callable, Optional +from .runtime_proc import ( + _ERROR_PATTERNS, + _FRONTEND_BIND_POLL_INTERVAL, + _FRONTEND_BIND_TIMEOUT_SECONDS, + _LOG_BUFFER_LINES, + _MAX_IDLE_RUNTIMES, + _RECENT_ERRORS_MAX, + _TERMINATE_GRACE_SECONDS, + _background_priority_kwargs, + _find_free_port, + _is_new_mode, + _is_port_free, + _kill_descendant_tree, + _read_env_value, + _resume_process_tree, + _suspend_process_tree, + _write_env_value, +) + logger = logging.getLogger(__name__) -# 2000 lines per runtime; lets a Terminal tab opened mid-session replay context. ~few hundred KB at worst. -_LOG_BUFFER_LINES = 2000 - -# SIGTERM grace; well-behaved servers shut down under a second so 3s is enough. -_TERMINATE_GRACE_SECONDS = 3 - -# 180s covers npm install (60-90s on typical hardware) plus the Vite bind. -_FRONTEND_BIND_TIMEOUT_SECONDS = 180 -# 80ms probe: dropping from 500ms was pure user-visible preview latency win; cheap on localhost. -_FRONTEND_BIND_POLL_INTERVAL = 0.08 - - # Module-level lock so only ONE vite optimizeDeps runs at a time; must be acquired before manager._lock to avoid deadlock with manager.attach. _vite_boot_lock = asyncio.Lock() -# Idle runtimes kept in LRU; trades memory for instant switch-back, beyond 1 because typical users ping-pong 2-3 apps. -_MAX_IDLE_RUNTIMES = 3 - -# Cap on recent error lines the agent gets; 50 is enough for babel error + stack + a few warnings. -_RECENT_ERRORS_MAX = 50 - -# Narrow regex for build errors (vite, babel, tsc, uvicorn); keeps routine logs out of agent context. -import re as _re -_ERROR_PATTERNS = _re.compile( - r"(?:" - r"\[plugin:[^\]]+\]|" # vite plugin errors - r"SyntaxError|" # node / babel - r"Unexpected token|" # babel / tsc parser - r"\berror TS\d+|" # tsc diagnostics - r"ERROR\s+in\s|" # webpack-style - r"Traceback \(most recent call last\)|" # python - r"ModuleNotFoundError|" - r"ImportError|" - r"AttributeError:|" - r"Failed to compile|" - r"Cannot find module|" - r"Cannot resolve" - r")" -) - - -def _suspend_process_tree(proc: Optional[asyncio.subprocess.Process]) -> None: - """Send SIGSTOP to a workspace's subprocess so it consumes 0% CPU - while sitting in the LRU idle pool. The signal is delivered to the - PROCESS GROUP (negative PID) when the child is a session leader, - so vite + uvicorn + their npm/python subchildren all pause together. - - No-op on Windows (SIGSTOP has no equivalent; the `OpenProcessToken` + - `NtSuspendProcess` route works but isn't worth the win32 surface - here; idle Windows runtimes just stay running, which is the current - behavior). Failures here are swallowed; if the process already died - a stop signal is meaningless.""" - if proc is None or os.name == "nt": - return - try: - if proc.returncode is not None: - return - os.kill(proc.pid, signal.SIGSTOP) - except (ProcessLookupError, PermissionError, OSError): - # Already-dead or out-of-permission; both safe to ignore. - pass - - -def _resume_process_tree(proc: Optional[asyncio.subprocess.Process]) -> None: - """SIGCONT a previously-suspended workspace process. Pair with - _suspend_process_tree. Microsecond cost; idempotent if the process - was never paused.""" - if proc is None or os.name == "nt": - return - try: - if proc.returncode is not None: - return - os.kill(proc.pid, signal.SIGCONT) - except (ProcessLookupError, PermissionError, OSError): - pass - - -def _background_priority_kwargs() -> dict: - """Return the kwargs that lower the spawned subprocess's OS priority - to a "background" level. On POSIX this is `preexec_fn=os.nice(10)`, - which sets the child's nice to +10 BEFORE exec (so the renice covers - the entire bash → vite + uvicorn process tree). On Windows it's - `creationflags=BELOW_NORMAL_PRIORITY_CLASS`. The OS scheduler then - yields workspace cycles to whichever agent or browser tab is in the - user's foreground, so an in-background app build doesn't starve a - live chat session. - - We intentionally do NOT pass `start_new_session=True` here even - though it would defend against an errant `kill 0` inside the - workspace propagating into the OpenSwarm group: doing so also - detaches the workspace from the terminal's foreground process - group, so a user Ctrl+C only reaches OpenSwarm itself and the - cleanup path has to chase every workspace by hand. If that path - is even slightly slow or gets interrupted by a second Ctrl+C, the - workspace's uvicorn / vite leaks past shutdown and the next - `bash run.sh` hits Errno 48 on port 8324. The `kill 0` propagation - is fixed at its source in the workspace template's run.sh - (uses `kill_tree` on tracked PIDs, never `kill 0`).""" - if os.name == "nt": - # subprocess.BELOW_NORMAL_PRIORITY_CLASS == 0x4000 - return {"creationflags": subprocess.BELOW_NORMAL_PRIORITY_CLASS} - return {"preexec_fn": lambda: os.nice(10)} - - -def _find_free_port() -> int: - """Ask the kernel for an unused localhost port. There's a tiny race - between this socket closing and the backend re-binding, but we hand - each port to exactly one runtime so no caller competes for it, and - the kernel won't immediately recycle a freshly-closed port anyway.""" - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("127.0.0.1", 0)) - return s.getsockname()[1] - - -def _kill_descendant_tree(pid: int, sig_name: str = "TERM") -> None: - """Recursively signal every descendant of `pid`, leaves-first. The - webapp template's run.sh installs `trap cleanup EXIT` (no TERM), so a - plain SIGTERM to the bash wrapper exits bash silently and leaves - vite/uvicorn grandchildren reparented to PID 1, squatting on the - workspace's ports. Walking the tree ourselves bypasses the template's - signal-handling habits entirely. POSIX uses `pgrep -P` to enumerate - direct children; Windows is covered by `taskkill /T /F` (job-object - walk). All failures are swallowed; missing PIDs mean the process - already exited, which is the desired state anyway.""" - if os.name == "nt": - try: - subprocess.run( - ["taskkill", "/PID", str(pid), "/T", "/F"], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL, - timeout=5, - ) - except Exception: - pass - return - try: - out = subprocess.run( - ["pgrep", "-P", str(pid)], - capture_output=True, - text=True, - timeout=2, - ) - children = [int(p) for p in out.stdout.split() if p.strip().isdigit()] - except Exception: - children = [] - for child in children: - _kill_descendant_tree(child, sig_name) - sig = getattr(signal, f"SIG{sig_name}", signal.SIGTERM) - for child in children: - try: - os.kill(child, sig) - except (ProcessLookupError, PermissionError, OSError): - pass - - -def _is_port_free(port: int) -> bool: - """True if nothing currently holds a TCP listener on 127.0.0.1:port. - Cheap kernel-probe; resolves on bind success. Used as the cross-session - safety net: if a prior OpenSwarm run left a ghost subprocess holding - the .env-persisted FRONTEND_PORT, we detect it here and reallocate - rather than handing run.sh a port that will EADDRINUSE.""" - try: - with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: - s.bind(("127.0.0.1", port)) - return True - except OSError: - return False - - -def _write_env_value(env_path: str, key: str, value: str) -> None: - """Update KEY=VALUE in an existing `.env`, preserving every other - line. Creates the file if missing. Used when a persisted port collides - with a ghost from a prior session and we have to reallocate before - spawning run.sh.""" - lines: list[str] = [] - found = False - if os.path.exists(env_path): - try: - with open(env_path, encoding="utf-8") as f: - lines = f.readlines() - except Exception: - lines = [] - for i, raw in enumerate(lines): - stripped = raw.strip() - if not stripped or stripped.startswith("#") or "=" not in stripped: - continue - k = stripped.split("=", 1)[0].strip() - if k == key: - lines[i] = f"{key}={value}\n" - found = True - break - if not found: - if lines and not lines[-1].endswith("\n"): - lines[-1] = lines[-1] + "\n" - lines.append(f"{key}={value}\n") - try: - with open(env_path, "w", encoding="utf-8") as f: - f.writelines(lines) - except Exception: - logger.exception("failed writing %s=%s to %s", key, value, env_path) - - -def _is_new_mode(workspace_path: str) -> bool: - """A workspace is "new-mode" (webapp-template scaffold) if it has a - `run.sh` at its root. Old-mode workspaces are flat `index.html`-only - apps that pre-date the template swap; they're served by OpenSwarm's - own `/api/outputs/workspace/{ws}/serve/...` FastAPI route and have an - optional `backend.py` we spawn directly. - - Single-file probe so the check is cheap to call on every runtime - start, status query, and serve request.""" - return os.path.isfile(os.path.join(workspace_path, "run.sh")) - - -def _read_env_value(env_path: str, key: str) -> Optional[str]: - """Parse one value out of a workspace's `.env` without the cost of a - full subprocess-source. Strips quotes + trailing comments. Returns - None if the file or key is missing.""" - if not os.path.exists(env_path): - return None - try: - with open(env_path, encoding="utf-8") as f: - for raw in f: - line = raw.strip() - if not line or line.startswith("#"): - continue - if "=" not in line: - continue - k, _, v = line.partition("=") - if k.strip() != key: - continue - v = v.strip() - # Strip an inline `# comment`. Naive; bash semantics are - # more permissive, but values we write don't contain `#`. - if "#" in v: - v = v.split("#", 1)[0].rstrip() - if (v.startswith('"') and v.endswith('"')) or (v.startswith("'") and v.endswith("'")): - v = v[1:-1] - return v - except Exception: - logger.exception("failed reading %s from %s", key, env_path) - return None - @dataclass class LogLine: diff --git a/backend/apps/outputs/runtime_proc.py b/backend/apps/outputs/runtime_proc.py new file mode 100644 index 00000000..9d699444 --- /dev/null +++ b/backend/apps/outputs/runtime_proc.py @@ -0,0 +1,252 @@ +"""OS/process/port primitives for the per-workspace runtime: signal-based +suspend/resume, descendant-tree kills, free-port allocation, and .env +read/write. No asyncio runtime state lives here; AppRuntime (runtime.py) owns +that and just calls into these.""" + +import logging +import os +import re +import signal +import socket +import subprocess +from typing import Optional + +logger = logging.getLogger(__name__) + +# SIGTERM grace; well-behaved servers shut down under a second so 3s is enough. +_TERMINATE_GRACE_SECONDS = 3 + +# 180s covers npm install (60-90s on typical hardware) plus the Vite bind. +_FRONTEND_BIND_TIMEOUT_SECONDS = 180 +# 80ms probe: dropping from 500ms was pure user-visible preview latency win; cheap on localhost. +_FRONTEND_BIND_POLL_INTERVAL = 0.08 + +# 2000 lines per runtime; lets a Terminal tab opened mid-session replay context. ~few hundred KB at worst. +_LOG_BUFFER_LINES = 2000 + +# Idle runtimes kept in LRU; trades memory for instant switch-back, beyond 1 because typical users ping-pong 2-3 apps. +_MAX_IDLE_RUNTIMES = 3 + +# Cap on recent error lines the agent gets; 50 is enough for babel error + stack + a few warnings. +_RECENT_ERRORS_MAX = 50 + +# Narrow regex for build errors (vite, babel, tsc, uvicorn); keeps routine logs out of agent context. +_ERROR_PATTERNS = re.compile( + r"(?:" + r"\[plugin:[^\]]+\]|" # vite plugin errors + r"SyntaxError|" # node / babel + r"Unexpected token|" # babel / tsc parser + r"\berror TS\d+|" # tsc diagnostics + r"ERROR\s+in\s|" # webpack-style + r"Traceback \(most recent call last\)|" # python + r"ModuleNotFoundError|" + r"ImportError|" + r"AttributeError:|" + r"Failed to compile|" + r"Cannot find module|" + r"Cannot resolve" + r")" +) + + +def _suspend_process_tree(proc) -> None: + """Send SIGSTOP to a workspace's subprocess so it consumes 0% CPU + while sitting in the LRU idle pool. The signal is delivered to the + PROCESS GROUP (negative PID) when the child is a session leader, + so vite + uvicorn + their npm/python subchildren all pause together. + + No-op on Windows (SIGSTOP has no equivalent; the `OpenProcessToken` + + `NtSuspendProcess` route works but isn't worth the win32 surface + here; idle Windows runtimes just stay running, which is the current + behavior). Failures here are swallowed; if the process already died + a stop signal is meaningless.""" + if proc is None or os.name == "nt": + return + try: + if proc.returncode is not None: + return + os.kill(proc.pid, signal.SIGSTOP) + except (ProcessLookupError, PermissionError, OSError): + # Already-dead or out-of-permission; both safe to ignore. + pass + + +def _resume_process_tree(proc) -> None: + """SIGCONT a previously-suspended workspace process. Pair with + _suspend_process_tree. Microsecond cost; idempotent if the process + was never paused.""" + if proc is None or os.name == "nt": + return + try: + if proc.returncode is not None: + return + os.kill(proc.pid, signal.SIGCONT) + except (ProcessLookupError, PermissionError, OSError): + pass + + +def _background_priority_kwargs() -> dict: + """Return the kwargs that lower the spawned subprocess's OS priority + to a "background" level. On POSIX this is `preexec_fn=os.nice(10)`, + which sets the child's nice to +10 BEFORE exec (so the renice covers + the entire bash → vite + uvicorn process tree). On Windows it's + `creationflags=BELOW_NORMAL_PRIORITY_CLASS`. The OS scheduler then + yields workspace cycles to whichever agent or browser tab is in the + user's foreground, so an in-background app build doesn't starve a + live chat session. + + We intentionally do NOT pass `start_new_session=True` here even + though it would defend against an errant `kill 0` inside the + workspace propagating into the OpenSwarm group: doing so also + detaches the workspace from the terminal's foreground process + group, so a user Ctrl+C only reaches OpenSwarm itself and the + cleanup path has to chase every workspace by hand. If that path + is even slightly slow or gets interrupted by a second Ctrl+C, the + workspace's uvicorn / vite leaks past shutdown and the next + `bash run.sh` hits Errno 48 on port 8324. The `kill 0` propagation + is fixed at its source in the workspace template's run.sh + (uses `kill_tree` on tracked PIDs, never `kill 0`).""" + if os.name == "nt": + # subprocess.BELOW_NORMAL_PRIORITY_CLASS == 0x4000 + return {"creationflags": subprocess.BELOW_NORMAL_PRIORITY_CLASS} + return {"preexec_fn": lambda: os.nice(10)} + + +def _find_free_port() -> int: + """Ask the kernel for an unused localhost port. There's a tiny race + between this socket closing and the backend re-binding, but we hand + each port to exactly one runtime so no caller competes for it, and + the kernel won't immediately recycle a freshly-closed port anyway.""" + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", 0)) + return s.getsockname()[1] + + +def _kill_descendant_tree(pid: int, sig_name: str = "TERM") -> None: + """Recursively signal every descendant of `pid`, leaves-first. The + webapp template's run.sh installs `trap cleanup EXIT` (no TERM), so a + plain SIGTERM to the bash wrapper exits bash silently and leaves + vite/uvicorn grandchildren reparented to PID 1, squatting on the + workspace's ports. Walking the tree ourselves bypasses the template's + signal-handling habits entirely. POSIX uses `pgrep -P` to enumerate + direct children; Windows is covered by `taskkill /T /F` (job-object + walk). All failures are swallowed; missing PIDs mean the process + already exited, which is the desired state anyway.""" + if os.name == "nt": + try: + subprocess.run( + ["taskkill", "/PID", str(pid), "/T", "/F"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL, + timeout=5, + ) + except Exception: + pass + return + try: + out = subprocess.run( + ["pgrep", "-P", str(pid)], + capture_output=True, + text=True, + timeout=2, + ) + children = [int(p) for p in out.stdout.split() if p.strip().isdigit()] + except Exception: + children = [] + for child in children: + _kill_descendant_tree(child, sig_name) + sig = getattr(signal, f"SIG{sig_name}", signal.SIGTERM) + for child in children: + try: + os.kill(child, sig) + except (ProcessLookupError, PermissionError, OSError): + pass + + +def _is_port_free(port: int) -> bool: + """True if nothing currently holds a TCP listener on 127.0.0.1:port. + Cheap kernel-probe; resolves on bind success. Used as the cross-session + safety net: if a prior OpenSwarm run left a ghost subprocess holding + the .env-persisted FRONTEND_PORT, we detect it here and reallocate + rather than handing run.sh a port that will EADDRINUSE.""" + try: + with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s: + s.bind(("127.0.0.1", port)) + return True + except OSError: + return False + + +def _write_env_value(env_path: str, key: str, value: str) -> None: + """Update KEY=VALUE in an existing `.env`, preserving every other + line. Creates the file if missing. Used when a persisted port collides + with a ghost from a prior session and we have to reallocate before + spawning run.sh.""" + lines: list[str] = [] + found = False + if os.path.exists(env_path): + try: + with open(env_path, encoding="utf-8") as f: + lines = f.readlines() + except Exception: + lines = [] + for i, raw in enumerate(lines): + stripped = raw.strip() + if not stripped or stripped.startswith("#") or "=" not in stripped: + continue + k = stripped.split("=", 1)[0].strip() + if k == key: + lines[i] = f"{key}={value}\n" + found = True + break + if not found: + if lines and not lines[-1].endswith("\n"): + lines[-1] = lines[-1] + "\n" + lines.append(f"{key}={value}\n") + try: + with open(env_path, "w", encoding="utf-8") as f: + f.writelines(lines) + except Exception: + logger.exception("failed writing %s=%s to %s", key, value, env_path) + + +def _is_new_mode(workspace_path: str) -> bool: + """A workspace is "new-mode" (webapp-template scaffold) if it has a + `run.sh` at its root. Old-mode workspaces are flat `index.html`-only + apps that pre-date the template swap; they're served by OpenSwarm's + own `/api/outputs/workspace/{ws}/serve/...` FastAPI route and have an + optional `backend.py` we spawn directly. + + Single-file probe so the check is cheap to call on every runtime + start, status query, and serve request.""" + return os.path.isfile(os.path.join(workspace_path, "run.sh")) + + +def _read_env_value(env_path: str, key: str) -> Optional[str]: + """Parse one value out of a workspace's `.env` without the cost of a + full subprocess-source. Strips quotes + trailing comments. Returns + None if the file or key is missing.""" + if not os.path.exists(env_path): + return None + try: + with open(env_path, encoding="utf-8") as f: + for raw in f: + line = raw.strip() + if not line or line.startswith("#"): + continue + if "=" not in line: + continue + k, _, v = line.partition("=") + if k.strip() != key: + continue + v = v.strip() + # Strip an inline `# comment`. Naive; bash semantics are + # more permissive, but values we write don't contain `#`. + if "#" in v: + v = v.split("#", 1)[0].rstrip() + if (v.startswith('"') and v.endswith('"')) or (v.startswith("'") and v.endswith("'")): + v = v[1:-1] + return v + except Exception: + logger.exception("failed reading %s from %s", key, env_path) + return None diff --git a/backend/apps/outputs/workspace_io.py b/backend/apps/outputs/workspace_io.py new file mode 100644 index 00000000..1e35ed62 --- /dev/null +++ b/backend/apps/outputs/workspace_io.py @@ -0,0 +1,117 @@ +"""On-disk persistence for outputs: Output JSON records under DATA_DIR and +the workspace file-tree walk used by the polling read endpoint.""" + +import json +import os + +from fastapi import HTTPException + +from backend.apps.outputs.models import Output +from backend.config.paths import OUTPUTS_DIR as DATA_DIR + + +def _load_all() -> list[Output]: + result = [] + if not os.path.exists(DATA_DIR): + return result + for fname in os.listdir(DATA_DIR): + if fname.endswith(".json"): + with open(os.path.join(DATA_DIR, fname)) as f: + result.append(Output(**json.load(f))) + return result + + +def _save(output: Output): + with open(os.path.join(DATA_DIR, f"{output.id}.json"), "w") as f: + json.dump(output.model_dump(), f, indent=2) + + +def _load(output_id: str) -> Output: + path = os.path.join(DATA_DIR, f"{output_id}.json") + if not os.path.exists(path): + raise HTTPException(status_code=404, detail="Output not found") + with open(path) as f: + return Output(**json.load(f)) + + +def load_output(output_id: str) -> Output | None: + """Public helper for other modules to resolve an output by ID.""" + path = os.path.join(DATA_DIR, f"{output_id}.json") + if not os.path.exists(path): + return None + with open(path) as f: + return Output(**json.load(f)) + + +# Build/install/cache directories that the polling endpoint must never +# descend into. Without this skip-list the workspace endpoint reads +# `node_modules/` (300 MB of MUI source, when it's a real dir and not a +# symlink), `.venv/` (10k+ Python files from the hardlinked cache), +# `__pycache__/`, `dist/`, `.git/`, etc; every 2 seconds while the +# agent is active. Result: backend CPU pegged on JSON-serializing +# auto-generated chunks the frontend will then throw away. The frontend +# already filters these for display; this skip is the real fix. +_WALK_SKIP_DIRS = frozenset({ + "node_modules", + ".vite", + ".vite-cache", + ".vite_cache", + ".git", + "dist", + ".next", + "__pycache__", + ".venv", + "venv", + ".pytest_cache", + ".mypy_cache", + ".ruff_cache", +}) + +# Cap per-file response size at 256 KB. Hand-written source rarely +# exceeds this; auto-generated bundles routinely run into the MBs and +# they're not what the user/agent is editing. Anything over the cap +# returns a truncated stub the frontend treats as "open the file +# directly to see full contents." +_WALK_MAX_FILE_BYTES = 256 * 1024 + + +def _walk_directory(folder: str) -> dict[str, str]: + """Walk a directory tree and return {relative_path: content} for all + text files the user is actually authoring. Skips build/install + directories AND truncates oversize files; both critical for the + polling endpoint, which is called every 2 s while the agent is + writing code and would otherwise serialize hundreds of MB per poll.""" + files: dict[str, str] = {} + if not os.path.isdir(folder): + return files + for root, dirs, filenames in os.walk(folder): + # Mutate `dirs` in place; that's how os.walk skips a subtree. + # Doing it here means we never even stat the children, so a + # 10k-file `.venv/` costs ~one stat (on the dir itself) instead + # of 10k. + dirs[:] = [d for d in dirs if d not in _WALK_SKIP_DIRS] + for fname in filenames: + full_path = os.path.join(root, fname) + # Normalize to forward-slash keys so the frontend's + # `path.split('/')` and `.startsWith(prefix)` checks work + # the same on Windows (where os.sep is '\\') as on macOS. + # Without this, every workspace file came back as + # `backend\\app.py` on Windows and the file tree silently + # mis-parsed. + rel_path = os.path.relpath(full_path, folder).replace(os.sep, "/") + try: + # Stat first; cheap, lets us skip giant files without + # opening + reading them. + size = os.path.getsize(full_path) + if size > _WALK_MAX_FILE_BYTES: + files[rel_path] = ( + f"// [openswarm] file truncated ({size} bytes > " + f"{_WALK_MAX_FILE_BYTES} byte cap). Open directly " + f"to view full contents." + ) + continue + with open(full_path) as f: + files[rel_path] = f.read() + except Exception: + pass + return files