mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-07 10:17:43 +02:00
[eric] split: extract outputs html/workspace + runtime process helpers
This commit is contained in:
ciregenz
@@ -0,0 +1,162 @@
|
||||
"""HTML data-injection + relative-URL token rewriting for served outputs.
|
||||
|
||||
The token rewrite (`_inject_token_into_relative_urls`) is a security boundary:
|
||||
iframe sub-resource fetches drop the parent's ?token= query, so the serve
|
||||
routes re-stamp it onto every relative href/src or they 401. Keep it wired to
|
||||
the serve routes."""
|
||||
|
||||
import base64
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
|
||||
from jsonschema import validate as schema_validate, ValidationError as SchemaValidationError
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MODEL_MAP = {
|
||||
"sonnet": "claude-sonnet-4-20250514",
|
||||
"opus": "claude-opus-4-20250514",
|
||||
"haiku": "claude-haiku-4-5-20251001",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_model(short_name: str) -> str:
|
||||
return MODEL_MAP.get(short_name, short_name)
|
||||
|
||||
|
||||
def _get_anthropic_client(api_model: str | None = None):
|
||||
"""Create an AsyncAnthropic client using the API key from app settings.
|
||||
|
||||
When `api_model` is provided and carries a 9Router prefix (cc/, cx/, gc/),
|
||||
the client is pointed at 9Router so non-Anthropic aux calls don't 400 on
|
||||
api.anthropic.com. Without an api_model we fall back to the default
|
||||
connection-mode-driven client.
|
||||
"""
|
||||
from backend.apps.settings.credentials import (
|
||||
get_anthropic_client,
|
||||
get_anthropic_client_for_model,
|
||||
)
|
||||
from backend.apps.settings.settings import load_settings
|
||||
settings = load_settings()
|
||||
if api_model:
|
||||
return get_anthropic_client_for_model(settings, api_model)
|
||||
return get_anthropic_client(settings)
|
||||
|
||||
|
||||
def _validate_against_schema(data: dict, schema: dict) -> str | None:
|
||||
"""Validate *data* against *schema*. Return an error string or None."""
|
||||
try:
|
||||
schema_validate(instance=data, schema=schema)
|
||||
return None
|
||||
except SchemaValidationError as exc:
|
||||
path = " -> ".join(str(p) for p in exc.absolute_path) if exc.absolute_path else "(root)"
|
||||
return f"Schema validation failed at {path}: {exc.message}"
|
||||
|
||||
|
||||
def _build_data_injection(input_json: str, result_json: str, backend_url_json: str = "null") -> str:
|
||||
"""Build a <script> tag that sets OUTPUT_INPUT / OUTPUT_BACKEND_RESULT /
|
||||
OUTPUT_BACKEND_URL and listens for postMessage updates.
|
||||
|
||||
OUTPUT_BACKEND_URL is `null` when the app has no live `backend.py`
|
||||
process; otherwise it's `http://localhost:<port>` and app code can
|
||||
`fetch(window.OUTPUT_BACKEND_URL + '/route')` to hit the persistent
|
||||
backend's endpoints."""
|
||||
return (
|
||||
"<script>\n"
|
||||
"(function() {\n"
|
||||
" window.OUTPUT_INPUT = " + input_json + ";\n"
|
||||
" window.OUTPUT_BACKEND_RESULT = " + result_json + ";\n"
|
||||
" window.OUTPUT_BACKEND_URL = " + backend_url_json + ";\n"
|
||||
" window.addEventListener('message', function(e) {\n"
|
||||
" if (e.data && e.data.type === 'OUTPUT_DATA') {\n"
|
||||
" window.OUTPUT_INPUT = e.data.input || {};\n"
|
||||
" window.OUTPUT_BACKEND_RESULT = e.data.backendResult || null;\n"
|
||||
" if (e.data.backendUrl !== undefined) window.OUTPUT_BACKEND_URL = e.data.backendUrl;\n"
|
||||
" window.dispatchEvent(new CustomEvent('output-data-ready'));\n"
|
||||
" }\n"
|
||||
" });\n"
|
||||
"})();\n"
|
||||
"</script>"
|
||||
)
|
||||
|
||||
|
||||
def _inject_data_into_html(html: str, input_json: str = "{}", result_json: str = "null", backend_url_json: str = "null") -> str:
|
||||
injection = _build_data_injection(input_json, result_json, backend_url_json)
|
||||
if "</head>" in html:
|
||||
return html.replace("</head>", f"{injection}\n</head>", 1)
|
||||
if "<body" in html:
|
||||
return html.replace("<body", f"{injection}\n<body", 1)
|
||||
return f"{injection}\n{html}"
|
||||
|
||||
|
||||
def _backend_url_for_workspace(workspace_id: str) -> str:
|
||||
"""Return the JSON-encoded backend URL for the given workspace, or
|
||||
"null" if no runtime is active. Cheap inline lookup so serve_workspace_file
|
||||
doesn't have to think about it."""
|
||||
try:
|
||||
from backend.apps.outputs.runtime import manager as runtime_manager
|
||||
rt = runtime_manager.get(workspace_id)
|
||||
if rt and rt.running and rt.port:
|
||||
return json.dumps(f"http://127.0.0.1:{rt.port}")
|
||||
except Exception:
|
||||
logger.exception("backend url lookup failed for %s", workspace_id)
|
||||
return "null"
|
||||
|
||||
|
||||
# URL schemes / prefixes that must NOT have ?token= appended. These are either
|
||||
# external (CDNs, mailto) or non-network references that the auth middleware
|
||||
# never sees. Anything else is treated as a same-origin relative URL pointing
|
||||
# at our /api/outputs/.../serve/ subtree, which DOES need the token.
|
||||
_ABSOLUTE_URL_PREFIXES = (
|
||||
"http://", "https://", "//", "data:", "blob:",
|
||||
"mailto:", "tel:", "javascript:", "about:", "#",
|
||||
)
|
||||
|
||||
_HREF_SRC_ATTR_RE = re.compile(
|
||||
r"""(\s(?:href|src))\s*=\s*(["'])([^"']+)\2""",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _inject_token_into_relative_urls(html: str, token: str) -> str:
|
||||
"""Append `?token=<t>` to every relative href/src in the served HTML.
|
||||
|
||||
Browsers strip the parent iframe URL's query string before resolving
|
||||
relative `<link href="styles.css">` / `<script src="x.js">`, so without
|
||||
this rewrite the sub-resource fetch lands at the auth middleware with no
|
||||
credentials and gets a 401. Idempotent: skips URLs that already carry a
|
||||
`token=` param. Skips absolute URLs (CDN, data:, etc.); see prefix list.
|
||||
"""
|
||||
if not token:
|
||||
return html
|
||||
|
||||
def _patch(match: re.Match) -> str:
|
||||
attr, quote, url = match.group(1), match.group(2), match.group(3)
|
||||
lowered = url.lower().lstrip()
|
||||
if lowered.startswith(_ABSOLUTE_URL_PREFIXES):
|
||||
return match.group(0)
|
||||
if "token=" in url:
|
||||
return match.group(0)
|
||||
# Split off any hash fragment so `?token=` lands in the query, not in
|
||||
# the fragment: `page.html?v=1#sec` → `page.html?v=1&token=X#sec`.
|
||||
hash_idx = url.find("#")
|
||||
if hash_idx >= 0:
|
||||
base, frag = url[:hash_idx], url[hash_idx:]
|
||||
else:
|
||||
base, frag = url, ""
|
||||
sep = "&" if "?" in base else "?"
|
||||
return f'{attr}={quote}{base}{sep}token={token}{frag}{quote}'
|
||||
|
||||
return _HREF_SRC_ATTR_RE.sub(_patch, html)
|
||||
|
||||
|
||||
def _decode_data_param(d: str) -> tuple[str, str]:
|
||||
"""Decode the base64-encoded _d query param into (input_json, result_json)."""
|
||||
try:
|
||||
decoded = json.loads(base64.b64decode(d))
|
||||
input_json = json.dumps(decoded.get("i", {}))
|
||||
result_json = json.dumps(decoded.get("r", None))
|
||||
return input_json, result_json
|
||||
except Exception:
|
||||
return "{}", "null"
|
||||
+20
-275
@@ -1,16 +1,13 @@
|
||||
import json
|
||||
import os
|
||||
import re
|
||||
import logging
|
||||
import mimetypes
|
||||
import base64
|
||||
from datetime import datetime
|
||||
from typing import Optional
|
||||
from contextlib import asynccontextmanager
|
||||
from fastapi import HTTPException, Query
|
||||
from fastapi.responses import Response
|
||||
from backend.auth import get_auth_token
|
||||
from jsonschema import validate as schema_validate, ValidationError as SchemaValidationError
|
||||
from backend.config.Apps import SubApp
|
||||
from backend.apps.outputs.models import (
|
||||
Output, OutputCreate, OutputUpdate, OutputExecute, OutputExecuteResult,
|
||||
@@ -23,157 +20,29 @@ from backend.apps.outputs.view_builder_templates import (
|
||||
seed_webapp_template_workspace,
|
||||
)
|
||||
from backend.apps.settings.settings import load_settings
|
||||
from backend.config.paths import OUTPUTS_DIR as DATA_DIR, OUTPUTS_WORKSPACE_DIR as WORKSPACE_DIR
|
||||
from backend.apps.outputs.html_inject import (
|
||||
MODEL_MAP,
|
||||
_resolve_model,
|
||||
_get_anthropic_client,
|
||||
_validate_against_schema,
|
||||
_build_data_injection,
|
||||
_inject_data_into_html,
|
||||
_backend_url_for_workspace,
|
||||
_inject_token_into_relative_urls,
|
||||
_decode_data_param,
|
||||
)
|
||||
from backend.apps.outputs.workspace_io import (
|
||||
_load_all,
|
||||
_save,
|
||||
_load,
|
||||
load_output,
|
||||
_walk_directory,
|
||||
)
|
||||
from backend.apps.outputs.prompts import VIBE_CODE_SYSTEM_PROMPT
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
MODEL_MAP = {
|
||||
"sonnet": "claude-sonnet-4-20250514",
|
||||
"opus": "claude-opus-4-20250514",
|
||||
"haiku": "claude-haiku-4-5-20251001",
|
||||
}
|
||||
|
||||
|
||||
def _resolve_model(short_name: str) -> str:
|
||||
return MODEL_MAP.get(short_name, short_name)
|
||||
|
||||
|
||||
def _get_anthropic_client(api_model: str | None = None):
|
||||
"""Create an AsyncAnthropic client using the API key from app settings.
|
||||
|
||||
When `api_model` is provided and carries a 9Router prefix (cc/, cx/, gc/),
|
||||
the client is pointed at 9Router so non-Anthropic aux calls don't 400 on
|
||||
api.anthropic.com. Without an api_model we fall back to the default
|
||||
connection-mode-driven client.
|
||||
"""
|
||||
from backend.apps.settings.credentials import (
|
||||
get_anthropic_client,
|
||||
get_anthropic_client_for_model,
|
||||
)
|
||||
settings = load_settings()
|
||||
if api_model:
|
||||
return get_anthropic_client_for_model(settings, api_model)
|
||||
return get_anthropic_client(settings)
|
||||
|
||||
|
||||
def _validate_against_schema(data: dict, schema: dict) -> str | None:
|
||||
"""Validate *data* against *schema*. Return an error string or None."""
|
||||
try:
|
||||
schema_validate(instance=data, schema=schema)
|
||||
return None
|
||||
except SchemaValidationError as exc:
|
||||
path = " -> ".join(str(p) for p in exc.absolute_path) if exc.absolute_path else "(root)"
|
||||
return f"Schema validation failed at {path}: {exc.message}"
|
||||
|
||||
from backend.config.paths import OUTPUTS_DIR as DATA_DIR, OUTPUTS_WORKSPACE_DIR as WORKSPACE_DIR
|
||||
|
||||
|
||||
def _build_data_injection(input_json: str, result_json: str, backend_url_json: str = "null") -> str:
|
||||
"""Build a <script> tag that sets OUTPUT_INPUT / OUTPUT_BACKEND_RESULT /
|
||||
OUTPUT_BACKEND_URL and listens for postMessage updates.
|
||||
|
||||
OUTPUT_BACKEND_URL is `null` when the app has no live `backend.py`
|
||||
process; otherwise it's `http://localhost:<port>` and app code can
|
||||
`fetch(window.OUTPUT_BACKEND_URL + '/route')` to hit the persistent
|
||||
backend's endpoints."""
|
||||
return (
|
||||
"<script>\n"
|
||||
"(function() {\n"
|
||||
" window.OUTPUT_INPUT = " + input_json + ";\n"
|
||||
" window.OUTPUT_BACKEND_RESULT = " + result_json + ";\n"
|
||||
" window.OUTPUT_BACKEND_URL = " + backend_url_json + ";\n"
|
||||
" window.addEventListener('message', function(e) {\n"
|
||||
" if (e.data && e.data.type === 'OUTPUT_DATA') {\n"
|
||||
" window.OUTPUT_INPUT = e.data.input || {};\n"
|
||||
" window.OUTPUT_BACKEND_RESULT = e.data.backendResult || null;\n"
|
||||
" if (e.data.backendUrl !== undefined) window.OUTPUT_BACKEND_URL = e.data.backendUrl;\n"
|
||||
" window.dispatchEvent(new CustomEvent('output-data-ready'));\n"
|
||||
" }\n"
|
||||
" });\n"
|
||||
"})();\n"
|
||||
"</script>"
|
||||
)
|
||||
|
||||
|
||||
def _inject_data_into_html(html: str, input_json: str = "{}", result_json: str = "null", backend_url_json: str = "null") -> str:
|
||||
injection = _build_data_injection(input_json, result_json, backend_url_json)
|
||||
if "</head>" in html:
|
||||
return html.replace("</head>", f"{injection}\n</head>", 1)
|
||||
if "<body" in html:
|
||||
return html.replace("<body", f"{injection}\n<body", 1)
|
||||
return f"{injection}\n{html}"
|
||||
|
||||
|
||||
def _backend_url_for_workspace(workspace_id: str) -> str:
|
||||
"""Return the JSON-encoded backend URL for the given workspace, or
|
||||
"null" if no runtime is active. Cheap inline lookup so serve_workspace_file
|
||||
doesn't have to think about it."""
|
||||
try:
|
||||
from backend.apps.outputs.runtime import manager as runtime_manager
|
||||
rt = runtime_manager.get(workspace_id)
|
||||
if rt and rt.running and rt.port:
|
||||
return json.dumps(f"http://127.0.0.1:{rt.port}")
|
||||
except Exception:
|
||||
logger.exception("backend url lookup failed for %s", workspace_id)
|
||||
return "null"
|
||||
|
||||
|
||||
# URL schemes / prefixes that must NOT have ?token= appended. These are either
|
||||
# external (CDNs, mailto) or non-network references that the auth middleware
|
||||
# never sees. Anything else is treated as a same-origin relative URL pointing
|
||||
# at our /api/outputs/.../serve/ subtree, which DOES need the token.
|
||||
_ABSOLUTE_URL_PREFIXES = (
|
||||
"http://", "https://", "//", "data:", "blob:",
|
||||
"mailto:", "tel:", "javascript:", "about:", "#",
|
||||
)
|
||||
|
||||
_HREF_SRC_ATTR_RE = re.compile(
|
||||
r"""(\s(?:href|src))\s*=\s*(["'])([^"']+)\2""",
|
||||
re.IGNORECASE,
|
||||
)
|
||||
|
||||
|
||||
def _inject_token_into_relative_urls(html: str, token: str) -> str:
|
||||
"""Append `?token=<t>` to every relative href/src in the served HTML.
|
||||
|
||||
Browsers strip the parent iframe URL's query string before resolving
|
||||
relative `<link href="styles.css">` / `<script src="x.js">`, so without
|
||||
this rewrite the sub-resource fetch lands at the auth middleware with no
|
||||
credentials and gets a 401. Idempotent: skips URLs that already carry a
|
||||
`token=` param. Skips absolute URLs (CDN, data:, etc.); see prefix list.
|
||||
"""
|
||||
if not token:
|
||||
return html
|
||||
|
||||
def _patch(match: re.Match) -> str:
|
||||
attr, quote, url = match.group(1), match.group(2), match.group(3)
|
||||
lowered = url.lower().lstrip()
|
||||
if lowered.startswith(_ABSOLUTE_URL_PREFIXES):
|
||||
return match.group(0)
|
||||
if "token=" in url:
|
||||
return match.group(0)
|
||||
# Split off any hash fragment so `?token=` lands in the query, not in
|
||||
# the fragment: `page.html?v=1#sec` → `page.html?v=1&token=X#sec`.
|
||||
hash_idx = url.find("#")
|
||||
if hash_idx >= 0:
|
||||
base, frag = url[:hash_idx], url[hash_idx:]
|
||||
else:
|
||||
base, frag = url, ""
|
||||
sep = "&" if "?" in base else "?"
|
||||
return f'{attr}={quote}{base}{sep}token={token}{frag}{quote}'
|
||||
|
||||
return _HREF_SRC_ATTR_RE.sub(_patch, html)
|
||||
|
||||
|
||||
def _decode_data_param(d: str) -> tuple[str, str]:
|
||||
"""Decode the base64-encoded _d query param into (input_json, result_json)."""
|
||||
try:
|
||||
decoded = json.loads(base64.b64decode(d))
|
||||
input_json = json.dumps(decoded.get("i", {}))
|
||||
result_json = json.dumps(decoded.get("r", None))
|
||||
return input_json, result_json
|
||||
except Exception:
|
||||
return "{}", "null"
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def outputs_lifespan():
|
||||
@@ -198,113 +67,6 @@ async def outputs_lifespan():
|
||||
outputs = SubApp("outputs", outputs_lifespan)
|
||||
|
||||
|
||||
def _load_all() -> list[Output]:
|
||||
result = []
|
||||
if not os.path.exists(DATA_DIR):
|
||||
return result
|
||||
for fname in os.listdir(DATA_DIR):
|
||||
if fname.endswith(".json"):
|
||||
with open(os.path.join(DATA_DIR, fname)) as f:
|
||||
result.append(Output(**json.load(f)))
|
||||
return result
|
||||
|
||||
|
||||
def _save(output: Output):
|
||||
with open(os.path.join(DATA_DIR, f"{output.id}.json"), "w") as f:
|
||||
json.dump(output.model_dump(), f, indent=2)
|
||||
|
||||
|
||||
def _load(output_id: str) -> Output:
|
||||
path = os.path.join(DATA_DIR, f"{output_id}.json")
|
||||
if not os.path.exists(path):
|
||||
raise HTTPException(status_code=404, detail="Output not found")
|
||||
with open(path) as f:
|
||||
return Output(**json.load(f))
|
||||
|
||||
|
||||
def load_output(output_id: str) -> Output | None:
|
||||
"""Public helper for other modules to resolve an output by ID."""
|
||||
path = os.path.join(DATA_DIR, f"{output_id}.json")
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
with open(path) as f:
|
||||
return Output(**json.load(f))
|
||||
|
||||
|
||||
# Build/install/cache directories that the polling endpoint must never
|
||||
# descend into. Without this skip-list the workspace endpoint reads
|
||||
# `node_modules/` (300 MB of MUI source, when it's a real dir and not a
|
||||
# symlink), `.venv/` (10k+ Python files from the hardlinked cache),
|
||||
# `__pycache__/`, `dist/`, `.git/`, etc; every 2 seconds while the
|
||||
# agent is active. Result: backend CPU pegged on JSON-serializing
|
||||
# auto-generated chunks the frontend will then throw away. The frontend
|
||||
# already filters these for display; this skip is the real fix.
|
||||
_WALK_SKIP_DIRS = frozenset({
|
||||
"node_modules",
|
||||
".vite",
|
||||
".vite-cache",
|
||||
".vite_cache",
|
||||
".git",
|
||||
"dist",
|
||||
".next",
|
||||
"__pycache__",
|
||||
".venv",
|
||||
"venv",
|
||||
".pytest_cache",
|
||||
".mypy_cache",
|
||||
".ruff_cache",
|
||||
})
|
||||
|
||||
# Cap per-file response size at 256 KB. Hand-written source rarely
|
||||
# exceeds this; auto-generated bundles routinely run into the MBs and
|
||||
# they're not what the user/agent is editing. Anything over the cap
|
||||
# returns a truncated stub the frontend treats as "open the file
|
||||
# directly to see full contents."
|
||||
_WALK_MAX_FILE_BYTES = 256 * 1024
|
||||
|
||||
|
||||
def _walk_directory(folder: str) -> dict[str, str]:
|
||||
"""Walk a directory tree and return {relative_path: content} for all
|
||||
text files the user is actually authoring. Skips build/install
|
||||
directories AND truncates oversize files; both critical for the
|
||||
polling endpoint, which is called every 2 s while the agent is
|
||||
writing code and would otherwise serialize hundreds of MB per poll."""
|
||||
files: dict[str, str] = {}
|
||||
if not os.path.isdir(folder):
|
||||
return files
|
||||
for root, dirs, filenames in os.walk(folder):
|
||||
# Mutate `dirs` in place; that's how os.walk skips a subtree.
|
||||
# Doing it here means we never even stat the children, so a
|
||||
# 10k-file `.venv/` costs ~one stat (on the dir itself) instead
|
||||
# of 10k.
|
||||
dirs[:] = [d for d in dirs if d not in _WALK_SKIP_DIRS]
|
||||
for fname in filenames:
|
||||
full_path = os.path.join(root, fname)
|
||||
# Normalize to forward-slash keys so the frontend's
|
||||
# `path.split('/')` and `.startsWith(prefix)` checks work
|
||||
# the same on Windows (where os.sep is '\\') as on macOS.
|
||||
# Without this, every workspace file came back as
|
||||
# `backend\\app.py` on Windows and the file tree silently
|
||||
# mis-parsed.
|
||||
rel_path = os.path.relpath(full_path, folder).replace(os.sep, "/")
|
||||
try:
|
||||
# Stat first; cheap, lets us skip giant files without
|
||||
# opening + reading them.
|
||||
size = os.path.getsize(full_path)
|
||||
if size > _WALK_MAX_FILE_BYTES:
|
||||
files[rel_path] = (
|
||||
f"// [openswarm] file truncated ({size} bytes > "
|
||||
f"{_WALK_MAX_FILE_BYTES} byte cap). Open directly "
|
||||
f"to view full contents."
|
||||
)
|
||||
continue
|
||||
with open(full_path) as f:
|
||||
files[rel_path] = f.read()
|
||||
except Exception:
|
||||
pass
|
||||
return files
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# File-serving endpoints (for iframe preview with multi-file support)
|
||||
# ---------------------------------------------------------------------------
|
||||
@@ -801,23 +563,6 @@ async def delete_output(output_id: str):
|
||||
return {"ok": True}
|
||||
|
||||
|
||||
VIBE_CODE_SYSTEM_PROMPT = """\
|
||||
You are an expert at building self-contained HTML/JS/CSS applications that run in an iframe.
|
||||
|
||||
The user will describe what they want, and you will generate:
|
||||
1. **frontend_code**: A complete HTML document. React 18 is available via esm.sh CDN.
|
||||
- Use: <script type="importmap">{"imports":{"react":"https://esm.sh/react@18","react-dom/client":"https://esm.sh/react-dom@18/client"}}</script>
|
||||
- Input data is at window.OUTPUT_INPUT (object), backend result at window.OUTPUT_BACKEND_RESULT.
|
||||
2. **input_schema**: A JSON Schema object defining the structured input.
|
||||
3. **backend_code** (optional): Python code where input_data is a global dict and result is a global dict to assign to.
|
||||
4. **name**: A short name for the view.
|
||||
5. **description**: A one-sentence description.
|
||||
6. **message**: A brief explanation of what you did/changed.
|
||||
|
||||
Return ONLY valid JSON with these keys. No markdown fences, no extra text.\
|
||||
"""
|
||||
|
||||
|
||||
@outputs.router.post("/vibe-code")
|
||||
async def vibe_code(body: VibeCodeRequest):
|
||||
"""Use an LLM to generate or iterate on Output code from a natural language prompt."""
|
||||
|
||||
@@ -0,0 +1,17 @@
|
||||
"""System prompts for the outputs SubApp's LLM-driven generators."""
|
||||
|
||||
VIBE_CODE_SYSTEM_PROMPT = """\
|
||||
You are an expert at building self-contained HTML/JS/CSS applications that run in an iframe.
|
||||
|
||||
The user will describe what they want, and you will generate:
|
||||
1. **frontend_code**: A complete HTML document. React 18 is available via esm.sh CDN.
|
||||
- Use: <script type="importmap">{"imports":{"react":"https://esm.sh/react@18","react-dom/client":"https://esm.sh/react-dom@18/client"}}</script>
|
||||
- Input data is at window.OUTPUT_INPUT (object), backend result at window.OUTPUT_BACKEND_RESULT.
|
||||
2. **input_schema**: A JSON Schema object defining the structured input.
|
||||
3. **backend_code** (optional): Python code where input_data is a global dict and result is a global dict to assign to.
|
||||
4. **name**: A short name for the view.
|
||||
5. **description**: A one-sentence description.
|
||||
6. **message**: A brief explanation of what you did/changed.
|
||||
|
||||
Return ONLY valid JSON with these keys. No markdown fences, no extra text.\
|
||||
"""
|
||||
+19
-243
@@ -3,259 +3,35 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
import sys
|
||||
from collections import deque, OrderedDict
|
||||
from dataclasses import dataclass
|
||||
from typing import Callable, Optional
|
||||
|
||||
from .runtime_proc import (
|
||||
_ERROR_PATTERNS,
|
||||
_FRONTEND_BIND_POLL_INTERVAL,
|
||||
_FRONTEND_BIND_TIMEOUT_SECONDS,
|
||||
_LOG_BUFFER_LINES,
|
||||
_MAX_IDLE_RUNTIMES,
|
||||
_RECENT_ERRORS_MAX,
|
||||
_TERMINATE_GRACE_SECONDS,
|
||||
_background_priority_kwargs,
|
||||
_find_free_port,
|
||||
_is_new_mode,
|
||||
_is_port_free,
|
||||
_kill_descendant_tree,
|
||||
_read_env_value,
|
||||
_resume_process_tree,
|
||||
_suspend_process_tree,
|
||||
_write_env_value,
|
||||
)
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# 2000 lines per runtime; lets a Terminal tab opened mid-session replay context. ~few hundred KB at worst.
|
||||
_LOG_BUFFER_LINES = 2000
|
||||
|
||||
# SIGTERM grace; well-behaved servers shut down under a second so 3s is enough.
|
||||
_TERMINATE_GRACE_SECONDS = 3
|
||||
|
||||
# 180s covers npm install (60-90s on typical hardware) plus the Vite bind.
|
||||
_FRONTEND_BIND_TIMEOUT_SECONDS = 180
|
||||
# 80ms probe: dropping from 500ms was pure user-visible preview latency win; cheap on localhost.
|
||||
_FRONTEND_BIND_POLL_INTERVAL = 0.08
|
||||
|
||||
|
||||
# Module-level lock so only ONE vite optimizeDeps runs at a time; must be acquired before manager._lock to avoid deadlock with manager.attach.
|
||||
_vite_boot_lock = asyncio.Lock()
|
||||
|
||||
# Idle runtimes kept in LRU; trades memory for instant switch-back, beyond 1 because typical users ping-pong 2-3 apps.
|
||||
_MAX_IDLE_RUNTIMES = 3
|
||||
|
||||
# Cap on recent error lines the agent gets; 50 is enough for babel error + stack + a few warnings.
|
||||
_RECENT_ERRORS_MAX = 50
|
||||
|
||||
# Narrow regex for build errors (vite, babel, tsc, uvicorn); keeps routine logs out of agent context.
|
||||
import re as _re
|
||||
_ERROR_PATTERNS = _re.compile(
|
||||
r"(?:"
|
||||
r"\[plugin:[^\]]+\]|" # vite plugin errors
|
||||
r"SyntaxError|" # node / babel
|
||||
r"Unexpected token|" # babel / tsc parser
|
||||
r"\berror TS\d+|" # tsc diagnostics
|
||||
r"ERROR\s+in\s|" # webpack-style
|
||||
r"Traceback \(most recent call last\)|" # python
|
||||
r"ModuleNotFoundError|"
|
||||
r"ImportError|"
|
||||
r"AttributeError:|"
|
||||
r"Failed to compile|"
|
||||
r"Cannot find module|"
|
||||
r"Cannot resolve"
|
||||
r")"
|
||||
)
|
||||
|
||||
|
||||
def _suspend_process_tree(proc: Optional[asyncio.subprocess.Process]) -> None:
|
||||
"""Send SIGSTOP to a workspace's subprocess so it consumes 0% CPU
|
||||
while sitting in the LRU idle pool. The signal is delivered to the
|
||||
PROCESS GROUP (negative PID) when the child is a session leader,
|
||||
so vite + uvicorn + their npm/python subchildren all pause together.
|
||||
|
||||
No-op on Windows (SIGSTOP has no equivalent; the `OpenProcessToken` +
|
||||
`NtSuspendProcess` route works but isn't worth the win32 surface
|
||||
here; idle Windows runtimes just stay running, which is the current
|
||||
behavior). Failures here are swallowed; if the process already died
|
||||
a stop signal is meaningless."""
|
||||
if proc is None or os.name == "nt":
|
||||
return
|
||||
try:
|
||||
if proc.returncode is not None:
|
||||
return
|
||||
os.kill(proc.pid, signal.SIGSTOP)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
# Already-dead or out-of-permission; both safe to ignore.
|
||||
pass
|
||||
|
||||
|
||||
def _resume_process_tree(proc: Optional[asyncio.subprocess.Process]) -> None:
|
||||
"""SIGCONT a previously-suspended workspace process. Pair with
|
||||
_suspend_process_tree. Microsecond cost; idempotent if the process
|
||||
was never paused."""
|
||||
if proc is None or os.name == "nt":
|
||||
return
|
||||
try:
|
||||
if proc.returncode is not None:
|
||||
return
|
||||
os.kill(proc.pid, signal.SIGCONT)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
pass
|
||||
|
||||
|
||||
def _background_priority_kwargs() -> dict:
|
||||
"""Return the kwargs that lower the spawned subprocess's OS priority
|
||||
to a "background" level. On POSIX this is `preexec_fn=os.nice(10)`,
|
||||
which sets the child's nice to +10 BEFORE exec (so the renice covers
|
||||
the entire bash → vite + uvicorn process tree). On Windows it's
|
||||
`creationflags=BELOW_NORMAL_PRIORITY_CLASS`. The OS scheduler then
|
||||
yields workspace cycles to whichever agent or browser tab is in the
|
||||
user's foreground, so an in-background app build doesn't starve a
|
||||
live chat session.
|
||||
|
||||
We intentionally do NOT pass `start_new_session=True` here even
|
||||
though it would defend against an errant `kill 0` inside the
|
||||
workspace propagating into the OpenSwarm group: doing so also
|
||||
detaches the workspace from the terminal's foreground process
|
||||
group, so a user Ctrl+C only reaches OpenSwarm itself and the
|
||||
cleanup path has to chase every workspace by hand. If that path
|
||||
is even slightly slow or gets interrupted by a second Ctrl+C, the
|
||||
workspace's uvicorn / vite leaks past shutdown and the next
|
||||
`bash run.sh` hits Errno 48 on port 8324. The `kill 0` propagation
|
||||
is fixed at its source in the workspace template's run.sh
|
||||
(uses `kill_tree` on tracked PIDs, never `kill 0`)."""
|
||||
if os.name == "nt":
|
||||
# subprocess.BELOW_NORMAL_PRIORITY_CLASS == 0x4000
|
||||
return {"creationflags": subprocess.BELOW_NORMAL_PRIORITY_CLASS}
|
||||
return {"preexec_fn": lambda: os.nice(10)}
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
"""Ask the kernel for an unused localhost port. There's a tiny race
|
||||
between this socket closing and the backend re-binding, but we hand
|
||||
each port to exactly one runtime so no caller competes for it, and
|
||||
the kernel won't immediately recycle a freshly-closed port anyway."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _kill_descendant_tree(pid: int, sig_name: str = "TERM") -> None:
|
||||
"""Recursively signal every descendant of `pid`, leaves-first. The
|
||||
webapp template's run.sh installs `trap cleanup EXIT` (no TERM), so a
|
||||
plain SIGTERM to the bash wrapper exits bash silently and leaves
|
||||
vite/uvicorn grandchildren reparented to PID 1, squatting on the
|
||||
workspace's ports. Walking the tree ourselves bypasses the template's
|
||||
signal-handling habits entirely. POSIX uses `pgrep -P` to enumerate
|
||||
direct children; Windows is covered by `taskkill /T /F` (job-object
|
||||
walk). All failures are swallowed; missing PIDs mean the process
|
||||
already exited, which is the desired state anyway."""
|
||||
if os.name == "nt":
|
||||
try:
|
||||
subprocess.run(
|
||||
["taskkill", "/PID", str(pid), "/T", "/F"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=5,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["pgrep", "-P", str(pid)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2,
|
||||
)
|
||||
children = [int(p) for p in out.stdout.split() if p.strip().isdigit()]
|
||||
except Exception:
|
||||
children = []
|
||||
for child in children:
|
||||
_kill_descendant_tree(child, sig_name)
|
||||
sig = getattr(signal, f"SIG{sig_name}", signal.SIGTERM)
|
||||
for child in children:
|
||||
try:
|
||||
os.kill(child, sig)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
pass
|
||||
|
||||
|
||||
def _is_port_free(port: int) -> bool:
|
||||
"""True if nothing currently holds a TCP listener on 127.0.0.1:port.
|
||||
Cheap kernel-probe; resolves on bind success. Used as the cross-session
|
||||
safety net: if a prior OpenSwarm run left a ghost subprocess holding
|
||||
the .env-persisted FRONTEND_PORT, we detect it here and reallocate
|
||||
rather than handing run.sh a port that will EADDRINUSE."""
|
||||
try:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", port))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _write_env_value(env_path: str, key: str, value: str) -> None:
|
||||
"""Update KEY=VALUE in an existing `.env`, preserving every other
|
||||
line. Creates the file if missing. Used when a persisted port collides
|
||||
with a ghost from a prior session and we have to reallocate before
|
||||
spawning run.sh."""
|
||||
lines: list[str] = []
|
||||
found = False
|
||||
if os.path.exists(env_path):
|
||||
try:
|
||||
with open(env_path, encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
except Exception:
|
||||
lines = []
|
||||
for i, raw in enumerate(lines):
|
||||
stripped = raw.strip()
|
||||
if not stripped or stripped.startswith("#") or "=" not in stripped:
|
||||
continue
|
||||
k = stripped.split("=", 1)[0].strip()
|
||||
if k == key:
|
||||
lines[i] = f"{key}={value}\n"
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
if lines and not lines[-1].endswith("\n"):
|
||||
lines[-1] = lines[-1] + "\n"
|
||||
lines.append(f"{key}={value}\n")
|
||||
try:
|
||||
with open(env_path, "w", encoding="utf-8") as f:
|
||||
f.writelines(lines)
|
||||
except Exception:
|
||||
logger.exception("failed writing %s=%s to %s", key, value, env_path)
|
||||
|
||||
|
||||
def _is_new_mode(workspace_path: str) -> bool:
|
||||
"""A workspace is "new-mode" (webapp-template scaffold) if it has a
|
||||
`run.sh` at its root. Old-mode workspaces are flat `index.html`-only
|
||||
apps that pre-date the template swap; they're served by OpenSwarm's
|
||||
own `/api/outputs/workspace/{ws}/serve/...` FastAPI route and have an
|
||||
optional `backend.py` we spawn directly.
|
||||
|
||||
Single-file probe so the check is cheap to call on every runtime
|
||||
start, status query, and serve request."""
|
||||
return os.path.isfile(os.path.join(workspace_path, "run.sh"))
|
||||
|
||||
|
||||
def _read_env_value(env_path: str, key: str) -> Optional[str]:
|
||||
"""Parse one value out of a workspace's `.env` without the cost of a
|
||||
full subprocess-source. Strips quotes + trailing comments. Returns
|
||||
None if the file or key is missing."""
|
||||
if not os.path.exists(env_path):
|
||||
return None
|
||||
try:
|
||||
with open(env_path, encoding="utf-8") as f:
|
||||
for raw in f:
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" not in line:
|
||||
continue
|
||||
k, _, v = line.partition("=")
|
||||
if k.strip() != key:
|
||||
continue
|
||||
v = v.strip()
|
||||
# Strip an inline `# comment`. Naive; bash semantics are
|
||||
# more permissive, but values we write don't contain `#`.
|
||||
if "#" in v:
|
||||
v = v.split("#", 1)[0].rstrip()
|
||||
if (v.startswith('"') and v.endswith('"')) or (v.startswith("'") and v.endswith("'")):
|
||||
v = v[1:-1]
|
||||
return v
|
||||
except Exception:
|
||||
logger.exception("failed reading %s from %s", key, env_path)
|
||||
return None
|
||||
|
||||
|
||||
@dataclass
|
||||
class LogLine:
|
||||
|
||||
@@ -0,0 +1,252 @@
|
||||
"""OS/process/port primitives for the per-workspace runtime: signal-based
|
||||
suspend/resume, descendant-tree kills, free-port allocation, and .env
|
||||
read/write. No asyncio runtime state lives here; AppRuntime (runtime.py) owns
|
||||
that and just calls into these."""
|
||||
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
import signal
|
||||
import socket
|
||||
import subprocess
|
||||
from typing import Optional
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
# SIGTERM grace; well-behaved servers shut down under a second so 3s is enough.
|
||||
_TERMINATE_GRACE_SECONDS = 3
|
||||
|
||||
# 180s covers npm install (60-90s on typical hardware) plus the Vite bind.
|
||||
_FRONTEND_BIND_TIMEOUT_SECONDS = 180
|
||||
# 80ms probe: dropping from 500ms was pure user-visible preview latency win; cheap on localhost.
|
||||
_FRONTEND_BIND_POLL_INTERVAL = 0.08
|
||||
|
||||
# 2000 lines per runtime; lets a Terminal tab opened mid-session replay context. ~few hundred KB at worst.
|
||||
_LOG_BUFFER_LINES = 2000
|
||||
|
||||
# Idle runtimes kept in LRU; trades memory for instant switch-back, beyond 1 because typical users ping-pong 2-3 apps.
|
||||
_MAX_IDLE_RUNTIMES = 3
|
||||
|
||||
# Cap on recent error lines the agent gets; 50 is enough for babel error + stack + a few warnings.
|
||||
_RECENT_ERRORS_MAX = 50
|
||||
|
||||
# Narrow regex for build errors (vite, babel, tsc, uvicorn); keeps routine logs out of agent context.
|
||||
_ERROR_PATTERNS = re.compile(
|
||||
r"(?:"
|
||||
r"\[plugin:[^\]]+\]|" # vite plugin errors
|
||||
r"SyntaxError|" # node / babel
|
||||
r"Unexpected token|" # babel / tsc parser
|
||||
r"\berror TS\d+|" # tsc diagnostics
|
||||
r"ERROR\s+in\s|" # webpack-style
|
||||
r"Traceback \(most recent call last\)|" # python
|
||||
r"ModuleNotFoundError|"
|
||||
r"ImportError|"
|
||||
r"AttributeError:|"
|
||||
r"Failed to compile|"
|
||||
r"Cannot find module|"
|
||||
r"Cannot resolve"
|
||||
r")"
|
||||
)
|
||||
|
||||
|
||||
def _suspend_process_tree(proc) -> None:
|
||||
"""Send SIGSTOP to a workspace's subprocess so it consumes 0% CPU
|
||||
while sitting in the LRU idle pool. The signal is delivered to the
|
||||
PROCESS GROUP (negative PID) when the child is a session leader,
|
||||
so vite + uvicorn + their npm/python subchildren all pause together.
|
||||
|
||||
No-op on Windows (SIGSTOP has no equivalent; the `OpenProcessToken` +
|
||||
`NtSuspendProcess` route works but isn't worth the win32 surface
|
||||
here; idle Windows runtimes just stay running, which is the current
|
||||
behavior). Failures here are swallowed; if the process already died
|
||||
a stop signal is meaningless."""
|
||||
if proc is None or os.name == "nt":
|
||||
return
|
||||
try:
|
||||
if proc.returncode is not None:
|
||||
return
|
||||
os.kill(proc.pid, signal.SIGSTOP)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
# Already-dead or out-of-permission; both safe to ignore.
|
||||
pass
|
||||
|
||||
|
||||
def _resume_process_tree(proc) -> None:
|
||||
"""SIGCONT a previously-suspended workspace process. Pair with
|
||||
_suspend_process_tree. Microsecond cost; idempotent if the process
|
||||
was never paused."""
|
||||
if proc is None or os.name == "nt":
|
||||
return
|
||||
try:
|
||||
if proc.returncode is not None:
|
||||
return
|
||||
os.kill(proc.pid, signal.SIGCONT)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
pass
|
||||
|
||||
|
||||
def _background_priority_kwargs() -> dict:
|
||||
"""Return the kwargs that lower the spawned subprocess's OS priority
|
||||
to a "background" level. On POSIX this is `preexec_fn=os.nice(10)`,
|
||||
which sets the child's nice to +10 BEFORE exec (so the renice covers
|
||||
the entire bash → vite + uvicorn process tree). On Windows it's
|
||||
`creationflags=BELOW_NORMAL_PRIORITY_CLASS`. The OS scheduler then
|
||||
yields workspace cycles to whichever agent or browser tab is in the
|
||||
user's foreground, so an in-background app build doesn't starve a
|
||||
live chat session.
|
||||
|
||||
We intentionally do NOT pass `start_new_session=True` here even
|
||||
though it would defend against an errant `kill 0` inside the
|
||||
workspace propagating into the OpenSwarm group: doing so also
|
||||
detaches the workspace from the terminal's foreground process
|
||||
group, so a user Ctrl+C only reaches OpenSwarm itself and the
|
||||
cleanup path has to chase every workspace by hand. If that path
|
||||
is even slightly slow or gets interrupted by a second Ctrl+C, the
|
||||
workspace's uvicorn / vite leaks past shutdown and the next
|
||||
`bash run.sh` hits Errno 48 on port 8324. The `kill 0` propagation
|
||||
is fixed at its source in the workspace template's run.sh
|
||||
(uses `kill_tree` on tracked PIDs, never `kill 0`)."""
|
||||
if os.name == "nt":
|
||||
# subprocess.BELOW_NORMAL_PRIORITY_CLASS == 0x4000
|
||||
return {"creationflags": subprocess.BELOW_NORMAL_PRIORITY_CLASS}
|
||||
return {"preexec_fn": lambda: os.nice(10)}
|
||||
|
||||
|
||||
def _find_free_port() -> int:
|
||||
"""Ask the kernel for an unused localhost port. There's a tiny race
|
||||
between this socket closing and the backend re-binding, but we hand
|
||||
each port to exactly one runtime so no caller competes for it, and
|
||||
the kernel won't immediately recycle a freshly-closed port anyway."""
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", 0))
|
||||
return s.getsockname()[1]
|
||||
|
||||
|
||||
def _kill_descendant_tree(pid: int, sig_name: str = "TERM") -> None:
|
||||
"""Recursively signal every descendant of `pid`, leaves-first. The
|
||||
webapp template's run.sh installs `trap cleanup EXIT` (no TERM), so a
|
||||
plain SIGTERM to the bash wrapper exits bash silently and leaves
|
||||
vite/uvicorn grandchildren reparented to PID 1, squatting on the
|
||||
workspace's ports. Walking the tree ourselves bypasses the template's
|
||||
signal-handling habits entirely. POSIX uses `pgrep -P` to enumerate
|
||||
direct children; Windows is covered by `taskkill /T /F` (job-object
|
||||
walk). All failures are swallowed; missing PIDs mean the process
|
||||
already exited, which is the desired state anyway."""
|
||||
if os.name == "nt":
|
||||
try:
|
||||
subprocess.run(
|
||||
["taskkill", "/PID", str(pid), "/T", "/F"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.DEVNULL,
|
||||
timeout=5,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
return
|
||||
try:
|
||||
out = subprocess.run(
|
||||
["pgrep", "-P", str(pid)],
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=2,
|
||||
)
|
||||
children = [int(p) for p in out.stdout.split() if p.strip().isdigit()]
|
||||
except Exception:
|
||||
children = []
|
||||
for child in children:
|
||||
_kill_descendant_tree(child, sig_name)
|
||||
sig = getattr(signal, f"SIG{sig_name}", signal.SIGTERM)
|
||||
for child in children:
|
||||
try:
|
||||
os.kill(child, sig)
|
||||
except (ProcessLookupError, PermissionError, OSError):
|
||||
pass
|
||||
|
||||
|
||||
def _is_port_free(port: int) -> bool:
|
||||
"""True if nothing currently holds a TCP listener on 127.0.0.1:port.
|
||||
Cheap kernel-probe; resolves on bind success. Used as the cross-session
|
||||
safety net: if a prior OpenSwarm run left a ghost subprocess holding
|
||||
the .env-persisted FRONTEND_PORT, we detect it here and reallocate
|
||||
rather than handing run.sh a port that will EADDRINUSE."""
|
||||
try:
|
||||
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
|
||||
s.bind(("127.0.0.1", port))
|
||||
return True
|
||||
except OSError:
|
||||
return False
|
||||
|
||||
|
||||
def _write_env_value(env_path: str, key: str, value: str) -> None:
|
||||
"""Update KEY=VALUE in an existing `.env`, preserving every other
|
||||
line. Creates the file if missing. Used when a persisted port collides
|
||||
with a ghost from a prior session and we have to reallocate before
|
||||
spawning run.sh."""
|
||||
lines: list[str] = []
|
||||
found = False
|
||||
if os.path.exists(env_path):
|
||||
try:
|
||||
with open(env_path, encoding="utf-8") as f:
|
||||
lines = f.readlines()
|
||||
except Exception:
|
||||
lines = []
|
||||
for i, raw in enumerate(lines):
|
||||
stripped = raw.strip()
|
||||
if not stripped or stripped.startswith("#") or "=" not in stripped:
|
||||
continue
|
||||
k = stripped.split("=", 1)[0].strip()
|
||||
if k == key:
|
||||
lines[i] = f"{key}={value}\n"
|
||||
found = True
|
||||
break
|
||||
if not found:
|
||||
if lines and not lines[-1].endswith("\n"):
|
||||
lines[-1] = lines[-1] + "\n"
|
||||
lines.append(f"{key}={value}\n")
|
||||
try:
|
||||
with open(env_path, "w", encoding="utf-8") as f:
|
||||
f.writelines(lines)
|
||||
except Exception:
|
||||
logger.exception("failed writing %s=%s to %s", key, value, env_path)
|
||||
|
||||
|
||||
def _is_new_mode(workspace_path: str) -> bool:
|
||||
"""A workspace is "new-mode" (webapp-template scaffold) if it has a
|
||||
`run.sh` at its root. Old-mode workspaces are flat `index.html`-only
|
||||
apps that pre-date the template swap; they're served by OpenSwarm's
|
||||
own `/api/outputs/workspace/{ws}/serve/...` FastAPI route and have an
|
||||
optional `backend.py` we spawn directly.
|
||||
|
||||
Single-file probe so the check is cheap to call on every runtime
|
||||
start, status query, and serve request."""
|
||||
return os.path.isfile(os.path.join(workspace_path, "run.sh"))
|
||||
|
||||
|
||||
def _read_env_value(env_path: str, key: str) -> Optional[str]:
|
||||
"""Parse one value out of a workspace's `.env` without the cost of a
|
||||
full subprocess-source. Strips quotes + trailing comments. Returns
|
||||
None if the file or key is missing."""
|
||||
if not os.path.exists(env_path):
|
||||
return None
|
||||
try:
|
||||
with open(env_path, encoding="utf-8") as f:
|
||||
for raw in f:
|
||||
line = raw.strip()
|
||||
if not line or line.startswith("#"):
|
||||
continue
|
||||
if "=" not in line:
|
||||
continue
|
||||
k, _, v = line.partition("=")
|
||||
if k.strip() != key:
|
||||
continue
|
||||
v = v.strip()
|
||||
# Strip an inline `# comment`. Naive; bash semantics are
|
||||
# more permissive, but values we write don't contain `#`.
|
||||
if "#" in v:
|
||||
v = v.split("#", 1)[0].rstrip()
|
||||
if (v.startswith('"') and v.endswith('"')) or (v.startswith("'") and v.endswith("'")):
|
||||
v = v[1:-1]
|
||||
return v
|
||||
except Exception:
|
||||
logger.exception("failed reading %s from %s", key, env_path)
|
||||
return None
|
||||
@@ -0,0 +1,117 @@
|
||||
"""On-disk persistence for outputs: Output JSON records under DATA_DIR and
|
||||
the workspace file-tree walk used by the polling read endpoint."""
|
||||
|
||||
import json
|
||||
import os
|
||||
|
||||
from fastapi import HTTPException
|
||||
|
||||
from backend.apps.outputs.models import Output
|
||||
from backend.config.paths import OUTPUTS_DIR as DATA_DIR
|
||||
|
||||
|
||||
def _load_all() -> list[Output]:
|
||||
result = []
|
||||
if not os.path.exists(DATA_DIR):
|
||||
return result
|
||||
for fname in os.listdir(DATA_DIR):
|
||||
if fname.endswith(".json"):
|
||||
with open(os.path.join(DATA_DIR, fname)) as f:
|
||||
result.append(Output(**json.load(f)))
|
||||
return result
|
||||
|
||||
|
||||
def _save(output: Output):
|
||||
with open(os.path.join(DATA_DIR, f"{output.id}.json"), "w") as f:
|
||||
json.dump(output.model_dump(), f, indent=2)
|
||||
|
||||
|
||||
def _load(output_id: str) -> Output:
|
||||
path = os.path.join(DATA_DIR, f"{output_id}.json")
|
||||
if not os.path.exists(path):
|
||||
raise HTTPException(status_code=404, detail="Output not found")
|
||||
with open(path) as f:
|
||||
return Output(**json.load(f))
|
||||
|
||||
|
||||
def load_output(output_id: str) -> Output | None:
|
||||
"""Public helper for other modules to resolve an output by ID."""
|
||||
path = os.path.join(DATA_DIR, f"{output_id}.json")
|
||||
if not os.path.exists(path):
|
||||
return None
|
||||
with open(path) as f:
|
||||
return Output(**json.load(f))
|
||||
|
||||
|
||||
# Build/install/cache directories that the polling endpoint must never
|
||||
# descend into. Without this skip-list the workspace endpoint reads
|
||||
# `node_modules/` (300 MB of MUI source, when it's a real dir and not a
|
||||
# symlink), `.venv/` (10k+ Python files from the hardlinked cache),
|
||||
# `__pycache__/`, `dist/`, `.git/`, etc; every 2 seconds while the
|
||||
# agent is active. Result: backend CPU pegged on JSON-serializing
|
||||
# auto-generated chunks the frontend will then throw away. The frontend
|
||||
# already filters these for display; this skip is the real fix.
|
||||
_WALK_SKIP_DIRS = frozenset({
|
||||
"node_modules",
|
||||
".vite",
|
||||
".vite-cache",
|
||||
".vite_cache",
|
||||
".git",
|
||||
"dist",
|
||||
".next",
|
||||
"__pycache__",
|
||||
".venv",
|
||||
"venv",
|
||||
".pytest_cache",
|
||||
".mypy_cache",
|
||||
".ruff_cache",
|
||||
})
|
||||
|
||||
# Cap per-file response size at 256 KB. Hand-written source rarely
|
||||
# exceeds this; auto-generated bundles routinely run into the MBs and
|
||||
# they're not what the user/agent is editing. Anything over the cap
|
||||
# returns a truncated stub the frontend treats as "open the file
|
||||
# directly to see full contents."
|
||||
_WALK_MAX_FILE_BYTES = 256 * 1024
|
||||
|
||||
|
||||
def _walk_directory(folder: str) -> dict[str, str]:
|
||||
"""Walk a directory tree and return {relative_path: content} for all
|
||||
text files the user is actually authoring. Skips build/install
|
||||
directories AND truncates oversize files; both critical for the
|
||||
polling endpoint, which is called every 2 s while the agent is
|
||||
writing code and would otherwise serialize hundreds of MB per poll."""
|
||||
files: dict[str, str] = {}
|
||||
if not os.path.isdir(folder):
|
||||
return files
|
||||
for root, dirs, filenames in os.walk(folder):
|
||||
# Mutate `dirs` in place; that's how os.walk skips a subtree.
|
||||
# Doing it here means we never even stat the children, so a
|
||||
# 10k-file `.venv/` costs ~one stat (on the dir itself) instead
|
||||
# of 10k.
|
||||
dirs[:] = [d for d in dirs if d not in _WALK_SKIP_DIRS]
|
||||
for fname in filenames:
|
||||
full_path = os.path.join(root, fname)
|
||||
# Normalize to forward-slash keys so the frontend's
|
||||
# `path.split('/')` and `.startsWith(prefix)` checks work
|
||||
# the same on Windows (where os.sep is '\\') as on macOS.
|
||||
# Without this, every workspace file came back as
|
||||
# `backend\\app.py` on Windows and the file tree silently
|
||||
# mis-parsed.
|
||||
rel_path = os.path.relpath(full_path, folder).replace(os.sep, "/")
|
||||
try:
|
||||
# Stat first; cheap, lets us skip giant files without
|
||||
# opening + reading them.
|
||||
size = os.path.getsize(full_path)
|
||||
if size > _WALK_MAX_FILE_BYTES:
|
||||
files[rel_path] = (
|
||||
f"// [openswarm] file truncated ({size} bytes > "
|
||||
f"{_WALK_MAX_FILE_BYTES} byte cap). Open directly "
|
||||
f"to view full contents."
|
||||
)
|
||||
continue
|
||||
with open(full_path) as f:
|
||||
files[rel_path] = f.read()
|
||||
except Exception:
|
||||
pass
|
||||
return files
|
||||
Reference in New Issue
Block a user