Ready
Describe what you want to build and the agent will update this app.
"""
VIEW_TEMPLATE_SCHEMA = """\
{
"type": "object",
"properties": {},
"required": []
}
"""
VIEW_TEMPLATE_META = """\
{
"name": "",
"description": ""
}
"""
VIEW_TEMPLATE_FILES = {
"index.html": VIEW_TEMPLATE_INDEX,
"schema.json": VIEW_TEMPLATE_SCHEMA,
"meta.json": VIEW_TEMPLATE_META,
}
# --------------------------------------------------------------------------- webapp_template (new-mode) seed helpers ---------------------------------------------------------------------------
def p_ignore_backend(src: str, names: list[str]) -> list[str]:
"""copytree filter; when copying the template root, drop only the
top-level `backend/` directory. Subdirectories named `backend` deeper
in the tree (none today, but defensively scoped) are unaffected."""
if os.path.abspath(src) == os.path.abspath(WEBAPP_TEMPLATE_DIR):
return [n for n in names if n == "backend"]
return []
DEBUGGER_PATH = os.path.abspath(
os.path.join(os.path.dirname(__file__), "..", "..", "..", "debugger")
)
TEMPLATE_BACKEND_PATH = os.path.abspath(os.path.join(WEBAPP_TEMPLATE_DIR, "backend"))
# --------------------------------------------------------------------------- Shared node_modules cache; every new webapp-template workspace symlinks its frontend/node_modules to a single warm directory. First-app create pays the ~22s npm-install cost once; every subsequent app is instant (just a symlink + vite startup, ~1s). Cache directory is keyed by a sha of the template's package.json, so a template dep bump invalidates the cache automatically; old caches sit until the user clears ~/.openswarm/cache. ---------------------------------------------------------------------------
p_warm_cache_lock = threading.Lock()
p_warm_cache_thread: threading.Thread | None = None
# Pre-built node_modules archive bundled with packaged releases. Generated by `scripts/build-template-archive.sh` and shipped at this path inside the app's resources. When present (and tagged with the current package.json sha), extract instead of running npm; decompression is ~3 s vs ~22 s for the live install. Stale archives (package.json bumped but archive not rebuilt) are silently ignored, so the live-install fallback always wins on correctness.
P_BUNDLED_ARCHIVE_DIR = os.path.join(
os.path.dirname(__file__), "webapp_template_cache"
)
def p_bundled_archive_path_for(digest: str) -> str:
"""Sha-tagged archive path so a stale archive from a prior template
version is automatically skipped instead of overwriting the cache with
out-of-date modules."""
return os.path.join(P_BUNDLED_ARCHIVE_DIR, f"node_modules.{digest}.tar.gz")
def bundled_extracted_modules() -> str | None:
"""A node_modules tree shipped ALREADY EXTRACTED in resources (digest-tagged),
so a workspace can junction straight at it with ZERO extract. This skips the
~14s first-app tar-extract on Windows (the extract is dominated by Defender
scanning ~tens of thousands of small files as they're written; shipping it
extracted moves that scan to install time, once). Returns the read-only path
or None when no extracted tree is shipped (e.g. the Mac build, which ships
the .tar.gz and uses the extract path instead). vite only reads node_modules
(its optimize cache lives elsewhere), so a read-only shared tree is safe."""
cand = os.path.join(P_BUNDLED_ARCHIVE_DIR, warm_cache_digest(), "node_modules")
return cand if os.path.isdir(cand) else None
def p_try_extract_bundled_archive(cache_dir: str, digest: str) -> bool:
"""Unpack the sha-tagged bundled archive into `cache_dir` if one
exists for the current template digest. Returns True on success,
False to signal the caller should fall back to a live `npm install`.
The archive is built from the same package.json + package-lock.json
sha so the extracted tree is byte-equivalent to `npm ci`."""
archive_path = p_bundled_archive_path_for(digest)
if not os.path.exists(archive_path):
return False
try:
logger.info(
"webapp-template: unpacking bundled warm-cache archive %s",
archive_path,
)
os.makedirs(cache_dir, exist_ok=True)
# Archive root is `node_modules/`; extracting into cache_dir places it at the expected path. tarfile uses zlib internally for .gz, no extra dep needed.
with tarfile.open(archive_path, "r:gz") as tar:
tar.extractall(cache_dir)
cache_modules = os.path.join(cache_dir, "node_modules")
if os.path.isdir(cache_modules):
return True
logger.warning(
"webapp-template: bundled archive extracted but no node_modules/ "
"directory at %s; falling back to npm install",
cache_modules,
)
return False
except Exception as exc:
logger.warning(
"webapp-template: bundled-archive extract failed (%s); "
"falling back to npm install",
exc,
)
return False
def warm_cache_digest() -> str:
"""Sha of the template's frontend/package.json; used as the cache
key + the bundled-archive filename suffix so a package.json bump
invalidates both at once."""
pkg_path = os.path.join(WEBAPP_TEMPLATE_DIR, "frontend", "package.json")
try:
with open(pkg_path, "rb") as fh:
return hashlib.sha256(fh.read()).hexdigest()[:12]
except OSError:
return "fallback"
def p_warm_cache_dir() -> str:
"""Path the warm node_modules lives under. Hashed by package.json so
upgrades automatically force a re-populate."""
base = os.environ.get("OPENSWARM_WEBAPP_CACHE_DIR") or os.path.expanduser(
"~/.openswarm/cache/webapp_template_node_modules"
)
return os.path.join(base, warm_cache_digest())
def warm_cache_is_complete(cache_modules: str) -> bool:
"""A populated node_modules/ dir is not proof of a *finished* install.
npm links package bins (node_modules/.bin/*) in the final phase, so an
install killed partway (e.g. Electron quit mid-warm) leaves the package
trees on disk but no .bin/. The old `os.path.isdir(node_modules)` check
then trusted that half-tree forever, every app symlinked to it, and
`npm run dev` died with `vite: command not found`. Require the one bin
every webapp-template app actually launches with so a partial cache is
treated as not-ready and repopulated instead of cached as good."""
return os.path.exists(os.path.join(cache_modules, ".bin", "vite"))
def ensure_warm_cache() -> str | None:
"""Populate the warm-cache node_modules if missing or incomplete.
Returns the absolute path to the populated `node_modules` directory, or
None on failure. Thread-safe; concurrent callers block on a single
install instead of racing. Idempotent and fast after the first call."""
cache_dir = p_warm_cache_dir()
cache_modules = os.path.join(cache_dir, "node_modules")
if warm_cache_is_complete(cache_modules):
return cache_modules
# Prefer a pre-extracted bundled tree: junction the workspace straight at it, no tar-extract and no npm. This is the #9 first-app speed win on Windows.
bundled = bundled_extracted_modules()
if bundled:
logger.info("webapp-template: using bundled pre-extracted node_modules (zero extract)")
return bundled
with p_warm_cache_lock:
if warm_cache_is_complete(cache_modules):
return cache_modules
# A node_modules that exists but flunks the completeness check is a half-finished install; wipe it so the rebuild below starts on clean ground instead of layering onto a broken tree.
if os.path.isdir(cache_modules):
shutil.rmtree(cache_modules, ignore_errors=True)
# Fast path: pre-built archive shipped inside the release. The build script generates this so users hitting OpenSwarm for the first time skip the ~22 s live `npm install`. Falls through on any failure so dev installs (no archive) keep working.
if p_try_extract_bundled_archive(cache_dir, warm_cache_digest()):
if warm_cache_is_complete(cache_modules):
logger.info("webapp-template: warm cache ready from bundled archive")
return cache_modules
# Archive unpacked a tree without the launch bin; don't trust it.
shutil.rmtree(cache_modules, ignore_errors=True)
try:
os.makedirs(cache_dir, exist_ok=True)
# Copy package.json + lockfile (if it exists) into the cache dir so npm has something to install from. We don't write back to the template; the lockfile generated here stays local to the cache.
tmpl_pkg = os.path.join(WEBAPP_TEMPLATE_DIR, "frontend", "package.json")
tmpl_lock = os.path.join(WEBAPP_TEMPLATE_DIR, "frontend", "package-lock.json")
shutil.copyfile(tmpl_pkg, os.path.join(cache_dir, "package.json"))
base_flags = ["--prefer-offline", "--no-audit", "--no-fund", "--loglevel=error"]
npm = p_resolve_npm()
if npm is None:
logger.info("webapp-template: no npm available; skipping warm cache (workspace will install on first run)")
return None
if os.path.exists(tmpl_lock):
shutil.copyfile(tmpl_lock, os.path.join(cache_dir, "package-lock.json"))
cmd = [*npm, "ci", *base_flags]
else:
# No lockfile yet; `npm install` resolves the tree and writes one into the cache dir for future use.
cmd = [*npm, "install", *base_flags]
logger.info("webapp-template: warming node_modules cache at %s", cache_dir)
result = subprocess.run(
cmd, cwd=cache_dir, capture_output=True, text=True, timeout=600
)
# --prefer-offline reuses npm's metadata cache, which can be stale: if a pinned transitive (e.g. a @babel/* helper) was published after the cache snapshot, resolution fails ETARGET even though the registry has it. Retry once online (drops --prefer-offline) so a partially-stale cache self-heals instead of dead-ending the whole App Builder frontend.
if result.returncode != 0 and "ETARGET" in (result.stderr or ""):
online_cmd = [c for c in cmd if c != "--prefer-offline"]
logger.info("webapp-template: warm-cache offline pass hit ETARGET; retrying online")
result = subprocess.run(
online_cmd, cwd=cache_dir, capture_output=True, text=True, timeout=600
)
if result.returncode != 0:
logger.warning(
"webapp-template warm-cache install failed (rc=%s): %s",
result.returncode,
(result.stderr or "")[-1500:],
)
return None
# Never hand back a tree the workspace can't actually launch from.
if not warm_cache_is_complete(cache_modules):
logger.warning("webapp-template: warm-cache install left no .bin/vite; not caching")
return None
return cache_modules
except Exception as exc:
logger.warning("webapp-template warm-cache failed: %s", exc)
return None
def p_try_link_dir(src: str, target: str) -> bool:
"""Point `target` at `src` as cheaply as possible. Prefer a symlink (instant,
shared, zero disk). On Windows os.symlink needs admin / Developer Mode, which
a normal user account lacks, so fall back to a directory junction (mklink /J,
no privilege required), then to a full copy as a last resort so even a
locked-down Windows box ends up with a usable node_modules. Returns True if
`target` now resolves to the dependency tree."""
try:
os.symlink(src, target)
return True
except OSError:
pass
if os.name == "nt":
try:
r = subprocess.run(
["cmd", "/c", "mklink", "/J", target, src],
capture_output=True, text=True, timeout=15,
)
if r.returncode == 0 and os.path.isdir(target):
return True
except Exception:
pass
try:
# Slow + uses disk, but guarantees the workspace can boot vite even when neither symlink nor junction is available.
shutil.copytree(src, target, dirs_exist_ok=True)
return True
except OSError as exc:
logger.warning("webapp-template link/copy failed (%s) for %s", exc, target)
return False
def p_link_node_modules(workspace_dir: str) -> None:
"""After copytree, point the workspace's frontend/node_modules at
the warm-cache directory. Safe fallback; if the cache isn't ready,
the workspace's run.sh will fall through to its own install path."""
cache_modules = ensure_warm_cache()
if not cache_modules:
return
target = os.path.join(workspace_dir, "frontend", "node_modules")
if os.path.islink(target):
try:
if os.readlink(target) == cache_modules:
return
except OSError:
pass
try:
os.unlink(target)
except OSError:
return
elif os.path.isdir(target):
# If the dir is EMPTY (left over from copytree of the template's placeholder node_modules; `.gitkeep`-style scenarios) nuke it so we can symlink to the warm cache. A non-empty directory is treated as a real npm install; respect it and bail.
try:
has_content = any(True for _ in os.scandir(target))
except OSError:
return
if has_content:
return
try:
os.rmdir(target)
except OSError:
return
try:
os.makedirs(os.path.dirname(target), exist_ok=True)
except OSError as exc:
logger.warning("webapp-template mkdir failed (%s) for %s", exc, workspace_dir)
return
if p_try_link_dir(cache_modules, target):
logger.info("webapp-template: linked %s -> %s", target, cache_modules)
# --------------------------------------------------------------------------- Shared Python venv cache; same pattern as the node_modules cache, but for the workspace backend's FastAPI + transitive deps. Eliminates the ~25s `python -m venv` + `pip install -e .` that backend_init.sh otherwise pays per workspace. ---------------------------------------------------------------------------
p_warm_venv_lock = threading.Lock()
def warm_venv_dir() -> str:
"""Cache root for the shared backend venv, keyed by a sha of the
template backend's pyproject.toml so a dep bump auto-invalidates."""
pyproject = os.path.join(WEBAPP_TEMPLATE_DIR, "backend", "pyproject.toml")
try:
with open(pyproject, "rb") as fh:
digest = hashlib.sha256(fh.read()).hexdigest()[:12]
except OSError:
digest = "fallback"
base = os.environ.get("OPENSWARM_BACKEND_VENV_CACHE_DIR") or os.path.expanduser(
"~/.openswarm/cache/webapp_template_backend_venv"
)
return os.path.join(base, digest)
def p_ensure_warm_python_venv() -> str | None:
"""Populate the warm-cache backend venv if missing. Returns the
absolute path to the populated `.venv` directory, or None on
failure. Thread-safe and idempotent; fast return after first call."""
cache_dir = warm_venv_dir()
venv_dir = os.path.join(cache_dir, ".venv")
sentinel = os.path.join(cache_dir, ".populated")
if os.path.isfile(sentinel) and os.path.isdir(venv_dir):
return venv_dir
with p_warm_venv_lock:
if os.path.isfile(sentinel) and os.path.isdir(venv_dir):
return venv_dir
try:
os.makedirs(cache_dir, exist_ok=True)
# Pick the same python the workspace's run.sh would have picked, so the venv's binary is compatible. Includes bare `python` as the last fallback for Windows, where there's no `python3` symlink; the installer ships just `python.exe`. On macOS/Linux the versioned candidates match first so we don't accidentally pick a system Python 2.x via the bare name.
py = p_resolve_python()
# Wipe any half-populated venv from a previous crashed run.
if os.path.isdir(venv_dir):
shutil.rmtree(venv_dir, ignore_errors=True)
logger.info("webapp-template: creating warm backend venv at %s", venv_dir)
r = subprocess.run(
[py, "-m", "venv", venv_dir],
capture_output=True, text=True, timeout=120,
)
if r.returncode != 0:
logger.warning("warm-venv create failed: %s", r.stderr[-1500:])
return None
# Install the template's dependencies (fastapi[standard], typeguard, transitives); NOT the workspace's own backend, which gets editable-installed per-workspace by run.sh after the cache copy. The venv layout differs by platform: POSIX puts executables in `bin/`, Windows in `Scripts/`, and the executable name itself gets `.exe`.
if os.name == "nt":
pip = os.path.join(venv_dir, "Scripts", "pip.exe")
else:
pip = os.path.join(venv_dir, "bin", "pip")
deps = ["fastapi[standard]", "typeguard==4.4.2"]
r = subprocess.run(
[pip, "install", "--disable-pip-version-check", *deps],
capture_output=True, text=True, timeout=600,
)
if r.returncode != 0:
logger.warning("warm-venv pip install failed: %s", r.stderr[-1500:])
return None
with open(sentinel, "w", encoding="utf-8") as fh:
fh.write("ok\n")
logger.info("webapp-template: warm backend venv ready at %s", venv_dir)
return venv_dir
except Exception as exc:
logger.warning("warm python venv failed: %s", exc)
return None
def warm_cache_in_background() -> None:
"""Kick off a one-shot daemon thread that pre-populates BOTH the
node_modules cache and the backend-venv cache so the user's FIRST
webapp-template seed doesn't pay the install costs. No-op (fast
return) if both caches are already there or a thread is in flight."""
global p_warm_cache_thread
if p_warm_cache_thread is not None and p_warm_cache_thread.is_alive():
return
node_done = warm_cache_is_complete(os.path.join(p_warm_cache_dir(), "node_modules"))
venv_done = os.path.isfile(os.path.join(warm_venv_dir(), ".populated"))
if node_done and venv_done:
return
def p_runner() -> None:
try:
ensure_warm_cache()
except Exception:
logger.exception("background warm node_modules crashed")
try:
p_ensure_warm_python_venv()
except Exception:
logger.exception("background warm python venv crashed")
p_warm_cache_thread = threading.Thread(
target=p_runner, daemon=True, name="webapp-template-warm-cache"
)
p_warm_cache_thread.start()
# Trigger pre-warm on module import; backend startup hits this and the installs run in parallel with the rest of the boot. By the time the user creates their first app, node_modules + the backend venv are usually ready.
warm_cache_in_background()
def patch_env_port(env_path: str, key: str, value: str) -> None:
"""Idempotent in-place rewrite: `KEY=...` → `KEY=value`. Appends if
the key isn't present. Preserves surrounding lines untouched."""
if not os.path.exists(env_path):
return
with open(env_path, encoding="utf-8") as f:
text = f.read()
pat = re.compile(rf"^{re.escape(key)}=.*$", re.MULTILINE)
new_line = f"{key}={value}"
if pat.search(text):
text = pat.sub(new_line, text)
else:
if text and not text.endswith("\n"):
text += "\n"
text += new_line + "\n"
with open(env_path, "w", encoding="utf-8") as f:
f.write(text)
def seed_webapp_template_workspace(workspace_dir: str, frontend_port: int) -> None:
"""Copy the vendored webapp-template snapshot into `workspace_dir`,
excluding the master template's `backend/` (brought in on-demand by
the workspace's own `backend_init.sh`). Then:
1. Copy `.env.example` → `.env` verbatim (preserves the upstream
defaults `FRONTEND_PORT=4949` and `BACKEND_PORT=NONE`).
2. Sed both `.env` and `.env.example` to set `FRONTEND_PORT=