mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-08-17 18:25:42 +02:00
[eric] runner: boot the real Electron shell under Xvfb so a cloud run drives a real browser
Co-Authored-By: Claude Opus 5 <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_015q1TVSLNNsXAoyM7sJuVKP
This commit is contained in:
co-authored by
Claude Opus 5
parent
949655e86c
commit
aa372dee13
@@ -10,11 +10,23 @@
|
||||
# /app/backend the FastAPI orchestrator
|
||||
# /app/router 9router's standalone server, found by p_find_9router_dir()
|
||||
# /app/python-env UV_PYTHON target probed by tools_lib/mcp_config.py
|
||||
#
|
||||
# Plus the renderer half, which exists so browser tools work the way they do on a
|
||||
# laptop instead of being denied:
|
||||
# /app/electron-runtime the same CastLabs Electron build the desktop app ships
|
||||
# /app/electron the desktop shell's own main process, unmodified
|
||||
# /app/frontend the production webpack bundle, served off loopback
|
||||
#
|
||||
# amd64 only. CastLabs publishes no linux-arm64 build, and running a DIFFERENT
|
||||
# Electron than the desktop app ships would quietly undo the point of this image.
|
||||
|
||||
ARG PYTHON_VERSION=3.13
|
||||
ARG NODE_VERSION=20
|
||||
ARG ROUTER_VERSION=0.3.60
|
||||
ARG UV_VERSION=0.11.8
|
||||
# Must track electron/package.json's devDependency, or the container drives a different browser than the laptop does.
|
||||
ARG ELECTRON_VERSION=42.3.3+wvcus
|
||||
ARG ELECTRON_SHA256=5b6ce3a4d13f07fc63d79e884f6a40d1bc8a1cdf82cb2d130c26e8c1530649cb
|
||||
|
||||
FROM node:${NODE_VERSION}-bookworm-slim AS node
|
||||
|
||||
@@ -27,6 +39,35 @@ RUN printf '{"name":"router-stage","version":"0.0.0","private":true}\n' > packag
|
||||
&& test -f node_modules/9router/app/server.js \
|
||||
&& test -z "$(find node_modules/9router -name '*.node' -print -quit)"
|
||||
|
||||
# Webpack output is architecture-independent, so this runs natively on the build host rather than under emulation.
|
||||
FROM --platform=$BUILDPLATFORM node:${NODE_VERSION}-bookworm-slim AS frontend
|
||||
WORKDIR /src
|
||||
COPY frontend/package.json frontend/package-lock.json ./
|
||||
RUN npm ci --no-audit --no-fund --silent
|
||||
COPY frontend ./
|
||||
RUN npm run build && test -f dist/index.html
|
||||
|
||||
# The shell's runtime deps only. --ignore-scripts leaves uiohook-napi without its prebuilt addon, which is correct: it taps a real keyboard, there isn't one here, and voiceHotkey already requires it inside a try.
|
||||
FROM --platform=$BUILDPLATFORM node:${NODE_VERSION}-bookworm-slim AS shell-deps
|
||||
WORKDIR /stage
|
||||
COPY electron/package.json electron/package-lock.json ./
|
||||
RUN npm install --omit=dev --ignore-scripts --no-audit --no-fund --silent
|
||||
|
||||
FROM debian:bookworm-slim AS electron
|
||||
ARG ELECTRON_VERSION
|
||||
ARG ELECTRON_SHA256
|
||||
ARG TARGETARCH
|
||||
RUN set -eux; \
|
||||
test "${TARGETARCH}" = "amd64" || { echo "the renderer half is amd64-only: CastLabs ships no linux-${TARGETARCH} Electron" >&2; exit 1; }; \
|
||||
apt-get update && apt-get install -y --no-install-recommends curl ca-certificates unzip; \
|
||||
url="https://github.com/castlabs/electron-releases/releases/download/v${ELECTRON_VERSION}/electron-v${ELECTRON_VERSION}-linux-x64.zip"; \
|
||||
curl -fsSL -o /tmp/electron.zip "${url}"; \
|
||||
echo "${ELECTRON_SHA256} /tmp/electron.zip" | sha256sum -c -; \
|
||||
mkdir -p /stage; \
|
||||
unzip -q /tmp/electron.zip -d /stage; \
|
||||
rm /tmp/electron.zip; \
|
||||
test -x /stage/electron
|
||||
|
||||
FROM python:${PYTHON_VERSION}-slim-bookworm AS uv
|
||||
ARG UV_VERSION
|
||||
ARG TARGETARCH
|
||||
@@ -50,21 +91,33 @@ RUN pip install --no-cache-dir --require-hashes --only-binary=:all: \
|
||||
|
||||
FROM python:${PYTHON_VERSION}-slim-bookworm
|
||||
|
||||
# The X server plus every shared object `ldd` reports the Electron binary wanting, and the fonts without which every page renders as boxes. Derived from ldd on the real binary, not from a blog post.
|
||||
RUN set -eux; \
|
||||
apt-get update; \
|
||||
apt-get install -y --no-install-recommends git ca-certificates; \
|
||||
apt-get install -y --no-install-recommends \
|
||||
git ca-certificates \
|
||||
xvfb fonts-liberation \
|
||||
libasound2 libatk-bridge2.0-0 libatk1.0-0 libatspi2.0-0 libcairo2 libcups2 \
|
||||
libdbus-1-3 libdrm2 libexpat1 libgbm1 libglib2.0-0 libgtk-3-0 libnss3 \
|
||||
libpango-1.0-0 libx11-6 libxcb1 libxcomposite1 libxdamage1 libxext6 \
|
||||
libxfixes3 libxkbcommon0 libxrandr2 libxtst6; \
|
||||
rm -rf /var/lib/apt/lists/*
|
||||
|
||||
COPY --from=node /usr/local/bin/node /usr/local/bin/node
|
||||
COPY --from=pydeps /opt/pydeps /usr/local
|
||||
COPY --from=router /stage/node_modules/9router/app /app/router
|
||||
|
||||
COPY backend /app/backend
|
||||
COPY openswarm-runner/runner /app/runner
|
||||
# Numeric owner on every /app copy, because a `chown -R /app` afterwards rewrites the whole tree into a second layer and the image pays for it twice (that cost 493MB before this line existed). Numeric, not `runner`, because the user is created further down.
|
||||
COPY --from=router --chown=10001:10001 /stage/node_modules/9router/app /app/router
|
||||
COPY --chown=10001:10001 backend /app/backend
|
||||
COPY --chown=10001:10001 openswarm-runner/runner /app/runner
|
||||
COPY --chown=10001:10001 electron /app/electron
|
||||
COPY --from=shell-deps --chown=10001:10001 /stage/node_modules /app/electron/node_modules
|
||||
COPY --from=electron --chown=10001:10001 /stage /app/electron-runtime
|
||||
COPY --from=frontend --chown=10001:10001 /src/dist /app/frontend
|
||||
|
||||
# After backend/, never before: mcp_config.resolve_command probes uv-bin last, and the repo's own copy is Mach-O.
|
||||
COPY --from=uv /stage/uv /app/backend/uv-bin/uv
|
||||
COPY --from=uv /stage/uvx /app/backend/uv-bin/uvx
|
||||
COPY --from=uv --chown=10001:10001 /stage/uv /app/backend/uv-bin/uv
|
||||
COPY --from=uv --chown=10001:10001 /stage/uvx /app/backend/uv-bin/uvx
|
||||
|
||||
RUN set -eux; \
|
||||
if ls /app/backend/.env* >/dev/null 2>&1; then echo "a dotenv reached the image; fix Dockerfile.dockerignore" >&2; exit 1; fi; \
|
||||
@@ -73,7 +126,10 @@ RUN set -eux; \
|
||||
find /app/backend -name '__pycache__' -type d -prune -exec rm -rf {} +; \
|
||||
useradd --create-home --uid 10001 --shell /usr/sbin/nologin runner; \
|
||||
mkdir -p /data; \
|
||||
chown -R runner:runner /app /data
|
||||
ln -s /data/openswarm /app/backend/data; \
|
||||
mkdir -p /tmp/.X11-unix; \
|
||||
chmod 1777 /tmp/.X11-unix; \
|
||||
chown runner:runner /data
|
||||
|
||||
USER runner
|
||||
WORKDIR /app
|
||||
@@ -87,6 +143,7 @@ ENV HOME=/home/runner \
|
||||
OPENSWARM_HOST=127.0.0.1 \
|
||||
OPENSWARM_PORT=8324 \
|
||||
DATA_DIR=/data/9router \
|
||||
NODE_ENV=production
|
||||
NODE_ENV=production \
|
||||
ELECTRON_BIN=/app/electron-runtime/electron
|
||||
|
||||
ENTRYPOINT ["python3", "-m", "runner.main"]
|
||||
|
||||
@@ -1,6 +1,8 @@
|
||||
*
|
||||
!backend
|
||||
!openswarm-runner/runner
|
||||
!electron
|
||||
!frontend
|
||||
|
||||
# A developer's real OAuth client secrets live here; baking them into an image that
|
||||
# gets pushed to a registry is how a laptop leaks credentials. The Dockerfile asserts
|
||||
@@ -13,6 +15,19 @@ backend/.venv
|
||||
backend/uv-bin
|
||||
backend/tests
|
||||
backend/.pytest_cache
|
||||
|
||||
# The bundle is built in a stage inside the image. A developer's stale local dist must
|
||||
# never be what a cloud run renders.
|
||||
frontend/dist
|
||||
frontend/node_modules
|
||||
electron/node_modules
|
||||
electron/dist
|
||||
electron/build-staging
|
||||
electron/python-env
|
||||
# Prebuilt Mach-O addons for the mac trackpad; main.js already skips a missing one.
|
||||
electron/native
|
||||
**/*.test.js
|
||||
|
||||
**/__pycache__
|
||||
**/*.pyc
|
||||
**/.DS_Store
|
||||
|
||||
@@ -5,13 +5,17 @@ One Fly Firecracker machine per run, no state kept.
|
||||
|
||||
## Build
|
||||
|
||||
The build context is the **repo root**, not this directory (the image needs `backend/`
|
||||
and `backend/requirements.lock`):
|
||||
The build context is the **repo root**, not this directory (the image needs `backend/`,
|
||||
`electron/`, `frontend/` and `backend/requirements.lock`). **amd64 only**, see the
|
||||
renderer section:
|
||||
|
||||
```bash
|
||||
docker build --platform linux/amd64 -f openswarm-runner/Dockerfile -t openswarm-runner .
|
||||
```
|
||||
|
||||
Nothing has to be built on the host first: the frontend bundle and the shell's node
|
||||
modules are built in their own stages inside the image.
|
||||
|
||||
## Run
|
||||
|
||||
The container is told everything it needs by one JSON run spec in `OPENSWARM_RUN_SPEC`
|
||||
@@ -29,12 +33,49 @@ The container is told everything it needs by one JSON run spec in `OPENSWARM_RUN
|
||||
],
|
||||
"callback": { "url": "https://api.openswarm.com/api/cloud-runs/cr_01J.../report",
|
||||
"token": "<two-party runner token, not a user credential>" },
|
||||
"max_run_seconds": 1800
|
||||
"max_run_seconds": 1800,
|
||||
"needs_browser": true
|
||||
}
|
||||
```
|
||||
|
||||
Exit codes: `0` ok, `1` runner crash, `2` bad spec, `3` credential expired on arrival,
|
||||
`4` backend never came up, `5` workflow failed, `6` wall-clock cap hit.
|
||||
`4` backend never came up, `5` workflow failed, `6` wall-clock cap hit,
|
||||
`7` no Electron window ever registered.
|
||||
|
||||
## The renderer
|
||||
|
||||
OpenSwarm's browser tier is not an HTTP client. Element serialization and every click,
|
||||
type and scroll live in `frontend/src/shared/browserCommandHandler.ts` and drive a live
|
||||
Electron `<webview>`; the backend only relays commands over the dashboard WebSocket. So
|
||||
the container runs **the real desktop shell**, unmodified, on a virtual display:
|
||||
|
||||
```
|
||||
Xvfb :99 -> Electron (ELECTRON_DEV=1, OPENSWARM_DEV_URL=<bundle>#/dashboard/cloud-run)
|
||||
-> registers on /ws/dashboard -> browser tools are live
|
||||
```
|
||||
|
||||
`ELECTRON_DEV=1` is the same path `bash run.sh` uses: the shell attaches to the backend
|
||||
already running here instead of spawning a second one. The bundle is served off loopback
|
||||
on `:4173`, the same port the packaged app prefers, and deep-linked at the run's one
|
||||
dashboard so no human has to click anything.
|
||||
|
||||
Three things follow from this and are worth knowing before you touch it:
|
||||
|
||||
- **amd64 only.** CastLabs (whose Electron the desktop app ships) publishes no
|
||||
linux-arm64 build. Running a *different* Electron in the cloud than users run on their
|
||||
laptops would quietly undo the point of the image, so the build refuses other arches.
|
||||
- **`--no-sandbox`.** Chromium's setuid sandbox needs a root-owned binary and its
|
||||
namespace sandbox needs unprivileged user namespaces; a non-root container under
|
||||
Docker's default seccomp has neither. The wall this run relies on is the Firecracker VM
|
||||
around the whole container, not Chromium's own layer. The flag lives in a named constant
|
||||
in `runner/renderer_process.py` rather than inside a launch string, on purpose.
|
||||
- **`needs_browser: false` skips it.** Boot costs roughly 15s and ~500MB of the run's
|
||||
memory, so a workflow that never opens a page can opt out. Default is on: parity is the
|
||||
reason this image exists, and opting out should be the thing you have to say.
|
||||
|
||||
If Electron starts but no window ever registers, the run **fails** (exit 7) rather than
|
||||
proceeding without a browser. A browser workflow that silently ran blind produces a
|
||||
confident wrong answer, which is worse than no answer.
|
||||
|
||||
## The credential rule
|
||||
|
||||
@@ -62,6 +103,11 @@ An access token that arrives expired fails the run (exit 3). The runner never re
|
||||
PYTHONPATH=.:openswarm-runner backend/.venv/bin/python3 -m pytest openswarm-runner/tests -q
|
||||
```
|
||||
|
||||
The Electron boot itself needs Linux and a display, so the tests pin the contract around
|
||||
it (the deep link, the bundle check, the three ways "no window" ends) rather than the
|
||||
boot. Proving the browser tier actually behaves means running a real page in both places
|
||||
and comparing; see the parity matrix in the cloud-browser work notes.
|
||||
|
||||
## Deploy
|
||||
|
||||
Not deployed. `fly.toml` is written but never applied; read its header first, the app
|
||||
|
||||
@@ -0,0 +1,233 @@
|
||||
"""Boot the real Electron shell inside the container so browser tools have a window to drive.
|
||||
|
||||
The browser tier is not an HTTP client: the element serialization and every click, type and
|
||||
scroll live in the frontend and drive a live Electron `<webview>`. There is no way to get
|
||||
laptop-identical behaviour without the laptop's actual renderer, so this starts one on a
|
||||
virtual display and points it at the backend that is already running in this container.
|
||||
|
||||
The Electron process runs the same `ELECTRON_DEV=1` path a developer uses (`bash run.sh`):
|
||||
the shell attaches to an existing backend on OPENSWARM_PORT instead of spawning its own, and
|
||||
loads whatever OPENSWARM_DEV_URL says. Here that URL is the packaged frontend bundle served
|
||||
off loopback, deep-linked straight at the run's dashboard so the window registers without a
|
||||
human clicking anything.
|
||||
"""
|
||||
|
||||
import functools
|
||||
import http.server
|
||||
import logging
|
||||
import os
|
||||
import shutil
|
||||
import socketserver
|
||||
import subprocess
|
||||
import threading
|
||||
import time
|
||||
from typing import Dict, List, Optional
|
||||
|
||||
import httpx
|
||||
from pydantic import BaseModel, ConfigDict, InstanceOf
|
||||
from typeguard import typechecked
|
||||
|
||||
logger = logging.getLogger("runner.renderer")
|
||||
|
||||
HOST = "127.0.0.1"
|
||||
RENDERER_HEALTH_PATH = "/api/health/renderer"
|
||||
# Same port the packaged app prefers, so the renderer's origin (and therefore its localStorage) is the one the frontend was built expecting.
|
||||
FRONTEND_PORT = 4173
|
||||
DISPLAY = ":99"
|
||||
XVFB_SCREEN = "1920x1080x24"
|
||||
# Cold Electron under a virtual display: Xvfb, Chromium boot, React mount, then the deferred dashboard socket. Measured in the low tens of seconds, so the budget is generous rather than tight.
|
||||
RENDERER_TIMEOUT_SECONDS = 180.0
|
||||
XVFB_READY_TIMEOUT_SECONDS = 20.0
|
||||
SHUTDOWN_GRACE_SECONDS = 5.0
|
||||
|
||||
# Chromium's setuid sandbox needs a root-owned binary and its namespace sandbox needs unprivileged
|
||||
# user namespaces, neither of which a non-root container under Docker's default seccomp profile has.
|
||||
# The isolation this run relies on is the Firecracker VM around the whole container, not Chromium's
|
||||
# own layer. Stated here rather than buried in a launch string, because dropping a sandbox is a
|
||||
# choice and the reader deserves to see it made.
|
||||
SANDBOX_FLAGS: List[str] = ["--no-sandbox"]
|
||||
# /dev/shm defaults to 64MB in a container, which Chromium overruns and then crashes on.
|
||||
CONTAINER_CHROMIUM_FLAGS: List[str] = ["--disable-dev-shm-usage", "--disable-gpu"]
|
||||
|
||||
|
||||
class RendererUnavailable(RuntimeError):
|
||||
"""No Electron window ever registered, so browser tools would be dead this run."""
|
||||
|
||||
|
||||
class RendererProcess(BaseModel):
|
||||
"""The virtual display and the Electron shell drawing into it. Dies with the run."""
|
||||
|
||||
model_config = ConfigDict(validate_assignment=True)
|
||||
|
||||
xvfb: InstanceOf[subprocess.Popen]
|
||||
electron: InstanceOf[subprocess.Popen]
|
||||
url: str
|
||||
|
||||
@typechecked
|
||||
def is_alive(self) -> bool:
|
||||
return self.electron.poll() is None and self.xvfb.poll() is None
|
||||
|
||||
|
||||
@typechecked
|
||||
def dashboard_url(port: int, dashboard_id: str) -> str:
|
||||
"""The frontend deep-link that mounts a dashboard directly. HashRouter, so the route is a fragment."""
|
||||
return f"http://{HOST}:{port}/index.html#/dashboard/{dashboard_id}"
|
||||
|
||||
|
||||
@typechecked
|
||||
def serve_frontend(frontend_dir: str) -> int:
|
||||
"""Serve the built bundle off loopback in a daemon thread; returns the port it landed on.
|
||||
|
||||
Falls back to an OS-assigned port if 4173 is held, exactly like the packaged shell does.
|
||||
"""
|
||||
if not os.path.isfile(os.path.join(frontend_dir, "index.html")):
|
||||
raise RendererUnavailable(
|
||||
f"no frontend bundle at {frontend_dir}; the image must be built with frontend/dist in it"
|
||||
)
|
||||
|
||||
handler = functools.partial(http.server.SimpleHTTPRequestHandler, directory=frontend_dir)
|
||||
|
||||
class p_Server(socketserver.ThreadingTCPServer):
|
||||
daemon_threads = True
|
||||
allow_reuse_address = True
|
||||
|
||||
try:
|
||||
server = p_Server((HOST, FRONTEND_PORT), handler)
|
||||
except OSError:
|
||||
server = p_Server((HOST, 0), handler)
|
||||
port = server.server_address[1]
|
||||
threading.Thread(target=server.serve_forever, daemon=True, name="frontend-server").start()
|
||||
logger.info("frontend bundle served from %s on %s:%d", frontend_dir, HOST, port)
|
||||
return port
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_x_socket_ready(display: str) -> bool:
|
||||
return os.path.exists(f"/tmp/.X11-unix/X{display.lstrip(':')}")
|
||||
|
||||
|
||||
@typechecked
|
||||
def start_xvfb(display: str = DISPLAY) -> subprocess.Popen:
|
||||
"""Bring up the virtual display and wait for its socket, so Electron never races it."""
|
||||
if shutil.which("Xvfb") is None:
|
||||
raise RendererUnavailable("Xvfb is not installed in this image, so there is no display to draw on")
|
||||
process = subprocess.Popen(
|
||||
["Xvfb", display, "-screen", "0", XVFB_SCREEN, "-nolisten", "tcp"],
|
||||
stdout=subprocess.DEVNULL,
|
||||
stderr=subprocess.PIPE,
|
||||
)
|
||||
budget = time.monotonic() + XVFB_READY_TIMEOUT_SECONDS
|
||||
while time.monotonic() < budget:
|
||||
if process.poll() is not None:
|
||||
raise RendererUnavailable(f"Xvfb exited immediately with code {process.returncode}")
|
||||
if p_x_socket_ready(display):
|
||||
logger.info("virtual display %s up (%s)", display, XVFB_SCREEN)
|
||||
return process
|
||||
time.sleep(0.1)
|
||||
p_stop(process)
|
||||
raise RendererUnavailable(f"Xvfb never created a socket for {display}")
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_electron_env(backend_port: int, url: str, display: str) -> Dict[str, str]:
|
||||
environment = dict(os.environ)
|
||||
# The dev path: attach to the backend already running here rather than spawning a second one, and load the bundle we are serving instead of a webpack dev server.
|
||||
environment["ELECTRON_DEV"] = "1"
|
||||
environment["OPENSWARM_DEV_URL"] = url
|
||||
environment["OPENSWARM_PORT"] = str(backend_port)
|
||||
environment["DISPLAY"] = display
|
||||
environment["ELECTRON_DISABLE_SECURITY_WARNINGS"] = "1"
|
||||
environment.pop("OPENSWARM_PACKAGED", None)
|
||||
return environment
|
||||
|
||||
|
||||
@typechecked
|
||||
def start_electron(app_root: str, backend_port: int, url: str, display: str = DISPLAY) -> subprocess.Popen:
|
||||
binary = os.environ.get("ELECTRON_BIN", "/app/electron-runtime/electron")
|
||||
if not os.path.isfile(binary):
|
||||
raise RendererUnavailable(f"no Electron binary at {binary}; the image was built without a renderer")
|
||||
app_dir = os.path.join(app_root, "electron")
|
||||
command = [binary, app_dir, *SANDBOX_FLAGS, *CONTAINER_CHROMIUM_FLAGS]
|
||||
logger.info("starting Electron: %s", " ".join(command))
|
||||
return subprocess.Popen(command, cwd=app_dir, env=p_electron_env(backend_port, url, display))
|
||||
|
||||
|
||||
@typechecked
|
||||
def await_registration(
|
||||
base_url: str,
|
||||
headers: Dict[str, str],
|
||||
electron: subprocess.Popen,
|
||||
deadline: float,
|
||||
) -> None:
|
||||
"""Block until the backend reports a renderer on its dashboard socket, or raise.
|
||||
|
||||
Polls the backend rather than the Electron process because "the window is up" and "the
|
||||
window can be driven" are different claims, and only the second one matters.
|
||||
"""
|
||||
budget = min(time.monotonic() + RENDERER_TIMEOUT_SECONDS, deadline)
|
||||
with httpx.Client(timeout=5.0) as client:
|
||||
while time.monotonic() < budget:
|
||||
if electron.poll() is not None:
|
||||
raise RendererUnavailable(
|
||||
f"Electron exited with code {electron.returncode} before any window registered"
|
||||
)
|
||||
try:
|
||||
body = client.get(f"{base_url}{RENDERER_HEALTH_PATH}", headers=headers).json()
|
||||
except (httpx.HTTPError, ValueError):
|
||||
body = {}
|
||||
if body.get("attached"):
|
||||
logger.info("renderer registered (%s dashboard connection(s))", body.get("connections"))
|
||||
return
|
||||
time.sleep(0.5)
|
||||
raise RendererUnavailable(
|
||||
"Electron started but no renderer ever registered on the dashboard WebSocket within "
|
||||
f"{RENDERER_TIMEOUT_SECONDS:.0f}s, so browser tools would be dead this run"
|
||||
)
|
||||
|
||||
|
||||
@typechecked
|
||||
def start_renderer(
|
||||
app_root: str,
|
||||
frontend_dir: str,
|
||||
backend_base_url: str,
|
||||
backend_headers: Dict[str, str],
|
||||
backend_port: int,
|
||||
dashboard_id: str,
|
||||
deadline: float,
|
||||
) -> RendererProcess:
|
||||
"""Display, bundle server, Electron, then block until the window is actually drivable."""
|
||||
url = dashboard_url(serve_frontend(frontend_dir), dashboard_id)
|
||||
xvfb = start_xvfb()
|
||||
try:
|
||||
electron = start_electron(app_root, backend_port, url)
|
||||
except BaseException:
|
||||
p_stop(xvfb)
|
||||
raise
|
||||
try:
|
||||
await_registration(backend_base_url, backend_headers, electron, deadline)
|
||||
except BaseException:
|
||||
p_stop(electron)
|
||||
p_stop(xvfb)
|
||||
raise
|
||||
return RendererProcess(xvfb=xvfb, electron=electron, url=url)
|
||||
|
||||
|
||||
@typechecked
|
||||
def p_stop(process: Optional[subprocess.Popen]) -> None:
|
||||
if process is None or process.poll() is not None:
|
||||
return
|
||||
process.terminate()
|
||||
try:
|
||||
process.wait(timeout=SHUTDOWN_GRACE_SECONDS)
|
||||
except subprocess.TimeoutExpired:
|
||||
process.kill()
|
||||
process.wait(timeout=SHUTDOWN_GRACE_SECONDS)
|
||||
|
||||
|
||||
@typechecked
|
||||
def stop_renderer(renderer: Optional[RendererProcess]) -> None:
|
||||
"""Electron first, then the display it was drawing on."""
|
||||
if renderer is None:
|
||||
return
|
||||
p_stop(renderer.electron)
|
||||
p_stop(renderer.xvfb)
|
||||
@@ -16,9 +16,10 @@ from typing import Optional
|
||||
from pydantic import BaseModel, ConfigDict
|
||||
from typeguard import typechecked
|
||||
|
||||
from runner.backend_process import BackendProcess, BackendUnavailable, start_backend, stop_backend
|
||||
from runner.boot.backend_process import BackendProcess, BackendUnavailable, start_backend, stop_backend
|
||||
from runner.boot.renderer_process import RendererProcess, RendererUnavailable, start_renderer, stop_renderer
|
||||
from runner.report import RunReport, send_report
|
||||
from runner.run_spec import CallbackTarget, InvalidRunSpec, RunSpec, load_run_spec
|
||||
from runner.run_spec import CLOUD_RUN_DASHBOARD_ID, CallbackTarget, InvalidRunSpec, RunSpec, load_run_spec
|
||||
from runner.seed.data_root import seed_data_root
|
||||
from runner.seed.router_credentials import write_router_db
|
||||
from runner.workflow_run import RunOutcome, RunProgress, WorkflowRunFailed, execute_workflow
|
||||
@@ -30,8 +31,10 @@ EXIT_CREDENTIAL_EXPIRED = 3
|
||||
EXIT_BACKEND_UNAVAILABLE = 4
|
||||
EXIT_WORKFLOW_FAILED = 5
|
||||
EXIT_DEADLINE = 6
|
||||
EXIT_RENDERER_UNAVAILABLE = 7
|
||||
|
||||
DEFAULT_APP_ROOT = "/app"
|
||||
DEFAULT_FRONTEND_DIR = "/app/frontend"
|
||||
DEFAULT_DATA_ROOT = "/data/openswarm"
|
||||
DEFAULT_ROUTER_DATA_DIR = "/data/9router"
|
||||
DEFAULT_PORT = 8324
|
||||
@@ -138,11 +141,24 @@ def p_run(spec: RunSpec, deadline: float) -> int:
|
||||
|
||||
backend: Optional[BackendProcess] = None
|
||||
process: Optional[subprocess.Popen] = None
|
||||
renderer: Optional[RendererProcess] = None
|
||||
try:
|
||||
backend = start_backend(app_root, data_root, port, deadline)
|
||||
process = backend.process
|
||||
logger.info("backend healthy at %s", backend.base_url)
|
||||
|
||||
if spec.needs_browser:
|
||||
renderer = start_renderer(
|
||||
app_root=app_root,
|
||||
frontend_dir=os.environ.get("OPENSWARM_FRONTEND_DIR", DEFAULT_FRONTEND_DIR),
|
||||
backend_base_url=backend.base_url,
|
||||
backend_headers=backend.headers(),
|
||||
backend_port=port,
|
||||
dashboard_id=CLOUD_RUN_DASHBOARD_ID,
|
||||
deadline=deadline,
|
||||
)
|
||||
logger.info("renderer attached at %s, browser tools are live", renderer.url)
|
||||
|
||||
send_report(spec.callback, RunReport(run_id=spec.run_id, phase="started", status="running"))
|
||||
heartbeat = Heartbeat(
|
||||
run_id=spec.run_id,
|
||||
@@ -152,9 +168,14 @@ def p_run(spec: RunSpec, deadline: float) -> int:
|
||||
outcome = execute_workflow(backend, spec.workflow.id, deadline, heartbeat.maybe_send)
|
||||
except BackendUnavailable as exc:
|
||||
return p_fail(spec, "failure", str(exc), EXIT_BACKEND_UNAVAILABLE)
|
||||
except RendererUnavailable as exc:
|
||||
# Loud, not silent: a browser workflow that quietly ran without a window produces a
|
||||
# confident wrong answer, which is worse than no answer.
|
||||
return p_fail(spec, "failure", str(exc), EXIT_RENDERER_UNAVAILABLE)
|
||||
except WorkflowRunFailed as exc:
|
||||
return p_fail(spec, "failure", str(exc), EXIT_WORKFLOW_FAILED)
|
||||
finally:
|
||||
stop_renderer(renderer)
|
||||
stop_backend(process)
|
||||
|
||||
code = p_exit_code_for(outcome)
|
||||
|
||||
@@ -19,6 +19,11 @@ from backend.apps.workflows.models import Workflow
|
||||
SPEC_ENV = "OPENSWARM_RUN_SPEC"
|
||||
SPEC_FILE_ENV = "OPENSWARM_RUN_SPEC_FILE"
|
||||
|
||||
# The one dashboard a cloud run has. Fixed rather than generated so the Electron window can be
|
||||
# deep-linked at it before the backend has even booted, and so a workflow arriving with the
|
||||
# laptop dashboard id it was authored against gets repointed at a dashboard that exists here.
|
||||
CLOUD_RUN_DASHBOARD_ID = "cloud-run"
|
||||
|
||||
# Headroom the access token must still have on arrival. The control plane refreshes right before dispatch; anything thinner than this means its clock or its queue is broken, and we must not paper over that by refreshing ourselves.
|
||||
MIN_TOKEN_LIFETIME = timedelta(minutes=2)
|
||||
|
||||
@@ -88,6 +93,10 @@ class RunSpec(BaseModel):
|
||||
callback: Optional[CallbackTarget] = None
|
||||
# Hard wall-clock ceiling. Fly bills by machine-second, so an agent that wedges must cost a bounded amount.
|
||||
max_run_seconds: int = Field(default=1800, ge=60, le=7200)
|
||||
# Boot Electron under a virtual display so browser steps work. On by default: parity is the
|
||||
# point of running in a container at all, and a workflow that never touches a browser is the
|
||||
# exception that should have to say so. Costs a few seconds and a few hundred MB when on.
|
||||
needs_browser: bool = True
|
||||
|
||||
@typechecked
|
||||
def expired_credentials(self, now: datetime) -> List[ProviderCredential]:
|
||||
@@ -106,8 +115,13 @@ class RunSpec(BaseModel):
|
||||
A cloud-executed workflow arrives with its schedule still configured. Left
|
||||
enabled, the container's own scheduler would fire it a second time inside
|
||||
the box, so the timer is stripped here rather than trusted to stay off.
|
||||
|
||||
It also arrives pointing at whatever dashboard it was authored on, which does not
|
||||
exist in this container; left alone, the first browser card would 404 looking for
|
||||
it. Repointed at the one dashboard this run has.
|
||||
"""
|
||||
copy = self.workflow.model_copy(deep=True)
|
||||
copy.dashboard_id = CLOUD_RUN_DASHBOARD_ID
|
||||
copy.schedule.enabled = False
|
||||
copy.deleted_at = None
|
||||
copy.draft_steps = None
|
||||
|
||||
@@ -13,8 +13,9 @@ from typing import Any, Dict
|
||||
|
||||
from typeguard import typechecked
|
||||
|
||||
from backend.apps.dashboards.models import Dashboard
|
||||
from backend.apps.settings.models import AppSettings
|
||||
from runner.run_spec import RunSpec
|
||||
from runner.run_spec import CLOUD_RUN_DASHBOARD_ID, RunSpec
|
||||
|
||||
# 9Router provider id -> the AppSettings field the backend reads a raw key from.
|
||||
API_KEY_SETTINGS_FIELD: Dict[str, str] = {
|
||||
@@ -69,7 +70,12 @@ def settings_for_run(spec: RunSpec) -> AppSettings:
|
||||
|
||||
@typechecked
|
||||
def seed_data_root(data_root: str, spec: RunSpec) -> None:
|
||||
"""Write the workflow record and the settings file the backend will read at boot."""
|
||||
"""Write the workflow, settings and dashboard records the backend will read at boot.
|
||||
|
||||
The dashboard exists so the Electron window has somewhere to land and browser cards have
|
||||
somewhere to render. Writing it here rather than letting the backend's first-boot migration
|
||||
invent one keeps its id knowable before anything has started.
|
||||
"""
|
||||
workflow = spec.workflow_for_disk()
|
||||
p_write_json(
|
||||
os.path.join(data_root, "workflows", f"{workflow.id}.json"),
|
||||
@@ -79,3 +85,8 @@ def seed_data_root(data_root: str, spec: RunSpec) -> None:
|
||||
os.path.join(data_root, "settings", "settings.json"),
|
||||
settings_for_run(spec).model_dump(mode="json"),
|
||||
)
|
||||
dashboard = Dashboard(id=CLOUD_RUN_DASHBOARD_ID, name=spec.workflow.title or "Cloud run")
|
||||
p_write_json(
|
||||
os.path.join(data_root, "dashboards", f"{dashboard.id}.json"),
|
||||
dashboard.model_dump(mode="json"),
|
||||
)
|
||||
|
||||
@@ -13,7 +13,7 @@ import httpx
|
||||
from pydantic import BaseModel, ConfigDict, Field
|
||||
from typeguard import typechecked
|
||||
|
||||
from runner.backend_process import BackendProcess
|
||||
from runner.boot.backend_process import BackendProcess
|
||||
|
||||
TERMINAL_STATUSES = ("success", "failure", "ran_late", "skipped")
|
||||
POLL_INTERVAL_SECONDS = 1.0
|
||||
|
||||
@@ -0,0 +1,99 @@
|
||||
"""The renderer half: a run that asked for a browser must never quietly proceed without one.
|
||||
|
||||
The Electron boot itself needs a Linux container and a real display, so what is pinned here is
|
||||
the contract around it: the deep link the window opens on, the bundle check, and the three ways
|
||||
"no window" is allowed to end (loudly, every time).
|
||||
"""
|
||||
|
||||
import subprocess
|
||||
|
||||
import pytest
|
||||
|
||||
from runner.boot import renderer_process
|
||||
from runner.boot.renderer_process import (
|
||||
CONTAINER_CHROMIUM_FLAGS,
|
||||
RendererUnavailable,
|
||||
SANDBOX_FLAGS,
|
||||
await_registration,
|
||||
dashboard_url,
|
||||
serve_frontend,
|
||||
start_electron,
|
||||
)
|
||||
from runner.run_spec import CLOUD_RUN_DASHBOARD_ID
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def dead_electron():
|
||||
"""A real Popen that has already exited; await_registration is typechecked on Popen."""
|
||||
process = subprocess.Popen(["/bin/sh", "-c", "exit 9"])
|
||||
process.wait()
|
||||
return process
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def live_electron():
|
||||
"""A real Popen that stays up long enough for a poll loop to run against it."""
|
||||
process = subprocess.Popen(["/bin/sh", "-c", "sleep 30"])
|
||||
yield process
|
||||
process.kill()
|
||||
process.wait()
|
||||
|
||||
|
||||
def test_the_window_opens_on_the_run_dashboard_not_the_picker() -> None:
|
||||
# HashRouter, so the route has to be a fragment or the static server 404s on it.
|
||||
assert dashboard_url(4173, CLOUD_RUN_DASHBOARD_ID) == (
|
||||
"http://127.0.0.1:4173/index.html#/dashboard/cloud-run"
|
||||
)
|
||||
|
||||
|
||||
def test_a_missing_bundle_says_so_instead_of_serving_an_empty_dir(tmp_path) -> None:
|
||||
with pytest.raises(RendererUnavailable, match="no frontend bundle"):
|
||||
serve_frontend(str(tmp_path))
|
||||
|
||||
|
||||
def test_a_missing_electron_binary_fails_the_run_rather_than_the_workflow(tmp_path, monkeypatch) -> None:
|
||||
monkeypatch.setenv("ELECTRON_BIN", str(tmp_path / "nope"))
|
||||
with pytest.raises(RendererUnavailable, match="built without a renderer"):
|
||||
start_electron(str(tmp_path), 8324, "http://127.0.0.1:4173/index.html")
|
||||
|
||||
|
||||
def test_chromium_is_launched_unsandboxed_on_purpose_and_out_of_shared_memory() -> None:
|
||||
# Dropping Chromium's own sandbox is a real tradeoff (the Firecracker VM is the wall that's
|
||||
# left), so it lives in a named constant a reviewer trips over, not inside a launch string.
|
||||
assert SANDBOX_FLAGS == ["--no-sandbox"]
|
||||
assert "--disable-dev-shm-usage" in CONTAINER_CHROMIUM_FLAGS
|
||||
|
||||
|
||||
def test_a_dead_electron_is_reported_as_dead_not_waited_out(monkeypatch, dead_electron) -> None:
|
||||
monkeypatch.setattr(renderer_process, "RENDERER_TIMEOUT_SECONDS", 30.0)
|
||||
with pytest.raises(RendererUnavailable, match="exited with code 9"):
|
||||
await_registration("http://127.0.0.1:1", {}, dead_electron, deadline=1e9)
|
||||
|
||||
|
||||
def test_a_window_that_never_registers_times_out_loudly(monkeypatch, live_electron) -> None:
|
||||
monkeypatch.setattr(renderer_process, "RENDERER_TIMEOUT_SECONDS", 0.5)
|
||||
with pytest.raises(RendererUnavailable, match="no renderer ever registered"):
|
||||
await_registration("http://127.0.0.1:1", {}, live_electron, deadline=1e9)
|
||||
|
||||
|
||||
def test_registration_is_believed_only_when_the_backend_says_a_socket_is_attached(monkeypatch, live_electron) -> None:
|
||||
replies = [{"attached": False, "ever_attached": False, "connections": 0},
|
||||
{"attached": True, "ever_attached": True, "connections": 1}]
|
||||
|
||||
class p_Response:
|
||||
def json(self):
|
||||
return replies.pop(0)
|
||||
|
||||
class p_Client:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, *exc):
|
||||
return False
|
||||
|
||||
def get(self, url, headers=None):
|
||||
return p_Response()
|
||||
|
||||
monkeypatch.setattr(renderer_process.httpx, "Client", lambda **kw: p_Client())
|
||||
await_registration("http://127.0.0.1:1", {}, live_electron, deadline=1e9)
|
||||
assert replies == []
|
||||
Reference in New Issue
Block a user