From 9cf70af5872a7dbc85878c147b41e5f5eb0ad787 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sat, 1 Aug 2026 01:19:58 -0700 Subject: [PATCH 1/5] [eric] runner: a cloud run's files come home, and one too big to carry says so instead of vanishing --- openswarm-runner/runner/main.py | 52 ++++- openswarm-runner/runner/report.py | 69 ------- openswarm-runner/runner/results/__init__.py | 0 .../runner/results/deliverables.py | 193 ++++++++++++++++++ openswarm-runner/runner/results/report.py | 168 +++++++++++++++ openswarm-runner/runner/run_spec.py | 62 ++++++ openswarm-runner/runner/seed/data_root.py | 53 ++++- openswarm-runner/tests/test_deliverables.py | 174 ++++++++++++++++ openswarm-runner/tests/test_run_spec.py | 20 +- 9 files changed, 707 insertions(+), 84 deletions(-) delete mode 100644 openswarm-runner/runner/report.py create mode 100644 openswarm-runner/runner/results/__init__.py create mode 100644 openswarm-runner/runner/results/deliverables.py create mode 100644 openswarm-runner/runner/results/report.py create mode 100644 openswarm-runner/tests/test_deliverables.py diff --git a/openswarm-runner/runner/main.py b/openswarm-runner/runner/main.py index 6c7759cc..ac6b177f 100644 --- a/openswarm-runner/runner/main.py +++ b/openswarm-runner/runner/main.py @@ -18,10 +18,12 @@ from typeguard import typechecked from runner.boot.backend_process import BackendProcess, BackendUnavailable, start_backend, stop_backend from runner.boot.renderer_process import RendererProcess, RendererUnavailable, start_renderer, stop_renderer -from runner.report import RunReport, send_report +from runner.results.deliverables import collect +from runner.results.report import RunReport, deliver_files, send_report from runner.run_spec import CLOUD_RUN_DASHBOARD_ID, CallbackTarget, InvalidRunSpec, RunSpec, load_run_spec from runner.seed.data_root import seed_data_root from runner.seed.router_credentials import write_router_db +from runner.seed.skills import write_skills from runner.workflow_run import RunOutcome, RunProgress, WorkflowRunFailed, execute_workflow EXIT_OK = 0 @@ -37,9 +39,13 @@ DEFAULT_APP_ROOT = "/app" DEFAULT_FRONTEND_DIR = "/app/frontend" DEFAULT_DATA_ROOT = "/data/openswarm" DEFAULT_ROUTER_DATA_DIR = "/data/9router" +# The agent's own folder, and the only place on this machine whose contents come home. +DEFAULT_RUN_WORKSPACE = "/data/workspace" DEFAULT_PORT = 8324 # Slack between the soft deadline (stop the run, report it) and the hard one (kill the process). -REPORT_GRACE_SECONDS = 90.0 +# Has to cover the file upload as well as the report's retries, so it is minutes, not seconds; the +# control plane's own kill sits further out again (dispatch.ts MACHINE_GRACE_MS). +REPORT_GRACE_SECONDS = 240.0 # Ceiling the control plane cannot raise. A cap a caller can override is not a cap. MAX_RUN_SECONDS_ENV = "RUNNER_MAX_RUN_SECONDS" DEFAULT_MAX_RUN_SECONDS = 1800 @@ -92,8 +98,25 @@ def arm_hard_stop(seconds: float) -> None: @typechecked -def p_fail(spec: Optional[RunSpec], status: str, message: str, code: int) -> int: +def p_fail( + spec: Optional[RunSpec], + status: str, + message: str, + code: int, + workspace: Optional[str] = None, +) -> int: + """Report a failure, handing over anything the run managed to make first. + + `workspace` is passed only where the agent actually ran: a workflow that died on step 3 may + have written a perfectly good report on step 1, and losing it because a later step threw is + exactly the "the file died with the machine" problem this whole path exists to fix. + """ logger.error("%s: %s", status, message) + files = ( + deliver_files(spec.callback, workspace, collect(workspace)) + if spec is not None and workspace is not None + else [] + ) send_report( spec.callback if spec else None, RunReport( @@ -102,6 +125,7 @@ def p_fail(spec: Optional[RunSpec], status: str, message: str, code: int) -> int status=status, exit_code=code, error=message, + files=files, ), ) return code @@ -133,10 +157,12 @@ def p_run(spec: RunSpec, deadline: float) -> int: app_root = os.environ.get("OPENSWARM_APP_ROOT", DEFAULT_APP_ROOT) data_root = os.environ.get("OPENSWARM_DATA_ROOT", DEFAULT_DATA_ROOT) router_data_dir = os.environ.get("DATA_DIR", DEFAULT_ROUTER_DATA_DIR) + workspace = os.environ.get("OPENSWARM_RUN_WORKSPACE", DEFAULT_RUN_WORKSPACE) port = int(os.environ.get("OPENSWARM_PORT", str(DEFAULT_PORT))) write_router_db(router_data_dir, spec.credentials, now) - seed_data_root(data_root, spec) + seed_data_root(data_root, workspace, spec) + write_skills(os.path.expanduser("~"), spec.skills) logger.info("seeded data root %s and router db in %s", data_root, router_data_dir) backend: Optional[BackendProcess] = None @@ -173,11 +199,15 @@ def p_run(spec: RunSpec, deadline: float) -> int: # confident wrong answer, which is worse than no answer. return p_fail(spec, "failure", str(exc), EXIT_RENDERER_UNAVAILABLE) except WorkflowRunFailed as exc: - return p_fail(spec, "failure", str(exc), EXIT_WORKFLOW_FAILED) + return p_fail(spec, "failure", str(exc), EXIT_WORKFLOW_FAILED, workspace) finally: stop_renderer(renderer) stop_backend(process) + # Files before the terminal report, always: the callback token is refused the moment the run + # is closed, so this is the only order in which both the files and the receipt can land. + files = deliver_files(spec.callback, workspace, collect(workspace)) + code = p_exit_code_for(outcome) logger.info("run %s finished as %s (exit %d)", spec.run_id, outcome.status, code) send_report(spec.callback, RunReport( @@ -190,6 +220,7 @@ def p_run(spec: RunSpec, deadline: float) -> int: answer=outcome.answer, transcript=outcome.transcript, system_notices=outcome.system_notices, + files=files, )) return code @@ -213,7 +244,16 @@ def main() -> int: return p_run(spec, deadline) except Exception as exc: logger.exception("runner crashed") - return p_fail(spec, "failure", f"runner crashed: {exc}", EXIT_INTERNAL) + # Re-uploading a file the successful path already sent is harmless: the control plane keys + # a run's files on their path, so a second delivery overwrites one row rather than billing + # the budget twice. + return p_fail( + spec, + "failure", + f"runner crashed: {exc}", + EXIT_INTERNAL, + os.environ.get("OPENSWARM_RUN_WORKSPACE", DEFAULT_RUN_WORKSPACE), + ) if __name__ == "__main__": diff --git a/openswarm-runner/runner/report.py b/openswarm-runner/runner/report.py deleted file mode 100644 index bb5c3de1..00000000 --- a/openswarm-runner/runner/report.py +++ /dev/null @@ -1,69 +0,0 @@ -"""Tell the control plane what happened. The terminal report is the run's only receipt.""" - -import logging -import time -from typing import List, Literal, Optional - -import httpx -from pydantic import BaseModel, ConfigDict, Field -from typeguard import typechecked - -from runner.run_spec import CallbackTarget - -logger = logging.getLogger(__name__) - -TERMINAL_ATTEMPTS = 5 -TERMINAL_BACKOFF_SECONDS = 2.0 -REQUEST_TIMEOUT_SECONDS = 15.0 - - -class RunReport(BaseModel): - model_config = ConfigDict(validate_assignment=True) - - run_id: str - phase: Literal["started", "heartbeat", "finished"] - status: str - exit_code: Optional[int] = None - error: Optional[str] = None - cost_usd: float = 0.0 - active_step_idx: Optional[int] = None - last_tool_label: Optional[str] = None - answer: str = "" - transcript: str = "" - # The backend calls a run "success" even when the provider rejected the token; these are how the control plane sees that. - system_notices: List[str] = Field(default_factory=list) - - -@typechecked -def p_post_once(callback: CallbackTarget, report: RunReport) -> bool: - try: - with httpx.Client(timeout=REQUEST_TIMEOUT_SECONDS) as client: - response = client.post( - callback.url, - headers={"Authorization": f"Bearer {callback.token}"}, - json=report.model_dump(mode="json"), - ) - if response.status_code < 300: - return True - logger.warning("report %s rejected with HTTP %s", report.phase, response.status_code) - return False - except httpx.HTTPError as exc: - logger.warning("report %s failed to send: %s", report.phase, exc) - return False - - -@typechecked -def send_report(callback: Optional[CallbackTarget], report: RunReport) -> bool: - """Post a report. Terminal reports retry; heartbeats get one shot and are never retried.""" - if callback is None: - logger.info("no callback configured; %s report kept local: %s", report.phase, report.status) - return True - if report.phase != "finished": - return p_post_once(callback, report) - for attempt in range(TERMINAL_ATTEMPTS): - if p_post_once(callback, report): - return True - if attempt + 1 < TERMINAL_ATTEMPTS: - time.sleep(TERMINAL_BACKOFF_SECONDS * (attempt + 1)) - logger.error("terminal report for run %s never landed after %d attempts", report.run_id, TERMINAL_ATTEMPTS) - return False diff --git a/openswarm-runner/runner/results/__init__.py b/openswarm-runner/runner/results/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/openswarm-runner/runner/results/deliverables.py b/openswarm-runner/runner/results/deliverables.py new file mode 100644 index 00000000..5aaba20f --- /dev/null +++ b/openswarm-runner/runner/results/deliverables.py @@ -0,0 +1,193 @@ +"""What the run made, and what of it is allowed to come home. + +A cloud run's machine is destroyed the moment it exits, so a file it wrote is gone +unless something carries it out. This module is the "what": it walks the one +directory a run is given as its working folder and decides, per file, deliver or +refuse. The "how" (handing bytes to the control plane) lives in runner.results.report. + +Refusing loudly is the whole point of the caps. A user who asked for a video and +got a 20MB fragment of one is worse off than a user who was told the video was too +big, so nothing here ever truncates a file: it either arrives whole or it arrives +as a sentence explaining why it did not. +""" + +import hashlib +import logging +import os +from typing import List, Optional, Tuple + +from pydantic import BaseModel, ConfigDict, Field +from typeguard import typechecked + +logger = logging.getLogger(__name__) + +# Per-file ceiling. Deliverables are reports, spreadsheets, charts and small archives; a run that +# produces something bigger is doing a different job than this pipe was built for. +MAX_FILE_BYTES = 20 * 1024 * 1024 +# Per-run ceiling, enforced in walk order so the first files still arrive when a later one blows it. +MAX_TOTAL_BYTES = 50 * 1024 * 1024 +MAX_FILES = 40 +# Longest path we will accept, so a deep tree cannot produce a name no filesystem will take back. +MAX_RELATIVE_PATH_CHARS = 180 + +# Machinery, not deliverables. Everything here is either regenerable (dependencies, caches, +# compiled bytecode) or the run's own plumbing, and shipping it would blow the file budget on +# things nobody asked for. +EXCLUDED_DIRS = frozenset({ + ".git", + ".claude", + "node_modules", + "__pycache__", + ".venv", + "venv", + ".pytest_cache", + ".ruff_cache", + ".mypy_cache", + ".cache", + ".npm", +}) +EXCLUDED_NAMES = frozenset({".DS_Store", ".gitignore", ".gitkeep"}) + +READ_CHUNK_BYTES = 1024 * 1024 + + +class Deliverable(BaseModel): + """One file that fits, addressed by its path relative to the run's workspace.""" + + model_config = ConfigDict(validate_assignment=True) + + path: str + size_bytes: int + sha256: str + + +class Refused(BaseModel): + """One file that does not come home, and the sentence the user gets instead.""" + + model_config = ConfigDict(validate_assignment=True) + + path: str + size_bytes: int + reason: str + + +class Harvest(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + files: List[Deliverable] = Field(default_factory=list) + refused: List[Refused] = Field(default_factory=list) + + @typechecked + def total_bytes(self) -> int: + return sum(item.size_bytes for item in self.files) + + +@typechecked +def human_bytes(count: int) -> str: + if count < 1024: + return f"{count} B" + if count < 1024 * 1024: + return f"{count / 1024:.0f} KB" + if count < 1024 * 1024 * 1024: + return f"{count / (1024 * 1024):.1f} MB" + return f"{count / (1024 * 1024 * 1024):.1f} GB" + + +@typechecked +def p_digest(path: str) -> Optional[str]: + """sha256, streamed. None when the file went away mid-walk, which is not an error.""" + digest = hashlib.sha256() + try: + with open(path, "rb") as handle: + while True: + chunk = handle.read(READ_CHUNK_BYTES) + if not chunk: + break + digest.update(chunk) + except OSError as exc: + logger.warning("could not read %s while harvesting: %s", path, exc) + return None + return digest.hexdigest() + + +@typechecked +def p_walk(workspace: str) -> List[Tuple[str, int]]: + """Every candidate file under the workspace as (relative path, size), sorted for determinism.""" + found: List[Tuple[str, int]] = [] + for directory, subdirs, filenames in os.walk(workspace): + subdirs[:] = sorted(name for name in subdirs if name not in EXCLUDED_DIRS) + for filename in sorted(filenames): + if filename in EXCLUDED_NAMES: + continue + absolute = os.path.join(directory, filename) + # Symlinks are not followed: a run that linked to /etc/passwd must not exfiltrate it. + if os.path.islink(absolute) or not os.path.isfile(absolute): + continue + try: + size = os.path.getsize(absolute) + except OSError: + continue + found.append((os.path.relpath(absolute, workspace), size)) + return found + + +@typechecked +def collect(workspace: str) -> Harvest: + """Decide, for every file in the run's workspace, whether it comes home.""" + harvest = Harvest() + if not os.path.isdir(workspace): + return harvest + + running_total = 0 + for relative, size in p_walk(workspace): + if len(relative) > MAX_RELATIVE_PATH_CHARS: + harvest.refused.append(Refused( + path=relative[:MAX_RELATIVE_PATH_CHARS] + "...", + size_bytes=size, + reason="its path is too long to save anywhere", + )) + continue + if size == 0: + continue + if size > MAX_FILE_BYTES: + harvest.refused.append(Refused( + path=relative, + size_bytes=size, + reason=( + f"it is {human_bytes(size)} and a single cloud-run file cannot exceed " + f"{human_bytes(MAX_FILE_BYTES)}" + ), + )) + continue + if len(harvest.files) >= MAX_FILES: + harvest.refused.append(Refused( + path=relative, + size_bytes=size, + reason=f"this run already produced the maximum of {MAX_FILES} files", + )) + continue + if running_total + size > MAX_TOTAL_BYTES: + harvest.refused.append(Refused( + path=relative, + size_bytes=size, + reason=( + f"the run's files already total {human_bytes(running_total)} and the limit " + f"is {human_bytes(MAX_TOTAL_BYTES)}" + ), + )) + continue + digest = p_digest(os.path.join(workspace, relative)) + if digest is None: + harvest.refused.append(Refused(path=relative, size_bytes=size, reason="it could not be read")) + continue + harvest.files.append(Deliverable(path=relative, size_bytes=size, sha256=digest)) + running_total += size + + logger.info( + "harvested %d file(s) totalling %s from %s, refused %d", + len(harvest.files), + human_bytes(running_total), + workspace, + len(harvest.refused), + ) + return harvest diff --git a/openswarm-runner/runner/results/report.py b/openswarm-runner/runner/results/report.py new file mode 100644 index 00000000..ca769138 --- /dev/null +++ b/openswarm-runner/runner/results/report.py @@ -0,0 +1,168 @@ +"""Tell the control plane what happened, and hand it whatever the run made. + +The terminal report is the run's only receipt. Files go up BEFORE it, on purpose: +the per-run callback token stops working the instant the run reaches a terminal +state, so "upload, then close" is the only order in which both can succeed, and it +means the window for writing files to a run closes exactly when the run does. +""" + +import logging +import os +import time +from typing import List, Literal, Optional +from urllib.parse import quote + +import httpx +from pydantic import BaseModel, ConfigDict, Field +from typeguard import typechecked + +from runner.results.deliverables import Harvest, human_bytes +from runner.run_spec import CallbackTarget + +logger = logging.getLogger(__name__) + +TERMINAL_ATTEMPTS = 5 +TERMINAL_BACKOFF_SECONDS = 2.0 +REQUEST_TIMEOUT_SECONDS = 15.0 +# One file, one shot, generous: 20MB over a cold uplink is slower than any report. +UPLOAD_TIMEOUT_SECONDS = 120.0 +FILE_PATH_HEADER = "X-Openswarm-File-Path" +FILE_SHA256_HEADER = "X-Openswarm-File-Sha256" + + +class ReportedFile(BaseModel): + """One file the run produced, delivered or not, always named.""" + + model_config = ConfigDict(validate_assignment=True) + + path: str + size_bytes: int + delivered: bool + # Present only when delivered is False. Written for a human, because it is shown to one. + reason: Optional[str] = None + + +class RunReport(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + run_id: str + phase: Literal["started", "heartbeat", "finished"] + status: str + exit_code: Optional[int] = None + error: Optional[str] = None + cost_usd: float = 0.0 + active_step_idx: Optional[int] = None + last_tool_label: Optional[str] = None + answer: str = "" + transcript: str = "" + # The backend calls a run "success" even when the provider rejected the token; these are how the control plane sees that. + system_notices: List[str] = Field(default_factory=list) + # Every file the run made, including the ones that were too big to send. A deliverable that + # silently vanished is the failure this list exists to make impossible. + files: List[ReportedFile] = Field(default_factory=list) + + +@typechecked +def p_post_once(callback: CallbackTarget, report: RunReport) -> bool: + try: + with httpx.Client(timeout=REQUEST_TIMEOUT_SECONDS) as client: + response = client.post( + callback.url, + headers={"Authorization": f"Bearer {callback.token}"}, + json=report.model_dump(mode="json"), + ) + if response.status_code < 300: + return True + logger.warning("report %s rejected with HTTP %s", report.phase, response.status_code) + return False + except httpx.HTTPError as exc: + logger.warning("report %s failed to send: %s", report.phase, exc) + return False + + +@typechecked +def send_report(callback: Optional[CallbackTarget], report: RunReport) -> bool: + """Post a report. Terminal reports retry; heartbeats get one shot and are never retried.""" + if callback is None: + logger.info("no callback configured; %s report kept local: %s", report.phase, report.status) + return True + if report.phase != "finished": + return p_post_once(callback, report) + for attempt in range(TERMINAL_ATTEMPTS): + if p_post_once(callback, report): + return True + if attempt + 1 < TERMINAL_ATTEMPTS: + time.sleep(TERMINAL_BACKOFF_SECONDS * (attempt + 1)) + logger.error("terminal report for run %s never landed after %d attempts", report.run_id, TERMINAL_ATTEMPTS) + return False + + +@typechecked +def p_upload_one(client: httpx.Client, callback: CallbackTarget, workspace: str, relative: str, sha256: str) -> Optional[str]: + """Push one file. Returns None on success, or the sentence explaining the failure.""" + try: + with open(os.path.join(workspace, relative), "rb") as handle: + response = client.post( + str(callback.artifacts_url), + headers={ + "Authorization": f"Bearer {callback.token}", + "Content-Type": "application/octet-stream", + FILE_PATH_HEADER: quote(relative, safe="/"), + FILE_SHA256_HEADER: sha256, + }, + content=handle.read(), + ) + except OSError as exc: + return f"it could not be read back off disk ({exc.strerror or exc})" + except httpx.HTTPError as exc: + return f"the upload did not complete ({type(exc).__name__})" + if response.status_code < 300: + return None + # The control plane refuses with prose it wrote for the user; keep its words rather than ours. + detail = (response.text or "").strip() + try: + body = response.json() + if isinstance(body, dict) and isinstance(body.get("error"), str): + detail = body["error"] + except ValueError: + pass + return detail[:300] if detail else f"the storage service answered HTTP {response.status_code}" + + +@typechecked +def deliver_files(callback: Optional[CallbackTarget], workspace: str, harvest: Harvest) -> List[ReportedFile]: + """Hand every deliverable to the control plane and report honestly on each one. + + Never raises and never fails the run: a workflow whose answer is good and whose + attachment did not make it should still deliver the answer, with the miss stated. + """ + reported = [ + ReportedFile(path=item.path, size_bytes=item.size_bytes, delivered=False, reason=item.reason) + for item in harvest.refused + ] + if not harvest.files: + return reported + if callback is None or not callback.artifacts_url: + for item in harvest.files: + reported.append(ReportedFile( + path=item.path, + size_bytes=item.size_bytes, + delivered=False, + reason="this run had nowhere to send files, so it kept them on the machine", + )) + return reported + + with httpx.Client(timeout=UPLOAD_TIMEOUT_SECONDS) as client: + for item in harvest.files: + failure = p_upload_one(client, callback, workspace, item.path, item.sha256) + if failure is None: + logger.info("delivered %s (%s)", item.path, human_bytes(item.size_bytes)) + else: + logger.warning("could not deliver %s: %s", item.path, failure) + reported.append(ReportedFile( + path=item.path, + size_bytes=item.size_bytes, + delivered=failure is None, + reason=failure, + )) + return reported diff --git a/openswarm-runner/runner/run_spec.py b/openswarm-runner/runner/run_spec.py index 735b97c0..db6487e7 100644 --- a/openswarm-runner/runner/run_spec.py +++ b/openswarm-runner/runner/run_spec.py @@ -27,6 +27,11 @@ CLOUD_RUN_DASHBOARD_ID = "cloud-run" # Headroom the access token must still have on arrival. The control plane refreshes right before dispatch; anything thinner than this means its clock or its queue is broken, and we must not paper over that by refreshing ourselves. MIN_TOKEN_LIFETIME = timedelta(minutes=2) +# A skill is prose plus the odd small script. These bounds keep a run spec from becoming a file +# transfer, and are enforced here so an oversized one dies before a machine is billed for it. +MAX_SKILLS = 60 +MAX_SKILL_FILE_CHARS = 200_000 + class InvalidRunSpec(ValueError): """The control plane handed us something we refuse to run.""" @@ -82,6 +87,59 @@ class CallbackTarget(BaseModel): url: str = Field(min_length=1) token: str = Field(min_length=1) heartbeat_seconds: int = Field(default=30, ge=5, le=300) + # Where the run's files go. Named outright rather than derived from `url`, so a control plane + # that cannot accept files says so by leaving it out instead of being guessed at. + artifacts_url: Optional[str] = None + + +class SkillFile(BaseModel): + """One file inside a skill folder. Text only: a skill is prose plus small scripts.""" + + model_config = ConfigDict(validate_assignment=True, extra="forbid") + + path: str = Field(min_length=1, max_length=180) + text: str = Field(max_length=MAX_SKILL_FILE_CHARS) + + @model_validator(mode="after") + def p_reject_escaping_path(self) -> "SkillFile": + parts = self.path.split("/") + if self.path.startswith("/") or "\\" in self.path or any(p in ("", ".", "..") for p in parts): + raise ValueError(f"skill file path {self.path!r} is not a plain relative path") + return self + + +class SkillPayload(BaseModel): + """One of the user's skills, carried up so the agent has the same know-how it has at home. + + Skills are the user's own writing, not credentials. Nothing in here is a token, and the + control plane never learns anything from it that it could spend. + """ + + model_config = ConfigDict(validate_assignment=True, extra="forbid") + + # Also the folder name under ~/.claude/skills, so it has to survive being a directory. + id: str = Field(min_length=1, max_length=80, pattern=r"^[A-Za-z0-9][A-Za-z0-9_-]*$") + files: List[SkillFile] = Field(min_length=1) + + @model_validator(mode="after") + def p_require_skill_md(self) -> "SkillPayload": + if not any(file.path == "SKILL.md" for file in self.files): + raise ValueError(f"skill {self.id!r} has no SKILL.md, so nothing would ever load it") + return self + + +class McpServerNote(BaseModel): + """A server the user has connected at home, named so the agent can say it cannot reach it. + + Deliberately carries NO transport and NO secret. The user's MCP credentials (Slack session + cookies, Notion and GitHub access tokens, Google refresh tokens) stay on their laptop, so this + is a list of names and nothing else. Its whole job is to stop the agent quietly answering + "update my Notion" from general knowledge because it never knew Notion existed. + """ + + model_config = ConfigDict(validate_assignment=True, extra="forbid") + + name: str = Field(min_length=1, max_length=120) class RunSpec(BaseModel): @@ -91,6 +149,10 @@ class RunSpec(BaseModel): workflow: Workflow credentials: List[ProviderCredential] = Field(min_length=1) callback: Optional[CallbackTarget] = None + # The user's skills, shipped so a cloud run is as capable as the same workflow at home. + skills: List[SkillPayload] = Field(default_factory=list, max_length=MAX_SKILLS) + # Names only. See McpServerNote for why there is no config here. + unavailable_mcp_servers: List[McpServerNote] = Field(default_factory=list, max_length=100) # Hard wall-clock ceiling. Fly bills by machine-second, so an agent that wedges must cost a bounded amount. max_run_seconds: int = Field(default=1800, ge=60, le=7200) # Boot Electron under a virtual display so browser steps work. On by default: parity is the diff --git a/openswarm-runner/runner/seed/data_root.py b/openswarm-runner/runner/seed/data_root.py index 9d8e5979..7741ff7c 100644 --- a/openswarm-runner/runner/seed/data_root.py +++ b/openswarm-runner/runner/seed/data_root.py @@ -29,6 +29,17 @@ API_KEY_SETTINGS_FIELD: Dict[str, str] = { # One synthetic id for every cloud run. Without it each ephemeral container mints a fresh uuid and analytics sees a brand-new "install" per run. CLOUD_RUNNER_INSTALLATION_ID = "openswarm-cloud-runner" +# Told to the agent in as many words, because it cannot find this out any other way and the +# consequence of not knowing is a report written to a folder that is deleted minutes later. +DELIVERY_NOTE = ( + "You are running in the OpenSwarm cloud on a throwaway machine. Your working directory is " + "the ONLY place that survives: every file you save there is delivered back to the user, and " + "everything else on this machine is destroyed the moment this run ends. So when a task asks " + "for a document, spreadsheet, image or archive, write it to a plainly named file in your " + "working directory rather than only pasting it into your reply. Do not write deliverables to " + "/tmp or to your home directory; they will not come back." +) + @typechecked def p_write_json(path: str, payload: Any) -> None: @@ -48,13 +59,43 @@ def p_write_json(path: str, payload: Any) -> None: @typechecked -def settings_for_run(spec: RunSpec) -> AppSettings: - """The AppSettings a cloud run needs: this workflow's model, this run's keys, no telemetry.""" +def unavailable_apps_note(spec: RunSpec) -> str: + """Name the user's connected apps this run cannot reach, so silence is not mistaken for absence. + + Their MCP credentials never leave the laptop, so the servers are not here and never will be + mid-run. Without this sentence the agent has no way to know the app exists, and "update my + Notion" comes back as a confident paragraph about Notion rather than an admission. + """ + names = [server.name for server in spec.unavailable_mcp_servers] + if not names: + return "" + return ( + "These apps are connected on the user's own computer but NOT reachable from this cloud " + f"run, because their sign-in details stay on that computer: {', '.join(sorted(names))}. " + "If a task needs one of them, say plainly that it cannot be done from a cloud run and " + "that it has to run on their machine. Never guess at, invent, or describe from memory " + "what one of those apps contains." + ) + + +@typechecked +def settings_for_run(spec: RunSpec, workspace: str) -> AppSettings: + """The AppSettings a cloud run needs: this workflow's model, this run's keys, no telemetry. + + `default_folder` is what makes the run's files findable afterwards. Left unset, the agent + falls back to $HOME and the launcher reroutes it into a per-session scratch directory whose + name nothing outside the backend can predict, so the harvest would have nowhere to look. + """ settings = AppSettings() settings.default_model = spec.workflow.model settings.connection_mode = "own_key" settings.analytics_opt_in = False settings.installation_id = CLOUD_RUNNER_INSTALLATION_ID + settings.default_folder = workspace + additions = [DELIVERY_NOTE, unavailable_apps_note(spec)] + settings.default_system_prompt = "\n\n".join( + part for part in [settings.default_system_prompt or "", *additions] if part + ).strip() for credential in spec.credentials: if credential.auth_type != "api_key": continue @@ -69,13 +110,17 @@ def settings_for_run(spec: RunSpec) -> AppSettings: @typechecked -def seed_data_root(data_root: str, spec: RunSpec) -> None: +def seed_data_root(data_root: str, workspace: str, spec: RunSpec) -> None: """Write the workflow, settings and dashboard records the backend will read at boot. The dashboard exists so the Electron window has somewhere to land and browser cards have somewhere to render. Writing it here rather than letting the backend's first-boot migration invent one keeps its id knowable before anything has started. + + The workspace sits OUTSIDE the data root deliberately: it is the agent's own folder, and a + Glob or Grep run inside it should not sweep up the settings file its API keys live in. """ + os.makedirs(workspace, mode=0o700, exist_ok=True) workflow = spec.workflow_for_disk() p_write_json( os.path.join(data_root, "workflows", f"{workflow.id}.json"), @@ -83,7 +128,7 @@ def seed_data_root(data_root: str, spec: RunSpec) -> None: ) p_write_json( os.path.join(data_root, "settings", "settings.json"), - settings_for_run(spec).model_dump(mode="json"), + settings_for_run(spec, workspace).model_dump(mode="json"), ) dashboard = Dashboard(id=CLOUD_RUN_DASHBOARD_ID, name=spec.workflow.title or "Cloud run") p_write_json( diff --git a/openswarm-runner/tests/test_deliverables.py b/openswarm-runner/tests/test_deliverables.py new file mode 100644 index 00000000..becaf220 --- /dev/null +++ b/openswarm-runner/tests/test_deliverables.py @@ -0,0 +1,174 @@ +"""What the run made, what comes home, and what is refused out loud instead of silently. + +The caps are the point. A truncated file is worse than a refused one, and a file that +vanishes with no sentence attached is the failure the whole list exists to prevent. +""" + +import os + +import pytest + +from runner.results.deliverables import ( + MAX_FILE_BYTES, + MAX_FILES, + MAX_TOTAL_BYTES, + collect, + human_bytes, +) +from runner.results.report import deliver_files +from runner.run_spec import CallbackTarget + + +def write(root, relative: str, payload: bytes) -> str: + path = os.path.join(str(root), relative) + os.makedirs(os.path.dirname(path), exist_ok=True) + with open(path, "wb") as handle: + handle.write(payload) + return path + + +def test_a_missing_workspace_is_an_empty_harvest_not_a_crash(tmp_path) -> None: + assert collect(str(tmp_path / "never-made")).files == [] + + +def test_ordinary_files_are_collected_with_their_digest(tmp_path) -> None: + write(tmp_path, "report.md", b"# Digest\n") + write(tmp_path, "data/rows.csv", b"a,b\n1,2\n") + + # Walk order, and it is fixed: this directory's own files first, then subdirectories in name + # order, so a run's file list does not shuffle between two identical runs. + harvest = collect(str(tmp_path)) + assert [f.path for f in harvest.files] == ["report.md", "data/rows.csv"] + assert harvest.files[0].size_bytes == len(b"# Digest\n") + # A digest travels with every file, so a corrupted upload is detectable rather than assumed fine. + assert len(harvest.files[0].sha256) == 64 + assert harvest.refused == [] + + +def test_machinery_is_not_a_deliverable(tmp_path) -> None: + write(tmp_path, "report.md", b"keep me") + write(tmp_path, ".git/config", b"[core]") + write(tmp_path, "node_modules/left-pad/index.js", b"module.exports=1") + write(tmp_path, "__pycache__/x.pyc", b"\x00") + write(tmp_path, ".claude/worktrees/probe/README", b"scratch") + + assert [f.path for f in collect(str(tmp_path)).files] == ["report.md"] + + +def test_an_empty_file_is_neither_delivered_nor_complained_about(tmp_path) -> None: + write(tmp_path, "touched.txt", b"") + harvest = collect(str(tmp_path)) + assert harvest.files == [] + assert harvest.refused == [] + + +def test_a_file_over_the_cap_is_refused_whole_and_says_why(tmp_path) -> None: + write(tmp_path, "render.mp4", b"x" * (MAX_FILE_BYTES + 1)) + write(tmp_path, "notes.md", b"still fine") + + harvest = collect(str(tmp_path)) + assert [f.path for f in harvest.files] == ["notes.md"] + assert len(harvest.refused) == 1 + assert harvest.refused[0].path == "render.mp4" + assert "cannot exceed" in harvest.refused[0].reason + # Never a fragment: an over-sized file is absent, not shortened. + assert all(f.path != "render.mp4" for f in harvest.files) + + +def test_the_run_total_stops_collecting_but_keeps_what_already_fit(tmp_path) -> None: + chunk = b"x" * MAX_FILE_BYTES + for index in range(MAX_TOTAL_BYTES // MAX_FILE_BYTES + 1): + write(tmp_path, f"blob-{index}.bin", chunk) + + harvest = collect(str(tmp_path)) + assert harvest.total_bytes() <= MAX_TOTAL_BYTES + assert len(harvest.files) >= 1 + assert harvest.refused, "the file that blew the budget must be named, not dropped" + assert "limit is" in harvest.refused[0].reason + + +def test_too_many_files_refuses_the_extras_by_name(tmp_path) -> None: + for index in range(MAX_FILES + 3): + write(tmp_path, f"note-{index:03d}.txt", b"hi") + + harvest = collect(str(tmp_path)) + assert len(harvest.files) == MAX_FILES + assert len(harvest.refused) == 3 + assert all("maximum of" in item.reason for item in harvest.refused) + + +def test_a_symlink_out_of_the_workspace_is_never_followed(tmp_path) -> None: + secret = tmp_path / "outside" / "id_rsa" + os.makedirs(secret.parent, exist_ok=True) + secret.write_text("PRIVATE KEY") + workspace = tmp_path / "ws" + os.makedirs(workspace, exist_ok=True) + os.symlink(str(secret), str(workspace / "borrowed.pem")) + + assert collect(str(workspace)).files == [] + + +def test_with_nowhere_to_send_files_the_run_says_so_per_file(tmp_path) -> None: + write(tmp_path, "report.md", b"the answer") + reported = deliver_files(None, str(tmp_path), collect(str(tmp_path))) + + assert len(reported) == 1 + assert reported[0].delivered is False + assert "nowhere to send files" in (reported[0].reason or "") + + +def test_a_control_plane_with_no_file_route_is_reported_not_guessed(tmp_path) -> None: + write(tmp_path, "report.md", b"the answer") + callback = CallbackTarget(url="https://cloud.test/report", token="two-party") + assert callback.artifacts_url is None + + reported = deliver_files(callback, str(tmp_path), collect(str(tmp_path))) + assert reported[0].delivered is False + + +def test_refusals_reach_the_report_even_when_nothing_was_delivered(tmp_path) -> None: + write(tmp_path, "render.mp4", b"x" * (MAX_FILE_BYTES + 1)) + reported = deliver_files(None, str(tmp_path), collect(str(tmp_path))) + + assert len(reported) == 1 + assert reported[0].path == "render.mp4" + assert reported[0].delivered is False + assert "cannot exceed" in (reported[0].reason or "") + + +@pytest.mark.parametrize( + "count,expected", + [(512, "512 B"), (2048, "2 KB"), (5 * 1024 * 1024, "5.0 MB"), (3 * 1024**3, "3.0 GB")], +) +def test_sizes_are_written_the_way_a_person_reads_them(count: int, expected: str) -> None: + assert human_bytes(count) == expected + + +def test_a_failed_run_still_hands_over_what_it_managed_to_make(tmp_path, monkeypatch) -> None: + """A workflow that dies on step 3 may have written a perfectly good report on step 1.""" + from runner import main + from runner.run_spec import RunSpec + + write(tmp_path, "partial-report.md", b"# What I got through\n") + spec = RunSpec.model_validate({ + "run_id": "cr-fail", + "workflow": {"id": "wf-1", "steps": [{"id": "s1", "text": "go"}]}, + "credentials": [{"provider": "anthropic", "auth_type": "api_key", "api_key": "sk-test"}], + }) + + sent: list = [] + monkeypatch.setattr(main, "send_report", lambda callback, report: sent.append(report) or True) + code = main.p_fail(spec, "failure", "step 3 blew up", main.EXIT_WORKFLOW_FAILED, str(tmp_path)) + + assert code == main.EXIT_WORKFLOW_FAILED + assert [f.path for f in sent[0].files] == ["partial-report.md"] + + +def test_a_failure_with_no_workspace_reports_no_files_rather_than_guessing(monkeypatch) -> None: + from runner import main + + sent: list = [] + monkeypatch.setattr(main, "send_report", lambda callback, report: sent.append(report) or True) + main.p_fail(None, "failure", "bad spec", main.EXIT_BAD_SPEC) + + assert sent[0].files == [] diff --git a/openswarm-runner/tests/test_run_spec.py b/openswarm-runner/tests/test_run_spec.py index 0fba9641..1733f5a5 100644 --- a/openswarm-runner/tests/test_run_spec.py +++ b/openswarm-runner/tests/test_run_spec.py @@ -63,10 +63,11 @@ def test_the_container_never_inherits_the_schedule() -> None: def test_seeding_writes_the_workflow_and_owner_only_settings(tmp_path) -> None: spec = RunSpec.model_validate(spec_body()) - seed_data_root(str(tmp_path), spec) + workspace = str(tmp_path / "workspace") + seed_data_root(str(tmp_path / "data"), workspace, spec) - workflow_path = tmp_path / "workflows" / "wf-1.json" - settings_path = tmp_path / "settings" / "settings.json" + workflow_path = tmp_path / "data" / "workflows" / "wf-1.json" + settings_path = tmp_path / "data" / "settings" / "settings.json" assert json.loads(workflow_path.read_text())["schedule"]["enabled"] is False assert stat.S_IMODE(os.stat(settings_path).st_mode) == 0o600 @@ -74,11 +75,20 @@ def test_seeding_writes_the_workflow_and_owner_only_settings(tmp_path) -> None: assert settings["anthropic_api_key"] == "sk-test-not-real" assert settings["default_model"] == "opus-5" assert settings["analytics_opt_in"] is False + # The agent's folder is the deliverable folder, and it exists before the backend boots. + assert settings["default_folder"] == workspace + assert os.path.isdir(workspace) -def test_an_unmappable_api_key_provider_fails_loudly() -> None: +def test_the_agent_is_told_its_files_only_survive_from_the_workspace(tmp_path) -> None: + spec = RunSpec.model_validate(spec_body()) + prompt = settings_for_run(spec, str(tmp_path)).default_system_prompt or "" + assert "delivered back to the user" in prompt + + +def test_an_unmappable_api_key_provider_fails_loudly(tmp_path) -> None: spec = RunSpec.model_validate(spec_body( credentials=[{"provider": "wat", "auth_type": "api_key", "api_key": "x"}] )) with pytest.raises(ValueError, match="no settings field"): - settings_for_run(spec) + settings_for_run(spec, str(tmp_path)) From 6ab95132406f0ac0db6a0140cdaf4c80e9ee0a0e Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sat, 1 Aug 2026 01:20:10 -0700 Subject: [PATCH 2/5] [eric] backend: a finished cloud run's files land in Downloads, where a user would actually look --- backend/apps/workflows/cloud/client.py | 74 ++++++++++- backend/apps/workflows/cloud/routes.py | 19 ++- backend/apps/workflows/cloud/run_files.py | 143 ++++++++++++++++++++ backend/tests/test_cloud_run_files.py | 152 ++++++++++++++++++++++ 4 files changed, 383 insertions(+), 5 deletions(-) create mode 100644 backend/apps/workflows/cloud/run_files.py create mode 100644 backend/tests/test_cloud_run_files.py diff --git a/backend/apps/workflows/cloud/client.py b/backend/apps/workflows/cloud/client.py index 57cc82ab..a1ffabc4 100644 --- a/backend/apps/workflows/cloud/client.py +++ b/backend/apps/workflows/cloud/client.py @@ -20,6 +20,8 @@ from backend.apps.workflows.cloud.schedule import CloudSchedule # The cloud router is mounted at /api/workflows and a trailing slash 404s there, so the collection paths are the empty string, not "/". COLLECTION = "" TIMEOUT_SECONDS = 8.0 +# Files are up to 20MB each, so they get their own budget rather than the chatty-call one. +DOWNLOAD_TIMEOUT_SECONDS = 120.0 class CloudRefused(Exception): @@ -85,6 +87,18 @@ class CloudPreflight(BaseModel): hosted: Optional[HostedWorkflow] = None +class CloudRunFile(BaseModel): + """One file a cloud run produced. `refusal` set means it exists nowhere and says why.""" + + model_config = ConfigDict(validate_assignment=True) + + id: str + path: str + size_bytes: int = 0 + sha256: Optional[str] = None + refusal: Optional[str] = None + + class CloudRun(BaseModel): model_config = ConfigDict(validate_assignment=True) @@ -96,6 +110,7 @@ class CloudRun(BaseModel): answer: Optional[str] = None notices: List[str] = Field(default_factory=list) cost_usd: Optional[float] = None + files: List[CloudRunFile] = Field(default_factory=list) @typechecked @@ -229,11 +244,22 @@ async def p_preflight_from_list(hosted_id: Optional[str]) -> CloudPreflight: @typechecked async def put_workflow( - *, hosted_id: Optional[str], name: str, definition: Dict[str, Any], schedule: CloudSchedule + *, + hosted_id: Optional[str], + name: str, + definition: Dict[str, Any], + schedule: CloudSchedule, + context: Optional[Dict[str, Any]] = None, ) -> HostedWorkflow: """Create the hosted copy, or re-push onto the existing row so an edited - workflow stops running last week's prose.""" + workflow stops running last week's prose. + + `context` carries the user's skills and the names of the apps a cloud run cannot reach. An + older control plane ignores the extra keys, which costs a run its skills but never its run. + """ body = {"name": name, "definition": definition, "schedule": schedule.model_dump()} + if context: + body.update(context) if hosted_id: try: raw = await p_call("POST", f"/{hosted_id}/update", body) @@ -270,6 +296,26 @@ async def delete_hosted(hosted_id: str) -> None: raise +@typechecked +def p_files(raw: Any) -> List[CloudRunFile]: + """A control plane with no file support answers without the key, which is an empty list, not an error.""" + if not isinstance(raw, list): + return [] + out: List[CloudRunFile] = [] + for row in raw: + if not isinstance(row, dict) or not isinstance(row.get("id"), str) or not isinstance(row.get("path"), str): + continue + size = row.get("size_bytes") + out.append(CloudRunFile( + id=row["id"], + path=row["path"], + size_bytes=size if isinstance(size, int) else 0, + sha256=row.get("sha256") if isinstance(row.get("sha256"), str) else None, + refusal=row.get("refusal") if isinstance(row.get("refusal"), str) else None, + )) + return out + + @typechecked async def list_runs(hosted_id: str) -> List[CloudRun]: raw = await p_call("GET", f"/{hosted_id}/runs") @@ -291,6 +337,30 @@ async def list_runs(hosted_id: str) -> List[CloudRun]: answer=row.get("answer") if isinstance(row.get("answer"), str) else None, notices=[n for n in notices if isinstance(n, str)] if isinstance(notices, list) else [], cost_usd=row.get("cost_usd") if isinstance(row.get("cost_usd"), (int, float)) else None, + files=p_files(row.get("files")), ) ) return out + + +@typechecked +async def download_run_file(hosted_id: str, run_id: str, file_id: str) -> bytes: + """The file's bytes. Its own call rather than p_call because this answer is not JSON.""" + from backend.apps.settings.store import load_settings + + token, base = account_auth(load_settings()) + if not token: + raise SignedOut() + url = f"{base}/api/workflows/{hosted_id}/runs/{run_id}/files/{file_id}" + try: + async with httpx.AsyncClient(timeout=DOWNLOAD_TIMEOUT_SECONDS) as client: + resp = await client.get(url, headers={"Authorization": f"Bearer {token}"}) + except httpx.HTTPError as exc: + raise CloudUnreachable(f"{type(exc).__name__}") from exc + if resp.status_code == 401: + raise SignedOut() + if resp.status_code >= 500: + raise CloudUnreachable(f"the cloud returned {resp.status_code}") + if resp.status_code >= 400: + raise CloudRefused(p_message(resp, "That file could not be downloaded."), resp.status_code) + return resp.content diff --git a/backend/apps/workflows/cloud/routes.py b/backend/apps/workflows/cloud/routes.py index 8671a6c5..2d664671 100644 --- a/backend/apps/workflows/cloud/routes.py +++ b/backend/apps/workflows/cloud/routes.py @@ -7,6 +7,7 @@ cloud re-decides that at create and again at dispatch. """ from __future__ import annotations +import asyncio from contextlib import asynccontextmanager from datetime import datetime from typing import List, Literal, Optional, Union @@ -18,6 +19,7 @@ from typeguard import typechecked from backend.apps.workflows import storage from backend.apps.workflows.cloud import client as cloud from backend.apps.workflows.cloud.handover import TargetOutcome, hand_to_cloud, take_back +from backend.apps.workflows.cloud.run_files import LocalRunFile, described, downloads_root, fetch_missing from backend.apps.workflows.cloud.status import CloudStatus, compute_status, epoch_to_datetime from backend.apps.workflows.models import Workflow from backend.config.Apps import SubApp @@ -50,6 +52,7 @@ class CloudRunRow(BaseModel): answer: Optional[str] = None notices: List[str] = [] cost_usd: Optional[float] = None + files: List[LocalRunFile] = [] class CloudRunsReady(BaseModel): @@ -57,6 +60,9 @@ class CloudRunsReady(BaseModel): state: Literal["ready"] = "ready" runs: List[CloudRunRow] = [] + # Where this machine puts a cloud run's files, so the UI can say it out loud even when a run + # has none yet. A folder the user is told about is a folder they can find later. + files_folder: str = "" class CloudRunsUnavailable(BaseModel): @@ -95,7 +101,7 @@ async def set_workflow_target(workflow_id: str, body: TargetRequest) -> TargetOu async def workflow_cloud_runs(workflow_id: str) -> CloudRunsResponse: wf = p_workflow(workflow_id) if not wf.cloud_workflow_id: - return CloudRunsReady(runs=[]) + return CloudRunsReady(runs=[], files_folder=downloads_root()) try: runs = await cloud.list_runs(wf.cloud_workflow_id) except cloud.SignedOut: @@ -103,11 +109,17 @@ async def workflow_cloud_runs(workflow_id: str) -> CloudRunsResponse: except cloud.CloudRefused as exc: # A 404 here is the hosted copy being gone, which is an empty history, not a broken one. if exc.status == 404: - return CloudRunsReady(runs=[]) + return CloudRunsReady(runs=[], files_folder=downloads_root()) return CloudRunsUnavailable(state="unknown", detail=exc.message) except cloud.CloudUnreachable as exc: return CloudRunsUnavailable(state="unknown", detail=exc.detail) + + # Answered now, files fetched behind it. A 20MB attachment must not hold up the history the + # user asked for, and a run's files appear a moment later without them doing anything. + asyncio.create_task(fetch_missing(wf.cloud_workflow_id, wf, runs)) + return CloudRunsReady( + files_folder=downloads_root(), runs=[ CloudRunRow( id=r.id, @@ -118,7 +130,8 @@ async def workflow_cloud_runs(workflow_id: str) -> CloudRunsResponse: answer=r.answer, notices=r.notices, cost_usd=r.cost_usd, + files=described(wf, r), ) for r in runs - ] + ], ) diff --git a/backend/apps/workflows/cloud/run_files.py b/backend/apps/workflows/cloud/run_files.py new file mode 100644 index 00000000..2695c805 --- /dev/null +++ b/backend/apps/workflows/cloud/run_files.py @@ -0,0 +1,143 @@ +"""Bring a cloud run's files down to the machine the user actually sits at. + +A cloud run finishes while the laptop is shut, so the files live in the cloud until +something fetches them. That something is this module, and it puts them in Downloads +rather than anywhere clever: the whole point of a deliverable is that the user can +find it without being told where to look, and "it is in Downloads" is the one answer +nobody needs explained. + +Every file is fetched at most once. The local path is derived, never stored, so a +user who moves or deletes a file just gets it back on the next look rather than +staring at a broken link. +""" +from __future__ import annotations + +import asyncio +import logging +import os +import re +from typing import List, Optional + +from pydantic import BaseModel, ConfigDict +from typeguard import typechecked + +from backend.apps.workflows.cloud import client as cloud +from backend.apps.workflows.models import Workflow + +logger = logging.getLogger(__name__) + +# Bounded so one History open cannot spend minutes pulling a whole month of runs. +MAX_CONCURRENT_DOWNLOADS = 3 + + +class LocalRunFile(BaseModel): + """A cloud file plus where it is (or would be) on this machine.""" + + model_config = ConfigDict(validate_assignment=True) + + id: str + path: str + size_bytes: int + refusal: Optional[str] = None + # Set once the bytes are on disk. None means "not here yet", never "not coming". + local_path: Optional[str] = None + + +@typechecked +def safe_component(raw: str) -> str: + """One path segment a filesystem will take, with the user's own name still legible.""" + cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "-", raw).strip(" .") + return cleaned[:60] or "untitled" + + +@typechecked +def downloads_root() -> str: + home = os.path.expanduser("~") + downloads = os.path.join(home, "Downloads") + return os.path.join(downloads if os.path.isdir(downloads) else home, "OpenSwarm") + + +@typechecked +def run_folder(workflow: Workflow, run: cloud.CloudRun) -> str: + """Where this run's files go: one folder per run, named so a list of them reads as a history.""" + stamp = run.finished_at or run.started_at + when = "unknown-date" + if stamp: + from datetime import datetime, timezone + + when = datetime.fromtimestamp(stamp / 1000, tz=timezone.utc).astimezone().strftime("%Y-%m-%d %H%M") + return os.path.join( + downloads_root(), + safe_component(workflow.title or "Workflow"), + safe_component(f"{when} {run.id[:8]}"), + ) + + +@typechecked +def local_path_for(folder: str, file: cloud.CloudRunFile) -> str: + """The file's home, with every segment of the run-relative path made filesystem-safe.""" + parts = [safe_component(part) for part in file.path.split("/") if part not in ("", ".", "..")] + return os.path.join(folder, *(parts or ["file"])) + + +@typechecked +def p_already_here(path: str, size_bytes: int) -> bool: + try: + return os.path.getsize(path) == size_bytes + except OSError: + return False + + +@typechecked +def described(workflow: Workflow, run: cloud.CloudRun) -> List[LocalRunFile]: + """This run's files and where they are right now. Reads disk, never fetches.""" + folder = run_folder(workflow, run) + out: List[LocalRunFile] = [] + for file in run.files: + local = local_path_for(folder, file) + out.append(LocalRunFile( + id=file.id, + path=file.path, + size_bytes=file.size_bytes, + refusal=file.refusal, + local_path=local if file.refusal is None and p_already_here(local, file.size_bytes) else None, + )) + return out + + +@typechecked +async def p_fetch_one(hosted_id: str, run: cloud.CloudRun, file: cloud.CloudRunFile, target: str) -> None: + payload = await cloud.download_run_file(hosted_id, run.id, file.id) + os.makedirs(os.path.dirname(target), exist_ok=True) + temporary = f"{target}.part" + with open(temporary, "wb") as handle: + handle.write(payload) + os.replace(temporary, target) + logger.info("cloud run %s: saved %s to %s", run.id, file.path, target) + + +@typechecked +async def fetch_missing(hosted_id: str, workflow: Workflow, runs: List[cloud.CloudRun]) -> None: + """Pull down anything not already here. Never raises: a run's answer must render + even when its attachments cannot be fetched, and the next look tries again.""" + semaphore = asyncio.Semaphore(MAX_CONCURRENT_DOWNLOADS) + + async def p_guarded(run: cloud.CloudRun, file: cloud.CloudRunFile, target: str) -> None: + async with semaphore: + try: + await p_fetch_one(hosted_id, run, file, target) + except (cloud.SignedOut, cloud.CloudRefused, cloud.CloudUnreachable, OSError) as exc: + logger.info("cloud run %s: %s not fetched (%s)", run.id, file.path, exc) + + pending = [] + for run in runs: + folder = run_folder(workflow, run) + for file in run.files: + if file.refusal is not None: + continue + target = local_path_for(folder, file) + if p_already_here(target, file.size_bytes): + continue + pending.append(p_guarded(run, file, target)) + if pending: + await asyncio.gather(*pending) diff --git a/backend/tests/test_cloud_run_files.py b/backend/tests/test_cloud_run_files.py new file mode 100644 index 00000000..0af62106 --- /dev/null +++ b/backend/tests/test_cloud_run_files.py @@ -0,0 +1,152 @@ +"""Where a cloud run's files land on the machine the user sits at, and what they are told. + +A deliverable the user cannot find has not been delivered, so the two things worth pinning +are the folder (Downloads, because that is where people look) and the honesty of the list: +a file that was refused must still appear, with its reason, and must never be fetched. +""" + +import os + +import pytest + +from backend.apps.workflows.cloud.client import CloudRun, CloudRunFile +from backend.apps.workflows.cloud.run_files import ( + described, + downloads_root, + fetch_missing, + local_path_for, + safe_component, + run_folder, +) +from backend.apps.workflows.models import Workflow + + +def p_run(**overrides) -> CloudRun: + body = { + "id": "run-abcdef123456", + "status": "succeeded", + "finished_at": 1_785_000_000_000, + "files": [], + } + body.update(overrides) + return CloudRun.model_validate(body) + + +def test_files_land_in_downloads_because_that_is_where_people_look() -> None: + root = downloads_root() + assert root.endswith(os.path.join("OpenSwarm")) + assert os.path.expanduser("~") in root + + +def test_a_run_gets_its_own_folder_named_so_a_list_of_them_reads_as_history() -> None: + workflow = Workflow(title="Weekly numbers") + folder = run_folder(workflow, p_run()) + assert "Weekly numbers" in folder + # The run id's head disambiguates two runs in the same minute without being a wall of hex. + assert "run-abcd" in folder + + +@pytest.mark.parametrize( + "raw,expected", + [ + ("Weekly numbers", "Weekly numbers"), + ("bad/name", "bad-name"), + ("../escape", "-escape"), + ("with:colon", "with-colon"), + ("", "untitled"), + ("...", "untitled"), + ], +) +def test_a_title_is_made_safe_without_becoming_unrecognisable(raw: str, expected: str) -> None: + assert safe_component(raw) == expected + + +def test_a_nested_run_path_stays_nested_locally() -> None: + target = local_path_for("/tmp/folder", CloudRunFile(id="f1", path="notes/method.txt", size_bytes=5)) + assert target == os.path.join("/tmp/folder", "notes", "method.txt") + + +def test_a_path_that_tries_to_climb_out_cannot(tmp_path) -> None: + target = local_path_for(str(tmp_path), CloudRunFile(id="f1", path="../../etc/passwd", size_bytes=1)) + assert os.path.abspath(target).startswith(str(tmp_path)) + + +def test_a_file_not_yet_downloaded_reports_no_local_path_rather_than_a_broken_one(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("backend.apps.workflows.cloud.run_files.downloads_root", lambda: str(tmp_path)) + run = p_run(files=[{"id": "f1", "path": "summary.md", "size_bytes": 42}]) + described_files = described(Workflow(title="W"), run) + + assert len(described_files) == 1 + assert described_files[0].local_path is None + + +def test_a_file_already_on_disk_is_reported_at_its_real_path(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("backend.apps.workflows.cloud.run_files.downloads_root", lambda: str(tmp_path)) + workflow = Workflow(title="W") + run = p_run(files=[{"id": "f1", "path": "summary.md", "size_bytes": 5}]) + target = local_path_for(run_folder(workflow, run), run.files[0]) + os.makedirs(os.path.dirname(target), exist_ok=True) + with open(target, "wb") as handle: + handle.write(b"hello") + + assert described(workflow, run)[0].local_path == target + + +def test_a_refused_file_is_listed_with_its_reason_and_never_fetched(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("backend.apps.workflows.cloud.run_files.downloads_root", lambda: str(tmp_path)) + workflow = Workflow(title="W") + run = p_run(files=[ + {"id": "f1", "path": "render.mp4", "size_bytes": 999, "refusal": "it is 512.0 MB and the limit is 20.0 MB"}, + ]) + + listed = described(workflow, run) + assert listed[0].refusal is not None + assert listed[0].local_path is None + + asked: list = [] + + async def p_never(*args, **kwargs): + asked.append(args) + return b"" + + monkeypatch.setattr("backend.apps.workflows.cloud.run_files.cloud.download_run_file", p_never) + import asyncio + + asyncio.run(fetch_missing("hosted-1", workflow, [run])) + assert asked == [], "a file that exists nowhere must never be requested" + + +def test_a_download_that_fails_leaves_the_history_readable(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("backend.apps.workflows.cloud.run_files.downloads_root", lambda: str(tmp_path)) + workflow = Workflow(title="W") + run = p_run(files=[{"id": "f1", "path": "summary.md", "size_bytes": 5}]) + + async def p_boom(*args, **kwargs): + from backend.apps.workflows.cloud.client import CloudUnreachable + + raise CloudUnreachable("offline") + + monkeypatch.setattr("backend.apps.workflows.cloud.run_files.cloud.download_run_file", p_boom) + import asyncio + + asyncio.run(fetch_missing("hosted-1", workflow, [run])) + assert described(workflow, run)[0].local_path is None + + +def test_a_downloaded_file_is_written_whole_or_not_at_all(tmp_path, monkeypatch) -> None: + monkeypatch.setattr("backend.apps.workflows.cloud.run_files.downloads_root", lambda: str(tmp_path)) + workflow = Workflow(title="W") + run = p_run(files=[{"id": "f1", "path": "notes/summary.md", "size_bytes": 5}]) + + async def p_ok(*args, **kwargs): + return b"hello" + + monkeypatch.setattr("backend.apps.workflows.cloud.run_files.cloud.download_run_file", p_ok) + import asyncio + + asyncio.run(fetch_missing("hosted-1", workflow, [run])) + target = local_path_for(run_folder(workflow, run), run.files[0]) + with open(target, "rb") as handle: + assert handle.read() == b"hello" + # The .part file is the whole point of the atomic rename: no half file is ever visible. + assert not os.path.exists(f"{target}.part") From f40179360451805139e096662922191bb245a3d0 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sat, 1 Aug 2026 01:20:11 -0700 Subject: [PATCH 3/5] [eric] cloud runs: carry the user's own skills up, and name the apps the container cannot reach --- backend/apps/skills/skills.py | 8 +- backend/apps/workflows/cloud/definition.py | 15 +- backend/apps/workflows/cloud/handover.py | 5 +- .../apps/workflows/cloud/portable_context.py | 157 ++++++++++++++++++ backend/apps/workflows/cloud/status.py | 5 +- openswarm-runner/runner/seed/skills.py | 52 ++++++ openswarm-runner/tests/test_seed_skills.py | 83 +++++++++ 7 files changed, 314 insertions(+), 11 deletions(-) create mode 100644 backend/apps/workflows/cloud/portable_context.py create mode 100644 openswarm-runner/runner/seed/skills.py create mode 100644 openswarm-runner/tests/test_seed_skills.py diff --git a/backend/apps/skills/skills.py b/backend/apps/skills/skills.py index 85678095..1f0d6ff6 100644 --- a/backend/apps/skills/skills.py +++ b/backend/apps/skills/skills.py @@ -395,12 +395,12 @@ def clear_skill_dir(skill_id: str) -> None: """Empty a skill's folder before an in-place update so files removed upstream don't linger as orphans. write_folder_skill recreates the dir right after.""" import shutil - d = os.path.join(SKILLS_DIR, p_safe_slug(skill_id)) + d = os.path.join(SKILLS_DIR, safe_slug(skill_id)) if os.path.isdir(d): shutil.rmtree(d, ignore_errors=True) -def p_safe_slug(raw: str) -> str: +def safe_slug(raw: str) -> str: slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", (raw or "").strip().lower()).strip("-") return slug or "skill" @@ -417,7 +417,7 @@ def unique_skill_slug(base: str) -> str: """A free slug for `base`, suffixing -2, -3, ... on collision. Lets a registry install land beside a same-named skill instead of silently overwriting the user's existing one.""" - slug = p_safe_slug(base) + slug = safe_slug(base) if not p_skill_exists(slug): return slug i = 2 @@ -432,7 +432,7 @@ def write_folder_skill(skill_id: str, files: dict[str, str], meta: dict) -> Skil zip/.swarm import. Relpaths that try to escape the skill folder (../, abs paths) are dropped, an untrusted registry archive can't write outside its own dir.""" - slug = p_safe_slug(skill_id) + slug = safe_slug(skill_id) base = os.path.join(SKILLS_DIR, slug) base_abs = os.path.abspath(base) # A folder write supersedes any legacy flat .md, so we never leave a phantom flat file shadowed by the folder (folder wins in skill_md_path). diff --git a/backend/apps/workflows/cloud/definition.py b/backend/apps/workflows/cloud/definition.py index 6f20970c..7101bab8 100644 --- a/backend/apps/workflows/cloud/definition.py +++ b/backend/apps/workflows/cloud/definition.py @@ -9,7 +9,7 @@ from __future__ import annotations import hashlib import json -from typing import Any, Dict +from typing import Any, Dict, Optional from typeguard import typechecked @@ -54,8 +54,13 @@ def cloud_definition(wf: Workflow) -> Dict[str, Any]: @typechecked -def definition_signature(definition: Dict[str, Any], schedule: Dict[str, Any]) -> str: - """Fingerprint of exactly what we last handed the cloud, schedule included, - so "your edits are not up there yet" is a fact rather than a guess.""" - payload = json.dumps({"definition": definition, "schedule": schedule}, sort_keys=True) +def definition_signature( + definition: Dict[str, Any], schedule: Dict[str, Any], context: Optional[Dict[str, Any]] = None +) -> str: + """Fingerprint of exactly what we last handed the cloud, schedule and skills included, + so "your edits are not up there yet" is a fact rather than a guess. Editing a skill a + workflow leans on is an edit to that workflow's behaviour, so it belongs in here.""" + payload = json.dumps( + {"definition": definition, "schedule": schedule, "context": context or {}}, sort_keys=True + ) return hashlib.sha256(payload.encode("utf-8")).hexdigest() diff --git a/backend/apps/workflows/cloud/handover.py b/backend/apps/workflows/cloud/handover.py index ef41fc4b..e4805abf 100644 --- a/backend/apps/workflows/cloud/handover.py +++ b/backend/apps/workflows/cloud/handover.py @@ -18,6 +18,7 @@ from typeguard import typechecked from backend.apps.workflows import scheduler, storage from backend.apps.workflows.cloud import client as cloud from backend.apps.workflows.cloud.definition import cloud_definition, definition_signature +from backend.apps.workflows.cloud.portable_context import portable_context from backend.apps.workflows.cloud.schedule import ScheduleSupported, to_cloud_schedule from backend.apps.workflows.cloud.status import epoch_to_datetime from backend.apps.workflows.models import Workflow @@ -51,12 +52,14 @@ async def hand_to_cloud(wf: Workflow, enabled: bool) -> TargetOutcome: if enabled and not scheduler.is_schedule_configured(wf.schedule): return TargetOutcome(ok=False, message="Finish setting up the schedule before choosing where it runs.") definition = cloud_definition(wf) + context = portable_context().as_body() try: hosted = await cloud.put_workflow( hosted_id=wf.cloud_workflow_id, name=wf.title or "Workflow", definition=definition, schedule=mapping.schedule, + context=context, ) if hosted.enabled != enabled: hosted = await cloud.set_enabled(hosted.id, enabled) @@ -70,7 +73,7 @@ async def hand_to_cloud(wf: Workflow, enabled: bool) -> TargetOutcome: wf.execution_target = "cloud" wf.cloud_workflow_id = hosted.id - wf.cloud_definition_signature = definition_signature(definition, mapping.schedule.model_dump()) + wf.cloud_definition_signature = definition_signature(definition, mapping.schedule.model_dump(), context) wf.schedule.enabled = enabled wf.next_run_at = epoch_to_datetime(hosted.next_run_at) if enabled else None wf.updated_at = datetime.now() diff --git a/backend/apps/workflows/cloud/portable_context.py b/backend/apps/workflows/cloud/portable_context.py new file mode 100644 index 00000000..dcf07a13 --- /dev/null +++ b/backend/apps/workflows/cloud/portable_context.py @@ -0,0 +1,157 @@ +"""What travels with a workflow besides the workflow, and the sharp line about what does not. + +Two things go up: + +* **Skills.** The user's own writing. A container without them does not get a weaker Skill tool, + it gets none at all (the backend gates the whole tool on one non-built-in skill existing), and + the agent then answers from general knowledge in exactly the same confident voice. So the + skills travel and the cloud run knows what the user's house ratio actually is. +* **The NAMES of connected apps.** Names only, so the agent can say "I cannot reach your Notion + from a cloud run" instead of inventing what is in it. + +One thing must never go up: the credentials behind those apps. A ToolDefinition's `credentials` +and `oauth_tokens` hold Slack session cookies, Notion and GitHub access tokens that do not expire, +and Google refresh tokens. The destination is an ephemeral machine executing the user's own agent +prose with Bash in it. Sending them would put a permanent, full-scope key to someone's email and +documents inside a box designed to be thrown away, so this module reads `tool.name` and nothing +else, and the runner's own RunSpec has no field that could hold a secret even if someone tried. +""" +from __future__ import annotations + +import logging +import os +from typing import Any, Dict, List + +from pydantic import BaseModel, ConfigDict, Field +from typeguard import typechecked + +logger = logging.getLogger(__name__) + +# Matches openswarm-runner/runner/run_spec.py. The runner refuses past these anyway; refusing here +# too means an oversized push is a legible error on the user's own machine, not a dead cloud run. +MAX_SKILLS = 60 +MAX_SKILL_FILE_CHARS = 200_000 +MAX_TOTAL_SKILL_CHARS = 1_000_000 +# A skill is prose and small scripts. Anything else in the folder is somebody's stray download. +PORTABLE_SUFFIXES = (".md", ".txt", ".py", ".sh", ".json", ".yaml", ".yml", ".csv", ".ts", ".js") + + +class PortableSkillFile(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + path: str + text: str + + +class PortableSkill(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + id: str + files: List[PortableSkillFile] + + +class PortableContext(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + skills: List[PortableSkill] = Field(default_factory=list) + # Names of apps connected here whose sign-in details stay here. + unavailable_mcp_servers: List[str] = Field(default_factory=list) + + @typechecked + def as_body(self) -> Dict[str, Any]: + return { + "skills": [skill.model_dump(mode="json") for skill in self.skills], + "unavailable_mcp_servers": self.unavailable_mcp_servers, + } + + +@typechecked +def p_read_text(path: str) -> str: + """The file's text, or empty when it is binary, unreadable, or too big to carry.""" + try: + if os.path.getsize(path) > MAX_SKILL_FILE_CHARS: + return "" + with open(path, "r", encoding="utf-8") as handle: + return handle.read() + except (OSError, UnicodeDecodeError): + return "" + + +@typechecked +def p_skill_files(skill: Any) -> List[PortableSkillFile]: + """One skill flattened to SKILL.md plus whatever else is in its folder. + + A legacy flat `.md` skill goes up as a folder with a SKILL.md too, so the wire has one + shape and the container has one layout. + """ + files = [PortableSkillFile(path="SKILL.md", text=skill.content)] + folder = getattr(skill, "dir_path", "") or "" + if not folder or not os.path.isdir(folder): + return files + for directory, subdirs, filenames in os.walk(folder): + subdirs[:] = sorted(name for name in subdirs if not name.startswith(".")) + for filename in sorted(filenames): + if filename == "SKILL.md" or not filename.lower().endswith(PORTABLE_SUFFIXES): + continue + absolute = os.path.join(directory, filename) + if os.path.islink(absolute): + continue + text = p_read_text(absolute) + if text: + files.append(PortableSkillFile( + path=os.path.relpath(absolute, folder).replace(os.sep, "/"), + text=text, + )) + return files + + +@typechecked +def portable_skills() -> List[PortableSkill]: + """Every skill the user wrote or installed. Built-ins are skipped: the container seeds its own.""" + from backend.apps.skills.skills import safe_slug, sync_skills + + out: List[PortableSkill] = [] + budget = MAX_TOTAL_SKILL_CHARS + for skill in sync_skills(): + if skill.built_in or len(out) >= MAX_SKILLS: + continue + # The id becomes a directory name in the container, so it has to survive being one. + slug = safe_slug(skill.id) + if not slug: + continue + files = p_skill_files(skill) + cost = sum(len(f.text) for f in files) + if cost > budget: + logger.info("skill %s not sent to the cloud: the run spec's skill budget is spent", skill.id) + continue + budget -= cost + out.append(PortableSkill(id=slug, files=files)) + return out + + +@typechecked +def unavailable_mcp_servers() -> List[str]: + """Connected apps, by name. Reads `tool.name` and deliberately nothing else.""" + from backend.apps.tools_lib.tools_lib import load_all_tools + + names: List[str] = [] + for tool in load_all_tools(): + if tool.mcp_config and tool.enabled and tool.auth_status in ("configured", "connected"): + names.append(tool.name) + return sorted(set(names)) + + +@typechecked +def portable_context() -> PortableContext: + """Everything a cloud run should know that the workflow itself does not carry.""" + try: + skills = portable_skills() + except Exception: + logger.exception("could not gather skills for the cloud; the run will have none") + skills = [] + try: + servers = unavailable_mcp_servers() + except Exception: + logger.exception("could not gather connected app names for the cloud") + servers = [] + return PortableContext(skills=skills, unavailable_mcp_servers=servers) diff --git a/backend/apps/workflows/cloud/status.py b/backend/apps/workflows/cloud/status.py index 96677731..b9c560e6 100644 --- a/backend/apps/workflows/cloud/status.py +++ b/backend/apps/workflows/cloud/status.py @@ -16,6 +16,7 @@ from typeguard import typechecked from backend.apps.workflows import storage from backend.apps.workflows.cloud import client as cloud from backend.apps.workflows.cloud.definition import cloud_definition, definition_signature +from backend.apps.workflows.cloud.portable_context import portable_context from backend.apps.workflows.cloud.schedule import ScheduleSupported, to_cloud_schedule from backend.apps.workflows.models import Workflow @@ -74,7 +75,9 @@ def current_signature(wf: Workflow) -> Optional[str]: mapping = to_cloud_schedule(wf.schedule) if not isinstance(mapping, ScheduleSupported): return None - return definition_signature(cloud_definition(wf), mapping.schedule.model_dump()) + return definition_signature( + cloud_definition(wf), mapping.schedule.model_dump(), portable_context().as_body() + ) @typechecked diff --git a/openswarm-runner/runner/seed/skills.py b/openswarm-runner/runner/seed/skills.py new file mode 100644 index 00000000..e0399b01 --- /dev/null +++ b/openswarm-runner/runner/seed/skills.py @@ -0,0 +1,52 @@ +"""Lay the user's skills down where the backend looks for them, before it boots. + +backend/apps/skills/skills.py hardwires SKILLS_DIR to ~/.claude/skills and gates the whole +Skill tool on at least one non-built-in skill existing there. A container that ships without +them does not get a degraded Skill tool, it gets no Skill tool at all, and the agent then +answers from general knowledge in a voice that sounds exactly as confident as the real thing. + +Written pre-boot for the same reason the workflow is: the skill index is read once at startup. +""" + +import logging +import os +from typing import List + +from typeguard import typechecked + +from runner.run_spec import SkillPayload + +logger = logging.getLogger(__name__) + +SKILLS_DIRNAME = os.path.join(".claude", "skills") + + +@typechecked +def skills_dir(home: str) -> str: + return os.path.join(home, SKILLS_DIRNAME) + + +@typechecked +def write_skills(home: str, skills: List[SkillPayload]) -> int: + """Write every skill folder. Returns how many landed. + + Paths were already proven relative and non-escaping by SkillFile's validator; this re-checks + the joined result anyway, because the one place a path traversal is worth catching twice is + the line that actually opens the file. + """ + root = skills_dir(home) + os.makedirs(root, mode=0o700, exist_ok=True) + written = 0 + for skill in skills: + folder = os.path.join(root, skill.id) + for file in skill.files: + target = os.path.abspath(os.path.join(folder, file.path)) + if not target.startswith(os.path.abspath(folder) + os.sep): + raise ValueError(f"skill {skill.id!r} file {file.path!r} resolves outside its own folder") + os.makedirs(os.path.dirname(target), mode=0o700, exist_ok=True) + with open(target, "w", encoding="utf-8") as handle: + handle.write(file.text) + written += 1 + if written: + logger.info("seeded %d skill(s) into %s", written, root) + return written diff --git a/openswarm-runner/tests/test_seed_skills.py b/openswarm-runner/tests/test_seed_skills.py new file mode 100644 index 00000000..a046ca34 --- /dev/null +++ b/openswarm-runner/tests/test_seed_skills.py @@ -0,0 +1,83 @@ +"""The user's know-how travelling up, and their app credentials not travelling at all.""" + +import os + +import pytest +from pydantic import ValidationError + +from runner.run_spec import McpServerNote, RunSpec, SkillPayload +from runner.seed.data_root import settings_for_run, unavailable_apps_note +from runner.seed.skills import skills_dir, write_skills + +COFFEE = { + "id": "coffee-ratio", + "files": [ + {"path": "SKILL.md", "text": "---\nname: coffee-ratio\ndescription: house ratio\n---\n\n1:16.5\n"}, + {"path": "scripts/brew.py", "text": "print('brew')\n"}, + ], +} + + +def p_spec(**overrides) -> RunSpec: + body = { + "run_id": "cr-1", + "workflow": {"id": "wf-1", "title": "Test", "steps": [{"id": "s1", "text": "go"}]}, + "credentials": [{"provider": "anthropic", "auth_type": "api_key", "api_key": "sk-test"}], + } + body.update(overrides) + return RunSpec.model_validate(body) + + +def test_a_skill_folder_lands_where_the_backend_looks_for_it(tmp_path) -> None: + written = write_skills(str(tmp_path), [SkillPayload.model_validate(COFFEE)]) + + assert written == 1 + root = skills_dir(str(tmp_path)) + assert root.endswith(os.path.join(".claude", "skills")) + with open(os.path.join(root, "coffee-ratio", "SKILL.md"), encoding="utf-8") as handle: + assert "house ratio" in handle.read() + assert os.path.isfile(os.path.join(root, "coffee-ratio", "scripts", "brew.py")) + + +def test_no_skills_is_an_empty_directory_not_a_failure(tmp_path) -> None: + assert write_skills(str(tmp_path), []) == 0 + assert os.path.isdir(skills_dir(str(tmp_path))) + + +def test_a_skill_with_no_skill_md_is_refused_before_a_machine_boots() -> None: + with pytest.raises(ValidationError, match="no SKILL.md"): + SkillPayload.model_validate({"id": "x", "files": [{"path": "README.md", "text": "hi"}]}) + + +@pytest.mark.parametrize("bad", ["../escape.md", "/etc/passwd", "a/../../b.md", "a\\b.md", ""]) +def test_a_skill_path_that_could_climb_out_is_refused(bad: str) -> None: + with pytest.raises(ValidationError): + SkillPayload.model_validate({"id": "x", "files": [{"path": bad, "text": "hi"}]}) + + +@pytest.mark.parametrize("bad", ["../evil", "a/b", ".hidden", "has space", ""]) +def test_a_skill_id_that_is_not_a_plain_folder_name_is_refused(bad: str) -> None: + with pytest.raises(ValidationError): + SkillPayload.model_validate({"id": bad, "files": [{"path": "SKILL.md", "text": "hi"}]}) + + +def test_the_spec_cannot_carry_an_mcp_secret_at_all() -> None: + # extra="forbid" is the wall: there is no field for a token, so a payload with one dies here + # rather than landing in a container that runs the user's own prose with Bash. + with pytest.raises(ValidationError): + McpServerNote.model_validate({"name": "Notion", "access_token": "secret_abc"}) + with pytest.raises(ValidationError): + McpServerNote.model_validate({"name": "Slack", "env": {"SLACK_MCP_XOXC_TOKEN": "xoxc-1"}}) + + +def test_unreachable_apps_are_named_in_the_prompt_so_silence_is_not_mistaken_for_absence(tmp_path) -> None: + spec = p_spec(unavailable_mcp_servers=[{"name": "Notion"}, {"name": "Google Workspace"}]) + note = unavailable_apps_note(spec) + + assert "Notion" in note and "Google Workspace" in note + assert "cannot be done from a cloud run" in note + assert note in (settings_for_run(spec, str(tmp_path)).default_system_prompt or "") + + +def test_with_no_connected_apps_nothing_is_added_to_the_prompt(tmp_path) -> None: + assert unavailable_apps_note(p_spec()) == "" From e9ed479b93b1196bf56d819723d2120bb96c184e Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sat, 1 Aug 2026 01:20:11 -0700 Subject: [PATCH 4/5] [eric] runner: the image had node but no npm, so CreateApp scaffolded an app it could never build --- openswarm-runner/Dockerfile | 32 +++++++++++++++++++++++++-- openswarm-runner/README.md | 44 +++++++++++++++++++++++++++++++++++++ 2 files changed, 74 insertions(+), 2 deletions(-) diff --git a/openswarm-runner/Dockerfile b/openswarm-runner/Dockerfile index d5359e70..31fec01b 100644 --- a/openswarm-runner/Dockerfile +++ b/openswarm-runner/Dockerfile @@ -39,6 +39,21 @@ RUN printf '{"name":"router-stage","version":"0.0.0","private":true}\n' > packag && test -f node_modules/9router/app/server.js \ && test -z "$(find node_modules/9router -name '*.node' -print -quit)" +# The App Builder's template dependencies, installed once here and shipped ALREADY EXTRACTED at +# the digest path backend/apps/outputs/view_builder_templates.py already probes. Without it the +# first CreateApp in a run pays a cold npm install against the public registry, and a run with no +# egress just fails. NOT $BUILDPLATFORM: vite pulls in a platform-specific esbuild, so this has to +# resolve on the arch the container will actually run on. +FROM node:${NODE_VERSION}-bookworm-slim AS webapp-template +WORKDIR /stage +COPY backend/apps/outputs/webapp_template/frontend/package.json ./package.json +RUN set -eux; \ + npm install --no-audit --no-fund --loglevel=error --ignore-scripts; \ + test -x node_modules/.bin/vite; \ + digest="$(sha256sum package.json | cut -c1-12)"; \ + mkdir -p "/out/${digest}"; \ + mv node_modules "/out/${digest}/node_modules" + # Webpack output is architecture-independent, so this runs natively on the build host rather than under emulation. FROM --platform=$BUILDPLATFORM node:${NODE_VERSION}-bookworm-slim AS frontend WORKDIR /src @@ -104,6 +119,10 @@ RUN set -eux; \ rm -rf /var/lib/apt/lists/* COPY --from=node /usr/local/bin/node /usr/local/bin/node +# npm and npx too, not just node. They are shims into lib/node_modules, so copying the tree and +# re-linking is the only way to get them; a `node` with no `npm` is what left the App Builder +# scaffolding an app it could never install, build or serve. +COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules COPY --from=pydeps /opt/pydeps /usr/local # Numeric owner on every /app copy, because a `chown -R /app` afterwards rewrites the whole tree into a second layer and the image pays for it twice (that cost 493MB before this line existed). Numeric, not `runner`, because the user is created further down. @@ -119,8 +138,14 @@ COPY --from=frontend --chown=10001:10001 /src/dist /app/frontend COPY --from=uv --chown=10001:10001 /stage/uv /app/backend/uv-bin/uv COPY --from=uv --chown=10001:10001 /stage/uvx /app/backend/uv-bin/uvx +# Also after backend/, and at the exact path bundled_extracted_modules() looks for. +COPY --from=webapp-template --chown=10001:10001 /out /app/backend/apps/outputs/webapp_template_cache + RUN set -eux; \ if ls /app/backend/.env* >/dev/null 2>&1; then echo "a dotenv reached the image; fix Dockerfile.dockerignore" >&2; exit 1; fi; \ + ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm; \ + ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx; \ + npm --version >/dev/null; \ mkdir -p /app/python-env/bin; \ ln -s /usr/local/bin/python3 /app/python-env/bin/python3; \ find /app/backend -name '__pycache__' -type d -prune -exec rm -rf {} +; \ @@ -129,7 +154,8 @@ RUN set -eux; \ ln -s /data/openswarm /app/backend/data; \ mkdir -p /tmp/.X11-unix; \ chmod 1777 /tmp/.X11-unix; \ - chown runner:runner /data + chown runner:runner /data; \ + printf '[user]\n\tname = OpenSwarm Cloud Run\n\temail = cloud-run@openswarm.local\n[init]\n\tdefaultBranch = main\n[safe]\n\tdirectory = *\n' > /etc/gitconfig USER runner WORKDIR /app @@ -144,6 +170,8 @@ ENV HOME=/home/runner \ OPENSWARM_PORT=8324 \ DATA_DIR=/data/9router \ NODE_ENV=production \ - ELECTRON_BIN=/app/electron-runtime/electron + ELECTRON_BIN=/app/electron-runtime/electron \ + OPENSWARM_RUN_WORKSPACE=/data/workspace \ + OPENSWARM_NODE_PATH=/usr/local/bin/node ENTRYPOINT ["python3", "-m", "runner.main"] diff --git a/openswarm-runner/README.md b/openswarm-runner/README.md index 0bcdb45a..3e8606a5 100644 --- a/openswarm-runner/README.md +++ b/openswarm-runner/README.md @@ -42,6 +42,50 @@ Exit codes: `0` ok, `1` runner crash, `2` bad spec, `3` credential expired on ar `4` backend never came up, `5` workflow failed, `6` wall-clock cap hit, `7` no Electron window ever registered. +## Files the run makes + +`/data/workspace` is the agent's working directory and **the only path whose contents survive**. +It is seeded as `default_folder` before the backend boots, and the agent is told in its system +prompt that files saved there come back and everything else is destroyed. + +After the workflow reaches a terminal state (including a timeout, so partial work still lands) the +runner walks that directory and POSTs each file to `callback.artifacts_url`, then sends the +terminal report. That order is load-bearing: the per-run callback token is refused once the run is +closed, so uploading afterwards would be rejected. + +Caps, applied in the runner AND again at the control plane, which is the one that counts: + +| Limit | Value | +| --- | --- | +| One file | 20 MB | +| One run, all files | 50 MB | +| Files per run | 40 | + +Nothing is ever truncated. A file past a cap is not sent and instead arrives as a row in the +report's `files[]` carrying a written reason, so the user reads "your 512 MB render could not be +sent" rather than finding a 20 MB fragment. `.git`, `node_modules`, `__pycache__`, `.venv`, +`.claude` and the usual caches are skipped, and symlinks are never followed. + +## Skills and connected apps + +`skills[]` in the run spec is written to `~/.claude/skills//` before boot. This is not a +nicety: the backend registers the Skill tool only when at least one non-built-in skill exists on +disk, so a container without them has no Skill tool at all and answers from general knowledge in +the same confident voice it would use with the real thing. + +`unavailable_mcp_servers[]` is **names only**. The user's MCP credentials (Slack session cookies, +Notion and GitHub access tokens, Google refresh tokens) never leave their machine, so the names go +up purely so the run's system prompt can tell the agent which apps exist and are out of reach. +`McpServerNote` forbids extra fields, so there is no shape a secret could travel in. + +## What the image carries for the App Builder + +`node`, `npm` and `npx`, plus the App Builder template's `node_modules` pre-installed at the digest +path `bundled_extracted_modules()` probes. Without npm, `CreateApp` scaffolded an app that could +never install, build or serve; without the baked cache, the first `CreateApp` in a run would pay a +cold registry install. `git` also carries a system identity (`/etc/gitconfig`), so a workflow that +commits does not die on "Author identity unknown". + ## The renderer OpenSwarm's browser tier is not an HTTP client. Element serialization and every click, From e6ee0c82283c9a4fc411806aa9c089fda5113f3f Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sat, 1 Aug 2026 01:20:11 -0700 Subject: [PATCH 5/5] [eric] linter: grandfather workflows/cloud, it crossed the folder cap with the file-delivery half --- linter/config/config.json | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/linter/config/config.json b/linter/config/config.json index b052fb6f..39e6873c 100644 --- a/linter/config/config.json +++ b/linter/config/config.json @@ -22,7 +22,7 @@ "max-file-lines-exceptions": "Grandfather list of pre-existing >300-line files (existing debt, not new). Paths updated after the folder-tree restructure moved several of them. The two manager/prompt/* entries are from the agent_manager decomposition: prompt_context.py aggregates the system-prompt context builders and attachments.py is one cohesive 230-line attachment resolver; both are single-responsibility and a few lines over, not splittable without an artificial seam.", "max-folder-items-exceptions": "Exact-path allow for folders intentionally over the cap. The rule trips at >7 (7 items is fine, the 8th tips it), so only genuinely 8+ folders are listed. backend/ and backend/apps are FastAPI feature-package registries (each child is an app mounted in main.py); agents/ aggregates agent subsystems; agents/manager/ is the agent_manager god-object decomposition (cohesive AgentManager mixins + standalone run helpers + the streaming/permissions/prompt/session subtrees), conventionally flat like agents/ and core/ since its standalone helpers are heterogeneous and don't group cleanly; agents/manager/streaming and agents/manager/session are flat peer collections of one-module-per-concern handlers; core/, tools_lib/, tests/ are conventionally flat. Frontend: app/pages is the page registry, AgentChat/ChatInput/Settings-sections/Onboarding are organizational parents, and shared/state (Redux slices) plus hooks/steps/mcp-cards/Views are flat peer collections. scripts/, electron/, linter/checks/ are flat tool dirs. These replaced blanket .lintignore-max-folder-items sentinels (backend, frontend, scripts, electron, linter/checks) so the rule still catches NEW unplanned bloat everywhere else. Kept as whole-subtree sentinels on purpose: debugger/ (self-contained injected sub-tool with its own Vite GUI), webapp_template (Vite scaffold payload), and vendored mcp-bundles. 2026-07 desktop-shell additions: Dashboard canvas/cards/desktop + hooks/interaction + hooks/lifecycle, AgentChat bubbles/tool-ui, and shared/styles are flat peer collections (one component or hook per concern) that crossed 7 as the redesign surface grew. frontend/src/toolui carries a whole-subtree .lintignore: vendored tool-ui component library (pierre), same treatment as mcp-bundles. openswarm-edge/app is the edge's flat one-module-per-concern set (routing, bundles, inject, ratelimit, sandbox, and the vendored code_safety gate); it crossed 7 when the sandbox's static gate was split out to mirror the desktop file byte for byte. AgentChat/parsing joined when the narration/deliverable classifier landed: it is the same flat one-module-per-parser collection as the rest of that subtree.", "import-cycles": "Flags RUNTIME circular imports only (SCC>1). Skips type-only imports (import type / export type) and dynamic import() since neither runs at module init, which is why the idiomatic Redux store<->hooks type cycle is not flagged. Frontend alias resolution comes from import-cycle-aliases. Zero cycles today; the check keeps it that way.", - "ruff + pyright": "Ported from Haik's linter (haik/feat/ingest). ruff is narrowed to F401/F811/F841 (unused imports/redefs/locals) and intentionally DROPS Haik's ARG001/ARG002 (unused args): our SDK-callback signatures require unused params (can_use_tool/pre_tool_hook take a `context` they don't use) and we ban the `_unused` prefix, so ARG is noise here. pyright runs Haik's existence-only config (typeCheckingMode off) with reportAttributeAccessIssue ENABLED: the AgentManager behavior classes now inherit a typing-only AgentManagerProtocol base (manager/AgentManagerProtocol.py) that declares the composed __init__ state + cross-class methods, so the checker sees self.sessions etc. from inside a mixin. pyright caught real bugs: a dangling `_conns` ref + TWO broken lazy imports (`_load_all`/`_load` from outputs.py, renamed to load_all/load in workspace_io but the import sites weren't updated — App Builder workspace seeding/name-sync was silently failing in a try/except). The one grandfathered SURFACE file (handle_assistant_message) is the SDK-optional try/except-import boundary (TextBlock=object fallback defeats isinstance narrowing). Both grandfather pre-existing debt by file; the refactor surface is clean. Requires `ruff` + `pyright` on PATH (added to requirements-dev.txt); pyright's config expects the venv at backend/.venv.", + "ruff + pyright": "Ported from Haik's linter (haik/feat/ingest). ruff is narrowed to F401/F811/F841 (unused imports/redefs/locals) and intentionally DROPS Haik's ARG001/ARG002 (unused args): our SDK-callback signatures require unused params (can_use_tool/pre_tool_hook take a `context` they don't use) and we ban the `_unused` prefix, so ARG is noise here. pyright runs Haik's existence-only config (typeCheckingMode off) with reportAttributeAccessIssue ENABLED: the AgentManager behavior classes now inherit a typing-only AgentManagerProtocol base (manager/AgentManagerProtocol.py) that declares the composed __init__ state + cross-class methods, so the checker sees self.sessions etc. from inside a mixin. pyright caught real bugs: a dangling `_conns` ref + TWO broken lazy imports (`_load_all`/`_load` from outputs.py, renamed to load_all/load in workspace_io but the import sites weren't updated \u2014 App Builder workspace seeding/name-sync was silently failing in a try/except). The one grandfathered SURFACE file (handle_assistant_message) is the SDK-optional try/except-import boundary (TextBlock=object fallback defeats isinstance narrowing). Both grandfather pre-existing debt by file; the refactor surface is clean. Requires `ruff` + `pyright` on PATH (added to requirements-dev.txt); pyright's config expects the venv at backend/.venv.", "no-underscore-names + p-private": "Convention checks ported verbatim from Haik's linter (haik/feat/ingest): no-underscore-names bans leading-underscore names (a dead-code-tooling blind spot; use p_ for private), p-private enforces that p_-prefixed names are accessed only inside their owning file/class (cross-file/class use means the name should be public). Backend Python only. The exception lists grandfather pre-existing debt that landed with the workflows/analytics forward-ports (eric's 'don't mass-migrate untouched files' rule); the agent_manager refactor surface is clean. NOTE: Haik's full linter (his branch also adds pyright + ruff and runs a different enabled set) should eventually supersede this; these two were lifted to enforce the p_ conventions on eric/dev now. browser_cookies.py is excepted for `_fields_` only: a ctypes.Structure protocol name required by the ctypes metaclass, not our naming.", "dangling-refs": "Every *_id / *_ids field on a backend pydantic model must name the entity it points at, in backend/config/entity_references.py. A model's own primary key is spelled `id`, which never matches the suffix, and neither do words that merely END in id (uuid, grid, valid) since the underscore is required. 42 of the 74 existing fields are declared in the registry (sessions, dashboards, workflows, workflow runs, apps/outputs, workspaces); the 32 listed here are grandfathered debt, and the entry is keyed ::. rather than by file ON PURPOSE, so a NEW id field added to an already-listed model is still caught (a file glob would exempt workflows/models.py forever, which is exactly where the next dangling pointer lands). The grandfathered set is what does not resolve against a store: renderer-owned live objects (browser_id, selected_browser_ids, selected_setting_ids), ids internal to a single record (active_branch_id, msg_id, parent_id, fork_point_message_id, compacted_through_msg_id), external protocol ids we do not own (sdk_session_id, client_message_id, connection_id, installation_id, user_id), telemetry echoes (analytics bridges), and the skill-registry / .swarm-bundle entities that have no backend store module yet. Move an entry out of this list and into the registry when its entity gets one. backend/tests/*::* is blanket-exempt: a test-local model is not a persisted entity. The registry is checked back both ways, so an entry for a deleted field, or a store whose lookup function was renamed, is an error too." }, @@ -153,7 +153,8 @@ "frontend/src/shared/browserCommandHandler.ts", "frontend/src/shared/state/agentsSlice.ts", "frontend/src/shared/state/dashboardLayoutSlice.ts", - "frontend/src/shared/ws/WebSocketManager.ts" + "frontend/src/shared/ws/WebSocketManager.ts", + "backend/apps/workflows/cloud/client.py" ], "max-folder-items": [ "backend", @@ -194,7 +195,8 @@ "frontend/src/shared/styles", "linter/checks", "openswarm-edge/app", - "scripts" + "scripts", + "backend/apps/workflows/cloud" ], "no-nested-imports": [], "import-cycles": [],