[eric] merge eric/runner-parity: files come back, apps build, skills load, and MCP refuses honestly

This commit is contained in:
ciregenz
2026-08-01 01:41:11 -07:00
23 changed files with 1477 additions and 105 deletions
+4 -4
View File
@@ -395,12 +395,12 @@ def clear_skill_dir(skill_id: str) -> None:
"""Empty a skill's folder before an in-place update so files removed upstream
don't linger as orphans. write_folder_skill recreates the dir right after."""
import shutil
d = os.path.join(SKILLS_DIR, p_safe_slug(skill_id))
d = os.path.join(SKILLS_DIR, safe_slug(skill_id))
if os.path.isdir(d):
shutil.rmtree(d, ignore_errors=True)
def p_safe_slug(raw: str) -> str:
def safe_slug(raw: str) -> str:
slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", (raw or "").strip().lower()).strip("-")
return slug or "skill"
@@ -417,7 +417,7 @@ def unique_skill_slug(base: str) -> str:
"""A free slug for `base`, suffixing -2, -3, ... on collision. Lets a
registry install land beside a same-named skill instead of silently
overwriting the user's existing one."""
slug = p_safe_slug(base)
slug = safe_slug(base)
if not p_skill_exists(slug):
return slug
i = 2
@@ -432,7 +432,7 @@ def write_folder_skill(skill_id: str, files: dict[str, str], meta: dict) -> Skil
zip/.swarm import. Relpaths that try to escape the skill folder (../, abs
paths) are dropped, an untrusted registry archive can't write outside its
own dir."""
slug = p_safe_slug(skill_id)
slug = safe_slug(skill_id)
base = os.path.join(SKILLS_DIR, slug)
base_abs = os.path.abspath(base)
# A folder write supersedes any legacy flat <slug>.md, so we never leave a phantom flat file shadowed by the folder (folder wins in skill_md_path).
+65 -2
View File
@@ -20,6 +20,8 @@ from backend.apps.workflows.cloud.schedule import CloudSchedule, wire
# The cloud router is mounted at /api/workflows and a trailing slash 404s there, so the collection paths are the empty string, not "/".
COLLECTION = ""
TIMEOUT_SECONDS = 8.0
# Files are up to 20MB each, so they get their own budget rather than the chatty-call one.
DOWNLOAD_TIMEOUT_SECONDS = 120.0
class CloudRefused(Exception):
@@ -88,6 +90,18 @@ class CloudPreflight(BaseModel):
hosted: Optional[HostedWorkflow] = None
class CloudRunFile(BaseModel):
"""One file a cloud run produced. `refusal` set means it exists nowhere and says why."""
model_config = ConfigDict(validate_assignment=True)
id: str
path: str
size_bytes: int = 0
sha256: Optional[str] = None
refusal: Optional[str] = None
class CloudRun(BaseModel):
model_config = ConfigDict(validate_assignment=True)
@@ -99,6 +113,7 @@ class CloudRun(BaseModel):
answer: Optional[str] = None
notices: List[str] = Field(default_factory=list)
cost_usd: Optional[float] = None
files: List[CloudRunFile] = Field(default_factory=list)
@typechecked
@@ -232,12 +247,16 @@ async def p_preflight_from_list(hosted_id: Optional[str]) -> CloudPreflight:
@typechecked
async def put_workflow(
*, hosted_id: Optional[str], name: str, definition: Dict[str, Any],
schedule: CloudSchedule, runs_before: int = 0,
schedule: CloudSchedule, runs_before: int = 0, context: Optional[Dict[str, Any]] = None,
) -> HostedWorkflow:
"""Create the hosted copy, or re-push onto the existing row so an edited workflow stops running
last week's prose. runs_before rides only on the create: an edit that resent it would hand a
nearly-spent run cap its whole budget back."""
nearly-spent run cap its whole budget back. `context` carries the user's skills and the names of
the apps a cloud run cannot reach; an older control plane ignores the extra keys, which costs a
run its skills but never its run."""
body: Dict[str, Any] = {"name": name, "definition": definition, "schedule": wire(schedule)}
if context:
body.update(context)
if hosted_id:
try:
raw = await p_call("POST", f"/{hosted_id}/update", body)
@@ -274,6 +293,26 @@ async def delete_hosted(hosted_id: str) -> None:
raise
@typechecked
def p_files(raw: Any) -> List[CloudRunFile]:
"""A control plane with no file support answers without the key, which is an empty list, not an error."""
if not isinstance(raw, list):
return []
out: List[CloudRunFile] = []
for row in raw:
if not isinstance(row, dict) or not isinstance(row.get("id"), str) or not isinstance(row.get("path"), str):
continue
size = row.get("size_bytes")
out.append(CloudRunFile(
id=row["id"],
path=row["path"],
size_bytes=size if isinstance(size, int) else 0,
sha256=row.get("sha256") if isinstance(row.get("sha256"), str) else None,
refusal=row.get("refusal") if isinstance(row.get("refusal"), str) else None,
))
return out
@typechecked
async def list_runs(hosted_id: str) -> List[CloudRun]:
raw = await p_call("GET", f"/{hosted_id}/runs")
@@ -294,5 +333,29 @@ async def list_runs(hosted_id: str) -> List[CloudRun]:
answer=row.get("answer") if isinstance(row.get("answer"), str) else None,
notices=[n for n in notices if isinstance(n, str)] if isinstance(notices, list) else [],
cost_usd=row.get("cost_usd") if isinstance(row.get("cost_usd"), (int, float)) else None,
files=p_files(row.get("files")),
))
return out
@typechecked
async def download_run_file(hosted_id: str, run_id: str, file_id: str) -> bytes:
"""The file's bytes. Its own call rather than p_call because this answer is not JSON."""
from backend.apps.settings.store import load_settings
token, base = account_auth(load_settings())
if not token:
raise SignedOut()
url = f"{base}/api/workflows/{hosted_id}/runs/{run_id}/files/{file_id}"
try:
async with httpx.AsyncClient(timeout=DOWNLOAD_TIMEOUT_SECONDS) as client:
resp = await client.get(url, headers={"Authorization": f"Bearer {token}"})
except httpx.HTTPError as exc:
raise CloudUnreachable(f"{type(exc).__name__}") from exc
if resp.status_code == 401:
raise SignedOut()
if resp.status_code >= 500:
raise CloudUnreachable(f"the cloud returned {resp.status_code}")
if resp.status_code >= 400:
raise CloudRefused(p_message(resp, "That file could not be downloaded."), resp.status_code)
return resp.content
+10 -5
View File
@@ -9,7 +9,7 @@ from __future__ import annotations
import hashlib
import json
from typing import Any, Dict
from typing import Any, Dict, Optional
from typeguard import typechecked
@@ -54,8 +54,13 @@ def cloud_definition(wf: Workflow) -> Dict[str, Any]:
@typechecked
def definition_signature(definition: Dict[str, Any], schedule: Dict[str, Any]) -> str:
"""Fingerprint of exactly what we last handed the cloud, schedule included,
so "your edits are not up there yet" is a fact rather than a guess."""
payload = json.dumps({"definition": definition, "schedule": schedule}, sort_keys=True)
def definition_signature(
definition: Dict[str, Any], schedule: Dict[str, Any], context: Optional[Dict[str, Any]] = None
) -> str:
"""Fingerprint of exactly what we last handed the cloud, schedule and skills included,
so "your edits are not up there yet" is a fact rather than a guess. Editing a skill a
workflow leans on is an edit to that workflow's behaviour, so it belongs in here."""
payload = json.dumps(
{"definition": definition, "schedule": schedule, "context": context or {}}, sort_keys=True
)
return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+4 -1
View File
@@ -18,6 +18,7 @@ from typeguard import typechecked
from backend.apps.workflows import scheduler, storage
from backend.apps.workflows.cloud import client as cloud
from backend.apps.workflows.cloud.definition import cloud_definition, definition_signature
from backend.apps.workflows.cloud.portable_context import portable_context
from backend.apps.workflows.cloud.schedule import ScheduleSupported, to_cloud_schedule, wire
from backend.apps.workflows.cloud.status import epoch_to_datetime
from backend.apps.workflows.models import Workflow
@@ -51,6 +52,7 @@ async def hand_to_cloud(wf: Workflow, enabled: bool) -> TargetOutcome:
if enabled and not scheduler.is_schedule_configured(wf.schedule):
return TargetOutcome(ok=False, message="Finish setting up the schedule before choosing where it runs.")
definition = cloud_definition(wf)
context = portable_context().as_body()
try:
hosted = await cloud.put_workflow(
hosted_id=wf.cloud_workflow_id,
@@ -58,6 +60,7 @@ async def hand_to_cloud(wf: Workflow, enabled: bool) -> TargetOutcome:
definition=definition,
schedule=mapping.schedule,
runs_before=wf.schedule.runs_count,
context=context,
)
if hosted.enabled != enabled:
hosted = await cloud.set_enabled(hosted.id, enabled)
@@ -71,7 +74,7 @@ async def hand_to_cloud(wf: Workflow, enabled: bool) -> TargetOutcome:
wf.execution_target = "cloud"
wf.cloud_workflow_id = hosted.id
wf.cloud_definition_signature = definition_signature(definition, wire(mapping.schedule))
wf.cloud_definition_signature = definition_signature(definition, wire(mapping.schedule), context)
wf.schedule.enabled = enabled
wf.next_run_at = epoch_to_datetime(hosted.next_run_at) if enabled else None
wf.updated_at = datetime.now()
@@ -0,0 +1,157 @@
"""What travels with a workflow besides the workflow, and the sharp line about what does not.
Two things go up:
* **Skills.** The user's own writing. A container without them does not get a weaker Skill tool,
it gets none at all (the backend gates the whole tool on one non-built-in skill existing), and
the agent then answers from general knowledge in exactly the same confident voice. So the
skills travel and the cloud run knows what the user's house ratio actually is.
* **The NAMES of connected apps.** Names only, so the agent can say "I cannot reach your Notion
from a cloud run" instead of inventing what is in it.
One thing must never go up: the credentials behind those apps. A ToolDefinition's `credentials`
and `oauth_tokens` hold Slack session cookies, Notion and GitHub access tokens that do not expire,
and Google refresh tokens. The destination is an ephemeral machine executing the user's own agent
prose with Bash in it. Sending them would put a permanent, full-scope key to someone's email and
documents inside a box designed to be thrown away, so this module reads `tool.name` and nothing
else, and the runner's own RunSpec has no field that could hold a secret even if someone tried.
"""
from __future__ import annotations
import logging
import os
from typing import Any, Dict, List
from pydantic import BaseModel, ConfigDict, Field
from typeguard import typechecked
logger = logging.getLogger(__name__)
# Matches openswarm-runner/runner/run_spec.py. The runner refuses past these anyway; refusing here
# too means an oversized push is a legible error on the user's own machine, not a dead cloud run.
MAX_SKILLS = 60
MAX_SKILL_FILE_CHARS = 200_000
MAX_TOTAL_SKILL_CHARS = 1_000_000
# A skill is prose and small scripts. Anything else in the folder is somebody's stray download.
PORTABLE_SUFFIXES = (".md", ".txt", ".py", ".sh", ".json", ".yaml", ".yml", ".csv", ".ts", ".js")
class PortableSkillFile(BaseModel):
model_config = ConfigDict(validate_assignment=True)
path: str
text: str
class PortableSkill(BaseModel):
model_config = ConfigDict(validate_assignment=True)
id: str
files: List[PortableSkillFile]
class PortableContext(BaseModel):
model_config = ConfigDict(validate_assignment=True)
skills: List[PortableSkill] = Field(default_factory=list)
# Names of apps connected here whose sign-in details stay here.
unavailable_mcp_servers: List[str] = Field(default_factory=list)
@typechecked
def as_body(self) -> Dict[str, Any]:
return {
"skills": [skill.model_dump(mode="json") for skill in self.skills],
"unavailable_mcp_servers": self.unavailable_mcp_servers,
}
@typechecked
def p_read_text(path: str) -> str:
"""The file's text, or empty when it is binary, unreadable, or too big to carry."""
try:
if os.path.getsize(path) > MAX_SKILL_FILE_CHARS:
return ""
with open(path, "r", encoding="utf-8") as handle:
return handle.read()
except (OSError, UnicodeDecodeError):
return ""
@typechecked
def p_skill_files(skill: Any) -> List[PortableSkillFile]:
"""One skill flattened to SKILL.md plus whatever else is in its folder.
A legacy flat `<id>.md` skill goes up as a folder with a SKILL.md too, so the wire has one
shape and the container has one layout.
"""
files = [PortableSkillFile(path="SKILL.md", text=skill.content)]
folder = getattr(skill, "dir_path", "") or ""
if not folder or not os.path.isdir(folder):
return files
for directory, subdirs, filenames in os.walk(folder):
subdirs[:] = sorted(name for name in subdirs if not name.startswith("."))
for filename in sorted(filenames):
if filename == "SKILL.md" or not filename.lower().endswith(PORTABLE_SUFFIXES):
continue
absolute = os.path.join(directory, filename)
if os.path.islink(absolute):
continue
text = p_read_text(absolute)
if text:
files.append(PortableSkillFile(
path=os.path.relpath(absolute, folder).replace(os.sep, "/"),
text=text,
))
return files
@typechecked
def portable_skills() -> List[PortableSkill]:
"""Every skill the user wrote or installed. Built-ins are skipped: the container seeds its own."""
from backend.apps.skills.skills import safe_slug, sync_skills
out: List[PortableSkill] = []
budget = MAX_TOTAL_SKILL_CHARS
for skill in sync_skills():
if skill.built_in or len(out) >= MAX_SKILLS:
continue
# The id becomes a directory name in the container, so it has to survive being one.
slug = safe_slug(skill.id)
if not slug:
continue
files = p_skill_files(skill)
cost = sum(len(f.text) for f in files)
if cost > budget:
logger.info("skill %s not sent to the cloud: the run spec's skill budget is spent", skill.id)
continue
budget -= cost
out.append(PortableSkill(id=slug, files=files))
return out
@typechecked
def unavailable_mcp_servers() -> List[str]:
"""Connected apps, by name. Reads `tool.name` and deliberately nothing else."""
from backend.apps.tools_lib.tools_lib import load_all_tools
names: List[str] = []
for tool in load_all_tools():
if tool.mcp_config and tool.enabled and tool.auth_status in ("configured", "connected"):
names.append(tool.name)
return sorted(set(names))
@typechecked
def portable_context() -> PortableContext:
"""Everything a cloud run should know that the workflow itself does not carry."""
try:
skills = portable_skills()
except Exception:
logger.exception("could not gather skills for the cloud; the run will have none")
skills = []
try:
servers = unavailable_mcp_servers()
except Exception:
logger.exception("could not gather connected app names for the cloud")
servers = []
return PortableContext(skills=skills, unavailable_mcp_servers=servers)
+16 -3
View File
@@ -7,6 +7,7 @@ cloud re-decides that at create and again at dispatch.
"""
from __future__ import annotations
import asyncio
from contextlib import asynccontextmanager
from datetime import datetime
from typing import List, Literal, Optional, Union
@@ -18,6 +19,7 @@ from typeguard import typechecked
from backend.apps.workflows import storage
from backend.apps.workflows.cloud import client as cloud
from backend.apps.workflows.cloud.handover import TargetOutcome, hand_to_cloud, take_back
from backend.apps.workflows.cloud.run_files import LocalRunFile, described, downloads_root, fetch_missing
from backend.apps.workflows.cloud.status import CloudStatus, compute_status, epoch_to_datetime
from backend.apps.workflows.models import Workflow
from backend.config.Apps import SubApp
@@ -50,6 +52,7 @@ class CloudRunRow(BaseModel):
answer: Optional[str] = None
notices: List[str] = []
cost_usd: Optional[float] = None
files: List[LocalRunFile] = []
class CloudRunsReady(BaseModel):
@@ -57,6 +60,9 @@ class CloudRunsReady(BaseModel):
state: Literal["ready"] = "ready"
runs: List[CloudRunRow] = []
# Where this machine puts a cloud run's files, so the UI can say it out loud even when a run
# has none yet. A folder the user is told about is a folder they can find later.
files_folder: str = ""
class CloudRunsUnavailable(BaseModel):
@@ -95,7 +101,7 @@ async def set_workflow_target(workflow_id: str, body: TargetRequest) -> TargetOu
async def workflow_cloud_runs(workflow_id: str) -> CloudRunsResponse:
wf = p_workflow(workflow_id)
if not wf.cloud_workflow_id:
return CloudRunsReady(runs=[])
return CloudRunsReady(runs=[], files_folder=downloads_root())
try:
runs = await cloud.list_runs(wf.cloud_workflow_id)
except cloud.SignedOut:
@@ -103,11 +109,17 @@ async def workflow_cloud_runs(workflow_id: str) -> CloudRunsResponse:
except cloud.CloudRefused as exc:
# A 404 here is the hosted copy being gone, which is an empty history, not a broken one.
if exc.status == 404:
return CloudRunsReady(runs=[])
return CloudRunsReady(runs=[], files_folder=downloads_root())
return CloudRunsUnavailable(state="unknown", detail=exc.message)
except cloud.CloudUnreachable as exc:
return CloudRunsUnavailable(state="unknown", detail=exc.detail)
# Answered now, files fetched behind it. A 20MB attachment must not hold up the history the
# user asked for, and a run's files appear a moment later without them doing anything.
asyncio.create_task(fetch_missing(wf.cloud_workflow_id, wf, runs))
return CloudRunsReady(
files_folder=downloads_root(),
runs=[
CloudRunRow(
id=r.id,
@@ -118,7 +130,8 @@ async def workflow_cloud_runs(workflow_id: str) -> CloudRunsResponse:
answer=r.answer,
notices=r.notices,
cost_usd=r.cost_usd,
files=described(wf, r),
)
for r in runs
]
],
)
+143
View File
@@ -0,0 +1,143 @@
"""Bring a cloud run's files down to the machine the user actually sits at.
A cloud run finishes while the laptop is shut, so the files live in the cloud until
something fetches them. That something is this module, and it puts them in Downloads
rather than anywhere clever: the whole point of a deliverable is that the user can
find it without being told where to look, and "it is in Downloads" is the one answer
nobody needs explained.
Every file is fetched at most once. The local path is derived, never stored, so a
user who moves or deletes a file just gets it back on the next look rather than
staring at a broken link.
"""
from __future__ import annotations
import asyncio
import logging
import os
import re
from typing import List, Optional
from pydantic import BaseModel, ConfigDict
from typeguard import typechecked
from backend.apps.workflows.cloud import client as cloud
from backend.apps.workflows.models import Workflow
logger = logging.getLogger(__name__)
# Bounded so one History open cannot spend minutes pulling a whole month of runs.
MAX_CONCURRENT_DOWNLOADS = 3
class LocalRunFile(BaseModel):
"""A cloud file plus where it is (or would be) on this machine."""
model_config = ConfigDict(validate_assignment=True)
id: str
path: str
size_bytes: int
refusal: Optional[str] = None
# Set once the bytes are on disk. None means "not here yet", never "not coming".
local_path: Optional[str] = None
@typechecked
def safe_component(raw: str) -> str:
"""One path segment a filesystem will take, with the user's own name still legible."""
cleaned = re.sub(r'[<>:"/\\|?*\x00-\x1f]', "-", raw).strip(" .")
return cleaned[:60] or "untitled"
@typechecked
def downloads_root() -> str:
home = os.path.expanduser("~")
downloads = os.path.join(home, "Downloads")
return os.path.join(downloads if os.path.isdir(downloads) else home, "OpenSwarm")
@typechecked
def run_folder(workflow: Workflow, run: cloud.CloudRun) -> str:
"""Where this run's files go: one folder per run, named so a list of them reads as a history."""
stamp = run.finished_at or run.started_at
when = "unknown-date"
if stamp:
from datetime import datetime, timezone
when = datetime.fromtimestamp(stamp / 1000, tz=timezone.utc).astimezone().strftime("%Y-%m-%d %H%M")
return os.path.join(
downloads_root(),
safe_component(workflow.title or "Workflow"),
safe_component(f"{when} {run.id[:8]}"),
)
@typechecked
def local_path_for(folder: str, file: cloud.CloudRunFile) -> str:
"""The file's home, with every segment of the run-relative path made filesystem-safe."""
parts = [safe_component(part) for part in file.path.split("/") if part not in ("", ".", "..")]
return os.path.join(folder, *(parts or ["file"]))
@typechecked
def p_already_here(path: str, size_bytes: int) -> bool:
try:
return os.path.getsize(path) == size_bytes
except OSError:
return False
@typechecked
def described(workflow: Workflow, run: cloud.CloudRun) -> List[LocalRunFile]:
"""This run's files and where they are right now. Reads disk, never fetches."""
folder = run_folder(workflow, run)
out: List[LocalRunFile] = []
for file in run.files:
local = local_path_for(folder, file)
out.append(LocalRunFile(
id=file.id,
path=file.path,
size_bytes=file.size_bytes,
refusal=file.refusal,
local_path=local if file.refusal is None and p_already_here(local, file.size_bytes) else None,
))
return out
@typechecked
async def p_fetch_one(hosted_id: str, run: cloud.CloudRun, file: cloud.CloudRunFile, target: str) -> None:
payload = await cloud.download_run_file(hosted_id, run.id, file.id)
os.makedirs(os.path.dirname(target), exist_ok=True)
temporary = f"{target}.part"
with open(temporary, "wb") as handle:
handle.write(payload)
os.replace(temporary, target)
logger.info("cloud run %s: saved %s to %s", run.id, file.path, target)
@typechecked
async def fetch_missing(hosted_id: str, workflow: Workflow, runs: List[cloud.CloudRun]) -> None:
"""Pull down anything not already here. Never raises: a run's answer must render
even when its attachments cannot be fetched, and the next look tries again."""
semaphore = asyncio.Semaphore(MAX_CONCURRENT_DOWNLOADS)
async def p_guarded(run: cloud.CloudRun, file: cloud.CloudRunFile, target: str) -> None:
async with semaphore:
try:
await p_fetch_one(hosted_id, run, file, target)
except (cloud.SignedOut, cloud.CloudRefused, cloud.CloudUnreachable, OSError) as exc:
logger.info("cloud run %s: %s not fetched (%s)", run.id, file.path, exc)
pending = []
for run in runs:
folder = run_folder(workflow, run)
for file in run.files:
if file.refusal is not None:
continue
target = local_path_for(folder, file)
if p_already_here(target, file.size_bytes):
continue
pending.append(p_guarded(run, file, target))
if pending:
await asyncio.gather(*pending)
+5 -1
View File
@@ -16,6 +16,7 @@ from typeguard import typechecked
from backend.apps.workflows import storage
from backend.apps.workflows.cloud import client as cloud
from backend.apps.workflows.cloud.definition import cloud_definition, definition_signature
from backend.apps.workflows.cloud.portable_context import portable_context
from backend.apps.workflows.cloud.schedule import ScheduleSupported, to_cloud_schedule, wire
from backend.apps.workflows.models import Workflow
@@ -74,7 +75,10 @@ def current_signature(wf: Workflow) -> Optional[str]:
mapping = to_cloud_schedule(wf.schedule)
if not isinstance(mapping, ScheduleSupported):
return None
return definition_signature(cloud_definition(wf), wire(mapping.schedule))
# wire() is the richer serializer (weekly, zones, ends_at); portable_context is what the runner needs to reproduce the user's setup.
return definition_signature(
cloud_definition(wf), wire(mapping.schedule), portable_context().as_body()
)
@typechecked
+152
View File
@@ -0,0 +1,152 @@
"""Where a cloud run's files land on the machine the user sits at, and what they are told.
A deliverable the user cannot find has not been delivered, so the two things worth pinning
are the folder (Downloads, because that is where people look) and the honesty of the list:
a file that was refused must still appear, with its reason, and must never be fetched.
"""
import os
import pytest
from backend.apps.workflows.cloud.client import CloudRun, CloudRunFile
from backend.apps.workflows.cloud.run_files import (
described,
downloads_root,
fetch_missing,
local_path_for,
safe_component,
run_folder,
)
from backend.apps.workflows.models import Workflow
def p_run(**overrides) -> CloudRun:
body = {
"id": "run-abcdef123456",
"status": "succeeded",
"finished_at": 1_785_000_000_000,
"files": [],
}
body.update(overrides)
return CloudRun.model_validate(body)
def test_files_land_in_downloads_because_that_is_where_people_look() -> None:
root = downloads_root()
assert root.endswith(os.path.join("OpenSwarm"))
assert os.path.expanduser("~") in root
def test_a_run_gets_its_own_folder_named_so_a_list_of_them_reads_as_history() -> None:
workflow = Workflow(title="Weekly numbers")
folder = run_folder(workflow, p_run())
assert "Weekly numbers" in folder
# The run id's head disambiguates two runs in the same minute without being a wall of hex.
assert "run-abcd" in folder
@pytest.mark.parametrize(
"raw,expected",
[
("Weekly numbers", "Weekly numbers"),
("bad/name", "bad-name"),
("../escape", "-escape"),
("with:colon", "with-colon"),
("", "untitled"),
("...", "untitled"),
],
)
def test_a_title_is_made_safe_without_becoming_unrecognisable(raw: str, expected: str) -> None:
assert safe_component(raw) == expected
def test_a_nested_run_path_stays_nested_locally() -> None:
target = local_path_for("/tmp/folder", CloudRunFile(id="f1", path="notes/method.txt", size_bytes=5))
assert target == os.path.join("/tmp/folder", "notes", "method.txt")
def test_a_path_that_tries_to_climb_out_cannot(tmp_path) -> None:
target = local_path_for(str(tmp_path), CloudRunFile(id="f1", path="../../etc/passwd", size_bytes=1))
assert os.path.abspath(target).startswith(str(tmp_path))
def test_a_file_not_yet_downloaded_reports_no_local_path_rather_than_a_broken_one(tmp_path, monkeypatch) -> None:
monkeypatch.setattr("backend.apps.workflows.cloud.run_files.downloads_root", lambda: str(tmp_path))
run = p_run(files=[{"id": "f1", "path": "summary.md", "size_bytes": 42}])
described_files = described(Workflow(title="W"), run)
assert len(described_files) == 1
assert described_files[0].local_path is None
def test_a_file_already_on_disk_is_reported_at_its_real_path(tmp_path, monkeypatch) -> None:
monkeypatch.setattr("backend.apps.workflows.cloud.run_files.downloads_root", lambda: str(tmp_path))
workflow = Workflow(title="W")
run = p_run(files=[{"id": "f1", "path": "summary.md", "size_bytes": 5}])
target = local_path_for(run_folder(workflow, run), run.files[0])
os.makedirs(os.path.dirname(target), exist_ok=True)
with open(target, "wb") as handle:
handle.write(b"hello")
assert described(workflow, run)[0].local_path == target
def test_a_refused_file_is_listed_with_its_reason_and_never_fetched(tmp_path, monkeypatch) -> None:
monkeypatch.setattr("backend.apps.workflows.cloud.run_files.downloads_root", lambda: str(tmp_path))
workflow = Workflow(title="W")
run = p_run(files=[
{"id": "f1", "path": "render.mp4", "size_bytes": 999, "refusal": "it is 512.0 MB and the limit is 20.0 MB"},
])
listed = described(workflow, run)
assert listed[0].refusal is not None
assert listed[0].local_path is None
asked: list = []
async def p_never(*args, **kwargs):
asked.append(args)
return b""
monkeypatch.setattr("backend.apps.workflows.cloud.run_files.cloud.download_run_file", p_never)
import asyncio
asyncio.run(fetch_missing("hosted-1", workflow, [run]))
assert asked == [], "a file that exists nowhere must never be requested"
def test_a_download_that_fails_leaves_the_history_readable(tmp_path, monkeypatch) -> None:
monkeypatch.setattr("backend.apps.workflows.cloud.run_files.downloads_root", lambda: str(tmp_path))
workflow = Workflow(title="W")
run = p_run(files=[{"id": "f1", "path": "summary.md", "size_bytes": 5}])
async def p_boom(*args, **kwargs):
from backend.apps.workflows.cloud.client import CloudUnreachable
raise CloudUnreachable("offline")
monkeypatch.setattr("backend.apps.workflows.cloud.run_files.cloud.download_run_file", p_boom)
import asyncio
asyncio.run(fetch_missing("hosted-1", workflow, [run]))
assert described(workflow, run)[0].local_path is None
def test_a_downloaded_file_is_written_whole_or_not_at_all(tmp_path, monkeypatch) -> None:
monkeypatch.setattr("backend.apps.workflows.cloud.run_files.downloads_root", lambda: str(tmp_path))
workflow = Workflow(title="W")
run = p_run(files=[{"id": "f1", "path": "notes/summary.md", "size_bytes": 5}])
async def p_ok(*args, **kwargs):
return b"hello"
monkeypatch.setattr("backend.apps.workflows.cloud.run_files.cloud.download_run_file", p_ok)
import asyncio
asyncio.run(fetch_missing("hosted-1", workflow, [run]))
target = local_path_for(run_folder(workflow, run), run.files[0])
with open(target, "rb") as handle:
assert handle.read() == b"hello"
# The .part file is the whole point of the atomic rename: no half file is ever visible.
assert not os.path.exists(f"{target}.part")
+5 -3
View File
@@ -22,7 +22,7 @@
"max-file-lines-exceptions": "Grandfather list of pre-existing >300-line files (existing debt, not new). Paths updated after the folder-tree restructure moved several of them. The two manager/prompt/* entries are from the agent_manager decomposition: prompt_context.py aggregates the system-prompt context builders and attachments.py is one cohesive 230-line attachment resolver; both are single-responsibility and a few lines over, not splittable without an artificial seam.",
"max-folder-items-exceptions": "Exact-path allow for folders intentionally over the cap. The rule trips at >7 (7 items is fine, the 8th tips it), so only genuinely 8+ folders are listed. backend/ and backend/apps are FastAPI feature-package registries (each child is an app mounted in main.py); agents/ aggregates agent subsystems; agents/manager/ is the agent_manager god-object decomposition (cohesive AgentManager mixins + standalone run helpers + the streaming/permissions/prompt/session subtrees), conventionally flat like agents/ and core/ since its standalone helpers are heterogeneous and don't group cleanly; agents/manager/streaming and agents/manager/session are flat peer collections of one-module-per-concern handlers; core/, tools_lib/, tests/ are conventionally flat. Frontend: app/pages is the page registry, AgentChat/ChatInput/Settings-sections/Onboarding are organizational parents, and shared/state (Redux slices) plus hooks/steps/mcp-cards/Views are flat peer collections. scripts/, electron/, linter/checks/ are flat tool dirs. These replaced blanket .lintignore-max-folder-items sentinels (backend, frontend, scripts, electron, linter/checks) so the rule still catches NEW unplanned bloat everywhere else. Kept as whole-subtree sentinels on purpose: debugger/ (self-contained injected sub-tool with its own Vite GUI), webapp_template (Vite scaffold payload), and vendored mcp-bundles. 2026-07 desktop-shell additions: Dashboard canvas/cards/desktop + hooks/interaction + hooks/lifecycle, AgentChat bubbles/tool-ui, and shared/styles are flat peer collections (one component or hook per concern) that crossed 7 as the redesign surface grew. frontend/src/toolui carries a whole-subtree .lintignore: vendored tool-ui component library (pierre), same treatment as mcp-bundles. openswarm-edge/app is the edge's flat one-module-per-concern set (routing, bundles, inject, ratelimit, sandbox, and the vendored code_safety gate); it crossed 7 when the sandbox's static gate was split out to mirror the desktop file byte for byte. AgentChat/parsing joined when the narration/deliverable classifier landed: it is the same flat one-module-per-parser collection as the rest of that subtree.",
"import-cycles": "Flags RUNTIME circular imports only (SCC>1). Skips type-only imports (import type / export type) and dynamic import() since neither runs at module init, which is why the idiomatic Redux store<->hooks type cycle is not flagged. Frontend alias resolution comes from import-cycle-aliases. Zero cycles today; the check keeps it that way.",
"ruff + pyright": "Ported from Haik's linter (haik/feat/ingest). ruff is narrowed to F401/F811/F841 (unused imports/redefs/locals) and intentionally DROPS Haik's ARG001/ARG002 (unused args): our SDK-callback signatures require unused params (can_use_tool/pre_tool_hook take a `context` they don't use) and we ban the `_unused` prefix, so ARG is noise here. pyright runs Haik's existence-only config (typeCheckingMode off) with reportAttributeAccessIssue ENABLED: the AgentManager behavior classes now inherit a typing-only AgentManagerProtocol base (manager/AgentManagerProtocol.py) that declares the composed __init__ state + cross-class methods, so the checker sees self.sessions etc. from inside a mixin. pyright caught real bugs: a dangling `_conns` ref + TWO broken lazy imports (`_load_all`/`_load` from outputs.py, renamed to load_all/load in workspace_io but the import sites weren't updated App Builder workspace seeding/name-sync was silently failing in a try/except). The one grandfathered SURFACE file (handle_assistant_message) is the SDK-optional try/except-import boundary (TextBlock=object fallback defeats isinstance narrowing). Both grandfather pre-existing debt by file; the refactor surface is clean. Requires `ruff` + `pyright` on PATH (added to requirements-dev.txt); pyright's config expects the venv at backend/.venv.",
"ruff + pyright": "Ported from Haik's linter (haik/feat/ingest). ruff is narrowed to F401/F811/F841 (unused imports/redefs/locals) and intentionally DROPS Haik's ARG001/ARG002 (unused args): our SDK-callback signatures require unused params (can_use_tool/pre_tool_hook take a `context` they don't use) and we ban the `_unused` prefix, so ARG is noise here. pyright runs Haik's existence-only config (typeCheckingMode off) with reportAttributeAccessIssue ENABLED: the AgentManager behavior classes now inherit a typing-only AgentManagerProtocol base (manager/AgentManagerProtocol.py) that declares the composed __init__ state + cross-class methods, so the checker sees self.sessions etc. from inside a mixin. pyright caught real bugs: a dangling `_conns` ref + TWO broken lazy imports (`_load_all`/`_load` from outputs.py, renamed to load_all/load in workspace_io but the import sites weren't updated \u2014 App Builder workspace seeding/name-sync was silently failing in a try/except). The one grandfathered SURFACE file (handle_assistant_message) is the SDK-optional try/except-import boundary (TextBlock=object fallback defeats isinstance narrowing). Both grandfather pre-existing debt by file; the refactor surface is clean. Requires `ruff` + `pyright` on PATH (added to requirements-dev.txt); pyright's config expects the venv at backend/.venv.",
"no-underscore-names + p-private": "Convention checks ported verbatim from Haik's linter (haik/feat/ingest): no-underscore-names bans leading-underscore names (a dead-code-tooling blind spot; use p_ for private), p-private enforces that p_-prefixed names are accessed only inside their owning file/class (cross-file/class use means the name should be public). Backend Python only. The exception lists grandfather pre-existing debt that landed with the workflows/analytics forward-ports (eric's 'don't mass-migrate untouched files' rule); the agent_manager refactor surface is clean. NOTE: Haik's full linter (his branch also adds pyright + ruff and runs a different enabled set) should eventually supersede this; these two were lifted to enforce the p_ conventions on eric/dev now. browser_cookies.py is excepted for `_fields_` only: a ctypes.Structure protocol name required by the ctypes metaclass, not our naming.",
"dangling-refs": "Every *_id / *_ids field on a backend pydantic model must name the entity it points at, in backend/config/entity_references.py. A model's own primary key is spelled `id`, which never matches the suffix, and neither do words that merely END in id (uuid, grid, valid) since the underscore is required. 42 of the 74 existing fields are declared in the registry (sessions, dashboards, workflows, workflow runs, apps/outputs, workspaces); the 32 listed here are grandfathered debt, and the entry is keyed <path>::<Model>.<field> rather than by file ON PURPOSE, so a NEW id field added to an already-listed model is still caught (a file glob would exempt workflows/models.py forever, which is exactly where the next dangling pointer lands). The grandfathered set is what does not resolve against a store: renderer-owned live objects (browser_id, selected_browser_ids, selected_setting_ids), ids internal to a single record (active_branch_id, msg_id, parent_id, fork_point_message_id, compacted_through_msg_id), external protocol ids we do not own (sdk_session_id, client_message_id, connection_id, installation_id, user_id), telemetry echoes (analytics bridges), and the skill-registry / .swarm-bundle entities that have no backend store module yet. Move an entry out of this list and into the registry when its entity gets one. backend/tests/*::* is blanket-exempt: a test-local model is not a persisted entity. The registry is checked back both ways, so an entry for a deleted field, or a store whose lookup function was renamed, is an error too."
},
@@ -153,7 +153,8 @@
"frontend/src/shared/browserCommandHandler.ts",
"frontend/src/shared/state/agentsSlice.ts",
"frontend/src/shared/state/dashboardLayoutSlice.ts",
"frontend/src/shared/ws/WebSocketManager.ts"
"frontend/src/shared/ws/WebSocketManager.ts",
"backend/apps/workflows/cloud/client.py"
],
"max-folder-items": [
"backend",
@@ -194,7 +195,8 @@
"frontend/src/shared/styles",
"linter/checks",
"openswarm-edge/app",
"scripts"
"scripts",
"backend/apps/workflows/cloud"
],
"no-nested-imports": [],
"import-cycles": [],
+30 -2
View File
@@ -39,6 +39,21 @@ RUN printf '{"name":"router-stage","version":"0.0.0","private":true}\n' > packag
&& test -f node_modules/9router/app/server.js \
&& test -z "$(find node_modules/9router -name '*.node' -print -quit)"
# The App Builder's template dependencies, installed once here and shipped ALREADY EXTRACTED at
# the digest path backend/apps/outputs/view_builder_templates.py already probes. Without it the
# first CreateApp in a run pays a cold npm install against the public registry, and a run with no
# egress just fails. NOT $BUILDPLATFORM: vite pulls in a platform-specific esbuild, so this has to
# resolve on the arch the container will actually run on.
FROM node:${NODE_VERSION}-bookworm-slim AS webapp-template
WORKDIR /stage
COPY backend/apps/outputs/webapp_template/frontend/package.json ./package.json
RUN set -eux; \
npm install --no-audit --no-fund --loglevel=error --ignore-scripts; \
test -x node_modules/.bin/vite; \
digest="$(sha256sum package.json | cut -c1-12)"; \
mkdir -p "/out/${digest}"; \
mv node_modules "/out/${digest}/node_modules"
# Webpack output is architecture-independent, so this runs natively on the build host rather than under emulation.
FROM --platform=$BUILDPLATFORM node:${NODE_VERSION}-bookworm-slim AS frontend
WORKDIR /src
@@ -104,6 +119,10 @@ RUN set -eux; \
rm -rf /var/lib/apt/lists/*
COPY --from=node /usr/local/bin/node /usr/local/bin/node
# npm and npx too, not just node. They are shims into lib/node_modules, so copying the tree and
# re-linking is the only way to get them; a `node` with no `npm` is what left the App Builder
# scaffolding an app it could never install, build or serve.
COPY --from=node /usr/local/lib/node_modules /usr/local/lib/node_modules
COPY --from=pydeps /opt/pydeps /usr/local
# Numeric owner on every /app copy, because a `chown -R /app` afterwards rewrites the whole tree into a second layer and the image pays for it twice (that cost 493MB before this line existed). Numeric, not `runner`, because the user is created further down.
@@ -119,8 +138,14 @@ COPY --from=frontend --chown=10001:10001 /src/dist /app/frontend
COPY --from=uv --chown=10001:10001 /stage/uv /app/backend/uv-bin/uv
COPY --from=uv --chown=10001:10001 /stage/uvx /app/backend/uv-bin/uvx
# Also after backend/, and at the exact path bundled_extracted_modules() looks for.
COPY --from=webapp-template --chown=10001:10001 /out /app/backend/apps/outputs/webapp_template_cache
RUN set -eux; \
if ls /app/backend/.env* >/dev/null 2>&1; then echo "a dotenv reached the image; fix Dockerfile.dockerignore" >&2; exit 1; fi; \
ln -s ../lib/node_modules/npm/bin/npm-cli.js /usr/local/bin/npm; \
ln -s ../lib/node_modules/npm/bin/npx-cli.js /usr/local/bin/npx; \
npm --version >/dev/null; \
mkdir -p /app/python-env/bin; \
ln -s /usr/local/bin/python3 /app/python-env/bin/python3; \
find /app/backend -name '__pycache__' -type d -prune -exec rm -rf {} +; \
@@ -129,7 +154,8 @@ RUN set -eux; \
ln -s /data/openswarm /app/backend/data; \
mkdir -p /tmp/.X11-unix; \
chmod 1777 /tmp/.X11-unix; \
chown runner:runner /data
chown runner:runner /data; \
printf '[user]\n\tname = OpenSwarm Cloud Run\n\temail = cloud-run@openswarm.local\n[init]\n\tdefaultBranch = main\n[safe]\n\tdirectory = *\n' > /etc/gitconfig
USER runner
WORKDIR /app
@@ -144,6 +170,8 @@ ENV HOME=/home/runner \
OPENSWARM_PORT=8324 \
DATA_DIR=/data/9router \
NODE_ENV=production \
ELECTRON_BIN=/app/electron-runtime/electron
ELECTRON_BIN=/app/electron-runtime/electron \
OPENSWARM_RUN_WORKSPACE=/data/workspace \
OPENSWARM_NODE_PATH=/usr/local/bin/node
ENTRYPOINT ["python3", "-m", "runner.main"]
+44
View File
@@ -42,6 +42,50 @@ Exit codes: `0` ok, `1` runner crash, `2` bad spec, `3` credential expired on ar
`4` backend never came up, `5` workflow failed, `6` wall-clock cap hit,
`7` no Electron window ever registered.
## Files the run makes
`/data/workspace` is the agent's working directory and **the only path whose contents survive**.
It is seeded as `default_folder` before the backend boots, and the agent is told in its system
prompt that files saved there come back and everything else is destroyed.
After the workflow reaches a terminal state (including a timeout, so partial work still lands) the
runner walks that directory and POSTs each file to `callback.artifacts_url`, then sends the
terminal report. That order is load-bearing: the per-run callback token is refused once the run is
closed, so uploading afterwards would be rejected.
Caps, applied in the runner AND again at the control plane, which is the one that counts:
| Limit | Value |
| --- | --- |
| One file | 20 MB |
| One run, all files | 50 MB |
| Files per run | 40 |
Nothing is ever truncated. A file past a cap is not sent and instead arrives as a row in the
report's `files[]` carrying a written reason, so the user reads "your 512 MB render could not be
sent" rather than finding a 20 MB fragment. `.git`, `node_modules`, `__pycache__`, `.venv`,
`.claude` and the usual caches are skipped, and symlinks are never followed.
## Skills and connected apps
`skills[]` in the run spec is written to `~/.claude/skills/<id>/` before boot. This is not a
nicety: the backend registers the Skill tool only when at least one non-built-in skill exists on
disk, so a container without them has no Skill tool at all and answers from general knowledge in
the same confident voice it would use with the real thing.
`unavailable_mcp_servers[]` is **names only**. The user's MCP credentials (Slack session cookies,
Notion and GitHub access tokens, Google refresh tokens) never leave their machine, so the names go
up purely so the run's system prompt can tell the agent which apps exist and are out of reach.
`McpServerNote` forbids extra fields, so there is no shape a secret could travel in.
## What the image carries for the App Builder
`node`, `npm` and `npx`, plus the App Builder template's `node_modules` pre-installed at the digest
path `bundled_extracted_modules()` probes. Without npm, `CreateApp` scaffolded an app that could
never install, build or serve; without the baked cache, the first `CreateApp` in a run would pay a
cold registry install. `git` also carries a system identity (`/etc/gitconfig`), so a workflow that
commits does not die on "Author identity unknown".
## The renderer
OpenSwarm's browser tier is not an HTTP client. Element serialization and every click,
+46 -6
View File
@@ -18,10 +18,12 @@ from typeguard import typechecked
from runner.boot.backend_process import BackendProcess, BackendUnavailable, start_backend, stop_backend
from runner.boot.renderer_process import RendererProcess, RendererUnavailable, start_renderer, stop_renderer
from runner.report import RunReport, send_report
from runner.results.deliverables import collect
from runner.results.report import RunReport, deliver_files, send_report
from runner.run_spec import CLOUD_RUN_DASHBOARD_ID, CallbackTarget, InvalidRunSpec, RunSpec, load_run_spec
from runner.seed.data_root import seed_data_root
from runner.seed.router_credentials import write_router_db
from runner.seed.skills import write_skills
from runner.workflow_run import RunOutcome, RunProgress, WorkflowRunFailed, execute_workflow
EXIT_OK = 0
@@ -37,9 +39,13 @@ DEFAULT_APP_ROOT = "/app"
DEFAULT_FRONTEND_DIR = "/app/frontend"
DEFAULT_DATA_ROOT = "/data/openswarm"
DEFAULT_ROUTER_DATA_DIR = "/data/9router"
# The agent's own folder, and the only place on this machine whose contents come home.
DEFAULT_RUN_WORKSPACE = "/data/workspace"
DEFAULT_PORT = 8324
# Slack between the soft deadline (stop the run, report it) and the hard one (kill the process).
REPORT_GRACE_SECONDS = 90.0
# Has to cover the file upload as well as the report's retries, so it is minutes, not seconds; the
# control plane's own kill sits further out again (dispatch.ts MACHINE_GRACE_MS).
REPORT_GRACE_SECONDS = 240.0
# Ceiling the control plane cannot raise. A cap a caller can override is not a cap.
MAX_RUN_SECONDS_ENV = "RUNNER_MAX_RUN_SECONDS"
DEFAULT_MAX_RUN_SECONDS = 1800
@@ -92,8 +98,25 @@ def arm_hard_stop(seconds: float) -> None:
@typechecked
def p_fail(spec: Optional[RunSpec], status: str, message: str, code: int) -> int:
def p_fail(
spec: Optional[RunSpec],
status: str,
message: str,
code: int,
workspace: Optional[str] = None,
) -> int:
"""Report a failure, handing over anything the run managed to make first.
`workspace` is passed only where the agent actually ran: a workflow that died on step 3 may
have written a perfectly good report on step 1, and losing it because a later step threw is
exactly the "the file died with the machine" problem this whole path exists to fix.
"""
logger.error("%s: %s", status, message)
files = (
deliver_files(spec.callback, workspace, collect(workspace))
if spec is not None and workspace is not None
else []
)
send_report(
spec.callback if spec else None,
RunReport(
@@ -102,6 +125,7 @@ def p_fail(spec: Optional[RunSpec], status: str, message: str, code: int) -> int
status=status,
exit_code=code,
error=message,
files=files,
),
)
return code
@@ -133,10 +157,12 @@ def p_run(spec: RunSpec, deadline: float) -> int:
app_root = os.environ.get("OPENSWARM_APP_ROOT", DEFAULT_APP_ROOT)
data_root = os.environ.get("OPENSWARM_DATA_ROOT", DEFAULT_DATA_ROOT)
router_data_dir = os.environ.get("DATA_DIR", DEFAULT_ROUTER_DATA_DIR)
workspace = os.environ.get("OPENSWARM_RUN_WORKSPACE", DEFAULT_RUN_WORKSPACE)
port = int(os.environ.get("OPENSWARM_PORT", str(DEFAULT_PORT)))
write_router_db(router_data_dir, spec.credentials, now)
seed_data_root(data_root, spec)
seed_data_root(data_root, workspace, spec)
write_skills(os.path.expanduser("~"), spec.skills)
logger.info("seeded data root %s and router db in %s", data_root, router_data_dir)
backend: Optional[BackendProcess] = None
@@ -173,11 +199,15 @@ def p_run(spec: RunSpec, deadline: float) -> int:
# confident wrong answer, which is worse than no answer.
return p_fail(spec, "failure", str(exc), EXIT_RENDERER_UNAVAILABLE)
except WorkflowRunFailed as exc:
return p_fail(spec, "failure", str(exc), EXIT_WORKFLOW_FAILED)
return p_fail(spec, "failure", str(exc), EXIT_WORKFLOW_FAILED, workspace)
finally:
stop_renderer(renderer)
stop_backend(process)
# Files before the terminal report, always: the callback token is refused the moment the run
# is closed, so this is the only order in which both the files and the receipt can land.
files = deliver_files(spec.callback, workspace, collect(workspace))
code = p_exit_code_for(outcome)
logger.info("run %s finished as %s (exit %d)", spec.run_id, outcome.status, code)
send_report(spec.callback, RunReport(
@@ -190,6 +220,7 @@ def p_run(spec: RunSpec, deadline: float) -> int:
answer=outcome.answer,
transcript=outcome.transcript,
system_notices=outcome.system_notices,
files=files,
))
return code
@@ -213,7 +244,16 @@ def main() -> int:
return p_run(spec, deadline)
except Exception as exc:
logger.exception("runner crashed")
return p_fail(spec, "failure", f"runner crashed: {exc}", EXIT_INTERNAL)
# Re-uploading a file the successful path already sent is harmless: the control plane keys
# a run's files on their path, so a second delivery overwrites one row rather than billing
# the budget twice.
return p_fail(
spec,
"failure",
f"runner crashed: {exc}",
EXIT_INTERNAL,
os.environ.get("OPENSWARM_RUN_WORKSPACE", DEFAULT_RUN_WORKSPACE),
)
if __name__ == "__main__":
-69
View File
@@ -1,69 +0,0 @@
"""Tell the control plane what happened. The terminal report is the run's only receipt."""
import logging
import time
from typing import List, Literal, Optional
import httpx
from pydantic import BaseModel, ConfigDict, Field
from typeguard import typechecked
from runner.run_spec import CallbackTarget
logger = logging.getLogger(__name__)
TERMINAL_ATTEMPTS = 5
TERMINAL_BACKOFF_SECONDS = 2.0
REQUEST_TIMEOUT_SECONDS = 15.0
class RunReport(BaseModel):
model_config = ConfigDict(validate_assignment=True)
run_id: str
phase: Literal["started", "heartbeat", "finished"]
status: str
exit_code: Optional[int] = None
error: Optional[str] = None
cost_usd: float = 0.0
active_step_idx: Optional[int] = None
last_tool_label: Optional[str] = None
answer: str = ""
transcript: str = ""
# The backend calls a run "success" even when the provider rejected the token; these are how the control plane sees that.
system_notices: List[str] = Field(default_factory=list)
@typechecked
def p_post_once(callback: CallbackTarget, report: RunReport) -> bool:
try:
with httpx.Client(timeout=REQUEST_TIMEOUT_SECONDS) as client:
response = client.post(
callback.url,
headers={"Authorization": f"Bearer {callback.token}"},
json=report.model_dump(mode="json"),
)
if response.status_code < 300:
return True
logger.warning("report %s rejected with HTTP %s", report.phase, response.status_code)
return False
except httpx.HTTPError as exc:
logger.warning("report %s failed to send: %s", report.phase, exc)
return False
@typechecked
def send_report(callback: Optional[CallbackTarget], report: RunReport) -> bool:
"""Post a report. Terminal reports retry; heartbeats get one shot and are never retried."""
if callback is None:
logger.info("no callback configured; %s report kept local: %s", report.phase, report.status)
return True
if report.phase != "finished":
return p_post_once(callback, report)
for attempt in range(TERMINAL_ATTEMPTS):
if p_post_once(callback, report):
return True
if attempt + 1 < TERMINAL_ATTEMPTS:
time.sleep(TERMINAL_BACKOFF_SECONDS * (attempt + 1))
logger.error("terminal report for run %s never landed after %d attempts", report.run_id, TERMINAL_ATTEMPTS)
return False
@@ -0,0 +1,193 @@
"""What the run made, and what of it is allowed to come home.
A cloud run's machine is destroyed the moment it exits, so a file it wrote is gone
unless something carries it out. This module is the "what": it walks the one
directory a run is given as its working folder and decides, per file, deliver or
refuse. The "how" (handing bytes to the control plane) lives in runner.results.report.
Refusing loudly is the whole point of the caps. A user who asked for a video and
got a 20MB fragment of one is worse off than a user who was told the video was too
big, so nothing here ever truncates a file: it either arrives whole or it arrives
as a sentence explaining why it did not.
"""
import hashlib
import logging
import os
from typing import List, Optional, Tuple
from pydantic import BaseModel, ConfigDict, Field
from typeguard import typechecked
logger = logging.getLogger(__name__)
# Per-file ceiling. Deliverables are reports, spreadsheets, charts and small archives; a run that
# produces something bigger is doing a different job than this pipe was built for.
MAX_FILE_BYTES = 20 * 1024 * 1024
# Per-run ceiling, enforced in walk order so the first files still arrive when a later one blows it.
MAX_TOTAL_BYTES = 50 * 1024 * 1024
MAX_FILES = 40
# Longest path we will accept, so a deep tree cannot produce a name no filesystem will take back.
MAX_RELATIVE_PATH_CHARS = 180
# Machinery, not deliverables. Everything here is either regenerable (dependencies, caches,
# compiled bytecode) or the run's own plumbing, and shipping it would blow the file budget on
# things nobody asked for.
EXCLUDED_DIRS = frozenset({
".git",
".claude",
"node_modules",
"__pycache__",
".venv",
"venv",
".pytest_cache",
".ruff_cache",
".mypy_cache",
".cache",
".npm",
})
EXCLUDED_NAMES = frozenset({".DS_Store", ".gitignore", ".gitkeep"})
READ_CHUNK_BYTES = 1024 * 1024
class Deliverable(BaseModel):
"""One file that fits, addressed by its path relative to the run's workspace."""
model_config = ConfigDict(validate_assignment=True)
path: str
size_bytes: int
sha256: str
class Refused(BaseModel):
"""One file that does not come home, and the sentence the user gets instead."""
model_config = ConfigDict(validate_assignment=True)
path: str
size_bytes: int
reason: str
class Harvest(BaseModel):
model_config = ConfigDict(validate_assignment=True)
files: List[Deliverable] = Field(default_factory=list)
refused: List[Refused] = Field(default_factory=list)
@typechecked
def total_bytes(self) -> int:
return sum(item.size_bytes for item in self.files)
@typechecked
def human_bytes(count: int) -> str:
if count < 1024:
return f"{count} B"
if count < 1024 * 1024:
return f"{count / 1024:.0f} KB"
if count < 1024 * 1024 * 1024:
return f"{count / (1024 * 1024):.1f} MB"
return f"{count / (1024 * 1024 * 1024):.1f} GB"
@typechecked
def p_digest(path: str) -> Optional[str]:
"""sha256, streamed. None when the file went away mid-walk, which is not an error."""
digest = hashlib.sha256()
try:
with open(path, "rb") as handle:
while True:
chunk = handle.read(READ_CHUNK_BYTES)
if not chunk:
break
digest.update(chunk)
except OSError as exc:
logger.warning("could not read %s while harvesting: %s", path, exc)
return None
return digest.hexdigest()
@typechecked
def p_walk(workspace: str) -> List[Tuple[str, int]]:
"""Every candidate file under the workspace as (relative path, size), sorted for determinism."""
found: List[Tuple[str, int]] = []
for directory, subdirs, filenames in os.walk(workspace):
subdirs[:] = sorted(name for name in subdirs if name not in EXCLUDED_DIRS)
for filename in sorted(filenames):
if filename in EXCLUDED_NAMES:
continue
absolute = os.path.join(directory, filename)
# Symlinks are not followed: a run that linked to /etc/passwd must not exfiltrate it.
if os.path.islink(absolute) or not os.path.isfile(absolute):
continue
try:
size = os.path.getsize(absolute)
except OSError:
continue
found.append((os.path.relpath(absolute, workspace), size))
return found
@typechecked
def collect(workspace: str) -> Harvest:
"""Decide, for every file in the run's workspace, whether it comes home."""
harvest = Harvest()
if not os.path.isdir(workspace):
return harvest
running_total = 0
for relative, size in p_walk(workspace):
if len(relative) > MAX_RELATIVE_PATH_CHARS:
harvest.refused.append(Refused(
path=relative[:MAX_RELATIVE_PATH_CHARS] + "...",
size_bytes=size,
reason="its path is too long to save anywhere",
))
continue
if size == 0:
continue
if size > MAX_FILE_BYTES:
harvest.refused.append(Refused(
path=relative,
size_bytes=size,
reason=(
f"it is {human_bytes(size)} and a single cloud-run file cannot exceed "
f"{human_bytes(MAX_FILE_BYTES)}"
),
))
continue
if len(harvest.files) >= MAX_FILES:
harvest.refused.append(Refused(
path=relative,
size_bytes=size,
reason=f"this run already produced the maximum of {MAX_FILES} files",
))
continue
if running_total + size > MAX_TOTAL_BYTES:
harvest.refused.append(Refused(
path=relative,
size_bytes=size,
reason=(
f"the run's files already total {human_bytes(running_total)} and the limit "
f"is {human_bytes(MAX_TOTAL_BYTES)}"
),
))
continue
digest = p_digest(os.path.join(workspace, relative))
if digest is None:
harvest.refused.append(Refused(path=relative, size_bytes=size, reason="it could not be read"))
continue
harvest.files.append(Deliverable(path=relative, size_bytes=size, sha256=digest))
running_total += size
logger.info(
"harvested %d file(s) totalling %s from %s, refused %d",
len(harvest.files),
human_bytes(running_total),
workspace,
len(harvest.refused),
)
return harvest
+168
View File
@@ -0,0 +1,168 @@
"""Tell the control plane what happened, and hand it whatever the run made.
The terminal report is the run's only receipt. Files go up BEFORE it, on purpose:
the per-run callback token stops working the instant the run reaches a terminal
state, so "upload, then close" is the only order in which both can succeed, and it
means the window for writing files to a run closes exactly when the run does.
"""
import logging
import os
import time
from typing import List, Literal, Optional
from urllib.parse import quote
import httpx
from pydantic import BaseModel, ConfigDict, Field
from typeguard import typechecked
from runner.results.deliverables import Harvest, human_bytes
from runner.run_spec import CallbackTarget
logger = logging.getLogger(__name__)
TERMINAL_ATTEMPTS = 5
TERMINAL_BACKOFF_SECONDS = 2.0
REQUEST_TIMEOUT_SECONDS = 15.0
# One file, one shot, generous: 20MB over a cold uplink is slower than any report.
UPLOAD_TIMEOUT_SECONDS = 120.0
FILE_PATH_HEADER = "X-Openswarm-File-Path"
FILE_SHA256_HEADER = "X-Openswarm-File-Sha256"
class ReportedFile(BaseModel):
"""One file the run produced, delivered or not, always named."""
model_config = ConfigDict(validate_assignment=True)
path: str
size_bytes: int
delivered: bool
# Present only when delivered is False. Written for a human, because it is shown to one.
reason: Optional[str] = None
class RunReport(BaseModel):
model_config = ConfigDict(validate_assignment=True)
run_id: str
phase: Literal["started", "heartbeat", "finished"]
status: str
exit_code: Optional[int] = None
error: Optional[str] = None
cost_usd: float = 0.0
active_step_idx: Optional[int] = None
last_tool_label: Optional[str] = None
answer: str = ""
transcript: str = ""
# The backend calls a run "success" even when the provider rejected the token; these are how the control plane sees that.
system_notices: List[str] = Field(default_factory=list)
# Every file the run made, including the ones that were too big to send. A deliverable that
# silently vanished is the failure this list exists to make impossible.
files: List[ReportedFile] = Field(default_factory=list)
@typechecked
def p_post_once(callback: CallbackTarget, report: RunReport) -> bool:
try:
with httpx.Client(timeout=REQUEST_TIMEOUT_SECONDS) as client:
response = client.post(
callback.url,
headers={"Authorization": f"Bearer {callback.token}"},
json=report.model_dump(mode="json"),
)
if response.status_code < 300:
return True
logger.warning("report %s rejected with HTTP %s", report.phase, response.status_code)
return False
except httpx.HTTPError as exc:
logger.warning("report %s failed to send: %s", report.phase, exc)
return False
@typechecked
def send_report(callback: Optional[CallbackTarget], report: RunReport) -> bool:
"""Post a report. Terminal reports retry; heartbeats get one shot and are never retried."""
if callback is None:
logger.info("no callback configured; %s report kept local: %s", report.phase, report.status)
return True
if report.phase != "finished":
return p_post_once(callback, report)
for attempt in range(TERMINAL_ATTEMPTS):
if p_post_once(callback, report):
return True
if attempt + 1 < TERMINAL_ATTEMPTS:
time.sleep(TERMINAL_BACKOFF_SECONDS * (attempt + 1))
logger.error("terminal report for run %s never landed after %d attempts", report.run_id, TERMINAL_ATTEMPTS)
return False
@typechecked
def p_upload_one(client: httpx.Client, callback: CallbackTarget, workspace: str, relative: str, sha256: str) -> Optional[str]:
"""Push one file. Returns None on success, or the sentence explaining the failure."""
try:
with open(os.path.join(workspace, relative), "rb") as handle:
response = client.post(
str(callback.artifacts_url),
headers={
"Authorization": f"Bearer {callback.token}",
"Content-Type": "application/octet-stream",
FILE_PATH_HEADER: quote(relative, safe="/"),
FILE_SHA256_HEADER: sha256,
},
content=handle.read(),
)
except OSError as exc:
return f"it could not be read back off disk ({exc.strerror or exc})"
except httpx.HTTPError as exc:
return f"the upload did not complete ({type(exc).__name__})"
if response.status_code < 300:
return None
# The control plane refuses with prose it wrote for the user; keep its words rather than ours.
detail = (response.text or "").strip()
try:
body = response.json()
if isinstance(body, dict) and isinstance(body.get("error"), str):
detail = body["error"]
except ValueError:
pass
return detail[:300] if detail else f"the storage service answered HTTP {response.status_code}"
@typechecked
def deliver_files(callback: Optional[CallbackTarget], workspace: str, harvest: Harvest) -> List[ReportedFile]:
"""Hand every deliverable to the control plane and report honestly on each one.
Never raises and never fails the run: a workflow whose answer is good and whose
attachment did not make it should still deliver the answer, with the miss stated.
"""
reported = [
ReportedFile(path=item.path, size_bytes=item.size_bytes, delivered=False, reason=item.reason)
for item in harvest.refused
]
if not harvest.files:
return reported
if callback is None or not callback.artifacts_url:
for item in harvest.files:
reported.append(ReportedFile(
path=item.path,
size_bytes=item.size_bytes,
delivered=False,
reason="this run had nowhere to send files, so it kept them on the machine",
))
return reported
with httpx.Client(timeout=UPLOAD_TIMEOUT_SECONDS) as client:
for item in harvest.files:
failure = p_upload_one(client, callback, workspace, item.path, item.sha256)
if failure is None:
logger.info("delivered %s (%s)", item.path, human_bytes(item.size_bytes))
else:
logger.warning("could not deliver %s: %s", item.path, failure)
reported.append(ReportedFile(
path=item.path,
size_bytes=item.size_bytes,
delivered=failure is None,
reason=failure,
))
return reported
+62
View File
@@ -27,6 +27,11 @@ CLOUD_RUN_DASHBOARD_ID = "cloud-run"
# Headroom the access token must still have on arrival. The control plane refreshes right before dispatch; anything thinner than this means its clock or its queue is broken, and we must not paper over that by refreshing ourselves.
MIN_TOKEN_LIFETIME = timedelta(minutes=2)
# A skill is prose plus the odd small script. These bounds keep a run spec from becoming a file
# transfer, and are enforced here so an oversized one dies before a machine is billed for it.
MAX_SKILLS = 60
MAX_SKILL_FILE_CHARS = 200_000
class InvalidRunSpec(ValueError):
"""The control plane handed us something we refuse to run."""
@@ -82,6 +87,59 @@ class CallbackTarget(BaseModel):
url: str = Field(min_length=1)
token: str = Field(min_length=1)
heartbeat_seconds: int = Field(default=30, ge=5, le=300)
# Where the run's files go. Named outright rather than derived from `url`, so a control plane
# that cannot accept files says so by leaving it out instead of being guessed at.
artifacts_url: Optional[str] = None
class SkillFile(BaseModel):
"""One file inside a skill folder. Text only: a skill is prose plus small scripts."""
model_config = ConfigDict(validate_assignment=True, extra="forbid")
path: str = Field(min_length=1, max_length=180)
text: str = Field(max_length=MAX_SKILL_FILE_CHARS)
@model_validator(mode="after")
def p_reject_escaping_path(self) -> "SkillFile":
parts = self.path.split("/")
if self.path.startswith("/") or "\\" in self.path or any(p in ("", ".", "..") for p in parts):
raise ValueError(f"skill file path {self.path!r} is not a plain relative path")
return self
class SkillPayload(BaseModel):
"""One of the user's skills, carried up so the agent has the same know-how it has at home.
Skills are the user's own writing, not credentials. Nothing in here is a token, and the
control plane never learns anything from it that it could spend.
"""
model_config = ConfigDict(validate_assignment=True, extra="forbid")
# Also the folder name under ~/.claude/skills, so it has to survive being a directory.
id: str = Field(min_length=1, max_length=80, pattern=r"^[A-Za-z0-9][A-Za-z0-9_-]*$")
files: List[SkillFile] = Field(min_length=1)
@model_validator(mode="after")
def p_require_skill_md(self) -> "SkillPayload":
if not any(file.path == "SKILL.md" for file in self.files):
raise ValueError(f"skill {self.id!r} has no SKILL.md, so nothing would ever load it")
return self
class McpServerNote(BaseModel):
"""A server the user has connected at home, named so the agent can say it cannot reach it.
Deliberately carries NO transport and NO secret. The user's MCP credentials (Slack session
cookies, Notion and GitHub access tokens, Google refresh tokens) stay on their laptop, so this
is a list of names and nothing else. Its whole job is to stop the agent quietly answering
"update my Notion" from general knowledge because it never knew Notion existed.
"""
model_config = ConfigDict(validate_assignment=True, extra="forbid")
name: str = Field(min_length=1, max_length=120)
class RunSpec(BaseModel):
@@ -91,6 +149,10 @@ class RunSpec(BaseModel):
workflow: Workflow
credentials: List[ProviderCredential] = Field(min_length=1)
callback: Optional[CallbackTarget] = None
# The user's skills, shipped so a cloud run is as capable as the same workflow at home.
skills: List[SkillPayload] = Field(default_factory=list, max_length=MAX_SKILLS)
# Names only. See McpServerNote for why there is no config here.
unavailable_mcp_servers: List[McpServerNote] = Field(default_factory=list, max_length=100)
# Hard wall-clock ceiling. Fly bills by machine-second, so an agent that wedges must cost a bounded amount.
max_run_seconds: int = Field(default=1800, ge=60, le=7200)
# Boot Electron under a virtual display so browser steps work. On by default: parity is the
+49 -4
View File
@@ -29,6 +29,17 @@ API_KEY_SETTINGS_FIELD: Dict[str, str] = {
# One synthetic id for every cloud run. Without it each ephemeral container mints a fresh uuid and analytics sees a brand-new "install" per run.
CLOUD_RUNNER_INSTALLATION_ID = "openswarm-cloud-runner"
# Told to the agent in as many words, because it cannot find this out any other way and the
# consequence of not knowing is a report written to a folder that is deleted minutes later.
DELIVERY_NOTE = (
"You are running in the OpenSwarm cloud on a throwaway machine. Your working directory is "
"the ONLY place that survives: every file you save there is delivered back to the user, and "
"everything else on this machine is destroyed the moment this run ends. So when a task asks "
"for a document, spreadsheet, image or archive, write it to a plainly named file in your "
"working directory rather than only pasting it into your reply. Do not write deliverables to "
"/tmp or to your home directory; they will not come back."
)
@typechecked
def p_write_json(path: str, payload: Any) -> None:
@@ -48,13 +59,43 @@ def p_write_json(path: str, payload: Any) -> None:
@typechecked
def settings_for_run(spec: RunSpec) -> AppSettings:
"""The AppSettings a cloud run needs: this workflow's model, this run's keys, no telemetry."""
def unavailable_apps_note(spec: RunSpec) -> str:
"""Name the user's connected apps this run cannot reach, so silence is not mistaken for absence.
Their MCP credentials never leave the laptop, so the servers are not here and never will be
mid-run. Without this sentence the agent has no way to know the app exists, and "update my
Notion" comes back as a confident paragraph about Notion rather than an admission.
"""
names = [server.name for server in spec.unavailable_mcp_servers]
if not names:
return ""
return (
"These apps are connected on the user's own computer but NOT reachable from this cloud "
f"run, because their sign-in details stay on that computer: {', '.join(sorted(names))}. "
"If a task needs one of them, say plainly that it cannot be done from a cloud run and "
"that it has to run on their machine. Never guess at, invent, or describe from memory "
"what one of those apps contains."
)
@typechecked
def settings_for_run(spec: RunSpec, workspace: str) -> AppSettings:
"""The AppSettings a cloud run needs: this workflow's model, this run's keys, no telemetry.
`default_folder` is what makes the run's files findable afterwards. Left unset, the agent
falls back to $HOME and the launcher reroutes it into a per-session scratch directory whose
name nothing outside the backend can predict, so the harvest would have nowhere to look.
"""
settings = AppSettings()
settings.default_model = spec.workflow.model
settings.connection_mode = "own_key"
settings.analytics_opt_in = False
settings.installation_id = CLOUD_RUNNER_INSTALLATION_ID
settings.default_folder = workspace
additions = [DELIVERY_NOTE, unavailable_apps_note(spec)]
settings.default_system_prompt = "\n\n".join(
part for part in [settings.default_system_prompt or "", *additions] if part
).strip()
for credential in spec.credentials:
if credential.auth_type != "api_key":
continue
@@ -69,13 +110,17 @@ def settings_for_run(spec: RunSpec) -> AppSettings:
@typechecked
def seed_data_root(data_root: str, spec: RunSpec) -> None:
def seed_data_root(data_root: str, workspace: str, spec: RunSpec) -> None:
"""Write the workflow, settings and dashboard records the backend will read at boot.
The dashboard exists so the Electron window has somewhere to land and browser cards have
somewhere to render. Writing it here rather than letting the backend's first-boot migration
invent one keeps its id knowable before anything has started.
The workspace sits OUTSIDE the data root deliberately: it is the agent's own folder, and a
Glob or Grep run inside it should not sweep up the settings file its API keys live in.
"""
os.makedirs(workspace, mode=0o700, exist_ok=True)
workflow = spec.workflow_for_disk()
p_write_json(
os.path.join(data_root, "workflows", f"{workflow.id}.json"),
@@ -83,7 +128,7 @@ def seed_data_root(data_root: str, spec: RunSpec) -> None:
)
p_write_json(
os.path.join(data_root, "settings", "settings.json"),
settings_for_run(spec).model_dump(mode="json"),
settings_for_run(spec, workspace).model_dump(mode="json"),
)
dashboard = Dashboard(id=CLOUD_RUN_DASHBOARD_ID, name=spec.workflow.title or "Cloud run")
p_write_json(
+52
View File
@@ -0,0 +1,52 @@
"""Lay the user's skills down where the backend looks for them, before it boots.
backend/apps/skills/skills.py hardwires SKILLS_DIR to ~/.claude/skills and gates the whole
Skill tool on at least one non-built-in skill existing there. A container that ships without
them does not get a degraded Skill tool, it gets no Skill tool at all, and the agent then
answers from general knowledge in a voice that sounds exactly as confident as the real thing.
Written pre-boot for the same reason the workflow is: the skill index is read once at startup.
"""
import logging
import os
from typing import List
from typeguard import typechecked
from runner.run_spec import SkillPayload
logger = logging.getLogger(__name__)
SKILLS_DIRNAME = os.path.join(".claude", "skills")
@typechecked
def skills_dir(home: str) -> str:
return os.path.join(home, SKILLS_DIRNAME)
@typechecked
def write_skills(home: str, skills: List[SkillPayload]) -> int:
"""Write every skill folder. Returns how many landed.
Paths were already proven relative and non-escaping by SkillFile's validator; this re-checks
the joined result anyway, because the one place a path traversal is worth catching twice is
the line that actually opens the file.
"""
root = skills_dir(home)
os.makedirs(root, mode=0o700, exist_ok=True)
written = 0
for skill in skills:
folder = os.path.join(root, skill.id)
for file in skill.files:
target = os.path.abspath(os.path.join(folder, file.path))
if not target.startswith(os.path.abspath(folder) + os.sep):
raise ValueError(f"skill {skill.id!r} file {file.path!r} resolves outside its own folder")
os.makedirs(os.path.dirname(target), mode=0o700, exist_ok=True)
with open(target, "w", encoding="utf-8") as handle:
handle.write(file.text)
written += 1
if written:
logger.info("seeded %d skill(s) into %s", written, root)
return written
+174
View File
@@ -0,0 +1,174 @@
"""What the run made, what comes home, and what is refused out loud instead of silently.
The caps are the point. A truncated file is worse than a refused one, and a file that
vanishes with no sentence attached is the failure the whole list exists to prevent.
"""
import os
import pytest
from runner.results.deliverables import (
MAX_FILE_BYTES,
MAX_FILES,
MAX_TOTAL_BYTES,
collect,
human_bytes,
)
from runner.results.report import deliver_files
from runner.run_spec import CallbackTarget
def write(root, relative: str, payload: bytes) -> str:
path = os.path.join(str(root), relative)
os.makedirs(os.path.dirname(path), exist_ok=True)
with open(path, "wb") as handle:
handle.write(payload)
return path
def test_a_missing_workspace_is_an_empty_harvest_not_a_crash(tmp_path) -> None:
assert collect(str(tmp_path / "never-made")).files == []
def test_ordinary_files_are_collected_with_their_digest(tmp_path) -> None:
write(tmp_path, "report.md", b"# Digest\n")
write(tmp_path, "data/rows.csv", b"a,b\n1,2\n")
# Walk order, and it is fixed: this directory's own files first, then subdirectories in name
# order, so a run's file list does not shuffle between two identical runs.
harvest = collect(str(tmp_path))
assert [f.path for f in harvest.files] == ["report.md", "data/rows.csv"]
assert harvest.files[0].size_bytes == len(b"# Digest\n")
# A digest travels with every file, so a corrupted upload is detectable rather than assumed fine.
assert len(harvest.files[0].sha256) == 64
assert harvest.refused == []
def test_machinery_is_not_a_deliverable(tmp_path) -> None:
write(tmp_path, "report.md", b"keep me")
write(tmp_path, ".git/config", b"[core]")
write(tmp_path, "node_modules/left-pad/index.js", b"module.exports=1")
write(tmp_path, "__pycache__/x.pyc", b"\x00")
write(tmp_path, ".claude/worktrees/probe/README", b"scratch")
assert [f.path for f in collect(str(tmp_path)).files] == ["report.md"]
def test_an_empty_file_is_neither_delivered_nor_complained_about(tmp_path) -> None:
write(tmp_path, "touched.txt", b"")
harvest = collect(str(tmp_path))
assert harvest.files == []
assert harvest.refused == []
def test_a_file_over_the_cap_is_refused_whole_and_says_why(tmp_path) -> None:
write(tmp_path, "render.mp4", b"x" * (MAX_FILE_BYTES + 1))
write(tmp_path, "notes.md", b"still fine")
harvest = collect(str(tmp_path))
assert [f.path for f in harvest.files] == ["notes.md"]
assert len(harvest.refused) == 1
assert harvest.refused[0].path == "render.mp4"
assert "cannot exceed" in harvest.refused[0].reason
# Never a fragment: an over-sized file is absent, not shortened.
assert all(f.path != "render.mp4" for f in harvest.files)
def test_the_run_total_stops_collecting_but_keeps_what_already_fit(tmp_path) -> None:
chunk = b"x" * MAX_FILE_BYTES
for index in range(MAX_TOTAL_BYTES // MAX_FILE_BYTES + 1):
write(tmp_path, f"blob-{index}.bin", chunk)
harvest = collect(str(tmp_path))
assert harvest.total_bytes() <= MAX_TOTAL_BYTES
assert len(harvest.files) >= 1
assert harvest.refused, "the file that blew the budget must be named, not dropped"
assert "limit is" in harvest.refused[0].reason
def test_too_many_files_refuses_the_extras_by_name(tmp_path) -> None:
for index in range(MAX_FILES + 3):
write(tmp_path, f"note-{index:03d}.txt", b"hi")
harvest = collect(str(tmp_path))
assert len(harvest.files) == MAX_FILES
assert len(harvest.refused) == 3
assert all("maximum of" in item.reason for item in harvest.refused)
def test_a_symlink_out_of_the_workspace_is_never_followed(tmp_path) -> None:
secret = tmp_path / "outside" / "id_rsa"
os.makedirs(secret.parent, exist_ok=True)
secret.write_text("PRIVATE KEY")
workspace = tmp_path / "ws"
os.makedirs(workspace, exist_ok=True)
os.symlink(str(secret), str(workspace / "borrowed.pem"))
assert collect(str(workspace)).files == []
def test_with_nowhere_to_send_files_the_run_says_so_per_file(tmp_path) -> None:
write(tmp_path, "report.md", b"the answer")
reported = deliver_files(None, str(tmp_path), collect(str(tmp_path)))
assert len(reported) == 1
assert reported[0].delivered is False
assert "nowhere to send files" in (reported[0].reason or "")
def test_a_control_plane_with_no_file_route_is_reported_not_guessed(tmp_path) -> None:
write(tmp_path, "report.md", b"the answer")
callback = CallbackTarget(url="https://cloud.test/report", token="two-party")
assert callback.artifacts_url is None
reported = deliver_files(callback, str(tmp_path), collect(str(tmp_path)))
assert reported[0].delivered is False
def test_refusals_reach_the_report_even_when_nothing_was_delivered(tmp_path) -> None:
write(tmp_path, "render.mp4", b"x" * (MAX_FILE_BYTES + 1))
reported = deliver_files(None, str(tmp_path), collect(str(tmp_path)))
assert len(reported) == 1
assert reported[0].path == "render.mp4"
assert reported[0].delivered is False
assert "cannot exceed" in (reported[0].reason or "")
@pytest.mark.parametrize(
"count,expected",
[(512, "512 B"), (2048, "2 KB"), (5 * 1024 * 1024, "5.0 MB"), (3 * 1024**3, "3.0 GB")],
)
def test_sizes_are_written_the_way_a_person_reads_them(count: int, expected: str) -> None:
assert human_bytes(count) == expected
def test_a_failed_run_still_hands_over_what_it_managed_to_make(tmp_path, monkeypatch) -> None:
"""A workflow that dies on step 3 may have written a perfectly good report on step 1."""
from runner import main
from runner.run_spec import RunSpec
write(tmp_path, "partial-report.md", b"# What I got through\n")
spec = RunSpec.model_validate({
"run_id": "cr-fail",
"workflow": {"id": "wf-1", "steps": [{"id": "s1", "text": "go"}]},
"credentials": [{"provider": "anthropic", "auth_type": "api_key", "api_key": "sk-test"}],
})
sent: list = []
monkeypatch.setattr(main, "send_report", lambda callback, report: sent.append(report) or True)
code = main.p_fail(spec, "failure", "step 3 blew up", main.EXIT_WORKFLOW_FAILED, str(tmp_path))
assert code == main.EXIT_WORKFLOW_FAILED
assert [f.path for f in sent[0].files] == ["partial-report.md"]
def test_a_failure_with_no_workspace_reports_no_files_rather_than_guessing(monkeypatch) -> None:
from runner import main
sent: list = []
monkeypatch.setattr(main, "send_report", lambda callback, report: sent.append(report) or True)
main.p_fail(None, "failure", "bad spec", main.EXIT_BAD_SPEC)
assert sent[0].files == []
+15 -5
View File
@@ -63,10 +63,11 @@ def test_the_container_never_inherits_the_schedule() -> None:
def test_seeding_writes_the_workflow_and_owner_only_settings(tmp_path) -> None:
spec = RunSpec.model_validate(spec_body())
seed_data_root(str(tmp_path), spec)
workspace = str(tmp_path / "workspace")
seed_data_root(str(tmp_path / "data"), workspace, spec)
workflow_path = tmp_path / "workflows" / "wf-1.json"
settings_path = tmp_path / "settings" / "settings.json"
workflow_path = tmp_path / "data" / "workflows" / "wf-1.json"
settings_path = tmp_path / "data" / "settings" / "settings.json"
assert json.loads(workflow_path.read_text())["schedule"]["enabled"] is False
assert stat.S_IMODE(os.stat(settings_path).st_mode) == 0o600
@@ -74,11 +75,20 @@ def test_seeding_writes_the_workflow_and_owner_only_settings(tmp_path) -> None:
assert settings["anthropic_api_key"] == "sk-test-not-real"
assert settings["default_model"] == "opus-5"
assert settings["analytics_opt_in"] is False
# The agent's folder is the deliverable folder, and it exists before the backend boots.
assert settings["default_folder"] == workspace
assert os.path.isdir(workspace)
def test_an_unmappable_api_key_provider_fails_loudly() -> None:
def test_the_agent_is_told_its_files_only_survive_from_the_workspace(tmp_path) -> None:
spec = RunSpec.model_validate(spec_body())
prompt = settings_for_run(spec, str(tmp_path)).default_system_prompt or ""
assert "delivered back to the user" in prompt
def test_an_unmappable_api_key_provider_fails_loudly(tmp_path) -> None:
spec = RunSpec.model_validate(spec_body(
credentials=[{"provider": "wat", "auth_type": "api_key", "api_key": "x"}]
))
with pytest.raises(ValueError, match="no settings field"):
settings_for_run(spec)
settings_for_run(spec, str(tmp_path))
@@ -0,0 +1,83 @@
"""The user's know-how travelling up, and their app credentials not travelling at all."""
import os
import pytest
from pydantic import ValidationError
from runner.run_spec import McpServerNote, RunSpec, SkillPayload
from runner.seed.data_root import settings_for_run, unavailable_apps_note
from runner.seed.skills import skills_dir, write_skills
COFFEE = {
"id": "coffee-ratio",
"files": [
{"path": "SKILL.md", "text": "---\nname: coffee-ratio\ndescription: house ratio\n---\n\n1:16.5\n"},
{"path": "scripts/brew.py", "text": "print('brew')\n"},
],
}
def p_spec(**overrides) -> RunSpec:
body = {
"run_id": "cr-1",
"workflow": {"id": "wf-1", "title": "Test", "steps": [{"id": "s1", "text": "go"}]},
"credentials": [{"provider": "anthropic", "auth_type": "api_key", "api_key": "sk-test"}],
}
body.update(overrides)
return RunSpec.model_validate(body)
def test_a_skill_folder_lands_where_the_backend_looks_for_it(tmp_path) -> None:
written = write_skills(str(tmp_path), [SkillPayload.model_validate(COFFEE)])
assert written == 1
root = skills_dir(str(tmp_path))
assert root.endswith(os.path.join(".claude", "skills"))
with open(os.path.join(root, "coffee-ratio", "SKILL.md"), encoding="utf-8") as handle:
assert "house ratio" in handle.read()
assert os.path.isfile(os.path.join(root, "coffee-ratio", "scripts", "brew.py"))
def test_no_skills_is_an_empty_directory_not_a_failure(tmp_path) -> None:
assert write_skills(str(tmp_path), []) == 0
assert os.path.isdir(skills_dir(str(tmp_path)))
def test_a_skill_with_no_skill_md_is_refused_before_a_machine_boots() -> None:
with pytest.raises(ValidationError, match="no SKILL.md"):
SkillPayload.model_validate({"id": "x", "files": [{"path": "README.md", "text": "hi"}]})
@pytest.mark.parametrize("bad", ["../escape.md", "/etc/passwd", "a/../../b.md", "a\\b.md", ""])
def test_a_skill_path_that_could_climb_out_is_refused(bad: str) -> None:
with pytest.raises(ValidationError):
SkillPayload.model_validate({"id": "x", "files": [{"path": bad, "text": "hi"}]})
@pytest.mark.parametrize("bad", ["../evil", "a/b", ".hidden", "has space", ""])
def test_a_skill_id_that_is_not_a_plain_folder_name_is_refused(bad: str) -> None:
with pytest.raises(ValidationError):
SkillPayload.model_validate({"id": bad, "files": [{"path": "SKILL.md", "text": "hi"}]})
def test_the_spec_cannot_carry_an_mcp_secret_at_all() -> None:
# extra="forbid" is the wall: there is no field for a token, so a payload with one dies here
# rather than landing in a container that runs the user's own prose with Bash.
with pytest.raises(ValidationError):
McpServerNote.model_validate({"name": "Notion", "access_token": "secret_abc"})
with pytest.raises(ValidationError):
McpServerNote.model_validate({"name": "Slack", "env": {"SLACK_MCP_XOXC_TOKEN": "xoxc-1"}})
def test_unreachable_apps_are_named_in_the_prompt_so_silence_is_not_mistaken_for_absence(tmp_path) -> None:
spec = p_spec(unavailable_mcp_servers=[{"name": "Notion"}, {"name": "Google Workspace"}])
note = unavailable_apps_note(spec)
assert "Notion" in note and "Google Workspace" in note
assert "cannot be done from a cloud run" in note
assert note in (settings_for_run(spec, str(tmp_path)).default_system_prompt or "")
def test_with_no_connected_apps_nothing_is_added_to_the_prompt(tmp_path) -> None:
assert unavailable_apps_note(p_spec()) == ""