From f40179360451805139e096662922191bb245a3d0 Mon Sep 17 00:00:00 2001 From: ciregenz Date: Sat, 1 Aug 2026 01:20:11 -0700 Subject: [PATCH] [eric] cloud runs: carry the user's own skills up, and name the apps the container cannot reach --- backend/apps/skills/skills.py | 8 +- backend/apps/workflows/cloud/definition.py | 15 +- backend/apps/workflows/cloud/handover.py | 5 +- .../apps/workflows/cloud/portable_context.py | 157 ++++++++++++++++++ backend/apps/workflows/cloud/status.py | 5 +- openswarm-runner/runner/seed/skills.py | 52 ++++++ openswarm-runner/tests/test_seed_skills.py | 83 +++++++++ 7 files changed, 314 insertions(+), 11 deletions(-) create mode 100644 backend/apps/workflows/cloud/portable_context.py create mode 100644 openswarm-runner/runner/seed/skills.py create mode 100644 openswarm-runner/tests/test_seed_skills.py diff --git a/backend/apps/skills/skills.py b/backend/apps/skills/skills.py index 85678095..1f0d6ff6 100644 --- a/backend/apps/skills/skills.py +++ b/backend/apps/skills/skills.py @@ -395,12 +395,12 @@ def clear_skill_dir(skill_id: str) -> None: """Empty a skill's folder before an in-place update so files removed upstream don't linger as orphans. write_folder_skill recreates the dir right after.""" import shutil - d = os.path.join(SKILLS_DIR, p_safe_slug(skill_id)) + d = os.path.join(SKILLS_DIR, safe_slug(skill_id)) if os.path.isdir(d): shutil.rmtree(d, ignore_errors=True) -def p_safe_slug(raw: str) -> str: +def safe_slug(raw: str) -> str: slug = re.sub(r"[^a-zA-Z0-9_-]+", "-", (raw or "").strip().lower()).strip("-") return slug or "skill" @@ -417,7 +417,7 @@ def unique_skill_slug(base: str) -> str: """A free slug for `base`, suffixing -2, -3, ... on collision. Lets a registry install land beside a same-named skill instead of silently overwriting the user's existing one.""" - slug = p_safe_slug(base) + slug = safe_slug(base) if not p_skill_exists(slug): return slug i = 2 @@ -432,7 +432,7 @@ def write_folder_skill(skill_id: str, files: dict[str, str], meta: dict) -> Skil zip/.swarm import. Relpaths that try to escape the skill folder (../, abs paths) are dropped, an untrusted registry archive can't write outside its own dir.""" - slug = p_safe_slug(skill_id) + slug = safe_slug(skill_id) base = os.path.join(SKILLS_DIR, slug) base_abs = os.path.abspath(base) # A folder write supersedes any legacy flat .md, so we never leave a phantom flat file shadowed by the folder (folder wins in skill_md_path). diff --git a/backend/apps/workflows/cloud/definition.py b/backend/apps/workflows/cloud/definition.py index 6f20970c..7101bab8 100644 --- a/backend/apps/workflows/cloud/definition.py +++ b/backend/apps/workflows/cloud/definition.py @@ -9,7 +9,7 @@ from __future__ import annotations import hashlib import json -from typing import Any, Dict +from typing import Any, Dict, Optional from typeguard import typechecked @@ -54,8 +54,13 @@ def cloud_definition(wf: Workflow) -> Dict[str, Any]: @typechecked -def definition_signature(definition: Dict[str, Any], schedule: Dict[str, Any]) -> str: - """Fingerprint of exactly what we last handed the cloud, schedule included, - so "your edits are not up there yet" is a fact rather than a guess.""" - payload = json.dumps({"definition": definition, "schedule": schedule}, sort_keys=True) +def definition_signature( + definition: Dict[str, Any], schedule: Dict[str, Any], context: Optional[Dict[str, Any]] = None +) -> str: + """Fingerprint of exactly what we last handed the cloud, schedule and skills included, + so "your edits are not up there yet" is a fact rather than a guess. Editing a skill a + workflow leans on is an edit to that workflow's behaviour, so it belongs in here.""" + payload = json.dumps( + {"definition": definition, "schedule": schedule, "context": context or {}}, sort_keys=True + ) return hashlib.sha256(payload.encode("utf-8")).hexdigest() diff --git a/backend/apps/workflows/cloud/handover.py b/backend/apps/workflows/cloud/handover.py index ef41fc4b..e4805abf 100644 --- a/backend/apps/workflows/cloud/handover.py +++ b/backend/apps/workflows/cloud/handover.py @@ -18,6 +18,7 @@ from typeguard import typechecked from backend.apps.workflows import scheduler, storage from backend.apps.workflows.cloud import client as cloud from backend.apps.workflows.cloud.definition import cloud_definition, definition_signature +from backend.apps.workflows.cloud.portable_context import portable_context from backend.apps.workflows.cloud.schedule import ScheduleSupported, to_cloud_schedule from backend.apps.workflows.cloud.status import epoch_to_datetime from backend.apps.workflows.models import Workflow @@ -51,12 +52,14 @@ async def hand_to_cloud(wf: Workflow, enabled: bool) -> TargetOutcome: if enabled and not scheduler.is_schedule_configured(wf.schedule): return TargetOutcome(ok=False, message="Finish setting up the schedule before choosing where it runs.") definition = cloud_definition(wf) + context = portable_context().as_body() try: hosted = await cloud.put_workflow( hosted_id=wf.cloud_workflow_id, name=wf.title or "Workflow", definition=definition, schedule=mapping.schedule, + context=context, ) if hosted.enabled != enabled: hosted = await cloud.set_enabled(hosted.id, enabled) @@ -70,7 +73,7 @@ async def hand_to_cloud(wf: Workflow, enabled: bool) -> TargetOutcome: wf.execution_target = "cloud" wf.cloud_workflow_id = hosted.id - wf.cloud_definition_signature = definition_signature(definition, mapping.schedule.model_dump()) + wf.cloud_definition_signature = definition_signature(definition, mapping.schedule.model_dump(), context) wf.schedule.enabled = enabled wf.next_run_at = epoch_to_datetime(hosted.next_run_at) if enabled else None wf.updated_at = datetime.now() diff --git a/backend/apps/workflows/cloud/portable_context.py b/backend/apps/workflows/cloud/portable_context.py new file mode 100644 index 00000000..dcf07a13 --- /dev/null +++ b/backend/apps/workflows/cloud/portable_context.py @@ -0,0 +1,157 @@ +"""What travels with a workflow besides the workflow, and the sharp line about what does not. + +Two things go up: + +* **Skills.** The user's own writing. A container without them does not get a weaker Skill tool, + it gets none at all (the backend gates the whole tool on one non-built-in skill existing), and + the agent then answers from general knowledge in exactly the same confident voice. So the + skills travel and the cloud run knows what the user's house ratio actually is. +* **The NAMES of connected apps.** Names only, so the agent can say "I cannot reach your Notion + from a cloud run" instead of inventing what is in it. + +One thing must never go up: the credentials behind those apps. A ToolDefinition's `credentials` +and `oauth_tokens` hold Slack session cookies, Notion and GitHub access tokens that do not expire, +and Google refresh tokens. The destination is an ephemeral machine executing the user's own agent +prose with Bash in it. Sending them would put a permanent, full-scope key to someone's email and +documents inside a box designed to be thrown away, so this module reads `tool.name` and nothing +else, and the runner's own RunSpec has no field that could hold a secret even if someone tried. +""" +from __future__ import annotations + +import logging +import os +from typing import Any, Dict, List + +from pydantic import BaseModel, ConfigDict, Field +from typeguard import typechecked + +logger = logging.getLogger(__name__) + +# Matches openswarm-runner/runner/run_spec.py. The runner refuses past these anyway; refusing here +# too means an oversized push is a legible error on the user's own machine, not a dead cloud run. +MAX_SKILLS = 60 +MAX_SKILL_FILE_CHARS = 200_000 +MAX_TOTAL_SKILL_CHARS = 1_000_000 +# A skill is prose and small scripts. Anything else in the folder is somebody's stray download. +PORTABLE_SUFFIXES = (".md", ".txt", ".py", ".sh", ".json", ".yaml", ".yml", ".csv", ".ts", ".js") + + +class PortableSkillFile(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + path: str + text: str + + +class PortableSkill(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + id: str + files: List[PortableSkillFile] + + +class PortableContext(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + skills: List[PortableSkill] = Field(default_factory=list) + # Names of apps connected here whose sign-in details stay here. + unavailable_mcp_servers: List[str] = Field(default_factory=list) + + @typechecked + def as_body(self) -> Dict[str, Any]: + return { + "skills": [skill.model_dump(mode="json") for skill in self.skills], + "unavailable_mcp_servers": self.unavailable_mcp_servers, + } + + +@typechecked +def p_read_text(path: str) -> str: + """The file's text, or empty when it is binary, unreadable, or too big to carry.""" + try: + if os.path.getsize(path) > MAX_SKILL_FILE_CHARS: + return "" + with open(path, "r", encoding="utf-8") as handle: + return handle.read() + except (OSError, UnicodeDecodeError): + return "" + + +@typechecked +def p_skill_files(skill: Any) -> List[PortableSkillFile]: + """One skill flattened to SKILL.md plus whatever else is in its folder. + + A legacy flat `.md` skill goes up as a folder with a SKILL.md too, so the wire has one + shape and the container has one layout. + """ + files = [PortableSkillFile(path="SKILL.md", text=skill.content)] + folder = getattr(skill, "dir_path", "") or "" + if not folder or not os.path.isdir(folder): + return files + for directory, subdirs, filenames in os.walk(folder): + subdirs[:] = sorted(name for name in subdirs if not name.startswith(".")) + for filename in sorted(filenames): + if filename == "SKILL.md" or not filename.lower().endswith(PORTABLE_SUFFIXES): + continue + absolute = os.path.join(directory, filename) + if os.path.islink(absolute): + continue + text = p_read_text(absolute) + if text: + files.append(PortableSkillFile( + path=os.path.relpath(absolute, folder).replace(os.sep, "/"), + text=text, + )) + return files + + +@typechecked +def portable_skills() -> List[PortableSkill]: + """Every skill the user wrote or installed. Built-ins are skipped: the container seeds its own.""" + from backend.apps.skills.skills import safe_slug, sync_skills + + out: List[PortableSkill] = [] + budget = MAX_TOTAL_SKILL_CHARS + for skill in sync_skills(): + if skill.built_in or len(out) >= MAX_SKILLS: + continue + # The id becomes a directory name in the container, so it has to survive being one. + slug = safe_slug(skill.id) + if not slug: + continue + files = p_skill_files(skill) + cost = sum(len(f.text) for f in files) + if cost > budget: + logger.info("skill %s not sent to the cloud: the run spec's skill budget is spent", skill.id) + continue + budget -= cost + out.append(PortableSkill(id=slug, files=files)) + return out + + +@typechecked +def unavailable_mcp_servers() -> List[str]: + """Connected apps, by name. Reads `tool.name` and deliberately nothing else.""" + from backend.apps.tools_lib.tools_lib import load_all_tools + + names: List[str] = [] + for tool in load_all_tools(): + if tool.mcp_config and tool.enabled and tool.auth_status in ("configured", "connected"): + names.append(tool.name) + return sorted(set(names)) + + +@typechecked +def portable_context() -> PortableContext: + """Everything a cloud run should know that the workflow itself does not carry.""" + try: + skills = portable_skills() + except Exception: + logger.exception("could not gather skills for the cloud; the run will have none") + skills = [] + try: + servers = unavailable_mcp_servers() + except Exception: + logger.exception("could not gather connected app names for the cloud") + servers = [] + return PortableContext(skills=skills, unavailable_mcp_servers=servers) diff --git a/backend/apps/workflows/cloud/status.py b/backend/apps/workflows/cloud/status.py index 96677731..b9c560e6 100644 --- a/backend/apps/workflows/cloud/status.py +++ b/backend/apps/workflows/cloud/status.py @@ -16,6 +16,7 @@ from typeguard import typechecked from backend.apps.workflows import storage from backend.apps.workflows.cloud import client as cloud from backend.apps.workflows.cloud.definition import cloud_definition, definition_signature +from backend.apps.workflows.cloud.portable_context import portable_context from backend.apps.workflows.cloud.schedule import ScheduleSupported, to_cloud_schedule from backend.apps.workflows.models import Workflow @@ -74,7 +75,9 @@ def current_signature(wf: Workflow) -> Optional[str]: mapping = to_cloud_schedule(wf.schedule) if not isinstance(mapping, ScheduleSupported): return None - return definition_signature(cloud_definition(wf), mapping.schedule.model_dump()) + return definition_signature( + cloud_definition(wf), mapping.schedule.model_dump(), portable_context().as_body() + ) @typechecked diff --git a/openswarm-runner/runner/seed/skills.py b/openswarm-runner/runner/seed/skills.py new file mode 100644 index 00000000..e0399b01 --- /dev/null +++ b/openswarm-runner/runner/seed/skills.py @@ -0,0 +1,52 @@ +"""Lay the user's skills down where the backend looks for them, before it boots. + +backend/apps/skills/skills.py hardwires SKILLS_DIR to ~/.claude/skills and gates the whole +Skill tool on at least one non-built-in skill existing there. A container that ships without +them does not get a degraded Skill tool, it gets no Skill tool at all, and the agent then +answers from general knowledge in a voice that sounds exactly as confident as the real thing. + +Written pre-boot for the same reason the workflow is: the skill index is read once at startup. +""" + +import logging +import os +from typing import List + +from typeguard import typechecked + +from runner.run_spec import SkillPayload + +logger = logging.getLogger(__name__) + +SKILLS_DIRNAME = os.path.join(".claude", "skills") + + +@typechecked +def skills_dir(home: str) -> str: + return os.path.join(home, SKILLS_DIRNAME) + + +@typechecked +def write_skills(home: str, skills: List[SkillPayload]) -> int: + """Write every skill folder. Returns how many landed. + + Paths were already proven relative and non-escaping by SkillFile's validator; this re-checks + the joined result anyway, because the one place a path traversal is worth catching twice is + the line that actually opens the file. + """ + root = skills_dir(home) + os.makedirs(root, mode=0o700, exist_ok=True) + written = 0 + for skill in skills: + folder = os.path.join(root, skill.id) + for file in skill.files: + target = os.path.abspath(os.path.join(folder, file.path)) + if not target.startswith(os.path.abspath(folder) + os.sep): + raise ValueError(f"skill {skill.id!r} file {file.path!r} resolves outside its own folder") + os.makedirs(os.path.dirname(target), mode=0o700, exist_ok=True) + with open(target, "w", encoding="utf-8") as handle: + handle.write(file.text) + written += 1 + if written: + logger.info("seeded %d skill(s) into %s", written, root) + return written diff --git a/openswarm-runner/tests/test_seed_skills.py b/openswarm-runner/tests/test_seed_skills.py new file mode 100644 index 00000000..a046ca34 --- /dev/null +++ b/openswarm-runner/tests/test_seed_skills.py @@ -0,0 +1,83 @@ +"""The user's know-how travelling up, and their app credentials not travelling at all.""" + +import os + +import pytest +from pydantic import ValidationError + +from runner.run_spec import McpServerNote, RunSpec, SkillPayload +from runner.seed.data_root import settings_for_run, unavailable_apps_note +from runner.seed.skills import skills_dir, write_skills + +COFFEE = { + "id": "coffee-ratio", + "files": [ + {"path": "SKILL.md", "text": "---\nname: coffee-ratio\ndescription: house ratio\n---\n\n1:16.5\n"}, + {"path": "scripts/brew.py", "text": "print('brew')\n"}, + ], +} + + +def p_spec(**overrides) -> RunSpec: + body = { + "run_id": "cr-1", + "workflow": {"id": "wf-1", "title": "Test", "steps": [{"id": "s1", "text": "go"}]}, + "credentials": [{"provider": "anthropic", "auth_type": "api_key", "api_key": "sk-test"}], + } + body.update(overrides) + return RunSpec.model_validate(body) + + +def test_a_skill_folder_lands_where_the_backend_looks_for_it(tmp_path) -> None: + written = write_skills(str(tmp_path), [SkillPayload.model_validate(COFFEE)]) + + assert written == 1 + root = skills_dir(str(tmp_path)) + assert root.endswith(os.path.join(".claude", "skills")) + with open(os.path.join(root, "coffee-ratio", "SKILL.md"), encoding="utf-8") as handle: + assert "house ratio" in handle.read() + assert os.path.isfile(os.path.join(root, "coffee-ratio", "scripts", "brew.py")) + + +def test_no_skills_is_an_empty_directory_not_a_failure(tmp_path) -> None: + assert write_skills(str(tmp_path), []) == 0 + assert os.path.isdir(skills_dir(str(tmp_path))) + + +def test_a_skill_with_no_skill_md_is_refused_before_a_machine_boots() -> None: + with pytest.raises(ValidationError, match="no SKILL.md"): + SkillPayload.model_validate({"id": "x", "files": [{"path": "README.md", "text": "hi"}]}) + + +@pytest.mark.parametrize("bad", ["../escape.md", "/etc/passwd", "a/../../b.md", "a\\b.md", ""]) +def test_a_skill_path_that_could_climb_out_is_refused(bad: str) -> None: + with pytest.raises(ValidationError): + SkillPayload.model_validate({"id": "x", "files": [{"path": bad, "text": "hi"}]}) + + +@pytest.mark.parametrize("bad", ["../evil", "a/b", ".hidden", "has space", ""]) +def test_a_skill_id_that_is_not_a_plain_folder_name_is_refused(bad: str) -> None: + with pytest.raises(ValidationError): + SkillPayload.model_validate({"id": bad, "files": [{"path": "SKILL.md", "text": "hi"}]}) + + +def test_the_spec_cannot_carry_an_mcp_secret_at_all() -> None: + # extra="forbid" is the wall: there is no field for a token, so a payload with one dies here + # rather than landing in a container that runs the user's own prose with Bash. + with pytest.raises(ValidationError): + McpServerNote.model_validate({"name": "Notion", "access_token": "secret_abc"}) + with pytest.raises(ValidationError): + McpServerNote.model_validate({"name": "Slack", "env": {"SLACK_MCP_XOXC_TOKEN": "xoxc-1"}}) + + +def test_unreachable_apps_are_named_in_the_prompt_so_silence_is_not_mistaken_for_absence(tmp_path) -> None: + spec = p_spec(unavailable_mcp_servers=[{"name": "Notion"}, {"name": "Google Workspace"}]) + note = unavailable_apps_note(spec) + + assert "Notion" in note and "Google Workspace" in note + assert "cannot be done from a cloud run" in note + assert note in (settings_for_run(spec, str(tmp_path)).default_system_prompt or "") + + +def test_with_no_connected_apps_nothing_is_added_to_the_prompt(tmp_path) -> None: + assert unavailable_apps_note(p_spec()) == ""