From 8afec84986102b63d89b65967af33ce3864b9c8a Mon Sep 17 00:00:00 2001 From: ciregenz Date: Wed, 2 Sep 2026 15:32:10 -0700 Subject: [PATCH] [eric] marketplace: the published .swarm catalog is the store tab, installed through the existing import door Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_012G8kyALnPjsA7aJFmMBq3R --- backend/apps/marketplace/__init__.py | 0 backend/apps/marketplace/catalog.py | 178 +++++++++++ backend/apps/marketplace/marketplace.py | 63 ++++ backend/apps/marketplace/package_download.py | 76 +++++ backend/apps/swarm/swarm.py | 15 +- backend/main.py | 3 +- backend/tests/test_marketplace_catalog.py | 130 ++++++++ .../pages/Directory/DirectoryPackagesTab.tsx | 233 ++++++++++++++ .../app/pages/Directory/MarketplaceBody.tsx | 20 +- .../app/pages/Directory/openMarketplace.ts | 2 +- .../Directory/packages/PackageBundleCard.tsx | 147 +++++++++ .../pages/Directory/packages/PackageCard.tsx | 118 +++++++ .../pages/Directory/packages/catalog.test.ts | 71 +++++ .../app/pages/Directory/packages/catalog.ts | 141 +++++++++ .../packages/detail/PackageBundleDialog.tsx | 229 ++++++++++++++ .../packages/detail/PackageDetails.tsx | 195 ++++++++++++ .../packages/detail/PackageDialog.tsx | 160 ++++++++++ .../packages/detail/PackageVideoSection.tsx | 211 +++++++++++++ .../packages/detail/YouTubePlayer.tsx | 292 ++++++++++++++++++ .../Directory/packages/installPackage.ts | 25 ++ .../pages/Directory/packages/notionDetails.ts | 43 +++ .../shared/state/marketplaceCatalogSlice.ts | 62 ++++ frontend/src/shared/state/store.ts | 2 + 23 files changed, 2400 insertions(+), 16 deletions(-) create mode 100644 backend/apps/marketplace/__init__.py create mode 100644 backend/apps/marketplace/catalog.py create mode 100644 backend/apps/marketplace/marketplace.py create mode 100644 backend/apps/marketplace/package_download.py create mode 100644 backend/tests/test_marketplace_catalog.py create mode 100644 frontend/src/app/pages/Directory/DirectoryPackagesTab.tsx create mode 100644 frontend/src/app/pages/Directory/packages/PackageBundleCard.tsx create mode 100644 frontend/src/app/pages/Directory/packages/PackageCard.tsx create mode 100644 frontend/src/app/pages/Directory/packages/catalog.test.ts create mode 100644 frontend/src/app/pages/Directory/packages/catalog.ts create mode 100644 frontend/src/app/pages/Directory/packages/detail/PackageBundleDialog.tsx create mode 100644 frontend/src/app/pages/Directory/packages/detail/PackageDetails.tsx create mode 100644 frontend/src/app/pages/Directory/packages/detail/PackageDialog.tsx create mode 100644 frontend/src/app/pages/Directory/packages/detail/PackageVideoSection.tsx create mode 100644 frontend/src/app/pages/Directory/packages/detail/YouTubePlayer.tsx create mode 100644 frontend/src/app/pages/Directory/packages/installPackage.ts create mode 100644 frontend/src/app/pages/Directory/packages/notionDetails.ts create mode 100644 frontend/src/shared/state/marketplaceCatalogSlice.ts diff --git a/backend/apps/marketplace/__init__.py b/backend/apps/marketplace/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/backend/apps/marketplace/catalog.py b/backend/apps/marketplace/catalog.py new file mode 100644 index 00000000..89c9761c --- /dev/null +++ b/backend/apps/marketplace/catalog.py @@ -0,0 +1,178 @@ +"""The marketplace catalog: a public Google Sheet read as CSV and normalized into listings. + +The sheet is the publisher's surface, so a package appears by adding a row and needs no OpenSwarm +deploy and no credential. Drive share links are rewritten to direct-download form because a share +link serves an HTML page, not the bundle. +""" + +import csv +import io +import os +import re +import time +import urllib.request +from typing import Dict, List, Optional + +from pydantic import BaseModel, ConfigDict, Field +from typeguard import typechecked + +# The published catalog. Overridable so a fork or a staging sheet can be pointed at without a build. +DEFAULT_SHEET_ID = "1HInP47Vj-daPYKIl02YrAbyKsX1GQ_FSPSm_YF2QIWw" +SHEET_ID_ENV = "OPENSWARM_MARKETPLACE_SHEET_ID" + +CACHE_TTL_SECONDS = 600 +FETCH_TIMEOUT_SECONDS = 15 +# A sheet is text; anything this large is a wrong URL answering, not a catalog. +MAX_SHEET_BYTES = 4 * 1024 * 1024 + +P_SHEET_ID_RE = re.compile(r"/spreadsheets/d/([A-Za-z0-9_-]+)") +P_DRIVE_ID_RE = re.compile(r"(?:/d/|id=)([A-Za-z0-9_-]{20,})") + + +class Listing(BaseModel): + """One row of the sheet. Every field is a string because a spreadsheet cell is a string, and a + publisher leaving one blank must never break the catalog for everyone else.""" + + model_config = ConfigDict(validate_assignment=True) + + id: str + title: str = "" + kind: str = "" + version: str = "" + author: str = "" + description: str = "" + tags: str = "" + download_url: str = "" + icon_url: str = "" + video_url: str = "" + size: str = "" + updated_at: str = "" + # A bundle groups already-published listings; this holds their comma-separated ids. + bundle_items: str = "" + # The publisher's long-form page, converted to our block JSON at publish time. + notion_url: str = "" + details_json: str = "" + + +class CatalogResponse(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + source: str + count: int + listings: List[Listing] = Field(default_factory=list) + error: str = "" + fetched_at: float = 0.0 + + +KNOWN_FIELDS = tuple(Listing.model_fields.keys()) + +p_cache: Optional[CatalogResponse] = None + + +@typechecked +def sheet_id() -> str: + """The configured sheet, accepting either a raw id or a pasted sheet URL.""" + raw = (os.environ.get(SHEET_ID_ENV) or "").strip() or DEFAULT_SHEET_ID + match = P_SHEET_ID_RE.search(raw) + return match.group(1) if match else raw + + +@typechecked +def csv_export_url(sid: str) -> str: + return f"https://docs.google.com/spreadsheets/d/{sid}/export?format=csv" + + +@typechecked +def normalize_download_url(url: str) -> str: + """A Drive share link serves a viewer page; the installer needs the bytes.""" + url = (url or "").strip() + if not url or "drive.google.com" not in url: + return url + match = P_DRIVE_ID_RE.search(url) + return f"https://drive.google.com/uc?export=download&id={match.group(1)}" if match else url + + +@typechecked +def normalize_video_url(url: str) -> str: + """Drive video links are stored in several hand-pasted shapes; keep one embeddable form.""" + url = (url or "").strip() + if not url or "drive.google.com" not in url or "/preview" in url: + return url + match = P_DRIVE_ID_RE.search(url) + return f"https://drive.google.com/file/d/{match.group(1)}/preview" if match else url + + +@typechecked +def normalize_video_field(raw: str) -> str: + """One cell can carry several demo videos, one URL per line, primary first.""" + lines = [normalize_video_url(line) for line in re.split(r"[\r\n]+", raw or "") if line.strip()] + return "\n".join(line for line in lines if line) + + +@typechecked +def slug_from_title(title: str) -> str: + return re.sub(r"[^a-z0-9]+", "-", title.lower()).strip("-") + + +@typechecked +def parse_csv(text: str) -> List[Listing]: + """Rows the sheet's own header names, loosely matched so a sloppy header still parses. Unknown + columns are ignored rather than rejected, so a publisher can add their own without a release.""" + listings: List[Listing] = [] + for row in csv.DictReader(io.StringIO(text)): + norm: Dict[str, str] = { + (key or "").strip().lower().replace(" ", "_"): (value or "").strip() + for key, value in row.items() + if key + } + values = {field: norm.get(field, "") for field in KNOWN_FIELDS} + if not values["id"] and not values["title"]: + continue + if not values["id"]: + values["id"] = slug_from_title(values["title"]) + if not values["id"]: + continue + values["download_url"] = normalize_download_url(values["download_url"]) + values["video_url"] = normalize_video_field(values["video_url"]) + listings.append(Listing(**values)) + return listings + + +@typechecked +def fetch_sheet_csv(url: str) -> str: + """Stdlib only: this runs on end-user machines, and the sheet is public so there is no auth.""" + request = urllib.request.Request(url, headers={"User-Agent": "OpenSwarm-Marketplace/1.0"}) + with urllib.request.urlopen(request, timeout=FETCH_TIMEOUT_SECONDS) as response: + raw = response.read(MAX_SHEET_BYTES + 1) + if len(raw) > MAX_SHEET_BYTES: + raise ValueError("catalog response is too large to be a sheet") + charset = response.headers.get_content_charset() or "utf-8" + return raw.decode(charset, errors="replace") + + +@typechecked +def load_catalog(force: bool = False) -> CatalogResponse: + """The catalog, from cache when fresh. A failed fetch serves the last good copy rather than an + empty store, and says so, because a flaky network must not look like an empty marketplace.""" + global p_cache + now = time.time() + if not force and p_cache is not None and now - p_cache.fetched_at < CACHE_TTL_SECONDS: + return p_cache + try: + listings = parse_csv(fetch_sheet_csv(csv_export_url(sheet_id()))) + except Exception as e: + if p_cache is not None: + return p_cache.model_copy(update={"source": "cache", "error": f"Could not refresh the catalog: {e}"}) + return CatalogResponse(source="empty", count=0, listings=[], error=f"Could not reach the catalog: {e}", fetched_at=now) + p_cache = CatalogResponse(source="sheet", count=len(listings), listings=listings, fetched_at=now) + return p_cache + + +@typechecked +def find_listing(listing_id: str) -> Optional[Listing]: + """Resolve an id against OUR fetched catalog; a caller never gets to name a URL to fetch.""" + wanted = (listing_id or "").strip() + for listing in load_catalog().listings: + if listing.id == wanted: + return listing + return None diff --git a/backend/apps/marketplace/marketplace.py b/backend/apps/marketplace/marketplace.py new file mode 100644 index 00000000..2987d6ea --- /dev/null +++ b/backend/apps/marketplace/marketplace.py @@ -0,0 +1,63 @@ +"""Marketplace routes: browse the published catalog, and stage a package for install. + +Install deliberately owns no writing of its own. It downloads the bundle and hands the bytes to the +same staging door a dropped .swarm goes through, so the conflict check, the secret review and the +"a skill never installs silently" rule are the ones already shipped, not a second copy that can +drift away from them. +""" + +import asyncio +import logging +from contextlib import asynccontextmanager + +from fastapi import HTTPException +from pydantic import BaseModel, ConfigDict + +from backend.apps.marketplace import catalog +from backend.apps.marketplace.package_download import ( + DownloadRefused, + download_package, + package_filename, +) +from backend.apps.swarm.models import ImportPreflightResponse +from backend.apps.swarm.swarm import stage_bundle_for_import +from backend.config.Apps import SubApp + +logger = logging.getLogger(__name__) + + +class InstallRequest(BaseModel): + model_config = ConfigDict(validate_assignment=True) + + id: str + + +@asynccontextmanager +async def marketplace_lifespan(): + yield + + +marketplace = SubApp("marketplace", marketplace_lifespan) + + +@marketplace.router.get("/listings") +async def get_listings(refresh: bool = False) -> catalog.CatalogResponse: + """The published catalog. Never raises: an unreachable sheet answers with the last good copy + and an error string, because a browse surface that 500s reads as "the marketplace is gone".""" + return await asyncio.to_thread(catalog.load_catalog, refresh) + + +@marketplace.router.post("/install/preflight") +async def install_preflight(body: InstallRequest) -> ImportPreflightResponse: + """Download the listing's bundle and stage it for the ordinary import confirm flow.""" + listing = await asyncio.to_thread(catalog.find_listing, body.id) + if listing is None: + raise HTTPException(status_code=404, detail="that package is not in the catalog any more") + if not listing.download_url: + raise HTTPException(status_code=400, detail="this listing has no package file yet") + try: + raw = await asyncio.to_thread(download_package, listing.download_url) + except DownloadRefused as e: + logger.warning("marketplace download refused for %s: %s", listing.id, e) + raise HTTPException(status_code=400, detail=str(e)) + return stage_bundle_for_import(raw, package_filename(listing.id, listing.title)) diff --git a/backend/apps/marketplace/package_download.py b/backend/apps/marketplace/package_download.py new file mode 100644 index 00000000..d101378a --- /dev/null +++ b/backend/apps/marketplace/package_download.py @@ -0,0 +1,76 @@ +"""Fetch a published .swarm, with the host allowlist that keeps this from being an SSRF hole. + +The catalog is a spreadsheet other people can edit, so a row's URL is untrusted input that would +otherwise make OUR backend fetch whatever it names, including localhost and cloud metadata. Every +hop of the redirect chain is checked, not just the first, because a permitted host may redirect. +""" + +import urllib.error +import urllib.request +from urllib.parse import urlparse + +from typeguard import typechecked + +ALLOWED_DOWNLOAD_HOSTS = ( + "drive.google.com", + "drive.usercontent.google.com", + "docs.google.com", + "github.com", + "objects.githubusercontent.com", + "raw.githubusercontent.com", +) + +DOWNLOAD_TIMEOUT_SECONDS = 60 +MAX_PACKAGE_BYTES = 200 * 1024 * 1024 + + +class DownloadRefused(Exception): + """The URL is not somewhere we are willing to fetch from.""" + + +@typechecked +def host_allowed(url: str) -> bool: + parsed = urlparse((url or "").strip()) + if parsed.scheme != "https" or not parsed.hostname: + return False + host = parsed.hostname.lower() + return any(host == allowed or host.endswith("." + allowed) for allowed in ALLOWED_DOWNLOAD_HOSTS) + + +class AllowlistRedirectHandler(urllib.request.HTTPRedirectHandler): + """Drive answers a download with a redirect, so redirects have to be followed; each new + location is re-checked so a permitted host cannot bounce us onto a private address.""" + + def redirect_request(self, req, fp, code, msg, headers, newurl): # type: ignore[override] + if not host_allowed(newurl): + raise DownloadRefused(f"refused a redirect to {urlparse(newurl).hostname or newurl}") + return super().redirect_request(req, fp, code, msg, headers, newurl) + + +@typechecked +def download_package(url: str) -> bytes: + """The bundle's bytes, or DownloadRefused. Never returns a partial or oversized body.""" + if not host_allowed(url): + raise DownloadRefused("this package is not hosted somewhere OpenSwarm will download from") + opener = urllib.request.build_opener(AllowlistRedirectHandler()) + request = urllib.request.Request(url, headers={"User-Agent": "OpenSwarm-Marketplace/1.0"}) + try: + with opener.open(request, timeout=DOWNLOAD_TIMEOUT_SECONDS) as response: + raw = response.read(MAX_PACKAGE_BYTES + 1) + except urllib.error.HTTPError as e: + raise DownloadRefused(f"the download returned {e.code}") + except DownloadRefused: + raise + except Exception as e: + raise DownloadRefused(f"the download failed: {e}") + if len(raw) > MAX_PACKAGE_BYTES: + raise DownloadRefused("the package is too large") + if not raw: + raise DownloadRefused("the download was empty") + return raw + + +@typechecked +def package_filename(listing_id: str, title: str) -> str: + base = (listing_id or title or "package").strip() or "package" + return f"{base}.swarm" diff --git a/backend/apps/swarm/swarm.py b/backend/apps/swarm/swarm.py index ceca79ca..5216456e 100644 --- a/backend/apps/swarm/swarm.py +++ b/backend/apps/swarm/swarm.py @@ -82,13 +82,14 @@ async def export_bundle(body: ExportRequest) -> Response: ) -@swarm.router.post("/import/preflight") -async def import_preflight(file: UploadFile = File(...)) -> ImportPreflightResponse: - raw = await file.read() +def stage_bundle_for_import(raw: bytes, filename: str) -> ImportPreflightResponse: + """Stage bytes for import and hand back the review. Every door that installs a bundle comes + through here (a dropped file, a marketplace package), so the conflict/secret review and the + one staging store can never diverge per door.""" if len(raw) > MAX_TOTAL_BYTES: raise HTTPException(status_code=400, detail="file is too large") try: - sandbox, manifest, warnings = closure.stage_upload(raw, file.filename or "") + sandbox, manifest, warnings = closure.stage_upload(raw, filename) conflicts = closure.detect_conflicts(sandbox, manifest) review = closure.review_bundle(sandbox, manifest) except BundleError as e: @@ -105,6 +106,12 @@ async def import_preflight(file: UploadFile = File(...)) -> ImportPreflightRespo ) +@swarm.router.post("/import/preflight") +async def import_preflight(file: UploadFile = File(...)) -> ImportPreflightResponse: + raw = await file.read() + return stage_bundle_for_import(raw, file.filename or "") + + @swarm.router.post("/import/commit") async def import_commit(body: ImportCommitRequest) -> ImportCommitResponse: entry = P_STAGING.get(body.staging_token) diff --git a/backend/main.py b/backend/main.py index 47ad0bba..13d3da71 100644 --- a/backend/main.py +++ b/backend/main.py @@ -43,6 +43,7 @@ from backend.apps.outputs.outputs import outputs from backend.apps.outputs.versions_routes import output_versions from backend.apps.dashboards.dashboards import dashboards from backend.apps.swarm.swarm import swarm +from backend.apps.marketplace.marketplace import marketplace from backend.apps.service.service import service from backend.apps.subscription.router import subscription from backend.apps.auth.router import auth @@ -60,7 +61,7 @@ from fastapi.middleware.cors import CORSMiddleware from fastapi import WebSocket, WebSocketDisconnect import json -main_app = MainApp([health, agents, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, output_versions, dashboards, swarm, service, subscription, auth, web, onboarding, voice, memory, help_app, anthropic_proxy, workflows, cloud_workflows, openai_passthrough, apps_sdk]) +main_app = MainApp([health, agents, skills, tools_lib, modes, settings, mcp_registry, skill_registry, outputs, output_versions, dashboards, swarm, marketplace, service, subscription, auth, web, onboarding, voice, memory, help_app, anthropic_proxy, workflows, cloud_workflows, openai_passthrough, apps_sdk]) app = main_app.app # Generate per-install auth token BEFORE we bind the HTTP port. By the time any request lands, the token file exists. See backend/auth.py. diff --git a/backend/tests/test_marketplace_catalog.py b/backend/tests/test_marketplace_catalog.py new file mode 100644 index 00000000..612aa6ac --- /dev/null +++ b/backend/tests/test_marketplace_catalog.py @@ -0,0 +1,130 @@ +"""The marketplace catalog and its one install door. + +Two things here are load-bearing and are asserted rather than assumed: the catalog is a spreadsheet +other people can edit, so a row's download URL is untrusted input, and install must go through the +SAME staging door a dropped .swarm uses, or the secret review and the never-install-a-skill-silently +rule would exist in one door and not the other. +""" + +import inspect + +import pytest + +from backend.apps.marketplace import catalog, marketplace +from backend.apps.marketplace.package_download import ( + DownloadRefused, + download_package, + host_allowed, +) + +SHEET = ( + "id,title,kind,version,author,description,tags,icon_url,download_url,size,updated_at,video_url,bundle_items\n" + "hello,Hello World,skill,1.0.0,Test,Says hello,demo,,https://drive.google.com/file/d/1AbCdEfGhIjKlMnOpQrStUv/view,12 KB,2026-09-01,,\n" + ",Derived Title,app,,Someone,No id column,,,https://example.com/x.swarm,,,,\n" + ",,,,,,,,,,,,\n" + "pack,Starter Pack,bundle,,Team,Two things,,,,,2026-09-02,,hello, hello ,\n" +) + + +def p_listing(rows, listing_id): + return next(row for row in rows if row.id == listing_id) + + +def test_a_drive_share_link_becomes_something_an_installer_can_fetch(): + rows = catalog.parse_csv(SHEET) + assert p_listing(rows, "hello").download_url == ( + "https://drive.google.com/uc?export=download&id=1AbCdEfGhIjKlMnOpQrStUv" + ) + + +def test_a_row_without_an_id_gets_one_from_its_title_and_a_blank_row_is_dropped(): + rows = catalog.parse_csv(SHEET) + assert [row.id for row in rows] == ["hello", "derived-title", "pack"] + + +def test_an_unknown_column_is_ignored_rather_than_rejected(): + rows = catalog.parse_csv("id,title,kind,publisher_notes\nx,X,skill,anything at all\n") + assert rows[0].id == "x" and rows[0].kind == "skill" + + +def test_several_demo_videos_in_one_cell_each_normalize_and_keep_their_order(): + raw = "https://drive.google.com/file/d/1AbCdEfGhIjKlMnOpQrStUv/view\nhttps://youtu.be/abcdefghijk" + rows = catalog.parse_csv(f"id,title,video_url\nv,V,\"{raw}\"\n") + assert rows[0].video_url.split("\n") == [ + "https://drive.google.com/file/d/1AbCdEfGhIjKlMnOpQrStUv/preview", + "https://youtu.be/abcdefghijk", + ] + + +@pytest.mark.parametrize( + "url,allowed", + [ + ("https://drive.google.com/uc?export=download&id=x", True), + ("https://drive.usercontent.google.com/download?id=x", True), + ("https://raw.githubusercontent.com/o/r/main/a.swarm", True), + # The sheet is editable by other people, so these are the shapes that must never resolve. + ("http://drive.google.com/uc?id=x", False), + ("https://drive.google.com.evil.example/a.swarm", False), + ("https://127.0.0.1/a.swarm", False), + ("https://169.254.169.254/latest/meta-data", False), + ("file:///etc/passwd", False), + ("", False), + ], +) +def test_only_hosts_we_publish_from_are_fetchable(url, allowed): + assert host_allowed(url) is allowed + + +def test_a_refused_host_never_reaches_the_network(): + with pytest.raises(DownloadRefused): + download_package("https://evil.example/package.swarm") + + +def test_a_redirect_off_the_allowlist_is_refused_mid_chain(): + from backend.apps.marketplace.package_download import AllowlistRedirectHandler + + handler = AllowlistRedirectHandler() + with pytest.raises(DownloadRefused): + handler.redirect_request(None, None, 302, "Found", {}, "https://127.0.0.1/pkg.swarm") + + +def test_an_unreachable_sheet_serves_the_last_good_catalog_instead_of_an_empty_store(monkeypatch): + monkeypatch.setattr(catalog, "fetch_sheet_csv", lambda url: SHEET) + good = catalog.load_catalog(force=True) + assert good.source == "sheet" and good.count == 3 + + def p_down(url): + raise OSError("no network") + + monkeypatch.setattr(catalog, "fetch_sheet_csv", p_down) + stale = catalog.load_catalog(force=True) + assert stale.source == "cache", "a flaky network must not look like an empty marketplace" + assert [row.id for row in stale.listings] == [row.id for row in good.listings] + assert "Could not refresh" in stale.error + + +def test_with_no_cache_at_all_an_unreachable_sheet_is_honest_rather_than_silent(monkeypatch): + monkeypatch.setattr(catalog, "p_cache", None) + + def p_down(url): + raise OSError("no network") + + monkeypatch.setattr(catalog, "fetch_sheet_csv", p_down) + empty = catalog.load_catalog(force=True) + assert empty.source == "empty" and empty.count == 0 and empty.error + + +def test_a_caller_names_a_listing_id_never_a_url(): + """The install request model carries only an id, so no caller can make the backend fetch a URL + of their choosing; the URL is resolved from the catalog WE fetched.""" + assert set(marketplace.InstallRequest.model_fields) == {"id"} + + +def test_install_stages_through_the_same_door_as_a_dropped_file(): + from backend.apps.swarm import swarm as swarm_routes + + assert marketplace.stage_bundle_for_import is swarm_routes.stage_bundle_for_import + body = inspect.getsource(marketplace.install_preflight) + assert "stage_bundle_for_import" in body + for forbidden in ("write_folder_skill", "closure.commit", "import_commit"): + assert forbidden not in body, "install must never write; it stages and lets the user confirm" diff --git a/frontend/src/app/pages/Directory/DirectoryPackagesTab.tsx b/frontend/src/app/pages/Directory/DirectoryPackagesTab.tsx new file mode 100644 index 00000000..32e4b219 --- /dev/null +++ b/frontend/src/app/pages/Directory/DirectoryPackagesTab.tsx @@ -0,0 +1,233 @@ +import React, { useEffect, useMemo, useState } from 'react'; +import Box from '@mui/material/Box'; +import Alert from '@mui/material/Alert'; +import Snackbar from '@mui/material/Snackbar'; +import Typography from '@mui/material/Typography'; +import CircularProgress from '@mui/material/CircularProgress'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { useAppDispatch, useAppSelector } from '@/shared/hooks'; +import { fetchMarketplaceListings } from '@/shared/state/marketplaceCatalogSlice'; +import { fetchSkills } from '@/shared/state/skillsSlice'; +import { fetchOutputs } from '@/shared/state/outputsSlice'; +import { fetchWorkflows } from '@/shared/state/workflowsSlice'; +import ImportModal from '@/app/components/share/ImportModal'; +import { importNeedsConfirm } from '@/app/components/share/importNeedsConfirm'; +import { importCommit } from '@/app/components/share/shareApi'; +import type { ImportPreflight } from '@/app/components/share/shareTypes'; +import DirectoryFilterBar from './DirectoryFilterBar'; +import PackageCard from './packages/PackageCard'; +import PackageDialog from './packages/detail/PackageDialog'; +import PackageBundleCard from './packages/PackageBundleCard'; +import PackageBundleDialog from './packages/detail/PackageBundleDialog'; +import { stagePackageInstall } from './packages/installPackage'; +import { KIND_LABELS, isBundle, resolveBundleMembers, type Listing } from './packages/catalog'; + +type Toast = { message: string; severity: 'success' | 'error' } | null; + +// The store tab: packages published to the marketplace sheet. Install downloads the .swarm and hands +// it to the ordinary bundle import, so what a package can do to this machine is reviewed the same way +// a dropped file is. +const DirectoryPackagesTab: React.FC<{ onInstalled?: (rootType: string) => void }> = ({ onInstalled }) => { + const c = useClaudeTokens(); + const dispatch = useAppDispatch(); + const { listings, loading, loaded, source, error } = useAppSelector((s) => s.marketplaceCatalog); + const [query, setQuery] = useState(''); + const [kinds, setKinds] = useState([]); + const [sort, setSort] = useState('newest'); + const [openListing, setOpenListing] = useState(null); + const [openBundle, setOpenBundle] = useState(null); + const [installingId, setInstallingId] = useState(null); + const [installedIds, setInstalledIds] = useState([]); + const [confirm, setConfirm] = useState<{ preflight: ImportPreflight; listingId: string } | null>(null); + const [committing, setCommitting] = useState(false); + const [toast, setToast] = useState(null); + + useEffect(() => { + dispatch(fetchMarketplaceListings(false)); + }, [dispatch]); + + const packages = useMemo(() => listings.filter((l) => !isBundle(l)), [listings]); + const bundles = useMemo(() => listings.filter(isBundle), [listings]); + const kindOptions = useMemo( + () => Array.from(new Set(packages.map((l) => l.kind).filter(Boolean))).sort() + .map((k) => ({ value: k, label: KIND_LABELS[k] || k })), + [packages], + ); + + const matchesQuery = (l: Listing, q: string): boolean => + !q || [l.title, l.description, l.tags, l.author].some((field) => field.toLowerCase().includes(q)); + + const visible = useMemo(() => { + const q = query.trim().toLowerCase(); + const rows = packages.filter((l) => (kinds.length === 0 || kinds.includes(l.kind)) && matchesQuery(l, q)); + return [...rows].sort((a, b) => { + if (sort === 'name') return a.title.localeCompare(b.title); + if (sort === 'kind') return a.kind.localeCompare(b.kind); + return (b.updated_at || '').localeCompare(a.updated_at || ''); + }); + }, [packages, query, kinds, sort]); + + // A bundle has no single kind, so it leaves the view entirely once the user narrows to one. + const visibleBundles = useMemo(() => { + if (kinds.length > 0) return []; + const q = query.trim().toLowerCase(); + return bundles.filter((b) => matchesQuery(b, q)); + }, [bundles, query, kinds]); + + const finish = (preflight: ImportPreflight, listingId: string, rootType: string) => { + setInstalledIds((prev) => (prev.includes(listingId) ? prev : [...prev, listingId])); + setToast({ message: `Added ${preflight.summary.root.name}`, severity: 'success' }); + // Nothing else refetches these on import, so an installed package would otherwise stay invisible. + if (rootType === 'skill') dispatch(fetchSkills()); + if (rootType === 'app') dispatch(fetchOutputs()); + if (rootType === 'workflow') dispatch(fetchWorkflows(undefined)); + onInstalled?.(rootType); + }; + + const commit = async (preflight: ImportPreflight, listingId: string) => { + setCommitting(true); + try { + const res = await importCommit(preflight.staging_token); + finish(preflight, listingId, res.root_type); + setConfirm(null); + } catch (e: unknown) { + setToast({ message: e instanceof Error ? e.message : "We couldn't finish the install.", severity: 'error' }); + } finally { + setCommitting(false); + } + }; + + const install = async (listingId: string) => { + setInstallingId(listingId); + try { + const preflight = await stagePackageInstall(listingId); + if (importNeedsConfirm(preflight)) setConfirm({ preflight, listingId }); + else await commit(preflight, listingId); + } catch (e: unknown) { + setToast({ message: e instanceof Error ? e.message : "We couldn't download this package.", severity: 'error' }); + } finally { + setInstallingId(null); + } + }; + + const installBundle = async (bundle: Listing) => { + const members = resolveBundleMembers(bundle, listings).filter((m) => m.download_url); + if (members.length === 0) { + setToast({ message: 'This bundle has no installable packages yet.', severity: 'error' }); + return; + } + // One at a time: each member gets its own review, and a bundle must not be a way to skip one. + for (const member of members) { + await install(member.id); + } + }; + + const body = (): React.ReactElement => { + if (loading && !loaded) { + return ( + + + + ); + } + if (visible.length === 0 && visibleBundles.length === 0) { + return ( + + {error ? error : 'No packages match that search yet.'} + + ); + } + return ( + <> + {visibleBundles.length > 0 && ( + + + Collections + + + {visibleBundles.map((bundle) => ( + setOpenBundle(bundle)} + /> + ))} + + + )} + + {visible.map((listing) => ( + setOpenListing(listing)} + onTag={(tag) => setQuery(tag)} + /> + ))} + + + ); + }; + + return ( + + setKinds((prev) => (prev.includes(value) ? prev.filter((k) => k !== value) : [...prev, value]))} + sortOptions={[ + { value: 'newest', label: 'Newest' }, + { value: 'name', label: 'Name' }, + { value: 'kind', label: 'Kind' }, + ]} + sortValue={sort} + onSort={setSort} + /> + {source === 'cache' && ( + + Showing the last catalog we loaded; the marketplace could not be reached just now. + + )} + {body()} + { if (openListing) void install(openListing.id); }} + onClose={() => setOpenListing(null)} + /> + { if (openBundle) void installBundle(openBundle); }} + onInstallMember={(id) => { void install(id); }} + onOpenMember={(member) => { setOpenBundle(null); setOpenListing(member); }} + onClose={() => setOpenBundle(null)} + /> + confirm && commit(confirm.preflight, confirm.listingId)} + onClose={() => setConfirm(null)} + /> + setToast(null)} + anchorOrigin={{ vertical: 'bottom', horizontal: 'center' }} + > + setToast(null)}> + {toast?.message} + + + + ); +}; + +export default DirectoryPackagesTab; diff --git a/frontend/src/app/pages/Directory/MarketplaceBody.tsx b/frontend/src/app/pages/Directory/MarketplaceBody.tsx index cddff735..522153b1 100644 --- a/frontend/src/app/pages/Directory/MarketplaceBody.tsx +++ b/frontend/src/app/pages/Directory/MarketplaceBody.tsx @@ -2,31 +2,31 @@ import React, { useState, useEffect } from 'react'; import Box from '@mui/material/Box'; import Typography from '@mui/material/Typography'; import CircularProgress from '@mui/material/CircularProgress'; -import DescriptionOutlinedIcon from '@mui/icons-material/DescriptionOutlined'; +import Inventory2OutlinedIcon from '@mui/icons-material/Inventory2Outlined'; import GridViewOutlinedIcon from '@mui/icons-material/GridViewOutlined'; import FolderSpecialOutlinedIcon from '@mui/icons-material/FolderSpecialOutlined'; import PowerOutlinedIcon from '@mui/icons-material/PowerOutlined'; import { useClaudeTokens } from '@/shared/styles/ThemeContext'; import { useAppDispatch, useAppSelector } from '@/shared/hooks'; import { clearMarketplaceRequestedTab } from '@/shared/state/dashboardLayoutSlice'; -import DirectorySkillsTab from './DirectorySkillsTab'; +import DirectoryPackagesTab from './DirectoryPackagesTab'; import DirectoryConnectorsTab from './DirectoryConnectorsTab'; // Lazy: the manage views pull the full Skills/Tools pages (markdown, MCP cards) and the marketplace opens from the dock. const SkillsManage = React.lazy(() => import('../Skills/Skills')); const ToolsManage = React.lazy(() => import('../Tools/Tools')); -export type DirectoryTab = 'skills' | 'connectors' | 'my-skills' | 'my-connectors'; +export type DirectoryTab = 'packages' | 'connectors' | 'my-skills' | 'my-connectors'; -const VALID_TABS: DirectoryTab[] = ['skills', 'connectors', 'my-skills', 'my-connectors']; +const VALID_TABS: DirectoryTab[] = ['packages', 'connectors', 'my-skills', 'my-connectors']; -// The Marketplace window body: claude.ai's Directory grids land first (the store), with the +// The Marketplace window body: the published package catalog lands first (the store), with the // installed-item manage pages as their own rail rows below the divider. const MarketplaceBody: React.FC = () => { const c = useClaudeTokens(); const dispatch = useAppDispatch(); const requestedTab = useAppSelector((s) => s.dashboardLayout.marketplaceRequestedTab); - const [view, setView] = useState('skills'); + const [view, setView] = useState('packages'); const [focusSkillId, setFocusSkillId] = useState(null); const [focusToolId, setFocusToolId] = useState(null); @@ -65,12 +65,12 @@ const MarketplaceBody: React.FC = () => { const content = (): React.ReactElement => { switch (view) { - case 'skills': - return { setFocusSkillId(id); setView('my-skills'); }} />; + case 'packages': + return { if (rootType === 'skill') { setFocusSkillId(null); setView('my-skills'); } }} />; case 'connectors': return { setFocusToolId(id); setView('my-connectors'); }} />; case 'my-skills': - return setView('skills')} focusSkillId={focusSkillId} />; + return setView('packages')} focusSkillId={focusSkillId} />; case 'my-connectors': return setView('connectors')} expandToolId={focusToolId} />; } @@ -80,7 +80,7 @@ const MarketplaceBody: React.FC = () => { - {railRow('skills', 'Skills', )} + {railRow('packages', 'Packages', )} {railRow('connectors', 'Connectors', )} {railRow('my-skills', 'My skills', )} diff --git a/frontend/src/app/pages/Directory/openMarketplace.ts b/frontend/src/app/pages/Directory/openMarketplace.ts index 794a22df..dde35dec 100644 --- a/frontend/src/app/pages/Directory/openMarketplace.ts +++ b/frontend/src/app/pages/Directory/openMarketplace.ts @@ -2,6 +2,6 @@ import { store } from '@/shared/state/store'; import { openMarketplaceCard } from '@/shared/state/dashboardLayoutSlice'; import type { DirectoryTab } from './MarketplaceBody'; -export function openMarketplace(tab: DirectoryTab = 'skills'): void { +export function openMarketplace(tab: DirectoryTab = 'packages'): void { store.dispatch(openMarketplaceCard({ tab })); } diff --git a/frontend/src/app/pages/Directory/packages/PackageBundleCard.tsx b/frontend/src/app/pages/Directory/packages/PackageBundleCard.tsx new file mode 100644 index 00000000..bfdc7787 --- /dev/null +++ b/frontend/src/app/pages/Directory/packages/PackageBundleCard.tsx @@ -0,0 +1,147 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Stack from '@mui/material/Stack'; +import Typography from '@mui/material/Typography'; +import Chip from '@mui/material/Chip'; +import Inventory2Icon from '@mui/icons-material/Inventory2'; +import ExtensionIcon from '@mui/icons-material/Extension'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { KIND_LABELS, type Listing } from './catalog'; + +interface Props { + bundle: Listing; + members: Listing[]; + onOpen: () => void; +} + +// Leads with a stacked-icon motif of its first few members and a count, so a bundle reads as a collection at a glance rather than as another single package. +export default function PackageBundleCard({ bundle, members, onOpen }: Props) { + const c = useClaudeTokens(); + const preview = members.slice(0, 4); + const kinds = Array.from(new Set(members.map((m) => KIND_LABELS[m.kind] || m.kind).filter(Boolean))); + + return ( + + + + {bundle.icon_url ? ( + + ) : ( + + )} + + + + {bundle.title} + + + Bundle · {members.length} {members.length === 1 ? 'package' : 'packages'} + + + + + + {bundle.description || 'A curated collection of packages.'} + + + + {preview.map((m, i) => ( + + {m.icon_url ? ( + + ) : ( + + )} + + ))} + {members.length > preview.length && ( + + +{members.length - preview.length} more + + )} + {kinds.length > 0 && ( + + )} + + + ); +} diff --git a/frontend/src/app/pages/Directory/packages/PackageCard.tsx b/frontend/src/app/pages/Directory/packages/PackageCard.tsx new file mode 100644 index 00000000..5812801f --- /dev/null +++ b/frontend/src/app/pages/Directory/packages/PackageCard.tsx @@ -0,0 +1,118 @@ +import React from 'react'; +import Box from '@mui/material/Box'; +import Stack from '@mui/material/Stack'; +import Typography from '@mui/material/Typography'; +import Chip from '@mui/material/Chip'; +import ExtensionIcon from '@mui/icons-material/Extension'; +import { useClaudeTokens } from '@/shared/styles/ThemeContext'; +import { parseTags, KIND_LABELS, type Listing } from './catalog'; + +interface Props { + listing: Listing; + onOpen: () => void; + onTag: (tag: string) => void; +} + +export default function PackageCard({ listing, onOpen, onTag }: Props) { + const c = useClaudeTokens(); + const tags = parseTags(listing.tags).slice(0, 3); + + return ( + + + + {listing.icon_url ? ( + + ) : ( + + )} + + + + {listing.title} + + + {KIND_LABELS[listing.kind] || listing.kind || 'Package'} + {listing.version ? ` · v${listing.version}` : ''} + + + + + + {listing.description || 'No description provided.'} + + + + {tags.map((t) => ( + ) => { + e.stopPropagation(); + onTag(t); + }} + sx={{ + height: 22, + fontSize: '0.72rem', + bgcolor: c.bg.secondary, + color: c.text.tertiary, + '&:hover': { color: c.text.primary }, + }} + /> + ))} + + + ); +} diff --git a/frontend/src/app/pages/Directory/packages/catalog.test.ts b/frontend/src/app/pages/Directory/packages/catalog.test.ts new file mode 100644 index 00000000..51cde0f6 --- /dev/null +++ b/frontend/src/app/pages/Directory/packages/catalog.test.ts @@ -0,0 +1,71 @@ +import { test } from 'node:test'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { driveStreamUrl, isBundle, parseBundleItems, parseTags, parseVideoUrls, resolveBundleMembers, videoEmbed, type Listing } from './catalog'; + +function listing(over: Partial): Listing { + return { + id: 'x', title: 'X', kind: 'skill', version: '', author: '', description: '', tags: '', + download_url: '', icon_url: '', video_url: '', size: '', updated_at: '', + bundle_items: '', notion_url: '', details_json: '', ...over, + }; +} + +test('a bundle resolves its members in its own order and drops ids that no longer exist', () => { + const bundle = listing({ id: 'pack', kind: 'bundle', bundle_items: 'b, a, gone, a' }); + const all = [bundle, listing({ id: 'a' }), listing({ id: 'b' })]; + assert.deepEqual(resolveBundleMembers(bundle, all).map((m) => m.id), ['b', 'a']); + assert.equal(isBundle(bundle), true); + assert.deepEqual(parseBundleItems('b, a, , a'), ['b', 'a'], 'a repeated id is listed once'); +}); + +test('a bundle never lists itself, which would install it forever', () => { + const bundle = listing({ id: 'pack', kind: 'bundle', bundle_items: 'pack, a' }); + assert.deepEqual(resolveBundleMembers(bundle, [bundle, listing({ id: 'a' })]).map((m) => m.id), ['a']); +}); + +test('tags survive sloppy spacing and empty cells', () => { + assert.deepEqual(parseTags(' notion , productivity ,,'), ['notion', 'productivity']); + assert.deepEqual(parseTags(''), []); +}); + +test('a Drive video link becomes a byte stream a video tag can actually play', () => { + assert.equal( + driveStreamUrl('https://drive.google.com/file/d/1AbCdEf/preview'), + 'https://drive.usercontent.google.com/download?id=1AbCdEf&export=download', + ); +}); + +test('a YouTube listing embeds the player, a Drive listing streams the file', () => { + assert.equal(videoEmbed('https://youtu.be/abcdefghijk')?.kind, 'youtube'); + assert.equal(videoEmbed('https://drive.google.com/file/d/1AbCdEf/preview')?.kind, 'file'); + assert.equal(videoEmbed(''), null); +}); + +test('several demo videos in one cell keep their order, primary first', () => { + assert.deepEqual(parseVideoUrls('https://a\n\nhttps://b\n'), ['https://a', 'https://b']); +}); + +// The store tab is the marketplace's front door, so it must be the row that opens by default. +test('Packages is the default marketplace tab and the old skills store is gone', () => { + const dir = path.join(process.cwd(), 'src/app/pages/Directory'); + const body = fs.readFileSync(path.join(dir, 'MarketplaceBody.tsx'), 'utf8'); + assert.match(body, /useState\('packages'\)/); + assert.match(body, /railRow\('packages', 'Packages'/); + assert.doesNotMatch(body, /DirectorySkillsTab/); + assert.equal(fs.existsSync(path.join(dir, 'DirectorySkillsTab.tsx')), false); + const open = fs.readFileSync(path.join(dir, 'openMarketplace.ts'), 'utf8'); + assert.match(open, /DirectoryTab = 'packages'/); +}); + +// Install must not grow a second write path; it stages and lets the shared confirm surface decide. +test('the packages tab installs through the shared bundle import, not its own writer', () => { + const tab = fs.readFileSync(path.join(process.cwd(), 'src/app/pages/Directory/DirectoryPackagesTab.tsx'), 'utf8'); + assert.match(tab, /importNeedsConfirm/); + assert.match(tab, /importCommit/); + assert.match(tab, / 0 && commit > gate, 'the confirm gate is asked BEFORE anything is committed'); +}); diff --git a/frontend/src/app/pages/Directory/packages/catalog.ts b/frontend/src/app/pages/Directory/packages/catalog.ts new file mode 100644 index 00000000..0d05528e --- /dev/null +++ b/frontend/src/app/pages/Directory/packages/catalog.ts @@ -0,0 +1,141 @@ +import { API_BASE } from '@/shared/config'; + +const CATALOG_LISTINGS_URL = `${API_BASE}/catalog/listings`; + +export interface Listing { + id: string; + title: string; + kind: string; + version: string; + author: string; + description: string; + tags: string; + download_url: string; + icon_url: string; + video_url: string; + size: string; + updated_at: string; + // Bundles (kind === 'bundle') list their member ids here, comma separated; empty on ordinary packages. + bundle_items: string; + notion_url: string; + // The linked Notion page converted to DetailDoc JSON at upload time; empty when there is no linked page. + details_json: string; +} + +export interface ListingsResponse { + source: 'sheet' | 'sample'; + count: number; + listings: Listing[]; + error: string; +} + +export async function fetchListings(): Promise { + const res = await fetch(CATALOG_LISTINGS_URL); + if (!res.ok) throw new Error(`Listings request failed (${res.status})`); + return (await res.json()) as ListingsResponse; +} + +export function parseTags(tags: string): string[] { + return (tags || '') + .split(',') + .map((t) => t.trim()) + .filter(Boolean); +} + +// Drive's /file/d/{id}/preview URL is a player page an HTML5