From 23182824d4a5542e166b9e59c5c9f6e60651540b Mon Sep 17 00:00:00 2001 From: Ethan Hanlon Date: Fri, 17 Jul 2026 18:13:08 -0700 Subject: [PATCH] [ethan] feat: add spotify integration. not completely reliable yet. --- backend/apps/agents/core/mcp_preflight.py | 5 + backend/apps/spotify_mcp_shim/__main__.py | 5 + backend/apps/spotify_mcp_shim/handlers.py | 222 ++++++++++++++++++ backend/apps/spotify_mcp_shim/server.py | 71 ++++++ backend/apps/spotify_mcp_shim/tools.py | 22 ++ backend/apps/tools_lib/mcp_config.py | 2 +- backend/requirements.txt | 4 +- frontend/src/app/pages/Tools/integrations.tsx | 13 + 8 files changed, 342 insertions(+), 2 deletions(-) create mode 100644 backend/apps/spotify_mcp_shim/__main__.py create mode 100644 backend/apps/spotify_mcp_shim/handlers.py create mode 100644 backend/apps/spotify_mcp_shim/server.py create mode 100644 backend/apps/spotify_mcp_shim/tools.py diff --git a/backend/apps/agents/core/mcp_preflight.py b/backend/apps/agents/core/mcp_preflight.py index a9383302..2b5745a2 100644 --- a/backend/apps/agents/core/mcp_preflight.py +++ b/backend/apps/agents/core/mcp_preflight.py @@ -26,6 +26,11 @@ CURATED_SHORTLIST: list[CuratedEntry] = [ "title": "Google Workspace", "description": "Gmail, Calendar, Drive, Docs, Sheets, Slides; for reading/sending email, checking the user's schedule, and pulling context from their documents.", }, + { + "id": "Spotify", + "title": "Spotify", + "description": "Control playback, search music, and manage playlists; when the task involves music or audio." + }, { "id": "Microsoft 365", "title": "Microsoft 365", diff --git a/backend/apps/spotify_mcp_shim/__main__.py b/backend/apps/spotify_mcp_shim/__main__.py new file mode 100644 index 00000000..ba566a59 --- /dev/null +++ b/backend/apps/spotify_mcp_shim/__main__.py @@ -0,0 +1,5 @@ +"""Module-level entrypoint so `python -m backend.apps.spotify_mcp_shim` works.""" +from backend.apps.spotify_mcp_shim.server import main + +if __name__ == "__main__": + main() diff --git a/backend/apps/spotify_mcp_shim/handlers.py b/backend/apps/spotify_mcp_shim/handlers.py new file mode 100644 index 00000000..c6d99148 --- /dev/null +++ b/backend/apps/spotify_mcp_shim/handlers.py @@ -0,0 +1,222 @@ +from typing import Optional, Any, Dict +import psutil +import json +import urllib.request +import urllib.error +import os + +# Bypass the OpenSwarm proxy for local Chrome CDP connections +os.environ["no_proxy"] = "*" + + +import subprocess +import time + +def p_ensure_chrome_cdp(): + """Ensure Chrome is running with CDP on port 9223. + If not, we launch a dedicated profile so we don't conflict with the user's main Chrome, + and we leave it running in the background so music keeps playing after the script exits! + """ + try: + req = urllib.request.Request("http://127.0.0.1:9223/json/version") + with urllib.request.urlopen(req, timeout=0.5) as response: + if response.status == 200: + return True + except Exception: + pass + + # CDP not responding. Let's auto-launch a dedicated Chrome instance! + profile_dir = os.path.expanduser("~/.openswarm/spotify_chrome_profile") + os.makedirs(profile_dir, exist_ok=True) + + subprocess.Popen([ + "/Applications/Google Chrome.app/Contents/MacOS/Google Chrome", + "--remote-debugging-port=9223", + f"--user-data-dir={profile_dir}", + "--no-first-run", + "--no-default-browser-check" + ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + + # Wait for it to spin up + for _ in range(10): + time.sleep(0.5) + try: + req = urllib.request.Request("http://127.0.0.1:9223/json/version") + with urllib.request.urlopen(req, timeout=0.5) as response: + if response.status == 200: + return True + except Exception: + continue + + return False + +def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]: + """ + Using the Playwright MCP, we'll: + + 1. Auto-launch a dedicated Chrome instance (with CDP) if it's not already running. + 2. Navigate to Spotify. If this is the user's first time on this dedicated profile, they will need to log in. + 3. Search for the song name. + 4. Play the song! + """ + + if not p_ensure_chrome_cdp(): + return { + "is_error": True, + "is_human_intervention": False, + "message": "Failed to auto-start Chrome with remote debugging on port 9223." + } + + import asyncio + + def run_async_play_track(): + async def inner(): + from playwright.async_api import async_playwright + import urllib.parse + import base64 + from io import BytesIO + from PIL import Image + + from backend.apps.settings.settings import load_settings + from backend.apps.settings.credentials import get_anthropic_client_for_model + from backend.apps.agents.providers.registry import resolve_aux_model + + settings = load_settings() + try: + model_id, _ = await resolve_aux_model(settings, preferred_tier="haiku") + client = get_anthropic_client_for_model(settings, model_id) + except Exception as e: + return { + "is_error": True, + "is_human_intervention": False, + "message": f"Could not load AI client for screenshot analysis: {e}" + } + + try: + async with async_playwright() as p: + browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223") + context = browser.contexts[0] + + page = None + for p_obj in context.pages: + if "spotify.com" in p_obj.url: + page = p_obj + break + + if not page: + page = await context.new_page() + + query = track_name + if artist: + query += f" {artist}" + + search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query)}" + await page.goto(search_url) + await page.wait_for_load_state("networkidle") + await asyncio.sleep(2) # Give elements time to render + + prompt = f""" +I want to play the track '{query}' on Spotify. Look at this screenshot of the Spotify web UI. +Determine the state and return ONLY a valid JSON object. Do not include markdown formatting or backticks, just the raw JSON. +You must choose one of the following exact JSON structures: + +1. If a login prompt or overlay is blocking the UI: +{{ + "action": "human_intervention", + "message": "User needs to log in" +}} + +2. If the track '{query}' is already playing (pause button is visible): +{{ + "action": "done", + "message": "Successfully started playback" +}} + +3. Otherwise, find the Play button for the top search result and return its exact center coordinates as percentages (0 to 100) of the image width and height: +{{ + "action": "click", + "x_percent": 50.5, + "y_percent": 25.0, + "message": "Clicking play button" +}} + +CRITICAL: Return strictly valid JSON. Double check your quotes and commas. +""".strip() + + for _ in range(3): + screenshot_bytes = await page.screenshot() + + img = Image.open(BytesIO(screenshot_bytes)) + max_width = 1024 + if img.width > max_width: + ratio = max_width / img.width + img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS) + buf = BytesIO() + img.convert("RGB").save(buf, format="JPEG", quality=45) + b64_img = base64.b64encode(buf.getvalue()).decode("utf-8") + + response = await client.messages.create( + model=model_id, + max_tokens=256, + messages=[{ + "role": "user", + "content": [ + {"type": "text", "text": prompt}, + {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64_img}} + ] + }] + ) + + try: + # Parse JSON from response + resp_text = response.content[0].text.strip() + if resp_text.startswith("```json"): + resp_text = resp_text[7:-3].strip() + elif resp_text.startswith("```"): + resp_text = resp_text[3:-3].strip() + + action_data = json.loads(resp_text) + + if action_data["action"] == "human_intervention": + return { + "is_error": False, + "is_human_intervention": True, + "message": action_data.get("message", "Human intervention needed.") + } + elif action_data["action"] == "done": + return { + "is_error": False, + "is_human_intervention": False, + "message": action_data.get("message", "Task completed.") + } + elif action_data["action"] == "click": + vp_w = await page.evaluate("window.innerWidth") + vp_h = await page.evaluate("window.innerHeight") + click_x = (float(action_data["x_percent"]) / 100.0) * vp_w + click_y = (float(action_data["y_percent"]) / 100.0) * vp_h + await page.mouse.click(click_x, click_y) + await asyncio.sleep(2) # wait for playback to start + # loop continues to verify + except Exception as e: + return { + "is_error": True, + "is_human_intervention": False, + "message": f"Failed to parse AI response: {e}\nResponse was: {response.content[0].text}" + } + + return { + "is_error": True, + "is_human_intervention": False, + "message": "AI failed to start playback after multiple attempts." + } + + except Exception as e: + return { + "is_error": True, + "is_human_intervention": False, + "message": f"An error occurred: {str(e)}" + } + + return asyncio.run(inner()) + + return run_async_play_track() \ No newline at end of file diff --git a/backend/apps/spotify_mcp_shim/server.py b/backend/apps/spotify_mcp_shim/server.py new file mode 100644 index 00000000..925b8baf --- /dev/null +++ b/backend/apps/spotify_mcp_shim/server.py @@ -0,0 +1,71 @@ +import sys +import json + +from backend.apps.spotify_mcp_shim.tools import TOOLS + +def p_send(id_, result=None, error=None): + msg = {"jsonrpc": "2.0", "id": id_} + if error is not None: + msg["error"] = error + else: + msg["result"] = result + sys.stdout.write(json.dumps(msg) + "\n") + sys.stdout.flush() + + +def p_err(text: str) -> dict: + return {"content": [{"type": "text", "text": f"Error: {text}"}], "isError": True} + +def p_ok(payload) -> dict: + if isinstance(payload, str): + return {"content": [{"type": "text", "text": payload}]} + return {"content": [{"type": "text", "text": json.dumps(payload, indent=2, default=str)}]} + +from backend.apps.spotify_mcp_shim.handlers import play_track + +def handle_tool_call(name: str, args: dict) -> dict: + match name: + case "play_track": + return p_ok(play_track(**args)) + + return p_err(f"Unknown tool: {name}") + +def main(): + for line in sys.stdin: + line = line.strip() + if not line: + continue + + try: + msg = json.loads(line) + except json.JSONDecodeError: + continue + + method = msg.get("method") + id_ = msg.get("id") + params = msg.get("params", {}) or {} + + if method == "initialize": + p_send(id_, { + "protocolVersion": "2024-11-05", + "capabilities": {"tools": {}}, + "serverInfo": {"name": "openswarm-spotify", "version": "1.0.0"}, + }) + elif method == "notifications/initialized": + pass + elif method == "tools/list": + p_send(id_, {"tools": TOOLS}) + elif method == "tools/call": + name = params.get("name", "") + args = params.get("arguments", {}) or {} + try: + p_send(id_, handle_tool_call(name, args)) + except Exception as e: + p_send(id_, p_err(f"shim crashed: {e!r}")) + elif method == "ping": + p_send(id_, {}) + elif id_ is not None: + p_send(id_, error={"code": -32601, "message": f"Method not found: {method}"}) + +if __name__ == "__main__": + main() \ No newline at end of file diff --git a/backend/apps/spotify_mcp_shim/tools.py b/backend/apps/spotify_mcp_shim/tools.py new file mode 100644 index 00000000..2b0510cf --- /dev/null +++ b/backend/apps/spotify_mcp_shim/tools.py @@ -0,0 +1,22 @@ +"""MCP tool surface for Spotify: the things a logged-in human does. + +Plays tracks. This is accomplished via a Playwright session, which may require +the user to manually input their credentials or solve a CAPTCHA. +""" + +OBJ = "object" + +TOOLS = [ + { + "name": "play_track", + "description": "Opens a browser and plays the requested song.", + "inputSchema": { + "type": OBJ, + "properties": { + "track_name": {"type": "string"}, + "artist": {"type": "string"} + }, + "required": ["track_name"] + } + } +] \ No newline at end of file diff --git a/backend/apps/tools_lib/mcp_config.py b/backend/apps/tools_lib/mcp_config.py index 898ba531..aa6977fb 100644 --- a/backend/apps/tools_lib/mcp_config.py +++ b/backend/apps/tools_lib/mcp_config.py @@ -152,7 +152,7 @@ def derive_mcp_config(tool: ToolDefinition) -> Optional[dict]: env["PYTHONPATH"] = (p_project_root + os.pathsep + existing_pp) if existing_pp else p_project_root # The session-borrow social shims (reddit/x/tiktok) each run as a Python shim that borrows the user's live browser session via the backend's cookie bridge, so they need the localhost port + auth token, plus PYTHONPATH to import themselves. - if tool.name.lower() in {"reddit", "x", "tiktok"} and config.get("type") == "stdio": + if tool.name.lower() in {"reddit", "x", "tiktok", "spotify"} and config.get("type") == "stdio": from backend.auth import get_auth_token env = config.setdefault("env", {}) env["OPENSWARM_PORT"] = os.environ.get("OPENSWARM_PORT", "8324") diff --git a/backend/requirements.txt b/backend/requirements.txt index a9b7fea0..0901942b 100644 --- a/backend/requirements.txt +++ b/backend/requirements.txt @@ -24,4 +24,6 @@ swarm-analytics==0.1.1 tzlocal==5.3.1 # Test deps (pytest, pytest-asyncio) live in requirements-dev.txt — they # never ship to production users and shaved ~3 MB / ~200 files off the -# Mac DMG when removed from the prod env. \ No newline at end of file +# Mac DMG when removed from the prod env. +psutil==7.2.2 +playwright==1.61.0 \ No newline at end of file diff --git a/frontend/src/app/pages/Tools/integrations.tsx b/frontend/src/app/pages/Tools/integrations.tsx index f1fe529c..25191cb7 100644 --- a/frontend/src/app/pages/Tools/integrations.tsx +++ b/frontend/src/app/pages/Tools/integrations.tsx @@ -40,6 +40,19 @@ export const INTEGRATIONS: Integration[] = [ ), }, + { + id: 'spotify', + name: "Spotify", + description: 'Control playback, search tracks, and manage playlists.', + mcp_config: { type: 'stdio', command: 'python', args: ['-m', 'backend.apps.spotify_mcp_shim'] }, + color: '#1DB954', + website: 'https://open.spotify.com', + icon: ( + + + + ), + }, { id: 'tiktok', name: 'TikTok',