mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-23 18:14:53 +02:00
[ethan] feat: add spotify integration. not completely reliable yet.
This commit is contained in:
Ethan Hanlon
@@ -26,6 +26,11 @@ CURATED_SHORTLIST: list[CuratedEntry] = [
|
||||
"title": "Google Workspace",
|
||||
"description": "Gmail, Calendar, Drive, Docs, Sheets, Slides; for reading/sending email, checking the user's schedule, and pulling context from their documents.",
|
||||
},
|
||||
{
|
||||
"id": "Spotify",
|
||||
"title": "Spotify",
|
||||
"description": "Control playback, search music, and manage playlists; when the task involves music or audio."
|
||||
},
|
||||
{
|
||||
"id": "Microsoft 365",
|
||||
"title": "Microsoft 365",
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
"""Module-level entrypoint so `python -m backend.apps.spotify_mcp_shim` works."""
|
||||
from backend.apps.spotify_mcp_shim.server import main
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,222 @@
|
||||
from typing import Optional, Any, Dict
|
||||
import psutil
|
||||
import json
|
||||
import urllib.request
|
||||
import urllib.error
|
||||
import os
|
||||
|
||||
# Bypass the OpenSwarm proxy for local Chrome CDP connections
|
||||
os.environ["no_proxy"] = "*"
|
||||
|
||||
|
||||
import subprocess
|
||||
import time
|
||||
|
||||
def p_ensure_chrome_cdp():
|
||||
"""Ensure Chrome is running with CDP on port 9223.
|
||||
If not, we launch a dedicated profile so we don't conflict with the user's main Chrome,
|
||||
and we leave it running in the background so music keeps playing after the script exits!
|
||||
"""
|
||||
try:
|
||||
req = urllib.request.Request("http://127.0.0.1:9223/json/version")
|
||||
with urllib.request.urlopen(req, timeout=0.5) as response:
|
||||
if response.status == 200:
|
||||
return True
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# CDP not responding. Let's auto-launch a dedicated Chrome instance!
|
||||
profile_dir = os.path.expanduser("~/.openswarm/spotify_chrome_profile")
|
||||
os.makedirs(profile_dir, exist_ok=True)
|
||||
|
||||
subprocess.Popen([
|
||||
"/Applications/Google Chrome.app/Contents/MacOS/Google Chrome",
|
||||
"--remote-debugging-port=9223",
|
||||
f"--user-data-dir={profile_dir}",
|
||||
"--no-first-run",
|
||||
"--no-default-browser-check"
|
||||
], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
|
||||
|
||||
# Wait for it to spin up
|
||||
for _ in range(10):
|
||||
time.sleep(0.5)
|
||||
try:
|
||||
req = urllib.request.Request("http://127.0.0.1:9223/json/version")
|
||||
with urllib.request.urlopen(req, timeout=0.5) as response:
|
||||
if response.status == 200:
|
||||
return True
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
return False
|
||||
|
||||
def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]:
|
||||
"""
|
||||
Using the Playwright MCP, we'll:
|
||||
|
||||
1. Auto-launch a dedicated Chrome instance (with CDP) if it's not already running.
|
||||
2. Navigate to Spotify. If this is the user's first time on this dedicated profile, they will need to log in.
|
||||
3. Search for the song name.
|
||||
4. Play the song!
|
||||
"""
|
||||
|
||||
if not p_ensure_chrome_cdp():
|
||||
return {
|
||||
"is_error": True,
|
||||
"is_human_intervention": False,
|
||||
"message": "Failed to auto-start Chrome with remote debugging on port 9223."
|
||||
}
|
||||
|
||||
import asyncio
|
||||
|
||||
def run_async_play_track():
|
||||
async def inner():
|
||||
from playwright.async_api import async_playwright
|
||||
import urllib.parse
|
||||
import base64
|
||||
from io import BytesIO
|
||||
from PIL import Image
|
||||
|
||||
from backend.apps.settings.settings import load_settings
|
||||
from backend.apps.settings.credentials import get_anthropic_client_for_model
|
||||
from backend.apps.agents.providers.registry import resolve_aux_model
|
||||
|
||||
settings = load_settings()
|
||||
try:
|
||||
model_id, _ = await resolve_aux_model(settings, preferred_tier="haiku")
|
||||
client = get_anthropic_client_for_model(settings, model_id)
|
||||
except Exception as e:
|
||||
return {
|
||||
"is_error": True,
|
||||
"is_human_intervention": False,
|
||||
"message": f"Could not load AI client for screenshot analysis: {e}"
|
||||
}
|
||||
|
||||
try:
|
||||
async with async_playwright() as p:
|
||||
browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223")
|
||||
context = browser.contexts[0]
|
||||
|
||||
page = None
|
||||
for p_obj in context.pages:
|
||||
if "spotify.com" in p_obj.url:
|
||||
page = p_obj
|
||||
break
|
||||
|
||||
if not page:
|
||||
page = await context.new_page()
|
||||
|
||||
query = track_name
|
||||
if artist:
|
||||
query += f" {artist}"
|
||||
|
||||
search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query)}"
|
||||
await page.goto(search_url)
|
||||
await page.wait_for_load_state("networkidle")
|
||||
await asyncio.sleep(2) # Give elements time to render
|
||||
|
||||
prompt = f"""
|
||||
I want to play the track '{query}' on Spotify. Look at this screenshot of the Spotify web UI.
|
||||
Determine the state and return ONLY a valid JSON object. Do not include markdown formatting or backticks, just the raw JSON.
|
||||
You must choose one of the following exact JSON structures:
|
||||
|
||||
1. If a login prompt or overlay is blocking the UI:
|
||||
{{
|
||||
"action": "human_intervention",
|
||||
"message": "User needs to log in"
|
||||
}}
|
||||
|
||||
2. If the track '{query}' is already playing (pause button is visible):
|
||||
{{
|
||||
"action": "done",
|
||||
"message": "Successfully started playback"
|
||||
}}
|
||||
|
||||
3. Otherwise, find the Play button for the top search result and return its exact center coordinates as percentages (0 to 100) of the image width and height:
|
||||
{{
|
||||
"action": "click",
|
||||
"x_percent": 50.5,
|
||||
"y_percent": 25.0,
|
||||
"message": "Clicking play button"
|
||||
}}
|
||||
|
||||
CRITICAL: Return strictly valid JSON. Double check your quotes and commas.
|
||||
""".strip()
|
||||
|
||||
for _ in range(3):
|
||||
screenshot_bytes = await page.screenshot()
|
||||
|
||||
img = Image.open(BytesIO(screenshot_bytes))
|
||||
max_width = 1024
|
||||
if img.width > max_width:
|
||||
ratio = max_width / img.width
|
||||
img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS)
|
||||
buf = BytesIO()
|
||||
img.convert("RGB").save(buf, format="JPEG", quality=45)
|
||||
b64_img = base64.b64encode(buf.getvalue()).decode("utf-8")
|
||||
|
||||
response = await client.messages.create(
|
||||
model=model_id,
|
||||
max_tokens=256,
|
||||
messages=[{
|
||||
"role": "user",
|
||||
"content": [
|
||||
{"type": "text", "text": prompt},
|
||||
{"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64_img}}
|
||||
]
|
||||
}]
|
||||
)
|
||||
|
||||
try:
|
||||
# Parse JSON from response
|
||||
resp_text = response.content[0].text.strip()
|
||||
if resp_text.startswith("```json"):
|
||||
resp_text = resp_text[7:-3].strip()
|
||||
elif resp_text.startswith("```"):
|
||||
resp_text = resp_text[3:-3].strip()
|
||||
|
||||
action_data = json.loads(resp_text)
|
||||
|
||||
if action_data["action"] == "human_intervention":
|
||||
return {
|
||||
"is_error": False,
|
||||
"is_human_intervention": True,
|
||||
"message": action_data.get("message", "Human intervention needed.")
|
||||
}
|
||||
elif action_data["action"] == "done":
|
||||
return {
|
||||
"is_error": False,
|
||||
"is_human_intervention": False,
|
||||
"message": action_data.get("message", "Task completed.")
|
||||
}
|
||||
elif action_data["action"] == "click":
|
||||
vp_w = await page.evaluate("window.innerWidth")
|
||||
vp_h = await page.evaluate("window.innerHeight")
|
||||
click_x = (float(action_data["x_percent"]) / 100.0) * vp_w
|
||||
click_y = (float(action_data["y_percent"]) / 100.0) * vp_h
|
||||
await page.mouse.click(click_x, click_y)
|
||||
await asyncio.sleep(2) # wait for playback to start
|
||||
# loop continues to verify
|
||||
except Exception as e:
|
||||
return {
|
||||
"is_error": True,
|
||||
"is_human_intervention": False,
|
||||
"message": f"Failed to parse AI response: {e}\nResponse was: {response.content[0].text}"
|
||||
}
|
||||
|
||||
return {
|
||||
"is_error": True,
|
||||
"is_human_intervention": False,
|
||||
"message": "AI failed to start playback after multiple attempts."
|
||||
}
|
||||
|
||||
except Exception as e:
|
||||
return {
|
||||
"is_error": True,
|
||||
"is_human_intervention": False,
|
||||
"message": f"An error occurred: {str(e)}"
|
||||
}
|
||||
|
||||
return asyncio.run(inner())
|
||||
|
||||
return run_async_play_track()
|
||||
@@ -0,0 +1,71 @@
|
||||
import sys
|
||||
import json
|
||||
|
||||
from backend.apps.spotify_mcp_shim.tools import TOOLS
|
||||
|
||||
def p_send(id_, result=None, error=None):
|
||||
msg = {"jsonrpc": "2.0", "id": id_}
|
||||
if error is not None:
|
||||
msg["error"] = error
|
||||
else:
|
||||
msg["result"] = result
|
||||
sys.stdout.write(json.dumps(msg) + "\n")
|
||||
sys.stdout.flush()
|
||||
|
||||
|
||||
def p_err(text: str) -> dict:
|
||||
return {"content": [{"type": "text", "text": f"Error: {text}"}], "isError": True}
|
||||
|
||||
def p_ok(payload) -> dict:
|
||||
if isinstance(payload, str):
|
||||
return {"content": [{"type": "text", "text": payload}]}
|
||||
return {"content": [{"type": "text", "text": json.dumps(payload, indent=2, default=str)}]}
|
||||
|
||||
from backend.apps.spotify_mcp_shim.handlers import play_track
|
||||
|
||||
def handle_tool_call(name: str, args: dict) -> dict:
|
||||
match name:
|
||||
case "play_track":
|
||||
return p_ok(play_track(**args))
|
||||
|
||||
return p_err(f"Unknown tool: {name}")
|
||||
|
||||
def main():
|
||||
for line in sys.stdin:
|
||||
line = line.strip()
|
||||
if not line:
|
||||
continue
|
||||
|
||||
try:
|
||||
msg = json.loads(line)
|
||||
except json.JSONDecodeError:
|
||||
continue
|
||||
|
||||
method = msg.get("method")
|
||||
id_ = msg.get("id")
|
||||
params = msg.get("params", {}) or {}
|
||||
|
||||
if method == "initialize":
|
||||
p_send(id_, {
|
||||
"protocolVersion": "2024-11-05",
|
||||
"capabilities": {"tools": {}},
|
||||
"serverInfo": {"name": "openswarm-spotify", "version": "1.0.0"},
|
||||
})
|
||||
elif method == "notifications/initialized":
|
||||
pass
|
||||
elif method == "tools/list":
|
||||
p_send(id_, {"tools": TOOLS})
|
||||
elif method == "tools/call":
|
||||
name = params.get("name", "")
|
||||
args = params.get("arguments", {}) or {}
|
||||
try:
|
||||
p_send(id_, handle_tool_call(name, args))
|
||||
except Exception as e:
|
||||
p_send(id_, p_err(f"shim crashed: {e!r}"))
|
||||
elif method == "ping":
|
||||
p_send(id_, {})
|
||||
elif id_ is not None:
|
||||
p_send(id_, error={"code": -32601, "message": f"Method not found: {method}"})
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,22 @@
|
||||
"""MCP tool surface for Spotify: the things a logged-in human does.
|
||||
|
||||
Plays tracks. This is accomplished via a Playwright session, which may require
|
||||
the user to manually input their credentials or solve a CAPTCHA.
|
||||
"""
|
||||
|
||||
OBJ = "object"
|
||||
|
||||
TOOLS = [
|
||||
{
|
||||
"name": "play_track",
|
||||
"description": "Opens a browser and plays the requested song.",
|
||||
"inputSchema": {
|
||||
"type": OBJ,
|
||||
"properties": {
|
||||
"track_name": {"type": "string"},
|
||||
"artist": {"type": "string"}
|
||||
},
|
||||
"required": ["track_name"]
|
||||
}
|
||||
}
|
||||
]
|
||||
@@ -152,7 +152,7 @@ def derive_mcp_config(tool: ToolDefinition) -> Optional[dict]:
|
||||
env["PYTHONPATH"] = (p_project_root + os.pathsep + existing_pp) if existing_pp else p_project_root
|
||||
|
||||
# The session-borrow social shims (reddit/x/tiktok) each run as a Python shim that borrows the user's live browser session via the backend's cookie bridge, so they need the localhost port + auth token, plus PYTHONPATH to import themselves.
|
||||
if tool.name.lower() in {"reddit", "x", "tiktok"} and config.get("type") == "stdio":
|
||||
if tool.name.lower() in {"reddit", "x", "tiktok", "spotify"} and config.get("type") == "stdio":
|
||||
from backend.auth import get_auth_token
|
||||
env = config.setdefault("env", {})
|
||||
env["OPENSWARM_PORT"] = os.environ.get("OPENSWARM_PORT", "8324")
|
||||
|
||||
@@ -24,4 +24,6 @@ swarm-analytics==0.1.1
|
||||
tzlocal==5.3.1
|
||||
# Test deps (pytest, pytest-asyncio) live in requirements-dev.txt — they
|
||||
# never ship to production users and shaved ~3 MB / ~200 files off the
|
||||
# Mac DMG when removed from the prod env.
|
||||
# Mac DMG when removed from the prod env.
|
||||
psutil==7.2.2
|
||||
playwright==1.61.0
|
||||
@@ -40,6 +40,19 @@ export const INTEGRATIONS: Integration[] = [
|
||||
</svg>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: 'spotify',
|
||||
name: "Spotify",
|
||||
description: 'Control playback, search tracks, and manage playlists.',
|
||||
mcp_config: { type: 'stdio', command: 'python', args: ['-m', 'backend.apps.spotify_mcp_shim'] },
|
||||
color: '#1DB954',
|
||||
website: 'https://open.spotify.com',
|
||||
icon: (
|
||||
<svg viewBox="0 0 24 24" width="22" height="22">
|
||||
<path fill="#1DB954" d="M12 0C5.373 0 0 5.373 0 12s5.373 12 12 12 12-5.373 12-12S18.627 0 12 0zm5.063 17.188a.75.75 0 0 1-1.031.254c-2.826-1.727-6.375-2.121-10.562-1.185a.75.75 0 1 1-.318-1.46c4.58-1.0 8.53-.544 11.6 1.337a.75.75 0 0 1 .31 1.054zm1.437-3.656a.937.937 0 0 1-1.29.316c-3.24-1.98-8.175-2.56-11.97-1.43a.937.937 0 1 1-.546-1.79c4.28-1.31 9.66-.66 13.48 1.67a.937.937 0 0 1 .325 1.23zM18.5 8.5a1.125 1.125 0 0 1-1.558.378c-3.63-2.45-9.72-2.67-13.0-1.49A1.125 1.125 0 0 1 4.06 5.6c3.8-1.34 10.22-1.08 14.06 1.77A1.125 1.125 0 0 1 18.5 8.5z"/>
|
||||
</svg>
|
||||
),
|
||||
},
|
||||
{
|
||||
id: 'tiktok',
|
||||
name: 'TikTok',
|
||||
|
||||
Reference in New Issue
Block a user