[ethan] fix: issues preventing playback, login persistence

backend/apps/spotify_mcp_shim/handlers.py

@@ -1,8 +1,7 @@
 from typing import Optional, Any, Dict
-import psutil
+import asyncio
 import json
 import urllib.request
-import urllib.error
 import os
 
 # Bypass the OpenSwarm proxy for local Chrome CDP connections
@@ -34,7 +33,9 @@ def p_ensure_chrome_cdp():
         "--remote-debugging-port=9223",
         f"--user-data-dir={profile_dir}",
         "--no-first-run",
-        "--no-default-browser-check"
+        "--no-default-browser-check",
+        "--restore-last-session",
+        "--password-store=basic"
     ], stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL)
     
     # Wait for it to spin up
@@ -50,7 +51,7 @@ def p_ensure_chrome_cdp():
             
     return False
 
-def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]:
+async def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]:
     """
     Using the Playwright MCP, we'll:
 
@@ -67,156 +68,191 @@ def play_track(track_name: str, artist: Optional[str]) -> Dict[str, Any]:
             "message": "Failed to auto-start Chrome with remote debugging on port 9223."
         }
 
-    import asyncio
+    from playwright.async_api import async_playwright
+    import urllib.parse
+    import base64
+    from io import BytesIO
+    from PIL import Image
     
-    def run_async_play_track():
-        async def inner():
-            from playwright.async_api import async_playwright
-            import urllib.parse
-            import base64
-            from io import BytesIO
-            from PIL import Image
+    from backend.apps.settings.settings import load_settings
+    from backend.apps.settings.credentials import get_anthropic_client_for_model
+    from backend.apps.agents.providers.registry import resolve_aux_model
+    
+    settings = load_settings()
+    try:
+        model_id, _ = await resolve_aux_model(settings, preferred_tier="haiku")
+        client = get_anthropic_client_for_model(settings, model_id)
+    except Exception as e:
+        return {
+            "is_error": True,
+            "is_human_intervention": False,
+            "message": f"Could not load AI client for screenshot analysis: {e}"
+        }
+    
+    try:
+        async with async_playwright() as p:
+            browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223")
+            context = browser.contexts[0]
             
-            from backend.apps.settings.settings import load_settings
-            from backend.apps.settings.credentials import get_anthropic_client_for_model
-            from backend.apps.agents.providers.registry import resolve_aux_model
+            page = None
+            for p_obj in context.pages:
+                if "spotify.com" in p_obj.url:
+                    page = p_obj
+                    break
             
-            settings = load_settings()
-            try:
-                model_id, _ = await resolve_aux_model(settings, preferred_tier="haiku")
-                client = get_anthropic_client_for_model(settings, model_id)
-            except Exception as e:
-                return {
-                    "is_error": True,
-                    "is_human_intervention": False,
-                    "message": f"Could not load AI client for screenshot analysis: {e}"
-                }
+            if not page:
+                page = await context.new_page()
             
-            try:
-                async with async_playwright() as p:
-                    browser = await p.chromium.connect_over_cdp("http://127.0.0.1:9223")
-                    context = browser.contexts[0]
-                    
-                    page = None
-                    for p_obj in context.pages:
-                        if "spotify.com" in p_obj.url:
-                            page = p_obj
-                            break
-                    
-                    if not page:
-                        page = await context.new_page()
-                    
-                    query = track_name
-                    if artist:
-                        query += f" {artist}"
-                        
-                    search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query)}"
-                    await page.goto(search_url)
-                    await page.wait_for_load_state("networkidle")
-                    await asyncio.sleep(2) # Give elements time to render
-
-                    prompt = f"""
+            query = track_name
+            if artist:
+                query += f" {artist}"
+                
+            search_url = f"https://open.spotify.com/search/{urllib.parse.quote(query)}"
+            await page.goto(search_url)
+            await page.wait_for_load_state("networkidle")
+            await asyncio.sleep(4) # Give elements time to render
+            
+            prompt = f"""
 I want to play the track '{query}' on Spotify. Look at this screenshot of the Spotify web UI.
 Determine the state and return ONLY a valid JSON object. Do not include markdown formatting or backticks, just the raw JSON.
 You must choose one of the following exact JSON structures:
 
 1. If a login prompt or overlay is blocking the UI:
 {{
-  "action": "human_intervention",
-  "message": "User needs to log in"
+"action": "human_intervention",
+"message": "User needs to log in"
 }}
 
-2. If the track '{query}' is already playing (pause button is visible):
+2. If the track '{query}' is already playing (a pause button is visible, or the now-playing bar at the bottom shows the track playing):
 {{
-  "action": "done",
-  "message": "Successfully started playback"
+"action": "done",
+"message": "Successfully started playback"
 }}
 
 3. Otherwise, find the Play button for the top search result and return its exact center coordinates as percentages (0 to 100) of the image width and height:
 {{
-  "action": "click",
-  "x_percent": 50.5,
-  "y_percent": 25.0,
-  "message": "Clicking play button"
+"action": "click",
+"x_percent": 50.5,
+"y_percent": 25.0,
+"message": "Clicking play button"
 }}
 
 CRITICAL: Return strictly valid JSON. Double check your quotes and commas.
 """.strip()
 
-                    for _ in range(3):
-                        screenshot_bytes = await page.screenshot()
+            for attempt in range(4):
+                if attempt > 0:
+                    # Wait a bit longer between clicks/checks to let the stream start
+                    await asyncio.sleep(4)
+                    
+                screenshot_bytes = await page.screenshot()
+                
+                img = Image.open(BytesIO(screenshot_bytes))
+                max_width = 1024
+                if img.width > max_width:
+                    ratio = max_width / img.width
+                    img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS)
+                buf = BytesIO()
+                img.convert("RGB").save(buf, format="JPEG", quality=45)
+                b64_img = base64.b64encode(buf.getvalue()).decode("utf-8")
+                
+                response = await client.messages.create(
+                    model=model_id,
+                    max_tokens=256,
+                    messages=[{
+                        "role": "user",
+                        "content": [
+                            {"type": "text", "text": prompt},
+                            {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64_img}}
+                        ]
+                    }]
+                )
+                
+                try:
+                    # Parse JSON from response safely, handling both Anthropic and OpenAI wrapper formats
+                    resp_text = ""
+                    if getattr(response, "content", None):
+                        content_block = response.content[0]
+                        resp_text = getattr(content_block, "text", str(content_block)).strip()
+                    else:
+                        choices = getattr(response, "choices", None)
+                        if not choices and hasattr(response, "model_dump"):
+                            choices = response.model_dump().get("choices")
+                        if not choices and hasattr(response, "dict"):
+                            choices = response.dict().get("choices")
+                        if choices and len(choices) > 0:
+                            choice = choices[0]
+                            if isinstance(choice, dict):
+                                msg = choice.get("message", {})
+                                if isinstance(msg, dict):
+                                    resp_text = msg.get("content", "")
+                                else:
+                                    resp_text = getattr(msg, "content", "")
+                            else:
+                                msg = getattr(choice, "message", None)
+                                resp_text = getattr(msg, "content", "")
+                    
+                    if not resp_text:
+                        raise ValueError(f"AI returned empty content or unsupported format! Full response: {response}")
+                    
+                    if resp_text.startswith("```json"):
+                        resp_text = resp_text[7:-3].strip()
+                    elif resp_text.startswith("```"):
+                        resp_text = resp_text[3:-3].strip()
                         
-                        img = Image.open(BytesIO(screenshot_bytes))
-                        max_width = 1024
-                        if img.width > max_width:
-                            ratio = max_width / img.width
-                            img = img.resize((max_width, int(img.height * ratio)), Image.LANCZOS)
-                        buf = BytesIO()
-                        img.convert("RGB").save(buf, format="JPEG", quality=45)
-                        b64_img = base64.b64encode(buf.getvalue()).decode("utf-8")
+                    action_data = json.loads(resp_text)
+                    
+                    if action_data["action"] == "human_intervention":
+                        return {
+                            "is_error": False,
+                            "is_human_intervention": True,
+                            "message": action_data.get("message", "Human intervention needed.")
+                        }
+                    elif action_data["action"] == "done":
+                        return {
+                            "is_error": False,
+                            "is_human_intervention": False,
+                            "message": action_data.get("message", "Task completed.")
+                        }
+                    elif action_data["action"] == "click":
+                        vp_w = await page.evaluate("window.innerWidth")
+                        vp_h = await page.evaluate("window.innerHeight")
                         
-                        response = await client.messages.create(
-                            model=model_id,
-                            max_tokens=256,
-                            messages=[{
-                                "role": "user",
-                                "content": [
-                                    {"type": "text", "text": prompt},
-                                    {"type": "image", "source": {"type": "base64", "media_type": "image/jpeg", "data": b64_img}}
-                                ]
-                            }]
-                        )
-                        
-                        try:
-                            # Parse JSON from response
-                            resp_text = response.content[0].text.strip()
-                            if resp_text.startswith("```json"):
-                                resp_text = resp_text[7:-3].strip()
-                            elif resp_text.startswith("```"):
-                                resp_text = resp_text[3:-3].strip()
-                                
-                            action_data = json.loads(resp_text)
+                        # Use exact image dimensions to calculate percentages, then map to CSS pixels
+                        if "x_percent" in action_data and "y_percent" in action_data:
+                            x_pct = float(action_data["x_percent"])
+                            y_pct = float(action_data["y_percent"])
+                            # If the AI accidentally returned pixels instead of percentages, cap them
+                            if x_pct > 100: x_pct = (x_pct / img.width) * 100
+                            if y_pct > 100: y_pct = (y_pct / img.height) * 100
                             
-                            if action_data["action"] == "human_intervention":
-                                return {
-                                    "is_error": False,
-                                    "is_human_intervention": True,
-                                    "message": action_data.get("message", "Human intervention needed.")
-                                }
-                            elif action_data["action"] == "done":
-                                return {
-                                    "is_error": False,
-                                    "is_human_intervention": False,
-                                    "message": action_data.get("message", "Task completed.")
-                                }
-                            elif action_data["action"] == "click":
-                                vp_w = await page.evaluate("window.innerWidth")
-                                vp_h = await page.evaluate("window.innerHeight")
-                                click_x = (float(action_data["x_percent"]) / 100.0) * vp_w
-                                click_y = (float(action_data["y_percent"]) / 100.0) * vp_h
-                                await page.mouse.click(click_x, click_y)
-                                await asyncio.sleep(2) # wait for playback to start
-                                # loop continues to verify
-                        except Exception as e:
-                            return {
-                                "is_error": True,
-                                "is_human_intervention": False,
-                                "message": f"Failed to parse AI response: {e}\nResponse was: {response.content[0].text}"
-                            }
+                            click_x = (x_pct / 100.0) * vp_w
+                            click_y = (y_pct / 100.0) * vp_h
+                        else:
+                            # Fallback if AI returned raw pixels (x, y)
+                            click_x = float(action_data["x"]) * (vp_w / img.width)
+                            click_y = float(action_data["y"]) * (vp_h / img.height)
                             
+                        await page.mouse.click(click_x, click_y)
+                        # sleep is now handled at the start of the next loop iteration
+                except Exception as e:
+                    raw_resp = getattr(response, 'content', 'No content attribute')
                     return {
                         "is_error": True,
                         "is_human_intervention": False,
-                        "message": "AI failed to start playback after multiple attempts."
+                        "message": f"Failed to parse AI response: {e}\nRaw content was: {raw_resp}"
                     }
                     
-            except Exception as e:
-                return {
-                    "is_error": True,
-                    "is_human_intervention": False,
-                    "message": f"An error occurred: {str(e)}"
-                }
-                
-        return asyncio.run(inner())
-        
-    return run_async_play_track()
+            return {
+                "is_error": True,
+                "is_human_intervention": False,
+                "message": "AI failed to start playback after multiple attempts."
+            }
+            
+    except Exception as e:
+        import traceback
+        return {
+            "is_error": True,
+            "is_human_intervention": False,
+            "message": f"An error occurred: {str(e)}\n\nTraceback:\n{traceback.format_exc()}"
+        }

backend/apps/spotify_mcp_shim/server.py

@@ -21,14 +21,20 @@ def p_ok(payload) -> dict:
         return {"content": [{"type": "text", "text": payload}]}
     return {"content": [{"type": "text", "text": json.dumps(payload, indent=2, default=str)}]}
 
-from backend.apps.spotify_mcp_shim.handlers import play_track
+import asyncio
+from backend.apps.spotify_mcp_shim import handlers
+
+def execute_tool_function(func, args: dict):
+    if asyncio.iscoroutinefunction(func):
+        return asyncio.run(func(**args))
+    return func(**args)
 
 def handle_tool_call(name: str, args: dict) -> dict:
-    match name:
-        case "play_track":
-            return p_ok(play_track(**args))
-
-    return p_err(f"Unknown tool: {name}")
+    handler = getattr(handlers, name, None)
+    if not handler or not callable(handler):
+        return p_err(f"Unknown tool: {name}")
+        
+    return p_ok(execute_tool_function(handler, args))
 
 def main():
     for line in sys.stdin:

frontend/src/app/pages/Tools/integrations.tsx

@@ -43,10 +43,14 @@ export const INTEGRATIONS: Integration[] = [
   {
     id: 'spotify',
     name: "Spotify",
-    description: 'Control playback, search tracks, and manage playlists.',
+    description: 'Control playback, search tracks, and manage playlists. (Opens an external Chrome window to bypass DRM)',
     mcp_config: { type: 'stdio', command: 'python', args: ['-m', 'backend.apps.spotify_mcp_shim'] },
     color: '#1DB954',
     website: 'https://open.spotify.com',
+    authType: 'browser_login',
+    connectLabel: 'Instructions',
+    loginUrl: '#',
+    connectInstructions: 'IMPORTANT: Spotify DRM blocks internal browsers. When you run a command for the first time, an external Google Chrome window will automatically open. Please sign into Spotify in that external window. Do NOT log in via the internal OpenSwarm browser, as playback will fail.',
     icon: (
       <svg viewBox="0 0 24 24" width="22" height="22">
         <path fill="#1DB954" d="M12 0C5.373 0 0 5.373 0 12s5.373 12 12 12 12-5.373 12-12S18.627 0 12 0zm5.063 17.188a.75.75 0 0 1-1.031.254c-2.826-1.727-6.375-2.121-10.562-1.185a.75.75 0 1 1-.318-1.46c4.58-1.0 8.53-.544 11.6 1.337a.75.75 0 0 1 .31 1.054zm1.437-3.656a.937.937 0 0 1-1.29.316c-3.24-1.98-8.175-2.56-11.97-1.43a.937.937 0 1 1-.546-1.79c4.28-1.31 9.66-.66 13.48 1.67a.937.937 0 0 1 .325 1.23zM18.5 8.5a1.125 1.125 0 0 1-1.558.378c-3.63-2.45-9.72-2.67-13.0-1.49A1.125 1.125 0 0 1 4.06 5.6c3.8-1.34 10.22-1.08 14.06 1.77A1.125 1.125 0 0 1 18.5 8.5z"/>