From e050cd6608b302ebace891d8d72d8b696c71effc Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 27 May 2026 16:00:19 -0700 Subject: [PATCH] [eric] test: add a playwright hand to drive the packaged app (launch, click, type, screenshot, read-dom, read-log) --- e2e/mcp/README.md | 67 ++++++++++++++++++++ e2e/mcp/electron-mcp.js | 132 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 199 insertions(+) create mode 100644 e2e/mcp/README.md create mode 100644 e2e/mcp/electron-mcp.js diff --git a/e2e/mcp/README.md b/e2e/mcp/README.md new file mode 100644 index 00000000..8f3a597b --- /dev/null +++ b/e2e/mcp/README.md @@ -0,0 +1,67 @@ +# openswarm-gui MCP: a Playwright hand for a Claude Code tester + +This is the "GUI hand" for the testing pyramid. The deterministic scripts in +`scripts/ci/` are the fast, free, binary CI gate (boot, signing, resilience, +network, agent turn). This MCP server covers the part scripts can't express: +**actual GUI behavior**, by letting a Claude Code instance you talk to drive the +real packaged app at Playwright (DOM) precision. + +It is NOT a replacement for the scripts. A CC tester *runs the scripts* for the +mechanical 95% and uses these tools for the exploratory/judgment 5% (does the +screen render, does clicking actually do something, does it look right) and for +escalation when a script is stuck or a result looks fake. + +## Setup (one-time, your call) + +Registering an auto-connecting MCP server modifies CC's own config, so add it +yourself, either: + +```bash +claude mcp add openswarm-gui -- node e2e/mcp/electron-mcp.js +``` + +or create `.mcp.json` at the repo root: + +```json +{ + "mcpServers": { + "openswarm-gui": { "command": "node", "args": ["e2e/mcp/electron-mcp.js"] } + } +} +``` + +Then `cd e2e && npm install` (pulls the MCP SDK + Playwright). Build the app first +so there's a packaged binary to drive (`electron/dist/win-unpacked` or the `.app`). + +## Tools + +| Tool | What it does | +| --- | --- | +| `app_launch` | launch the packaged app, wait for the main window, return backend port + build provenance | +| `app_close` | close the app | +| `screenshot` | PNG of the current window (the eyes) | +| `snapshot` | accessibility tree (structured "what's on screen", no pixels) | +| `click` / `fill` / `press` | drive inputs by Playwright selector (CSS, `text=`, `role=`) | +| `wait_for` | wait until a selector is visible | +| `eval` | run JS in the renderer and return JSON (inspect anything, incl. `window.openswarm`) | +| `read_log` | tail `backend.log` (provenance + `[perf]` marks + errors) | + +## Verification rubric (the prompt a CC tester follows) + +1. Run the deterministic gate first: `node scripts/ci/verify-all.js`. If anything + there fails, stop and report; the GUI walk only matters once boot/serve pass. +2. `app_launch`. Confirm the returned `build.sha` matches `git rev-parse HEAD`. +3. Walk every surface from `frontend/src/app/Main.tsx`: open each screen/tab, take + a `screenshot` + `snapshot`, and for each primary control `click` it and confirm + the follow-on state changes (new view, dialog opens, list updates). +4. Drive the core flow: start a new session, `fill` the prompt, send, confirm a + reply renders, then confirm `read_log` shows `[perf] first-agent-response` + (this is the renderer-driven mark the API-only agent-turn check can't assert). +5. Flag anything that renders blank, throws in the console (`eval` on + `window.__errors__` if present, or watch for empty `#root`), or looks visually + broken. Capture a screenshot with every flag. +6. `app_close`. + +A CC instance running this is the apex of the pyramid: interactive (you chat with +it), fully featured (it can also edit code, run the scripts, read any file), and +future-proof (any new CC capability is available the moment it ships). diff --git a/e2e/mcp/electron-mcp.js b/e2e/mcp/electron-mcp.js new file mode 100644 index 00000000..b13d28ae --- /dev/null +++ b/e2e/mcp/electron-mcp.js @@ -0,0 +1,132 @@ +#!/usr/bin/env node +// A tiny MCP server that hands a Claude Code instance the "hand + eyes" to drive +// the REAL packaged OpenSwarm app: launch it, click, type, screenshot, read the +// live DOM/accessibility tree, run JS in the renderer, and tail backend.log. +// +// Why this exists: the deterministic scripts in scripts/ci/ cover boot/serve/ +// resilience/network/agent-turn (fast, free, the CI gate). This covers the part +// scripts can't express - actual GUI behavior - by letting a CC instance you talk +// to drive the app at Playwright (DOM) precision. The official @playwright/mcp is +// browser-only, so we wrap Playwright's Electron (_electron) API ourselves. +// +// Register it (repo-root .mcp.json) and any CC instance auto-connects: +// { "mcpServers": { "openswarm-gui": { "command": "node", +// "args": ["e2e/mcp/electron-mcp.js"] } } } +// +// The launched ElectronApplication + main Page are held across calls, so a CC +// session clicks through a single live app the way a person would. + +'use strict'; +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { _electron } = require('@playwright/test'); +const { Server } = require('@modelcontextprotocol/sdk/server/index.js'); +const { StdioServerTransport } = require('@modelcontextprotocol/sdk/server/stdio.js'); +const { ListToolsRequestSchema, CallToolRequestSchema } = require('@modelcontextprotocol/sdk/types.js'); + +const REPO_ROOT = path.resolve(__dirname, '..', '..'); + +function packagedAppPath(explicit) { + if (explicit) return explicit; + if (process.env.E2E_APP_PATH) return process.env.E2E_APP_PATH; + const dist = path.join(REPO_ROOT, 'electron', 'dist'); + const candidates = process.platform === 'win32' + ? [path.join(dist, 'win-unpacked', 'OpenSwarm.exe')] + : process.platform === 'darwin' + ? ['mac-arm64', 'mac', 'mac-universal'].map((d) => path.join(dist, d, 'OpenSwarm.app', 'Contents', 'MacOS', 'OpenSwarm')) + : [path.join(dist, 'linux-unpacked', 'openswarm')]; + const found = candidates.find((c) => { try { return fs.statSync(c).isFile(); } catch { return false; } }); + if (!found) throw new Error(`packaged app not found; build first or pass appPath. Looked in:\n ${candidates.join('\n ')}`); + return found; +} + +function backendLogPath() { + if (process.platform === 'darwin') return path.join(os.homedir(), 'Library', 'Application Support', 'OpenSwarm', 'data', 'backend.log'); + if (process.platform === 'win32') return path.join(process.env.APPDATA || os.homedir(), 'OpenSwarm', 'data', 'backend.log'); + const xdg = process.env.XDG_DATA_HOME || path.join(os.homedir(), '.local', 'share'); + return path.join(xdg, 'OpenSwarm', 'data', 'backend.log'); +} + +let app = null; // ElectronApplication +let page = null; // main Page + +async function findMainWindow(timeoutMs = 120000) { + const deadline = Date.now() + timeoutMs; + while (Date.now() < deadline) { + for (const w of app.windows()) { + try { + const ready = await w.evaluate(() => { + const hasBridge = typeof window.openswarm?.getBackendPort === 'function'; + const root = document.getElementById('root'); + return hasBridge && !!root && root.childElementCount > 0; + }); + if (ready) return w; + } catch { /* navigating */ } + } + await new Promise((r) => setTimeout(r, 500)); + } + throw new Error('main window with a mounted React root never appeared'); +} + +function text(s) { return { content: [{ type: 'text', text: typeof s === 'string' ? s : JSON.stringify(s, null, 2) }] }; } +function err(s) { return { content: [{ type: 'text', text: `ERROR: ${s}` }], isError: true }; } +function needPage() { if (!page) throw new Error('no app open; call app_launch first'); } + +const TOOLS = [ + { name: 'app_launch', description: 'Launch the packaged OpenSwarm app and wait for the main window. Returns backend port + build provenance.', inputSchema: { type: 'object', properties: { appPath: { type: 'string', description: 'Optional explicit path to the packaged binary' } } } }, + { name: 'app_close', description: 'Close the running app.', inputSchema: { type: 'object', properties: {} } }, + { name: 'screenshot', description: 'Capture a PNG screenshot of the current main window (what a user would see).', inputSchema: { type: 'object', properties: { fullPage: { type: 'boolean' } } } }, + { name: 'snapshot', description: 'Return the accessibility tree of the page (structured "what is on screen" without pixels).', inputSchema: { type: 'object', properties: {} } }, + { name: 'click', description: 'Click an element by Playwright selector (CSS, text=..., role=..., etc.).', inputSchema: { type: 'object', properties: { selector: { type: 'string' } }, required: ['selector'] } }, + { name: 'fill', description: 'Type text into an input/textarea by selector (clears first).', inputSchema: { type: 'object', properties: { selector: { type: 'string' }, text: { type: 'string' } }, required: ['selector', 'text'] } }, + { name: 'press', description: 'Press a keyboard key (e.g. Enter, Escape, Control+A) on the focused element.', inputSchema: { type: 'object', properties: { key: { type: 'string' } }, required: ['key'] } }, + { name: 'wait_for', description: 'Wait until a selector is visible (default 30s).', inputSchema: { type: 'object', properties: { selector: { type: 'string' }, timeoutMs: { type: 'number' } }, required: ['selector'] } }, + { name: 'eval', description: 'Evaluate a JS expression in the renderer and return the JSON result (powerful: inspect anything, incl. window.openswarm).', inputSchema: { type: 'object', properties: { expression: { type: 'string' } }, required: ['expression'] } }, + { name: 'read_log', description: 'Return the tail of the app backend.log (provenance + [perf] marks + errors land here).', inputSchema: { type: 'object', properties: { tailLines: { type: 'number' } } } }, +]; + +async function handle(name, a) { + if (name === 'app_launch') { + if (app) { try { await app.close(); } catch { /* */ } app = null; page = null; } + app = await _electron.launch({ executablePath: packagedAppPath(a.appPath), args: [] }); + page = await findMainWindow(); + const info = await page.evaluate(async () => ({ + port: window.openswarm.getBackendPort ? await window.openswarm.getBackendPort() : null, + build: window.openswarm.getBuildInfo ? await window.openswarm.getBuildInfo() : null, + })); + return text({ launched: true, ...info }); + } + if (name === 'app_close') { + if (app) { try { await app.close(); } catch { /* */ } } + app = null; page = null; + return text('closed'); + } + if (name === 'screenshot') { needPage(); const buf = await page.screenshot({ fullPage: !!a.fullPage }); return { content: [{ type: 'image', data: buf.toString('base64'), mimeType: 'image/png' }] }; } + if (name === 'snapshot') { needPage(); return text(await page.accessibility.snapshot()); } + if (name === 'click') { needPage(); await page.click(a.selector, { timeout: 15000 }); return text(`clicked ${a.selector}`); } + if (name === 'fill') { needPage(); await page.fill(a.selector, a.text, { timeout: 15000 }); return text(`filled ${a.selector}`); } + if (name === 'press') { needPage(); await page.keyboard.press(a.key); return text(`pressed ${a.key}`); } + if (name === 'wait_for') { needPage(); await page.waitForSelector(a.selector, { state: 'visible', timeout: a.timeoutMs || 30000 }); return text(`visible: ${a.selector}`); } + if (name === 'eval') { needPage(); const r = await page.evaluate((expr) => eval(expr), a.expression); return text(r === undefined ? 'undefined' : r); } + if (name === 'read_log') { + const raw = (() => { try { return fs.readFileSync(backendLogPath(), 'utf8'); } catch { return ''; } })(); + const lines = raw.split(/\r?\n/); + const n = a.tailLines || 80; + return text(lines.slice(-n).join('\n') || '(backend.log empty or missing)'); + } + throw new Error(`unknown tool ${name}`); +} + +async function main() { + const server = new Server({ name: 'openswarm-gui', version: '0.1.0' }, { capabilities: { tools: {} } }); + server.setRequestHandler(ListToolsRequestSchema, async () => ({ tools: TOOLS })); + server.setRequestHandler(CallToolRequestSchema, async (req) => { + try { return await handle(req.params.name, req.params.arguments || {}); } + catch (e) { return err(e && e.message || String(e)); } + }); + process.on('exit', () => { try { app && app.close(); } catch { /* */ } }); + await server.connect(new StdioServerTransport()); +} + +main().catch((e) => { process.stderr.write(`electron-mcp fatal: ${e && e.stack || e}\n`); process.exit(1); });