From 5d8672b9a02abfa0961b9ef0f88f9a8f5e1be0a8 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 27 May 2026 15:41:13 -0700 Subject: [PATCH] [eric] test: add a check that runs one real agent turn on your own login and proves a genuine reply --- scripts/ci/lib/app-harness.js | 35 +++++++++ scripts/ci/verify-agent-turn.js | 123 ++++++++++++++++++++++++++++++++ 2 files changed, 158 insertions(+) create mode 100644 scripts/ci/verify-agent-turn.js diff --git a/scripts/ci/lib/app-harness.js b/scripts/ci/lib/app-harness.js index 974af6ff..1fe3bc29 100644 --- a/scripts/ci/lib/app-harness.js +++ b/scripts/ci/lib/app-harness.js @@ -86,6 +86,39 @@ function healthCode(port, timeoutMs = 3000) { }); } +// Authenticated JSON call to the running backend, the same way the app calls it. +// Returns { status, json, text }; status 0 means the request never completed. +function apiRequest(port, { method = 'GET', path = '/', token = '', body = null, timeoutMs = 30000 } = {}) { + return new Promise((resolve) => { + const data = body != null ? Buffer.from(JSON.stringify(body)) : null; + const headers = {}; + if (token) headers.Authorization = `Bearer ${token}`; + if (data) { headers['Content-Type'] = 'application/json'; headers['Content-Length'] = data.length; } + const req = http.request({ host: '127.0.0.1', port, path, method, headers }, (res) => { + let buf = ''; + res.on('data', (c) => { buf += c; }); + res.on('end', () => { let json = null; try { json = JSON.parse(buf); } catch { /* non-json */ } resolve({ status: res.statusCode, json, text: buf }); }); + }); + req.on('error', () => resolve({ status: 0, json: null, text: '' })); + req.setTimeout(timeoutMs, () => { req.destroy(); resolve({ status: 0, json: null, text: '' }); }); + if (data) req.write(data); + req.end(); + }); +} + +// Find an already-running app to reuse (so we exercise the user's logged-in +// creds) by reading the token off disk and the port out of the last backend.log, +// then confirming it actually answers. Returns { port, token } or null. +async function attachToRunning() { + const token = readFileSafe(authTokenPath()).trim(); + const m = readFileSafe(backendLogPath()).match(/Backend ready on port (\d+)/g); + if (!token || !m) return null; + const port = Number(m[m.length - 1].match(/(\d+)/)[1]); // last = most recent launch + if (!port) return null; + const code = await healthCode(port); + return code === 200 ? { port, token } : null; +} + function parseProvenanceSha(log) { const m = log.match(/\[provenance\] OpenSwarm \S+ sha=([0-9a-f]+)/); return m ? m[1] : null; @@ -138,6 +171,8 @@ module.exports = { spawnApp, killApp, healthCode, + apiRequest, + attachToRunning, parseProvenanceSha, parsePerfMarks, launchAndWait, diff --git a/scripts/ci/verify-agent-turn.js b/scripts/ci/verify-agent-turn.js new file mode 100644 index 00000000..865f234b --- /dev/null +++ b/scripts/ci/verify-agent-turn.js @@ -0,0 +1,123 @@ +#!/usr/bin/env node +// Drives a REAL agent turn end-to-end against the packaged app, using whatever +// provider the app is already logged into on THIS machine (your subscription / +// 9router / API key) — no separate key, no second agent. It proves the full +// round-trip: session launch -> message -> the model actually generated a reply. +// +// Why this is gated: a real turn spends a sliver of your LLM quota. So it only +// runs when OPENSWARM_E2E_AGENT=1; otherwise it prints a skip and exits 0, which +// keeps CI from silently billing you. +// +// How it proves a real reply (not a fake pass): it launches with NO tools (so the +// agent can't trip an approval gate and hang) and a trivial prompt, then polls the +// session until status is terminal and asserts the model produced output tokens +// (tokens.output > 0) with no error. Output tokens can only come from a live +// provider answering — they can't be faked by the harness. +// +// OPENSWARM_E2E_AGENT=1 node scripts/ci/verify-agent-turn.js [--app ] [--prompt "..."] +// +// Note: the [perf] first-agent-response mark is emitted by the RENDERER, so it is +// asserted in the GUI walkthrough, not here (this path is API-driven by design). + +'use strict'; +const h = require('./lib/app-harness'); + +function parseArgs(argv) { + const out = { app: null, prompt: 'Reply with exactly the single word: pong', timeoutMs: 120000 }; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--app') out.app = argv[++i]; + else if (argv[i] === '--prompt') out.prompt = argv[++i]; + else if (argv[i] === '--timeout-ms') out.timeoutMs = Number(argv[++i]); + } + return out; +} + +function providerForGroup(group) { + const g = group.toLowerCase(); + if (g.includes('openai')) return 'openai'; + if (g.includes('google') || g.includes('gemini')) return 'google'; + if (g.includes('openrouter')) return 'openrouter'; + return 'anthropic'; // Anthropic / OpenSwarm Pro / Claude / custom default +} + +// Pick a model from /api/agents/models, preferring a cheap Anthropic-family one. +function pickModel(modelsByGroup) { + const groups = Object.entries(modelsByGroup || {}); + const flat = groups.flatMap(([group, ms]) => (ms || []).map((m) => ({ value: m.value, group }))); + if (!flat.length) return null; + const pref = flat.find((m) => /haiku/i.test(m.value)) || flat.find((m) => /sonnet|claude/i.test(m.value)) || flat[0]; + return { model: pref.value, provider: providerForGroup(pref.group) }; +} + +async function main() { + if (process.env.OPENSWARM_E2E_AGENT !== '1') { + process.stdout.write('SKIP: agent-turn check is gated. Set OPENSWARM_E2E_AGENT=1 to run a real turn (spends LLM quota).\n'); + process.exit(0); + } + const args = parseArgs(process.argv.slice(2)); + + // Reuse the user's already-running, logged-in app if there is one; else launch. + let port, token, child = null, ownsApp = false; + const attached = await h.attachToRunning(); + if (attached) { + ({ port, token } = attached); + process.stdout.write(`Attached to running app on :${port}\n`); + } else { + const appPath = h.packagedAppPath(args.app); + process.stdout.write(`Launching: ${appPath}\n`); + const res = await h.launchAndWait({ appPath, timeoutMs: args.timeoutMs }); + child = res.child; ownsApp = true; port = res.port; + token = h.readFileSafe(h.authTokenPath()).trim(); + } + + const fail = (msg) => { if (ownsApp) h.killApp(child); process.stderr.write(`\nAGENT FAIL: ${msg}\n`); process.exit(1); }; + if (!port || !token) fail('no backend port / auth token available'); + + let sessionId = null; + try { + // 1. creds gate + const models = await h.apiRequest(port, { path: '/api/agents/models', token }); + if (models.status !== 200) fail(`GET /api/agents/models -> ${models.status}`); + const picked = pickModel(models.json && models.json.models); + if (!picked) { if (ownsApp) h.killApp(child); process.stdout.write('SKIP: no provider connected on this machine; a real turn is not testable here.\n'); process.exit(0); } + process.stdout.write(` using model ${picked.model} (provider ${picked.provider})\n`); + + // 2. launch a tool-free, single-turn session + const launch = await h.apiRequest(port, { + method: 'POST', path: '/api/agents/launch', token, + body: { name: 'e2e-agent-turn', model: picked.model, provider: picked.provider, mode: 'agent', allowed_tools: [], max_turns: 1 }, + }); + if (launch.status !== 200 || !launch.json || !launch.json.session_id) fail(`launch -> ${launch.status} ${launch.text.slice(0, 200)}`); + sessionId = launch.json.session_id; + + // 3. send the prompt + const send = await h.apiRequest(port, { method: 'POST', path: `/api/agents/sessions/${sessionId}/message`, token, body: { prompt: args.prompt } }); + if (send.status !== 200) fail(`send message -> ${send.status} ${send.text.slice(0, 200)}`); + + // 4. poll until terminal; a real reply == output tokens with no error + const TERMINAL = new Set(['completed', 'stopped', 'error']); + const t0 = Date.now(); + let session = null, status = ''; + while (Date.now() - t0 < args.timeoutMs) { + const g = await h.apiRequest(port, { path: `/api/agents/sessions/${sessionId}`, token }); + if (g.status === 200 && g.json) { session = g.json; status = session.status; if (TERMINAL.has(status)) break; } + await h.sleep(1500); + } + if (!session) fail('never read session state back'); + if (status === 'error') fail(`turn ended in error: ${JSON.stringify(session.error || session.last_error || 'unknown')}`); + if (!TERMINAL.has(status)) fail(`turn did not finish within ${Math.round(args.timeoutMs / 1000)}s (status=${status})`); + const out = (session.tokens && Number(session.tokens.output)) || 0; + if (out <= 0) fail(`turn completed but produced 0 output tokens (no real model reply)`); + + process.stdout.write(`\nAGENT PASS: real turn completed in ${Math.round((Date.now() - t0) / 1000)}s, ${out} output tokens from ${picked.model}.\n`); + } finally { + if (sessionId) { + await h.apiRequest(port, { method: 'POST', path: `/api/agents/sessions/${sessionId}/stop`, token }).catch(() => {}); + await h.apiRequest(port, { method: 'DELETE', path: `/api/agents/sessions/${sessionId}`, token }).catch(() => {}); + } + if (ownsApp) h.killApp(child); + } + process.exit(0); +} + +main().catch((e) => { process.stderr.write(`\nAGENT FAIL: ${e && e.message || e}\n`); process.exit(1); });