From 54a7cb96b4ab665b7416b8778bd5a5bdba172ac8 Mon Sep 17 00:00:00 2001 From: Eric Date: Wed, 27 May 2026 10:19:53 -0700 Subject: [PATCH] [eric] ci: boot the packaged backend in ci + check its health and agent endpoints --- .github/workflows/phase-tests.yml | 58 +++++++++++++++++ scripts/ci/smoke-backend.js | 100 ++++++++++++++++++++++++++++++ scripts/ci/test-smoke-backend.js | 84 +++++++++++++++++++++++++ scripts/run-phase-tests.js | 35 +++++++++++ 4 files changed, 277 insertions(+) create mode 100644 .github/workflows/phase-tests.yml create mode 100644 scripts/ci/smoke-backend.js create mode 100644 scripts/ci/test-smoke-backend.js create mode 100644 scripts/run-phase-tests.js diff --git a/.github/workflows/phase-tests.yml b/.github/workflows/phase-tests.yml new file mode 100644 index 00000000..8f44143e --- /dev/null +++ b/.github/workflows/phase-tests.yml @@ -0,0 +1,58 @@ +name: Phase tests (hermetic) + +# Runs every deterministic, no-build test harness from the build-parity plan as a +# CI gate on each push/PR. These need no packaged artifact, no secrets, and no +# network, so this job is fast and always meaningful. Phase tests that REQUIRE a +# packaged build, signing, or real devices are NOT here (they can't be hermetic) +# — those live in the smoke job below (manual) and RELEASE_CHECKLIST.md. + +on: + push: + pull_request: + workflow_dispatch: + +jobs: + hermetic-tests: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20.18.1' + # Phase 0 (boot timing + file count), Phase 3 (backend smoke logic), + # Phase 5a (release promotion gate) — all assert pass + failure paths. + - run: node scripts/run-phase-tests.js + + # Phase 3 real-artifact smoke. Manual until the bundled-backend launch command + # is confirmed on a runner. Builds the signed Windows artifact, then drives the + # actual backend the app spawns and asserts health + agent endpoints answer. + # NOTE: the backend-launch step is marked TODO — fill in how the packaged build + # starts its bundled python backend headlessly before relying on this job. + windows-artifact-smoke: + if: github.event_name == 'workflow_dispatch' + runs-on: windows-latest + timeout-minutes: 60 + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-node@v4 + with: + node-version: '20.18.1' + - uses: actions/setup-python@v5 + with: + python-version: '3.13' + - name: Build (unsigned, artifact only) + shell: pwsh + env: + GOOGLE_OAUTH_CLIENT_ID: ${{ secrets.GOOGLE_OAUTH_CLIENT_ID }} + GOOGLE_OAUTH_CLIENT_SECRET: ${{ secrets.GOOGLE_OAUTH_CLIENT_SECRET }} + run: pwsh -NoProfile -File scripts/build-app-win.ps1 + # TODO(user): launch the packaged backend headlessly here (spawn the bundled + # python-env\python.exe against backend\main.py with the same env main.js + # uses), capture its port + auth.token, then: + # node scripts/ci/smoke-backend.js --port --token-file + # Break the http origin/CORS in a test branch to confirm this goes red. + - name: Smoke the booted backend (enable once launch is wired) + if: ${{ false }} + shell: pwsh + run: node scripts/ci/smoke-backend.js --port 8324 --token-file "$env:APPDATA\OpenSwarm\data\auth.token" diff --git a/scripts/ci/smoke-backend.js b/scripts/ci/smoke-backend.js new file mode 100644 index 00000000..5164b371 --- /dev/null +++ b/scripts/ci/smoke-backend.js @@ -0,0 +1,100 @@ +#!/usr/bin/env node +// Phase 3 artifact smoke: drive the real backend the packaged app spawns and +// prove it actually serves. Polls the health endpoint until it answers 200 (or +// a hard timeout), then hits an authenticated agent-subapp endpoint and asserts +// 200. This is the check that turns "works on our machine" into "the artifact +// boots and answers" — break the http origin/CORS or auth and it goes red. +// +// Usage: +// node scripts/ci/smoke-backend.js --port 8324 [--token ] \ +// [--token-file ] [--health /api/health/check] \ +// [--agent /api/agents/models] [--timeout-ms 120000] [--origin ] +// +// Exit 0 = backend booted and both endpoints answered 200. Exit 1 = failed +// (prints reason). Always exits (never hangs): the poll has a wall-clock cap. + +'use strict'; +const fs = require('fs'); +const http = require('http'); + +function parseArgs(argv) { + const out = { + port: null, token: null, tokenFile: null, + health: '/api/health/check', agent: '/api/agents/models', + timeoutMs: 120000, origin: null, host: '127.0.0.1', + }; + for (let i = 0; i < argv.length; i++) { + const a = argv[i]; + if (a === '--port') out.port = Number(argv[++i]); + else if (a === '--token') out.token = argv[++i]; + else if (a === '--token-file') out.tokenFile = argv[++i]; + else if (a === '--health') out.health = argv[++i]; + else if (a === '--agent') out.agent = argv[++i]; + else if (a === '--timeout-ms') out.timeoutMs = Number(argv[++i]); + else if (a === '--origin') out.origin = argv[++i]; + else if (a === '--host') out.host = argv[++i]; + } + return out; +} + +function getOnce(host, port, path, headers) { + return new Promise((resolve) => { + let req; + try { + // http.get throws synchronously (not via 'error') on a malformed path or + // header value. A CI smoke tool must degrade to "not answering" (0), never + // crash, so the wait loop / failure message stays in control. + req = http.get({ host, port, path, headers }, (res) => { + res.on('data', () => {}); // drain so the socket frees + res.on('end', () => resolve(res.statusCode)); + }); + } catch { + resolve(0); + return; + } + req.on('error', () => resolve(0)); + req.setTimeout(4000, () => { req.destroy(); resolve(0); }); + }); +} + +function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); } + +async function main() { + const args = parseArgs(process.argv.slice(2)); + if (!args.port) { process.stderr.write('FAIL: --port is required\n'); process.exit(1); } + + let token = args.token || ''; + if (!token && args.tokenFile) { + try { token = fs.readFileSync(args.tokenFile, 'utf8').trim(); } catch { /* leave empty */ } + } + const headers = {}; + if (token) headers.Authorization = `Bearer ${token}`; + // The backend rejects cross-origin localhost callers; CI can pass --origin to + // prove the origin check is wired (a wrong origin must NOT 200 the agent call). + if (args.origin) headers.Origin = args.origin; + + // 1) Wait for health to answer 200, capped by wall clock. + const deadline = Date.now() + args.timeoutMs; + let healthCode = 0; + while (Date.now() < deadline) { + healthCode = await getOnce(args.host, args.port, args.health, headers); + if (healthCode === 200) break; + await sleep(500); + } + if (healthCode !== 200) { + process.stderr.write(`FAIL: health ${args.health} never returned 200 within ${args.timeoutMs}ms (last=${healthCode})\n`); + process.exit(1); + } + + // 2) Authenticated agent-subapp endpoint must answer 200. + const agentCode = await getOnce(args.host, args.port, args.agent, headers); + if (agentCode !== 200) { + process.stderr.write(`FAIL: agent ${args.agent} returned ${agentCode} (expected 200; auth/origin/CORS broken?)\n`); + process.exit(1); + } + + process.stdout.write(`SMOKE PASS: health 200, agent 200 on :${args.port}\n`); + process.exit(0); +} + +main(); diff --git a/scripts/ci/test-smoke-backend.js b/scripts/ci/test-smoke-backend.js new file mode 100644 index 00000000..20e20ba6 --- /dev/null +++ b/scripts/ci/test-smoke-backend.js @@ -0,0 +1,84 @@ +#!/usr/bin/env node +// Phase 3 test: stand up a fake backend on a real localhost port and prove the +// smoke checker passes a healthy one and fails the failure modes the gate +// exists for (health never ready, agent 401/500). Hermetic; no packaged build. +// +// node scripts/ci/test-smoke-backend.js + +'use strict'; +const http = require('http'); +const path = require('path'); +const { execFile } = require('child_process'); + +const NODE = process.execPath; +const SCRIPT = path.join(__dirname, 'smoke-backend.js'); +let passed = 0; + +function assert(cond, msg) { + if (!cond) { process.stderr.write(`\nASSERT FAILED: ${msg}\n`); process.exit(1); } + passed++; +} + +// Fake backend. behavior() decides the status code per request path so we can +// model healthy / unauthorized / broken-after-health cases. +function startServer(behavior) { + return new Promise((resolve) => { + const srv = http.createServer((req, res) => { + const code = behavior(req); + res.writeHead(code, { 'content-type': 'application/json' }); + res.end('{}'); + }); + srv.listen(0, '127.0.0.1', () => resolve({ srv, port: srv.address().port })); + }); +} + +// Async (not execFileSync): the fake server shares this process's event loop, +// so blocking it synchronously would stop the server answering. Returning a +// promise keeps the loop free to service the smoke child's real TCP requests. +function runSmoke(port, extra) { + return new Promise((resolve) => { + execFile(NODE, [SCRIPT, '--port', String(port), '--timeout-ms', '3000', ...extra], + (err) => resolve(err && err.code != null ? err.code : (err ? -1 : 0))); + }); +} + +async function main() { + // (1) healthy: both endpoints 200 -> pass + { + const { srv, port } = await startServer(() => 200); + assert(await runSmoke(port, ['--agent', '/api/agents/models']) === 0, 'healthy backend should pass'); + srv.close(); + } + + // (2) health ok but agent 401 (auth/origin broken) -> fail + { + const { srv, port } = await startServer((req) => (req.url.startsWith('/api/health') ? 200 : 401)); + assert(await runSmoke(port, ['--agent', '/api/agents/models']) === 1, 'agent 401 should fail the smoke'); + srv.close(); + } + + // (3) health ok but agent 500 -> fail + { + const { srv, port } = await startServer((req) => (req.url.startsWith('/api/health') ? 200 : 500)); + assert(await runSmoke(port, ['--agent', '/api/agents/models']) === 1, 'agent 500 should fail the smoke'); + srv.close(); + } + + // (4) health never ready (always 503) -> fail within timeout, no hang + { + const { srv, port } = await startServer(() => 503); + const t0 = Date.now(); + assert(await runSmoke(port, []) === 1, 'never-ready health should fail'); + assert(Date.now() - t0 < 15000, 'must fail via timeout, not hang'); + srv.close(); + } + + // (5) nothing listening on the port -> fail + { + assert(await runSmoke(59999, []) === 1, 'no server should fail'); + } + + process.stdout.write(`\nPhase 3 backend smoke: ${passed} assertions passed.\n`); +} + +main(); diff --git a/scripts/run-phase-tests.js b/scripts/run-phase-tests.js new file mode 100644 index 00000000..db74d52b --- /dev/null +++ b/scripts/run-phase-tests.js @@ -0,0 +1,35 @@ +#!/usr/bin/env node +// One green button for every hermetic phase test in this plan. These run with no +// packaged build, no network, and no platform assumptions, so they are safe in +// CI and on any dev machine. Phase tests that REQUIRE a packaged artifact, CI, +// macOS hardware, or a real device fleet are intentionally not here (they can't +// be hermetic); see RELEASE_CHECKLIST.md for those. +// +// node scripts/run-phase-tests.js +// +// Exit 0 only if every suite passes. + +'use strict'; +const path = require('path'); +const { execFileSync } = require('child_process'); + +const SUITES = [ + ['Phase 0 boot timing + file count', 'perf/test-perf.js'], + ['Phase 3 backend artifact smoke', 'ci/test-smoke-backend.js'], + ['Phase 5a release promotion gate', 'release/test-verify-release.js'], +]; + +let failures = 0; +for (const [label, rel] of SUITES) { + process.stdout.write(`\n=== ${label} ===\n`); + try { + const out = execFileSync(process.execPath, [path.join(__dirname, rel)], { encoding: 'utf8' }); + process.stdout.write(out.trim() + '\n'); + } catch (e) { + failures++; + process.stdout.write((e.stdout || '') + (e.stderr || '') + ` -> SUITE FAILED (${rel})\n`); + } +} + +process.stdout.write(`\n${failures === 0 ? 'ALL PHASE TEST SUITES PASSED' : failures + ' SUITE(S) FAILED'}\n`); +process.exit(failures === 0 ? 0 : 1);