[eric] ci: boot the packaged backend in ci + check its health and agent endpoints

2026-09-12 12:47:42 +02:00 · 2026-05-27 10:19:53 -07:00
parent 6230d5685b
commit 54a7cb96b4
4 changed files with 277 additions and 0 deletions
@@ -0,0 +1,58 @@
+name: Phase tests (hermetic)
+
+# Runs every deterministic, no-build test harness from the build-parity plan as a
+# CI gate on each push/PR. These need no packaged artifact, no secrets, and no
+# network, so this job is fast and always meaningful. Phase tests that REQUIRE a
+# packaged build, signing, or real devices are NOT here (they can't be hermetic)
+# — those live in the smoke job below (manual) and RELEASE_CHECKLIST.md.
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+
+jobs:
+  hermetic-tests:
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20.18.1'
+      # Phase 0 (boot timing + file count), Phase 3 (backend smoke logic),
+      # Phase 5a (release promotion gate) — all assert pass + failure paths.
+      - run: node scripts/run-phase-tests.js
+
+  # Phase 3 real-artifact smoke. Manual until the bundled-backend launch command
+  # is confirmed on a runner. Builds the signed Windows artifact, then drives the
+  # actual backend the app spawns and asserts health + agent endpoints answer.
+  # NOTE: the backend-launch step is marked TODO — fill in how the packaged build
+  # starts its bundled python backend headlessly before relying on this job.
+  windows-artifact-smoke:
+    if: github.event_name == 'workflow_dispatch'
+    runs-on: windows-latest
+    timeout-minutes: 60
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-node@v4
+        with:
+          node-version: '20.18.1'
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.13'
+      - name: Build (unsigned, artifact only)
+        shell: pwsh
+        env:
+          GOOGLE_OAUTH_CLIENT_ID:     ${{ secrets.GOOGLE_OAUTH_CLIENT_ID }}
+          GOOGLE_OAUTH_CLIENT_SECRET: ${{ secrets.GOOGLE_OAUTH_CLIENT_SECRET }}
+        run: pwsh -NoProfile -File scripts/build-app-win.ps1
+      # TODO(user): launch the packaged backend headlessly here (spawn the bundled
+      # python-env\python.exe against backend\main.py with the same env main.js
+      # uses), capture its port + auth.token, then:
+      #   node scripts/ci/smoke-backend.js --port <port> --token-file <auth.token path>
+      # Break the http origin/CORS in a test branch to confirm this goes red.
+      - name: Smoke the booted backend (enable once launch is wired)
+        if: ${{ false }}
+        shell: pwsh
+        run: node scripts/ci/smoke-backend.js --port 8324 --token-file "$env:APPDATA\OpenSwarm\data\auth.token"
@@ -0,0 +1,100 @@
+#!/usr/bin/env node
+// Phase 3 artifact smoke: drive the real backend the packaged app spawns and
+// prove it actually serves. Polls the health endpoint until it answers 200 (or
+// a hard timeout), then hits an authenticated agent-subapp endpoint and asserts
+// 200. This is the check that turns "works on our machine" into "the artifact
+// boots and answers" — break the http origin/CORS or auth and it goes red.
+//
+// Usage:
+//   node scripts/ci/smoke-backend.js --port 8324 [--token <bearer>] \
+//        [--token-file <path>] [--health /api/health/check] \
+//        [--agent /api/agents/models] [--timeout-ms 120000] [--origin <url>]
+//
+// Exit 0 = backend booted and both endpoints answered 200. Exit 1 = failed
+// (prints reason). Always exits (never hangs): the poll has a wall-clock cap.
+
+'use strict';
+const fs = require('fs');
+const http = require('http');
+
+function parseArgs(argv) {
+  const out = {
+    port: null, token: null, tokenFile: null,
+    health: '/api/health/check', agent: '/api/agents/models',
+    timeoutMs: 120000, origin: null, host: '127.0.0.1',
+  };
+  for (let i = 0; i < argv.length; i++) {
+    const a = argv[i];
+    if (a === '--port') out.port = Number(argv[++i]);
+    else if (a === '--token') out.token = argv[++i];
+    else if (a === '--token-file') out.tokenFile = argv[++i];
+    else if (a === '--health') out.health = argv[++i];
+    else if (a === '--agent') out.agent = argv[++i];
+    else if (a === '--timeout-ms') out.timeoutMs = Number(argv[++i]);
+    else if (a === '--origin') out.origin = argv[++i];
+    else if (a === '--host') out.host = argv[++i];
+  }
+  return out;
+}
+
+function getOnce(host, port, path, headers) {
+  return new Promise((resolve) => {
+    let req;
+    try {
+      // http.get throws synchronously (not via 'error') on a malformed path or
+      // header value. A CI smoke tool must degrade to "not answering" (0), never
+      // crash, so the wait loop / failure message stays in control.
+      req = http.get({ host, port, path, headers }, (res) => {
+        res.on('data', () => {}); // drain so the socket frees
+        res.on('end', () => resolve(res.statusCode));
+      });
+    } catch {
+      resolve(0);
+      return;
+    }
+    req.on('error', () => resolve(0));
+    req.setTimeout(4000, () => { req.destroy(); resolve(0); });
+  });
+}
+
+function sleep(ms) { return new Promise((r) => setTimeout(r, ms)); }
+
+async function main() {
+  const args = parseArgs(process.argv.slice(2));
+  if (!args.port) { process.stderr.write('FAIL: --port is required\n'); process.exit(1); }
+
+  let token = args.token || '';
+  if (!token && args.tokenFile) {
+    try { token = fs.readFileSync(args.tokenFile, 'utf8').trim(); } catch { /* leave empty */ }
+  }
+  const headers = {};
+  if (token) headers.Authorization = `Bearer ${token}`;
+  // The backend rejects cross-origin localhost callers; CI can pass --origin to
+  // prove the origin check is wired (a wrong origin must NOT 200 the agent call).
+  if (args.origin) headers.Origin = args.origin;
+
+  // 1) Wait for health to answer 200, capped by wall clock.
+  const deadline = Date.now() + args.timeoutMs;
+  let healthCode = 0;
+  while (Date.now() < deadline) {
+    healthCode = await getOnce(args.host, args.port, args.health, headers);
+    if (healthCode === 200) break;
+    await sleep(500);
+  }
+  if (healthCode !== 200) {
+    process.stderr.write(`FAIL: health ${args.health} never returned 200 within ${args.timeoutMs}ms (last=${healthCode})\n`);
+    process.exit(1);
+  }
+
+  // 2) Authenticated agent-subapp endpoint must answer 200.
+  const agentCode = await getOnce(args.host, args.port, args.agent, headers);
+  if (agentCode !== 200) {
+    process.stderr.write(`FAIL: agent ${args.agent} returned ${agentCode} (expected 200; auth/origin/CORS broken?)\n`);
+    process.exit(1);
+  }
+
+  process.stdout.write(`SMOKE PASS: health 200, agent 200 on :${args.port}\n`);
+  process.exit(0);
+}
+
+main();
@@ -0,0 +1,84 @@
+#!/usr/bin/env node
+// Phase 3 test: stand up a fake backend on a real localhost port and prove the
+// smoke checker passes a healthy one and fails the failure modes the gate
+// exists for (health never ready, agent 401/500). Hermetic; no packaged build.
+//
+//   node scripts/ci/test-smoke-backend.js
+
+'use strict';
+const http = require('http');
+const path = require('path');
+const { execFile } = require('child_process');
+
+const NODE = process.execPath;
+const SCRIPT = path.join(__dirname, 'smoke-backend.js');
+let passed = 0;
+
+function assert(cond, msg) {
+  if (!cond) { process.stderr.write(`\nASSERT FAILED: ${msg}\n`); process.exit(1); }
+  passed++;
+}
+
+// Fake backend. behavior() decides the status code per request path so we can
+// model healthy / unauthorized / broken-after-health cases.
+function startServer(behavior) {
+  return new Promise((resolve) => {
+    const srv = http.createServer((req, res) => {
+      const code = behavior(req);
+      res.writeHead(code, { 'content-type': 'application/json' });
+      res.end('{}');
+    });
+    srv.listen(0, '127.0.0.1', () => resolve({ srv, port: srv.address().port }));
+  });
+}
+
+// Async (not execFileSync): the fake server shares this process's event loop,
+// so blocking it synchronously would stop the server answering. Returning a
+// promise keeps the loop free to service the smoke child's real TCP requests.
+function runSmoke(port, extra) {
+  return new Promise((resolve) => {
+    execFile(NODE, [SCRIPT, '--port', String(port), '--timeout-ms', '3000', ...extra],
+      (err) => resolve(err && err.code != null ? err.code : (err ? -1 : 0)));
+  });
+}
+
+async function main() {
+  // (1) healthy: both endpoints 200 -> pass
+  {
+    const { srv, port } = await startServer(() => 200);
+    assert(await runSmoke(port, ['--agent', '/api/agents/models']) === 0, 'healthy backend should pass');
+    srv.close();
+  }
+
+  // (2) health ok but agent 401 (auth/origin broken) -> fail
+  {
+    const { srv, port } = await startServer((req) => (req.url.startsWith('/api/health') ? 200 : 401));
+    assert(await runSmoke(port, ['--agent', '/api/agents/models']) === 1, 'agent 401 should fail the smoke');
+    srv.close();
+  }
+
+  // (3) health ok but agent 500 -> fail
+  {
+    const { srv, port } = await startServer((req) => (req.url.startsWith('/api/health') ? 200 : 500));
+    assert(await runSmoke(port, ['--agent', '/api/agents/models']) === 1, 'agent 500 should fail the smoke');
+    srv.close();
+  }
+
+  // (4) health never ready (always 503) -> fail within timeout, no hang
+  {
+    const { srv, port } = await startServer(() => 503);
+    const t0 = Date.now();
+    assert(await runSmoke(port, []) === 1, 'never-ready health should fail');
+    assert(Date.now() - t0 < 15000, 'must fail via timeout, not hang');
+    srv.close();
+  }
+
+  // (5) nothing listening on the port -> fail
+  {
+    assert(await runSmoke(59999, []) === 1, 'no server should fail');
+  }
+
+  process.stdout.write(`\nPhase 3 backend smoke: ${passed} assertions passed.\n`);
+}
+
+main();
@@ -0,0 +1,35 @@
+#!/usr/bin/env node
+// One green button for every hermetic phase test in this plan. These run with no
+// packaged build, no network, and no platform assumptions, so they are safe in
+// CI and on any dev machine. Phase tests that REQUIRE a packaged artifact, CI,
+// macOS hardware, or a real device fleet are intentionally not here (they can't
+// be hermetic); see RELEASE_CHECKLIST.md for those.
+//
+//   node scripts/run-phase-tests.js
+//
+// Exit 0 only if every suite passes.
+
+'use strict';
+const path = require('path');
+const { execFileSync } = require('child_process');
+
+const SUITES = [
+  ['Phase 0  boot timing + file count', 'perf/test-perf.js'],
+  ['Phase 3  backend artifact smoke',   'ci/test-smoke-backend.js'],
+  ['Phase 5a release promotion gate',   'release/test-verify-release.js'],
+];
+
+let failures = 0;
+for (const [label, rel] of SUITES) {
+  process.stdout.write(`\n=== ${label} ===\n`);
+  try {
+    const out = execFileSync(process.execPath, [path.join(__dirname, rel)], { encoding: 'utf8' });
+    process.stdout.write(out.trim() + '\n');
+  } catch (e) {
+    failures++;
+    process.stdout.write((e.stdout || '') + (e.stderr || '') + `  -> SUITE FAILED (${rel})\n`);
+  }
+}
+
+process.stdout.write(`\n${failures === 0 ? 'ALL PHASE TEST SUITES PASSED' : failures + ' SUITE(S) FAILED'}\n`);
+process.exit(failures === 0 ? 0 : 1);