diff --git a/scripts/ci/lib/app-harness.js b/scripts/ci/lib/app-harness.js index 1fe3bc29..c5c41acd 100644 --- a/scripts/ci/lib/app-harness.js +++ b/scripts/ci/lib/app-harness.js @@ -133,6 +133,29 @@ function parsePerfMarks(log) { return marks; } +// Pure verdict on a backend.log: the log-based half of the boot check (provenance +// matches HEAD, the three perf marks exist, are ordered, and are not degenerate). +// Kept pure + exported so it can be mutation-tested (selftest-gate.js feeds it +// crafted broken logs and proves each guard fires) without launching the app. +// Returns { failures: string[], sha, marks }; empty failures == the log half passed. +function bootFailures({ log, headShort } = {}) { + const failures = []; + const sha = parseProvenanceSha(log || ''); + if (!sha) failures.push('no [provenance] line in backend.log'); + else if (headShort && sha !== headShort) failures.push(`provenance sha ${sha} != git HEAD ${headShort}`); + + const marks = parsePerfMarks(log || ''); + const missing = ['app-launch', 'first-paint', 'backend-http-ready'].filter((k) => !(k in marks)); + if (missing.length) missing.forEach((k) => failures.push(`missing [perf] ${k}`)); + else { + if (!(marks['app-launch'] <= marks['first-paint'] && marks['first-paint'] <= marks['backend-http-ready'])) { + failures.push(`[perf] marks out of order: ${JSON.stringify(marks)}`); + } + if (!(marks['backend-http-ready'] > 0)) failures.push('[perf] backend-http-ready not > 0 (degenerate marks)'); + } + return { failures, sha, marks }; +} + // Launch the app and poll its backend.log until it reports HTTP-ready (or time out). // Returns { child, log, port }. Caller is responsible for killApp(child). async function launchAndWait({ appPath, timeoutMs = 180000, freshLog = true } = {}) { @@ -175,5 +198,6 @@ module.exports = { attachToRunning, parseProvenanceSha, parsePerfMarks, + bootFailures, launchAndWait, }; diff --git a/scripts/ci/selftest-gate.js b/scripts/ci/selftest-gate.js new file mode 100644 index 00000000..4cc90569 --- /dev/null +++ b/scripts/ci/selftest-gate.js @@ -0,0 +1,58 @@ +#!/usr/bin/env node +// "Test the tests" - mutation testing of the gate's own logic. For each guard in +// the boot check, we feed a deliberately BROKEN backend.log and assert the guard +// fires (and that a good log passes). If breaking the input doesn't turn the gate +// red, the gate is theater; this script fails loudly when that happens. +// +// This covers the PURE, log-based assertions (provenance + perf). The live-process +// guards (signature --require-signed, wrong-token auth, verify-all aggregation, +// renderer paint) are fault-injected separately - see GATE_AUDIT.md. +// +// node scripts/ci/selftest-gate.js +// +// Exit 0 = every guard discriminates good from broken. Exit 1 = a guard is fake. + +'use strict'; +const h = require('./lib/app-harness'); + +let failed = 0; +function check(name, cond) { process.stdout.write(` ${cond ? 'ok ' : 'FAIL'} ${name}\n`); if (!cond) failed++; } +const caught = (log, head, re) => h.bootFailures({ log, headShort: head }).failures.some((f) => re.test(f)); + +const HEAD = 'abc123def456'; +const GOOD = [ + '[provenance] OpenSwarm 1.1.69 sha=abc123def456 channel=stable builtAt=2026-01-01T00:00:00Z', + '[perf] app-launch t=100', + '[perf] first-paint t=400', + '[perf] backend-http-ready t=4000', + 'Backend ready on port 8324', +].join('\n'); + +process.stdout.write('boot-check mutation tests:\n'); + +// Baseline: a good log must PASS (no false positives - the inverse failure mode). +check('good log -> 0 failures (no false alarm)', h.bootFailures({ log: GOOD, headShort: HEAD }).failures.length === 0); + +// Each mutation must be CAUGHT: +check('missing [provenance] -> caught', caught(GOOD.replace(/\[provenance\].*/, ''), HEAD, /provenance/)); +check('sha != HEAD -> caught', caught(GOOD.replace('abc123def456', '000000000000'), HEAD, /!= git HEAD/)); +check('missing first-paint mark -> caught', caught(GOOD.replace(/\[perf\] first-paint t=400\n/, ''), HEAD, /first-paint/)); +check('missing backend-http-ready mark -> caught', caught(GOOD.replace(/\[perf\] backend-http-ready t=4000/, ''), HEAD, /backend-http-ready/)); +check('out-of-order marks -> caught', caught(GOOD.replace('first-paint t=400', 'first-paint t=9999'), HEAD, /out of order/)); +check('degenerate all-zero marks -> caught', caught( + '[provenance] OpenSwarm 1 sha=abc123def456 channel=stable\n[perf] app-launch t=0\n[perf] first-paint t=0\n[perf] backend-http-ready t=0', + HEAD, /> 0|degenerate/)); + +// And a stale build (old sha) must be caught even with all marks fine - the exact +// real-world case we already saw fire live. +check('stale build (every mark fine, wrong sha) -> still caught', caught(GOOD.replace('abc123def456', 'deadbeef0000'), HEAD, /!= git HEAD/)); + +process.stdout.write('\nparse-function edge cases:\n'); +check('parseProvenanceSha reads a real line', h.parseProvenanceSha(GOOD) === 'abc123def456'); +check('parseProvenanceSha returns null on no marker', h.parseProvenanceSha('nothing here') === null); +check('parsePerfMarks finds all three', Object.keys(h.parsePerfMarks(GOOD)).length === 3); + +process.stdout.write(failed + ? `\nGATE SELFTEST FAIL: ${failed} guard(s) did not discriminate - the gate has theater in it.\n` + : '\nGATE SELFTEST PASS: every boot guard fires on a break and passes on good input.\n'); +process.exit(failed ? 1 : 0); diff --git a/scripts/ci/verify-packaged-app.js b/scripts/ci/verify-packaged-app.js index 7cab0523..7a051757 100644 --- a/scripts/ci/verify-packaged-app.js +++ b/scripts/ci/verify-packaged-app.js @@ -41,16 +41,9 @@ async function main() { child = res.child; const { log, port } = res; - // --- assertions --- - const provSha = h.parseProvenanceSha(log); - if (!provSha) fail('no [provenance] line in backend.log (app may not have booted)'); - if (headShort && provSha !== headShort) fail(`provenance sha ${provSha} != git HEAD ${headShort}`); - - const marks = h.parsePerfMarks(log); - for (const k of ['app-launch', 'first-paint', 'backend-http-ready']) if (!(k in marks)) fail(`missing [perf] ${k} in backend.log`); - if (!(marks['app-launch'] <= marks['first-paint'] && marks['first-paint'] <= marks['backend-http-ready'])) { - fail(`[perf] marks out of order: ${JSON.stringify(marks)}`); - } + // --- assertions (log half is pure + mutation-tested in selftest-gate.js) --- + const { failures, sha: provSha, marks } = h.bootFailures({ log, headShort }); + if (failures.length) fail(failures.join('; ')); if (port) { let code = 0;