From db6419fe7ec32550491384065e9ad5080a6a5fd4 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 28 May 2026 09:17:07 -0700 Subject: [PATCH] [eric] ci: add verify-dogfood that launches the packaged app, parses the [preflight2] line, INDEPENDENTLY verifies boot evidence (provenance==HEAD, perf marks ordered, health 200, no renderer-gone), and fails red when verdict and reality disagree, recording every run to dogfood-manifest.jsonl for the aggregator --- scripts/ci/verify-dogfood.js | 131 +++++++++++++++++++++++++++++++++++ 1 file changed, 131 insertions(+) create mode 100644 scripts/ci/verify-dogfood.js diff --git a/scripts/ci/verify-dogfood.js b/scripts/ci/verify-dogfood.js new file mode 100644 index 00000000..07d85275 --- /dev/null +++ b/scripts/ci/verify-dogfood.js @@ -0,0 +1,131 @@ +#!/usr/bin/env node +// Dogfood cross-check: launches the packaged app, extracts the [preflight2] verdict from backend.log, then INDEPENDENTLY verifies boot success (provenance == HEAD, perf marks ordered, health 200, no renderer-gone) and asserts the verdict actually correlates with reality. Records every run to dogfood-manifest.jsonl (one line per run) which the aggregator reads to compute false-positive/negative rates per check. The two bug classes this catches: +// verdict=ok but boot broken -> a missed check (preflight is theater) +// verdict=fail but boot fine -> false positive (will scare real users) +// Either causes a non-zero exit so the dogfood loop goes red and the v* tag is blocked. + +'use strict'; +const fs = require('fs'); +const path = require('path'); +const h = require('./lib/app-harness'); + +function parseArgs(argv) { + const out = { app: null, timeoutMs: 180000, manifest: null }; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--app') out.app = argv[++i]; + else if (argv[i] === '--timeout-ms') out.timeoutMs = Number(argv[++i]); + else if (argv[i] === '--manifest') out.manifest = argv[++i]; + } + return out; +} + +function parsePreflight2(log) { + // [preflight2] verdict= totalMs= + const m = log.match(/\[preflight2\] verdict=(\w+) totalMs=(\d+) (.*)/); + if (!m) return null; + const checks = {}; + const reasons = m[3].split(';').map((s) => s.trim()).filter(Boolean); + for (const r of reasons) { + const km = r.match(/^([a-z-]+):(\w+)\((.+)\)$/); + if (km) checks[km[1]] = { status: km[2], reason: km[3] }; + } + return { verdict: m[1], totalMs: Number(m[2]), checks, raw: m[0] }; +} + +function bootSuccessSignals(log, headShort, port) { + // Independent of the preflight: did the app actually paint and serve? + const { failures, sha, marks } = h.bootFailures({ log, headShort }); + return { + provenanceMatch: !failures.some((f) => /provenance|HEAD/.test(f)), + perfMarksOk: !failures.some((f) => /\[perf\]/.test(f)), + portParsed: !!port, + rendererCrashed: /renderer process gone/.test(log), + bootFailures: failures, + sha, + marks, + }; +} + +function classify(verdict, boot) { + // boot is "real success" iff provenance matches HEAD, perf marks ordered, port parsed, no renderer crash, no boot failures. + const realOk = boot.provenanceMatch && boot.perfMarksOk && boot.portParsed && !boot.rendererCrashed && boot.bootFailures.length === 0; + if (verdict === 'ok' && !realOk) return { mismatch: true, kind: 'false-negative', detail: 'verdict=ok but boot evidence disagrees' }; + if (verdict === 'fail' && realOk) return { mismatch: true, kind: 'false-positive', detail: 'verdict=fail but boot evidence is clean' }; + // warn is allowed in both directions; it's the soft-signal state. + return { mismatch: false, kind: realOk ? 'true-positive' : 'true-negative', detail: '' }; +} + +function appendManifest(manifestPath, entry) { + try { + fs.mkdirSync(path.dirname(manifestPath), { recursive: true }); + fs.appendFileSync(manifestPath, JSON.stringify(entry) + '\n'); + } catch (e) { process.stderr.write(` warn could not write manifest: ${e && e.message}\n`); } +} + +async function main() { + const args = parseArgs(process.argv.slice(2)); + const appPath = h.packagedAppPath(args.app); + const headShort = h.gitHeadShort(); + const manifestPath = args.manifest || path.join(h.REPO_ROOT, 'scripts', 'ci', 'dogfood-manifest.jsonl'); + + process.stdout.write(`Dogfood run: ${appPath}\n`); + const t0 = Date.now(); + const res = await h.launchAndWait({ appPath, timeoutMs: args.timeoutMs }); + const child = res.child; + let healthCode = 0; + try { + if (res.port) { for (let i = 0; i < 10 && healthCode !== 200; i++) { healthCode = await h.healthCode(res.port); if (healthCode !== 200) await h.sleep(1000); } } + // Wait an extra moment for the [preflight2] line to land (it fires async). + await h.sleep(3000); + const log = h.readFileSafe(res.logPath); + const preflight = parsePreflight2(log); + const boot = bootSuccessSignals(log, headShort, res.port); + const totalMs = Date.now() - t0; + + process.stdout.write(`\n boot.provenanceMatch = ${boot.provenanceMatch} (sha=${boot.sha})\n`); + process.stdout.write(` boot.perfMarksOk = ${boot.perfMarksOk} (${JSON.stringify(boot.marks)})\n`); + process.stdout.write(` boot.portParsed = ${boot.portParsed} (port=${res.port}, health=${healthCode})\n`); + process.stdout.write(` boot.rendererCrashed = ${boot.rendererCrashed}\n`); + process.stdout.write(` boot.failureList = ${JSON.stringify(boot.bootFailures)}\n`); + if (preflight) process.stdout.write(` preflight2.verdict = ${preflight.verdict} (totalMs=${preflight.totalMs})\n`); + else process.stdout.write(` preflight2 line = ABSENT (no verdict to cross-check)\n`); + + const verdict = preflight ? preflight.verdict : 'absent'; + const cls = classify(verdict, boot); + const entry = { + ts: new Date().toISOString(), + platform: process.platform, + arch: process.arch, + sha: headShort, + verdict, + preflightChecks: preflight ? preflight.checks : null, + boot, + healthCode, + classification: cls, + durationMs: totalMs, + }; + appendManifest(manifestPath, entry); + + if (cls.mismatch) { + process.stderr.write(`\nDOGFOOD FAIL: ${cls.kind} - ${cls.detail}\n`); + process.stderr.write(` This is the bug class the v* tag must NOT ship: verdict and reality disagree.\n`); + h.killApp(child); + process.exit(1); + } + if (!preflight) { + // Absent line is a separate failure mode (preflight didn't run or didn't log). + process.stderr.write(`\nDOGFOOD FAIL: no [preflight2] line in backend.log; the module did not fire or did not log\n`); + h.killApp(child); + process.exit(1); + } + h.killApp(child); + process.stdout.write(`\nDOGFOOD PASS: verdict=${verdict} agrees with independent boot evidence (${cls.kind}); recorded to ${manifestPath}\n`); + process.exit(0); + } catch (e) { + h.killApp(child); + process.stderr.write(`\nDOGFOOD FAIL: ${e && e.stack || e}\n`); + process.exit(1); + } +} + +main().catch((e) => { process.stderr.write(`\nDOGFOOD FAIL: ${e && e.stack || e}\n`); process.exit(1); });