From b0b3bff91df00e850270ae33ed206247f9b4e4a1 Mon Sep 17 00:00:00 2001 From: Eric Date: Thu, 28 May 2026 09:17:15 -0700 Subject: [PATCH] [eric] ci: add dogfood-aggregator that walks the rolling manifest, computes per-platform per-check warn/fail rates, emits preflight-tunings.json with a demote list when a check exceeds 2x the false-positive tolerance, and writes a release-readiness block the v* gate reads; preflight.js loadTunings/applyTunings honors the demote list at runtime so a chronically-noisy check gets silently downgraded to warn on subsequent boots --- electron/preflight.js | 29 ++++++- scripts/ci/dogfood-aggregator.js | 101 +++++++++++++++++++++++++ scripts/ci/verify-release-readiness.js | 54 +++++++++++++ 3 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 scripts/ci/dogfood-aggregator.js create mode 100644 scripts/ci/verify-release-readiness.js diff --git a/electron/preflight.js b/electron/preflight.js index 435de1e5..94510fee 100644 --- a/electron/preflight.js +++ b/electron/preflight.js @@ -196,6 +196,28 @@ async function checkClock(env, opts = {}) { }); } +// Load auto-tunings emitted by the dogfood aggregator: a check the loop has +// repeatedly seen false-positive on its platform is silently downgraded from +// fail -> warn here so a known-noisy probe can't single-handedly scare a user. +// File is bundled at build time (scripts/ci/preflight-tunings.json -> resources). +function loadTunings(env) { + const candidates = [ + path.join(__dirname, '..', 'scripts', 'ci', 'preflight-tunings.json'), + path.join(process.resourcesPath || '', 'preflight-tunings.json'), + ]; + for (const p of candidates) { + try { return JSON.parse(env.fs.readFileSync(p, 'utf8')); } catch {} + } + return null; +} + +function applyTunings(results, tunings, platform) { + if (!tunings || !Array.isArray(tunings.demote)) return results; + const demoted = new Set(tunings.demote.filter((d) => d.platform === platform).map((d) => d.check)); + if (!demoted.size) return results; + return results.map((r) => (r.status === 'fail' && demoted.has(r.name)) ? { ...r, status: 'warn', reason: `${r.reason} [auto-demoted by dogfood tuning]` } : r); +} + async function run(env, opts = {}) { env = env || defaultEnv(); const tasks = [ @@ -209,9 +231,11 @@ async function run(env, opts = {}) { withTimeout('dual-stack', () => checkDualStack(env, opts.dualStack), 3500), withTimeout('clock', () => checkClock(env, opts.clock), 3500), ]; - const results = await Promise.all(tasks); + const rawResults = await Promise.all(tasks); + const tunings = loadTunings(env); + const results = applyTunings(rawResults, tunings, env.platform); const verdict = results.some((r) => r.status === 'fail') ? 'fail' : results.some((r) => r.status === 'warn') ? 'warn' : 'ok'; - return { verdict, results, totalMs: Math.max(...results.map((r) => r.durationMs)), startedAt: env.now() }; + return { verdict, results, totalMs: Math.max(...results.map((r) => r.durationMs)), startedAt: env.now(), tuningsApplied: tunings ? tunings.demote.length : 0 }; } function cachePath(dataDir, appVersion) { return path.join(dataDir, `preflight-${appVersion}.json`); } @@ -246,4 +270,5 @@ module.exports = { checkOs, checkResources, checkAppdataWritable, checkSecurityBlock, checkSystemLibs, checkNetwork, checkGpu, checkDualStack, checkClock, run, cachePath, readCache, writeCache, pruneOldCaches, + loadTunings, applyTunings, }; diff --git a/scripts/ci/dogfood-aggregator.js b/scripts/ci/dogfood-aggregator.js new file mode 100644 index 00000000..ad4e2826 --- /dev/null +++ b/scripts/ci/dogfood-aggregator.js @@ -0,0 +1,101 @@ +#!/usr/bin/env node +// Reads every line of dogfood-manifest.jsonl, computes per-check warn+fail rates, identifies checks that DISAGREE with reality (verdict says fail but boot was fine, or check warned on >2x baseline runs), and emits preflight-tunings.json which the preflight module reads to auto-demote a noisy check (bump its timeout, or downgrade fail->warn). Also emits a release-readiness summary the gate uses. + +'use strict'; +const fs = require('fs'); +const path = require('path'); +const h = require('./lib/app-harness'); + +function parseArgs(argv) { + const out = { manifest: null, tunings: null, minRuns: 12, falsePositiveTolerance: 0.10 }; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--manifest') out.manifest = argv[++i]; + else if (argv[i] === '--tunings') out.tunings = argv[++i]; + else if (argv[i] === '--min-runs') out.minRuns = Number(argv[++i]); + else if (argv[i] === '--fp-tolerance') out.falsePositiveTolerance = Number(argv[++i]); + } + return out; +} + +function readManifest(p) { + let text = ''; + try { text = fs.readFileSync(p, 'utf8'); } catch { return []; } + return text.split(/\r?\n/).filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean); +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const manifestPath = args.manifest || path.join(h.REPO_ROOT, 'scripts', 'ci', 'dogfood-manifest.jsonl'); + const tuningsPath = args.tunings || path.join(h.REPO_ROOT, 'scripts', 'ci', 'preflight-tunings.json'); + const runs = readManifest(manifestPath); + + process.stdout.write(`Manifest: ${manifestPath}\n`); + process.stdout.write(`Runs: ${runs.length}\n`); + if (runs.length === 0) { process.stdout.write('\nAGGREGATE: no runs yet; nothing to tune.\n'); return; } + + // Per-platform stats so a noisy-on-Windows-only check doesn't get demoted globally. + const byPlatform = {}; + for (const r of runs) { + const p = r.platform || 'unknown'; + if (!byPlatform[p]) byPlatform[p] = { runs: [], total: 0, mismatches: 0, falsePositives: 0, falseNegatives: 0, checkStats: {} }; + const slot = byPlatform[p]; + slot.runs.push(r); + slot.total++; + if (r.classification && r.classification.mismatch) { + slot.mismatches++; + if (r.classification.kind === 'false-positive') slot.falsePositives++; + if (r.classification.kind === 'false-negative') slot.falseNegatives++; + } + for (const [name, info] of Object.entries(r.preflightChecks || {})) { + if (!slot.checkStats[name]) slot.checkStats[name] = { warn: 0, fail: 0, total: 0 }; + slot.checkStats[name].total++; + if (info.status === 'warn') slot.checkStats[name].warn++; + if (info.status === 'fail') slot.checkStats[name].fail++; + } + } + + // Identify tuning candidates: checks whose warn-rate on a platform exceeds the + // tolerance AND the platform's overall boots are mostly successful. These are + // false-positive sources that need either a longer timeout or a demoted threshold. + const tunings = { generatedAt: new Date().toISOString(), perPlatform: {}, demote: [] }; + for (const [p, slot] of Object.entries(byPlatform)) { + tunings.perPlatform[p] = { runs: slot.total, mismatches: slot.mismatches, falsePositiveRate: slot.total ? slot.falsePositives / slot.total : 0, falseNegativeRate: slot.total ? slot.falseNegatives / slot.total : 0 }; + for (const [name, st] of Object.entries(slot.checkStats)) { + const warnRate = st.warn / Math.max(1, st.total); + if (warnRate > 2 * args.falsePositiveTolerance && slot.falsePositives / Math.max(1, slot.total) > args.falsePositiveTolerance) { + tunings.demote.push({ platform: p, check: name, warnRate, action: 'demote-to-warn-only' }); + } + } + } + + fs.writeFileSync(tuningsPath, JSON.stringify(tunings, null, 2)); + process.stdout.write(`Tunings written: ${tuningsPath}\n`); + for (const [p, slot] of Object.entries(byPlatform)) { + process.stdout.write(`\n ${p}: ${slot.total} runs, ${slot.mismatches} mismatches (${slot.falsePositives} false-positive, ${slot.falseNegatives} false-negative)\n`); + for (const [name, st] of Object.entries(slot.checkStats)) { + const wr = ((st.warn / st.total) * 100).toFixed(1); + const fr = ((st.fail / st.total) * 100).toFixed(1); + process.stdout.write(` ${name.padEnd(20)} warn=${wr}% fail=${fr}% (n=${st.total})\n`); + } + } + + // Release readiness: consecutive-clean-runs window per platform. The v* tag + // gate fails unless every platform has >= minRuns runs with zero mismatches + // in its tail window. + let ready = true; + const readiness = {}; + for (const [p, slot] of Object.entries(byPlatform)) { + const tail = slot.runs.slice(-args.minRuns); + const tailMismatches = tail.filter((r) => r.classification && r.classification.mismatch).length; + const consecutiveClean = tail.length === args.minRuns && tailMismatches === 0; + readiness[p] = { tailSize: tail.length, tailMismatches, ready: consecutiveClean }; + if (!consecutiveClean) ready = false; + } + tunings.releaseReadiness = { ready, perPlatform: readiness, minRuns: args.minRuns }; + fs.writeFileSync(tuningsPath, JSON.stringify(tunings, null, 2)); + process.stdout.write(`\nRelease readiness: ${ready ? 'READY' : 'NOT READY'} (need ${args.minRuns} consecutive clean runs per platform)\n`); + for (const [p, r] of Object.entries(readiness)) process.stdout.write(` ${p}: ${r.tailSize}/${args.minRuns} clean=${r.tailMismatches === 0}\n`); + process.exit(0); +} + +main(); diff --git a/scripts/ci/verify-release-readiness.js b/scripts/ci/verify-release-readiness.js new file mode 100644 index 00000000..ec73d3bc --- /dev/null +++ b/scripts/ci/verify-release-readiness.js @@ -0,0 +1,54 @@ +#!/usr/bin/env node +// The v* tag gate. Reads preflight-tunings.json (produced by the aggregator) and asserts every platform has the required number of consecutive clean dogfood runs AND zero outstanding tuning candidates. Exits non-zero if any platform isn't ready, blocking the release workflow from publishing. + +'use strict'; +const fs = require('fs'); +const path = require('path'); +const h = require('./lib/app-harness'); + +function parseArgs(argv) { + const out = { tunings: null, requirePlatforms: ['win32', 'darwin'] }; + for (let i = 0; i < argv.length; i++) { + if (argv[i] === '--tunings') out.tunings = argv[++i]; + else if (argv[i] === '--require') out.requirePlatforms = argv[++i].split(','); + } + return out; +} + +function main() { + const args = parseArgs(process.argv.slice(2)); + const tuningsPath = args.tunings || path.join(h.REPO_ROOT, 'scripts', 'ci', 'preflight-tunings.json'); + let tunings; + try { tunings = JSON.parse(fs.readFileSync(tuningsPath, 'utf8')); } + catch (e) { process.stderr.write(`\nRELEASE-READINESS FAIL: cannot read ${tuningsPath}: ${e && e.message}\n Run dogfood-aggregator first to generate it.\n`); process.exit(1); } + + process.stdout.write(`Release readiness check: ${tuningsPath}\n`); + process.stdout.write(`Tunings generated at: ${tunings.generatedAt}\n`); + + let failed = 0; + const r = tunings.releaseReadiness || {}; + if (!r.perPlatform) { process.stderr.write(' FAIL no perPlatform block in tunings\n'); process.exit(1); } + + for (const p of args.requirePlatforms) { + const slot = r.perPlatform[p]; + if (!slot) { process.stderr.write(` FAIL required platform "${p}" has no runs yet\n`); failed++; continue; } + if (!slot.ready) { process.stderr.write(` FAIL ${p}: ${slot.tailSize}/${r.minRuns} runs in tail, ${slot.tailMismatches} mismatch(es)\n`); failed++; } + else process.stdout.write(` ok ${p}: ${slot.tailSize}/${r.minRuns} consecutive clean runs\n`); + } + + // No outstanding demotion candidates (each is an unresolved false-positive source). + const demoteCount = (tunings.demote || []).length; + if (demoteCount > 0) { + process.stderr.write(` FAIL ${demoteCount} check(s) flagged for demotion - resolve before tagging:\n`); + for (const d of tunings.demote) process.stderr.write(` - ${d.platform}/${d.check}: warnRate=${(d.warnRate * 100).toFixed(1)}%\n`); + failed++; + } else { + process.stdout.write(' ok no demotion candidates outstanding\n'); + } + + if (failed) { process.stderr.write(`\nRELEASE-READINESS FAIL: ${failed} blocker(s); do not tag v*.\n`); process.exit(1); } + process.stdout.write('\nRELEASE-READINESS PASS: every platform has the required consecutive clean dogfood runs; tagging is unblocked.\n'); + process.exit(0); +} + +main();