mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-28 12:34:50 +02:00
[eric] ci: add dogfood-aggregator that walks the rolling manifest, computes per-platform per-check warn/fail rates, emits preflight-tunings.json with a demote list when a check exceeds 2x the false-positive tolerance, and writes a release-readiness block the v* gate reads; preflight.js loadTunings/applyTunings honors the demote list at runtime so a chronically-noisy check gets silently downgraded to warn on subsequent boots
This commit is contained in:
Eric
+27
-2
@@ -196,6 +196,28 @@ async function checkClock(env, opts = {}) {
|
||||
});
|
||||
}
|
||||
|
||||
// Load auto-tunings emitted by the dogfood aggregator: a check the loop has
|
||||
// repeatedly seen false-positive on its platform is silently downgraded from
|
||||
// fail -> warn here so a known-noisy probe can't single-handedly scare a user.
|
||||
// File is bundled at build time (scripts/ci/preflight-tunings.json -> resources).
|
||||
function loadTunings(env) {
|
||||
const candidates = [
|
||||
path.join(__dirname, '..', 'scripts', 'ci', 'preflight-tunings.json'),
|
||||
path.join(process.resourcesPath || '', 'preflight-tunings.json'),
|
||||
];
|
||||
for (const p of candidates) {
|
||||
try { return JSON.parse(env.fs.readFileSync(p, 'utf8')); } catch {}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
function applyTunings(results, tunings, platform) {
|
||||
if (!tunings || !Array.isArray(tunings.demote)) return results;
|
||||
const demoted = new Set(tunings.demote.filter((d) => d.platform === platform).map((d) => d.check));
|
||||
if (!demoted.size) return results;
|
||||
return results.map((r) => (r.status === 'fail' && demoted.has(r.name)) ? { ...r, status: 'warn', reason: `${r.reason} [auto-demoted by dogfood tuning]` } : r);
|
||||
}
|
||||
|
||||
async function run(env, opts = {}) {
|
||||
env = env || defaultEnv();
|
||||
const tasks = [
|
||||
@@ -209,9 +231,11 @@ async function run(env, opts = {}) {
|
||||
withTimeout('dual-stack', () => checkDualStack(env, opts.dualStack), 3500),
|
||||
withTimeout('clock', () => checkClock(env, opts.clock), 3500),
|
||||
];
|
||||
const results = await Promise.all(tasks);
|
||||
const rawResults = await Promise.all(tasks);
|
||||
const tunings = loadTunings(env);
|
||||
const results = applyTunings(rawResults, tunings, env.platform);
|
||||
const verdict = results.some((r) => r.status === 'fail') ? 'fail' : results.some((r) => r.status === 'warn') ? 'warn' : 'ok';
|
||||
return { verdict, results, totalMs: Math.max(...results.map((r) => r.durationMs)), startedAt: env.now() };
|
||||
return { verdict, results, totalMs: Math.max(...results.map((r) => r.durationMs)), startedAt: env.now(), tuningsApplied: tunings ? tunings.demote.length : 0 };
|
||||
}
|
||||
|
||||
function cachePath(dataDir, appVersion) { return path.join(dataDir, `preflight-${appVersion}.json`); }
|
||||
@@ -246,4 +270,5 @@ module.exports = {
|
||||
checkOs, checkResources, checkAppdataWritable, checkSecurityBlock, checkSystemLibs,
|
||||
checkNetwork, checkGpu, checkDualStack, checkClock,
|
||||
run, cachePath, readCache, writeCache, pruneOldCaches,
|
||||
loadTunings, applyTunings,
|
||||
};
|
||||
|
||||
@@ -0,0 +1,101 @@
|
||||
#!/usr/bin/env node
|
||||
// Reads every line of dogfood-manifest.jsonl, computes per-check warn+fail rates, identifies checks that DISAGREE with reality (verdict says fail but boot was fine, or check warned on >2x baseline runs), and emits preflight-tunings.json which the preflight module reads to auto-demote a noisy check (bump its timeout, or downgrade fail->warn). Also emits a release-readiness summary the gate uses.
|
||||
|
||||
'use strict';
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const h = require('./lib/app-harness');
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = { manifest: null, tunings: null, minRuns: 12, falsePositiveTolerance: 0.10 };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--manifest') out.manifest = argv[++i];
|
||||
else if (argv[i] === '--tunings') out.tunings = argv[++i];
|
||||
else if (argv[i] === '--min-runs') out.minRuns = Number(argv[++i]);
|
||||
else if (argv[i] === '--fp-tolerance') out.falsePositiveTolerance = Number(argv[++i]);
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function readManifest(p) {
|
||||
let text = '';
|
||||
try { text = fs.readFileSync(p, 'utf8'); } catch { return []; }
|
||||
return text.split(/\r?\n/).filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const manifestPath = args.manifest || path.join(h.REPO_ROOT, 'scripts', 'ci', 'dogfood-manifest.jsonl');
|
||||
const tuningsPath = args.tunings || path.join(h.REPO_ROOT, 'scripts', 'ci', 'preflight-tunings.json');
|
||||
const runs = readManifest(manifestPath);
|
||||
|
||||
process.stdout.write(`Manifest: ${manifestPath}\n`);
|
||||
process.stdout.write(`Runs: ${runs.length}\n`);
|
||||
if (runs.length === 0) { process.stdout.write('\nAGGREGATE: no runs yet; nothing to tune.\n'); return; }
|
||||
|
||||
// Per-platform stats so a noisy-on-Windows-only check doesn't get demoted globally.
|
||||
const byPlatform = {};
|
||||
for (const r of runs) {
|
||||
const p = r.platform || 'unknown';
|
||||
if (!byPlatform[p]) byPlatform[p] = { runs: [], total: 0, mismatches: 0, falsePositives: 0, falseNegatives: 0, checkStats: {} };
|
||||
const slot = byPlatform[p];
|
||||
slot.runs.push(r);
|
||||
slot.total++;
|
||||
if (r.classification && r.classification.mismatch) {
|
||||
slot.mismatches++;
|
||||
if (r.classification.kind === 'false-positive') slot.falsePositives++;
|
||||
if (r.classification.kind === 'false-negative') slot.falseNegatives++;
|
||||
}
|
||||
for (const [name, info] of Object.entries(r.preflightChecks || {})) {
|
||||
if (!slot.checkStats[name]) slot.checkStats[name] = { warn: 0, fail: 0, total: 0 };
|
||||
slot.checkStats[name].total++;
|
||||
if (info.status === 'warn') slot.checkStats[name].warn++;
|
||||
if (info.status === 'fail') slot.checkStats[name].fail++;
|
||||
}
|
||||
}
|
||||
|
||||
// Identify tuning candidates: checks whose warn-rate on a platform exceeds the
|
||||
// tolerance AND the platform's overall boots are mostly successful. These are
|
||||
// false-positive sources that need either a longer timeout or a demoted threshold.
|
||||
const tunings = { generatedAt: new Date().toISOString(), perPlatform: {}, demote: [] };
|
||||
for (const [p, slot] of Object.entries(byPlatform)) {
|
||||
tunings.perPlatform[p] = { runs: slot.total, mismatches: slot.mismatches, falsePositiveRate: slot.total ? slot.falsePositives / slot.total : 0, falseNegativeRate: slot.total ? slot.falseNegatives / slot.total : 0 };
|
||||
for (const [name, st] of Object.entries(slot.checkStats)) {
|
||||
const warnRate = st.warn / Math.max(1, st.total);
|
||||
if (warnRate > 2 * args.falsePositiveTolerance && slot.falsePositives / Math.max(1, slot.total) > args.falsePositiveTolerance) {
|
||||
tunings.demote.push({ platform: p, check: name, warnRate, action: 'demote-to-warn-only' });
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fs.writeFileSync(tuningsPath, JSON.stringify(tunings, null, 2));
|
||||
process.stdout.write(`Tunings written: ${tuningsPath}\n`);
|
||||
for (const [p, slot] of Object.entries(byPlatform)) {
|
||||
process.stdout.write(`\n ${p}: ${slot.total} runs, ${slot.mismatches} mismatches (${slot.falsePositives} false-positive, ${slot.falseNegatives} false-negative)\n`);
|
||||
for (const [name, st] of Object.entries(slot.checkStats)) {
|
||||
const wr = ((st.warn / st.total) * 100).toFixed(1);
|
||||
const fr = ((st.fail / st.total) * 100).toFixed(1);
|
||||
process.stdout.write(` ${name.padEnd(20)} warn=${wr}% fail=${fr}% (n=${st.total})\n`);
|
||||
}
|
||||
}
|
||||
|
||||
// Release readiness: consecutive-clean-runs window per platform. The v* tag
|
||||
// gate fails unless every platform has >= minRuns runs with zero mismatches
|
||||
// in its tail window.
|
||||
let ready = true;
|
||||
const readiness = {};
|
||||
for (const [p, slot] of Object.entries(byPlatform)) {
|
||||
const tail = slot.runs.slice(-args.minRuns);
|
||||
const tailMismatches = tail.filter((r) => r.classification && r.classification.mismatch).length;
|
||||
const consecutiveClean = tail.length === args.minRuns && tailMismatches === 0;
|
||||
readiness[p] = { tailSize: tail.length, tailMismatches, ready: consecutiveClean };
|
||||
if (!consecutiveClean) ready = false;
|
||||
}
|
||||
tunings.releaseReadiness = { ready, perPlatform: readiness, minRuns: args.minRuns };
|
||||
fs.writeFileSync(tuningsPath, JSON.stringify(tunings, null, 2));
|
||||
process.stdout.write(`\nRelease readiness: ${ready ? 'READY' : 'NOT READY'} (need ${args.minRuns} consecutive clean runs per platform)\n`);
|
||||
for (const [p, r] of Object.entries(readiness)) process.stdout.write(` ${p}: ${r.tailSize}/${args.minRuns} clean=${r.tailMismatches === 0}\n`);
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main();
|
||||
@@ -0,0 +1,54 @@
|
||||
#!/usr/bin/env node
|
||||
// The v* tag gate. Reads preflight-tunings.json (produced by the aggregator) and asserts every platform has the required number of consecutive clean dogfood runs AND zero outstanding tuning candidates. Exits non-zero if any platform isn't ready, blocking the release workflow from publishing.
|
||||
|
||||
'use strict';
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const h = require('./lib/app-harness');
|
||||
|
||||
function parseArgs(argv) {
|
||||
const out = { tunings: null, requirePlatforms: ['win32', 'darwin'] };
|
||||
for (let i = 0; i < argv.length; i++) {
|
||||
if (argv[i] === '--tunings') out.tunings = argv[++i];
|
||||
else if (argv[i] === '--require') out.requirePlatforms = argv[++i].split(',');
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
function main() {
|
||||
const args = parseArgs(process.argv.slice(2));
|
||||
const tuningsPath = args.tunings || path.join(h.REPO_ROOT, 'scripts', 'ci', 'preflight-tunings.json');
|
||||
let tunings;
|
||||
try { tunings = JSON.parse(fs.readFileSync(tuningsPath, 'utf8')); }
|
||||
catch (e) { process.stderr.write(`\nRELEASE-READINESS FAIL: cannot read ${tuningsPath}: ${e && e.message}\n Run dogfood-aggregator first to generate it.\n`); process.exit(1); }
|
||||
|
||||
process.stdout.write(`Release readiness check: ${tuningsPath}\n`);
|
||||
process.stdout.write(`Tunings generated at: ${tunings.generatedAt}\n`);
|
||||
|
||||
let failed = 0;
|
||||
const r = tunings.releaseReadiness || {};
|
||||
if (!r.perPlatform) { process.stderr.write(' FAIL no perPlatform block in tunings\n'); process.exit(1); }
|
||||
|
||||
for (const p of args.requirePlatforms) {
|
||||
const slot = r.perPlatform[p];
|
||||
if (!slot) { process.stderr.write(` FAIL required platform "${p}" has no runs yet\n`); failed++; continue; }
|
||||
if (!slot.ready) { process.stderr.write(` FAIL ${p}: ${slot.tailSize}/${r.minRuns} runs in tail, ${slot.tailMismatches} mismatch(es)\n`); failed++; }
|
||||
else process.stdout.write(` ok ${p}: ${slot.tailSize}/${r.minRuns} consecutive clean runs\n`);
|
||||
}
|
||||
|
||||
// No outstanding demotion candidates (each is an unresolved false-positive source).
|
||||
const demoteCount = (tunings.demote || []).length;
|
||||
if (demoteCount > 0) {
|
||||
process.stderr.write(` FAIL ${demoteCount} check(s) flagged for demotion - resolve before tagging:\n`);
|
||||
for (const d of tunings.demote) process.stderr.write(` - ${d.platform}/${d.check}: warnRate=${(d.warnRate * 100).toFixed(1)}%\n`);
|
||||
failed++;
|
||||
} else {
|
||||
process.stdout.write(' ok no demotion candidates outstanding\n');
|
||||
}
|
||||
|
||||
if (failed) { process.stderr.write(`\nRELEASE-READINESS FAIL: ${failed} blocker(s); do not tag v*.\n`); process.exit(1); }
|
||||
process.stdout.write('\nRELEASE-READINESS PASS: every platform has the required consecutive clean dogfood runs; tagging is unblocked.\n');
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
main();
|
||||
Reference in New Issue
Block a user