openswarm/scripts/ci/dogfood-aggregator.js

#!/usr/bin/env node
// Reads every line of dogfood-manifest.jsonl, computes per-check warn+fail rates, identifies checks that DISAGREE with reality (verdict says fail but boot was fine, or check warned on >2x baseline runs), and emits preflight-tunings.json which the preflight module reads to auto-demote a noisy check (bump its timeout, or downgrade fail->warn). Also emits a release-readiness summary the gate uses.

'use strict';
const fs = require('fs');
const path = require('path');
const h = require('./lib/app-harness');

function parseArgs(argv) {
  const out = { manifest: null, tunings: null, minRuns: 12, falsePositiveTolerance: 0.10 };
  for (let i = 0; i < argv.length; i++) {
    if (argv[i] === '--manifest') out.manifest = argv[++i];
    else if (argv[i] === '--tunings') out.tunings = argv[++i];
    else if (argv[i] === '--min-runs') out.minRuns = Number(argv[++i]);
    else if (argv[i] === '--fp-tolerance') out.falsePositiveTolerance = Number(argv[++i]);
  }
  return out;
}

function readManifest(p) {
  let text = '';
  try { text = fs.readFileSync(p, 'utf8'); } catch { return []; }
  return text.split(/\r?\n/).filter(Boolean).map((l) => { try { return JSON.parse(l); } catch { return null; } }).filter(Boolean);
}

function main() {
  const args = parseArgs(process.argv.slice(2));
  const manifestPath = args.manifest || path.join(h.REPO_ROOT, 'scripts', 'ci', 'dogfood-manifest.jsonl');
  const tuningsPath = args.tunings || path.join(h.REPO_ROOT, 'scripts', 'ci', 'preflight-tunings.json');
  const runs = readManifest(manifestPath);

  process.stdout.write(`Manifest: ${manifestPath}\n`);
  process.stdout.write(`Runs:     ${runs.length}\n`);
  if (runs.length === 0) { process.stdout.write('\nAGGREGATE: no runs yet; nothing to tune.\n'); return; }

  // Per-platform stats so a noisy-on-Windows-only check doesn't get demoted globally.
  const byPlatform = {};
  for (const r of runs) {
    const p = r.platform || 'unknown';
    if (!byPlatform[p]) byPlatform[p] = { runs: [], total: 0, mismatches: 0, falsePositives: 0, falseNegatives: 0, checkStats: {} };
    const slot = byPlatform[p];
    slot.runs.push(r);
    slot.total++;
    if (r.classification && r.classification.mismatch) {
      slot.mismatches++;
      if (r.classification.kind === 'false-positive') slot.falsePositives++;
      if (r.classification.kind === 'false-negative') slot.falseNegatives++;
    }
    for (const [name, info] of Object.entries(r.preflightChecks || {})) {
      if (!slot.checkStats[name]) slot.checkStats[name] = { warn: 0, fail: 0, total: 0 };
      slot.checkStats[name].total++;
      if (info.status === 'warn') slot.checkStats[name].warn++;
      if (info.status === 'fail') slot.checkStats[name].fail++;
    }
  }

  // Identify tuning candidates: checks whose warn-rate on a platform exceeds the
  // tolerance AND the platform's overall boots are mostly successful. These are
  // false-positive sources that need either a longer timeout or a demoted threshold.
  const tunings = { generatedAt: new Date().toISOString(), perPlatform: {}, demote: [] };
  for (const [p, slot] of Object.entries(byPlatform)) {
    tunings.perPlatform[p] = { runs: slot.total, mismatches: slot.mismatches, falsePositiveRate: slot.total ? slot.falsePositives / slot.total : 0, falseNegativeRate: slot.total ? slot.falseNegatives / slot.total : 0 };
    for (const [name, st] of Object.entries(slot.checkStats)) {
      const warnRate = st.warn / Math.max(1, st.total);
      if (warnRate > 2 * args.falsePositiveTolerance && slot.falsePositives / Math.max(1, slot.total) > args.falsePositiveTolerance) {
        tunings.demote.push({ platform: p, check: name, warnRate, action: 'demote-to-warn-only' });
      }
    }
  }

  fs.writeFileSync(tuningsPath, JSON.stringify(tunings, null, 2));
  process.stdout.write(`Tunings written: ${tuningsPath}\n`);
  for (const [p, slot] of Object.entries(byPlatform)) {
    process.stdout.write(`\n  ${p}: ${slot.total} runs, ${slot.mismatches} mismatches (${slot.falsePositives} false-positive, ${slot.falseNegatives} false-negative)\n`);
    for (const [name, st] of Object.entries(slot.checkStats)) {
      const wr = ((st.warn / st.total) * 100).toFixed(1);
      const fr = ((st.fail / st.total) * 100).toFixed(1);
      process.stdout.write(`    ${name.padEnd(20)} warn=${wr}% fail=${fr}% (n=${st.total})\n`);
    }
  }

  // Release readiness: consecutive-clean-runs window per platform. The v* tag
  // gate fails unless every platform has >= minRuns runs with zero mismatches
  // in its tail window.
  let ready = true;
  const readiness = {};
  for (const [p, slot] of Object.entries(byPlatform)) {
    const tail = slot.runs.slice(-args.minRuns);
    const tailMismatches = tail.filter((r) => r.classification && r.classification.mismatch).length;
    const consecutiveClean = tail.length === args.minRuns && tailMismatches === 0;
    readiness[p] = { tailSize: tail.length, tailMismatches, ready: consecutiveClean };
    if (!consecutiveClean) ready = false;
  }
  tunings.releaseReadiness = { ready, perPlatform: readiness, minRuns: args.minRuns };
  fs.writeFileSync(tuningsPath, JSON.stringify(tunings, null, 2));
  process.stdout.write(`\nRelease readiness: ${ready ? 'READY' : 'NOT READY'} (need ${args.minRuns} consecutive clean runs per platform)\n`);
  for (const [p, r] of Object.entries(readiness)) process.stdout.write(`  ${p}: ${r.tailSize}/${args.minRuns} clean=${r.tailMismatches === 0}\n`);
  process.exit(0);
}

main();