ECC/scripts/dev/gateguard-eval.js

#!/usr/bin/env node
'use strict';

const { execFileSync } = require('child_process');
const fs = require('fs');
const os = require('os');
const path = require('path');
const { performance } = require('perf_hooks');
const { Worker, isMainThread, parentPort, workerData } = require('worker_threads');

const REPO_ROOT = path.join(__dirname, '..', '..');
const HOOK_RELATIVE_PATH = 'scripts/hooks/gateguard-fact-force.js';
const DEFAULT_CORPUS_DIR = path.join(REPO_ROOT, 'tests', 'fixtures', 'gateguard-scenarios');
const DEFAULT_BASELINE = 'upstream/main';
const BASELINE_ALIASES = Object.freeze({ pr: '58a4a0d0' });
const EXPECTATIONS = new Set(['deny', 'allow']);
const TOKEN_CHARS = 4;
const CONDENSED_PREFIX = /^\[Fact-Forcing Gate\] \(denial #\d+ this session\) First (edit|creation) of /;

const QUESTION_IDS = Object.freeze([
  'importers',
  'public-api',
  'local-callers',
  'callers',
  'no-duplicate',
  'data-schema',
  'loader',
  'behaviour-change',
  'no-duplicate-instruction',
  'under-test',
  'existing-tests',
  'supersedes',
  'linked-from',
  'why-new-file',
  'references',
  'corrects-or-adds',
  'config-reader',
  'config-effect',
  'no-plaintext-secrets',
  'quote-instruction'
]);

const CONDENSED_PHRASES = Object.freeze([
  { phrase: 'call sites in this file or its module', ids: { edit: 'local-callers', creation: 'local-callers' } },
  { phrase: 'importers/callers', ids: { edit: 'importers', creation: 'callers' } },
  { phrase: 'affected API', ids: { edit: 'public-api', creation: 'public-api' } },
  { phrase: 'data schemas', ids: { edit: 'data-schema', creation: 'data-schema' } },
  { phrase: 'verbatim instruction', ids: { edit: 'quote-instruction', creation: 'quote-instruction' } }
]);

// --- corpus ---

function fail(message) {
  throw new Error(message);
}

function validateStep(scenario, step, index) {
  const where = `${scenario.name} step ${index + 1}`;
  if (!step || typeof step !== 'object') fail(`${where}: not an object`);
  if (typeof step.id !== 'string' || !step.id) fail(`${where}: missing id`);
  if (!step.payload || typeof step.payload !== 'object') fail(`${where}: missing payload`);
  if (typeof step.payload.tool_name !== 'string') fail(`${where}: payload.tool_name must be a string`);
  if (!EXPECTATIONS.has(step.expect)) fail(`${where}: expect must be deny or allow`);
  if (typeof step.mustDeny !== 'boolean') fail(`${where}: mustDeny must be a boolean`);
  if (typeof step.redundant !== 'boolean') fail(`${where}: redundant must be a boolean`);
  if (step.mustDeny && step.expect !== 'deny') fail(`${where}: mustDeny requires expect deny`);
  if (step.redundant && step.expect !== 'allow') fail(`${where}: redundant requires expect allow`);
  if (step.relevantQuestions !== undefined) {
    const ids = step.relevantQuestions;
    if (!Array.isArray(ids) || ids.some(id => !QUESTION_IDS.includes(id))) fail(`${where}: unknown relevantQuestions id`);
  }
  if (step.transcript !== undefined && !Array.isArray(step.transcript)) fail(`${where}: transcript must be an array`);
}

function validateScenario(scenario, file) {
  if (!scenario || typeof scenario !== 'object') fail(`${file}: not an object`);
  if (typeof scenario.name !== 'string' || !scenario.name) fail(`${file}: missing name`);
  if (!Array.isArray(scenario.steps) || scenario.steps.length === 0) fail(`${file}: steps must be a non-empty array`);
  const ids = new Set();
  scenario.steps.forEach((step, index) => {
    validateStep(scenario, step, index);
    if (ids.has(step.id)) fail(`${scenario.name}: duplicate step id ${step.id}`);
    ids.add(step.id);
  });
}

/** Load and validate every scenario file in a corpus directory, sorted by file name. */
function loadCorpus(dir = DEFAULT_CORPUS_DIR) {
  return fs
    .readdirSync(dir)
    .filter(name => name.endsWith('.json'))
    .sort()
    .map(name => {
      const scenario = JSON.parse(fs.readFileSync(path.join(dir, name), 'utf8'));
      validateScenario(scenario, name);
      return { ...scenario, file: name };
    });
}

// --- baseline materialisation ---

function gitShow(ref, relativePath) {
  return execFileSync('git', ['show', `${ref}:${relativePath}`], {
    cwd: REPO_ROOT,
    encoding: 'utf8',
    maxBuffer: 16 * 1024 * 1024,
    stdio: ['ignore', 'pipe', 'pipe']
  });
}

function relativeRequires(source) {
  const specs = [];
  const pattern = /require\(\s*['"](\.{1,2}\/[^'"]+)['"]\s*\)/g;
  for (let match = pattern.exec(source); match; match = pattern.exec(source)) specs.push(match[1]);
  return specs;
}

function requireTarget(fromFile, spec) {
  const joined = path.posix.normalize(path.posix.join(path.posix.dirname(fromFile), spec));
  return path.posix.extname(joined) ? joined : `${joined}.js`;
}

function resolveBaselineRef(ref) {
  return Object.hasOwn(BASELINE_ALIASES, ref) ? BASELINE_ALIASES[ref] : ref;
}

/** Write the hook at a git ref, and every file it requires relatively, into a temp tree; returns the hook path. */
function materializeHook(ref, outDir) {
  const seen = new Set();
  const queue = [HOOK_RELATIVE_PATH];
  while (queue.length > 0) {
    const relative = queue.shift();
    if (seen.has(relative)) continue;
    seen.add(relative);
    let source;
    try {
      source = gitShow(ref, relative);
    } catch (error) {
      fail(`cannot read ${relative} at ${ref} (fetch the ref first): ${String(error.stderr || error.message).trim()}`);
    }
    const target = path.join(outDir, ...relative.split('/'));
    fs.mkdirSync(path.dirname(target), { recursive: true });
    fs.writeFileSync(target, source, 'utf8');
    if (relative.endsWith('.js')) relativeRequires(source).forEach(spec => queue.push(requireTarget(relative, spec)));
  }
  return path.join(outDir, ...HOOK_RELATIVE_PATH.split('/'));
}

// --- scenario setup ---

function substitute(value, vars) {
  if (typeof value === 'string') return value.replace(/\{\{(root|transcript)\}\}/g, (_, name) => vars[name]);
  if (Array.isArray(value)) return value.map(item => substitute(item, vars));
  if (value && typeof value === 'object') {
    return Object.fromEntries(Object.entries(value).map(([key, item]) => [key, substitute(item, vars)]));
  }
  return value;
}

function createSymlink(root, link, target) {
  const linkPath = path.join(root, ...link.split('/'));
  fs.mkdirSync(path.dirname(linkPath), { recursive: true });
  const resolved = path.resolve(path.dirname(linkPath), ...target.split('/'));
  const isDir = fs.existsSync(resolved) && fs.statSync(resolved).isDirectory();
  if (process.platform === 'win32' && isDir) fs.symlinkSync(resolved, linkPath, 'junction');
  else fs.symlinkSync(target, linkPath, isDir ? 'dir' : 'file');
}

function prepareScenario(scenario, workDir) {
  const root = scenario.root || fs.realpathSync(fs.mkdtempSync(path.join(workDir, 'project-')));
  if (!scenario.root) {
    for (const [relative, content] of Object.entries(scenario.files || {})) {
      const file = path.join(root, ...relative.split('/'));
      fs.mkdirSync(path.dirname(file), { recursive: true });
      fs.writeFileSync(file, content, 'utf8');
    }
    for (const dir of scenario.dirs || []) fs.mkdirSync(path.join(root, ...dir.split('/')), { recursive: true });
    try {
      for (const [link, target] of Object.entries(scenario.symlinks || {})) createSymlink(root, link, target);
    } catch (error) {
      return { skipped: `symlinks unavailable: ${error.code || error.message}` };
    }
  }
  const stateDir = fs.mkdtempSync(path.join(workDir, 'state-'));
  const home = fs.mkdtempSync(path.join(workDir, 'home-'));
  const transcript = path.join(fs.mkdtempSync(path.join(workDir, 'transcript-')), 'session.jsonl');
  const vars = { root, transcript };
  const env = {
    PATH: process.env.PATH || '',
    HOME: home,
    USERPROFILE: home,
    GATEGUARD_STATE_DIR: stateDir,
    CLAUDE_PROJECT_DIR: root,
    ...(process.env.SystemRoot ? { SystemRoot: process.env.SystemRoot } : {}),
    ...(scenario.env || {})
  };
  const steps = scenario.steps.map(step => ({
    id: step.id,
    records: substitute(step.transcript || [], vars),
    payload: substitute(step.payload, vars)
  }));
  return { env, transcript, steps };
}

// --- hook execution ---

function runStepsInWorker(hookFile, prepared) {
  return new Promise((resolve, reject) => {
    const worker = new Worker(__filename, {
      env: prepared.env,
      workerData: { gateguardEval: true, hookFile, transcript: prepared.transcript, steps: prepared.steps }
    });
    let outcome = null;
    worker.once('message', message => {
      outcome = message;
    });
    worker.once('error', reject);
    worker.once('exit', code => {
      if (outcome) resolve(outcome);
      else reject(new Error(`scenario worker exited with code ${code}`));
    });
  });
}

function hookOutput(result) {
  if (!result || typeof result !== 'object') return { decision: null, reason: '', context: '' };
  let parsed = null;
  if (typeof result.stdout === 'string' && result.stdout) {
    try {
      parsed = JSON.parse(result.stdout);
    } catch (_) {
      parsed = null;
    }
  }
  const specific = parsed && parsed.hookSpecificOutput ? parsed.hookSpecificOutput : {};
  const context = [].concat(result.additionalContext || specific.additionalContext || []).join('\n');
  return { decision: specific.permissionDecision || null, reason: specific.permissionDecisionReason || '', context };
}

function allowKind(context) {
  if (context.includes('Prior search seen in this turn')) return 'credit';
  if (context.includes('Sibling of ')) return 'sibling';
  if (context.includes('Comment or whitespace-only change')) return 'trivial';
  return 'pass';
}

function workerMain() {
  const { hookFile, transcript, steps } = workerData;
  const hook = require(hookFile);
  const results = [];
  for (const step of steps) {
    if (step.records.length > 0) {
      fs.appendFileSync(transcript, step.records.map(record => `${JSON.stringify(record)}\n`).join(''), 'utf8');
    }
    const raw = JSON.stringify(step.payload);
    let result;
    let error = null;
    const started = performance.now();
    try {
      result = hook.run(raw);
    } catch (thrown) {
      error = String(thrown && thrown.message ? thrown.message : thrown);
      result = raw;
    }
    const latencyMs = performance.now() - started;
    const output = hookOutput(result);
    const denied = output.decision === 'deny';
    results.push({
      id: step.id,
      decision: denied ? 'deny' : 'allow',
      kind: denied ? 'deny' : allowKind(output.context),
      explicitAllow: output.decision === 'allow',
      reason: output.reason,
      latencyMs,
      error
    });
  }
  parentPort.postMessage({ results });
}

/** Run every scenario against one hook file; each scenario gets its own project, state dir, transcript and clean env. */
async function runCorpus(hookFile, scenarios) {
  const workDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-eval-'));
  try {
    const runs = [];
    for (const scenario of scenarios) {
      const prepared = prepareScenario(scenario, workDir);
      if (prepared.skipped) {
        runs.push({ scenario, skipped: prepared.skipped, results: [] });
        continue;
      }
      const { results } = await runStepsInWorker(hookFile, prepared);
      runs.push({ scenario, skipped: null, results });
    }
    return runs;
  } finally {
    fs.rmSync(workDir, { recursive: true, force: true });
  }
}

// --- question mapping ---

function questionTextTable() {
  const { questionText } = require(path.join(REPO_ROOT, 'scripts', 'lib', 'gateguard-target-class'));
  return new Map(QUESTION_IDS.map(id => [questionText(id), id]));
}

/** Question ids a denial asked, read from its numbered list or its condensed hint. */
function questionsAsked(reason, table) {
  const numbered = reason
    .split('\n')
    .map(line => line.match(/^\d+\. (.+)$/))
    .filter(Boolean)
    .map(match => table.get(match[1]) || `unmapped:${match[1]}`);
  if (numbered.length > 0) return numbered;
  const condensed = reason.match(CONDENSED_PREFIX);
  if (!condensed) return [];
  return CONDENSED_PHRASES.filter(({ phrase }) => reason.includes(phrase)).map(({ ids }) => ids[condensed[1]]);
}

// --- metrics ---

function percentile(sorted, fraction) {
  if (sorted.length === 0) return 0;
  const index = Math.min(sorted.length - 1, Math.max(0, Math.ceil(fraction * sorted.length) - 1));
  return sorted[index];
}

/** Metrics for one hook's run of the corpus, plus per-step detail. */
function summarize(runs, table = questionTextTable()) {
  const totals = {
    scenarios: 0,
    skippedScenarios: 0,
    steps: 0,
    denials: 0,
    redundantDenials: 0,
    mustDenyBypasses: 0,
    mismatches: 0,
    irrelevantQuestions: 0,
    irrelevantInCondensed: 0,
    missedQuestions: 0,
    unmappedQuestions: 0,
    explicitAllows: 0,
    errors: 0,
    denialTokens: 0,
    allowsByKind: { credit: 0, sibling: 0, trivial: 0, pass: 0 }
  };
  const latencies = [];
  const steps = [];
  const perScenario = [];
  for (const run of runs) {
    if (run.skipped) {
      totals.skippedScenarios += 1;
      perScenario.push({ name: run.scenario.name, skipped: run.skipped, steps: 0, denials: 0 });
      continue;
    }
    totals.scenarios += 1;
    let scenarioDenials = 0;
    run.results.forEach((result, index) => {
      const step = run.scenario.steps[index];
      const denied = result.decision === 'deny';
      const relevant = Array.isArray(step.relevantQuestions) ? step.relevantQuestions : null;
      const asked = denied && relevant ? questionsAsked(result.reason, table) : [];
      const irrelevant = denied && relevant ? asked.filter(id => !relevant.includes(id)) : [];
      const missed = denied && relevant ? relevant.filter(id => !asked.includes(id)) : [];
      totals.steps += 1;
      latencies.push(result.latencyMs);
      if (denied) {
        totals.denials += 1;
        scenarioDenials += 1;
        totals.denialTokens += Math.ceil(result.reason.length / TOKEN_CHARS);
        if (step.redundant) totals.redundantDenials += 1;
      } else {
        totals.allowsByKind[result.kind] += 1;
        if (step.mustDeny) totals.mustDenyBypasses += 1;
      }
      if (result.decision !== step.expect) totals.mismatches += 1;
      const condensed = denied && CONDENSED_PREFIX.test(result.reason);
      totals.irrelevantQuestions += irrelevant.length;
      if (condensed) totals.irrelevantInCondensed += irrelevant.length;
      totals.missedQuestions += missed.length;
      totals.unmappedQuestions += asked.filter(id => id.startsWith('unmapped:')).length;
      if (result.explicitAllow) totals.explicitAllows += 1;
      if (result.error) totals.errors += 1;
      steps.push({
        scenario: run.scenario.name,
        step: step.id,
        expect: step.expect,
        decision: result.decision,
        kind: result.kind,
        mustDeny: step.mustDeny,
        redundant: step.redundant,
        condensed,
        questions: asked,
        irrelevant,
        missed,
        latencyMs: Number(result.latencyMs.toFixed(3)),
        ...(result.error ? { error: result.error } : {})
      });
    });
    perScenario.push({ name: run.scenario.name, skipped: null, steps: run.results.length, denials: scenarioDenials });
  }
  const sorted = latencies.slice().sort((a, b) => a - b);
  return {
    totals,
    latency: { p50Ms: Number(percentile(sorted, 0.5).toFixed(3)), p95Ms: Number(percentile(sorted, 0.95).toFixed(3)) },
    perScenario,
    steps
  };
}

// --- reporting ---

const METRIC_ROWS = [
  ['Steps', s => s.totals.steps],
  ['Denials', s => s.totals.denials],
  ['Redundant denials', s => s.totals.redundantDenials],
  ['Must-deny bypasses', s => s.totals.mustDenyBypasses],
  ['Expectation mismatches', s => s.totals.mismatches],
  ['Irrelevant questions asked', s => s.totals.irrelevantQuestions],
  ['Irrelevant questions in condensed denials', s => s.totals.irrelevantInCondensed],
  ['Warranted questions not asked', s => s.totals.missedQuestions],
  ['Estimated denial tokens', s => s.totals.denialTokens],
  ['Allows with a credit note', s => s.totals.allowsByKind.credit],
  ['Allows with a sibling note', s => s.totals.allowsByKind.sibling],
  ['Allows with a trivial-edit note', s => s.totals.allowsByKind.trivial],
  ['Hook latency p50 (ms)', s => s.latency.p50Ms.toFixed(2)],
  ['Hook latency p95 (ms)', s => s.latency.p95Ms.toFixed(2)]
];

function markdownTable(header, rows) {
  const line = cells => `| ${cells.join(' | ')} |`;
  return [line(header), line(header.map((_, i) => (i === 0 ? '---' : '---:'))), ...rows.map(line)].join('\n');
}

/** Markdown report: metric table, per-scenario denials, and any working-tree mismatches. */
function renderMarkdown(report) {
  const labels = report.hooks.map(hook => hook.label);
  const lines = [
    `Corpus: ${report.corpus.scenarios} scenarios, ${report.corpus.steps} steps.`,
    '',
    markdownTable(['Metric', ...labels], METRIC_ROWS.map(([name, pick]) => [name, ...report.hooks.map(hook => String(pick(hook.summary)))])),
    '',
    markdownTable(
      ['Scenario', 'Steps', ...labels.map(label => `Denials: ${label}`)],
      report.hooks[0].summary.perScenario.map((row, index) => [
        row.name,
        String(row.steps),
        ...report.hooks.map(hook => {
          const entry = hook.summary.perScenario[index];
          return entry.skipped ? 'skipped' : String(entry.denials);
        })
      ])
    )
  ];
  const working = report.hooks[0].summary;
  const mismatched = working.steps.filter(step => step.decision !== step.expect);
  if (mismatched.length > 0) {
    lines.push('', `${report.hooks[0].label} mismatches:`, '');
    mismatched.forEach(step => lines.push(`- ${step.scenario} / ${step.step}: expected ${step.expect}, got ${step.decision} (${step.kind})`));
  }
  return `${lines.join('\n')}\n`;
}

// --- CLI ---

const USAGE = [
  'Usage: node scripts/dev/gateguard-eval.js [--markdown | --json] [--baseline <ref>]... [--corpus <dir>]',
  '',
  '  --baseline <ref>  git ref whose hook is compared with the working tree (repeatable;',
  `                    default ${DEFAULT_BASELINE}; "pr" means ${BASELINE_ALIASES.pr})`,
  '  --corpus <dir>    scenario directory (default tests/fixtures/gateguard-scenarios)',
  '  --markdown        print markdown tables (default)',
  '  --json            print the full report as JSON'
].join('\n');

function parseArgs(argv) {
  const options = { format: 'markdown', baselines: [], corpus: DEFAULT_CORPUS_DIR, help: false };
  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i];
    if (arg === '--markdown') options.format = 'markdown';
    else if (arg === '--json') options.format = 'json';
    else if (arg === '--help' || arg === '-h') options.help = true;
    else if ((arg === '--baseline' || arg === '--corpus') && i + 1 < argv.length) {
      if (arg === '--baseline') options.baselines.push(argv[++i]);
      else options.corpus = path.resolve(argv[++i]);
    } else fail(`unknown or incomplete argument: ${arg}\n\n${USAGE}`);
  }
  if (options.baselines.length === 0) options.baselines.push(DEFAULT_BASELINE);
  return options;
}

/** Evaluate the working-tree hook and each baseline ref over a corpus. */
async function evaluate({ baselines = [DEFAULT_BASELINE], corpus = DEFAULT_CORPUS_DIR } = {}) {
  const scenarios = loadCorpus(corpus);
  const table = questionTextTable();
  const hooks = [{ label: 'working tree', ref: null, file: path.join(REPO_ROOT, ...HOOK_RELATIVE_PATH.split('/')) }];
  const materialized = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-baseline-'));
  try {
    baselines.forEach((label, index) => {
      const ref = resolveBaselineRef(label);
      hooks.push({ label, ref, file: materializeHook(ref, path.join(materialized, String(index))) });
    });
    const evaluated = [];
    for (const hook of hooks) {
      evaluated.push({ label: hook.label, ref: hook.ref, summary: summarize(await runCorpus(hook.file, scenarios), table) });
    }
    return {
      corpus: { scenarios: scenarios.length, steps: scenarios.reduce((sum, scenario) => sum + scenario.steps.length, 0) },
      hooks: evaluated
    };
  } finally {
    fs.rmSync(materialized, { recursive: true, force: true });
  }
}

async function main() {
  const options = parseArgs(process.argv.slice(2));
  if (options.help) {
    process.stdout.write(`${USAGE}\n`);
    return;
  }
  const report = await evaluate(options);
  process.stdout.write(options.format === 'json' ? `${JSON.stringify(report, null, 2)}\n` : renderMarkdown(report));
  const working = report.hooks[0].summary.totals;
  if (working.mustDenyBypasses > 0 || working.mismatches > 0 || working.explicitAllows > 0 || working.errors > 0) {
    process.exitCode = 1;
  }
}

if (!isMainThread && workerData && workerData.gateguardEval) {
  workerMain();
} else if (require.main === module) {
  main().catch(error => {
    process.stderr.write(`[gateguard-eval] ${error.message}\n`);
    process.exitCode = 2;
  });
}

module.exports = { loadCorpus, materializeHook, runCorpus, summarize, questionsAsked, renderMarkdown, evaluate };