feat(gateguard): write scenario gate results as SARIF

tests/scripts/gateguard-eval.test.js

@@ -0,0 +1,153 @@
+/**
+ * Tests for scripts/dev/gateguard-eval.js argument parsing and SARIF output.
+ */
+
+'use strict';
+
+const assert = require('assert');
+const path = require('path');
+const { parseArgs, renderSarif } = require('../../scripts/dev/gateguard-eval');
+
+let passed = 0;
+let failed = 0;
+
+function test(name, fn) {
+  try {
+    fn();
+    console.log(`  ✓ ${name}`);
+    passed++;
+  } catch (error) {
+    console.log(`  ✗ ${name}`);
+    console.log(`    Error: ${error.message}`);
+    failed++;
+  }
+}
+
+function step(overrides) {
+  return {
+    scenario: 'sensitive-targets',
+    file: '09-sensitive-targets.json',
+    step: 'edit-env',
+    expect: 'deny',
+    decision: 'deny',
+    kind: 'deny',
+    mustDeny: true,
+    redundant: false,
+    ...overrides
+  };
+}
+
+function summary(steps, totals = {}) {
+  return {
+    totals: { steps: steps.length, denials: 0, mustDenyBypasses: 0, mismatches: 0, explicitAllows: 0, errors: 0, skippedScenarios: 0, ...totals },
+    latency: { p50Ms: 1.234, p95Ms: 5.678 },
+    perScenario: [],
+    steps
+  };
+}
+
+function report(workingSteps, baselineTotals = { mustDenyBypasses: 8 }) {
+  return {
+    corpus: { scenarios: 1, steps: workingSteps.length },
+    hooks: [
+      { label: 'working tree', ref: null, summary: summary(workingSteps) },
+      { label: 'upstream/main', ref: 'upstream/main', summary: summary([], baselineTotals) }
+    ]
+  };
+}
+
+function results(log) {
+  return log.runs[0].results;
+}
+
+console.log('\n=== Testing gateguard-eval ===\n');
+
+test('--sarif takes a file path and keeps the other options', () => {
+  const options = parseArgs(['--json', '--sarif', 'out/gate.sarif', '--baseline', 'main']);
+  assert.strictEqual(options.format, 'json');
+  assert.strictEqual(options.sarif, path.resolve('out/gate.sarif'));
+  assert.deepStrictEqual(options.baselines, ['main']);
+});
+
+test('--sarif without a value is rejected', () => {
+  assert.throws(() => parseArgs(['--sarif']), /unknown or incomplete argument: --sarif/);
+});
+
+test('a clean working tree produces a valid SARIF log with no results', () => {
+  const log = JSON.parse(renderSarif(report([step({}), step({ step: 'read-only', mustDeny: false, expect: 'allow', decision: 'allow', kind: 'pass' })])));
+  assert.strictEqual(log.version, '2.1.0');
+  assert.strictEqual(log.runs.length, 1);
+  assert.strictEqual(log.runs[0].tool.driver.name, 'gateguard-eval');
+  assert.deepStrictEqual(results(log), []);
+});
+
+test('every rule declares a default level', () => {
+  const rules = JSON.parse(renderSarif(report([]))).runs[0].tool.driver.rules;
+  assert.deepStrictEqual(
+    rules.map(rule => [rule.id, rule.defaultConfiguration.level]),
+    [
+      ['gateguard/must-deny-bypass', 'error'],
+      ['gateguard/explicit-allow', 'error'],
+      ['gateguard/hook-error', 'error'],
+      ['gateguard/expectation-mismatch', 'warning']
+    ]
+  );
+});
+
+test('a must-deny step that was allowed is an error on its scenario file', () => {
+  const log = JSON.parse(renderSarif(report([step({ decision: 'allow', kind: 'credit' })])));
+  assert.strictEqual(results(log).length, 1);
+  const [result] = results(log);
+  assert.strictEqual(result.ruleId, 'gateguard/must-deny-bypass');
+  assert.strictEqual(result.level, 'error');
+  assert.strictEqual(result.locations[0].physicalLocation.artifactLocation.uri, 'tests/fixtures/gateguard-scenarios/09-sensitive-targets.json');
+  assert.strictEqual(result.locations[0].physicalLocation.artifactLocation.uriBaseId, '%SRCROOT%');
+  assert.match(result.message.text, /sensitive-targets \/ edit-env: expected deny, got allow \(credit\)/);
+});
+
+test('an explicit allow and a hook error are reported as errors', () => {
+  const log = JSON.parse(
+    renderSarif(
+      report([
+        step({ step: 'a', mustDeny: false, expect: 'allow', decision: 'allow', kind: 'pass', explicitAllow: true }),
+        step({ step: 'b', mustDeny: false, error: 'boom' })
+      ])
+    )
+  );
+  assert.deepStrictEqual(results(log).map(result => [result.ruleId, result.level]), [
+    ['gateguard/explicit-allow', 'error'],
+    ['gateguard/hook-error', 'error']
+  ]);
+});
+
+test('a wrong decision that is not a bypass is a warning', () => {
+  const log = JSON.parse(renderSarif(report([step({ mustDeny: false, expect: 'allow', decision: 'deny', kind: 'deny' })])));
+  assert.deepStrictEqual(results(log).map(result => [result.ruleId, result.level]), [['gateguard/expectation-mismatch', 'warning']]);
+});
+
+test('a bypass is not reported a second time as a mismatch', () => {
+  const log = JSON.parse(renderSarif(report([step({ decision: 'allow', kind: 'pass' })])));
+  assert.deepStrictEqual(results(log).map(result => result.ruleId), ['gateguard/must-deny-bypass']);
+});
+
+test('baseline totals are recorded without their failures becoming results', () => {
+  const log = JSON.parse(renderSarif(report([step({})])));
+  const hooks = log.runs[0].properties.hooks;
+  assert.deepStrictEqual(hooks.map(hook => [hook.label, hook.totals.mustDenyBypasses]), [
+    ['working tree', 0],
+    ['upstream/main', 8]
+  ]);
+  assert.deepStrictEqual(results(log), []);
+});
+
+test('output leaves out timing so reruns are byte-identical', () => {
+  const text = renderSarif(report([step({ latencyMs: 3.21 })]));
+  assert.ok(!/latency|p50|p95|Time/i.test(text), 'timing leaked into SARIF');
+  const slower = report([step({ latencyMs: 9.87 })]);
+  slower.hooks[0].summary.latency = { p50Ms: 9, p95Ms: 99 };
+  assert.strictEqual(renderSarif(slower), text);
+});
+
+console.log(`\nPassed: ${passed}`);
+console.log(`Failed: ${failed}`);
+process.exitCode = failed > 0 ? 1 : 0;