feat(gateguard): write scenario gate results as SARIF

docs/gateguard/evaluation.md

@@ -190,6 +190,7 @@ git fetch upstream main
 node scripts/dev/gateguard-eval.js --markdown
 node scripts/dev/gateguard-eval.js --markdown --baseline upstream/main --baseline 58a4a0d0
 node scripts/dev/gateguard-eval.js --json > gateguard-eval.json
+node scripts/dev/gateguard-eval.js --baseline upstream/main --sarif gateguard-eval.sarif
 node tests/hooks/gateguard-scenarios.test.js
 ```
 
@@ -198,6 +199,10 @@ node tests/hooks/gateguard-scenarios.test.js
 against it. `--corpus <dir>` points at another
 scenario directory. The script exits non-zero when the working tree has a
 mismatch, a must-deny bypass, an explicit `allow` decision or a thrown error.
+`--sarif <file>` also writes those working-tree failures as SARIF 2.1.0
+(bypasses, explicit allows and errors as `error`, other mismatches as
+`warning`) with each hook's totals in the run properties. Timing is left out,
+so the file is identical across reruns of the same tree and corpus.
 
 ## Corpus format
 

scripts/dev/gateguard-eval.js

@@ -396,6 +396,7 @@ function summarize(runs, table = questionTextTable()) {
       if (result.error) totals.errors += 1;
       steps.push({
         scenario: run.scenario.name,
+        file: run.scenario.file,
         step: step.id,
         expect: step.expect,
         decision: result.decision,
@@ -407,6 +408,7 @@ function summarize(runs, table = questionTextTable()) {
         irrelevant,
         missed,
         latencyMs: Number(result.latencyMs.toFixed(3)),
+        ...(result.explicitAllow ? { explicitAllow: true } : {}),
         ...(result.error ? { error: result.error } : {})
       });
     });
@@ -474,27 +476,121 @@ function renderMarkdown(report) {
   return `${lines.join('\n')}\n`;
 }
 
+const SARIF_RULES = Object.freeze([
+  {
+    id: 'gateguard/must-deny-bypass',
+    level: 'error',
+    name: 'MustDenyBypass',
+    text: 'A step that must be denied (sensitive target, bypass attempt, or mutating first command) was allowed.'
+  },
+  {
+    id: 'gateguard/explicit-allow',
+    level: 'error',
+    name: 'ExplicitAllow',
+    text: 'The hook emitted permissionDecision "allow", which skips the host permission prompt.'
+  },
+  {
+    id: 'gateguard/hook-error',
+    level: 'error',
+    name: 'HookError',
+    text: 'The hook threw or produced unparseable output for a step.'
+  },
+  {
+    id: 'gateguard/expectation-mismatch',
+    level: 'warning',
+    name: 'ExpectationMismatch',
+    text: 'The hook decision differs from the decision the scenario expects.'
+  }
+]);
+
+function sarifFindings(step) {
+  const findings = [];
+  if (step.mustDeny && step.decision !== 'deny') findings.push('gateguard/must-deny-bypass');
+  if (step.explicitAllow) findings.push('gateguard/explicit-allow');
+  if (step.error) findings.push('gateguard/hook-error');
+  if (step.decision !== step.expect && findings.length === 0) findings.push('gateguard/expectation-mismatch');
+  return findings;
+}
+
+function sarifTotals(summary) {
+  const { steps, denials, mustDenyBypasses, mismatches, explicitAllows, errors, skippedScenarios } = summary.totals;
+  return { steps, denials, mustDenyBypasses, mismatches, explicitAllows, errors, skippedScenarios };
+}
+
+/** SARIF 2.1.0 log of working-tree gate failures, with per-hook totals; timing is omitted so output is reproducible. */
+function renderSarif(report) {
+  const working = report.hooks[0].summary;
+  const levels = new Map(SARIF_RULES.map(rule => [rule.id, rule.level]));
+  const results = [];
+  for (const step of working.steps) {
+    for (const ruleId of sarifFindings(step)) {
+      results.push({
+        ruleId,
+        level: levels.get(ruleId),
+        message: { text: `${step.scenario} / ${step.step}: expected ${step.expect}, got ${step.decision} (${step.kind})` },
+        locations: [
+          {
+            physicalLocation: {
+              artifactLocation: { uri: `tests/fixtures/gateguard-scenarios/${step.file}`, uriBaseId: '%SRCROOT%' }
+            }
+          }
+        ]
+      });
+    }
+  }
+  const log = {
+    version: '2.1.0',
+    $schema: 'https://json.schemastore.org/sarif-2.1.0.json',
+    runs: [
+      {
+        tool: {
+          driver: {
+            name: 'gateguard-eval',
+            informationUri: 'https://github.com/affaan-m/ECC/blob/main/docs/gateguard/evaluation.md',
+            rules: SARIF_RULES.map(rule => ({
+              id: rule.id,
+              name: rule.name,
+              shortDescription: { text: rule.text },
+              defaultConfiguration: { level: rule.level }
+            }))
+          }
+        },
+        automationDetails: { id: 'gateguard/scenario-corpus' },
+        invocations: [{ executionSuccessful: true }],
+        properties: {
+          corpus: report.corpus,
+          hooks: report.hooks.map(hook => ({ label: hook.label, totals: sarifTotals(hook.summary) }))
+        },
+        results
+      }
+    ]
+  };
+  return `${JSON.stringify(log, null, 2)}\n`;
+}
+
 // --- CLI ---
 
 const USAGE = [
-  'Usage: node scripts/dev/gateguard-eval.js [--markdown | --json] [--baseline <ref>]... [--corpus <dir>]',
+  'Usage: node scripts/dev/gateguard-eval.js [--markdown | --json] [--baseline <ref>]... [--corpus <dir>] [--sarif <file>]',
   '',
   '  --baseline <ref>  git ref whose hook is compared with the working tree (repeatable;',
   `                    default ${DEFAULT_BASELINE})`,
   '  --corpus <dir>    scenario directory (default tests/fixtures/gateguard-scenarios)',
   '  --markdown        print markdown tables (default)',
-  '  --json            print the full report as JSON'
+  '  --json            print the full report as JSON',
+  '  --sarif <file>    also write working-tree gate failures as SARIF 2.1.0'
 ].join('\n');
 
 function parseArgs(argv) {
-  const options = { format: 'markdown', baselines: [], corpus: DEFAULT_CORPUS_DIR, help: false };
+  const options = { format: 'markdown', baselines: [], corpus: DEFAULT_CORPUS_DIR, sarif: null, help: false };
   for (let i = 0; i < argv.length; i++) {
     const arg = argv[i];
     if (arg === '--markdown') options.format = 'markdown';
     else if (arg === '--json') options.format = 'json';
     else if (arg === '--help' || arg === '-h') options.help = true;
-    else if ((arg === '--baseline' || arg === '--corpus') && i + 1 < argv.length) {
+    else if ((arg === '--baseline' || arg === '--corpus' || arg === '--sarif') && i + 1 < argv.length) {
       if (arg === '--baseline') options.baselines.push(argv[++i]);
+      else if (arg === '--sarif') options.sarif = path.resolve(argv[++i]);
       else options.corpus = path.resolve(argv[++i]);
     } else fail(`unknown or incomplete argument: ${arg}\n\n${USAGE}`);
   }
@@ -533,6 +629,7 @@ async function main() {
   }
   const report = await evaluate(options);
   process.stdout.write(options.format === 'json' ? `${JSON.stringify(report, null, 2)}\n` : renderMarkdown(report));
+  if (options.sarif) fs.writeFileSync(options.sarif, renderSarif(report), 'utf8');
   const working = report.hooks[0].summary.totals;
   if (working.mustDenyBypasses > 0 || working.mismatches > 0 || working.explicitAllows > 0 || working.errors > 0) {
     process.exitCode = 1;
@@ -548,4 +645,4 @@ if (!isMainThread && workerData && workerData.gateguardEval) {
   });
 }
 
-module.exports = { loadCorpus, materializeHook, runCorpus, summarize, questionsAsked, renderMarkdown, evaluate };
+module.exports = { loadCorpus, materializeHook, runCorpus, summarize, questionsAsked, renderMarkdown, renderSarif, parseArgs, evaluate };

tests/scripts/gateguard-eval.test.js

@@ -0,0 +1,153 @@
+/**
+ * Tests for scripts/dev/gateguard-eval.js argument parsing and SARIF output.
+ */
+
+'use strict';
+
+const assert = require('assert');
+const path = require('path');
+const { parseArgs, renderSarif } = require('../../scripts/dev/gateguard-eval');
+
+let passed = 0;
+let failed = 0;
+
+function test(name, fn) {
+  try {
+    fn();
+    console.log(`  ✓ ${name}`);
+    passed++;
+  } catch (error) {
+    console.log(`  ✗ ${name}`);
+    console.log(`    Error: ${error.message}`);
+    failed++;
+  }
+}
+
+function step(overrides) {
+  return {
+    scenario: 'sensitive-targets',
+    file: '09-sensitive-targets.json',
+    step: 'edit-env',
+    expect: 'deny',
+    decision: 'deny',
+    kind: 'deny',
+    mustDeny: true,
+    redundant: false,
+    ...overrides
+  };
+}
+
+function summary(steps, totals = {}) {
+  return {
+    totals: { steps: steps.length, denials: 0, mustDenyBypasses: 0, mismatches: 0, explicitAllows: 0, errors: 0, skippedScenarios: 0, ...totals },
+    latency: { p50Ms: 1.234, p95Ms: 5.678 },
+    perScenario: [],
+    steps
+  };
+}
+
+function report(workingSteps, baselineTotals = { mustDenyBypasses: 8 }) {
+  return {
+    corpus: { scenarios: 1, steps: workingSteps.length },
+    hooks: [
+      { label: 'working tree', ref: null, summary: summary(workingSteps) },
+      { label: 'upstream/main', ref: 'upstream/main', summary: summary([], baselineTotals) }
+    ]
+  };
+}
+
+function results(log) {
+  return log.runs[0].results;
+}
+
+console.log('\n=== Testing gateguard-eval ===\n');
+
+test('--sarif takes a file path and keeps the other options', () => {
+  const options = parseArgs(['--json', '--sarif', 'out/gate.sarif', '--baseline', 'main']);
+  assert.strictEqual(options.format, 'json');
+  assert.strictEqual(options.sarif, path.resolve('out/gate.sarif'));
+  assert.deepStrictEqual(options.baselines, ['main']);
+});
+
+test('--sarif without a value is rejected', () => {
+  assert.throws(() => parseArgs(['--sarif']), /unknown or incomplete argument: --sarif/);
+});
+
+test('a clean working tree produces a valid SARIF log with no results', () => {
+  const log = JSON.parse(renderSarif(report([step({}), step({ step: 'read-only', mustDeny: false, expect: 'allow', decision: 'allow', kind: 'pass' })])));
+  assert.strictEqual(log.version, '2.1.0');
+  assert.strictEqual(log.runs.length, 1);
+  assert.strictEqual(log.runs[0].tool.driver.name, 'gateguard-eval');
+  assert.deepStrictEqual(results(log), []);
+});
+
+test('every rule declares a default level', () => {
+  const rules = JSON.parse(renderSarif(report([]))).runs[0].tool.driver.rules;
+  assert.deepStrictEqual(
+    rules.map(rule => [rule.id, rule.defaultConfiguration.level]),
+    [
+      ['gateguard/must-deny-bypass', 'error'],
+      ['gateguard/explicit-allow', 'error'],
+      ['gateguard/hook-error', 'error'],
+      ['gateguard/expectation-mismatch', 'warning']
+    ]
+  );
+});
+
+test('a must-deny step that was allowed is an error on its scenario file', () => {
+  const log = JSON.parse(renderSarif(report([step({ decision: 'allow', kind: 'credit' })])));
+  assert.strictEqual(results(log).length, 1);
+  const [result] = results(log);
+  assert.strictEqual(result.ruleId, 'gateguard/must-deny-bypass');
+  assert.strictEqual(result.level, 'error');
+  assert.strictEqual(result.locations[0].physicalLocation.artifactLocation.uri, 'tests/fixtures/gateguard-scenarios/09-sensitive-targets.json');
+  assert.strictEqual(result.locations[0].physicalLocation.artifactLocation.uriBaseId, '%SRCROOT%');
+  assert.match(result.message.text, /sensitive-targets \/ edit-env: expected deny, got allow \(credit\)/);
+});
+
+test('an explicit allow and a hook error are reported as errors', () => {
+  const log = JSON.parse(
+    renderSarif(
+      report([
+        step({ step: 'a', mustDeny: false, expect: 'allow', decision: 'allow', kind: 'pass', explicitAllow: true }),
+        step({ step: 'b', mustDeny: false, error: 'boom' })
+      ])
+    )
+  );
+  assert.deepStrictEqual(results(log).map(result => [result.ruleId, result.level]), [
+    ['gateguard/explicit-allow', 'error'],
+    ['gateguard/hook-error', 'error']
+  ]);
+});
+
+test('a wrong decision that is not a bypass is a warning', () => {
+  const log = JSON.parse(renderSarif(report([step({ mustDeny: false, expect: 'allow', decision: 'deny', kind: 'deny' })])));
+  assert.deepStrictEqual(results(log).map(result => [result.ruleId, result.level]), [['gateguard/expectation-mismatch', 'warning']]);
+});
+
+test('a bypass is not reported a second time as a mismatch', () => {
+  const log = JSON.parse(renderSarif(report([step({ decision: 'allow', kind: 'pass' })])));
+  assert.deepStrictEqual(results(log).map(result => result.ruleId), ['gateguard/must-deny-bypass']);
+});
+
+test('baseline totals are recorded without their failures becoming results', () => {
+  const log = JSON.parse(renderSarif(report([step({})])));
+  const hooks = log.runs[0].properties.hooks;
+  assert.deepStrictEqual(hooks.map(hook => [hook.label, hook.totals.mustDenyBypasses]), [
+    ['working tree', 0],
+    ['upstream/main', 8]
+  ]);
+  assert.deepStrictEqual(results(log), []);
+});
+
+test('output leaves out timing so reruns are byte-identical', () => {
+  const text = renderSarif(report([step({ latencyMs: 3.21 })]));
+  assert.ok(!/latency|p50|p95|Time/i.test(text), 'timing leaked into SARIF');
+  const slower = report([step({ latencyMs: 9.87 })]);
+  slower.hooks[0].summary.latency = { p50Ms: 9, p95Ms: 99 };
+  assert.strictEqual(renderSarif(slower), text);
+});
+
+console.log(`\nPassed: ${passed}`);
+console.log(`Failed: ${failed}`);
+process.exitCode = failed > 0 ? 1 : 0;