diff --git a/docs/gateguard/evaluation.md b/docs/gateguard/evaluation.md index b5476ceb3..63028bb8e 100644 --- a/docs/gateguard/evaluation.md +++ b/docs/gateguard/evaluation.md @@ -190,6 +190,7 @@ git fetch upstream main node scripts/dev/gateguard-eval.js --markdown node scripts/dev/gateguard-eval.js --markdown --baseline upstream/main --baseline 58a4a0d0 node scripts/dev/gateguard-eval.js --json > gateguard-eval.json +node scripts/dev/gateguard-eval.js --baseline upstream/main --sarif gateguard-eval.sarif node tests/hooks/gateguard-scenarios.test.js ``` @@ -198,6 +199,10 @@ node tests/hooks/gateguard-scenarios.test.js against it. `--corpus ` points at another scenario directory. The script exits non-zero when the working tree has a mismatch, a must-deny bypass, an explicit `allow` decision or a thrown error. +`--sarif ` also writes those working-tree failures as SARIF 2.1.0 +(bypasses, explicit allows and errors as `error`, other mismatches as +`warning`) with each hook's totals in the run properties. Timing is left out, +so the file is identical across reruns of the same tree and corpus. ## Corpus format diff --git a/scripts/dev/gateguard-eval.js b/scripts/dev/gateguard-eval.js index 629d588d7..8bb74b0e2 100644 --- a/scripts/dev/gateguard-eval.js +++ b/scripts/dev/gateguard-eval.js @@ -396,6 +396,7 @@ function summarize(runs, table = questionTextTable()) { if (result.error) totals.errors += 1; steps.push({ scenario: run.scenario.name, + file: run.scenario.file, step: step.id, expect: step.expect, decision: result.decision, @@ -407,6 +408,7 @@ function summarize(runs, table = questionTextTable()) { irrelevant, missed, latencyMs: Number(result.latencyMs.toFixed(3)), + ...(result.explicitAllow ? { explicitAllow: true } : {}), ...(result.error ? { error: result.error } : {}) }); }); @@ -474,27 +476,121 @@ function renderMarkdown(report) { return `${lines.join('\n')}\n`; } +const SARIF_RULES = Object.freeze([ + { + id: 'gateguard/must-deny-bypass', + level: 'error', + name: 'MustDenyBypass', + text: 'A step that must be denied (sensitive target, bypass attempt, or mutating first command) was allowed.' + }, + { + id: 'gateguard/explicit-allow', + level: 'error', + name: 'ExplicitAllow', + text: 'The hook emitted permissionDecision "allow", which skips the host permission prompt.' + }, + { + id: 'gateguard/hook-error', + level: 'error', + name: 'HookError', + text: 'The hook threw or produced unparseable output for a step.' + }, + { + id: 'gateguard/expectation-mismatch', + level: 'warning', + name: 'ExpectationMismatch', + text: 'The hook decision differs from the decision the scenario expects.' + } +]); + +function sarifFindings(step) { + const findings = []; + if (step.mustDeny && step.decision !== 'deny') findings.push('gateguard/must-deny-bypass'); + if (step.explicitAllow) findings.push('gateguard/explicit-allow'); + if (step.error) findings.push('gateguard/hook-error'); + if (step.decision !== step.expect && findings.length === 0) findings.push('gateguard/expectation-mismatch'); + return findings; +} + +function sarifTotals(summary) { + const { steps, denials, mustDenyBypasses, mismatches, explicitAllows, errors, skippedScenarios } = summary.totals; + return { steps, denials, mustDenyBypasses, mismatches, explicitAllows, errors, skippedScenarios }; +} + +/** SARIF 2.1.0 log of working-tree gate failures, with per-hook totals; timing is omitted so output is reproducible. */ +function renderSarif(report) { + const working = report.hooks[0].summary; + const levels = new Map(SARIF_RULES.map(rule => [rule.id, rule.level])); + const results = []; + for (const step of working.steps) { + for (const ruleId of sarifFindings(step)) { + results.push({ + ruleId, + level: levels.get(ruleId), + message: { text: `${step.scenario} / ${step.step}: expected ${step.expect}, got ${step.decision} (${step.kind})` }, + locations: [ + { + physicalLocation: { + artifactLocation: { uri: `tests/fixtures/gateguard-scenarios/${step.file}`, uriBaseId: '%SRCROOT%' } + } + } + ] + }); + } + } + const log = { + version: '2.1.0', + $schema: 'https://json.schemastore.org/sarif-2.1.0.json', + runs: [ + { + tool: { + driver: { + name: 'gateguard-eval', + informationUri: 'https://github.com/affaan-m/ECC/blob/main/docs/gateguard/evaluation.md', + rules: SARIF_RULES.map(rule => ({ + id: rule.id, + name: rule.name, + shortDescription: { text: rule.text }, + defaultConfiguration: { level: rule.level } + })) + } + }, + automationDetails: { id: 'gateguard/scenario-corpus' }, + invocations: [{ executionSuccessful: true }], + properties: { + corpus: report.corpus, + hooks: report.hooks.map(hook => ({ label: hook.label, totals: sarifTotals(hook.summary) })) + }, + results + } + ] + }; + return `${JSON.stringify(log, null, 2)}\n`; +} + // --- CLI --- const USAGE = [ - 'Usage: node scripts/dev/gateguard-eval.js [--markdown | --json] [--baseline ]... [--corpus ]', + 'Usage: node scripts/dev/gateguard-eval.js [--markdown | --json] [--baseline ]... [--corpus ] [--sarif ]', '', ' --baseline git ref whose hook is compared with the working tree (repeatable;', ` default ${DEFAULT_BASELINE})`, ' --corpus scenario directory (default tests/fixtures/gateguard-scenarios)', ' --markdown print markdown tables (default)', - ' --json print the full report as JSON' + ' --json print the full report as JSON', + ' --sarif also write working-tree gate failures as SARIF 2.1.0' ].join('\n'); function parseArgs(argv) { - const options = { format: 'markdown', baselines: [], corpus: DEFAULT_CORPUS_DIR, help: false }; + const options = { format: 'markdown', baselines: [], corpus: DEFAULT_CORPUS_DIR, sarif: null, help: false }; for (let i = 0; i < argv.length; i++) { const arg = argv[i]; if (arg === '--markdown') options.format = 'markdown'; else if (arg === '--json') options.format = 'json'; else if (arg === '--help' || arg === '-h') options.help = true; - else if ((arg === '--baseline' || arg === '--corpus') && i + 1 < argv.length) { + else if ((arg === '--baseline' || arg === '--corpus' || arg === '--sarif') && i + 1 < argv.length) { if (arg === '--baseline') options.baselines.push(argv[++i]); + else if (arg === '--sarif') options.sarif = path.resolve(argv[++i]); else options.corpus = path.resolve(argv[++i]); } else fail(`unknown or incomplete argument: ${arg}\n\n${USAGE}`); } @@ -533,6 +629,7 @@ async function main() { } const report = await evaluate(options); process.stdout.write(options.format === 'json' ? `${JSON.stringify(report, null, 2)}\n` : renderMarkdown(report)); + if (options.sarif) fs.writeFileSync(options.sarif, renderSarif(report), 'utf8'); const working = report.hooks[0].summary.totals; if (working.mustDenyBypasses > 0 || working.mismatches > 0 || working.explicitAllows > 0 || working.errors > 0) { process.exitCode = 1; @@ -548,4 +645,4 @@ if (!isMainThread && workerData && workerData.gateguardEval) { }); } -module.exports = { loadCorpus, materializeHook, runCorpus, summarize, questionsAsked, renderMarkdown, evaluate }; +module.exports = { loadCorpus, materializeHook, runCorpus, summarize, questionsAsked, renderMarkdown, renderSarif, parseArgs, evaluate }; diff --git a/tests/scripts/gateguard-eval.test.js b/tests/scripts/gateguard-eval.test.js new file mode 100644 index 000000000..1c986709f --- /dev/null +++ b/tests/scripts/gateguard-eval.test.js @@ -0,0 +1,153 @@ +/** + * Tests for scripts/dev/gateguard-eval.js argument parsing and SARIF output. + */ + +'use strict'; + +const assert = require('assert'); +const path = require('path'); +const { parseArgs, renderSarif } = require('../../scripts/dev/gateguard-eval'); + +let passed = 0; +let failed = 0; + +function test(name, fn) { + try { + fn(); + console.log(` ✓ ${name}`); + passed++; + } catch (error) { + console.log(` ✗ ${name}`); + console.log(` Error: ${error.message}`); + failed++; + } +} + +function step(overrides) { + return { + scenario: 'sensitive-targets', + file: '09-sensitive-targets.json', + step: 'edit-env', + expect: 'deny', + decision: 'deny', + kind: 'deny', + mustDeny: true, + redundant: false, + ...overrides + }; +} + +function summary(steps, totals = {}) { + return { + totals: { steps: steps.length, denials: 0, mustDenyBypasses: 0, mismatches: 0, explicitAllows: 0, errors: 0, skippedScenarios: 0, ...totals }, + latency: { p50Ms: 1.234, p95Ms: 5.678 }, + perScenario: [], + steps + }; +} + +function report(workingSteps, baselineTotals = { mustDenyBypasses: 8 }) { + return { + corpus: { scenarios: 1, steps: workingSteps.length }, + hooks: [ + { label: 'working tree', ref: null, summary: summary(workingSteps) }, + { label: 'upstream/main', ref: 'upstream/main', summary: summary([], baselineTotals) } + ] + }; +} + +function results(log) { + return log.runs[0].results; +} + +console.log('\n=== Testing gateguard-eval ===\n'); + +test('--sarif takes a file path and keeps the other options', () => { + const options = parseArgs(['--json', '--sarif', 'out/gate.sarif', '--baseline', 'main']); + assert.strictEqual(options.format, 'json'); + assert.strictEqual(options.sarif, path.resolve('out/gate.sarif')); + assert.deepStrictEqual(options.baselines, ['main']); +}); + +test('--sarif without a value is rejected', () => { + assert.throws(() => parseArgs(['--sarif']), /unknown or incomplete argument: --sarif/); +}); + +test('a clean working tree produces a valid SARIF log with no results', () => { + const log = JSON.parse(renderSarif(report([step({}), step({ step: 'read-only', mustDeny: false, expect: 'allow', decision: 'allow', kind: 'pass' })]))); + assert.strictEqual(log.version, '2.1.0'); + assert.strictEqual(log.runs.length, 1); + assert.strictEqual(log.runs[0].tool.driver.name, 'gateguard-eval'); + assert.deepStrictEqual(results(log), []); +}); + +test('every rule declares a default level', () => { + const rules = JSON.parse(renderSarif(report([]))).runs[0].tool.driver.rules; + assert.deepStrictEqual( + rules.map(rule => [rule.id, rule.defaultConfiguration.level]), + [ + ['gateguard/must-deny-bypass', 'error'], + ['gateguard/explicit-allow', 'error'], + ['gateguard/hook-error', 'error'], + ['gateguard/expectation-mismatch', 'warning'] + ] + ); +}); + +test('a must-deny step that was allowed is an error on its scenario file', () => { + const log = JSON.parse(renderSarif(report([step({ decision: 'allow', kind: 'credit' })]))); + assert.strictEqual(results(log).length, 1); + const [result] = results(log); + assert.strictEqual(result.ruleId, 'gateguard/must-deny-bypass'); + assert.strictEqual(result.level, 'error'); + assert.strictEqual(result.locations[0].physicalLocation.artifactLocation.uri, 'tests/fixtures/gateguard-scenarios/09-sensitive-targets.json'); + assert.strictEqual(result.locations[0].physicalLocation.artifactLocation.uriBaseId, '%SRCROOT%'); + assert.match(result.message.text, /sensitive-targets \/ edit-env: expected deny, got allow \(credit\)/); +}); + +test('an explicit allow and a hook error are reported as errors', () => { + const log = JSON.parse( + renderSarif( + report([ + step({ step: 'a', mustDeny: false, expect: 'allow', decision: 'allow', kind: 'pass', explicitAllow: true }), + step({ step: 'b', mustDeny: false, error: 'boom' }) + ]) + ) + ); + assert.deepStrictEqual(results(log).map(result => [result.ruleId, result.level]), [ + ['gateguard/explicit-allow', 'error'], + ['gateguard/hook-error', 'error'] + ]); +}); + +test('a wrong decision that is not a bypass is a warning', () => { + const log = JSON.parse(renderSarif(report([step({ mustDeny: false, expect: 'allow', decision: 'deny', kind: 'deny' })]))); + assert.deepStrictEqual(results(log).map(result => [result.ruleId, result.level]), [['gateguard/expectation-mismatch', 'warning']]); +}); + +test('a bypass is not reported a second time as a mismatch', () => { + const log = JSON.parse(renderSarif(report([step({ decision: 'allow', kind: 'pass' })]))); + assert.deepStrictEqual(results(log).map(result => result.ruleId), ['gateguard/must-deny-bypass']); +}); + +test('baseline totals are recorded without their failures becoming results', () => { + const log = JSON.parse(renderSarif(report([step({})]))); + const hooks = log.runs[0].properties.hooks; + assert.deepStrictEqual(hooks.map(hook => [hook.label, hook.totals.mustDenyBypasses]), [ + ['working tree', 0], + ['upstream/main', 8] + ]); + assert.deepStrictEqual(results(log), []); +}); + +test('output leaves out timing so reruns are byte-identical', () => { + const text = renderSarif(report([step({ latencyMs: 3.21 })])); + assert.ok(!/latency|p50|p95|Time/i.test(text), 'timing leaked into SARIF'); + const slower = report([step({ latencyMs: 9.87 })]); + slower.hooks[0].summary.latency = { p50Ms: 9, p95Ms: 99 }; + assert.strictEqual(renderSarif(slower), text); +}); + +console.log(`\nPassed: ${passed}`); +console.log(`Failed: ${failed}`); +process.exitCode = failed > 0 ? 1 : 0;