diff --git a/scripts/dev/gateguard-eval.js b/scripts/dev/gateguard-eval.js index 8bb74b0e2..f44598d52 100644 --- a/scripts/dev/gateguard-eval.js +++ b/scripts/dev/gateguard-eval.js @@ -6,6 +6,7 @@ const fs = require('fs'); const os = require('os'); const path = require('path'); const { performance } = require('perf_hooks'); +const { pathToFileURL } = require('url'); const { Worker, isMainThread, parentPort, workerData } = require('worker_threads'); const REPO_ROOT = path.join(__dirname, '..', '..'); @@ -512,6 +513,15 @@ function sarifFindings(step) { return findings; } +function sarifArtifactLocation(corpusDir, file) { + const absolute = path.resolve(corpusDir || DEFAULT_CORPUS_DIR, file || ''); + const relative = path.relative(REPO_ROOT, absolute); + if (relative && !relative.startsWith('..') && !path.isAbsolute(relative)) { + return { uri: relative.split(path.sep).join('/'), uriBaseId: '%SRCROOT%' }; + } + return { uri: pathToFileURL(absolute).href }; +} + function sarifTotals(summary) { const { steps, denials, mustDenyBypasses, mismatches, explicitAllows, errors, skippedScenarios } = summary.totals; return { steps, denials, mustDenyBypasses, mismatches, explicitAllows, errors, skippedScenarios }; @@ -530,9 +540,7 @@ function renderSarif(report) { message: { text: `${step.scenario} / ${step.step}: expected ${step.expect}, got ${step.decision} (${step.kind})` }, locations: [ { - physicalLocation: { - artifactLocation: { uri: `tests/fixtures/gateguard-scenarios/${step.file}`, uriBaseId: '%SRCROOT%' } - } + physicalLocation: { artifactLocation: sarifArtifactLocation(report.corpusDir, step.file) } } ] }); @@ -614,6 +622,7 @@ async function evaluate({ baselines = [DEFAULT_BASELINE], corpus = DEFAULT_CORPU } return { corpus: { scenarios: scenarios.length, steps: scenarios.reduce((sum, scenario) => sum + scenario.steps.length, 0) }, + corpusDir: corpus, hooks: evaluated }; } finally { diff --git a/tests/scripts/gateguard-eval.test.js b/tests/scripts/gateguard-eval.test.js index 1c986709f..9935322a3 100644 --- a/tests/scripts/gateguard-eval.test.js +++ b/tests/scripts/gateguard-eval.test.js @@ -5,7 +5,9 @@ 'use strict'; const assert = require('assert'); +const os = require('os'); const path = require('path'); +const { pathToFileURL } = require('url'); const { parseArgs, renderSarif } = require('../../scripts/dev/gateguard-eval'); let passed = 0; @@ -105,6 +107,20 @@ test('a must-deny step that was allowed is an error on its scenario file', () => assert.match(result.message.text, /sensitive-targets \/ edit-env: expected deny, got allow \(credit\)/); }); +test('a finding from another corpus inside the repository links to that corpus', () => { + const failing = report([step({ decision: 'allow', kind: 'credit', file: 'custom.json' })]); + failing.corpusDir = path.join(__dirname, '..', 'fixtures', 'other-scenarios'); + const location = results(JSON.parse(renderSarif(failing)))[0].locations[0].physicalLocation.artifactLocation; + assert.deepStrictEqual(location, { uri: 'tests/fixtures/other-scenarios/custom.json', uriBaseId: '%SRCROOT%' }); +}); + +test('a finding from a corpus outside the repository links to its absolute file', () => { + const failing = report([step({ decision: 'allow', kind: 'credit', file: 'custom.json' })]); + failing.corpusDir = path.join(os.tmpdir(), 'gateguard-corpus'); + const location = results(JSON.parse(renderSarif(failing)))[0].locations[0].physicalLocation.artifactLocation; + assert.deepStrictEqual(location, { uri: pathToFileURL(path.join(os.tmpdir(), 'gateguard-corpus', 'custom.json')).href }); +}); + test('an explicit allow and a hook error are reported as errors', () => { const log = JSON.parse( renderSarif(