mirror of
https://github.com/affaan-m/ECC.git
synced 2026-10-18 05:57:40 +02:00
feat(gateguard): integrate evidence-aware first-touch questions
This commit is contained in:
haelyra
@@ -0,0 +1,277 @@
|
||||
/**
|
||||
* Tests for scripts/dev/gateguard-eval.js argument parsing and SARIF output.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const { pathToFileURL } = require('url');
|
||||
const {
|
||||
parseArgs, renderSarif, renderMarkdown, loadCorpus, runCorpus, summarize, measureColdLatency, coldSummary, gateFails
|
||||
} = require('../../scripts/dev/gateguard-eval');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function test(name, fn) {
|
||||
try {
|
||||
fn();
|
||||
console.log(` ✓ ${name}`);
|
||||
passed++;
|
||||
} catch (error) {
|
||||
console.log(` ✗ ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
function step(overrides) {
|
||||
return {
|
||||
scenario: 'sensitive-targets',
|
||||
file: '09-sensitive-targets.json',
|
||||
step: 'edit-env',
|
||||
expect: 'deny',
|
||||
decision: 'deny',
|
||||
kind: 'deny',
|
||||
mustDeny: true,
|
||||
redundant: false,
|
||||
...overrides
|
||||
};
|
||||
}
|
||||
|
||||
function summary(steps, totals = {}) {
|
||||
return {
|
||||
totals: { steps: steps.length, denials: 0, mustDenyBypasses: 0, mismatches: 0, explicitAllows: 0, errors: 0, skippedScenarios: 0, ...totals },
|
||||
latency: { p50Ms: 1.234, p90Ms: 4.321, p95Ms: 5.678 },
|
||||
perScenario: [],
|
||||
steps
|
||||
};
|
||||
}
|
||||
|
||||
function report(workingSteps, baselineTotals = { mustDenyBypasses: 8 }) {
|
||||
return {
|
||||
corpus: { scenarios: 1, steps: workingSteps.length },
|
||||
hooks: [
|
||||
{ label: 'working tree', ref: null, summary: summary(workingSteps) },
|
||||
{ label: 'upstream/main', ref: 'upstream/main', summary: summary([], baselineTotals) }
|
||||
]
|
||||
};
|
||||
}
|
||||
|
||||
function results(log) {
|
||||
return log.runs[0].results;
|
||||
}
|
||||
|
||||
console.log('\n=== Testing gateguard-eval ===\n');
|
||||
|
||||
test('--sarif takes a file path and keeps the other options', () => {
|
||||
const options = parseArgs(['--json', '--sarif', 'out/gate.sarif', '--baseline', 'main']);
|
||||
assert.strictEqual(options.format, 'json');
|
||||
assert.strictEqual(options.sarif, path.resolve('out/gate.sarif'));
|
||||
assert.deepStrictEqual(options.baselines, ['main']);
|
||||
});
|
||||
|
||||
test('--sarif without a value is rejected', () => {
|
||||
assert.throws(() => parseArgs(['--sarif']), /unknown or incomplete argument: --sarif/);
|
||||
});
|
||||
|
||||
test('a clean working tree produces a valid SARIF log with no results', () => {
|
||||
const log = JSON.parse(renderSarif(report([step({}), step({ step: 'read-only', mustDeny: false, expect: 'allow', decision: 'allow', kind: 'pass' })])));
|
||||
assert.strictEqual(log.version, '2.1.0');
|
||||
assert.strictEqual(log.runs.length, 1);
|
||||
assert.strictEqual(log.runs[0].tool.driver.name, 'gateguard-eval');
|
||||
assert.deepStrictEqual(results(log), []);
|
||||
});
|
||||
|
||||
test('every rule declares a default level', () => {
|
||||
const rules = JSON.parse(renderSarif(report([]))).runs[0].tool.driver.rules;
|
||||
assert.deepStrictEqual(
|
||||
rules.map(rule => [rule.id, rule.defaultConfiguration.level]),
|
||||
[
|
||||
['gateguard/must-deny-bypass', 'error'],
|
||||
['gateguard/explicit-allow', 'error'],
|
||||
['gateguard/hook-error', 'error'],
|
||||
['gateguard/expectation-mismatch', 'warning']
|
||||
]
|
||||
);
|
||||
});
|
||||
|
||||
test('a must-deny step that was allowed is an error on its scenario file', () => {
|
||||
const log = JSON.parse(renderSarif(report([step({ decision: 'allow', kind: 'credit' })])));
|
||||
assert.strictEqual(results(log).length, 1);
|
||||
const [result] = results(log);
|
||||
assert.strictEqual(result.ruleId, 'gateguard/must-deny-bypass');
|
||||
assert.strictEqual(result.level, 'error');
|
||||
assert.strictEqual(result.locations[0].physicalLocation.artifactLocation.uri, 'tests/fixtures/gateguard-scenarios/09-sensitive-targets.json');
|
||||
assert.strictEqual(result.locations[0].physicalLocation.artifactLocation.uriBaseId, '%SRCROOT%');
|
||||
assert.match(result.message.text, /sensitive-targets \/ edit-env: expected deny, got allow \(credit\)/);
|
||||
});
|
||||
|
||||
test('a finding from another corpus inside the repository links to that corpus', () => {
|
||||
const failing = report([step({ decision: 'allow', kind: 'credit', file: 'custom.json' })]);
|
||||
failing.corpusDir = path.join(__dirname, '..', 'fixtures', 'other-scenarios');
|
||||
const location = results(JSON.parse(renderSarif(failing)))[0].locations[0].physicalLocation.artifactLocation;
|
||||
assert.deepStrictEqual(location, { uri: 'tests/fixtures/other-scenarios/custom.json', uriBaseId: '%SRCROOT%' });
|
||||
});
|
||||
|
||||
test('a finding from a corpus outside the repository links to its absolute file', () => {
|
||||
const failing = report([step({ decision: 'allow', kind: 'credit', file: 'custom.json' })]);
|
||||
failing.corpusDir = path.join(os.tmpdir(), 'gateguard-corpus');
|
||||
const location = results(JSON.parse(renderSarif(failing)))[0].locations[0].physicalLocation.artifactLocation;
|
||||
assert.deepStrictEqual(location, { uri: pathToFileURL(path.join(os.tmpdir(), 'gateguard-corpus', 'custom.json')).href });
|
||||
});
|
||||
|
||||
test('an explicit allow and a hook error are reported as errors', () => {
|
||||
const log = JSON.parse(
|
||||
renderSarif(
|
||||
report([
|
||||
step({ step: 'a', mustDeny: false, expect: 'allow', decision: 'allow', kind: 'pass', explicitAllow: true }),
|
||||
step({ step: 'b', mustDeny: false, error: 'boom' })
|
||||
])
|
||||
)
|
||||
);
|
||||
assert.deepStrictEqual(results(log).map(result => [result.ruleId, result.level]), [
|
||||
['gateguard/explicit-allow', 'error'],
|
||||
['gateguard/hook-error', 'error']
|
||||
]);
|
||||
});
|
||||
|
||||
test('a wrong decision that is not a bypass is a warning', () => {
|
||||
const log = JSON.parse(renderSarif(report([step({ mustDeny: false, expect: 'allow', decision: 'deny', kind: 'deny' })])));
|
||||
assert.deepStrictEqual(results(log).map(result => [result.ruleId, result.level]), [['gateguard/expectation-mismatch', 'warning']]);
|
||||
});
|
||||
|
||||
test('a bypass is not reported a second time as a mismatch', () => {
|
||||
const log = JSON.parse(renderSarif(report([step({ decision: 'allow', kind: 'pass' })])));
|
||||
assert.deepStrictEqual(results(log).map(result => result.ruleId), ['gateguard/must-deny-bypass']);
|
||||
});
|
||||
|
||||
test('baseline totals are recorded without their failures becoming results', () => {
|
||||
const log = JSON.parse(renderSarif(report([step({})])));
|
||||
const hooks = log.runs[0].properties.hooks;
|
||||
assert.deepStrictEqual(hooks.map(hook => [hook.label, hook.totals.mustDenyBypasses]), [
|
||||
['working tree', 0],
|
||||
['upstream/main', 8]
|
||||
]);
|
||||
assert.deepStrictEqual(results(log), []);
|
||||
});
|
||||
|
||||
test('output leaves out timing so reruns are byte-identical', () => {
|
||||
const text = renderSarif(report([step({ latencyMs: 3.21 })]));
|
||||
assert.ok(!/latency|p50|p95|Time/i.test(text), 'timing leaked into SARIF');
|
||||
const slower = report([step({ latencyMs: 9.87 })]);
|
||||
slower.hooks[0].summary.latency = { p50Ms: 9, p90Ms: 90, p95Ms: 99 };
|
||||
assert.strictEqual(renderSarif(slower), text);
|
||||
});
|
||||
|
||||
test('--no-cold turns off the fresh-process pass', () => {
|
||||
assert.strictEqual(parseArgs([]).cold, true);
|
||||
assert.strictEqual(parseArgs(['--no-cold']).cold, false);
|
||||
});
|
||||
|
||||
// A fresh-process failure is recorded as an allow. Where the worker also allowed the
|
||||
// step it is no disagreement, so before failures were counted on their own it left
|
||||
// no trace in the report, the latency or the exit status.
|
||||
test('a fresh-process failure on a step the worker allows is still counted', () => {
|
||||
const cold = coldSummary(
|
||||
[{ scenario: 's', step: 'a', latencyMs: null, decision: 'allow', error: 'killed' }],
|
||||
[{ scenario: 's', step: 'a', decision: 'allow' }]
|
||||
);
|
||||
assert.strictEqual(cold.disagreements, 0);
|
||||
assert.strictEqual(cold.errors, 1);
|
||||
assert.strictEqual(cold.samples, 0);
|
||||
assert.strictEqual(cold.p50Ms, null, 'no measured step must not read as a 0 ms latency');
|
||||
});
|
||||
|
||||
test('a fresh-process failure fails the gate when nothing else does', () => {
|
||||
const clean = report([step()]);
|
||||
clean.hooks[0].summary.cold = { p50Ms: 1, p90Ms: 1, p95Ms: 1, samples: 1, errors: 0, disagreements: 0 };
|
||||
assert.strictEqual(gateFails(clean), false);
|
||||
const failed = report([step()]);
|
||||
failed.hooks[0].summary.cold = { p50Ms: null, p90Ms: null, p95Ms: null, samples: 0, errors: 1, disagreements: 0 };
|
||||
assert.strictEqual(gateFails(failed), true);
|
||||
});
|
||||
|
||||
test('the report names fresh-process failures and invents no latency', () => {
|
||||
// summarize([]) is the real summary shape with every count at zero, so this keeps
|
||||
// working as metrics are added, which a hand-built totals object would not.
|
||||
const working = { ...summarize([]), cold: { p50Ms: null, p90Ms: null, p95Ms: null, samples: 0, errors: 1, disagreements: 0 } };
|
||||
const failed = {
|
||||
corpus: { scenarios: 0, steps: 0 },
|
||||
hooks: [
|
||||
{ label: 'working tree', ref: null, summary: working },
|
||||
{ label: 'upstream/main', ref: 'upstream/main', summary: summarize([]) }
|
||||
]
|
||||
};
|
||||
const text = renderMarkdown(failed);
|
||||
assert.match(text, /1 step\(s\) failed in a fresh process/);
|
||||
assert.match(text, /no valid sample/);
|
||||
assert.ok(!/fresh process \(ms\)[^\n]*\b0\.00\b/.test(text), 'a latency was reported with no measured step');
|
||||
});
|
||||
|
||||
async function coldTests() {
|
||||
const corpusDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-eval-corpus-'));
|
||||
try {
|
||||
fs.copyFileSync(
|
||||
path.join(__dirname, '..', 'fixtures', 'gateguard-scenarios', '11-first-shell-commands.json'),
|
||||
path.join(corpusDir, 'shell.json')
|
||||
);
|
||||
const scenarios = loadCorpus(corpusDir);
|
||||
const hookFile = path.join(__dirname, '..', '..', 'scripts', 'hooks', 'gateguard-fact-force.js');
|
||||
const warm = summarize(await runCorpus(hookFile, scenarios));
|
||||
const cold = measureColdLatency(hookFile, scenarios);
|
||||
test('the fresh-process pass times every step', () => {
|
||||
assert.strictEqual(cold.length, scenarios[0].steps.length);
|
||||
assert.ok(cold.every(step => Number.isFinite(step.latencyMs) && step.latencyMs > 0));
|
||||
});
|
||||
test('fresh processes decide every step as the worker does', () => {
|
||||
assert.deepStrictEqual(cold.map(step => step.decision), warm.steps.map(step => step.decision));
|
||||
});
|
||||
|
||||
// A hook that throws used to end the fresh-process pass, so neither the report
|
||||
// nor the SARIF output was written for a run that had hit one bad step.
|
||||
const throwingDir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-eval-throw-'));
|
||||
try {
|
||||
const throwingHook = path.join(throwingDir, 'throwing-hook.js');
|
||||
fs.writeFileSync(throwingHook, "'use strict';\nmodule.exports = { run() { throw new Error('boom'); } };\n");
|
||||
const warmThrow = summarize(await runCorpus(throwingHook, scenarios));
|
||||
let coldThrow = null;
|
||||
let escaped = null;
|
||||
try {
|
||||
coldThrow = measureColdLatency(throwingHook, scenarios);
|
||||
} catch (error) {
|
||||
escaped = error;
|
||||
}
|
||||
test('a hook that throws does not end the fresh-process pass', () => {
|
||||
assert.strictEqual(escaped, null, escaped && escaped.message);
|
||||
assert.strictEqual(coldThrow.length, scenarios[0].steps.length);
|
||||
assert.ok(coldThrow.every(step => step.error === 'boom'), JSON.stringify(coldThrow.map(step => step.error)));
|
||||
});
|
||||
test('a throwing step is decided as the worker decides it, so it is no disagreement', () => {
|
||||
assert.deepStrictEqual(coldThrow.map(step => step.decision), warmThrow.steps.map(step => step.decision));
|
||||
});
|
||||
test('every throwing step is reported as a fresh-process failure', () => {
|
||||
assert.strictEqual(coldSummary(coldThrow, warmThrow.steps).errors, coldThrow.length);
|
||||
});
|
||||
} finally {
|
||||
fs.rmSync(throwingDir, { recursive: true, force: true });
|
||||
}
|
||||
} finally {
|
||||
fs.rmSync(corpusDir, { recursive: true, force: true });
|
||||
}
|
||||
}
|
||||
|
||||
coldTests().then(finish, error => {
|
||||
console.log(` ✗ fresh-process pass failed: ${error.message}`);
|
||||
failed++;
|
||||
finish();
|
||||
});
|
||||
|
||||
function finish() {
|
||||
console.log(`\nPassed: ${passed}`);
|
||||
console.log(`Failed: ${failed}`);
|
||||
process.exitCode = failed > 0 ? 1 : 0;
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/**
|
||||
* Tests for scripts/dev/gateguard-latency.js.
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const path = require('path');
|
||||
const { quantile, bootstrapDifference, verdict, summarize, measure, parseArgs } = require('../../scripts/dev/gateguard-latency');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function test(name, fn) {
|
||||
try {
|
||||
fn();
|
||||
console.log(` ✓ ${name}`);
|
||||
passed++;
|
||||
} catch (error) {
|
||||
console.log(` ✗ ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
console.log('\n=== Testing gateguard-latency ===\n');
|
||||
|
||||
test('quantiles interpolate between ranks', () => {
|
||||
assert.strictEqual(quantile([1, 2, 3, 4], 0.5), 2.5);
|
||||
assert.strictEqual(quantile([4, 1, 3, 2], 0), 1);
|
||||
assert.strictEqual(quantile([4, 1, 3, 2], 1), 4);
|
||||
assert.ok(Math.abs(quantile([1, 2, 3, 4, 5, 6, 7, 8, 9, 10], 0.9) - 9.1) < 1e-9);
|
||||
assert.ok(Number.isNaN(quantile([], 0.5)));
|
||||
});
|
||||
|
||||
const slow = Array.from({ length: 40 }, (_, i) => 20 + (i % 5));
|
||||
const fast = Array.from({ length: 40 }, (_, i) => 10 + (i % 5));
|
||||
const same = Array.from({ length: 40 }, (_, i) => 20 + ((i * 7) % 5));
|
||||
|
||||
test('a clearly faster sample is reported faster at p50 and p90', () => {
|
||||
assert.strictEqual(verdict(bootstrapDifference(slow, fast, 0.5)), 'faster');
|
||||
assert.strictEqual(verdict(bootstrapDifference(slow, fast, 0.9)), 'faster');
|
||||
});
|
||||
|
||||
test('a clearly slower sample is reported slower', () => {
|
||||
assert.strictEqual(verdict(bootstrapDifference(fast, slow, 0.5)), 'slower');
|
||||
});
|
||||
|
||||
test('samples from the same distribution show no measurable change', () => {
|
||||
assert.strictEqual(verdict(bootstrapDifference(slow, same, 0.5)), 'no measurable change');
|
||||
assert.strictEqual(verdict(bootstrapDifference(slow, same, 0.9)), 'no measurable change');
|
||||
});
|
||||
|
||||
test('the interval contains the observed difference and is reproducible', () => {
|
||||
const interval = bootstrapDifference(slow, fast, 0.5);
|
||||
const observed = quantile(fast, 0.5) - quantile(slow, 0.5);
|
||||
assert.ok(interval[0] <= observed && observed <= interval[1], `${observed} outside ${interval}`);
|
||||
assert.deepStrictEqual(bootstrapDifference(slow, fast, 0.5), interval);
|
||||
});
|
||||
|
||||
test('summaries compare the working tree with every baseline', () => {
|
||||
const rows = summarize({ call: { 'working tree': fast, main: slow } }, ['working tree', 'main']);
|
||||
assert.strictEqual(rows.length, 1);
|
||||
assert.strictEqual(rows[0].comparisons[0].baseline, 'main');
|
||||
assert.strictEqual(rows[0].comparisons[0].p50Verdict, 'faster');
|
||||
assert.strictEqual(rows[0].quantiles.main.p50, quantile(slow, 0.5));
|
||||
});
|
||||
|
||||
test('arguments default to upstream/main and validate --runs', () => {
|
||||
assert.deepStrictEqual(parseArgs([]).baselines, ['upstream/main']);
|
||||
assert.strictEqual(parseArgs(['--runs', '5']).runs, 5);
|
||||
assert.throws(() => parseArgs(['--runs', '0']), /incomplete argument/);
|
||||
assert.throws(() => parseArgs(['--runs']), /incomplete argument/);
|
||||
});
|
||||
|
||||
test('one round times every call type in fresh processes', () => {
|
||||
const hookFile = path.join(__dirname, '..', '..', 'scripts', 'hooks', 'gateguard-fact-force.js');
|
||||
const samples = measure([{ label: 'working tree', file: hookFile }], { runs: 1 });
|
||||
const calls = Object.keys(samples);
|
||||
assert.strictEqual(calls.length, 7);
|
||||
for (const call of calls) {
|
||||
assert.strictEqual(samples[call]['working tree'].length, 1);
|
||||
assert.ok(samples[call]['working tree'][0] > 0, call);
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`\nPassed: ${passed}`);
|
||||
console.log(`Failed: ${failed}`);
|
||||
process.exitCode = failed > 0 ? 1 : 0;
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Tests for scripts/gateguard-report.js
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const { spawnSync } = require('child_process');
|
||||
|
||||
const SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'gateguard-report.js');
|
||||
const FIXTURE_DIR = path.join(__dirname, '..', 'fixtures', 'gateguard-metrics');
|
||||
const { loadMetrics, summarize, formatReport, parseArgs } = require('../../scripts/gateguard-report');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function test(name, fn) {
|
||||
try {
|
||||
fn();
|
||||
console.log(` ✓ ${name}`);
|
||||
passed++;
|
||||
} catch (error) {
|
||||
console.log(` ✗ ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
function run(args, env = {}) {
|
||||
const result = spawnSync('node', [SCRIPT, ...args], {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, ...env },
|
||||
timeout: 10000
|
||||
});
|
||||
return { status: result.status, stdout: result.stdout || '', stderr: result.stderr || '' };
|
||||
}
|
||||
|
||||
console.log('\n=== Testing gateguard-report ===\n');
|
||||
|
||||
const loaded = loadMetrics(FIXTURE_DIR);
|
||||
const report = summarize(loaded.events);
|
||||
|
||||
test('reads the rotated file first, then the current file', () => {
|
||||
assert.strictEqual(loaded.events.length, 12);
|
||||
assert.strictEqual(loaded.events[0].ts, '2026-09-01T10:00:00.000Z');
|
||||
assert.strictEqual(loaded.events[11].decision, 'pass-checked');
|
||||
assert.deepStrictEqual(loaded.files.map(f => path.basename(f)), ['metrics.jsonl.1', 'metrics.jsonl']);
|
||||
});
|
||||
|
||||
test('skips malformed, unknown-version, unknown-decision and non-code lines', () => {
|
||||
assert.strictEqual(loaded.skipped, 6);
|
||||
assert.ok(loaded.events.every(e => e.session === 'aaaaaaaaaaaa' || e.session === 'bbbbbbbbbbbb'));
|
||||
assert.ok(!loaded.events.some(e => String(e.reason).includes('/')));
|
||||
});
|
||||
|
||||
test('counts decisions by type', () => {
|
||||
assert.deepStrictEqual(report.total.decisions, {
|
||||
deny: 4,
|
||||
cap: 1,
|
||||
credit: 1,
|
||||
'destructive-deny': 1,
|
||||
'pass-checked': 1,
|
||||
'routine-deny': 1,
|
||||
'routine-readonly': 1,
|
||||
sibling: 1,
|
||||
trivial: 1
|
||||
});
|
||||
assert.strictEqual(report.total.events, 12);
|
||||
});
|
||||
|
||||
test('breaks denials down by class and reason', () => {
|
||||
assert.strictEqual(report.total.denials.total, 6);
|
||||
assert.deepStrictEqual(report.total.denials.byClass, { code: 3, shell: 2, config: 1 });
|
||||
assert.deepStrictEqual(report.total.denials.byReason, {
|
||||
'first-touch': 2,
|
||||
destructive: 1,
|
||||
'first-command': 1,
|
||||
'near-miss:out-of-scope': 1,
|
||||
sensitive: 1
|
||||
});
|
||||
});
|
||||
|
||||
test('reports the near-miss share of first-touch denials', () => {
|
||||
assert.deepStrictEqual(report.total.nearMiss, { count: 1, of: 4, share: 0.25 });
|
||||
});
|
||||
|
||||
test('reports first-touch outcome rates', () => {
|
||||
const { firstTouch } = report.total;
|
||||
assert.strictEqual(firstTouch.total, 8);
|
||||
assert.deepStrictEqual(firstTouch.counts, { deny: 4, credit: 1, sibling: 1, cap: 1, trivial: 1 });
|
||||
assert.strictEqual(firstTouch.rates.deny, 0.5);
|
||||
assert.strictEqual(firstTouch.rates.trivial, 0.125);
|
||||
});
|
||||
|
||||
test('counts routine read-only passes and ranks the most-asked questions', () => {
|
||||
assert.strictEqual(report.total.routineReadonlyPasses, 1);
|
||||
assert.deepStrictEqual(report.total.topQuestions.slice(0, 3), [
|
||||
['quote-instruction', 4],
|
||||
['callers', 1],
|
||||
['config-effect', 1]
|
||||
]);
|
||||
});
|
||||
|
||||
test('summarizes each session separately in order of first appearance', () => {
|
||||
assert.deepStrictEqual(Object.keys(report.sessions), ['aaaaaaaaaaaa', 'bbbbbbbbbbbb']);
|
||||
const a = report.sessions.aaaaaaaaaaaa;
|
||||
assert.strictEqual(a.events, 6);
|
||||
assert.strictEqual(a.denials.total, 3);
|
||||
assert.deepStrictEqual(a.nearMiss, { count: 1, of: 2, share: 0.5 });
|
||||
assert.strictEqual(a.firstTouch.total, 4);
|
||||
assert.strictEqual(a.first, '2026-09-01T10:00:00.000Z');
|
||||
assert.strictEqual(a.last, '2026-09-01T10:05:00.000Z');
|
||||
const b = report.sessions.bbbbbbbbbbbb;
|
||||
assert.strictEqual(b.events, 6);
|
||||
assert.deepStrictEqual(b.denials.byClass, { code: 1, config: 1, shell: 1 });
|
||||
assert.strictEqual(b.routineReadonlyPasses, 0);
|
||||
});
|
||||
|
||||
test('reports empty rates as null instead of dividing by zero', () => {
|
||||
const empty = summarize([]);
|
||||
assert.strictEqual(empty.total.events, 0);
|
||||
assert.strictEqual(empty.total.nearMiss.share, null);
|
||||
assert.strictEqual(empty.total.firstTouch.rates.deny, null);
|
||||
});
|
||||
|
||||
test('formats a readable text report', () => {
|
||||
const text = formatReport({ ...loaded, report });
|
||||
assert.ok(text.includes('12 events in 2 sessions (6 malformed lines skipped)'));
|
||||
assert.ok(text.includes('Total'));
|
||||
assert.ok(text.includes('Session aaaaaaaaaaaa'));
|
||||
assert.ok(text.includes('near-miss share of first-touch denials: 25.0% (1 of 4)'));
|
||||
assert.ok(text.includes('routine read-only passes: 1'));
|
||||
assert.ok(text.includes('quote-instruction 4'));
|
||||
});
|
||||
|
||||
test('parses --dir, --dir=, --json and --help', () => {
|
||||
assert.deepStrictEqual(parseArgs(['--dir', '/x', '--json']), { dir: '/x', json: true, help: false });
|
||||
assert.deepStrictEqual(parseArgs(['--dir=/y']), { dir: '/y', json: false, help: false });
|
||||
assert.strictEqual(parseArgs(['-h']).help, true);
|
||||
assert.throws(() => parseArgs(['--dir']), /Missing value for --dir/);
|
||||
assert.throws(() => parseArgs(['--bogus']), /Unknown argument: --bogus/);
|
||||
});
|
||||
|
||||
test('CLI prints JSON with --json', () => {
|
||||
const result = run(['--dir', FIXTURE_DIR, '--json']);
|
||||
assert.strictEqual(result.status, 0, result.stderr);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
assert.strictEqual(parsed.events, 12);
|
||||
assert.strictEqual(parsed.skipped, 6);
|
||||
assert.strictEqual(parsed.total.denials.total, 6);
|
||||
assert.deepStrictEqual(Object.keys(parsed.sessions), ['aaaaaaaaaaaa', 'bbbbbbbbbbbb']);
|
||||
});
|
||||
|
||||
test('CLI reads GATEGUARD_STATE_DIR when --dir is not given', () => {
|
||||
const result = run([], { GATEGUARD_STATE_DIR: FIXTURE_DIR });
|
||||
assert.strictEqual(result.status, 0, result.stderr);
|
||||
assert.ok(result.stdout.includes('12 events in 2 sessions'));
|
||||
});
|
||||
|
||||
test('CLI explains how to enable metrics when none are found', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-report-'));
|
||||
try {
|
||||
const result = run(['--dir', dir]);
|
||||
assert.strictEqual(result.status, 0, result.stderr);
|
||||
assert.ok(result.stdout.includes('No GateGuard metrics'));
|
||||
assert.ok(result.stdout.includes('GATEGUARD_METRICS=1'));
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('CLI exits 1 on an unknown argument', () => {
|
||||
const result = run(['--bogus']);
|
||||
assert.strictEqual(result.status, 1);
|
||||
assert.ok(result.stderr.includes('Unknown argument: --bogus'));
|
||||
});
|
||||
|
||||
console.log(`\n ${passed} passed, ${failed} failed\n`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
@@ -108,6 +108,7 @@ function buildExpectedPublishPaths(repoRoot) {
|
||||
"scripts/welcome.js",
|
||||
"scripts/gemini-adapt-agents.js",
|
||||
"scripts/qwen-adapt-agents.js",
|
||||
"scripts/gateguard-report.js",
|
||||
"scripts/sync-ecc-to-codex.sh",
|
||||
"scripts/codex/legacy-sync-state.js",
|
||||
"scripts/codex/install-global-git-hooks.sh",
|
||||
|
||||
Reference in New Issue
Block a user