mirror of
https://github.com/affaan-m/ECC.git
synced 2026-10-21 07:21:46 +02:00
feat(gateguard): add metrics event schema and report script
This commit is contained in:
Jeffrey Montoya
@@ -124,6 +124,7 @@
|
||||
"scripts/memory.js",
|
||||
"scripts/memory-mcp.mjs",
|
||||
"scripts/gemini-adapt-agents.js",
|
||||
"scripts/gateguard-report.js",
|
||||
"scripts/harness-adapter-compliance.js",
|
||||
"scripts/harness-audit.js",
|
||||
"scripts/eval-harness.js",
|
||||
|
||||
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env node
|
||||
'use strict';
|
||||
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const { METRICS_FILE_NAME, isMetricsEvent } = require('./lib/gateguard-metrics');
|
||||
|
||||
const DENIAL_DECISIONS = new Set(['deny', 'routine-deny', 'destructive-deny']);
|
||||
const FIRST_TOUCH_DECISIONS = ['deny', 'credit', 'sibling', 'cap', 'trivial'];
|
||||
const NEAR_MISS_PREFIX = 'near-miss:';
|
||||
const TOP_QUESTIONS = 10;
|
||||
|
||||
function showHelp() {
|
||||
console.log(`
|
||||
Usage: node scripts/gateguard-report.js [options]
|
||||
|
||||
Summarises GateGuard decision metrics (written when GATEGUARD_METRICS=1).
|
||||
|
||||
Options:
|
||||
--dir <path> Directory holding metrics.jsonl (default: GATEGUARD_STATE_DIR or ~/.gateguard)
|
||||
--json Emit machine-readable JSON
|
||||
--help Show this help text
|
||||
`);
|
||||
}
|
||||
|
||||
function parseArgs(argv) {
|
||||
const options = { dir: null, json: false, help: false };
|
||||
for (let index = 0; index < argv.length; index++) {
|
||||
const arg = argv[index];
|
||||
if (arg === '--json') {
|
||||
options.json = true;
|
||||
} else if (arg === '--help' || arg === '-h') {
|
||||
options.help = true;
|
||||
} else if (arg === '--dir') {
|
||||
const value = argv[index + 1];
|
||||
if (!value || value.startsWith('--')) throw new Error('Missing value for --dir');
|
||||
options.dir = value;
|
||||
index++;
|
||||
} else if (arg.startsWith('--dir=')) {
|
||||
const value = arg.slice('--dir='.length);
|
||||
if (!value) throw new Error('Missing value for --dir');
|
||||
options.dir = value;
|
||||
} else {
|
||||
throw new Error(`Unknown argument: ${arg}`);
|
||||
}
|
||||
}
|
||||
return options;
|
||||
}
|
||||
|
||||
function defaultDir() {
|
||||
return process.env.GATEGUARD_STATE_DIR || path.join(process.env.HOME || process.env.USERPROFILE || '/tmp', '.gateguard');
|
||||
}
|
||||
|
||||
/** Valid events from `metrics.jsonl.1` then `metrics.jsonl` in `dir`, with the count of skipped lines. */
|
||||
function loadMetrics(dir) {
|
||||
const current = path.join(dir, METRICS_FILE_NAME);
|
||||
const files = [`${current}.1`, current].filter(file => {
|
||||
try {
|
||||
return fs.statSync(file).isFile();
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
});
|
||||
const events = [];
|
||||
let skipped = 0;
|
||||
for (const file of files) {
|
||||
for (const line of fs.readFileSync(file, 'utf8').split('\n')) {
|
||||
if (!line.trim()) continue;
|
||||
let parsed = null;
|
||||
try {
|
||||
parsed = JSON.parse(line);
|
||||
} catch (_) {
|
||||
parsed = null;
|
||||
}
|
||||
if (isMetricsEvent(parsed)) {
|
||||
events.push(parsed);
|
||||
} else {
|
||||
skipped++;
|
||||
}
|
||||
}
|
||||
}
|
||||
return { dir, files, events, skipped };
|
||||
}
|
||||
|
||||
function increment(counts, key) {
|
||||
counts[key] = (counts[key] || 0) + 1;
|
||||
}
|
||||
|
||||
function byCount(counts) {
|
||||
return Object.fromEntries(Object.entries(counts).sort(([a, x], [b, y]) => y - x || (a < b ? -1 : a > b ? 1 : 0)));
|
||||
}
|
||||
|
||||
function ratio(count, total) {
|
||||
return total > 0 ? count / total : null;
|
||||
}
|
||||
|
||||
function summarizeGroup(events) {
|
||||
const decisions = {};
|
||||
const denialClasses = {};
|
||||
const denialReasons = {};
|
||||
const questions = {};
|
||||
const firstTouchCounts = Object.fromEntries(FIRST_TOUCH_DECISIONS.map(decision => [decision, 0]));
|
||||
let denials = 0;
|
||||
let nearMiss = 0;
|
||||
for (const event of events) {
|
||||
increment(decisions, event.decision);
|
||||
if (Object.hasOwn(firstTouchCounts, event.decision)) firstTouchCounts[event.decision]++;
|
||||
if (DENIAL_DECISIONS.has(event.decision)) {
|
||||
denials++;
|
||||
increment(denialClasses, event.class || 'shell');
|
||||
increment(denialReasons, event.reason || 'unknown');
|
||||
}
|
||||
if (event.decision === 'deny') {
|
||||
if (typeof event.reason === 'string' && event.reason.startsWith(NEAR_MISS_PREFIX)) nearMiss++;
|
||||
for (const id of event.questions || []) increment(questions, id);
|
||||
}
|
||||
}
|
||||
const firstTouchTotal = FIRST_TOUCH_DECISIONS.reduce((sum, decision) => sum + firstTouchCounts[decision], 0);
|
||||
return {
|
||||
events: events.length,
|
||||
first: events.length > 0 ? events[0].ts : null,
|
||||
last: events.length > 0 ? events[events.length - 1].ts : null,
|
||||
decisions: byCount(decisions),
|
||||
denials: { total: denials, byClass: byCount(denialClasses), byReason: byCount(denialReasons) },
|
||||
nearMiss: { count: nearMiss, of: firstTouchCounts.deny, share: ratio(nearMiss, firstTouchCounts.deny) },
|
||||
firstTouch: {
|
||||
total: firstTouchTotal,
|
||||
counts: firstTouchCounts,
|
||||
rates: Object.fromEntries(FIRST_TOUCH_DECISIONS.map(decision => [decision, ratio(firstTouchCounts[decision], firstTouchTotal)]))
|
||||
},
|
||||
routineReadonlyPasses: decisions['routine-readonly'] || 0,
|
||||
topQuestions: Object.entries(byCount(questions)).slice(0, TOP_QUESTIONS)
|
||||
};
|
||||
}
|
||||
|
||||
/** Totals and per-session summaries of metrics events. */
|
||||
function summarize(events) {
|
||||
const bySession = new Map();
|
||||
for (const event of events) {
|
||||
if (!bySession.has(event.session)) bySession.set(event.session, []);
|
||||
bySession.get(event.session).push(event);
|
||||
}
|
||||
const sessions = {};
|
||||
for (const [session, sessionEvents] of bySession) sessions[session] = summarizeGroup(sessionEvents);
|
||||
return { total: summarizeGroup(events), sessions };
|
||||
}
|
||||
|
||||
function percent(value) {
|
||||
return value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`;
|
||||
}
|
||||
|
||||
function counts(entries) {
|
||||
const list = Array.isArray(entries) ? entries : Object.entries(entries);
|
||||
return list.length > 0 ? list.map(([key, count]) => `${key} ${count}`).join(', ') : 'none';
|
||||
}
|
||||
|
||||
function formatGroup(title, group) {
|
||||
const rates = FIRST_TOUCH_DECISIONS.map(decision => `${decision} ${percent(group.firstTouch.rates[decision])}`).join(', ');
|
||||
return [
|
||||
title,
|
||||
` decisions: ${counts(group.decisions)}`,
|
||||
` denials: ${group.denials.total}`,
|
||||
` denials by class: ${counts(group.denials.byClass)}`,
|
||||
` denials by reason: ${counts(group.denials.byReason)}`,
|
||||
` near-miss share of first-touch denials: ${percent(group.nearMiss.share)} (${group.nearMiss.count} of ${group.nearMiss.of})`,
|
||||
` first-touch outcomes (${group.firstTouch.total}): ${rates}`,
|
||||
` routine read-only passes: ${group.routineReadonlyPasses}`,
|
||||
` most-asked questions: ${counts(group.topQuestions)}`
|
||||
].join('\n');
|
||||
}
|
||||
|
||||
/** Plain-text report for `loadMetrics` output plus its `summarize` report. */
|
||||
function formatReport({ dir, events, skipped, report }) {
|
||||
const sessionIds = Object.keys(report.sessions);
|
||||
const header =
|
||||
`GateGuard metrics${dir ? ` in ${dir}` : ''}: ${events.length} events in ${sessionIds.length} sessions` +
|
||||
(skipped > 0 ? ` (${skipped} malformed lines skipped)` : '');
|
||||
const sections = [header, '', formatGroup('Total', report.total)];
|
||||
for (const session of sessionIds) {
|
||||
const group = report.sessions[session];
|
||||
sections.push('', formatGroup(`Session ${session} (${group.first} to ${group.last})`, group));
|
||||
}
|
||||
return `${sections.join('\n')}\n`;
|
||||
}
|
||||
|
||||
function main() {
|
||||
try {
|
||||
const options = parseArgs(process.argv.slice(2));
|
||||
if (options.help) {
|
||||
showHelp();
|
||||
return;
|
||||
}
|
||||
const loaded = loadMetrics(options.dir || defaultDir());
|
||||
const report = summarize(loaded.events);
|
||||
if (options.json) {
|
||||
const output = { dir: loaded.dir, files: loaded.files, events: loaded.events.length, skipped: loaded.skipped, ...report };
|
||||
process.stdout.write(`${JSON.stringify(output, null, 2)}\n`);
|
||||
} else if (loaded.events.length === 0) {
|
||||
process.stdout.write(
|
||||
`No GateGuard metrics in ${loaded.dir}` +
|
||||
(loaded.skipped > 0 ? ` (${loaded.skipped} malformed lines skipped)` : '') +
|
||||
'. Set GATEGUARD_METRICS=1 to record gate decisions.\n'
|
||||
);
|
||||
} else {
|
||||
process.stdout.write(formatReport({ ...loaded, report }));
|
||||
}
|
||||
} catch (error) {
|
||||
process.stderr.write(`Error: ${error.message}\n`);
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
if (require.main === module) {
|
||||
main();
|
||||
}
|
||||
|
||||
module.exports = { formatReport, loadMetrics, parseArgs, summarize };
|
||||
@@ -0,0 +1,120 @@
|
||||
'use strict';
|
||||
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
|
||||
const METRICS_SCHEMA_VERSION = 1;
|
||||
const METRICS_FILE_NAME = 'metrics.jsonl';
|
||||
const METRICS_MAX_BYTES = 1024 * 1024;
|
||||
const SESSION_DIGEST_LENGTH = 12;
|
||||
const CODE_MAX_LENGTH = 48;
|
||||
const CODE_PATTERN = /^[a-z][a-z0-9-]*(?::[a-z][a-z0-9-]*)?$/;
|
||||
const SESSION_PATTERN = /^[0-9a-f]{12}$/;
|
||||
|
||||
const DECISIONS = Object.freeze([
|
||||
'deny',
|
||||
'credit',
|
||||
'sibling',
|
||||
'cap',
|
||||
'trivial',
|
||||
'pass-checked',
|
||||
'pass-exempt',
|
||||
'pass-subagent',
|
||||
'routine-deny',
|
||||
'routine-readonly',
|
||||
'destructive-deny',
|
||||
'pass'
|
||||
]);
|
||||
const DECISION_SET = new Set(DECISIONS);
|
||||
const TOOLS = new Set(['Edit', 'Write', 'MultiEdit', 'Bash', 'PowerShell']);
|
||||
|
||||
function isCode(value) {
|
||||
return typeof value === 'string' && value.length <= CODE_MAX_LENGTH && CODE_PATTERN.test(value);
|
||||
}
|
||||
|
||||
function codeOrNull(value) {
|
||||
return isCode(value) ? value : null;
|
||||
}
|
||||
|
||||
function sessionDigest(sessionKey) {
|
||||
return crypto.createHash('sha256').update(String(sessionKey || '')).digest('hex').slice(0, SESSION_DIGEST_LENGTH);
|
||||
}
|
||||
|
||||
function profileFields(profile) {
|
||||
if (!profile || typeof profile !== 'object') return null;
|
||||
return {
|
||||
known: profile.known === true,
|
||||
language: codeOrNull(profile.language),
|
||||
touchesPublicSurface: profile.touchesPublicSurface === true,
|
||||
touchesData: profile.touchesData === true,
|
||||
trivial: profile.trivial === true
|
||||
};
|
||||
}
|
||||
|
||||
/** One metrics line for a gate decision, or null when the decision or tool is not recorded. */
|
||||
function metricsEvent({ sessionKey, tool, cls, decision, reason, questions, sensitive, profile, now = Date.now() }) {
|
||||
if (!DECISION_SET.has(decision) || !TOOLS.has(tool)) return null;
|
||||
return {
|
||||
v: METRICS_SCHEMA_VERSION,
|
||||
ts: new Date(now).toISOString(),
|
||||
session: sessionDigest(sessionKey),
|
||||
tool,
|
||||
class: codeOrNull(cls),
|
||||
decision,
|
||||
reason: codeOrNull(reason),
|
||||
questions: Array.isArray(questions) ? questions.filter(isCode) : null,
|
||||
sensitive: sensitive === true,
|
||||
profile: profileFields(profile)
|
||||
};
|
||||
}
|
||||
|
||||
/** Whether a parsed line has the shape `metricsEvent` writes. */
|
||||
function isMetricsEvent(value) {
|
||||
if (!value || typeof value !== 'object' || Array.isArray(value)) return false;
|
||||
if (value.v !== METRICS_SCHEMA_VERSION || typeof value.ts !== 'string') return false;
|
||||
if (typeof value.session !== 'string' || !SESSION_PATTERN.test(value.session)) return false;
|
||||
if (!TOOLS.has(value.tool) || !DECISION_SET.has(value.decision)) return false;
|
||||
if (value.class !== null && !isCode(value.class)) return false;
|
||||
if (value.reason !== null && !isCode(value.reason)) return false;
|
||||
if (value.questions !== null && !(Array.isArray(value.questions) && value.questions.every(isCode))) return false;
|
||||
return typeof value.sensitive === 'boolean';
|
||||
}
|
||||
|
||||
/** Append events to `<dir>/metrics.jsonl` in one write, rotating to `.1` past the size cap; never throws. */
|
||||
function appendMetrics(dir, events) {
|
||||
try {
|
||||
const lines = events
|
||||
.filter(Boolean)
|
||||
.map(event => `${JSON.stringify(event)}\n`)
|
||||
.join('');
|
||||
if (!lines) return false;
|
||||
const file = path.join(dir, METRICS_FILE_NAME);
|
||||
fs.mkdirSync(dir, { recursive: true });
|
||||
let size = 0;
|
||||
try {
|
||||
const stat = fs.statSync(file);
|
||||
size = stat.isFile() ? stat.size : 0;
|
||||
} catch (_) {
|
||||
size = 0;
|
||||
}
|
||||
if (size > 0 && size + Buffer.byteLength(lines) > METRICS_MAX_BYTES) {
|
||||
fs.renameSync(file, `${file}.1`);
|
||||
}
|
||||
fs.appendFileSync(file, lines, { encoding: 'utf8', mode: 0o600 });
|
||||
return true;
|
||||
} catch (_) {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
module.exports = {
|
||||
DECISIONS,
|
||||
METRICS_FILE_NAME,
|
||||
METRICS_MAX_BYTES,
|
||||
METRICS_SCHEMA_VERSION,
|
||||
appendMetrics,
|
||||
isMetricsEvent,
|
||||
metricsEvent,
|
||||
sessionDigest
|
||||
};
|
||||
+17
@@ -0,0 +1,17 @@
|
||||
{"v":1,"ts":"2026-09-01T10:02:00.000Z","session":"aaaaaaaaaaaa","tool":"Edit","class":"code","decision":"deny","reason":"near-miss:out-of-scope","questions":["local-callers","quote-instruction"],"sensitive":false,"profile":{"known":true,"language":"js","touchesPublicSurface":false,"touchesData":false,"trivial":false}}
|
||||
not json
|
||||
{"v":1,"ts":"2026-09-01T10:03:00.000Z","session":"aaaaaaaaaaaa","tool":"Edit","class":"code","decision":"credit","reason":"prior-search","questions":null,"sensitive":false,"profile":null}
|
||||
{"v":2,"ts":"2026-09-01T10:03:30.000Z","session":"aaaaaaaaaaaa","tool":"Edit","class":"code","decision":"deny","reason":"first-touch","questions":[],"sensitive":false,"profile":null}
|
||||
{"v":1,"ts":"2026-09-01T10:04:00.000Z","session":"aaaaaaaaaaaa","tool":"Edit","class":"code","decision":"trivial","reason":"comment-whitespace","questions":null,"sensitive":false,"profile":{"known":true,"language":"js","touchesPublicSurface":false,"touchesData":false,"trivial":true}}
|
||||
{"v":1,"ts":"2026-09-01T10:05:00.000Z","session":"aaaaaaaaaaaa","tool":"Bash","class":null,"decision":"routine-deny","reason":"first-command","questions":null,"sensitive":false,"profile":null}
|
||||
{"v":1,"ts":"2026-09-01T11:00:00.000Z","session":"NOT-A-HASH","tool":"Edit","class":"code","decision":"deny","reason":"first-touch","questions":[],"sensitive":false,"profile":null}
|
||||
|
||||
{"v":1,"ts":"2026-09-01T11:00:30.000Z","session":"bbbbbbbbbbbb","tool":"Edit","class":"code","decision":"allow","reason":"first-touch","questions":null,"sensitive":false,"profile":null}
|
||||
[]
|
||||
{"v":1,"ts":"2026-09-01T11:00:40.000Z","session":"bbbbbbbbbbbb","tool":"Edit","class":"code","decision":"deny","reason":"/home/user/secret.js","questions":null,"sensitive":false,"profile":null}
|
||||
{"v":1,"ts":"2026-09-01T11:01:00.000Z","session":"bbbbbbbbbbbb","tool":"Write","class":"code","decision":"deny","reason":"first-touch","questions":["callers","no-duplicate","quote-instruction"],"sensitive":false,"profile":{"known":true,"language":"py","touchesPublicSurface":true,"touchesData":false,"trivial":false}}
|
||||
{"v":1,"ts":"2026-09-01T11:02:00.000Z","session":"bbbbbbbbbbbb","tool":"Write","class":"code","decision":"sibling","reason":"same-turn-dir","questions":null,"sensitive":false,"profile":null}
|
||||
{"v":1,"ts":"2026-09-01T11:03:00.000Z","session":"bbbbbbbbbbbb","tool":"Edit","class":"config","decision":"deny","reason":"sensitive","questions":["config-reader","config-effect","no-plaintext-secrets","quote-instruction"],"sensitive":true,"profile":null}
|
||||
{"v":1,"ts":"2026-09-01T11:04:00.000Z","session":"bbbbbbbbbbbb","tool":"Edit","class":"code","decision":"cap","reason":"max-denials","questions":null,"sensitive":false,"profile":null}
|
||||
{"v":1,"ts":"2026-09-01T11:05:00.000Z","session":"bbbbbbbbbbbb","tool":"PowerShell","class":null,"decision":"destructive-deny","reason":"destructive","questions":null,"sensitive":false,"profile":null}
|
||||
{"v":1,"ts":"2026-09-01T11:06:00.000Z","session":"bbbbbbbbbbbb","tool":"Edit","class":"code","decision":"pass-checked","reason":"checked","questions":null,"sensitive":false,"profile":null}
|
||||
@@ -0,0 +1,2 @@
|
||||
{"v":1,"ts":"2026-09-01T10:00:00.000Z","session":"aaaaaaaaaaaa","tool":"Edit","class":"code","decision":"deny","reason":"first-touch","questions":["importers","public-api","quote-instruction"],"sensitive":false,"profile":{"known":true,"language":"js","touchesPublicSurface":true,"touchesData":false,"trivial":false}}
|
||||
{"v":1,"ts":"2026-09-01T10:01:00.000Z","session":"aaaaaaaaaaaa","tool":"Bash","class":null,"decision":"routine-readonly","reason":"readonly","questions":null,"sensitive":false,"profile":null}
|
||||
@@ -0,0 +1,153 @@
|
||||
/**
|
||||
* Tests for scripts/lib/gateguard-metrics.js
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const crypto = require('crypto');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
|
||||
const {
|
||||
DECISIONS,
|
||||
METRICS_FILE_NAME,
|
||||
METRICS_MAX_BYTES,
|
||||
metricsEvent,
|
||||
appendMetrics,
|
||||
isMetricsEvent
|
||||
} = require('../../scripts/lib/gateguard-metrics');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function test(name, fn) {
|
||||
try {
|
||||
fn();
|
||||
console.log(` \u2713 ${name}`);
|
||||
passed++;
|
||||
} catch (error) {
|
||||
console.log(` \u2717 ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
const NOW = Date.parse('2026-09-01T10:00:00.000Z');
|
||||
const base = { sessionKey: 'sess-1', tool: 'Edit', decision: 'deny', reason: 'first-touch', now: NOW };
|
||||
|
||||
console.log('\n=== Testing gateguard-metrics ===\n');
|
||||
|
||||
test('builds a schema 1 event with a 12-character session digest', () => {
|
||||
const event = metricsEvent({
|
||||
...base,
|
||||
cls: 'code',
|
||||
questions: ['importers', 'quote-instruction'],
|
||||
sensitive: false,
|
||||
profile: { known: true, language: 'js', touchesPublicSurface: true, touchesData: false, trivial: false, extra: 'x' }
|
||||
});
|
||||
assert.deepStrictEqual(event, {
|
||||
v: 1,
|
||||
ts: '2026-09-01T10:00:00.000Z',
|
||||
session: crypto.createHash('sha256').update('sess-1').digest('hex').slice(0, 12),
|
||||
tool: 'Edit',
|
||||
class: 'code',
|
||||
decision: 'deny',
|
||||
reason: 'first-touch',
|
||||
questions: ['importers', 'quote-instruction'],
|
||||
sensitive: false,
|
||||
profile: { known: true, language: 'js', touchesPublicSurface: true, touchesData: false, trivial: false }
|
||||
});
|
||||
assert.ok(isMetricsEvent(event));
|
||||
});
|
||||
|
||||
test('returns null for unknown decisions or tools', () => {
|
||||
assert.strictEqual(metricsEvent({ ...base, decision: 'allow' }), null);
|
||||
assert.strictEqual(metricsEvent({ ...base, tool: 'Read' }), null);
|
||||
});
|
||||
|
||||
test('drops any class, reason, question or language that is not a short code', () => {
|
||||
const event = metricsEvent({
|
||||
...base,
|
||||
cls: '/src/app.js',
|
||||
reason: 'rm -rf /',
|
||||
questions: ['importers', '/etc/passwd', 'x'.repeat(100)],
|
||||
profile: { known: true, language: 'C:\\x' }
|
||||
});
|
||||
assert.strictEqual(event.class, null);
|
||||
assert.strictEqual(event.reason, null);
|
||||
assert.deepStrictEqual(event.questions, ['importers']);
|
||||
assert.strictEqual(event.profile.language, null);
|
||||
});
|
||||
|
||||
test('keeps near-miss reason codes', () => {
|
||||
assert.strictEqual(metricsEvent({ ...base, reason: 'near-miss:out-of-scope' }).reason, 'near-miss:out-of-scope');
|
||||
});
|
||||
|
||||
test('lists every decision the hook records', () => {
|
||||
assert.deepStrictEqual([...DECISIONS].sort(), [
|
||||
'cap',
|
||||
'credit',
|
||||
'deny',
|
||||
'destructive-deny',
|
||||
'pass',
|
||||
'pass-checked',
|
||||
'pass-exempt',
|
||||
'pass-subagent',
|
||||
'routine-deny',
|
||||
'routine-readonly',
|
||||
'sibling',
|
||||
'trivial'
|
||||
]);
|
||||
});
|
||||
|
||||
test('rejects events with the wrong shape', () => {
|
||||
const good = metricsEvent(base);
|
||||
assert.ok(isMetricsEvent(good));
|
||||
for (const bad of [
|
||||
null,
|
||||
[],
|
||||
{ ...good, v: 2 },
|
||||
{ ...good, session: 'ABC' },
|
||||
{ ...good, decision: 'allow' },
|
||||
{ ...good, tool: 'Read' },
|
||||
{ ...good, reason: '/x' },
|
||||
{ ...good, class: 'a b' },
|
||||
{ ...good, questions: ['ok', '/x'] },
|
||||
{ ...good, sensitive: 'no' },
|
||||
{ ...good, ts: 5 }
|
||||
]) {
|
||||
assert.strictEqual(isMetricsEvent(bad), false, JSON.stringify(bad));
|
||||
}
|
||||
});
|
||||
|
||||
test('appends one line per event in a single write and rotates past the cap', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-metrics-lib-'));
|
||||
try {
|
||||
const file = path.join(dir, 'nested', METRICS_FILE_NAME);
|
||||
assert.strictEqual(appendMetrics(path.dirname(file), [metricsEvent(base), null, metricsEvent({ ...base, decision: 'credit' })]), true);
|
||||
assert.strictEqual(fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).length, 2);
|
||||
fs.writeFileSync(`${file}.1`, 'old\n');
|
||||
fs.writeFileSync(file, 'x'.repeat(METRICS_MAX_BYTES));
|
||||
assert.strictEqual(appendMetrics(path.dirname(file), [metricsEvent(base)]), true);
|
||||
assert.strictEqual(fs.statSync(`${file}.1`).size, METRICS_MAX_BYTES);
|
||||
assert.strictEqual(fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).length, 1);
|
||||
assert.strictEqual(appendMetrics(path.dirname(file), [null]), false);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('reports failure instead of throwing when the file cannot be written', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-metrics-lib-'));
|
||||
try {
|
||||
fs.mkdirSync(path.join(dir, METRICS_FILE_NAME));
|
||||
assert.strictEqual(appendMetrics(dir, [metricsEvent(base)]), false);
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
console.log(`\n ${passed} passed, ${failed} failed\n`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
@@ -0,0 +1,182 @@
|
||||
/**
|
||||
* Tests for scripts/gateguard-report.js
|
||||
*/
|
||||
|
||||
'use strict';
|
||||
|
||||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
const { spawnSync } = require('child_process');
|
||||
|
||||
const SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'gateguard-report.js');
|
||||
const FIXTURE_DIR = path.join(__dirname, '..', 'fixtures', 'gateguard-metrics');
|
||||
const { loadMetrics, summarize, formatReport, parseArgs } = require('../../scripts/gateguard-report');
|
||||
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
function test(name, fn) {
|
||||
try {
|
||||
fn();
|
||||
console.log(` ✓ ${name}`);
|
||||
passed++;
|
||||
} catch (error) {
|
||||
console.log(` ✗ ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
failed++;
|
||||
}
|
||||
}
|
||||
|
||||
function run(args, env = {}) {
|
||||
const result = spawnSync('node', [SCRIPT, ...args], {
|
||||
encoding: 'utf8',
|
||||
env: { ...process.env, ...env },
|
||||
timeout: 10000
|
||||
});
|
||||
return { status: result.status, stdout: result.stdout || '', stderr: result.stderr || '' };
|
||||
}
|
||||
|
||||
console.log('\n=== Testing gateguard-report ===\n');
|
||||
|
||||
const loaded = loadMetrics(FIXTURE_DIR);
|
||||
const report = summarize(loaded.events);
|
||||
|
||||
test('reads the rotated file first, then the current file', () => {
|
||||
assert.strictEqual(loaded.events.length, 12);
|
||||
assert.strictEqual(loaded.events[0].ts, '2026-09-01T10:00:00.000Z');
|
||||
assert.strictEqual(loaded.events[11].decision, 'pass-checked');
|
||||
assert.deepStrictEqual(loaded.files.map(f => path.basename(f)), ['metrics.jsonl.1', 'metrics.jsonl']);
|
||||
});
|
||||
|
||||
test('skips malformed, unknown-version, unknown-decision and non-code lines', () => {
|
||||
assert.strictEqual(loaded.skipped, 6);
|
||||
assert.ok(loaded.events.every(e => e.session === 'aaaaaaaaaaaa' || e.session === 'bbbbbbbbbbbb'));
|
||||
assert.ok(!loaded.events.some(e => String(e.reason).includes('/')));
|
||||
});
|
||||
|
||||
test('counts decisions by type', () => {
|
||||
assert.deepStrictEqual(report.total.decisions, {
|
||||
deny: 4,
|
||||
cap: 1,
|
||||
credit: 1,
|
||||
'destructive-deny': 1,
|
||||
'pass-checked': 1,
|
||||
'routine-deny': 1,
|
||||
'routine-readonly': 1,
|
||||
sibling: 1,
|
||||
trivial: 1
|
||||
});
|
||||
assert.strictEqual(report.total.events, 12);
|
||||
});
|
||||
|
||||
test('breaks denials down by class and reason', () => {
|
||||
assert.strictEqual(report.total.denials.total, 6);
|
||||
assert.deepStrictEqual(report.total.denials.byClass, { code: 3, shell: 2, config: 1 });
|
||||
assert.deepStrictEqual(report.total.denials.byReason, {
|
||||
'first-touch': 2,
|
||||
destructive: 1,
|
||||
'first-command': 1,
|
||||
'near-miss:out-of-scope': 1,
|
||||
sensitive: 1
|
||||
});
|
||||
});
|
||||
|
||||
test('reports the near-miss share of first-touch denials', () => {
|
||||
assert.deepStrictEqual(report.total.nearMiss, { count: 1, of: 4, share: 0.25 });
|
||||
});
|
||||
|
||||
test('reports first-touch outcome rates', () => {
|
||||
const { firstTouch } = report.total;
|
||||
assert.strictEqual(firstTouch.total, 8);
|
||||
assert.deepStrictEqual(firstTouch.counts, { deny: 4, credit: 1, sibling: 1, cap: 1, trivial: 1 });
|
||||
assert.strictEqual(firstTouch.rates.deny, 0.5);
|
||||
assert.strictEqual(firstTouch.rates.trivial, 0.125);
|
||||
});
|
||||
|
||||
test('counts routine read-only passes and ranks the most-asked questions', () => {
|
||||
assert.strictEqual(report.total.routineReadonlyPasses, 1);
|
||||
assert.deepStrictEqual(report.total.topQuestions.slice(0, 3), [
|
||||
['quote-instruction', 4],
|
||||
['callers', 1],
|
||||
['config-effect', 1]
|
||||
]);
|
||||
});
|
||||
|
||||
test('summarizes each session separately in order of first appearance', () => {
|
||||
assert.deepStrictEqual(Object.keys(report.sessions), ['aaaaaaaaaaaa', 'bbbbbbbbbbbb']);
|
||||
const a = report.sessions.aaaaaaaaaaaa;
|
||||
assert.strictEqual(a.events, 6);
|
||||
assert.strictEqual(a.denials.total, 3);
|
||||
assert.deepStrictEqual(a.nearMiss, { count: 1, of: 2, share: 0.5 });
|
||||
assert.strictEqual(a.firstTouch.total, 4);
|
||||
assert.strictEqual(a.first, '2026-09-01T10:00:00.000Z');
|
||||
assert.strictEqual(a.last, '2026-09-01T10:05:00.000Z');
|
||||
const b = report.sessions.bbbbbbbbbbbb;
|
||||
assert.strictEqual(b.events, 6);
|
||||
assert.deepStrictEqual(b.denials.byClass, { code: 1, config: 1, shell: 1 });
|
||||
assert.strictEqual(b.routineReadonlyPasses, 0);
|
||||
});
|
||||
|
||||
test('reports empty rates as null instead of dividing by zero', () => {
|
||||
const empty = summarize([]);
|
||||
assert.strictEqual(empty.total.events, 0);
|
||||
assert.strictEqual(empty.total.nearMiss.share, null);
|
||||
assert.strictEqual(empty.total.firstTouch.rates.deny, null);
|
||||
});
|
||||
|
||||
test('formats a readable text report', () => {
|
||||
const text = formatReport({ ...loaded, report });
|
||||
assert.ok(text.includes('12 events in 2 sessions (6 malformed lines skipped)'));
|
||||
assert.ok(text.includes('Total'));
|
||||
assert.ok(text.includes('Session aaaaaaaaaaaa'));
|
||||
assert.ok(text.includes('near-miss share of first-touch denials: 25.0% (1 of 4)'));
|
||||
assert.ok(text.includes('routine read-only passes: 1'));
|
||||
assert.ok(text.includes('quote-instruction 4'));
|
||||
});
|
||||
|
||||
test('parses --dir, --dir=, --json and --help', () => {
|
||||
assert.deepStrictEqual(parseArgs(['--dir', '/x', '--json']), { dir: '/x', json: true, help: false });
|
||||
assert.deepStrictEqual(parseArgs(['--dir=/y']), { dir: '/y', json: false, help: false });
|
||||
assert.strictEqual(parseArgs(['-h']).help, true);
|
||||
assert.throws(() => parseArgs(['--dir']), /Missing value for --dir/);
|
||||
assert.throws(() => parseArgs(['--bogus']), /Unknown argument: --bogus/);
|
||||
});
|
||||
|
||||
test('CLI prints JSON with --json', () => {
|
||||
const result = run(['--dir', FIXTURE_DIR, '--json']);
|
||||
assert.strictEqual(result.status, 0, result.stderr);
|
||||
const parsed = JSON.parse(result.stdout);
|
||||
assert.strictEqual(parsed.events, 12);
|
||||
assert.strictEqual(parsed.skipped, 6);
|
||||
assert.strictEqual(parsed.total.denials.total, 6);
|
||||
assert.deepStrictEqual(Object.keys(parsed.sessions), ['aaaaaaaaaaaa', 'bbbbbbbbbbbb']);
|
||||
});
|
||||
|
||||
test('CLI reads GATEGUARD_STATE_DIR when --dir is not given', () => {
|
||||
const result = run([], { GATEGUARD_STATE_DIR: FIXTURE_DIR });
|
||||
assert.strictEqual(result.status, 0, result.stderr);
|
||||
assert.ok(result.stdout.includes('12 events in 2 sessions'));
|
||||
});
|
||||
|
||||
test('CLI explains how to enable metrics when none are found', () => {
|
||||
const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-report-'));
|
||||
try {
|
||||
const result = run(['--dir', dir]);
|
||||
assert.strictEqual(result.status, 0, result.stderr);
|
||||
assert.ok(result.stdout.includes('No GateGuard metrics'));
|
||||
assert.ok(result.stdout.includes('GATEGUARD_METRICS=1'));
|
||||
} finally {
|
||||
fs.rmSync(dir, { recursive: true, force: true });
|
||||
}
|
||||
});
|
||||
|
||||
test('CLI exits 1 on an unknown argument', () => {
|
||||
const result = run(['--bogus']);
|
||||
assert.strictEqual(result.status, 1);
|
||||
assert.ok(result.stderr.includes('Unknown argument: --bogus'));
|
||||
});
|
||||
|
||||
console.log(`\n ${passed} passed, ${failed} failed\n`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
@@ -83,6 +83,7 @@ function buildExpectedPublishPaths(repoRoot) {
|
||||
"scripts/uninstall.js",
|
||||
"scripts/welcome.js",
|
||||
"scripts/gemini-adapt-agents.js",
|
||||
"scripts/gateguard-report.js",
|
||||
"scripts/sync-ecc-to-codex.sh",
|
||||
"scripts/codex/legacy-sync-state.js",
|
||||
"scripts/codex/install-global-git-hooks.sh",
|
||||
|
||||
Reference in New Issue
Block a user