diff --git a/package.json b/package.json index 57357af0e..59c1feda0 100644 --- a/package.json +++ b/package.json @@ -124,6 +124,7 @@ "scripts/memory.js", "scripts/memory-mcp.mjs", "scripts/gemini-adapt-agents.js", + "scripts/gateguard-report.js", "scripts/harness-adapter-compliance.js", "scripts/harness-audit.js", "scripts/eval-harness.js", diff --git a/scripts/gateguard-report.js b/scripts/gateguard-report.js new file mode 100644 index 000000000..10c3bafcb --- /dev/null +++ b/scripts/gateguard-report.js @@ -0,0 +1,217 @@ +#!/usr/bin/env node +'use strict'; + +const fs = require('fs'); +const path = require('path'); +const { METRICS_FILE_NAME, isMetricsEvent } = require('./lib/gateguard-metrics'); + +const DENIAL_DECISIONS = new Set(['deny', 'routine-deny', 'destructive-deny']); +const FIRST_TOUCH_DECISIONS = ['deny', 'credit', 'sibling', 'cap', 'trivial']; +const NEAR_MISS_PREFIX = 'near-miss:'; +const TOP_QUESTIONS = 10; + +function showHelp() { + console.log(` +Usage: node scripts/gateguard-report.js [options] + +Summarises GateGuard decision metrics (written when GATEGUARD_METRICS=1). + +Options: + --dir Directory holding metrics.jsonl (default: GATEGUARD_STATE_DIR or ~/.gateguard) + --json Emit machine-readable JSON + --help Show this help text +`); +} + +function parseArgs(argv) { + const options = { dir: null, json: false, help: false }; + for (let index = 0; index < argv.length; index++) { + const arg = argv[index]; + if (arg === '--json') { + options.json = true; + } else if (arg === '--help' || arg === '-h') { + options.help = true; + } else if (arg === '--dir') { + const value = argv[index + 1]; + if (!value || value.startsWith('--')) throw new Error('Missing value for --dir'); + options.dir = value; + index++; + } else if (arg.startsWith('--dir=')) { + const value = arg.slice('--dir='.length); + if (!value) throw new Error('Missing value for --dir'); + options.dir = value; + } else { + throw new Error(`Unknown argument: ${arg}`); + } + } + return options; +} + +function defaultDir() { + return process.env.GATEGUARD_STATE_DIR || path.join(process.env.HOME || process.env.USERPROFILE || '/tmp', '.gateguard'); +} + +/** Valid events from `metrics.jsonl.1` then `metrics.jsonl` in `dir`, with the count of skipped lines. */ +function loadMetrics(dir) { + const current = path.join(dir, METRICS_FILE_NAME); + const files = [`${current}.1`, current].filter(file => { + try { + return fs.statSync(file).isFile(); + } catch (_) { + return false; + } + }); + const events = []; + let skipped = 0; + for (const file of files) { + for (const line of fs.readFileSync(file, 'utf8').split('\n')) { + if (!line.trim()) continue; + let parsed = null; + try { + parsed = JSON.parse(line); + } catch (_) { + parsed = null; + } + if (isMetricsEvent(parsed)) { + events.push(parsed); + } else { + skipped++; + } + } + } + return { dir, files, events, skipped }; +} + +function increment(counts, key) { + counts[key] = (counts[key] || 0) + 1; +} + +function byCount(counts) { + return Object.fromEntries(Object.entries(counts).sort(([a, x], [b, y]) => y - x || (a < b ? -1 : a > b ? 1 : 0))); +} + +function ratio(count, total) { + return total > 0 ? count / total : null; +} + +function summarizeGroup(events) { + const decisions = {}; + const denialClasses = {}; + const denialReasons = {}; + const questions = {}; + const firstTouchCounts = Object.fromEntries(FIRST_TOUCH_DECISIONS.map(decision => [decision, 0])); + let denials = 0; + let nearMiss = 0; + for (const event of events) { + increment(decisions, event.decision); + if (Object.hasOwn(firstTouchCounts, event.decision)) firstTouchCounts[event.decision]++; + if (DENIAL_DECISIONS.has(event.decision)) { + denials++; + increment(denialClasses, event.class || 'shell'); + increment(denialReasons, event.reason || 'unknown'); + } + if (event.decision === 'deny') { + if (typeof event.reason === 'string' && event.reason.startsWith(NEAR_MISS_PREFIX)) nearMiss++; + for (const id of event.questions || []) increment(questions, id); + } + } + const firstTouchTotal = FIRST_TOUCH_DECISIONS.reduce((sum, decision) => sum + firstTouchCounts[decision], 0); + return { + events: events.length, + first: events.length > 0 ? events[0].ts : null, + last: events.length > 0 ? events[events.length - 1].ts : null, + decisions: byCount(decisions), + denials: { total: denials, byClass: byCount(denialClasses), byReason: byCount(denialReasons) }, + nearMiss: { count: nearMiss, of: firstTouchCounts.deny, share: ratio(nearMiss, firstTouchCounts.deny) }, + firstTouch: { + total: firstTouchTotal, + counts: firstTouchCounts, + rates: Object.fromEntries(FIRST_TOUCH_DECISIONS.map(decision => [decision, ratio(firstTouchCounts[decision], firstTouchTotal)])) + }, + routineReadonlyPasses: decisions['routine-readonly'] || 0, + topQuestions: Object.entries(byCount(questions)).slice(0, TOP_QUESTIONS) + }; +} + +/** Totals and per-session summaries of metrics events. */ +function summarize(events) { + const bySession = new Map(); + for (const event of events) { + if (!bySession.has(event.session)) bySession.set(event.session, []); + bySession.get(event.session).push(event); + } + const sessions = {}; + for (const [session, sessionEvents] of bySession) sessions[session] = summarizeGroup(sessionEvents); + return { total: summarizeGroup(events), sessions }; +} + +function percent(value) { + return value === null ? 'n/a' : `${(value * 100).toFixed(1)}%`; +} + +function counts(entries) { + const list = Array.isArray(entries) ? entries : Object.entries(entries); + return list.length > 0 ? list.map(([key, count]) => `${key} ${count}`).join(', ') : 'none'; +} + +function formatGroup(title, group) { + const rates = FIRST_TOUCH_DECISIONS.map(decision => `${decision} ${percent(group.firstTouch.rates[decision])}`).join(', '); + return [ + title, + ` decisions: ${counts(group.decisions)}`, + ` denials: ${group.denials.total}`, + ` denials by class: ${counts(group.denials.byClass)}`, + ` denials by reason: ${counts(group.denials.byReason)}`, + ` near-miss share of first-touch denials: ${percent(group.nearMiss.share)} (${group.nearMiss.count} of ${group.nearMiss.of})`, + ` first-touch outcomes (${group.firstTouch.total}): ${rates}`, + ` routine read-only passes: ${group.routineReadonlyPasses}`, + ` most-asked questions: ${counts(group.topQuestions)}` + ].join('\n'); +} + +/** Plain-text report for `loadMetrics` output plus its `summarize` report. */ +function formatReport({ dir, events, skipped, report }) { + const sessionIds = Object.keys(report.sessions); + const header = + `GateGuard metrics${dir ? ` in ${dir}` : ''}: ${events.length} events in ${sessionIds.length} sessions` + + (skipped > 0 ? ` (${skipped} malformed lines skipped)` : ''); + const sections = [header, '', formatGroup('Total', report.total)]; + for (const session of sessionIds) { + const group = report.sessions[session]; + sections.push('', formatGroup(`Session ${session} (${group.first} to ${group.last})`, group)); + } + return `${sections.join('\n')}\n`; +} + +function main() { + try { + const options = parseArgs(process.argv.slice(2)); + if (options.help) { + showHelp(); + return; + } + const loaded = loadMetrics(options.dir || defaultDir()); + const report = summarize(loaded.events); + if (options.json) { + const output = { dir: loaded.dir, files: loaded.files, events: loaded.events.length, skipped: loaded.skipped, ...report }; + process.stdout.write(`${JSON.stringify(output, null, 2)}\n`); + } else if (loaded.events.length === 0) { + process.stdout.write( + `No GateGuard metrics in ${loaded.dir}` + + (loaded.skipped > 0 ? ` (${loaded.skipped} malformed lines skipped)` : '') + + '. Set GATEGUARD_METRICS=1 to record gate decisions.\n' + ); + } else { + process.stdout.write(formatReport({ ...loaded, report })); + } + } catch (error) { + process.stderr.write(`Error: ${error.message}\n`); + process.exit(1); + } +} + +if (require.main === module) { + main(); +} + +module.exports = { formatReport, loadMetrics, parseArgs, summarize }; diff --git a/scripts/lib/gateguard-metrics.js b/scripts/lib/gateguard-metrics.js new file mode 100644 index 000000000..c8ac0651f --- /dev/null +++ b/scripts/lib/gateguard-metrics.js @@ -0,0 +1,120 @@ +'use strict'; + +const crypto = require('crypto'); +const fs = require('fs'); +const path = require('path'); + +const METRICS_SCHEMA_VERSION = 1; +const METRICS_FILE_NAME = 'metrics.jsonl'; +const METRICS_MAX_BYTES = 1024 * 1024; +const SESSION_DIGEST_LENGTH = 12; +const CODE_MAX_LENGTH = 48; +const CODE_PATTERN = /^[a-z][a-z0-9-]*(?::[a-z][a-z0-9-]*)?$/; +const SESSION_PATTERN = /^[0-9a-f]{12}$/; + +const DECISIONS = Object.freeze([ + 'deny', + 'credit', + 'sibling', + 'cap', + 'trivial', + 'pass-checked', + 'pass-exempt', + 'pass-subagent', + 'routine-deny', + 'routine-readonly', + 'destructive-deny', + 'pass' +]); +const DECISION_SET = new Set(DECISIONS); +const TOOLS = new Set(['Edit', 'Write', 'MultiEdit', 'Bash', 'PowerShell']); + +function isCode(value) { + return typeof value === 'string' && value.length <= CODE_MAX_LENGTH && CODE_PATTERN.test(value); +} + +function codeOrNull(value) { + return isCode(value) ? value : null; +} + +function sessionDigest(sessionKey) { + return crypto.createHash('sha256').update(String(sessionKey || '')).digest('hex').slice(0, SESSION_DIGEST_LENGTH); +} + +function profileFields(profile) { + if (!profile || typeof profile !== 'object') return null; + return { + known: profile.known === true, + language: codeOrNull(profile.language), + touchesPublicSurface: profile.touchesPublicSurface === true, + touchesData: profile.touchesData === true, + trivial: profile.trivial === true + }; +} + +/** One metrics line for a gate decision, or null when the decision or tool is not recorded. */ +function metricsEvent({ sessionKey, tool, cls, decision, reason, questions, sensitive, profile, now = Date.now() }) { + if (!DECISION_SET.has(decision) || !TOOLS.has(tool)) return null; + return { + v: METRICS_SCHEMA_VERSION, + ts: new Date(now).toISOString(), + session: sessionDigest(sessionKey), + tool, + class: codeOrNull(cls), + decision, + reason: codeOrNull(reason), + questions: Array.isArray(questions) ? questions.filter(isCode) : null, + sensitive: sensitive === true, + profile: profileFields(profile) + }; +} + +/** Whether a parsed line has the shape `metricsEvent` writes. */ +function isMetricsEvent(value) { + if (!value || typeof value !== 'object' || Array.isArray(value)) return false; + if (value.v !== METRICS_SCHEMA_VERSION || typeof value.ts !== 'string') return false; + if (typeof value.session !== 'string' || !SESSION_PATTERN.test(value.session)) return false; + if (!TOOLS.has(value.tool) || !DECISION_SET.has(value.decision)) return false; + if (value.class !== null && !isCode(value.class)) return false; + if (value.reason !== null && !isCode(value.reason)) return false; + if (value.questions !== null && !(Array.isArray(value.questions) && value.questions.every(isCode))) return false; + return typeof value.sensitive === 'boolean'; +} + +/** Append events to `/metrics.jsonl` in one write, rotating to `.1` past the size cap; never throws. */ +function appendMetrics(dir, events) { + try { + const lines = events + .filter(Boolean) + .map(event => `${JSON.stringify(event)}\n`) + .join(''); + if (!lines) return false; + const file = path.join(dir, METRICS_FILE_NAME); + fs.mkdirSync(dir, { recursive: true }); + let size = 0; + try { + const stat = fs.statSync(file); + size = stat.isFile() ? stat.size : 0; + } catch (_) { + size = 0; + } + if (size > 0 && size + Buffer.byteLength(lines) > METRICS_MAX_BYTES) { + fs.renameSync(file, `${file}.1`); + } + fs.appendFileSync(file, lines, { encoding: 'utf8', mode: 0o600 }); + return true; + } catch (_) { + return false; + } +} + +module.exports = { + DECISIONS, + METRICS_FILE_NAME, + METRICS_MAX_BYTES, + METRICS_SCHEMA_VERSION, + appendMetrics, + isMetricsEvent, + metricsEvent, + sessionDigest +}; diff --git a/tests/fixtures/gateguard-metrics/metrics.jsonl b/tests/fixtures/gateguard-metrics/metrics.jsonl new file mode 100644 index 000000000..06e6fd4ff --- /dev/null +++ b/tests/fixtures/gateguard-metrics/metrics.jsonl @@ -0,0 +1,17 @@ +{"v":1,"ts":"2026-09-01T10:02:00.000Z","session":"aaaaaaaaaaaa","tool":"Edit","class":"code","decision":"deny","reason":"near-miss:out-of-scope","questions":["local-callers","quote-instruction"],"sensitive":false,"profile":{"known":true,"language":"js","touchesPublicSurface":false,"touchesData":false,"trivial":false}} +not json +{"v":1,"ts":"2026-09-01T10:03:00.000Z","session":"aaaaaaaaaaaa","tool":"Edit","class":"code","decision":"credit","reason":"prior-search","questions":null,"sensitive":false,"profile":null} +{"v":2,"ts":"2026-09-01T10:03:30.000Z","session":"aaaaaaaaaaaa","tool":"Edit","class":"code","decision":"deny","reason":"first-touch","questions":[],"sensitive":false,"profile":null} +{"v":1,"ts":"2026-09-01T10:04:00.000Z","session":"aaaaaaaaaaaa","tool":"Edit","class":"code","decision":"trivial","reason":"comment-whitespace","questions":null,"sensitive":false,"profile":{"known":true,"language":"js","touchesPublicSurface":false,"touchesData":false,"trivial":true}} +{"v":1,"ts":"2026-09-01T10:05:00.000Z","session":"aaaaaaaaaaaa","tool":"Bash","class":null,"decision":"routine-deny","reason":"first-command","questions":null,"sensitive":false,"profile":null} +{"v":1,"ts":"2026-09-01T11:00:00.000Z","session":"NOT-A-HASH","tool":"Edit","class":"code","decision":"deny","reason":"first-touch","questions":[],"sensitive":false,"profile":null} + +{"v":1,"ts":"2026-09-01T11:00:30.000Z","session":"bbbbbbbbbbbb","tool":"Edit","class":"code","decision":"allow","reason":"first-touch","questions":null,"sensitive":false,"profile":null} +[] +{"v":1,"ts":"2026-09-01T11:00:40.000Z","session":"bbbbbbbbbbbb","tool":"Edit","class":"code","decision":"deny","reason":"/home/user/secret.js","questions":null,"sensitive":false,"profile":null} +{"v":1,"ts":"2026-09-01T11:01:00.000Z","session":"bbbbbbbbbbbb","tool":"Write","class":"code","decision":"deny","reason":"first-touch","questions":["callers","no-duplicate","quote-instruction"],"sensitive":false,"profile":{"known":true,"language":"py","touchesPublicSurface":true,"touchesData":false,"trivial":false}} +{"v":1,"ts":"2026-09-01T11:02:00.000Z","session":"bbbbbbbbbbbb","tool":"Write","class":"code","decision":"sibling","reason":"same-turn-dir","questions":null,"sensitive":false,"profile":null} +{"v":1,"ts":"2026-09-01T11:03:00.000Z","session":"bbbbbbbbbbbb","tool":"Edit","class":"config","decision":"deny","reason":"sensitive","questions":["config-reader","config-effect","no-plaintext-secrets","quote-instruction"],"sensitive":true,"profile":null} +{"v":1,"ts":"2026-09-01T11:04:00.000Z","session":"bbbbbbbbbbbb","tool":"Edit","class":"code","decision":"cap","reason":"max-denials","questions":null,"sensitive":false,"profile":null} +{"v":1,"ts":"2026-09-01T11:05:00.000Z","session":"bbbbbbbbbbbb","tool":"PowerShell","class":null,"decision":"destructive-deny","reason":"destructive","questions":null,"sensitive":false,"profile":null} +{"v":1,"ts":"2026-09-01T11:06:00.000Z","session":"bbbbbbbbbbbb","tool":"Edit","class":"code","decision":"pass-checked","reason":"checked","questions":null,"sensitive":false,"profile":null} diff --git a/tests/fixtures/gateguard-metrics/metrics.jsonl.1 b/tests/fixtures/gateguard-metrics/metrics.jsonl.1 new file mode 100644 index 000000000..34ea592b4 --- /dev/null +++ b/tests/fixtures/gateguard-metrics/metrics.jsonl.1 @@ -0,0 +1,2 @@ +{"v":1,"ts":"2026-09-01T10:00:00.000Z","session":"aaaaaaaaaaaa","tool":"Edit","class":"code","decision":"deny","reason":"first-touch","questions":["importers","public-api","quote-instruction"],"sensitive":false,"profile":{"known":true,"language":"js","touchesPublicSurface":true,"touchesData":false,"trivial":false}} +{"v":1,"ts":"2026-09-01T10:01:00.000Z","session":"aaaaaaaaaaaa","tool":"Bash","class":null,"decision":"routine-readonly","reason":"readonly","questions":null,"sensitive":false,"profile":null} diff --git a/tests/lib/gateguard-metrics.test.js b/tests/lib/gateguard-metrics.test.js new file mode 100644 index 000000000..12b204067 --- /dev/null +++ b/tests/lib/gateguard-metrics.test.js @@ -0,0 +1,153 @@ +/** + * Tests for scripts/lib/gateguard-metrics.js + */ + +'use strict'; + +const assert = require('assert'); +const crypto = require('crypto'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); + +const { + DECISIONS, + METRICS_FILE_NAME, + METRICS_MAX_BYTES, + metricsEvent, + appendMetrics, + isMetricsEvent +} = require('../../scripts/lib/gateguard-metrics'); + +let passed = 0; +let failed = 0; + +function test(name, fn) { + try { + fn(); + console.log(` \u2713 ${name}`); + passed++; + } catch (error) { + console.log(` \u2717 ${name}`); + console.log(` Error: ${error.message}`); + failed++; + } +} + +const NOW = Date.parse('2026-09-01T10:00:00.000Z'); +const base = { sessionKey: 'sess-1', tool: 'Edit', decision: 'deny', reason: 'first-touch', now: NOW }; + +console.log('\n=== Testing gateguard-metrics ===\n'); + +test('builds a schema 1 event with a 12-character session digest', () => { + const event = metricsEvent({ + ...base, + cls: 'code', + questions: ['importers', 'quote-instruction'], + sensitive: false, + profile: { known: true, language: 'js', touchesPublicSurface: true, touchesData: false, trivial: false, extra: 'x' } + }); + assert.deepStrictEqual(event, { + v: 1, + ts: '2026-09-01T10:00:00.000Z', + session: crypto.createHash('sha256').update('sess-1').digest('hex').slice(0, 12), + tool: 'Edit', + class: 'code', + decision: 'deny', + reason: 'first-touch', + questions: ['importers', 'quote-instruction'], + sensitive: false, + profile: { known: true, language: 'js', touchesPublicSurface: true, touchesData: false, trivial: false } + }); + assert.ok(isMetricsEvent(event)); +}); + +test('returns null for unknown decisions or tools', () => { + assert.strictEqual(metricsEvent({ ...base, decision: 'allow' }), null); + assert.strictEqual(metricsEvent({ ...base, tool: 'Read' }), null); +}); + +test('drops any class, reason, question or language that is not a short code', () => { + const event = metricsEvent({ + ...base, + cls: '/src/app.js', + reason: 'rm -rf /', + questions: ['importers', '/etc/passwd', 'x'.repeat(100)], + profile: { known: true, language: 'C:\\x' } + }); + assert.strictEqual(event.class, null); + assert.strictEqual(event.reason, null); + assert.deepStrictEqual(event.questions, ['importers']); + assert.strictEqual(event.profile.language, null); +}); + +test('keeps near-miss reason codes', () => { + assert.strictEqual(metricsEvent({ ...base, reason: 'near-miss:out-of-scope' }).reason, 'near-miss:out-of-scope'); +}); + +test('lists every decision the hook records', () => { + assert.deepStrictEqual([...DECISIONS].sort(), [ + 'cap', + 'credit', + 'deny', + 'destructive-deny', + 'pass', + 'pass-checked', + 'pass-exempt', + 'pass-subagent', + 'routine-deny', + 'routine-readonly', + 'sibling', + 'trivial' + ]); +}); + +test('rejects events with the wrong shape', () => { + const good = metricsEvent(base); + assert.ok(isMetricsEvent(good)); + for (const bad of [ + null, + [], + { ...good, v: 2 }, + { ...good, session: 'ABC' }, + { ...good, decision: 'allow' }, + { ...good, tool: 'Read' }, + { ...good, reason: '/x' }, + { ...good, class: 'a b' }, + { ...good, questions: ['ok', '/x'] }, + { ...good, sensitive: 'no' }, + { ...good, ts: 5 } + ]) { + assert.strictEqual(isMetricsEvent(bad), false, JSON.stringify(bad)); + } +}); + +test('appends one line per event in a single write and rotates past the cap', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-metrics-lib-')); + try { + const file = path.join(dir, 'nested', METRICS_FILE_NAME); + assert.strictEqual(appendMetrics(path.dirname(file), [metricsEvent(base), null, metricsEvent({ ...base, decision: 'credit' })]), true); + assert.strictEqual(fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).length, 2); + fs.writeFileSync(`${file}.1`, 'old\n'); + fs.writeFileSync(file, 'x'.repeat(METRICS_MAX_BYTES)); + assert.strictEqual(appendMetrics(path.dirname(file), [metricsEvent(base)]), true); + assert.strictEqual(fs.statSync(`${file}.1`).size, METRICS_MAX_BYTES); + assert.strictEqual(fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).length, 1); + assert.strictEqual(appendMetrics(path.dirname(file), [null]), false); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('reports failure instead of throwing when the file cannot be written', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-metrics-lib-')); + try { + fs.mkdirSync(path.join(dir, METRICS_FILE_NAME)); + assert.strictEqual(appendMetrics(dir, [metricsEvent(base)]), false); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +console.log(`\n ${passed} passed, ${failed} failed\n`); +process.exit(failed > 0 ? 1 : 0); diff --git a/tests/scripts/gateguard-report.test.js b/tests/scripts/gateguard-report.test.js new file mode 100644 index 000000000..d2111a7b4 --- /dev/null +++ b/tests/scripts/gateguard-report.test.js @@ -0,0 +1,182 @@ +/** + * Tests for scripts/gateguard-report.js + */ + +'use strict'; + +const assert = require('assert'); +const fs = require('fs'); +const os = require('os'); +const path = require('path'); +const { spawnSync } = require('child_process'); + +const SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'gateguard-report.js'); +const FIXTURE_DIR = path.join(__dirname, '..', 'fixtures', 'gateguard-metrics'); +const { loadMetrics, summarize, formatReport, parseArgs } = require('../../scripts/gateguard-report'); + +let passed = 0; +let failed = 0; + +function test(name, fn) { + try { + fn(); + console.log(` ✓ ${name}`); + passed++; + } catch (error) { + console.log(` ✗ ${name}`); + console.log(` Error: ${error.message}`); + failed++; + } +} + +function run(args, env = {}) { + const result = spawnSync('node', [SCRIPT, ...args], { + encoding: 'utf8', + env: { ...process.env, ...env }, + timeout: 10000 + }); + return { status: result.status, stdout: result.stdout || '', stderr: result.stderr || '' }; +} + +console.log('\n=== Testing gateguard-report ===\n'); + +const loaded = loadMetrics(FIXTURE_DIR); +const report = summarize(loaded.events); + +test('reads the rotated file first, then the current file', () => { + assert.strictEqual(loaded.events.length, 12); + assert.strictEqual(loaded.events[0].ts, '2026-09-01T10:00:00.000Z'); + assert.strictEqual(loaded.events[11].decision, 'pass-checked'); + assert.deepStrictEqual(loaded.files.map(f => path.basename(f)), ['metrics.jsonl.1', 'metrics.jsonl']); +}); + +test('skips malformed, unknown-version, unknown-decision and non-code lines', () => { + assert.strictEqual(loaded.skipped, 6); + assert.ok(loaded.events.every(e => e.session === 'aaaaaaaaaaaa' || e.session === 'bbbbbbbbbbbb')); + assert.ok(!loaded.events.some(e => String(e.reason).includes('/'))); +}); + +test('counts decisions by type', () => { + assert.deepStrictEqual(report.total.decisions, { + deny: 4, + cap: 1, + credit: 1, + 'destructive-deny': 1, + 'pass-checked': 1, + 'routine-deny': 1, + 'routine-readonly': 1, + sibling: 1, + trivial: 1 + }); + assert.strictEqual(report.total.events, 12); +}); + +test('breaks denials down by class and reason', () => { + assert.strictEqual(report.total.denials.total, 6); + assert.deepStrictEqual(report.total.denials.byClass, { code: 3, shell: 2, config: 1 }); + assert.deepStrictEqual(report.total.denials.byReason, { + 'first-touch': 2, + destructive: 1, + 'first-command': 1, + 'near-miss:out-of-scope': 1, + sensitive: 1 + }); +}); + +test('reports the near-miss share of first-touch denials', () => { + assert.deepStrictEqual(report.total.nearMiss, { count: 1, of: 4, share: 0.25 }); +}); + +test('reports first-touch outcome rates', () => { + const { firstTouch } = report.total; + assert.strictEqual(firstTouch.total, 8); + assert.deepStrictEqual(firstTouch.counts, { deny: 4, credit: 1, sibling: 1, cap: 1, trivial: 1 }); + assert.strictEqual(firstTouch.rates.deny, 0.5); + assert.strictEqual(firstTouch.rates.trivial, 0.125); +}); + +test('counts routine read-only passes and ranks the most-asked questions', () => { + assert.strictEqual(report.total.routineReadonlyPasses, 1); + assert.deepStrictEqual(report.total.topQuestions.slice(0, 3), [ + ['quote-instruction', 4], + ['callers', 1], + ['config-effect', 1] + ]); +}); + +test('summarizes each session separately in order of first appearance', () => { + assert.deepStrictEqual(Object.keys(report.sessions), ['aaaaaaaaaaaa', 'bbbbbbbbbbbb']); + const a = report.sessions.aaaaaaaaaaaa; + assert.strictEqual(a.events, 6); + assert.strictEqual(a.denials.total, 3); + assert.deepStrictEqual(a.nearMiss, { count: 1, of: 2, share: 0.5 }); + assert.strictEqual(a.firstTouch.total, 4); + assert.strictEqual(a.first, '2026-09-01T10:00:00.000Z'); + assert.strictEqual(a.last, '2026-09-01T10:05:00.000Z'); + const b = report.sessions.bbbbbbbbbbbb; + assert.strictEqual(b.events, 6); + assert.deepStrictEqual(b.denials.byClass, { code: 1, config: 1, shell: 1 }); + assert.strictEqual(b.routineReadonlyPasses, 0); +}); + +test('reports empty rates as null instead of dividing by zero', () => { + const empty = summarize([]); + assert.strictEqual(empty.total.events, 0); + assert.strictEqual(empty.total.nearMiss.share, null); + assert.strictEqual(empty.total.firstTouch.rates.deny, null); +}); + +test('formats a readable text report', () => { + const text = formatReport({ ...loaded, report }); + assert.ok(text.includes('12 events in 2 sessions (6 malformed lines skipped)')); + assert.ok(text.includes('Total')); + assert.ok(text.includes('Session aaaaaaaaaaaa')); + assert.ok(text.includes('near-miss share of first-touch denials: 25.0% (1 of 4)')); + assert.ok(text.includes('routine read-only passes: 1')); + assert.ok(text.includes('quote-instruction 4')); +}); + +test('parses --dir, --dir=, --json and --help', () => { + assert.deepStrictEqual(parseArgs(['--dir', '/x', '--json']), { dir: '/x', json: true, help: false }); + assert.deepStrictEqual(parseArgs(['--dir=/y']), { dir: '/y', json: false, help: false }); + assert.strictEqual(parseArgs(['-h']).help, true); + assert.throws(() => parseArgs(['--dir']), /Missing value for --dir/); + assert.throws(() => parseArgs(['--bogus']), /Unknown argument: --bogus/); +}); + +test('CLI prints JSON with --json', () => { + const result = run(['--dir', FIXTURE_DIR, '--json']); + assert.strictEqual(result.status, 0, result.stderr); + const parsed = JSON.parse(result.stdout); + assert.strictEqual(parsed.events, 12); + assert.strictEqual(parsed.skipped, 6); + assert.strictEqual(parsed.total.denials.total, 6); + assert.deepStrictEqual(Object.keys(parsed.sessions), ['aaaaaaaaaaaa', 'bbbbbbbbbbbb']); +}); + +test('CLI reads GATEGUARD_STATE_DIR when --dir is not given', () => { + const result = run([], { GATEGUARD_STATE_DIR: FIXTURE_DIR }); + assert.strictEqual(result.status, 0, result.stderr); + assert.ok(result.stdout.includes('12 events in 2 sessions')); +}); + +test('CLI explains how to enable metrics when none are found', () => { + const dir = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-report-')); + try { + const result = run(['--dir', dir]); + assert.strictEqual(result.status, 0, result.stderr); + assert.ok(result.stdout.includes('No GateGuard metrics')); + assert.ok(result.stdout.includes('GATEGUARD_METRICS=1')); + } finally { + fs.rmSync(dir, { recursive: true, force: true }); + } +}); + +test('CLI exits 1 on an unknown argument', () => { + const result = run(['--bogus']); + assert.strictEqual(result.status, 1); + assert.ok(result.stderr.includes('Unknown argument: --bogus')); +}); + +console.log(`\n ${passed} passed, ${failed} failed\n`); +process.exit(failed > 0 ? 1 : 0); diff --git a/tests/scripts/npm-publish-surface.test.js b/tests/scripts/npm-publish-surface.test.js index 02ca28943..a63f7f0dc 100644 --- a/tests/scripts/npm-publish-surface.test.js +++ b/tests/scripts/npm-publish-surface.test.js @@ -83,6 +83,7 @@ function buildExpectedPublishPaths(repoRoot) { "scripts/uninstall.js", "scripts/welcome.js", "scripts/gemini-adapt-agents.js", + "scripts/gateguard-report.js", "scripts/sync-ecc-to-codex.sh", "scripts/codex/legacy-sync-state.js", "scripts/codex/install-global-git-hooks.sh",