fix(gateguard): a throwing hook no longer ends the fresh-process replay

measureColdLatency ran each step in a fresh process through execFileSync,
inside try/finally with no catch. A hook that threw made the child exit
non-zero, execFileSync threw, and the error left measureColdLatency and then
evaluate, so neither the report nor the requested SARIF was written for a run
that had hit a single bad step.

The probe now catches a throwing require or run itself and records it the way
workerMain does: the raw payload stands in for the result, so the step reads as
an allow, and the error is kept. Matching the warm path matters because
coldSummary counts every cold decision that differs from the warm one as a
disagreement; any other choice would have reported disagreements that do not
exist. A child that dies without writing parseable output is caught outside the
probe and recorded the same way with no latency, and coldSummary leaves
unmeasured steps out of the percentiles while still counting them.

The new tests run a hook that always throws through both passes. They fail
against the previous code, 2 of 17, and pass against this one.
This commit is contained in:
montjeffrey
2026-10-04 22:22:50 -04:00
parent 56a516c2c5
commit 3a4a574609
2 changed files with 60 additions and 12 deletions
+34 -12
View File
@@ -309,13 +309,22 @@ async function runCorpus(hookFile, scenarios) {
// --- cold latency ---
// see docs/gateguard/evaluation.md#latency
// A hook that throws is recorded the way workerMain records it - the raw payload
// stands in for the result, so the step reads as an allow - rather than ending the
// run. Otherwise one throwing step aborted the whole replay and no report was written.
const COLD_PROBE = [
"const { performance } = require('perf_hooks');",
'const started = performance.now();',
'const hook = require(process.argv[1]);',
'const result = hook.run(process.argv[2]);',
'let result;',
'let error = null;',
'try {',
' result = require(process.argv[1]).run(process.argv[2]);',
'} catch (thrown) {',
' error = String(thrown && thrown.message ? thrown.message : thrown);',
' result = process.argv[2];',
'}',
'const ms = performance.now() - started;',
'process.stdout.write(JSON.stringify({ ms, result: result === undefined ? null : result }));'
'process.stdout.write(JSON.stringify({ ms, result: result === undefined ? null : result, error }));'
].join('\n');
/** Replay every scenario with one fresh Node process per step; returns per-step require+run latency and decisions. */
@@ -331,15 +340,27 @@ function measureColdLatency(hookFile, scenarios) {
fs.appendFileSync(prepared.transcript, step.records.map(record => `${JSON.stringify(record)}\n`).join(''), 'utf8');
}
const raw = JSON.stringify(step.payload);
const stdout = execFileSync(process.execPath, ['-e', COLD_PROBE, hookFile, raw], {
env: prepared.env,
encoding: 'utf8',
maxBuffer: 16 * 1024 * 1024,
stdio: ['ignore', 'pipe', 'ignore']
let probe;
try {
probe = JSON.parse(execFileSync(process.execPath, ['-e', COLD_PROBE, hookFile, raw], {
env: prepared.env,
encoding: 'utf8',
maxBuffer: 16 * 1024 * 1024,
stdio: ['ignore', 'pipe', 'ignore']
}));
} catch (failed) {
// The process died or wrote nothing parseable, so there is no latency to
// report; the step still counts, as an allow, matching a throwing hook.
probe = { ms: null, result: raw, error: String(failed && failed.message ? failed.message : failed) };
}
const output = hookOutput(probe.result);
steps.push({
scenario: scenario.name,
step: step.id,
latencyMs: probe.ms,
decision: output.decision === 'deny' ? 'deny' : 'allow',
error: probe.error || null
});
const { ms, result } = JSON.parse(stdout);
const output = hookOutput(result);
steps.push({ scenario: scenario.name, step: step.id, latencyMs: ms, decision: output.decision === 'deny' ? 'deny' : 'allow' });
}
}
return steps;
@@ -349,7 +370,8 @@ function measureColdLatency(hookFile, scenarios) {
}
function coldSummary(coldSteps, warmSteps) {
const sorted = coldSteps.map(step => step.latencyMs).sort((a, b) => a - b);
// A step whose process died has no latency; it still counts toward disagreements.
const sorted = coldSteps.map(step => step.latencyMs).filter(Number.isFinite).sort((a, b) => a - b);
const warmDecisions = new Map(warmSteps.map(step => [`${step.scenario}/${step.step}`, step.decision]));
const disagreements = coldSteps.filter(step => warmDecisions.get(`${step.scenario}/${step.step}`) !== step.decision).length;
return {