fix(gateguard): enforce evaluation validity guards

docker/gateguard-effectiveness/coverage.js

@@ -18,6 +18,11 @@ const { pairedComparison } = require('./lib');
 // Rough floor for 80% power on a 40-point pass-rate difference at a two-sided
 // 5% level. Below it the report refuses to let counts stand as evidence.
 const MIN_PAIRS_FOR_A_CLAIM = 20;
+const MAX_INVALID_TRIAL_FRACTION = 0.1;
+
+function isValidTrial(row) {
+  return !row.providerError && !row.timedOut && !row.judgeFailed;
+}
 
 /** Outcome classes. The four ending in -hole are the improvable cases. */
 const CLASSES = Object.freeze({
@@ -178,6 +183,20 @@ function renderEffect(rows, { referenceArm = 'gate', baselineArm = 'off' } = {})
 }
 
 function renderHoles(rows, { baselineArm = 'off' } = {}) {
+  const invalid = rows.filter(row => !isValidTrial(row));
+  const invalidFraction = rows.length ? invalid.length / rows.length : 0;
+  const validity = rows.length
+    ? `Excluded ${invalid.length}/${rows.length} invalid trial attempt(s) (provider error, timeout, or judge failure).`
+    : 'No trial attempts recorded.';
+  if (invalidFraction > MAX_INVALID_TRIAL_FRACTION) {
+    return [
+      '### Run validity',
+      '',
+      `INCONCLUSIVE: ${invalid.length}/${rows.length} trial attempts were invalid, above the 10% limit. Arm comparisons are suppressed.`,
+      ''
+    ].join('\n');
+  }
+  rows = rows.filter(isValidTrial);
   const arms = [...new Set(rows.map(row => row.arm))].sort();
   const classes = Object.keys(CLASSES);
   const lines = [];
@@ -186,6 +205,7 @@ function renderHoles(rows, { baselineArm = 'off' } = {}) {
   const verdicts = informative(rows);
 
   lines.push('### Trap strength and usability', '');
+  lines.push(validity, '');
   lines.push('| Scenario | Ungated passed | Rate | Trap | Separates arms |', '| --- | ---: | ---: | --- | --- |');
   const verdictOf = Object.fromEntries(verdicts.map(row => [row.scenario, row.label]));
   for (const row of strength) {
@@ -236,6 +256,6 @@ function renderHoles(rows, { baselineArm = 'off' } = {}) {
 }
 
 module.exports = {
-  CLASSES, MIN_PAIRS_FOR_A_CLAIM, classify, tally, difficulty, informative,
+  CLASSES, MIN_PAIRS_FOR_A_CLAIM, MAX_INVALID_TRIAL_FRACTION, isValidTrial, classify, tally, difficulty, informative,
   discoveredQuestions, unengaged, renderEffect, renderHoles
 };

docker/gateguard-effectiveness/intent-eval.js

@@ -11,7 +11,7 @@
 
 const fs = require('fs');
 const path = require('path');
-const { prepareWorkspace, commitWorkspace, grade, readMetrics } = require('./lib');
+const { prepareWorkspace, commitWorkspace, grade, readMetrics, hookWasObserved } = require('./lib');
 const { ARMS, TRIAL_FILE, trialPatch } = require('./arms');
 const { runConversation } = require('./session');
 const { classify } = require('./coverage');
@@ -39,12 +39,14 @@ function loadIntentScenarios(dir = SCENARIO_DIR, ids = null) {
       if (!Array.isArray(spec.evidence) || !spec.evidence.length) throw new Error(`scenario ${spec.id} has no evidence`);
       if (typeof intent.ambiguity !== 'string' || !intent.ambiguity) throw new Error(`scenario ${spec.id} intent has no ambiguity`);
       if (typeof intent.stonewall !== 'string' || !intent.stonewall) throw new Error(`scenario ${spec.id} intent has no stonewall`);
-      if (!Array.isArray(intent.facts) || !intent.facts.some(fact => fact.decisive)) {
-        throw new Error(`scenario ${spec.id} intent has no decisive fact`);
-      }
+      if (!Array.isArray(intent.facts)) throw new Error(`scenario ${spec.id} intent has no decisive fact`);
       for (const fact of intent.facts) {
         if (!fact.id || typeof fact.fact !== 'string' || !fact.fact) throw new Error(`scenario ${spec.id} has a malformed fact`);
       }
+      const decisiveCount = intent.facts.filter(fact => fact.decisive === true).length;
+      if (decisiveCount !== 1) {
+        throw new Error(`scenario ${spec.id} intent must have exactly one decisive fact; found ${decisiveCount}`);
+      }
       return {
         ...spec,
         root,
@@ -109,9 +111,14 @@ function runIntentTrial(trial, { workRoot, armSettingsByName, executable, model,
     judgeFailed: conversation.judgeFailed,
     providerError: conversation.providerError,
     providerMessage: conversation.providerMessage,
-    // Same guard as lib.js: a gated arm whose agent edited files with no metrics
-    // line and no denial means the hook never ran, so the trial proves nothing.
-    hookObserved: !ARMS[trial.arm].gate || conversation.editCalls === 0 || metrics.length > 0 || conversation.gateDenials > 0,
+    // Any hook-matched edit or shell call requires observable hook output.
+    hookObserved: hookWasObserved({
+      gated: ARMS[trial.arm].gate,
+      editCalls: conversation.editCalls,
+      shellCalls: conversation.shellCalls,
+      metrics,
+      gateDenials: conversation.gateDenials
+    }),
     gateDenials: conversation.gateDenials,
     denialsPerTurn: conversation.denialsPerTurn,
     questionsAsked: [...new Set(denials.flatMap(event => event.questions || []))].sort(),

docker/gateguard-effectiveness/lib.js

@@ -199,7 +199,7 @@ function toolResultText(content) {
 
 /** Reads a stream-json transcript: final result, assistant text and tool calls. */
 function parseStream(stdout) {
-  const parsed = { result: null, texts: [], tools: {}, editCalls: 0, gateDenials: 0 };
+  const parsed = { result: null, texts: [], tools: {}, editCalls: 0, shellCalls: 0, gateDenials: 0 };
   for (const line of String(stdout || '').split('\n')) {
     if (!line.trim()) continue;
     let event;
@@ -220,6 +220,7 @@ function parseStream(stdout) {
       if (block.type === 'tool_use') {
         parsed.tools[block.name] = (parsed.tools[block.name] || 0) + 1;
         if (['Edit', 'Write', 'MultiEdit', 'NotebookEdit'].includes(block.name)) parsed.editCalls++;
+        if (['Bash', 'PowerShell'].includes(block.name)) parsed.shellCalls++;
       }
     }
   }
@@ -243,6 +244,10 @@ function readMetrics(stateDir) {
     .filter(Boolean);
 }
 
+function hookWasObserved({ gated, editCalls, shellCalls, metrics, gateDenials }) {
+  return !gated || editCalls + shellCalls === 0 || metrics.length > 0 || gateDenials > 0;
+}
+
 /** Share of a scenario's evidence patterns that the agent's own text states. */
 function evidenceRecall(scenario, texts) {
   const text = texts.join('\n');
@@ -310,7 +315,13 @@ function runTrial(trial, { workRoot, armSettingsByName, executable, model, maxTu
     denials: denials.length,
     gateDenials: stream.gateDenials,
     questionsAsked: [...new Set(denials.flatMap(event => event.questions || []))].sort(),
-    hookObserved: !gated || stream.editCalls === 0 || metrics.length > 0 || stream.gateDenials > 0,
+    hookObserved: hookWasObserved({
+      gated,
+      editCalls: stream.editCalls,
+      shellCalls: stream.shellCalls,
+      metrics,
+      gateDenials: stream.gateDenials
+    }),
     ...usageOf(stream.result)
   };
 }
@@ -435,6 +446,7 @@ module.exports = {
   claudeArgs,
   parseStream,
   readMetrics,
+  hookWasObserved,
   evidenceRecall,
   runTrial,
   binomialTwoSided,

docker/gateguard-effectiveness/run-intent.js

@@ -15,7 +15,7 @@ const { spawnSync } = require('child_process');
 const { schedule } = require('./lib');
 const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms');
 const { loadIntentScenarios, graderIsSound, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');
-const { renderHoles, difficulty } = require('./coverage');
+const { renderHoles, difficulty, isValidTrial } = require('./coverage');
 
 const REPO_ROOT = path.resolve(__dirname, '..', '..');
 
@@ -91,6 +91,10 @@ function readResults(file) {
   return fs.readFileSync(file, 'utf8').split('\n').filter(Boolean).map(line => JSON.parse(line));
 }
 
+function completedTrialKeys(rows) {
+  return new Set(rows.filter(isValidTrial).map(row => row.key));
+}
+
 // Resuming a session needs persistence, so Claude writes a transcript folder per
 // trial workspace under its projects directory. A per-trial CLAUDE_CONFIG_DIR
 // would isolate those but also hides the credentials, so they are removed here
@@ -123,7 +127,13 @@ function writeReport(options, rows, meta) {
   fs.writeFileSync(path.join(options.out, 'holes.md'), header + body);
   fs.writeFileSync(
     path.join(options.out, 'summary.json'),
-    `${JSON.stringify({ meta, difficulty: difficulty(rows), rows: rows.length }, null, 2)}\n`
+    `${JSON.stringify({
+      meta,
+      difficulty: difficulty(rows.filter(isValidTrial)),
+      rows: rows.length,
+      validRows: rows.filter(isValidTrial).length,
+      invalidRows: rows.filter(row => !isValidTrial(row)).length
+    }, null, 2)}\n`
   );
   return header + body;
 }
@@ -197,7 +207,9 @@ function main() {
     const armSettingsByName = Object.fromEntries(
       options.arms.map(arm => [arm, armSettings(ARMS[arm].gate ? treeRoots[ARMS[arm].tree] : null)])
     );
-    const done = new Set(rows.map(row => row.key));
+    // Failed attempts remain in results.jsonl for audit, but the scheduled key
+    // is complete only after a valid trial has been recorded.
+    const done = completedTrialKeys(rows);
     const trials = schedule(scenarios, options.arms, options.reps, options.seed).filter(trial => !done.has(trial.key));
     process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`);
     if (options.dryRun) {
@@ -264,4 +276,4 @@ if (require.main === module) {
   }
 }
 
-module.exports = { parseArgs, writeReport, checkGraders };
+module.exports = { parseArgs, writeReport, checkGraders, completedTrialKeys };

docker/gateguard-effectiveness/session.js

@@ -88,6 +88,7 @@ function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, option
   const denialsPerTurn = [];
   let gateDenials = 0;
   let editCalls = 0;
+  let shellCalls = 0;
   const tools = {};
   let judgeFailed = false;
   let asked = false;
@@ -120,6 +121,7 @@ function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, option
     denialsPerTurn.push(stream.gateDenials);
     gateDenials += stream.gateDenials;
     editCalls += stream.editCalls;
+    shellCalls += stream.shellCalls;
     for (const [name, count] of Object.entries(stream.tools)) tools[name] = (tools[name] || 0) + count;
 
     const message = finalText(stream);
@@ -147,6 +149,7 @@ function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, option
     // after a non-zero is the latch, not the gate choosing to stay quiet.
     denialsPerTurn,
     editCalls,
+    shellCalls,
     tools,
     judgeFailed,
     exitStatus: lastStatus,