fix: distinguish auth failures from ordinary provider discussion

docker/gateguard-effectiveness/lib.js

@@ -227,6 +227,13 @@ function parseStream(stdout) {
   return parsed;
 }
 
+/** Recognize sign-in failure diagnostics, not ordinary authentication discussion. */
+function isAuthenticationFailure(message) {
+  if (typeof message !== 'string') return false;
+  return /^\s*(?:(?:API\s+)?Error:\s*)?(?:Not\s+(?:logged|signed)\s+in|Authentication\s+(?:is\s+)?(?:failed|required)|OAuth(?:\s+(?:token|authentication))?\s+(?:failed|expired|invalid|required)|(?:Login|Sign-in)\s+(?:failed|required)|Invalid\s+API\s+key)(?=\s*(?:$|[.:;!()\u00b7-])|[\r\n])/i.test(message)
+    || /^\s*(?:Please|Must)\s+(?:run\s+\/login|log\s+in|sign\s+in)\b/i.test(message);
+}
+
 function readMetrics(stateDir) {
   const file = path.join(stateDir, 'metrics.jsonl');
   if (!fs.existsSync(file)) return [];
@@ -461,6 +468,7 @@ module.exports = {
   childEnv,
   claudeArgs,
   parseStream,
+  isAuthenticationFailure,
   readMetrics,
   hookWasObserved,
   evidenceRecall,

docker/gateguard-effectiveness/run-intent.js

@@ -13,7 +13,7 @@ const crypto = require('crypto');
 const os = require('os');
 const path = require('path');
 const { spawnSync } = require('child_process');
-const { schedule, completedTrialKeys } = require('./lib');
+const { schedule, completedTrialKeys, isAuthenticationFailure } = require('./lib');
 const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms');
 const { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');
 const { renderHoles, difficulty, isValidTrial } = require('./coverage');
@@ -356,7 +356,7 @@ function main() {
       fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
       // An unauthenticated or hookless run must not be recorded: graded as data it
       // looks like a finding (the first such trial was filed as a coverage hole).
-      if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
+      if (row.providerError && isAuthenticationFailure(row.providerMessage)) {
         throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);
       }
       if (!row.hookObserved) {

docker/gateguard-effectiveness/run.js

@@ -7,7 +7,7 @@ const os = require('os');
 const path = require('path');
 const { spawnSync } = require('child_process');
 const { ARMS, DEFAULT_ARMS, resolveRef, materializeTree, armSettings } = require('./arms');
-const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys } = require('./lib');
+const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys, isAuthenticationFailure } = require('./lib');
 
 const REPO_ROOT = path.join(__dirname, '..', '..');
 
@@ -202,7 +202,7 @@ function main() {
         };
         fs.writeFileSync(path.join(options.out, row.transcript), transcript);
         fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
-        if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
+        if (row.providerError && isAuthenticationFailure(row.providerMessage)) {
           throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);
         }
         if (!row.hookObserved) throw new Error(`${trial.key}: the gate did not run although the agent edited files; check the hook setup before spending more sessions`);

tests/docker/gateguard-effectiveness.test.js

@@ -317,6 +317,24 @@ if (gnuTarDir) {
   });
 }
 
+test('authentication abort recognizes failure diagnostics rather than ordinary discussion', () => {
+  const failures = [
+    'Not logged in - Please run /login', 'Authentication failed: OAuth token expired',
+    'OAuth authentication failed', 'OAuth token expired; sign in again',
+    'Authentication is required', 'Login failed', 'Error: Not signed in.',
+    'Invalid API key · Please run /login', 'Please run /login', 'Must sign in'
+  ];
+  const ordinary = [
+    'Payment API key authentication uses the Authorization header.',
+    'OAuth tokens authorize payment API requests.', 'Login routes use OAuth.',
+    'Authentication uses a bearer token.', 'How to use OAuth login',
+    'Please explain login flows', 'Authentication failed examples belong in docs',
+    'Not logged in errors are covered in tests', '', null
+  ];
+  for (const text of failures) assert.strictEqual(lib.isAuthenticationFailure(text), true, text);
+  for (const text of ordinary) assert.strictEqual(lib.isAuthenticationFailure(text), false, String(text));
+});
+
 test('pilot summaries count valid trials only and report provider errors apart', () => {
   const row = (arm, rep, passed, extra = {}) => ({
     key: `s/${arm}/${rep}`, arm, scenario: 's', rep, question: 'q', passed, score: passed ? 1 : 0, evidenceRecall: 0, gateDenials: 0,
@@ -442,6 +460,24 @@ if (gitAvailable()) {
     } finally { fs.rmSync(root, { recursive: true, force: true }); }
   });
 
+  test('ordinary authentication discussion remains a failed-attempt row on missing result', () => {
+    for (const text of ['Payment API key authentication uses the Authorization header.', 'OAuth tokens authorize payment API requests.']) {
+      const root = tempDir('gg-eff-auth-discussion-');
+      try {
+        const scenario = scenarios.find(item => item.id === 'importers-date-shape');
+        const events = [{ type: 'assistant', message: { content: [{ type: 'text', text }] } }];
+        const { out, run, results } = syntheticPilotRunner(root, scenario, 'off', events);
+        const result = run({});
+        assert.strictEqual(result.status, 0, result.stderr);
+        assert.strictEqual(results().length, 1);
+        assert.strictEqual(results()[0].providerError, true);
+        assert.strictEqual(results()[0].providerMessage, text);
+        assert.strictEqual(fs.readdirSync(path.join(out, 'transcripts')).length, 1);
+        assert.ok(!fs.existsSync(path.join(out, '.run.lock')));
+      } finally { fs.rmSync(root, { recursive: true, force: true }); }
+    }
+  });
+
   test('the pilot runner stops before recording a gated trial whose hook never ran', () => {
     const root = tempDir('gg-eff-hookless-');
     try {