diff --git a/docker/gateguard-effectiveness/lib.js b/docker/gateguard-effectiveness/lib.js index 228e1a308..30cf59650 100644 --- a/docker/gateguard-effectiveness/lib.js +++ b/docker/gateguard-effectiveness/lib.js @@ -227,6 +227,13 @@ function parseStream(stdout) { return parsed; } +/** Recognize sign-in failure diagnostics, not ordinary authentication discussion. */ +function isAuthenticationFailure(message) { + if (typeof message !== 'string') return false; + return /^\s*(?:(?:API\s+)?Error:\s*)?(?:Not\s+(?:logged|signed)\s+in|Authentication\s+(?:is\s+)?(?:failed|required)|OAuth(?:\s+(?:token|authentication))?\s+(?:failed|expired|invalid|required)|(?:Login|Sign-in)\s+(?:failed|required)|Invalid\s+API\s+key)(?=\s*(?:$|[.:;!()\u00b7-])|[\r\n])/i.test(message) + || /^\s*(?:Please|Must)\s+(?:run\s+\/login|log\s+in|sign\s+in)\b/i.test(message); +} + function readMetrics(stateDir) { const file = path.join(stateDir, 'metrics.jsonl'); if (!fs.existsSync(file)) return []; @@ -461,6 +468,7 @@ module.exports = { childEnv, claudeArgs, parseStream, + isAuthenticationFailure, readMetrics, hookWasObserved, evidenceRecall, diff --git a/docker/gateguard-effectiveness/run-intent.js b/docker/gateguard-effectiveness/run-intent.js index 0fd0f07fa..22c3a5252 100644 --- a/docker/gateguard-effectiveness/run-intent.js +++ b/docker/gateguard-effectiveness/run-intent.js @@ -13,7 +13,7 @@ const crypto = require('crypto'); const os = require('os'); const path = require('path'); const { spawnSync } = require('child_process'); -const { schedule, completedTrialKeys } = require('./lib'); +const { schedule, completedTrialKeys, isAuthenticationFailure } = require('./lib'); const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms'); const { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval'); const { renderHoles, difficulty, isValidTrial } = require('./coverage'); @@ -356,7 +356,7 @@ function main() { fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true }); // An unauthenticated or hookless run must not be recorded: graded as data it // looks like a finding (the first such trial was filed as a coverage hole). - if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) { + if (row.providerError && isAuthenticationFailure(row.providerMessage)) { throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`); } if (!row.hookObserved) { diff --git a/docker/gateguard-effectiveness/run.js b/docker/gateguard-effectiveness/run.js index e0284e943..3c6e9d2e9 100644 --- a/docker/gateguard-effectiveness/run.js +++ b/docker/gateguard-effectiveness/run.js @@ -7,7 +7,7 @@ const os = require('os'); const path = require('path'); const { spawnSync } = require('child_process'); const { ARMS, DEFAULT_ARMS, resolveRef, materializeTree, armSettings } = require('./arms'); -const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys } = require('./lib'); +const { loadScenarios, schedule, runTrial, summarize, renderMarkdown, completedTrialKeys, isAuthenticationFailure } = require('./lib'); const REPO_ROOT = path.join(__dirname, '..', '..'); @@ -202,7 +202,7 @@ function main() { }; fs.writeFileSync(path.join(options.out, row.transcript), transcript); fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true }); - if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) { + if (row.providerError && isAuthenticationFailure(row.providerMessage)) { throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`); } if (!row.hookObserved) throw new Error(`${trial.key}: the gate did not run although the agent edited files; check the hook setup before spending more sessions`); diff --git a/tests/docker/gateguard-effectiveness.test.js b/tests/docker/gateguard-effectiveness.test.js index 48b33f023..9d4e4ba1c 100644 --- a/tests/docker/gateguard-effectiveness.test.js +++ b/tests/docker/gateguard-effectiveness.test.js @@ -317,6 +317,24 @@ if (gnuTarDir) { }); } +test('authentication abort recognizes failure diagnostics rather than ordinary discussion', () => { + const failures = [ + 'Not logged in - Please run /login', 'Authentication failed: OAuth token expired', + 'OAuth authentication failed', 'OAuth token expired; sign in again', + 'Authentication is required', 'Login failed', 'Error: Not signed in.', + 'Invalid API key ยท Please run /login', 'Please run /login', 'Must sign in' + ]; + const ordinary = [ + 'Payment API key authentication uses the Authorization header.', + 'OAuth tokens authorize payment API requests.', 'Login routes use OAuth.', + 'Authentication uses a bearer token.', 'How to use OAuth login', + 'Please explain login flows', 'Authentication failed examples belong in docs', + 'Not logged in errors are covered in tests', '', null + ]; + for (const text of failures) assert.strictEqual(lib.isAuthenticationFailure(text), true, text); + for (const text of ordinary) assert.strictEqual(lib.isAuthenticationFailure(text), false, String(text)); +}); + test('pilot summaries count valid trials only and report provider errors apart', () => { const row = (arm, rep, passed, extra = {}) => ({ key: `s/${arm}/${rep}`, arm, scenario: 's', rep, question: 'q', passed, score: passed ? 1 : 0, evidenceRecall: 0, gateDenials: 0, @@ -442,6 +460,24 @@ if (gitAvailable()) { } finally { fs.rmSync(root, { recursive: true, force: true }); } }); + test('ordinary authentication discussion remains a failed-attempt row on missing result', () => { + for (const text of ['Payment API key authentication uses the Authorization header.', 'OAuth tokens authorize payment API requests.']) { + const root = tempDir('gg-eff-auth-discussion-'); + try { + const scenario = scenarios.find(item => item.id === 'importers-date-shape'); + const events = [{ type: 'assistant', message: { content: [{ type: 'text', text }] } }]; + const { out, run, results } = syntheticPilotRunner(root, scenario, 'off', events); + const result = run({}); + assert.strictEqual(result.status, 0, result.stderr); + assert.strictEqual(results().length, 1); + assert.strictEqual(results()[0].providerError, true); + assert.strictEqual(results()[0].providerMessage, text); + assert.strictEqual(fs.readdirSync(path.join(out, 'transcripts')).length, 1); + assert.ok(!fs.existsSync(path.join(out, '.run.lock'))); + } finally { fs.rmSync(root, { recursive: true, force: true }); } + } + }); + test('the pilot runner stops before recording a gated trial whose hook never ran', () => { const root = tempDir('gg-eff-hookless-'); try {