From 8d969b0ec4a2355b93994098dfe5535ebddbd316 Mon Sep 17 00:00:00 2001 From: haelyra <49814733+haelyra@users.noreply.github.com> Date: Sun, 11 Oct 2026 19:01:56 -0400 Subject: [PATCH] fix: retain authentication diagnostics across empty stream blocks --- docker/gateguard-effectiveness/lib.js | 2 +- docker/gateguard-effectiveness/session.js | 3 ++- tests/docker/gateguard-effectiveness.test.js | 10 ++++++++-- tests/docker/gateguard-intent.test.js | 20 ++++++++++++++++++++ 4 files changed, 31 insertions(+), 4 deletions(-) diff --git a/docker/gateguard-effectiveness/lib.js b/docker/gateguard-effectiveness/lib.js index a533d1a59..228e1a308 100644 --- a/docker/gateguard-effectiveness/lib.js +++ b/docker/gateguard-effectiveness/lib.js @@ -308,7 +308,7 @@ function runTrial(trial, { workRoot, armSettingsByName, executable, model, maxTu providerMessage: stream.result && stream.result.is_error === true ? String(stream.result.result || '').slice(0, 200) : !stream.result - ? String(stream.texts[stream.texts.length - 1] || '').slice(0, 200) || null + ? String([...stream.texts].reverse().find(text => text.trim() !== '') || '').slice(0, 200) || null : null, wallMs: Date.now() - started, passed: outcome.passed, diff --git a/docker/gateguard-effectiveness/session.js b/docker/gateguard-effectiveness/session.js index 0dc1e53f5..880a3eab9 100644 --- a/docker/gateguard-effectiveness/session.js +++ b/docker/gateguard-effectiveness/session.js @@ -115,7 +115,8 @@ function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, option // conversation rather than being graded as a finding. if (!stream.result || stream.result.is_error === true) { providerError = true; - providerMessage = String((stream.result && stream.result.result) || finalText(stream) || '').slice(0, 200) || null; + providerMessage = String((stream.result && stream.result.result) + || [...stream.texts].reverse().find(text => text.trim() !== '') || '').slice(0, 200) || null; } addUsage(usage, stream.result); denialsPerTurn.push(stream.gateDenials); diff --git a/tests/docker/gateguard-effectiveness.test.js b/tests/docker/gateguard-effectiveness.test.js index dca3a5216..48b33f023 100644 --- a/tests/docker/gateguard-effectiveness.test.js +++ b/tests/docker/gateguard-effectiveness.test.js @@ -386,11 +386,15 @@ if (gitAvailable()) { }); test('assistant-only authentication failures retain transcripts and abort before pilot rows', () => { - for (const text of ['Not logged in - Please run /login', 'Authentication failed: OAuth token expired']) { + for (const [text, trailing] of [ + ['Not logged in - Please run /login', null], ['Authentication failed: OAuth token expired', null], + ['Not logged in - Please run /login', ''], ['Authentication failed: OAuth token expired', ' \t '] + ]) { const root = tempDir('gg-eff-auth-'); try { const scenario = scenarios.find(item => item.id === 'importers-date-shape'); const events = [{ type: 'assistant', message: { content: [{ type: 'text', text: 'Preparing' }, { type: 'text', text }] } }]; + if (trailing !== null) events.push({ type: 'assistant', message: { content: [{ type: 'text', text: trailing }] } }); const { out, run, results } = syntheticPilotRunner(root, scenario, 'off', events); const auth = run({}); assert.strictEqual(auth.status, 2, auth.stderr); @@ -418,8 +422,10 @@ if (gitAvailable()) { const text = value => ({ type: 'assistant', message: { content: [{ type: 'text', text: value }] } }); const cases = [ [[text('Provider connection closed')], true, 'Provider connection closed'], - [[text('old'), text('')], true, null], + [[text('old'), text('')], true, 'old'], [[], true, null], + [[text(' \t ')], true, null], + [[text('old'), text(' '), text('Latest diagnostic'), text('')], true, 'Latest diagnostic'], [[text('x'.repeat(250))], true, 'x'.repeat(200)], [[text('Not logged in'), { type: 'result', is_error: false, result: 'Done' }], false, null], [[text('Ignored'), { type: 'result', is_error: true, result: 'Result error' }], true, 'Result error'] diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js index 3edfb006d..54b74e784 100644 --- a/tests/docker/gateguard-intent.test.js +++ b/tests/docker/gateguard-intent.test.js @@ -741,6 +741,26 @@ test('a missing or errored result is a provider error, not a trial', () => { assert.match(result.providerMessage, /log ?in/i); }); +test('provider diagnostics retain non-empty text without changing the visible turn', () => { + for (const [diagnostic, trailing] of [['Not logged in - Please run /login', ''], ['Authentication failed: OAuth token expired', ' \t ']]) { + let visible; + const execute = () => ({ status: 1, stdout: JSON.stringify({ type: 'assistant', message: { + content: [{ type: 'text', text: diagnostic }, { type: 'text', text: trailing }] + } }) }); + const judge = (_intent, message) => { + visible = message; + return { isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false }; + }; + const result = session.runConversation( + { cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT }, + { model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge } + ); + assert.strictEqual(result.providerError, true); + assert.strictEqual(result.providerMessage, diagnostic); + assert.strictEqual(visible, trailing, 'provider diagnostic fallback does not rewrite conversation text'); + } +}); + test('denials are recorded per turn, so the once-per-session latch is visible', () => { let call = 0; const execute = () => {