merge: reviewed last nonblank diagnostics across trial runners

This commit is contained in:
haelyra
2026-10-11 19:03:36 -04:00
4 changed files with 31 additions and 4 deletions
+1 -1
View File
@@ -308,7 +308,7 @@ function runTrial(trial, { workRoot, armSettingsByName, executable, model, maxTu
providerMessage: stream.result && stream.result.is_error === true
? String(stream.result.result || '').slice(0, 200)
: !stream.result
? String(stream.texts[stream.texts.length - 1] || '').slice(0, 200) || null
? String([...stream.texts].reverse().find(text => text.trim() !== '') || '').slice(0, 200) || null
: null,
wallMs: Date.now() - started,
passed: outcome.passed,
+2 -1
View File
@@ -115,7 +115,8 @@ function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, option
// conversation rather than being graded as a finding.
if (!stream.result || stream.result.is_error === true) {
providerError = true;
providerMessage = String((stream.result && stream.result.result) || finalText(stream) || '').slice(0, 200) || null;
providerMessage = String((stream.result && stream.result.result)
|| [...stream.texts].reverse().find(text => text.trim() !== '') || '').slice(0, 200) || null;
}
addUsage(usage, stream.result);
denialsPerTurn.push(stream.gateDenials);
+8 -2
View File
@@ -386,11 +386,15 @@ if (gitAvailable()) {
});
test('assistant-only authentication failures retain transcripts and abort before pilot rows', () => {
for (const text of ['Not logged in - Please run /login', 'Authentication failed: OAuth token expired']) {
for (const [text, trailing] of [
['Not logged in - Please run /login', null], ['Authentication failed: OAuth token expired', null],
['Not logged in - Please run /login', ''], ['Authentication failed: OAuth token expired', ' \t ']
]) {
const root = tempDir('gg-eff-auth-');
try {
const scenario = scenarios.find(item => item.id === 'importers-date-shape');
const events = [{ type: 'assistant', message: { content: [{ type: 'text', text: 'Preparing' }, { type: 'text', text }] } }];
if (trailing !== null) events.push({ type: 'assistant', message: { content: [{ type: 'text', text: trailing }] } });
const { out, run, results } = syntheticPilotRunner(root, scenario, 'off', events);
const auth = run({});
assert.strictEqual(auth.status, 2, auth.stderr);
@@ -418,8 +422,10 @@ if (gitAvailable()) {
const text = value => ({ type: 'assistant', message: { content: [{ type: 'text', text: value }] } });
const cases = [
[[text('Provider connection closed')], true, 'Provider connection closed'],
[[text('old'), text('')], true, null],
[[text('old'), text('')], true, 'old'],
[[], true, null],
[[text(' \t ')], true, null],
[[text('old'), text(' '), text('Latest diagnostic'), text('')], true, 'Latest diagnostic'],
[[text('x'.repeat(250))], true, 'x'.repeat(200)],
[[text('Not logged in'), { type: 'result', is_error: false, result: 'Done' }], false, null],
[[text('Ignored'), { type: 'result', is_error: true, result: 'Result error' }], true, 'Result error']
+20
View File
@@ -741,6 +741,26 @@ test('a missing or errored result is a provider error, not a trial', () => {
assert.match(result.providerMessage, /log ?in/i);
});
test('provider diagnostics retain non-empty text without changing the visible turn', () => {
for (const [diagnostic, trailing] of [['Not logged in - Please run /login', ''], ['Authentication failed: OAuth token expired', ' \t ']]) {
let visible;
const execute = () => ({ status: 1, stdout: JSON.stringify({ type: 'assistant', message: {
content: [{ type: 'text', text: diagnostic }, { type: 'text', text: trailing }]
} }) });
const judge = (_intent, message) => {
visible = message;
return { isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false };
};
const result = session.runConversation(
{ cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT },
{ model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge }
);
assert.strictEqual(result.providerError, true);
assert.strictEqual(result.providerMessage, diagnostic);
assert.strictEqual(visible, trailing, 'provider diagnostic fallback does not rewrite conversation text');
}
});
test('denials are recorded per turn, so the once-per-session latch is visible', () => {
let call = 0;
const execute = () => {