mirror of
https://github.com/affaan-m/ECC.git
synced 2026-10-16 05:05:30 +02:00
merge: reviewed last nonblank diagnostics across trial runners
This commit is contained in:
@@ -308,7 +308,7 @@ function runTrial(trial, { workRoot, armSettingsByName, executable, model, maxTu
|
||||
providerMessage: stream.result && stream.result.is_error === true
|
||||
? String(stream.result.result || '').slice(0, 200)
|
||||
: !stream.result
|
||||
? String(stream.texts[stream.texts.length - 1] || '').slice(0, 200) || null
|
||||
? String([...stream.texts].reverse().find(text => text.trim() !== '') || '').slice(0, 200) || null
|
||||
: null,
|
||||
wallMs: Date.now() - started,
|
||||
passed: outcome.passed,
|
||||
|
||||
@@ -115,7 +115,8 @@ function runConversation({ cwd, stateDir, settingsPath, prompt, intent }, option
|
||||
// conversation rather than being graded as a finding.
|
||||
if (!stream.result || stream.result.is_error === true) {
|
||||
providerError = true;
|
||||
providerMessage = String((stream.result && stream.result.result) || finalText(stream) || '').slice(0, 200) || null;
|
||||
providerMessage = String((stream.result && stream.result.result)
|
||||
|| [...stream.texts].reverse().find(text => text.trim() !== '') || '').slice(0, 200) || null;
|
||||
}
|
||||
addUsage(usage, stream.result);
|
||||
denialsPerTurn.push(stream.gateDenials);
|
||||
|
||||
@@ -386,11 +386,15 @@ if (gitAvailable()) {
|
||||
});
|
||||
|
||||
test('assistant-only authentication failures retain transcripts and abort before pilot rows', () => {
|
||||
for (const text of ['Not logged in - Please run /login', 'Authentication failed: OAuth token expired']) {
|
||||
for (const [text, trailing] of [
|
||||
['Not logged in - Please run /login', null], ['Authentication failed: OAuth token expired', null],
|
||||
['Not logged in - Please run /login', ''], ['Authentication failed: OAuth token expired', ' \t ']
|
||||
]) {
|
||||
const root = tempDir('gg-eff-auth-');
|
||||
try {
|
||||
const scenario = scenarios.find(item => item.id === 'importers-date-shape');
|
||||
const events = [{ type: 'assistant', message: { content: [{ type: 'text', text: 'Preparing' }, { type: 'text', text }] } }];
|
||||
if (trailing !== null) events.push({ type: 'assistant', message: { content: [{ type: 'text', text: trailing }] } });
|
||||
const { out, run, results } = syntheticPilotRunner(root, scenario, 'off', events);
|
||||
const auth = run({});
|
||||
assert.strictEqual(auth.status, 2, auth.stderr);
|
||||
@@ -418,8 +422,10 @@ if (gitAvailable()) {
|
||||
const text = value => ({ type: 'assistant', message: { content: [{ type: 'text', text: value }] } });
|
||||
const cases = [
|
||||
[[text('Provider connection closed')], true, 'Provider connection closed'],
|
||||
[[text('old'), text('')], true, null],
|
||||
[[text('old'), text('')], true, 'old'],
|
||||
[[], true, null],
|
||||
[[text(' \t ')], true, null],
|
||||
[[text('old'), text(' '), text('Latest diagnostic'), text('')], true, 'Latest diagnostic'],
|
||||
[[text('x'.repeat(250))], true, 'x'.repeat(200)],
|
||||
[[text('Not logged in'), { type: 'result', is_error: false, result: 'Done' }], false, null],
|
||||
[[text('Ignored'), { type: 'result', is_error: true, result: 'Result error' }], true, 'Result error']
|
||||
|
||||
@@ -741,6 +741,26 @@ test('a missing or errored result is a provider error, not a trial', () => {
|
||||
assert.match(result.providerMessage, /log ?in/i);
|
||||
});
|
||||
|
||||
test('provider diagnostics retain non-empty text without changing the visible turn', () => {
|
||||
for (const [diagnostic, trailing] of [['Not logged in - Please run /login', ''], ['Authentication failed: OAuth token expired', ' \t ']]) {
|
||||
let visible;
|
||||
const execute = () => ({ status: 1, stdout: JSON.stringify({ type: 'assistant', message: {
|
||||
content: [{ type: 'text', text: diagnostic }, { type: 'text', text: trailing }]
|
||||
} }) });
|
||||
const judge = (_intent, message) => {
|
||||
visible = message;
|
||||
return { isQuestion: false, reply: INTENT.stonewall, disclosed: [], judgeFailed: false };
|
||||
};
|
||||
const result = session.runConversation(
|
||||
{ cwd: '.', stateDir: '.', settingsPath: 's', prompt: 'p', intent: INTENT },
|
||||
{ model: 'm', maxTurns: 5, timeoutMs: 1000, userTurns: 1, execute, judge }
|
||||
);
|
||||
assert.strictEqual(result.providerError, true);
|
||||
assert.strictEqual(result.providerMessage, diagnostic);
|
||||
assert.strictEqual(visible, trailing, 'provider diagnostic fallback does not rewrite conversation text');
|
||||
}
|
||||
});
|
||||
|
||||
test('denials are recorded per turn, so the once-per-session latch is visible', () => {
|
||||
let call = 0;
|
||||
const execute = () => {
|
||||
|
||||
Reference in New Issue
Block a user