diff --git a/docker/gateguard-effectiveness/lib.js b/docker/gateguard-effectiveness/lib.js index b166741f3..a533d1a59 100644 --- a/docker/gateguard-effectiveness/lib.js +++ b/docker/gateguard-effectiveness/lib.js @@ -305,7 +305,11 @@ function runTrial(trial, { workRoot, armSettingsByName, executable, model, maxTu exitStatus: run.status, timedOut: Boolean(run.error && run.error.code === 'ETIMEDOUT'), providerError: !stream.result || stream.result.is_error === true, - providerMessage: stream.result && stream.result.is_error === true ? String(stream.result.result || '').slice(0, 200) : null, + providerMessage: stream.result && stream.result.is_error === true + ? String(stream.result.result || '').slice(0, 200) + : !stream.result + ? String(stream.texts[stream.texts.length - 1] || '').slice(0, 200) || null + : null, wallMs: Date.now() - started, passed: outcome.passed, score: outcome.score, diff --git a/tests/docker/gateguard-effectiveness.test.js b/tests/docker/gateguard-effectiveness.test.js index 68a5834af..dca3a5216 100644 --- a/tests/docker/gateguard-effectiveness.test.js +++ b/tests/docker/gateguard-effectiveness.test.js @@ -339,14 +339,16 @@ test('pilot summaries count valid trials only and report provider errors apart', }); /** The pilot runner driven end to end with fake-claude.js as the provider. */ -function syntheticPilotRunner(root, scenario, arms) { +function syntheticPilotRunner(root, scenario, arms, streamEvents = null) { const out = path.join(root, 'out'); const preload = path.join(root, 'preload.js'); fs.writeFileSync(preload, ` const cp = require('child_process'); const originalSpawn = cp.spawnSync; cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude' - ? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options) + ? ${streamEvents === null + ? `originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)` + : JSON.stringify({ status: 0, stdout: streamEvents.map(event => JSON.stringify(event)).join('\n') + '\n', stderr: '' })} : originalSpawn(file, args, options); `); const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', arms, '--reps', '1', '--scenarios', scenario.id]; @@ -383,6 +385,57 @@ if (gitAvailable()) { } finally { fs.rmSync(root, { recursive: true, force: true }); } }); + test('assistant-only authentication failures retain transcripts and abort before pilot rows', () => { + for (const text of ['Not logged in - Please run /login', 'Authentication failed: OAuth token expired']) { + const root = tempDir('gg-eff-auth-'); + try { + const scenario = scenarios.find(item => item.id === 'importers-date-shape'); + const events = [{ type: 'assistant', message: { content: [{ type: 'text', text: 'Preparing' }, { type: 'text', text }] } }]; + const { out, run, results } = syntheticPilotRunner(root, scenario, 'off', events); + const auth = run({}); + assert.strictEqual(auth.status, 2, auth.stderr); + assert.match(auth.stderr, /not signed in/); + assert.deepStrictEqual(results(), [], 'authentication is not recorded as a completed or failed trial row'); + const files = fs.readdirSync(path.join(out, 'transcripts')); + assert.strictEqual(files.length, 1); + const retained = fs.readFileSync(path.join(out, 'transcripts', files[0]), 'utf8'); + assert.ok(retained.includes(text)); + assert.ok(!fs.existsSync(path.join(out, '.run.lock'))); + const resumed = syntheticPilotRunner(root, scenario, 'off').run({}); + assert.strictEqual(resumed.status, 0, resumed.stderr); + assert.strictEqual(results().length, 1); + assert.strictEqual(results()[0].providerError, false); + assert.strictEqual(fs.readdirSync(path.join(out, 'transcripts')).length, 2); + assert.strictEqual(fs.readFileSync(path.join(out, 'transcripts', files[0]), 'utf8'), retained); + } finally { fs.rmSync(root, { recursive: true, force: true }); } + } + }); + + test('missing-result messages preserve ordinary failures, empty text and result mapping', () => { + const root = tempDir('gg-eff-messages-'); + try { + const scenario = scenarios.find(item => item.id === 'importers-date-shape'); + const text = value => ({ type: 'assistant', message: { content: [{ type: 'text', text: value }] } }); + const cases = [ + [[text('Provider connection closed')], true, 'Provider connection closed'], + [[text('old'), text('')], true, null], + [[], true, null], + [[text('x'.repeat(250))], true, 'x'.repeat(200)], + [[text('Not logged in'), { type: 'result', is_error: false, result: 'Done' }], false, null], + [[text('Ignored'), { type: 'result', is_error: true, result: 'Result error' }], true, 'Result error'] + ]; + for (const [events, error, message] of cases) { + const row = lib.runTrial({ key: 'mapping/off/1', scenario, arm: 'off', rep: 1 }, { + workRoot: root, armSettingsByName: { off: arms.armSettings(null) }, executable: 'controlled', + model: 'fake', maxTurns: 1, timeoutMs: 1000, + execute: () => ({ status: 0, stdout: events.map(event => JSON.stringify(event)).join('\n'), stderr: '' }) + }); + assert.strictEqual(row.providerError, error); + assert.strictEqual(row.providerMessage, message); + } + } finally { fs.rmSync(root, { recursive: true, force: true }); } + }); + test('the pilot runner stops before recording a gated trial whose hook never ran', () => { const root = tempDir('gg-eff-hookless-'); try { diff --git a/tests/lib/state-store-concurrency.test.js b/tests/lib/state-store-concurrency.test.js index 79d137b64..de246ef9b 100644 --- a/tests/lib/state-store-concurrency.test.js +++ b/tests/lib/state-store-concurrency.test.js @@ -127,12 +127,17 @@ async function run() { await withWorkers(3, async workers => { // IPC acknowledgements form a barrier: every handle exists before any write. await Promise.all(workers.map(worker => worker.request('open', { dbPath }))); - await Promise.all(workers.map((worker, index) => worker.request('write', { worker: `worker-${index}`, count: 4 }))); + // One operation per request retains contention without assuming lock fairness + // for greedy batches beyond the documented five-second acquisition budget. + for (let round = 0; round < 4; round += 1) { + await Promise.all(workers.map((worker, index) => + worker.request('write', { worker: `worker-${index}-${round}`, count: 1 }))); + } await Promise.all(workers.map(worker => worker.request('close'))); }); const actual = (await readItems(dbPath)).map(item => item.id).sort(); const expected = Array.from({ length: 3 }, (_, worker) => - Array.from({ length: 4 }, (_, item) => `worker-${worker}-${item}`)).flat().sort(); + Array.from({ length: 4 }, (_, item) => `worker-${worker}-${item}-0`)).flat().sort(); assert.deepStrictEqual(actual, expected, 'Every acknowledged task must survive all worker exits'); }); diff --git a/tests/test_agent_self_evaluation.py b/tests/test_agent_self_evaluation.py index 9e245ac34..894d71213 100644 --- a/tests/test_agent_self_evaluation.py +++ b/tests/test_agent_self_evaluation.py @@ -122,11 +122,11 @@ def test_task_read_does_not_hide_other_file_errors(evaluator: ModuleType, monkey @pytest.fixture(params=[SCRIPT, PI_SCRIPT], ids=["canonical", "pi"]) -def evaluator(request: pytest.FixtureRequest) -> ModuleType: +def evaluator(request: pytest.FixtureRequest, monkeypatch: pytest.MonkeyPatch) -> ModuleType: spec = importlib.util.spec_from_file_location("task_input_evaluator", request.param) assert spec is not None and spec.loader is not None module = importlib.util.module_from_spec(spec) - sys.modules[spec.name] = module + monkeypatch.setitem(sys.modules, spec.name, module) spec.loader.exec_module(module) return module