mirror of
https://github.com/affaan-m/ECC.git
synced 2026-10-16 05:05:30 +02:00
502 lines
25 KiB
JavaScript
502 lines
25 KiB
JavaScript
/**
|
|
* Tests for docker/gateguard-effectiveness: scenarios, graders, arms and analysis. No model is called.
|
|
*/
|
|
|
|
'use strict';
|
|
|
|
const assert = require('assert');
|
|
const fs = require('fs');
|
|
const os = require('os');
|
|
const path = require('path');
|
|
const { spawnSync } = require('child_process');
|
|
|
|
const ROOT = path.join(__dirname, '..', '..');
|
|
const DIR = path.join(ROOT, 'docker', 'gateguard-effectiveness');
|
|
const HOOK = path.join(ROOT, 'scripts', 'hooks', 'gateguard-fact-force.js');
|
|
const lib = require(path.join(DIR, 'lib'));
|
|
const arms = require(path.join(DIR, 'arms'));
|
|
const { applyArm, armQuestionIds, PLACEBO_TEXT } = require(path.join(DIR, 'arm-patch'));
|
|
const { parseArgs, readOptionalText, writeMetadata, withOutputLock } = require(path.join(DIR, 'run'));
|
|
|
|
let passed = 0;
|
|
let failed = 0;
|
|
|
|
function test(name, fn) {
|
|
try {
|
|
fn();
|
|
console.log(` ✓ ${name}`);
|
|
passed++;
|
|
} catch (error) {
|
|
console.log(` ✗ ${name}`);
|
|
console.log(` Error: ${error.message}`);
|
|
failed++;
|
|
}
|
|
}
|
|
|
|
function tempDir(prefix) {
|
|
return fs.realpathSync(fs.mkdtempSync(path.join(os.tmpdir(), prefix)));
|
|
}
|
|
|
|
function gradeWith(scenario, overlay) {
|
|
const dir = tempDir('gg-eff-grade-');
|
|
try {
|
|
lib.prepareWorkspace(scenario, dir, overlay);
|
|
return lib.grade(scenario, dir);
|
|
} finally {
|
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
}
|
|
|
|
function gitAvailable() {
|
|
const result = spawnSync('git', ['--version'], { encoding: 'utf8' });
|
|
return result.status === 0 && !result.error;
|
|
}
|
|
|
|
/** Directory of the GNU tar that Git for Windows bundles, or null. GNU tar reads `C:\...` as a remote host:path. */
|
|
function gitBundledGnuTarDir() {
|
|
if (process.platform !== 'win32') return null;
|
|
const execPath = spawnSync('git', ['--exec-path'], { encoding: 'utf8' });
|
|
if (execPath.status !== 0 || execPath.error) return null;
|
|
// <git>/mingw64/libexec/git-core -> <git>/usr/bin
|
|
const dir = path.resolve(execPath.stdout.trim(), '..', '..', '..', 'usr', 'bin');
|
|
const version = spawnSync(path.join(dir, 'tar.exe'), ['--version'], { encoding: 'utf8' });
|
|
return version.status === 0 && /GNU tar/.test(version.stdout) ? dir : null;
|
|
}
|
|
|
|
console.log('\n=== Testing gateguard-effectiveness ===\n');
|
|
|
|
const scenarios = lib.loadScenarios();
|
|
|
|
test('six scenarios load with a target question each', () => {
|
|
assert.strictEqual(scenarios.length, 6);
|
|
for (const scenario of scenarios) assert.ok(scenario.targetQuestions.includes(scenario.question), scenario.id);
|
|
});
|
|
|
|
for (const scenario of scenarios) {
|
|
test(`${scenario.id}: the grader fails the start and naive workspaces and passes the reference`, () => {
|
|
const start = gradeWith(scenario, null);
|
|
const naive = gradeWith(scenario, 'naive');
|
|
const reference = gradeWith(scenario, 'reference');
|
|
assert.strictEqual(start.passed, false, `start ${start.score}`);
|
|
assert.strictEqual(naive.passed, false, `naive ${naive.score}`);
|
|
assert.deepStrictEqual(reference, { passed: true, score: 1 });
|
|
assert.ok(naive.score < 1 && start.score < 1);
|
|
});
|
|
|
|
test(`${scenario.id}: the gate asks a target question for the reference edits`, () => {
|
|
const asked = new Set(lib.questionsForReference(scenario, HOOK).flatMap(event => event.questions));
|
|
assert.ok(scenario.targetQuestions.some(question => asked.has(question)), `asked ${[...asked].join(', ')}`);
|
|
});
|
|
}
|
|
|
|
test('dotfiles are stored under dot- names and restored in the workspace', () => {
|
|
const scenario = scenarios.find(item => item.id === 'secrets-config-key');
|
|
const dir = tempDir('gg-eff-dot-');
|
|
try {
|
|
lib.prepareWorkspace(scenario, dir);
|
|
for (const name of ['.env', '.env.example', '.gitignore']) assert.ok(fs.existsSync(path.join(dir, name)), name);
|
|
assert.ok(!fs.readdirSync(dir).some(name => name.startsWith('dot-')));
|
|
} finally {
|
|
fs.rmSync(dir, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
function fakeTargetClass() {
|
|
return {
|
|
questionIdsFor: (cls, isWrite) => (cls === 'code' ? ['importers', 'public-api', 'data-schema', 'quote-instruction'] : isWrite ? ['config-reader', 'quote-instruction'] : ['config-effect', 'quote-instruction']),
|
|
questionText: id => `text:${id}`,
|
|
condensedQuestionPhrase: id => `phrase:${id}`,
|
|
condensedHintFor: cls => `original:${cls}`
|
|
};
|
|
}
|
|
|
|
test('dropping a question removes only that question and keeps other classes byte-identical', () => {
|
|
const patched = applyArm({ drop: ['importers'] }, fakeTargetClass());
|
|
assert.deepStrictEqual(patched.questionIdsFor('code', false), ['public-api', 'data-schema', 'quote-instruction']);
|
|
assert.strictEqual(patched.condensedHintFor('config', false), 'original:config');
|
|
assert.ok(!patched.condensedHintFor('code', false).includes('importers'));
|
|
assert.strictEqual(patched.questionText('public-api'), 'text:public-api');
|
|
});
|
|
|
|
test('the placebo keeps the question count and the verbatim-instruction question', () => {
|
|
const original = fakeTargetClass();
|
|
const ids = original.questionIdsFor('code', false);
|
|
const placebo = armQuestionIds({ placebo: true }, ids);
|
|
assert.strictEqual(placebo.length, ids.length);
|
|
assert.strictEqual(placebo[placebo.length - 1], 'quote-instruction');
|
|
assert.ok(placebo.slice(0, -1).every(id => Object.hasOwn(PLACEBO_TEXT, id)));
|
|
const patched = applyArm({ placebo: true }, fakeTargetClass());
|
|
assert.strictEqual(patched.questionText(placebo[0]), PLACEBO_TEXT[placebo[0]]);
|
|
assert.ok(patched.condensedHintFor('code', false).endsWith("the user's verbatim instruction, then retry."));
|
|
});
|
|
|
|
test('no patch leaves the module untouched', () => {
|
|
const original = fakeTargetClass();
|
|
const before = { ...original };
|
|
applyArm(null, original);
|
|
assert.deepStrictEqual(Object.keys(original), Object.keys(before));
|
|
assert.strictEqual(original.questionIdsFor, before.questionIdsFor);
|
|
});
|
|
|
|
test('trial patches follow the arm', () => {
|
|
const scenario = scenarios.find(item => item.id === 'importers-date-shape');
|
|
assert.strictEqual(arms.trialPatch('gate', scenario), null);
|
|
assert.strictEqual(arms.trialPatch('off', scenario), null);
|
|
assert.strictEqual(arms.trialPatch('main', scenario), null);
|
|
assert.deepStrictEqual(arms.trialPatch('minus-target', scenario), { drop: ['importers', 'local-callers'] });
|
|
assert.deepStrictEqual(arms.trialPatch('placebo', scenario), { placebo: true });
|
|
assert.throws(() => arms.trialPatch('nope', scenario), /unknown arm/);
|
|
});
|
|
|
|
test('the off arm registers no hooks and gate arms register the edit and shell matchers', () => {
|
|
assert.deepStrictEqual(arms.armSettings(null), { hooks: {} });
|
|
const settings = arms.armSettings(path.join(os.tmpdir(), 'tree'));
|
|
assert.deepStrictEqual(settings.hooks.PreToolUse.map(entry => entry.matcher), ['Edit|Write|MultiEdit|NotebookEdit', 'Bash|PowerShell']);
|
|
assert.ok(settings.hooks.PreToolUse.every(entry => entry.hooks[0].command.includes(arms.ARM_HOOK) && !entry.hooks[0].command.includes('\\')));
|
|
});
|
|
|
|
test('the schedule runs every arm once per scenario and rep, in a seeded order', () => {
|
|
const trials = lib.schedule(scenarios, arms.DEFAULT_ARMS, 3, 11);
|
|
assert.strictEqual(trials.length, scenarios.length * arms.DEFAULT_ARMS.length * 3);
|
|
assert.strictEqual(new Set(trials.map(trial => trial.key)).size, trials.length);
|
|
assert.deepStrictEqual(lib.schedule(scenarios, arms.DEFAULT_ARMS, 3, 11).map(trial => trial.key), trials.map(trial => trial.key));
|
|
const firstBlock = trials.slice(0, arms.DEFAULT_ARMS.length);
|
|
assert.deepStrictEqual(firstBlock.map(trial => trial.arm).sort(), arms.DEFAULT_ARMS.slice().sort());
|
|
});
|
|
|
|
test('the child environment drops harness and hook variables but keeps the OAuth token', () => {
|
|
const saved = { ...process.env };
|
|
try {
|
|
Object.assign(process.env, { CLAUDE_PLUGIN_ROOT: '/x', ECC_GATEGUARD: 'off', GATEGUARD_DISABLED: '1', CLAUDE_CODE_OAUTH_TOKEN: 'token' });
|
|
const env = lib.childEnv('/state');
|
|
assert.strictEqual(env.CLAUDE_PLUGIN_ROOT, undefined);
|
|
assert.strictEqual(env.ECC_GATEGUARD, undefined);
|
|
assert.strictEqual(env.GATEGUARD_DISABLED, undefined);
|
|
assert.strictEqual(env.CLAUDE_CODE_OAUTH_TOKEN, 'token');
|
|
assert.strictEqual(env.GATEGUARD_STATE_DIR, '/state');
|
|
} finally {
|
|
for (const key of Object.keys(process.env)) if (!(key in saved)) delete process.env[key];
|
|
Object.assign(process.env, saved);
|
|
}
|
|
});
|
|
|
|
test('the stream parser counts tools, gate denials and usage', () => {
|
|
const lines = [
|
|
{ type: 'assistant', message: { content: [{ type: 'text', text: 'I will check csv.js' }, { type: 'tool_use', name: 'Grep', input: {} }, { type: 'tool_use', name: 'Edit', input: {} }, { type: 'tool_use', name: 'Bash', input: {} }] } },
|
|
{ type: 'user', message: { content: [{ type: 'tool_result', is_error: true, content: [{ type: 'text', text: '[Fact-Forcing Gate]\n\nBefore editing' }] }] } },
|
|
{ type: 'user', message: { content: [{ type: 'tool_result', content: 'ok' }] } },
|
|
'not json',
|
|
{ type: 'result', is_error: false, num_turns: 4, usage: { output_tokens: 9 } }
|
|
].map(line => (typeof line === 'string' ? line : JSON.stringify(line))).join('\n');
|
|
const parsed = lib.parseStream(lines);
|
|
assert.deepStrictEqual(parsed.tools, { Grep: 1, Edit: 1, Bash: 1 });
|
|
assert.strictEqual(parsed.editCalls, 1);
|
|
assert.strictEqual(parsed.shellCalls, 1);
|
|
assert.strictEqual(parsed.gateDenials, 1);
|
|
assert.strictEqual(parsed.result.num_turns, 4);
|
|
assert.deepStrictEqual(parsed.texts, ['I will check csv.js']);
|
|
});
|
|
|
|
test('evidence recall counts the patterns the agent states', () => {
|
|
const scenario = scenarios.find(item => item.id === 'importers-date-shape');
|
|
assert.strictEqual(lib.evidenceRecall(scenario, ['updated src/report.js and lib/export/csv.js']), 2 / 3);
|
|
assert.strictEqual(lib.evidenceRecall(scenario, []), 0);
|
|
});
|
|
|
|
test('the exact McNemar test matches known values', () => {
|
|
assert.strictEqual(lib.binomialTwoSided(0, 0), 1);
|
|
assert.ok(Math.abs(lib.binomialTwoSided(0, 6) - 0.03125) < 1e-12);
|
|
assert.ok(Math.abs(lib.binomialTwoSided(2, 8) - 0.2890625) < 1e-12);
|
|
});
|
|
|
|
function row(arm, scenario, rep, passedTrial) {
|
|
return { arm, scenario, rep, question: 'q', passed: passedTrial, score: passedTrial ? 1 : 0, evidenceRecall: 0, gateDenials: 0, inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, costUsd: null, turns: 1, wallMs: 1, providerError: false, timedOut: false };
|
|
}
|
|
|
|
test('paired comparisons count discordant pairs and bound the difference', () => {
|
|
const rows = [];
|
|
for (let rep = 1; rep <= 6; rep++) {
|
|
rows.push(row('gate', 's', rep, true), row('off', 's', rep, rep > 5));
|
|
}
|
|
const comparison = lib.pairedComparison(rows, 'gate', 'off');
|
|
assert.strictEqual(comparison.pairs, 6);
|
|
assert.strictEqual(comparison.onlyReference, 5);
|
|
assert.strictEqual(comparison.onlyArm, 0);
|
|
assert.ok(Math.abs(comparison.passRateDifference + 5 / 6) < 1e-12);
|
|
assert.ok(comparison.interval[1] < 0);
|
|
const markdown = lib.renderMarkdown(lib.summarize(rows, ['gate', 'off']), { model: 'm', sha: 'abc', reps: 6 });
|
|
assert.ok(markdown.includes('| off | 6 | -83%'), markdown);
|
|
});
|
|
|
|
test('metadata publication replaces a symlink entry without touching its victim', () => {
|
|
const out = tempDir('gg-eff-meta-');
|
|
try {
|
|
const victim = path.join(out, 'victim');
|
|
const meta = path.join(out, 'meta.json');
|
|
fs.writeFileSync(victim, 'VICTIM_BYTES');
|
|
try { fs.symlinkSync(victim, meta); }
|
|
catch (error) {
|
|
if (process.platform !== 'win32') throw error;
|
|
fs.writeFileSync(meta, 'old');
|
|
}
|
|
writeMetadata(meta, { model: 'fake', sha: 'fixture' });
|
|
assert.strictEqual(fs.readFileSync(victim, 'utf8'), 'VICTIM_BYTES');
|
|
const fd = fs.openSync(meta, fs.constants.O_RDONLY | (fs.constants.O_NOFOLLOW || 0));
|
|
try {
|
|
assert.strictEqual(fs.fstatSync(fd).isFile(), true);
|
|
assert.deepStrictEqual(JSON.parse(fs.readFileSync(fd, 'utf8')), { model: 'fake', sha: 'fixture' });
|
|
} finally { fs.closeSync(fd); }
|
|
assert.ok(!fs.readdirSync(out).some(file => file.endsWith('.tmp')));
|
|
} finally { fs.rmSync(out, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('same-output workers are rejected and the lock is released on errors', () => {
|
|
const out = tempDir('gg-eff-lock-');
|
|
try {
|
|
assert.throws(() => withOutputLock(out, () => {
|
|
assert.throws(() => withOutputLock(out, () => assert.fail('second worker started')), /concurrent workers/);
|
|
throw new Error('trial failure');
|
|
}), /trial failure/);
|
|
assert.deepStrictEqual(fs.readdirSync(out), []);
|
|
assert.strictEqual(withOutputLock(out, () => 'resumed'), 'resumed');
|
|
assert.deepStrictEqual(fs.readdirSync(out), []);
|
|
} finally { fs.rmSync(out, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('arguments require an output folder, a model and the real-provider opt-in', () => {
|
|
assert.throws(() => parseArgs([]), /--out is required/);
|
|
assert.throws(() => parseArgs(['--out', 'x']), /--model is required/);
|
|
assert.throws(() => parseArgs(['--out', 'x', '--model', 'm']), /--allow-real-provider/);
|
|
assert.throws(() => parseArgs(['--out', 'x', '--arms', 'gate,nope', '--dry-run']), /unknown arm/);
|
|
const options = parseArgs(['--out', 'x', '--model', 'm', '--allow-real-provider', '--reps', '2', '--scenarios', 'a,b']);
|
|
assert.strictEqual(options.reps, 2);
|
|
assert.deepStrictEqual(options.scenarios, ['a', 'b']);
|
|
assert.deepStrictEqual(options.arms, arms.DEFAULT_ARMS);
|
|
});
|
|
|
|
test('materializeTree hands tar no host path, only a relative archive name run inside dest', () => {
|
|
const work = tempDir('gg-eff-tarargs-');
|
|
try {
|
|
const dest = path.join(work, 'tree');
|
|
const calls = [];
|
|
const spawn = (command, args, options) => {
|
|
calls.push({ command, args, cwd: options.cwd });
|
|
if (command === 'git') fs.writeFileSync(args[args.indexOf('-o') + 1], '');
|
|
if (command === 'tar') fs.mkdirSync(path.join(dest, 'scripts', 'hooks'), { recursive: true });
|
|
return { status: 0, stdout: '', stderr: '' };
|
|
};
|
|
arms.materializeTree(ROOT, 'deadbeef', dest, spawn);
|
|
const tar = calls.find(call => call.command === 'tar');
|
|
assert.strictEqual(tar.cwd, dest);
|
|
for (const arg of tar.args) {
|
|
assert.ok(!path.isAbsolute(arg) && !/^[A-Za-z]:/.test(arg), `tar argument looks like a host path: ${arg}`);
|
|
}
|
|
} finally {
|
|
fs.rmSync(work, { recursive: true, force: true });
|
|
}
|
|
});
|
|
|
|
const gnuTarDir = gitAvailable() ? gitBundledGnuTarDir() : null;
|
|
if (gnuTarDir) {
|
|
test('materializeTree extracts into a drive-letter destination when GNU tar is first on PATH', () => {
|
|
const work = tempDir('gg-eff-tar-');
|
|
const savedPath = process.env.PATH;
|
|
try {
|
|
const dest = path.join(work, 'tree');
|
|
assert.match(dest, /^[A-Za-z]:\\/);
|
|
process.env.PATH = `${gnuTarDir}${path.delimiter}${savedPath}`;
|
|
const tree = arms.materializeTree(ROOT, arms.resolveRef(ROOT, 'HEAD'), dest);
|
|
assert.strictEqual(tree, dest);
|
|
assert.ok(fs.existsSync(path.join(dest, 'scripts', 'hooks', 'gateguard-fact-force.js')));
|
|
assert.ok(fs.existsSync(path.join(dest, ...arms.ARM_HOOK.split('/'))));
|
|
assert.ok(!fs.existsSync(path.join(dest, 'scripts.tar')), 'the archive is removed after extraction');
|
|
} finally {
|
|
process.env.PATH = savedPath;
|
|
fs.rmSync(work, { recursive: true, force: true });
|
|
}
|
|
});
|
|
}
|
|
|
|
test('pilot summaries count valid trials only and report provider errors apart', () => {
|
|
const row = (arm, rep, passed, extra = {}) => ({
|
|
key: `s/${arm}/${rep}`, arm, scenario: 's', rep, question: 'q', passed, score: passed ? 1 : 0, evidenceRecall: 0, gateDenials: 0,
|
|
inputTokens: 1, cachedInputTokens: 0, outputTokens: 1, costUsd: null, turns: 1, wallMs: 1, providerError: false, timedOut: false, ...extra
|
|
});
|
|
const rows = [
|
|
row('gate', 1, true),
|
|
row('gate', 2, false, { providerError: true }),
|
|
row('gate', 2, true),
|
|
row('gate', 3, false, { timedOut: true }),
|
|
row('gate', 4, false, { judgeFailed: true }),
|
|
row('off', 1, true)
|
|
];
|
|
const gate = lib.summarize(rows, ['gate', 'off']).arms.find(arm => arm.arm === 'gate');
|
|
assert.strictEqual(gate.trials, 2, 'a provider error or timeout is not a trial');
|
|
assert.strictEqual(gate.passes, 2);
|
|
assert.strictEqual(gate.meanScore, 1);
|
|
assert.strictEqual(gate.providerErrors, 2);
|
|
assert.deepStrictEqual([...lib.completedTrialKeys(rows)].sort(), ['s/gate/1', 's/gate/2', 's/off/1']);
|
|
});
|
|
|
|
/** The pilot runner driven end to end with fake-claude.js as the provider. */
|
|
function syntheticPilotRunner(root, scenario, arms) {
|
|
const out = path.join(root, 'out');
|
|
const preload = path.join(root, 'preload.js');
|
|
fs.writeFileSync(preload, `
|
|
const cp = require('child_process');
|
|
const originalSpawn = cp.spawnSync;
|
|
cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude'
|
|
? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)
|
|
: originalSpawn(file, args, options);
|
|
`);
|
|
const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', arms, '--reps', '1', '--scenarios', scenario.id];
|
|
const env = { ...process.env, FAKE_CLAUDE_OVERLAY: path.join(scenario.root, 'reference') };
|
|
const run = extraEnv => spawnSync(process.execPath, args, { env: { ...env, ...extraEnv }, encoding: 'utf8', timeout: 60000 });
|
|
const results = () => {
|
|
const file = path.join(out, 'results.jsonl');
|
|
return fs.existsSync(file) ? fs.readFileSync(file, 'utf8').trim().split('\n').filter(Boolean).map(line => JSON.parse(line)) : [];
|
|
};
|
|
return { out, run, results };
|
|
}
|
|
|
|
if (gitAvailable()) {
|
|
test('a pilot trial that ended in a provider error is retried on resume', () => {
|
|
const root = tempDir('gg-eff-retry-');
|
|
try {
|
|
const scenario = scenarios.find(item => item.id === 'importers-date-shape');
|
|
const { out, run, results } = syntheticPilotRunner(root, scenario, 'off');
|
|
const failed = run({ FAKE_CLAUDE_PROVIDER_ERROR: '1' });
|
|
assert.strictEqual(failed.status, 0, failed.stderr);
|
|
assert.deepStrictEqual(results().map(row => row.providerError), [true]);
|
|
const retried = run({});
|
|
assert.strictEqual(retried.status, 0, retried.stderr);
|
|
assert.ok(retried.stderr.includes('1 trials to run'), retried.stderr);
|
|
const rows = results();
|
|
assert.deepStrictEqual(rows.map(row => row.providerError), [true, false], 'the failed attempt stays for audit');
|
|
assert.strictEqual(rows[1].passed, true);
|
|
assert.notStrictEqual(rows[0].transcript, rows[1].transcript, 'each attempt keeps its own transcript');
|
|
const { summary } = JSON.parse(fs.readFileSync(path.join(out, 'summary.json'), 'utf8'));
|
|
const off = summary.arms.find(arm => arm.arm === 'off');
|
|
assert.deepStrictEqual([off.trials, off.passes, off.providerErrors], [1, 1, 1]);
|
|
const again = run({});
|
|
assert.ok(again.stderr.includes('0 trials to run'), again.stderr);
|
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('the pilot runner stops before recording a gated trial whose hook never ran', () => {
|
|
const root = tempDir('gg-eff-hookless-');
|
|
try {
|
|
const scenario = scenarios.find(item => item.id === 'importers-date-shape');
|
|
const { run, results } = syntheticPilotRunner(root, scenario, 'gate');
|
|
const result = run({ FAKE_CLAUDE_SKIP_HOOK: '1' });
|
|
assert.notStrictEqual(result.status, 0, 'a hookless gated trial must stop the run');
|
|
assert.match(result.stderr, /the gate did not run although the agent edited files/);
|
|
assert.deepStrictEqual(results(), [], 'the hookless trial is not recorded as data');
|
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('actual fake-provider CLI resumes identity and resists a metadata publication symlink race', () => {
|
|
const root = tempDir('gg-eff-cli-');
|
|
const out = path.join(root, 'out');
|
|
const victim = path.join(root, 'victim');
|
|
const preload = path.join(root, 'preload.js');
|
|
const scenario = scenarios.find(item => item.id === 'importers-date-shape');
|
|
try {
|
|
fs.writeFileSync(victim, 'VICTIM_BYTES');
|
|
fs.writeFileSync(preload, `
|
|
const cp = require('child_process');
|
|
const originalSpawn = cp.spawnSync;
|
|
cp.spawnSync = (file, args, options) => file === 'ecc-fake-claude'
|
|
? originalSpawn(process.execPath, [${JSON.stringify(path.join(DIR, 'fake-claude.js'))}, ...args], options)
|
|
: originalSpawn(file, args, options);
|
|
if (process.env.ECC_TEST_META_RACE === '1') {
|
|
const fs = require('fs');
|
|
const rename = fs.renameSync;
|
|
fs.renameSync = (from, to) => {
|
|
if (String(to).endsWith('meta.json')) {
|
|
fs.unlinkSync(to);
|
|
fs.symlinkSync(${JSON.stringify(victim)}, to);
|
|
}
|
|
return rename(from, to);
|
|
};
|
|
}
|
|
`);
|
|
const args = ['--require', preload, path.join(DIR, 'run.js'), '--out', out, '--model', 'fake', '--claude', 'ecc-fake-claude', '--allow-real-provider', '--arms', 'off', '--reps', '1', '--scenarios', scenario.id];
|
|
const env = { ...process.env, FAKE_CLAUDE_OVERLAY: path.join(scenario.root, 'reference') };
|
|
const run = extraEnv => spawnSync(process.execPath, args, { env: { ...env, ...extraEnv }, encoding: 'utf8', timeout: 60000 });
|
|
const first = run({});
|
|
assert.strictEqual(first.status, 0, first.stderr);
|
|
const metaPath = path.join(out, 'meta.json');
|
|
const metadata = JSON.parse(fs.readFileSync(metaPath, 'utf8'));
|
|
assert.strictEqual(metadata.model, 'fake');
|
|
assert.strictEqual(metadata.sha, arms.resolveRef(ROOT, 'HEAD'));
|
|
const results = fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8');
|
|
assert.strictEqual(results.trim().split('\n').length, 1);
|
|
assert.strictEqual(JSON.parse(results.trim()).passed, true);
|
|
const second = run({});
|
|
assert.strictEqual(second.status, 0, second.stderr);
|
|
assert.ok(second.stderr.includes('0 trials to run'));
|
|
assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results);
|
|
assert.deepStrictEqual(JSON.parse(fs.readFileSync(metaPath, 'utf8')), metadata);
|
|
if (process.platform !== 'win32') {
|
|
const resultsPath = path.join(out, 'results.jsonl');
|
|
for (const file of [metaPath, resultsPath]) {
|
|
const original = fs.readFileSync(file, 'utf8');
|
|
fs.unlinkSync(file);
|
|
fs.symlinkSync(victim, file);
|
|
const early = run({});
|
|
assert.notStrictEqual(early.status, 0, 'early symlink must prevent a trial');
|
|
assert.strictEqual(fs.readFileSync(victim, 'utf8'), 'VICTIM_BYTES');
|
|
assert.throws(() => readOptionalText(file, { ...fs, constants: { ...fs.constants, O_NOFOLLOW: 0 } }), /without symlinks/);
|
|
fs.unlinkSync(file);
|
|
fs.writeFileSync(file, original, { flag: 'wx', mode: 0o600 });
|
|
}
|
|
assert.strictEqual(fs.readFileSync(resultsPath, 'utf8'), results);
|
|
const raced = run({ ECC_TEST_META_RACE: '1' });
|
|
assert.strictEqual(raced.status, 0, raced.stderr);
|
|
assert.strictEqual(fs.readFileSync(victim, 'utf8'), 'VICTIM_BYTES');
|
|
assert.deepStrictEqual(JSON.parse(fs.readFileSync(metaPath, 'utf8')), metadata);
|
|
assert.strictEqual(fs.readFileSync(path.join(out, 'results.jsonl'), 'utf8'), results);
|
|
}
|
|
assert.ok(!fs.readdirSync(out).includes('.run.lock'));
|
|
} finally { fs.rmSync(root, { recursive: true, force: true }); }
|
|
});
|
|
|
|
test('a trial wires the arm hook, counts the denial and grades the result (test double for Claude)', () => {
|
|
const work = tempDir('gg-eff-trial-');
|
|
const saved = process.env.FAKE_CLAUDE_OVERLAY;
|
|
try {
|
|
const sha = arms.resolveRef(ROOT, 'HEAD');
|
|
const tree = arms.materializeTree(ROOT, sha, path.join(work, 'tree'));
|
|
const scenario = scenarios.find(item => item.id === 'importers-date-shape');
|
|
process.env.FAKE_CLAUDE_OVERLAY = path.join(scenario.root, 'reference');
|
|
const execute = (_file, args, options) => spawnSync(process.execPath, [path.join(DIR, 'fake-claude.js'), ...args], options);
|
|
const settings = arms.armSettings(tree);
|
|
const common = { workRoot: work, armSettingsByName: { gate: settings, 'minus-target': settings, off: arms.armSettings(null) }, executable: 'claude', model: 'fake', maxTurns: 5, timeoutMs: 60000, execute };
|
|
const gated = lib.runTrial({ key: `${scenario.id}/gate/1`, scenario, arm: 'gate', rep: 1 }, common);
|
|
assert.strictEqual(gated.passed, true);
|
|
assert.strictEqual(gated.gateDenials, 1);
|
|
assert.strictEqual(gated.denials, 1);
|
|
assert.strictEqual(gated.hookObserved, true);
|
|
assert.ok(gated.questionsAsked.includes('importers'), gated.questionsAsked.join(','));
|
|
assert.strictEqual(gated.turns, 3);
|
|
assert.strictEqual(gated.inputTokens, 150);
|
|
assert.strictEqual(gated.evidenceRecall, 1);
|
|
const ablated = lib.runTrial({ key: `${scenario.id}/minus-target/1`, scenario, arm: 'minus-target', rep: 1 }, common);
|
|
assert.strictEqual(ablated.gateDenials, 1);
|
|
assert.ok(!ablated.questionsAsked.some(question => scenario.targetQuestions.includes(question)), ablated.questionsAsked.join(','));
|
|
const off = lib.runTrial({ key: `${scenario.id}/off/1`, scenario, arm: 'off', rep: 1 }, common);
|
|
assert.strictEqual(off.gateDenials, 0);
|
|
assert.strictEqual(off.passed, true);
|
|
} finally {
|
|
if (saved === undefined) delete process.env.FAKE_CLAUDE_OVERLAY;
|
|
else process.env.FAKE_CLAUDE_OVERLAY = saved;
|
|
fs.rmSync(work, { recursive: true, force: true });
|
|
}
|
|
});
|
|
}
|
|
|
|
console.log(`\nPassed: ${passed}`);
|
|
console.log(`Failed: ${failed}`);
|
|
process.exitCode = failed > 0 ? 1 : 0;
|