ECC/tests/lib/eval-harness/security.test.js

'use strict';

const assert = require('assert');
const fs = require('fs');
const path = require('path');
const { spawnSync } = require('child_process');
const { test, tempDir, cleanup, finish } = require('./helpers');
const gate = require('../../../scripts/lib/eval-harness/gate');
const library = path.resolve(__dirname, '../../../scripts/lib/eval-harness');
const refused = error => error.code === 'gate.isolation_required';
const invalidVariant = error => error.code === 'gate.variant_invalid';

function setup(fn) {
  const root = tempDir('security');
  try {
    const variant = path.join(root, 'variant');
    fs.mkdirSync(variant);
    fs.writeFileSync(path.join(variant, 'variant.json'), JSON.stringify({ name: 'candidate', effect_class: 'SE0' }));
    fs.writeFileSync(path.join(variant, 'run.js'), 'module.exports={solve:()=>1};');
    const taskset = path.join(root, 'answers.json');
    fs.writeFileSync(taskset, JSON.stringify({ version: '1', family: 'canary', tasks: [{ id: 't', input: 0, expected: 1 }] }));
    fn({ root, variant, taskset, work: path.join(root, 'work') });
  } finally {
    cleanup(root);
  }
}

test('gate refuses all execution modes before creating work, including prior trusted flags', () => setup(c => {
  for (const extra of [{}, { trusted_local: true }, { isolation: { verified: true } }, { executor: 'anything' }]) {
    assert.throws(() => gate.runGate({ taskset: c.taskset, baseline: c.variant, candidate: c.variant, work_dir: c.work, ...extra }), refused);
    assert.ok(!fs.existsSync(c.work));
  }
}));

test('refusal happens before reading configuration properties', () => {
  const config = new Proxy({}, { get() { throw new Error('configuration was inspected'); } });
  assert.throws(() => gate.runGate(config), refused);
  assert.throws(() => gate.runVariant(config), refused);
});

test('direct runner refuses caller-supplied trust and isolation claims', () => setup(c => {
  const variant = gate.loadVariant(c.variant);
  for (const options of [{}, { trusted_local: true }, { isolation: { verified: true } }]) {
    assert.throws(() => gate.runVariant(variant, [], c.work, options), refused);
  }
  assert.ok(!fs.existsSync(c.work));
}));

test('CLI refuses before reading a config or creating a capsule even with trusted-local', () => setup(c => {
  const cli = path.resolve(library, '../../eval-harness.js');
  const config = path.join(c.root, 'config.json');
  fs.writeFileSync(config, JSON.stringify({ taskset: c.taskset, baseline: c.variant, candidate: c.variant }));
  const capsule = path.join(c.root, 'capsule');
  for (const input of [config, path.join(c.root, 'missing.json')]) {
    const result = spawnSync(process.execPath, [cli, 'gate', 'run', input, '--capsule', capsule, '--trusted-local'], { encoding: 'utf8', timeout: 2000 });
    assert.strictEqual(result.status, 1);
    assert.match(result.stderr, /gate.isolation_required/);
    assert.ok(!fs.existsSync(capsule));
  }
}));

test('child and retired preload reject before loading an escaping canary payload', () => setup(c => {
  const marker = path.join(c.root, 'executed');
  const external = path.join(c.root, 'external.js');
  fs.writeFileSync(external, `require('fs').writeFileSync(${JSON.stringify(marker)},'bad');module.exports={solve:()=>1};`);
  for (const args of [[path.join(library, 'gate-child.js')], ['--require', path.join(library, 'effect-fence.js'), external]]) {
    const result = spawnSync(process.execPath, args, {
      cwd: c.variant, input: JSON.stringify({ entry: external, tasks: [] }), encoding: 'utf8', timeout: 2000,
      env: { ECC_EFFECT_FENCE_ROOT: c.variant, ECC_EFFECT_FENCE_LOG: path.join(c.root, 'log') },
    });
    assert.notStrictEqual(result.status, 0);
    assert.match(result.stderr, /gate.isolation_required/);
    assert.ok(!fs.existsSync(marker));
  }
}));

test('read, alternate builtin, descriptor and promise escape payloads never load', () => setup(c => {
  const marker = path.join(c.root, 'executed');
  const payloads = [
    `require('fs').readFileSync(${JSON.stringify(c.taskset)});`,
    "process.getBuiltinModule('ht'+'tp');", // Acquiring the API only; no request.
    `const fs=require('fs');const fd=fs.openSync(${JSON.stringify(marker)},'w');fs.writeSync(fd,'escape');fs.closeSync(fd);`,
    `require('fs/promises').writeFile(${JSON.stringify(marker)},'escape');`,
  ];
  for (const source of payloads) {
    const entry = path.join(c.variant, 'run.js');
    fs.writeFileSync(entry, `require('fs').writeFileSync(${JSON.stringify(marker)},'loaded');${source}`);
    const result = spawnSync(process.execPath, ['--require', path.join(library, 'effect-fence.js'), entry], { encoding: 'utf8', timeout: 2000 });
    assert.notStrictEqual(result.status, 0);
    assert.match(result.stderr, /gate.isolation_required/);
    assert.ok(!fs.existsSync(marker), 'payload must not begin executing');
  }
}));

test('unsafe names and escaping or undigested entry paths are rejected', () => setup(c => {
  const file = path.join(c.variant, 'variant.json');
  const cases = [
    { name: 'n/../../escaped' }, { name: '/abs' }, { name: 44 },
    { entry: path.join(c.root, 'external.js') }, { entry: '../run.js' },
    { entry: 'C:\\evil.js' }, { entry: 'node_modules/hidden.js' }, { entry: 42 },
  ];
  for (const extra of cases) {
    fs.writeFileSync(file, JSON.stringify({ name: 'candidate', effect_class: 'SE0', ...extra }));
    assert.throws(() => gate.loadVariant(c.variant), invalidVariant);
  }
}));

test('symlink manifests and symlink trees cannot hide from digest', () => setup(c => {
  const file = path.join(c.variant, 'variant.json');
  const outside = path.join(c.root, 'manifest.json');
  fs.renameSync(file, outside);
  fs.symlinkSync(outside, file);
  assert.throws(() => gate.loadVariant(c.variant), invalidVariant);
  fs.unlinkSync(file);
  fs.renameSync(outside, file);
  fs.symlinkSync(c.root, path.join(c.variant, 'link'));
  assert.throws(() => gate.loadVariant(c.variant), invalidVariant);
}));

test('valid nested entry is in digest; missing and excluded entries fail closed', () => setup(c => {
  fs.mkdirSync(path.join(c.variant, 'nested'));
  fs.writeFileSync(path.join(c.variant, 'nested', 'entry.js'), 'module.exports={solve:()=>2};');
  const file = path.join(c.variant, 'variant.json');
  fs.writeFileSync(file, JSON.stringify({ name: 'candidate', entry: 'nested/entry.js', effect_class: 'SE0' }));
  const variant = gate.loadVariant(c.variant);
  assert.strictEqual(variant.entry, path.join('nested', 'entry.js'));
  assert.match(variant.digest, /^[0-9a-f]{64}$/);
  for (const entry of ['missing.js', '.git/hidden.js']) {
    fs.writeFileSync(file, JSON.stringify({ name: 'candidate', entry, effect_class: 'SE0' }));
    assert.throws(() => gate.loadVariant(c.variant), invalidVariant);
  }
}));

test('result parser rejects empty, missing, duplicate, unexpected and ambiguous output', () => {
  const tasks = [{ id: 't' }];
  const cases = [
    {}, null, [], { results: {} }, { results: [] },
    { results: [{ id: 'wrong', output: 1 }] }, { results: [{ id: 't' }] },
    { results: [{ id: 't', output: 1, error: 'bad' }] },
    { results: [{ id: 't', output: 1 }, { id: 't', output: 1 }] }, { fatal: '' },
  ];
  for (const value of cases) {
    const result = gate.parseChildResult({ status: 0, stdout: JSON.stringify(value) }, tasks);
    assert.ok(result.fatal);
    assert.strictEqual(result.outputs.size, 0);
  }
  const valid = gate.parseChildResult({ status: 0, stdout: JSON.stringify({ results: [{ id: 't', output: 1 }] }) }, tasks);
  assert.strictEqual(valid.fatal, null);
  assert.strictEqual(valid.outputs.get('t').output, 1);
  assert.ok(gate.parseChildResult({ status: 0, stdout: 'x'.repeat(1024 * 1024 + 1) }, tasks).fatal);
  assert.ok(gate.parseChildResult(null, tasks).fatal);
});

test('fatal baseline classification rejects timeout, nonzero, signal and protocol failures', () => {
  const tasks = [{ id: 't' }];
  const cases = [
    { error: { code: 'ETIMEDOUT' } }, { error: { code: 'ENOENT' } },
    { status: 1, stdout: '{}' }, { status: null, signal: 'SIGTERM' },
    { status: 0, stdout: '{broken' }, { status: 0, stdout: JSON.stringify({ fatal: 'cannot load variant' }) },
  ];
  for (const child of cases) {
    const run = { ...gate.parseChildResult(child, tasks), exit_code: child.status, marker_intact: true, fence_events: [] };
    assert.ok(gate.baselineFailure(run, tasks));
  }
});

test('baseline validation requires complete unique error-free results and integrity', () => {
  const tasks = [{ id: 't' }];
  const run = { outputs: new Map([['t', { id: 't', output: 1 }]]), fatal: null, exit_code: 0, marker_intact: true, fence_events: [] };
  assert.strictEqual(gate.baselineFailure(run, tasks), null);
  const cases = [
    { outputs: new Map() }, { outputs: new Map([['t', { id: 'wrong', output: 1 }]]) },
    { outputs: new Map([['t', { id: 't', error: 'failure' }]]) }, { fatal: 'bad' },
    { exit_code: 1 }, { marker_intact: false }, { fence_events: [{ kind: 'effect' }] },
  ];
  for (const delta of cases) assert.ok(gate.baselineFailure({ ...run, ...delta }, tasks));
  for (const input of [undefined, [], [null], [{ id: 't' }, { id: 't' }]]) {
    assert.ok(gate.baselineFailure(run, input));
  }
});

test('duplicate task ids cannot erase per-task regression evidence', () => setup(c => {
  for (const tasks of [[{ id: 'same', input: 0, expected: 1 }, { id: 'same', input: 1, expected: 2 }], [null]]) {
    fs.writeFileSync(c.taskset, JSON.stringify({ version: '1', family: 'canary', tasks }));
    assert.throws(() => gate.loadTaskset(c.taskset), error => error.code === 'gate.taskset_invalid');
  }
}));

finish('security');