ECC/tests/lib/context-profile-eval-corpus.test.js

'use strict';
const assert = require('node:assert/strict');
const fs = require('node:fs');
const os = require('node:os');
const path = require('node:path');
const { spawnSync } = require('node:child_process');
const test = require('node:test');
const { loadContextRegistry } = require('../../scripts/lib/context-pack-registry');

const root = path.resolve(__dirname, '../..');
const corpus = JSON.parse(fs.readFileSync(path.join(root, 'docker/context-profiles/ai-corpus.json'), 'utf8'));
const references = JSON.parse(fs.readFileSync(path.join(__dirname, '../fixtures/context-eval-references.json'), 'utf8'));
const ID = /^[a-z][a-z0-9-]{0,63}$/;
const BLOCKS = ['excluded', 'native-authority', 'manual-only', 'opt-out-conflict', 'unknown-id'];
const bytes = text => Buffer.byteLength(text, 'utf8');

function assertSafePath(file) {
  assert.equal(typeof file, 'string');
  assert.ok(file.length > 0 && !path.isAbsolute(file) && !path.win32.isAbsolute(file), `absolute path: ${file}`);
  assert.ok(!file.includes('\\') && !file.includes('\0'), `unsafe path: ${file}`);
  for (const part of file.split('/')) {
    assert.ok(part && part !== '.' && part !== '..' && !part.startsWith('.'), `unsafe path segment: ${file}`);
  }
}

function writeTree(dir, files) {
  for (const [file, content] of Object.entries(files)) {
    const target = path.join(dir, file);
    fs.mkdirSync(path.dirname(target), { recursive: true });
    fs.writeFileSync(target, content);
  }
}

function runCheck(task, overlay) {
  const dir = fs.mkdtempSync(path.join(os.tmpdir(), `ecc-eval-${task.id}-`));
  try {
    writeTree(dir, task.files);
    if (overlay) writeTree(dir, overlay);
    fs.writeFileSync(path.join(dir, '.ecc-eval-check.cjs'), task.check);
    return spawnSync(process.execPath, ['.ecc-eval-check.cjs'], { cwd: dir, timeout: 10000, encoding: 'utf8' });
  } finally {
    fs.rmSync(dir, { recursive: true, force: true });
  }
}

test('corpus v2 header, ids and probe shapes are valid', () => {
  assert.equal(corpus.schemaVersion, 'ecc.context-eval-corpus.v2');
  assert.equal(corpus.id, 'coding-tasks@1');
  assert.equal(typeof corpus.sampling, 'string');
  assert.ok(corpus.sampling.length > 0);
  assert.equal(corpus.minimumDistinctTasks, 30);
  assert.equal(corpus.nonInferiorityMargin, 0.05);
  assert.equal(corpus.tasks.length, 30);
  assert.ok(corpus.selection.length >= 30);
  for (const cases of [corpus.selection, corpus.tasks]) {
    assert.equal(new Set(cases.map(c => c.id)).size, cases.length, 'duplicate id');
    for (const item of cases) assert.match(item.id, ID);
  }
  const categories = new Set(corpus.selection.map(p => p.category));
  for (const category of ['exact', 'paraphrase', 'no-workflow', 'policy']) assert.ok(categories.has(category), category);
  for (const probe of corpus.selection) {
    assert.equal(typeof probe.query, 'string');
    assert.ok(bytes(probe.query) > 0 && bytes(probe.query) <= 8192);
    if (probe.expectedBlock !== undefined) {
      assert.ok(BLOCKS.includes(probe.expectedBlock), probe.id);
    } else {
      assert.ok(Array.isArray(probe.expectedIds) && probe.expectedIds.length <= 1, probe.id);
    }
    if (probe.noWorkflow !== undefined) assert.equal(typeof probe.noWorkflow, 'boolean');
  }
});

test('every referenced skill ID exists in the registry', () => {
  const known = new Set(loadContextRegistry({ repoRoot: root }).entries.map(e => e.id));
  const ids = new Set();
  for (const probe of corpus.selection) {
    for (const key of ['expectedIds', 'exclude']) (probe[key] || []).forEach(id => ids.add(id));
    if (probe.expectedBlock !== 'unknown-id') (probe.explicitIds || []).forEach(id => ids.add(id));
  }
  corpus.tasks.forEach(task => task.manualIds.forEach(id => ids.add(id)));
  for (const id of ids) assert.ok(known.has(id), `unknown registry ID: ${id}`);
  for (const probe of corpus.selection.filter(p => p.expectedBlock === 'unknown-id')) {
    assert.ok(probe.explicitIds.some(id => !known.has(id)), probe.id);
  }
});

test('tasks respect shape, size and path-safety limits', () => {
  const skills = new Set();
  let noWorkflow = 0;
  for (const task of corpus.tasks) {
    assert.equal(typeof task.category, 'string');
    assert.ok(Array.isArray(task.manualIds) && task.manualIds.length <= 1, task.id);
    task.manualIds.forEach(id => skills.add(id));
    if (task.category === 'no-workflow') {
      noWorkflow++;
      assert.deepEqual(task.manualIds, [], task.id);
    }
    if (task.noWorkflow !== undefined) assert.equal(task.noWorkflow, true, task.id);
    assert.ok(bytes(task.query) > 0 && bytes(task.query) <= 1500, `${task.id} query is ${bytes(task.query)} bytes`);
    assert.ok(/dependenc/i.test(task.query), `${task.id} query must forbid new dependencies`);
    assert.ok(!/ecc-eval-check|hidden check/i.test(task.query), task.id);
    const files = Object.entries(task.files);
    assert.ok(files.length >= 1 && files.length <= 4, `${task.id} has ${files.length} files`);
    let total = 0;
    for (const [file, content] of files) {
      assertSafePath(file);
      assert.equal(typeof content, 'string');
      assert.ok(bytes(content) <= 4096, `${task.id}/${file} exceeds 4 KB`);
      total += bytes(content);
    }
    assert.ok(total <= 12288, `${task.id} files exceed 12 KB`);
    assert.equal(typeof task.check, 'string');
    assert.doesNotMatch(task.check, /child_process|worker_threads|node:net|node:http|writeFile|appendFile|mkdirSync|rmSync|unlinkSync|fetch\(/,
      `${task.id} check uses a forbidden API`);
    const overlay = references[task.id];
    assert.ok(overlay && Object.keys(overlay).length >= 1, `${task.id} has no reference`);
    for (const [file, content] of Object.entries(overlay)) {
      assertSafePath(file);
      assert.equal(typeof content, 'string');
    }
  }
  assert.ok(noWorkflow >= 6 && noWorkflow <= 10, `no-workflow tasks: ${noWorkflow}`);
  assert.ok(skills.size >= 10, `distinct skills: ${skills.size}`);
  assert.deepEqual(Object.keys(references).sort(), corpus.tasks.map(t => t.id).sort());
});

for (const task of corpus.tasks) {
  test(`hidden check fails on starter files and passes on the reference: ${task.id}`, () => {
    const before = runCheck(task, null);
    assert.notEqual(before.status, 0, `${task.id} check passed on the starter files`);
    assert.equal(before.error, undefined);
    const after = runCheck(task, references[task.id]);
    assert.equal(after.status, 0, `${task.id} reference failed:\n${after.stderr}${after.stdout}`);
  });
}