mirror of
https://github.com/affaan-m/ECC.git
synced 2026-09-28 20:45:11 +02:00
68 lines
3.4 KiB
JavaScript
68 lines
3.4 KiB
JavaScript
'use strict';
|
|
// Development tool: assembles a complex corpus JSON from a reviewed fixture
|
|
// tree. Usage: node build-corpus.js [casesDir=cases] [outFile=complex-corpus.json] [corpusId=complex-tasks@1]
|
|
// Run after editing any fixture, query, or grader; commit the tree and the
|
|
// regenerated corpus together.
|
|
const fs = require('node:fs');
|
|
const path = require('node:path');
|
|
|
|
const root = __dirname;
|
|
const casesDir = path.join(root, process.argv[2] || 'cases');
|
|
const OUT = path.join(root, '..', process.argv[3] || 'complex-corpus.json');
|
|
const corpusId = process.argv[4] || 'complex-tasks@1';
|
|
|
|
function collect(directory, prefix = '') {
|
|
const files = {};
|
|
for (const entry of fs.readdirSync(directory, { withFileTypes: true }).sort((a, b) => a.name.localeCompare(b.name))) {
|
|
const relative = prefix ? `${prefix}/${entry.name}` : entry.name;
|
|
if (entry.isDirectory()) Object.assign(files, collect(path.join(directory, entry.name), relative));
|
|
else if (entry.isFile()) files[relative] = fs.readFileSync(path.join(directory, entry.name), 'utf8');
|
|
}
|
|
return files;
|
|
}
|
|
|
|
const tasks = [];
|
|
const selection = [];
|
|
for (const id of fs.readdirSync(casesDir).sort()) {
|
|
const directory = path.join(casesDir, id);
|
|
const meta = JSON.parse(fs.readFileSync(path.join(directory, 'meta.json'), 'utf8'));
|
|
if (meta.id !== id || !/^[a-z][a-z0-9-]{0,63}$/.test(id)) throw new Error(`Invalid task metadata in ${id}`);
|
|
const files = collect(path.join(directory, 'files'));
|
|
const stepsDir = path.join(directory, 'steps');
|
|
let task;
|
|
if (fs.existsSync(stepsDir)) {
|
|
const steps = fs.readdirSync(stepsDir).sort().map((name, index) => ({
|
|
query: fs.readFileSync(path.join(stepsDir, name, 'query.md'), 'utf8').trim(),
|
|
check: fs.readFileSync(path.join(stepsDir, name, 'check.cjs'), 'utf8'),
|
|
...(meta.steps?.[index]?.manualIds ? { manualIds: meta.steps[index].manualIds } : {}),
|
|
...((meta.steps?.[index]?.checkTimeoutMs || meta.checkTimeoutMs)
|
|
? { checkTimeoutMs: meta.steps?.[index]?.checkTimeoutMs || meta.checkTimeoutMs } : {}),
|
|
}));
|
|
task = { id, category: meta.category, manualIds: meta.manualIds || [], files, steps };
|
|
} else {
|
|
const query = fs.readFileSync(path.join(directory, 'query.md'), 'utf8').trim();
|
|
task = { id, category: meta.category, manualIds: meta.manualIds,
|
|
...(meta.checkTimeoutMs ? { checkTimeoutMs: meta.checkTimeoutMs } : {}),
|
|
query, files, check: fs.readFileSync(path.join(directory, 'check.cjs'), 'utf8') };
|
|
}
|
|
tasks.push(task);
|
|
selection.push({ id: meta.selection.id, category: meta.selection.category,
|
|
query: meta.selection.query || task.query || task.steps.map(step => step.query).join(' '),
|
|
expectedIds: meta.selection.expectedIds });
|
|
}
|
|
|
|
const corpus = {
|
|
schemaVersion: 'ecc.context-eval-complex-corpus.v1',
|
|
id: corpusId,
|
|
sampling: 'Realistic multi-file engineering tasks, fixed before any provider call, with deterministic '
|
|
+ 'hidden graders scoring partial credit (ECC_EVAL_SCORE). Descriptive pilot: no '
|
|
+ 'population-representativeness claim. See complex-eval/DESIGN.md for the preregistered methodology.',
|
|
minimumDistinctTasks: tasks.length,
|
|
nonInferiorityMargin: 0.05,
|
|
selection,
|
|
tasks,
|
|
};
|
|
fs.writeFileSync(OUT, `${JSON.stringify(corpus, null, 1)}\n`);
|
|
console.log(`wrote ${path.basename(OUT)} (${corpusId}): ${tasks.length} tasks, ${selection.length} selection probes, `
|
|
+ `${tasks.reduce((sum, task) => sum + Object.keys(task.files).length, 0)} fixture files`);
|