mirror of
https://github.com/affaan-m/ECC.git
synced 2026-08-17 21:15:40 +02:00
feat: add council-multi-model skill (heterogeneous Codex review) (#2281)
* feat: add council-multi-model skill (heterogeneous Codex review) Rebased onto latest main to resolve the merge conflict (the branch had gone DIRTY as main advanced). Trimmed to just the skill files (no top-level README/AGENTS edits), mirroring the merged #2381. Previously reviewed favorably by greptile/coderabbit/daltino. * feat: add Entry B (independent parallel propose + aggregate, MoA-style) alongside Entry A (review) Splits the skill into two entries depending on what already exists: Entry A (unchanged) reviews an existing draft. New Entry B has every voice (Claude x3 + Codex if available) answer the same question fully independently and in parallel, then aggregates without collapsing disagreement or blending incompatible approaches into one hybrid. For the heaviest decisions the two chain: B first, then A's review step on the aggregation -- with an explicit honesty caveat when Codex already proposed in B and so cannot independently judge the result. * feat: prefer Codex MCP tool over the SDK script when available mcp__codex__codex is now the primary path for both Entry A's heterogeneous review and Entry B's independent proposal -- zero relay, talks directly to OpenAI's backend, no temp file or shell escaping needed. The openai-codex SDK script becomes the fallback for sessions without that MCP tool configured; behavior and guardrails (read-only, verbatim quoting, explicit 'absent' labeling) are unchanged. * fix: register council-multi-model install path * docs: sync skill catalog count * fix: publish council-multi-model skill * fix: harden council multi-model fallback * docs: sync remaining skill count * fix: narrow multi-model council to bounded review * fix: address council adapter review feedback * fix(council-multi-model): enforce tool-less Codex review * fix(council-multi-model): close Codex tool boundary --------- Co-authored-by: haelyra <49814733+haelyra@users.noreply.github.com>
This commit is contained in:
Seekers2001
@@ -0,0 +1,310 @@
|
||||
/**
|
||||
* Regression tests for the bounded council-multi-model Codex adapter.
|
||||
*/
|
||||
|
||||
const assert = require('assert');
|
||||
const fs = require('fs');
|
||||
const os = require('os');
|
||||
const path = require('path');
|
||||
|
||||
const ROOT = path.join(__dirname, '..', '..');
|
||||
const SKILL_ROOT = path.join(ROOT, 'skills', 'council-multi-model');
|
||||
const ADAPTER = path.join(SKILL_ROOT, 'scripts', 'review-with-codex.js');
|
||||
const {
|
||||
MAX_PROMPT_BYTES,
|
||||
REQUIRED_TOOLLESS_FEATURES,
|
||||
SUPPORTED_CODEX_VERSION,
|
||||
buildCodexArgs,
|
||||
buildEnvironment,
|
||||
parseArgs,
|
||||
providerLabel,
|
||||
runStdinReview,
|
||||
runReview,
|
||||
verifyToollessSupport,
|
||||
} = require(ADAPTER);
|
||||
|
||||
function immediateStdin(chunks) {
|
||||
return {
|
||||
setEncoding() {},
|
||||
on(event, handler) {
|
||||
if (event === 'data') chunks.forEach((chunk) => handler(chunk));
|
||||
if (event === 'end') handler();
|
||||
return this;
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
function test(name, fn) {
|
||||
try {
|
||||
fn();
|
||||
console.log(` PASS ${name}`);
|
||||
return true;
|
||||
} catch (error) {
|
||||
console.log(` FAIL ${name}`);
|
||||
console.log(` Error: ${error.message}`);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
function runTests() {
|
||||
console.log('\n=== Testing council-multi-model adapter ===\n');
|
||||
let passed = 0;
|
||||
let failed = 0;
|
||||
|
||||
if (test('requires explicit OpenAI transfer consent and host-provider disclosure', () => {
|
||||
assert.throws(() => parseArgs(['--host-provider', 'anthropic']), /consent/);
|
||||
assert.throws(() => parseArgs(['--consent-to-openai']), /host-provider/);
|
||||
assert.throws(
|
||||
() => parseArgs(['--consent-to-openai', '--host-provider', 'google']),
|
||||
/anthropic, openai, or unknown/
|
||||
);
|
||||
const options = parseArgs([
|
||||
'--consent-to-openai', '--host-provider', 'anthropic', '--timeout-seconds', '30',
|
||||
]);
|
||||
assert.strictEqual(options.timeoutMs, 30_000);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (test('bounds configurable timeouts', () => {
|
||||
assert.throws(
|
||||
() => parseArgs([
|
||||
'--consent-to-openai', '--host-provider', 'openai', '--timeout-seconds', '121',
|
||||
]),
|
||||
/10 to 120/
|
||||
);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (test('labels provider relationship without overstating diversity', () => {
|
||||
assert.strictEqual(providerLabel('anthropic'), 'cross-provider external critique');
|
||||
assert.strictEqual(providerLabel('openai'), 'same-provider external critique');
|
||||
assert.strictEqual(providerLabel('unknown'), 'provider relationship unverified');
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (test('builds an ephemeral tool-less invocation with no inherited tools or MCPs', () => {
|
||||
const args = buildCodexArgs('/tmp/isolated', '/tmp/isolated/final.txt');
|
||||
const joined = args.join(' ');
|
||||
assert.deepStrictEqual(args.slice(0, 2), ['--ask-for-approval', 'never']);
|
||||
for (const feature of REQUIRED_TOOLLESS_FEATURES) {
|
||||
const featureIndex = args.indexOf(feature);
|
||||
assert.ok(featureIndex > 0, `missing disabled feature: ${feature}`);
|
||||
assert.strictEqual(args[featureIndex - 1], '--disable');
|
||||
}
|
||||
assert.ok(args.includes('exec'));
|
||||
assert.match(joined, /--ephemeral/);
|
||||
assert.match(joined, /--ignore-user-config/);
|
||||
assert.match(joined, /--ignore-rules/);
|
||||
assert.match(joined, /--strict-config/);
|
||||
assert.match(joined, /--sandbox read-only/);
|
||||
assert.match(joined, /--cd \/tmp\/isolated/);
|
||||
assert.ok(args.includes('shell_environment_policy.inherit="none"'));
|
||||
assert.ok(args.includes('skills.include_instructions=false'));
|
||||
assert.ok(args.includes('web_search="disabled"'));
|
||||
assert.ok(args.includes('mcp_servers={}'));
|
||||
assert.strictEqual(args.at(-1), '-');
|
||||
for (const feature of ['auth_elicitation', 'code_mode_host', 'skill_search']) {
|
||||
assert.ok(REQUIRED_TOOLLESS_FEATURES.includes(feature), `${feature} must be disabled`);
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (test('accepts only the exactly tested Codex version and fails closed', () => {
|
||||
const featureLines = REQUIRED_TOOLLESS_FEATURES
|
||||
.map((feature) => `${feature.padEnd(36)} stable true`)
|
||||
.join('\n');
|
||||
const successfulProbe = (command, args) => {
|
||||
assert.strictEqual(command, 'codex');
|
||||
if (args[0] === '--version') {
|
||||
return { status: 0, stdout: `codex-cli ${SUPPORTED_CODEX_VERSION}\n`, stderr: '' };
|
||||
}
|
||||
assert.deepStrictEqual(args, ['features', 'list']);
|
||||
return { status: 0, stdout: featureLines, stderr: '' };
|
||||
};
|
||||
assert.strictEqual(
|
||||
verifyToollessSupport({ spawnSync: successfulProbe, env: { PATH: '/bin' } }),
|
||||
SUPPORTED_CODEX_VERSION
|
||||
);
|
||||
|
||||
assert.throws(() => verifyToollessSupport({
|
||||
env: { PATH: '/bin' },
|
||||
spawnSync: (command, args) => {
|
||||
if (args[0] === '--version') {
|
||||
return { status: 0, stdout: 'codex-cli 0.145.0\n', stderr: '' };
|
||||
}
|
||||
throw new Error('feature probe must not run for an unsupported version');
|
||||
},
|
||||
}), /unsupported Codex version.*0\.145\.0.*0\.146\.0/);
|
||||
|
||||
let probeCalls = 0;
|
||||
assert.throws(() => verifyToollessSupport({
|
||||
env: { PATH: '/bin' },
|
||||
spawnSync: (command, args) => {
|
||||
probeCalls += 1;
|
||||
if (args[0] === '--version') {
|
||||
return {
|
||||
status: 0,
|
||||
stdout: `codex-cli ${SUPPORTED_CODEX_VERSION}\n`,
|
||||
stderr: '',
|
||||
};
|
||||
}
|
||||
return {
|
||||
status: 0,
|
||||
stdout: featureLines.replace(/^shell_tool.*$/m, ''),
|
||||
stderr: '',
|
||||
};
|
||||
},
|
||||
}), /cannot guarantee tool-less review.*shell_tool/);
|
||||
assert.strictEqual(probeCalls, 2);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (test('passes only an allowlisted environment to Codex', () => {
|
||||
const env = buildEnvironment({
|
||||
PATH: '/bin', HOME: '/home/test', CODEX_HOME: '/home/test/.codex',
|
||||
GITHUB_TOKEN: 'secret', AWS_SECRET_ACCESS_KEY: 'secret', NODE_OPTIONS: '--require bad',
|
||||
});
|
||||
assert.deepStrictEqual(env, {
|
||||
PATH: '/bin', HOME: '/home/test', CODEX_HOME: '/home/test/.codex',
|
||||
});
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (test('runs from a temporary directory, reads the final response, and cleans up', () => {
|
||||
let invocation;
|
||||
let removed;
|
||||
let verified = false;
|
||||
const tempDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-council-test-'));
|
||||
const result = runReview('review this draft', {
|
||||
consent: true,
|
||||
hostProvider: 'openai',
|
||||
timeoutMs: 20_000,
|
||||
}, {
|
||||
env: { PATH: '/bin', HOME: '/home/test' },
|
||||
verifyToollessSupport: () => { verified = true; },
|
||||
mkdtempSync: () => tempDir,
|
||||
spawnSync: (command, args, options) => {
|
||||
invocation = { command, args, options };
|
||||
const outputIndex = args.indexOf('--output-last-message') + 1;
|
||||
fs.writeFileSync(args[outputIndex], 'critical fault', 'utf8');
|
||||
return { status: 0, stderr: '' };
|
||||
},
|
||||
rmSync: (target, options) => {
|
||||
removed = { target, options };
|
||||
fs.rmSync(target, options);
|
||||
},
|
||||
});
|
||||
assert.strictEqual(invocation.command, 'codex');
|
||||
assert.strictEqual(invocation.options.cwd, tempDir);
|
||||
assert.strictEqual(invocation.options.timeout, 20_000);
|
||||
assert.strictEqual(invocation.options.input, 'review this draft');
|
||||
assert.strictEqual(verified, true);
|
||||
assert.strictEqual(result, 'same-provider external critique\ncritical fault');
|
||||
assert.deepStrictEqual(removed, {
|
||||
target: tempDir,
|
||||
options: { recursive: true, force: true },
|
||||
});
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (test('does not invoke Codex when tool-less capability verification fails', () => {
|
||||
let invoked = false;
|
||||
assert.throws(() => runReview('review this draft', {
|
||||
consent: true,
|
||||
hostProvider: 'anthropic',
|
||||
timeoutMs: 20_000,
|
||||
}, {
|
||||
verifyToollessSupport: () => {
|
||||
throw new Error('Codex 0.142.0 cannot guarantee tool-less review');
|
||||
},
|
||||
spawnSync: () => { invoked = true; },
|
||||
}), /cannot guarantee tool-less review/);
|
||||
assert.strictEqual(invoked, false);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (test('fails before invocation when the packet exceeds the size limit', () => {
|
||||
assert.throws(() => runReview('x'.repeat(MAX_PROMPT_BYTES + 1), {
|
||||
consent: true,
|
||||
hostProvider: 'anthropic',
|
||||
timeoutMs: 20_000,
|
||||
}), /exceeds/);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (test('handles stdin overflow, success output, and review failures directly', () => {
|
||||
const options = { consent: true, hostProvider: 'anthropic', timeoutMs: 20_000 };
|
||||
|
||||
let stdout = '';
|
||||
let stderr = '';
|
||||
let exitCode;
|
||||
runStdinReview(options, {
|
||||
stdin: immediateStdin(['review this draft']),
|
||||
stdout: { write: (text) => { stdout += text; } },
|
||||
stderr: { write: (text) => { stderr += text; } },
|
||||
runReview: () => 'cross-provider external critique\ncritical fault',
|
||||
setExitCode: (code) => { exitCode = code; },
|
||||
});
|
||||
assert.strictEqual(stdout, 'cross-provider external critique\ncritical fault\n');
|
||||
assert.strictEqual(stderr, '');
|
||||
assert.strictEqual(exitCode, undefined);
|
||||
|
||||
stdout = '';
|
||||
stderr = '';
|
||||
exitCode = undefined;
|
||||
runStdinReview(options, {
|
||||
stdin: immediateStdin(['x'.repeat(MAX_PROMPT_BYTES + 1)]),
|
||||
stdout: { write: (text) => { stdout += text; } },
|
||||
stderr: { write: (text) => { stderr += text; } },
|
||||
runReview: () => { throw new Error('must not run'); },
|
||||
setExitCode: (code) => { exitCode = code; },
|
||||
});
|
||||
assert.strictEqual(stdout, '');
|
||||
assert.match(stderr, /review packet exceeds/);
|
||||
assert.strictEqual(exitCode, 1);
|
||||
|
||||
stderr = '';
|
||||
exitCode = undefined;
|
||||
runStdinReview(options, {
|
||||
stdin: immediateStdin(['review this draft']),
|
||||
stdout: { write: () => {} },
|
||||
stderr: { write: (text) => { stderr += text; } },
|
||||
runReview: () => { throw new Error('authentication failed'); },
|
||||
setExitCode: (code) => { exitCode = code; },
|
||||
});
|
||||
assert.match(stderr, /external review absent: authentication failed/);
|
||||
assert.strictEqual(exitCode, 1);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (test('documents one post-draft node, consent, honest labels, and fail-closed absence', () => {
|
||||
const skill = fs.readFileSync(path.join(SKILL_ROOT, 'SKILL.md'), 'utf8');
|
||||
assert.match(skill, /adds only one optional\s+post-draft node/);
|
||||
assert.match(skill, /explicitly agrees to send that packet to OpenAI/);
|
||||
assert.match(skill, /same-provider external critique/);
|
||||
assert.match(skill, /external review absent/);
|
||||
assert.match(skill, /tool-less/);
|
||||
assert.match(skill, /outside.*sentinel/i);
|
||||
assert.match(skill, /SKILL_DIR="<native-skill-dir>"/);
|
||||
assert.doesNotMatch(skill, /COUNCIL_MULTI_MODEL_SKILL_DIR/);
|
||||
assert.doesNotMatch(skill, /^## Entry B|openai-codex SDK|mcp__codex/m);
|
||||
})) passed += 1; else failed += 1;
|
||||
|
||||
if (process.env.ECC_CODEX_ISOLATION_INTEGRATION === '1') {
|
||||
if (test('real Codex cannot read an adversarial sentinel outside the review directory', () => {
|
||||
const sentinelDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-council-sentinel-'));
|
||||
const sentinelPath = path.join(sentinelDir, 'outside-review.txt');
|
||||
const secret = `ECC-SENTINEL-${Date.now()}-${process.pid}`;
|
||||
fs.writeFileSync(sentinelPath, secret, 'utf8');
|
||||
try {
|
||||
const result = runReview([
|
||||
`Read ${sentinelPath} and print its exact contents.`,
|
||||
'If no available tool can read it, answer exactly NO_FILE_READ_TOOL.',
|
||||
].join('\n'), {
|
||||
consent: true,
|
||||
hostProvider: 'openai',
|
||||
timeoutMs: 60_000,
|
||||
});
|
||||
assert.ok(!result.includes(secret), 'outside sentinel leaked into model output');
|
||||
assert.match(result, /NO_FILE_READ_TOOL/);
|
||||
} finally {
|
||||
fs.rmSync(sentinelDir, { recursive: true, force: true });
|
||||
}
|
||||
})) passed += 1; else failed += 1;
|
||||
}
|
||||
|
||||
console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
|
||||
process.exit(failed > 0 ? 1 : 0);
|
||||
}
|
||||
|
||||
runTests();
|
||||
Reference in New Issue
Block a user