mirror of
https://github.com/affaan-m/ECC.git
synced 2026-09-28 20:45:11 +02:00
Merge reviewed PR #2954 CI and security follow-up
This commit is contained in:
@@ -154,6 +154,12 @@ case "$EVAL_MODE" in
|
||||
esac
|
||||
|
||||
EVALUATOR_TOOLS=$(evaluator_tools_for_mode "$EVAL_MODE")
|
||||
EVALUATOR_OPTIONS=(--allowedTools "$EVALUATOR_TOOLS")
|
||||
if [ "$EVAL_MODE" != "playwright" ]; then
|
||||
# Allow rules only pre-approve calls. Deny this server's tools explicitly
|
||||
# in modes that inspect existing screenshots or source instead.
|
||||
EVALUATOR_OPTIONS+=(--disallowedTools 'mcp__playwright__*')
|
||||
fi
|
||||
|
||||
phase "GAN-STYLE HARNESS — Setup"
|
||||
|
||||
@@ -263,7 +269,7 @@ Update gan-harness/generator-state.md." \
|
||||
fi
|
||||
|
||||
claude -p --model "$EVALUATOR_MODEL" \
|
||||
--allowedTools "$EVALUATOR_TOOLS" \
|
||||
"${EVALUATOR_OPTIONS[@]}" \
|
||||
"You are the Evaluator in a GAN-style harness. Read agents/gan-evaluator.md for full instructions.
|
||||
|
||||
Iteration: $i
|
||||
|
||||
@@ -383,6 +383,26 @@ const results = Object.freeze([
|
||||
assertEvaluatorTools(calls, baseTools);
|
||||
});
|
||||
})),
|
||||
|
||||
test('evaluator mode explicitly denies Playwright tools without changing other phases', () => {
|
||||
for (const mode of ['playwright', 'screenshot', 'code-only']) {
|
||||
withHarnessRun({ mode }, ({ result, calls }) => {
|
||||
assert.strictEqual(result.status, 0, result.stderr);
|
||||
const [evaluator] = evaluatorCalls(calls);
|
||||
const denyIndex = evaluator.indexOf('--disallowedTools');
|
||||
if (mode === 'playwright') {
|
||||
assert.strictEqual(denyIndex, -1);
|
||||
} else {
|
||||
assert.ok(denyIndex >= 0, `${mode} must deny the configured Playwright server tools`);
|
||||
assert.strictEqual(evaluator[denyIndex + 1], 'mcp__playwright__*');
|
||||
assert.strictEqual(evaluator.filter(arg => arg === '--disallowedTools').length, 1);
|
||||
}
|
||||
for (const args of calls.filter(args => args[0] === '-p' && args !== evaluator)) {
|
||||
assert.strictEqual(args.includes('--disallowedTools'), false);
|
||||
}
|
||||
});
|
||||
}
|
||||
}),
|
||||
]);
|
||||
|
||||
const passed = results.filter(Boolean).length;
|
||||
|
||||
Reference in New Issue
Block a user