diff --git a/scripts/gan-harness.sh b/scripts/gan-harness.sh index 79e081620..3b4164bec 100755 --- a/scripts/gan-harness.sh +++ b/scripts/gan-harness.sh @@ -154,6 +154,12 @@ case "$EVAL_MODE" in esac EVALUATOR_TOOLS=$(evaluator_tools_for_mode "$EVAL_MODE") +EVALUATOR_OPTIONS=(--allowedTools "$EVALUATOR_TOOLS") +if [ "$EVAL_MODE" != "playwright" ]; then + # Allow rules only pre-approve calls. Deny this server's tools explicitly + # in modes that inspect existing screenshots or source instead. + EVALUATOR_OPTIONS+=(--disallowedTools 'mcp__playwright__*') +fi phase "GAN-STYLE HARNESS — Setup" @@ -263,7 +269,7 @@ Update gan-harness/generator-state.md." \ fi claude -p --model "$EVALUATOR_MODEL" \ - --allowedTools "$EVALUATOR_TOOLS" \ + "${EVALUATOR_OPTIONS[@]}" \ "You are the Evaluator in a GAN-style harness. Read agents/gan-evaluator.md for full instructions. Iteration: $i diff --git a/tests/gan-harness.test.js b/tests/gan-harness.test.js index 7bc05cba5..b2a591212 100644 --- a/tests/gan-harness.test.js +++ b/tests/gan-harness.test.js @@ -383,6 +383,26 @@ const results = Object.freeze([ assertEvaluatorTools(calls, baseTools); }); })), + + test('evaluator mode explicitly denies Playwright tools without changing other phases', () => { + for (const mode of ['playwright', 'screenshot', 'code-only']) { + withHarnessRun({ mode }, ({ result, calls }) => { + assert.strictEqual(result.status, 0, result.stderr); + const [evaluator] = evaluatorCalls(calls); + const denyIndex = evaluator.indexOf('--disallowedTools'); + if (mode === 'playwright') { + assert.strictEqual(denyIndex, -1); + } else { + assert.ok(denyIndex >= 0, `${mode} must deny the configured Playwright server tools`); + assert.strictEqual(evaluator[denyIndex + 1], 'mcp__playwright__*'); + assert.strictEqual(evaluator.filter(arg => arg === '--disallowedTools').length, 1); + } + for (const args of calls.filter(args => args[0] === '-p' && args !== evaluator)) { + assert.strictEqual(args.includes('--disallowedTools'), false); + } + }); + } + }), ]); const passed = results.filter(Boolean).length;