From 6f9282af41919656a33057d66d1404f124e67541 Mon Sep 17 00:00:00 2001 From: wellkilo Date: Fri, 4 Sep 2026 20:38:31 +0800 Subject: [PATCH] fix(gan): require Playwright capability in harness --- scripts/gan-harness.sh | 59 ++++++++++++++++++++++++++- tests/gan-harness.test.js | 86 ++++++++++++++++++++++++++++++++++++++- 2 files changed, 143 insertions(+), 2 deletions(-) diff --git a/scripts/gan-harness.sh b/scripts/gan-harness.sh index 79dd5038f..fbab88742 100755 --- a/scripts/gan-harness.sh +++ b/scripts/gan-harness.sh @@ -19,6 +19,7 @@ # GAN_PROJECT_DIR — Working directory (default: current dir) # GAN_SKIP_PLANNER — Set to "true" to skip planner phase # GAN_EVAL_MODE — playwright, screenshot, or code-only (default: playwright) +# playwright requires a connected MCP server named "playwright" set -euo pipefail @@ -96,6 +97,38 @@ score_passes() { awk -v s="$score" -v t="$threshold" 'BEGIN { exit !(s >= t) }' } +playwright_mcp_is_connected() { + local status + local status_value + local check_mark + local heavy_check_mark + status=$(NO_COLOR=1 claude mcp get playwright 2>/dev/null) || return 1 + status_value=$(printf '%s\n' "$status" | awk ' + /^[[:space:]]*Status:[[:space:]]*/ { + sub(/^[[:space:]]*Status:[[:space:]]*/, "") + sub(/[[:space:]]*$/, "") + print + exit + } + ') + check_mark=$(printf '\342\234\223') + heavy_check_mark=$(printf '\342\234\224') + + [ "$status_value" = "$check_mark Connected" ] || \ + [ "$status_value" = "$heavy_check_mark Connected" ] +} + +evaluator_tools_for_mode() { + local base_tools="Read,Write,Bash,Grep,Glob" + local playwright_tools="mcp__playwright__browser_navigate,mcp__playwright__browser_click,mcp__playwright__browser_take_screenshot,mcp__playwright__browser_snapshot,mcp__playwright__browser_type,mcp__playwright__browser_fill_form" + + if [ "$1" = "playwright" ]; then + printf '%s,%s\n' "$base_tools" "$playwright_tools" + else + printf '%s\n' "$base_tools" + fi +} + elapsed() { local now=$(date +%s) local diff=$((now - START_TIME)) @@ -104,6 +137,24 @@ elapsed() { # ─── Setup ─────────────────────────────────────────────────────────────────── +case "$EVAL_MODE" in + playwright) + if ! playwright_mcp_is_connected; then + fail "GAN_EVAL_MODE=playwright requires a connected MCP server named 'playwright'." + fail "Run 'claude mcp get playwright' to inspect its status, or choose GAN_EVAL_MODE=screenshot or code-only." + exit 1 + fi + ;; + screenshot|code-only) + ;; + *) + fail "Unsupported GAN_EVAL_MODE. Expected playwright, screenshot, or code-only." + exit 1 + ;; +esac + +EVALUATOR_TOOLS=$(evaluator_tools_for_mode "$EVAL_MODE") + phase "GAN-STYLE HARNESS — Setup" log "Brief: ${CYAN}${BRIEF}${NC}" @@ -205,8 +256,14 @@ Update gan-harness/generator-state.md." \ # ── EVALUATE ── echo -e "${RED}>> EVALUATOR (iteration $i)${NC}" + if [ "$EVAL_MODE" = "playwright" ] && ! playwright_mcp_is_connected; then + fail "The Playwright MCP server disconnected before evaluator iteration $i." + fail "Run 'claude mcp get playwright' to inspect its status, then retry the harness." + exit 1 + fi + claude -p --model "$EVALUATOR_MODEL" \ - --allowedTools "Read,Write,Bash,Grep,Glob" \ + --allowedTools "$EVALUATOR_TOOLS" \ "You are the Evaluator in a GAN-style harness. Read agents/gan-evaluator.md for full instructions. Iteration: $i diff --git a/tests/gan-harness.test.js b/tests/gan-harness.test.js index 36c7255ce..6a5041f92 100644 --- a/tests/gan-harness.test.js +++ b/tests/gan-harness.test.js @@ -12,7 +12,9 @@ const { spawnSync } = require('child_process'); const repoRoot = path.resolve(__dirname, '..'); const harnessPath = path.join(repoRoot, 'scripts', 'gan-harness.sh'); +const evaluatorPath = path.join(repoRoot, 'agents', 'gan-evaluator.md'); const harnessSource = fs.readFileSync(harnessPath, 'utf8'); +const evaluatorSource = fs.readFileSync(evaluatorPath, 'utf8'); if (process.platform === 'win32') { console.log('\n=== GAN harness helpers ===\n'); @@ -34,15 +36,22 @@ function test(name, fn) { } } -function runHarnessScript(script, args = []) { +function runHarnessScript(script, args = [], env = {}) { const bashExecutable = process.platform === 'win32' ? 'bash' : '/bin/bash'; const result = spawnSync(bashExecutable, ['-c', script, 'gan-harness-test', ...args], { encoding: 'utf8', + env: { ...process.env, ...env }, }); assert.strictEqual(result.status, 0, result.stderr || 'GAN harness script failed'); return result.stdout.trim(); } +function extractShellFunction(name) { + const functionMatch = harnessSource.match(new RegExp(`${name}\\(\\) \\{[\\s\\S]*?\\n\\}`)); + assert.ok(functionMatch, `expected scripts/gan-harness.sh to define ${name}`); + return functionMatch[0]; +} + function extractScore(feedback) { const functionMatch = harnessSource.match(/extract_score\(\) \{[\s\S]*?\n\}/); assert.ok(functionMatch, 'expected scripts/gan-harness.sh to define extract_score'); @@ -58,6 +67,39 @@ function extractScore(feedback) { } } +function probePlaywright(statusLine, commandStatus = 0) { + const script = [ + 'claude() {', + ' [ "$#" -eq 3 ] && [ "$1" = mcp ] && [ "$2" = get ] && [ "$3" = playwright ] || return 64', + ' [ "$NO_COLOR" = 1 ] || return 65', + " printf '%s\\n' \"$GAN_TEST_MCP_STATUS\"", + ' return "$GAN_TEST_MCP_EXIT"', + '}', + extractShellFunction('playwright_mcp_is_connected'), + 'if playwright_mcp_is_connected; then printf connected; else printf unavailable; fi', + ].join('\n'); + + return runHarnessScript(script, [], { + GAN_TEST_MCP_STATUS: statusLine, + GAN_TEST_MCP_EXIT: String(commandStatus), + }); +} + +function evaluatorToolsForMode(mode) { + return runHarnessScript( + `${extractShellFunction('evaluator_tools_for_mode')}\nevaluator_tools_for_mode "$1"`, + [mode] + ).split(','); +} + +function declaredEvaluatorTools() { + const frontmatter = evaluatorSource.match(/^---\r?\n([\s\S]*?)\r?\n---/); + assert.ok(frontmatter, 'expected agents/gan-evaluator.md to have frontmatter'); + const toolsLine = frontmatter[1].match(/^tools:\s*(.+)$/m); + assert.ok(toolsLine, 'expected agents/gan-evaluator.md to declare tools'); + return toolsLine[1].split(',').map(tool => tool.trim()); +} + console.log('\n=== GAN harness helpers ===\n'); const results = Object.freeze([ @@ -106,6 +148,48 @@ const results = Object.freeze([ assert.strictEqual(result, '0.0'); }), + test('Playwright preflight accepts only an explicitly connected server', () => { + assert.strictEqual(probePlaywright('Status: \u2713 Connected'), 'connected'); + assert.strictEqual(probePlaywright('Status: \u2714 Connected'), 'connected'); + for (const unavailableStatus of [ + 'Status: ! Connected \u00b7 tools fetch failed', + 'Status: ! Needs authentication', + 'Status: \u2718 Failed to connect', + 'Status: \u23f8 Pending approval', + 'Status: \u2298 Disabled for this project', + '', + ]) { + assert.strictEqual(probePlaywright(unavailableStatus), 'unavailable'); + } + assert.strictEqual(probePlaywright('Status: \u2713 Connected', 1), 'unavailable'); + assert.strictEqual( + probePlaywright('Status: \u2718 Failed to connect\nStatus: \u2713 Connected'), + 'unavailable' + ); + }), + + test('evaluator tools follow mode and reuse the approved agent contract', () => { + assert.deepStrictEqual(evaluatorToolsForMode('playwright'), declaredEvaluatorTools()); + for (const mode of ['screenshot', 'code-only']) { + assert.deepStrictEqual( + evaluatorToolsForMode(mode), + ['Read', 'Write', 'Bash', 'Grep', 'Glob'] + ); + } + }), + + test('Playwright is checked before setup and again before evaluator launch', () => { + const preflightCall = harnessSource.indexOf('if ! playwright_mcp_is_connected'); + const setupMutation = harnessSource.indexOf('mkdir -p "$FEEDBACK_DIR"'); + const runtimeCheck = harnessSource.indexOf('[ "$EVAL_MODE" = "playwright" ] && ! playwright_mcp_is_connected'); + const evaluatorLaunch = harnessSource.indexOf('claude -p --model "$EVALUATOR_MODEL"'); + + assert.ok(preflightCall >= 0 && preflightCall < setupMutation); + assert.ok(runtimeCheck >= 0 && runtimeCheck < evaluatorLaunch); + assert.match(harnessSource, /--allowedTools "\$EVALUATOR_TOOLS"/); + assert.match(harnessSource, /Unsupported GAN_EVAL_MODE/); + }), + test('final score lookup is compatible with the macOS Bash 3.2 runtime', () => { const finalScoreBlock = harnessSource.match( /NUM_ITERATIONS=\$\{#SCORES\[@\]\}\nif \[ "\$NUM_ITERATIONS"[\s\S]*?\nfi/