fix(gan): require Playwright capability in harness

2026-09-30 21:45:13 +02:00 · 2026-09-04 20:48:42 +08:00
parent e04ea0b9cc
commit 6f9282af41
2 changed files with 143 additions and 2 deletions
@@ -19,6 +19,7 @@
 #   GAN_PROJECT_DIR     — Working directory (default: current dir)
 #   GAN_SKIP_PLANNER    — Set to "true" to skip planner phase
 #   GAN_EVAL_MODE       — playwright, screenshot, or code-only (default: playwright)
+#                        playwright requires a connected MCP server named "playwright"

 set -euo pipefail

@@ -96,6 +97,38 @@ score_passes() {
  awk -v s="$score" -v t="$threshold" 'BEGIN { exit !(s >= t) }'
 }

+playwright_mcp_is_connected() {
+  local status
+  local status_value
+  local check_mark
+  local heavy_check_mark
+  status=$(NO_COLOR=1 claude mcp get playwright 2>/dev/null) || return 1
+  status_value=$(printf '%s\n' "$status" | awk '
+    /^[[:space:]]*Status:[[:space:]]*/ {
+      sub(/^[[:space:]]*Status:[[:space:]]*/, "")
+      sub(/[[:space:]]*$/, "")
+      print
+      exit
+    }
+  ')
+  check_mark=$(printf '\342\234\223')
+  heavy_check_mark=$(printf '\342\234\224')
+
+  [ "$status_value" = "$check_mark Connected" ] || \
+    [ "$status_value" = "$heavy_check_mark Connected" ]
+}
+
+evaluator_tools_for_mode() {
+  local base_tools="Read,Write,Bash,Grep,Glob"
+  local playwright_tools="mcp__playwright__browser_navigate,mcp__playwright__browser_click,mcp__playwright__browser_take_screenshot,mcp__playwright__browser_snapshot,mcp__playwright__browser_type,mcp__playwright__browser_fill_form"
+
+  if [ "$1" = "playwright" ]; then
+    printf '%s,%s\n' "$base_tools" "$playwright_tools"
+  else
+    printf '%s\n' "$base_tools"
+  fi
+}
+
 elapsed() {
  local now=$(date +%s)
  local diff=$((now - START_TIME))
@@ -104,6 +137,24 @@ elapsed() {

 # ─── Setup ───────────────────────────────────────────────────────────────────

+case "$EVAL_MODE" in
+  playwright)
+    if ! playwright_mcp_is_connected; then
+      fail "GAN_EVAL_MODE=playwright requires a connected MCP server named 'playwright'."
+      fail "Run 'claude mcp get playwright' to inspect its status, or choose GAN_EVAL_MODE=screenshot or code-only."
+      exit 1
+    fi
+    ;;
+  screenshot|code-only)
+    ;;
+  *)
+    fail "Unsupported GAN_EVAL_MODE. Expected playwright, screenshot, or code-only."
+    exit 1
+    ;;
+esac
+
+EVALUATOR_TOOLS=$(evaluator_tools_for_mode "$EVAL_MODE")
+
 phase "GAN-STYLE HARNESS — Setup"

 log "Brief: ${CYAN}${BRIEF}${NC}"
@@ -205,8 +256,14 @@ Update gan-harness/generator-state.md." \
  # ── EVALUATE ──
  echo -e "${RED}>> EVALUATOR (iteration $i)${NC}"

+  if [ "$EVAL_MODE" = "playwright" ] && ! playwright_mcp_is_connected; then
+    fail "The Playwright MCP server disconnected before evaluator iteration $i."
+    fail "Run 'claude mcp get playwright' to inspect its status, then retry the harness."
+    exit 1
+  fi
+
  claude -p --model "$EVALUATOR_MODEL" \
-    --allowedTools "Read,Write,Bash,Grep,Glob" \
+    --allowedTools "$EVALUATOR_TOOLS" \
    "You are the Evaluator in a GAN-style harness. Read agents/gan-evaluator.md for full instructions.

 Iteration: $i
@@ -12,7 +12,9 @@ const { spawnSync } = require('child_process');

 const repoRoot = path.resolve(__dirname, '..');
 const harnessPath = path.join(repoRoot, 'scripts', 'gan-harness.sh');
+const evaluatorPath = path.join(repoRoot, 'agents', 'gan-evaluator.md');
 const harnessSource = fs.readFileSync(harnessPath, 'utf8');
+const evaluatorSource = fs.readFileSync(evaluatorPath, 'utf8');

 if (process.platform === 'win32') {
  console.log('\n=== GAN harness helpers ===\n');
@@ -34,15 +36,22 @@ function test(name, fn) {
  }
 }

-function runHarnessScript(script, args = []) {
+function runHarnessScript(script, args = [], env = {}) {
  const bashExecutable = process.platform === 'win32' ? 'bash' : '/bin/bash';
  const result = spawnSync(bashExecutable, ['-c', script, 'gan-harness-test', ...args], {
    encoding: 'utf8',
+    env: { ...process.env, ...env },
  });
  assert.strictEqual(result.status, 0, result.stderr || 'GAN harness script failed');
  return result.stdout.trim();
 }

+function extractShellFunction(name) {
+  const functionMatch = harnessSource.match(new RegExp(`${name}\\(\\) \\{[\\s\\S]*?\\n\\}`));
+  assert.ok(functionMatch, `expected scripts/gan-harness.sh to define ${name}`);
+  return functionMatch[0];
+}
+
 function extractScore(feedback) {
  const functionMatch = harnessSource.match(/extract_score\(\) \{[\s\S]*?\n\}/);
  assert.ok(functionMatch, 'expected scripts/gan-harness.sh to define extract_score');
@@ -58,6 +67,39 @@ function extractScore(feedback) {
  }
 }

+function probePlaywright(statusLine, commandStatus = 0) {
+  const script = [
+    'claude() {',
+    '  [ "$#" -eq 3 ] && [ "$1" = mcp ] && [ "$2" = get ] && [ "$3" = playwright ] || return 64',
+    '  [ "$NO_COLOR" = 1 ] || return 65',
+    "  printf '%s\\n' \"$GAN_TEST_MCP_STATUS\"",
+    '  return "$GAN_TEST_MCP_EXIT"',
+    '}',
+    extractShellFunction('playwright_mcp_is_connected'),
+    'if playwright_mcp_is_connected; then printf connected; else printf unavailable; fi',
+  ].join('\n');
+
+  return runHarnessScript(script, [], {
+    GAN_TEST_MCP_STATUS: statusLine,
+    GAN_TEST_MCP_EXIT: String(commandStatus),
+  });
+}
+
+function evaluatorToolsForMode(mode) {
+  return runHarnessScript(
+    `${extractShellFunction('evaluator_tools_for_mode')}\nevaluator_tools_for_mode "$1"`,
+    [mode]
+  ).split(',');
+}
+
+function declaredEvaluatorTools() {
+  const frontmatter = evaluatorSource.match(/^---\r?\n([\s\S]*?)\r?\n---/);
+  assert.ok(frontmatter, 'expected agents/gan-evaluator.md to have frontmatter');
+  const toolsLine = frontmatter[1].match(/^tools:\s*(.+)$/m);
+  assert.ok(toolsLine, 'expected agents/gan-evaluator.md to declare tools');
+  return toolsLine[1].split(',').map(tool => tool.trim());
+}
+
 console.log('\n=== GAN harness helpers ===\n');

 const results = Object.freeze([
@@ -106,6 +148,48 @@ const results = Object.freeze([
    assert.strictEqual(result, '0.0');
  }),

+  test('Playwright preflight accepts only an explicitly connected server', () => {
+    assert.strictEqual(probePlaywright('Status: \u2713 Connected'), 'connected');
+    assert.strictEqual(probePlaywright('Status: \u2714 Connected'), 'connected');
+    for (const unavailableStatus of [
+      'Status: ! Connected \u00b7 tools fetch failed',
+      'Status: ! Needs authentication',
+      'Status: \u2718 Failed to connect',
+      'Status: \u23f8 Pending approval',
+      'Status: \u2298 Disabled for this project',
+      '',
+    ]) {
+      assert.strictEqual(probePlaywright(unavailableStatus), 'unavailable');
+    }
+    assert.strictEqual(probePlaywright('Status: \u2713 Connected', 1), 'unavailable');
+    assert.strictEqual(
+      probePlaywright('Status: \u2718 Failed to connect\nStatus: \u2713 Connected'),
+      'unavailable'
+    );
+  }),
+
+  test('evaluator tools follow mode and reuse the approved agent contract', () => {
+    assert.deepStrictEqual(evaluatorToolsForMode('playwright'), declaredEvaluatorTools());
+    for (const mode of ['screenshot', 'code-only']) {
+      assert.deepStrictEqual(
+        evaluatorToolsForMode(mode),
+        ['Read', 'Write', 'Bash', 'Grep', 'Glob']
+      );
+    }
+  }),
+
+  test('Playwright is checked before setup and again before evaluator launch', () => {
+    const preflightCall = harnessSource.indexOf('if ! playwright_mcp_is_connected');
+    const setupMutation = harnessSource.indexOf('mkdir -p "$FEEDBACK_DIR"');
+    const runtimeCheck = harnessSource.indexOf('[ "$EVAL_MODE" = "playwright" ] && ! playwright_mcp_is_connected');
+    const evaluatorLaunch = harnessSource.indexOf('claude -p --model "$EVALUATOR_MODEL"');
+
+    assert.ok(preflightCall >= 0 && preflightCall < setupMutation);
+    assert.ok(runtimeCheck >= 0 && runtimeCheck < evaluatorLaunch);
+    assert.match(harnessSource, /--allowedTools "\$EVALUATOR_TOOLS"/);
+    assert.match(harnessSource, /Unsupported GAN_EVAL_MODE/);
+  }),
+
  test('final score lookup is compatible with the macOS Bash 3.2 runtime', () => {
    const finalScoreBlock = harnessSource.match(
      /NUM_ITERATIONS=\$\{#SCORES\[@\]\}\nif \[ "\$NUM_ITERATIONS"[\s\S]*?\nfi/