ECC/tests/ci/validate-agents-tools.test.js

/**
 * Focused tests for validate-agents.js tools frontmatter rules.
 *
 * Run with: node tests/ci/validate-agents-tools.test.js
 */

const assert = require('assert');
const path = require('path');
const fs = require('fs');
const os = require('os');
const { execFileSync } = require('child_process');

const validatorsDir = path.join(__dirname, '..', '..', 'scripts', 'ci');
const repoRoot = path.join(__dirname, '..', '..');
const canonicalAgentsDir = path.join(repoRoot, 'agents');

function test(name, fn) {
  try {
    fn();
    console.log(`  \u2713 ${name}`);
    return true;
  } catch (err) {
    console.log(`  \u2717 ${name}`);
    console.log(`    Error: ${err.message}`);
    return false;
  }
}

function createTestDir() {
  return fs.mkdtempSync(path.join(os.tmpdir(), 'validate-agents-tools-test-'));
}

function cleanupTestDir(testDir) {
  fs.rmSync(testDir, { recursive: true, force: true });
}

function stripShebang(source) {
  let s = source;
  if (s.charCodeAt(0) === 0xFEFF) s = s.slice(1);
  if (s.startsWith('#!')) {
    const nl = s.indexOf('\n');
    s = nl === -1 ? '' : s.slice(nl + 1);
  }
  return s;
}

function runSourceViaTempFile(source) {
  const tmpFile = path.join(repoRoot, `.tmp-validator-${Date.now()}-${Math.random().toString(36).slice(2)}.js`);
  try {
    fs.writeFileSync(tmpFile, source, 'utf8');
    const stdout = execFileSync('node', [tmpFile], {
      encoding: 'utf8',
      stdio: ['pipe', 'pipe', 'pipe'],
      timeout: 10000,
      cwd: repoRoot,
    });
    return { code: 0, stdout, stderr: '' };
  } catch (err) {
    return {
      code: err.status || 1,
      stdout: err.stdout || '',
      stderr: err.stderr || '',
    };
  } finally {
    fs.rmSync(tmpFile, { force: true });
  }
}

function runValidatorWithDir(validatorName, dirConstant, overridePath) {
  const validatorPath = path.join(validatorsDir, `${validatorName}.js`);
  let source = fs.readFileSync(validatorPath, 'utf8');
  source = stripShebang(source);
  const dirRegex = new RegExp(`const ${dirConstant} = .*?;`);
  source = source.replace(dirRegex, `const ${dirConstant} = ${JSON.stringify(overridePath)};`);
  return runSourceViaTempFile(source);
}

function readCanonicalAgent(file) {
  const resolvedPath = path.resolve(canonicalAgentsDir, file);
  const agentsRoot = path.resolve(canonicalAgentsDir);
  assert.ok(
    resolvedPath.startsWith(`${agentsRoot}${path.sep}`),
    `${file} should resolve inside the canonical agents directory`
  );
  return fs.readFileSync(resolvedPath, 'utf8');
}

function runTests() {
  console.log('\n=== Testing validate-agents tools frontmatter ===\n');

  let passed = 0;
  let failed = 0;

  if (test('canonical agents declare tools as comma-separated scalars', () => {
    const agentFiles = fs.readdirSync(canonicalAgentsDir).filter(file => file.endsWith('.md'));

    for (const file of agentFiles) {
      const content = readCanonicalAgent(file);
      const frontmatter = content.match(/^---\r?\n([\s\S]*?)\r?\n---/);
      assert.ok(frontmatter, `${file} should have frontmatter`);

      const toolsLine = frontmatter[1].match(/^tools:\s*(.+)$/m);
      assert.ok(toolsLine, `${file} should declare a non-empty tools scalar`);
      assert.ok(
        !toolsLine[1].trim().startsWith('['),
        `${file} should use comma-separated scalar tools, not a YAML sequence`
      );
    }
  })) passed++; else failed++;

  if (test('accepts comma-separated scalar agent tools', () => {
    const testDir = createTestDir();
    try {
      fs.writeFileSync(path.join(testDir, 'scalar-tools.md'), '---\nmodel: sonnet\ntools: Read, Glob, Grep\n---\n# Agent');

      const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir);
      assert.strictEqual(result.code, 0, `Should accept scalar tools, got stderr: ${result.stderr}`);
    } finally {
      cleanupTestDir(testDir);
    }
  })) passed++; else failed++;

  if (test('rejects YAML sequence-form agent tools', () => {
    const testDir = createTestDir();
    try {
      fs.writeFileSync(path.join(testDir, 'sequence-tools.md'), '---\nmodel: sonnet\ntools: [Read, Glob, Grep]\n---\n# Agent');

      const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir);
      assert.strictEqual(result.code, 1, 'Should reject sequence-form tools');
      assert.ok(
        result.stderr.includes('comma-separated scalar'),
        `Should explain the supported tools format, got stderr: ${result.stderr}`
      );
    } finally {
      cleanupTestDir(testDir);
    }
  })) passed++; else failed++;

  if (test('rejects block sequence-form agent tools', () => {
    const testDir = createTestDir();
    try {
      fs.writeFileSync(path.join(testDir, 'block-sequence-tools.md'), '---\nmodel: sonnet\ntools:\n  - Read\n  - Glob\n  - Grep\n---\n# Agent');

      const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir);
      assert.strictEqual(result.code, 1, 'Should reject block sequence-form tools');
      assert.ok(
        result.stderr.includes('comma-separated scalar'),
        `Should explain the supported tools format, got stderr: ${result.stderr}`
      );
    } finally {
      cleanupTestDir(testDir);
    }
  })) passed++; else failed++;

  if (test('rejects explicitly tagged YAML sequence-form agent tools', () => {
    const testDir = createTestDir();
    try {
      fs.writeFileSync(path.join(testDir, 'tagged-sequence-tools.md'), '---\nmodel: sonnet\ntools: !!seq [Read, Glob, Grep]\n---\n# Agent');

      const result = runValidatorWithDir('validate-agents', 'AGENTS_DIR', testDir);
      assert.strictEqual(result.code, 1, 'Should reject tagged sequence-form tools');
      assert.ok(
        result.stderr.includes('comma-separated scalar'),
        `Should explain the supported tools format, got stderr: ${result.stderr}`
      );
    } finally {
      cleanupTestDir(testDir);
    }
  })) passed++; else failed++;

  console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
  process.exit(failed > 0 ? 1 : 0);
}

runTests();