ECC/tests/hooks/skill-run-tracker.test.js

/**
 * Tests for scripts/hooks/skill-run-tracker.js and the JSONL sink bounds in
 * scripts/lib/skill-evolution/tracker.js (#2463).
 *
 * Focus: the tracker records real runs, and it never persists prompt text,
 * unbounded strings, an unbounded file, or a world-readable sink.
 *
 * Run with: node tests/run-all.js
 */

'use strict';

const assert = require('assert');
const fs = require('fs');
const os = require('os');
const path = require('path');

const { buildRecord, deriveOutcome, extractSkillId, isSkillToolCall, run } = require('../../scripts/hooks/skill-run-tracker');
const { readHooksConfig } = require('../../scripts/lib/hooks-config');
const {
  MAX_RUN_RECORDS,
  RUNS_FILE_MODE,
  getRunsFilePath,
  recordSkillExecution,
  readSkillExecutionRecords,
} = require('../../scripts/lib/skill-evolution/tracker');

let passed = 0;
let failed = 0;

function test(name, fn) {
  try {
    fn();
    console.log(`  ✓ ${name}`);
    passed++;
  } catch (err) {
    console.log(`  ✗ ${name}`);
    console.log(`    Error: ${err.message}`);
    failed++;
  }
}

function withTempHome(fn) {
  const homeDir = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-skill-runs-'));
  try {
    return fn(homeDir);
  } finally {
    fs.rmSync(homeDir, { recursive: true, force: true });
  }
}

function payload(overrides = {}) {
  return {
    hook_event_name: 'PostToolUse',
    tool_name: 'Skill',
    tool_input: { skill_id: 'code-review', skill_version: '1.2.0' },
    tool_response: {},
    ...overrides,
  };
}

// ── skill id extraction and bounds ────────────────────────────────────────────

test('extractSkillId probes the field names Claude Code has used', () => {
  assert.strictEqual(extractSkillId({ skill_id: 'a' }), 'a');
  assert.strictEqual(extractSkillId({ skillId: 'b' }), 'b');
  assert.strictEqual(extractSkillId({ skill: 'c' }), 'c');
  assert.strictEqual(extractSkillId({ name: 'd' }), 'd');
  assert.strictEqual(extractSkillId({ command: 'e' }), 'e');
  assert.strictEqual(extractSkillId('bare-string'), 'bare-string');
});

test('extractSkillId returns null when no skill id is present', () => {
  assert.strictEqual(extractSkillId({}), null);
  assert.strictEqual(extractSkillId(null), null);
  assert.strictEqual(extractSkillId(42), null);
});

test('extractSkillId rejects an over-long identifier rather than truncating it', () => {
  assert.strictEqual(extractSkillId({ skill_id: 'x'.repeat(129) }), null);
  assert.strictEqual(extractSkillId({ skill_id: 'x'.repeat(128) }), 'x'.repeat(128));
});

test('extractSkillId rejects free text that is not identifier-shaped', () => {
  // A prompt smuggled into the skill field must not become a persisted id.
  assert.strictEqual(extractSkillId({ skill_id: 'summarize this: my api key is sk-abc' }), null);
  assert.strictEqual(extractSkillId({ skill_id: 'line\nbreak' }), null);
  assert.strictEqual(extractSkillId({ skill_id: '   ' }), null);
});

// ── privacy: no prompt text is persisted ─────────────────────────────────────

test('buildRecord synthesizes task_description and never copies prompt fields', () => {
  const secret = 'PROMPT-SECRET-do-not-persist';
  const record = buildRecord(payload({
    tool_input: {
      skill_id: 'code-review',
      skill_version: '1.2.0',
      task_description: secret,
      description: secret,
      prompt: secret,
      taskDescription: secret,
    },
  }));

  assert.strictEqual(record.task_description, 'Skill invocation: code-review');
  assert.ok(!JSON.stringify(record).includes(secret), 'record must not contain any prompt text');
});

test('buildRecord persists only the four dashboard fields', () => {
  const record = buildRecord(payload());
  assert.deepStrictEqual(
    Object.keys(record).sort(),
    ['outcome', 'skill_id', 'skill_version', 'task_description']
  );
});

test('buildRecord drops a non-identifier skill_version instead of persisting it', () => {
  const record = buildRecord(payload({
    tool_input: { skill_id: 'code-review', skill_version: 'v1 (as requested by the user in chat)' },
  }));
  assert.strictEqual(record.skill_version, 'unknown');
});

test('buildRecord returns null when the skill id is unusable', () => {
  assert.strictEqual(buildRecord(payload({ tool_input: {} })), null);
});

// ── outcome derivation ───────────────────────────────────────────────────────

test('deriveOutcome reports failure for PostToolUseFailure routing', () => {
  assert.strictEqual(deriveOutcome(payload({ hook_event_name: 'PostToolUseFailure' })), 'failure');
});

test('deriveOutcome reports failure for error-bearing tool responses', () => {
  assert.strictEqual(deriveOutcome(payload({ tool_response: { is_error: true } })), 'failure');
  assert.strictEqual(deriveOutcome(payload({ tool_response: { isError: true } })), 'failure');
  assert.strictEqual(deriveOutcome(payload({ tool_response: { status: 'ERROR' } })), 'failure');
  assert.strictEqual(deriveOutcome(payload({ tool_response: { error: 'boom' } })), 'failure');
});

test('deriveOutcome reports success otherwise', () => {
  assert.strictEqual(deriveOutcome(payload()), 'success');
  assert.strictEqual(deriveOutcome(payload({ tool_response: { status: 'ok' } })), 'success');
});

// ── hook behaviour ───────────────────────────────────────────────────────────

test('run ignores non-Skill tools and malformed input without throwing', () => {
  assert.doesNotThrow(() => run(JSON.stringify(payload({ tool_name: 'Bash' }))));
  assert.doesNotThrow(() => run('not json'));
  assert.doesNotThrow(() => run(''));
});

// ── tool-name tolerance ───────────────────────────────────────────────────────
// OpenCode names the tool `skill`. A case-sensitive matcher meant the hook ran,
// dropped every event, and the dashboard kept reporting "unmeasured" with no
// indication that a writer existed at all.

test('isSkillToolCall accepts both harness spellings and rejects everything else', () => {
  assert.strictEqual(isSkillToolCall({ tool_name: 'Skill' }), true);
  assert.strictEqual(isSkillToolCall({ tool_name: 'skill' }), true);
  assert.strictEqual(isSkillToolCall({ tool_name: '  SKILL  ' }), true);
  assert.strictEqual(isSkillToolCall({ tool_name: 'Bash' }), false);
  assert.strictEqual(isSkillToolCall({ tool_name: '' }), false);
  assert.strictEqual(isSkillToolCall({ tool_name: 42 }), false);
  assert.strictEqual(isSkillToolCall({}), false);
  assert.strictEqual(isSkillToolCall(null), false);
});

test('run records a run for a lowercase skill tool name', () => {
  withTempHome(homeDir => {
    const previousHome = process.env.HOME;
    const previousUserProfile = process.env.USERPROFILE;
    process.env.HOME = homeDir;
    process.env.USERPROFILE = homeDir;
    try {
      run(payload({ tool_name: 'skill' }));

      const records = readSkillExecutionRecords({ homeDir });
      assert.strictEqual(records.length, 1);
      assert.strictEqual(records[0].skill_id, 'code-review');
    } finally {
      if (previousHome === undefined) delete process.env.HOME; else process.env.HOME = previousHome;
      if (previousUserProfile === undefined) delete process.env.USERPROFILE; else process.env.USERPROFILE = previousUserProfile;
    }
  });
});

test('run still records nothing for a non-skill tool', () => {
  withTempHome(homeDir => {
    const previousHome = process.env.HOME;
    const previousUserProfile = process.env.USERPROFILE;
    process.env.HOME = homeDir;
    process.env.USERPROFILE = homeDir;
    try {
      run(payload({ tool_name: 'bash' }));
      assert.strictEqual(readSkillExecutionRecords({ homeDir }).length, 0);
    } finally {
      if (previousHome === undefined) delete process.env.HOME; else process.env.HOME = previousHome;
      if (previousUserProfile === undefined) delete process.env.USERPROFILE; else process.env.USERPROFILE = previousUserProfile;
    }
  });
});

// ── JSONL sink bounds ────────────────────────────────────────────────────────

test('recordSkillExecution writes the sink owner-only', function () {
  if (process.platform === 'win32') {
    return; // POSIX modes are not meaningful on Windows
  }
  withTempHome(homeDir => {
    recordSkillExecution(
      { skill_id: 'code-review', skill_version: '1.0.0', task_description: 'Skill invocation: code-review', outcome: 'success' },
      { homeDir }
    );
    const runsFilePath = getRunsFilePath({ homeDir });
    const mode = fs.statSync(runsFilePath).mode & 0o777;
    assert.strictEqual(mode, RUNS_FILE_MODE, `expected mode ${RUNS_FILE_MODE.toString(8)}, got ${mode.toString(8)}`);
  });
});

test('recordSkillExecution re-tightens an already world-readable sink', function () {
  if (process.platform === 'win32') {
    return;
  }
  withTempHome(homeDir => {
    const runsFilePath = getRunsFilePath({ homeDir });
    fs.mkdirSync(path.dirname(runsFilePath), { recursive: true });
    fs.writeFileSync(runsFilePath, '', { mode: 0o644 });

    recordSkillExecution(
      { skill_id: 'code-review', skill_version: '1.0.0', task_description: 'Skill invocation: code-review', outcome: 'success' },
      { homeDir }
    );

    assert.strictEqual(fs.statSync(runsFilePath).mode & 0o777, RUNS_FILE_MODE);
  });
});

test('the JSONL sink is bounded by a retention cap', () => {
  withTempHome(homeDir => {
    const maxRecords = 5;
    for (let i = 0; i < maxRecords + 4; i++) {
      recordSkillExecution(
        { skill_id: `skill-${i}`, skill_version: '1.0.0', task_description: `Skill invocation: skill-${i}`, outcome: 'success' },
        { homeDir, maxRecords }
      );
    }

    const records = readSkillExecutionRecords({ homeDir });
    assert.strictEqual(records.length, maxRecords, 'sink must be trimmed to the cap');
    // Trimming keeps the newest runs, so the dashboard still reflects recent activity.
    assert.strictEqual(records[records.length - 1].skill_id, `skill-${maxRecords + 3}`);
    assert.strictEqual(records[0].skill_id, `skill-${4}`);
  });
});

test('the default retention cap is a finite bound', () => {
  assert.ok(Number.isInteger(MAX_RUN_RECORDS) && MAX_RUN_RECORDS > 0, 'MAX_RUN_RECORDS must be a positive integer');
});

test('an end-to-end Skill hook run lands exactly one non-sensitive record', () => {
  withTempHome(homeDir => {
    const previousHome = process.env.HOME;
    const previousUserProfile = process.env.USERPROFILE;
    process.env.HOME = homeDir;
    process.env.USERPROFILE = homeDir;
    try {
      run(JSON.stringify(payload({
        tool_input: { skill_id: 'code-review', skill_version: '1.2.0', prompt: 'PROMPT-SECRET' },
      })));

      const records = readSkillExecutionRecords({ homeDir });
      assert.strictEqual(records.length, 1);
      assert.strictEqual(records[0].skill_id, 'code-review');
      assert.strictEqual(records[0].skill_version, '1.2.0');
      assert.strictEqual(records[0].outcome, 'success');
      assert.ok(!JSON.stringify(records[0]).includes('PROMPT-SECRET'));
    } finally {
      if (previousHome === undefined) delete process.env.HOME; else process.env.HOME = previousHome;
      if (previousUserProfile === undefined) delete process.env.USERPROFILE; else process.env.USERPROFILE = previousUserProfile;
    }
  });
});

// deriveOutcome treats PostToolUseFailure as a hard failure. That branch is
// only reachable if the hook is actually registered for the event: the
// PostToolUse dispatcher does not fan out PostToolUseFailure, so the tracker
// needs its own hooks.json entry. Without it, hard Skill failures are silently
// dropped and the dashboard's success rate is inflated.
test('the tracker is registered for PostToolUseFailure so hard failures are recorded', () => {
  const hooksConfig = readHooksConfig(path.join(__dirname, '..', '..', 'hooks', 'hooks.json'));
  const entries = (hooksConfig.hooks.PostToolUseFailure || [])
    .filter(entry => entry.id === 'post:skill:track');

  assert.strictEqual(entries.length, 1, 'expected one post:skill:track PostToolUseFailure entry');
  assert.strictEqual(entries[0].matcher, 'Skill', 'tracker must only match the Skill tool');
  assert.ok(
    entries[0].hooks[0].command.includes('scripts/hooks/skill-run-tracker.js'),
    'entry should invoke skill-run-tracker.js'
  );
});

console.log(`\nPassed: ${passed}`);
console.log(`Failed: ${failed}`);
if (failed > 0) {
  process.exitCode = 1;
}