mirror of
https://github.com/affaan-m/ECC.git
synced 2026-10-14 20:25:26 +02:00
Under OpenCode the skill-health dashboard had no writer and no honest fallback, so it reported every discovered skill as healthy (#2463). Root cause: `post:skill:track` is only wired for Claude Code, where `posttooluse-dispatcher.js` fans the event out to `skill-run-tracker.js`. The OpenCode plugin re-implements the harness events instead of reusing that dispatcher, so nothing ever wrote `skill-runs.jsonl` — and the tracker's matcher only accepted the exact Claude Code tool name `Skill`, while OpenCode calls the tool `skill`, so even wiring it would have dropped every event. Commands that locate ECC through the inlined resolver in `scripts/lib/resolve-ecc-root.js` also fell back to `~/.claude`, which does not contain ECC, so they could not find their scripts. Changes: - `.opencode/plugins/ecc-hooks.ts`: dispatch `post:skill:track` from `tool.execute.after`. The tracker module is imported lazily and behind try/catch, matching the existing `changed-files-store` pattern: this file is OpenCode's startup entry point, so a static import failure would crash the session before any hook runs (#2530). The module is reused rather than reimplemented, keeping its identifier validation, length bounds and "never persist prompt text" guarantees. - `scripts/hooks/skill-run-tracker.js`: match the tool name case-insensitively so `Skill` and `skill` are both accepted. - `scripts/lib/skill-evolution/health.js`: split the summary into measured/healthy/declining/unmeasured instead of deriving `healthy = total - declining`, which turned "no telemetry" into "everything is fine" for every skill. - `scripts/lib/skill-evolution/dashboard.js`: show the breakdown and warn explicitly when skills have no recorded runs, since absence of telemetry is not a health signal. Under OpenCode the resolver also needs `CLAUDE_PLUGIN_ROOT`, which only the plugin can supply: `shell.env` exports it when a root is found, without replacing a root the user set or one an earlier plugin already put in `output.env`. Root discovery is per capability and probes required runtime files — never optional content. Probing for `skills/continuous-learning-v2` (the `workflow-quality` module) rejected supported modular installs that ship the plugin and tracker without that optional module, disabling telemetry and the root export for exactly those users. Also fixes an incidental bug: the compiled plugin bundle is ESM, where `__dirname` is undefined, so the ECC version lookup silently fell back to its default. The module directory is now derived from `import.meta.url`. Tests: `tests/opencode-plugin-hooks.test.js` covers the telemetry dispatch (skill tool recorded, other tools ignored, minimal profile respected), a modular install without the optional skills module, the root export and both override paths; `tests/hooks/skill-run-tracker.test.js` covers the case-insensitive matcher; `tests/lib/skill-evolution.test.js` and `tests/lib/skill-dashboard.test.js` cover the unmeasured bucket, the warning and the aged-out-coverage wording. Refs #2463, #2530
748 lines
28 KiB
JavaScript
748 lines
28 KiB
JavaScript
/**
|
|
* Tests for skill health dashboard.
|
|
*
|
|
* Run with: node tests/lib/skill-dashboard.test.js
|
|
*/
|
|
|
|
const assert = require('assert');
|
|
const fs = require('fs');
|
|
const os = require('os');
|
|
const path = require('path');
|
|
const { spawnSync } = require('child_process');
|
|
|
|
const dashboard = require('../../scripts/lib/skill-evolution/dashboard');
|
|
const versioning = require('../../scripts/lib/skill-evolution/versioning');
|
|
const _provenance = require('../../scripts/lib/skill-evolution/provenance');
|
|
|
|
const HEALTH_SCRIPT = path.join(__dirname, '..', '..', 'scripts', 'skills-health.js');
|
|
|
|
function test(name, fn) {
|
|
try {
|
|
fn();
|
|
console.log(` \u2713 ${name}`);
|
|
return true;
|
|
} catch (error) {
|
|
console.log(` \u2717 ${name}`);
|
|
console.log(` Error: ${error.message}`);
|
|
return false;
|
|
}
|
|
}
|
|
|
|
function createTempDir(prefix) {
|
|
return fs.mkdtempSync(path.join(os.tmpdir(), prefix));
|
|
}
|
|
|
|
function cleanupTempDir(dirPath) {
|
|
fs.rmSync(dirPath, { recursive: true, force: true });
|
|
}
|
|
|
|
function createSkill(skillRoot, name, content) {
|
|
const skillDir = path.join(skillRoot, name);
|
|
fs.mkdirSync(skillDir, { recursive: true });
|
|
fs.writeFileSync(path.join(skillDir, 'SKILL.md'), content);
|
|
return skillDir;
|
|
}
|
|
|
|
function appendJsonl(filePath, rows) {
|
|
const lines = rows.map(row => JSON.stringify(row)).join('\n');
|
|
fs.mkdirSync(path.dirname(filePath), { recursive: true });
|
|
fs.writeFileSync(filePath, `${lines}\n`);
|
|
}
|
|
|
|
function runCli(args) {
|
|
return spawnSync(process.execPath, [HEALTH_SCRIPT, ...args], {
|
|
encoding: 'utf8',
|
|
});
|
|
}
|
|
|
|
function runTests() {
|
|
console.log('\n=== Testing skill dashboard ===\n');
|
|
|
|
let passed = 0;
|
|
let failed = 0;
|
|
|
|
const repoRoot = createTempDir('skill-dashboard-repo-');
|
|
const homeDir = createTempDir('skill-dashboard-home-');
|
|
const skillsRoot = path.join(repoRoot, 'skills');
|
|
const learnedRoot = path.join(homeDir, '.claude', 'skills', 'learned');
|
|
const importedRoot = path.join(homeDir, '.claude', 'skills', 'imported');
|
|
const runsFile = path.join(homeDir, '.claude', 'state', 'skill-runs.jsonl');
|
|
const now = '2026-03-15T12:00:00.000Z';
|
|
|
|
fs.mkdirSync(skillsRoot, { recursive: true });
|
|
fs.mkdirSync(learnedRoot, { recursive: true });
|
|
fs.mkdirSync(importedRoot, { recursive: true });
|
|
|
|
try {
|
|
console.log('Chart primitives:');
|
|
|
|
if (test('sparkline maps float values to Unicode block characters', () => {
|
|
const result = dashboard.sparkline([1, 0.5, 0]);
|
|
assert.strictEqual(result.length, 3);
|
|
assert.strictEqual(result[0], '\u2588');
|
|
assert.strictEqual(result[2], '\u2581');
|
|
})) passed++; else failed++;
|
|
|
|
if (test('sparkline returns empty string for empty array', () => {
|
|
assert.strictEqual(dashboard.sparkline([]), '');
|
|
})) passed++; else failed++;
|
|
|
|
if (test('sparkline renders null values as empty block', () => {
|
|
const result = dashboard.sparkline([null, 0.5, null]);
|
|
assert.strictEqual(result[0], '\u2591');
|
|
assert.strictEqual(result[2], '\u2591');
|
|
assert.strictEqual(result.length, 3);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('horizontalBar renders correct fill ratio', () => {
|
|
const result = dashboard.horizontalBar(5, 10, 10);
|
|
const filled = (result.match(/\u2588/g) || []).length;
|
|
const empty = (result.match(/\u2591/g) || []).length;
|
|
assert.strictEqual(filled, 5);
|
|
assert.strictEqual(empty, 5);
|
|
assert.strictEqual(result.length, 10);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('horizontalBar handles zero value', () => {
|
|
const result = dashboard.horizontalBar(0, 10, 10);
|
|
const filled = (result.match(/\u2588/g) || []).length;
|
|
assert.strictEqual(filled, 0);
|
|
assert.strictEqual(result.length, 10);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('panelBox renders box-drawing characters with title', () => {
|
|
const result = dashboard.panelBox('Test Panel', ['line one', 'line two'], 30);
|
|
assert.match(result, /\u250C/);
|
|
assert.match(result, /\u2510/);
|
|
assert.match(result, /\u2514/);
|
|
assert.match(result, /\u2518/);
|
|
assert.match(result, /Test Panel/);
|
|
assert.match(result, /line one/);
|
|
assert.match(result, /line two/);
|
|
})) passed++; else failed++;
|
|
|
|
console.log('\nTime-series bucketing:');
|
|
|
|
if (test('bucketByDay groups records into daily bins', () => {
|
|
const nowMs = Date.parse(now);
|
|
const records = [
|
|
{ skill_id: 'alpha', outcome: 'success', recorded_at: '2026-03-15T10:00:00.000Z' },
|
|
{ skill_id: 'alpha', outcome: 'failure', recorded_at: '2026-03-15T08:00:00.000Z' },
|
|
{ skill_id: 'alpha', outcome: 'success', recorded_at: '2026-03-14T10:00:00.000Z' },
|
|
];
|
|
|
|
const buckets = dashboard.bucketByDay(records, nowMs, 3);
|
|
assert.strictEqual(buckets.length, 3);
|
|
const todayBucket = buckets[buckets.length - 1];
|
|
assert.strictEqual(todayBucket.runs, 2);
|
|
assert.strictEqual(todayBucket.rate, 0.5);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('bucketByDay returns null rate for empty days', () => {
|
|
const nowMs = Date.parse(now);
|
|
const buckets = dashboard.bucketByDay([], nowMs, 5);
|
|
assert.strictEqual(buckets.length, 5);
|
|
for (const bucket of buckets) {
|
|
assert.strictEqual(bucket.rate, null);
|
|
assert.strictEqual(bucket.runs, 0);
|
|
}
|
|
})) passed++; else failed++;
|
|
|
|
console.log('\nPanel renderers:');
|
|
|
|
const alphaSkillDir = createSkill(skillsRoot, 'alpha', '# Alpha\n');
|
|
const betaSkillDir = createSkill(learnedRoot, 'beta', '# Beta\n');
|
|
|
|
versioning.createVersion(alphaSkillDir, {
|
|
timestamp: '2026-03-14T11:00:00.000Z',
|
|
author: 'observer',
|
|
reason: 'bootstrap',
|
|
});
|
|
|
|
fs.writeFileSync(path.join(alphaSkillDir, 'SKILL.md'), '# Alpha v2\n');
|
|
versioning.createVersion(alphaSkillDir, {
|
|
timestamp: '2026-03-15T11:00:00.000Z',
|
|
author: 'observer',
|
|
reason: 'accepted-amendment',
|
|
});
|
|
|
|
versioning.createVersion(betaSkillDir, {
|
|
timestamp: '2026-03-14T11:00:00.000Z',
|
|
author: 'observer',
|
|
reason: 'bootstrap',
|
|
});
|
|
|
|
const { appendFile } = require('../../scripts/lib/utils');
|
|
const alphaAmendmentsPath = path.join(alphaSkillDir, '.evolution', 'amendments.jsonl');
|
|
appendFile(alphaAmendmentsPath, JSON.stringify({
|
|
event: 'proposal',
|
|
status: 'pending',
|
|
created_at: '2026-03-15T07:00:00.000Z',
|
|
}) + '\n');
|
|
|
|
appendJsonl(runsFile, [
|
|
{
|
|
skill_id: 'alpha',
|
|
skill_version: 'v2',
|
|
task_description: 'Success task',
|
|
outcome: 'success',
|
|
failure_reason: null,
|
|
tokens_used: 100,
|
|
duration_ms: 1000,
|
|
user_feedback: 'accepted',
|
|
recorded_at: '2026-03-14T10:00:00.000Z',
|
|
},
|
|
{
|
|
skill_id: 'alpha',
|
|
skill_version: 'v2',
|
|
task_description: 'Failed task',
|
|
outcome: 'failure',
|
|
failure_reason: 'Regression',
|
|
tokens_used: 100,
|
|
duration_ms: 1000,
|
|
user_feedback: 'rejected',
|
|
recorded_at: '2026-03-13T10:00:00.000Z',
|
|
},
|
|
{
|
|
skill_id: 'alpha',
|
|
skill_version: 'v1',
|
|
task_description: 'Older success',
|
|
outcome: 'success',
|
|
failure_reason: null,
|
|
tokens_used: 100,
|
|
duration_ms: 1000,
|
|
user_feedback: 'accepted',
|
|
recorded_at: '2026-02-20T10:00:00.000Z',
|
|
},
|
|
{
|
|
skill_id: 'beta',
|
|
skill_version: 'v1',
|
|
task_description: 'Beta success',
|
|
outcome: 'success',
|
|
failure_reason: null,
|
|
tokens_used: 90,
|
|
duration_ms: 800,
|
|
user_feedback: 'accepted',
|
|
recorded_at: '2026-03-15T09:00:00.000Z',
|
|
},
|
|
{
|
|
skill_id: 'beta',
|
|
skill_version: 'v1',
|
|
task_description: 'Beta failure',
|
|
outcome: 'failure',
|
|
failure_reason: 'Bad import',
|
|
tokens_used: 90,
|
|
duration_ms: 800,
|
|
user_feedback: 'corrected',
|
|
recorded_at: '2026-02-20T09:00:00.000Z',
|
|
},
|
|
]);
|
|
|
|
const testRecords = [
|
|
{ skill_id: 'alpha', outcome: 'success', failure_reason: null, recorded_at: '2026-03-14T10:00:00.000Z' },
|
|
{ skill_id: 'alpha', outcome: 'failure', failure_reason: 'Regression', recorded_at: '2026-03-13T10:00:00.000Z' },
|
|
{ skill_id: 'alpha', outcome: 'success', failure_reason: null, recorded_at: '2026-02-20T10:00:00.000Z' },
|
|
{ skill_id: 'beta', outcome: 'success', failure_reason: null, recorded_at: '2026-03-15T09:00:00.000Z' },
|
|
{ skill_id: 'beta', outcome: 'failure', failure_reason: 'Bad import', recorded_at: '2026-02-20T09:00:00.000Z' },
|
|
];
|
|
|
|
if (test('renderSuccessRatePanel produces one row per skill with sparklines', () => {
|
|
const skills = [{ skill_id: 'alpha' }, { skill_id: 'beta' }];
|
|
const result = dashboard.renderSuccessRatePanel(testRecords, skills, { now });
|
|
|
|
assert.ok(result.text.includes('Success Rate'));
|
|
assert.ok(result.data.skills.length >= 2);
|
|
|
|
const alpha = result.data.skills.find(s => s.skill_id === 'alpha');
|
|
assert.ok(alpha);
|
|
assert.ok(Array.isArray(alpha.daily_rates));
|
|
assert.strictEqual(alpha.daily_rates.length, 30);
|
|
assert.ok(typeof alpha.sparkline === 'string');
|
|
assert.ok(alpha.sparkline.length > 0);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('renderFailureClusterPanel groups failures by reason', () => {
|
|
const failureRecords = [
|
|
{ skill_id: 'alpha', outcome: 'failure', failure_reason: 'Regression' },
|
|
{ skill_id: 'alpha', outcome: 'failure', failure_reason: 'Regression' },
|
|
{ skill_id: 'beta', outcome: 'failure', failure_reason: 'Bad import' },
|
|
{ skill_id: 'alpha', outcome: 'success', failure_reason: null },
|
|
];
|
|
|
|
const result = dashboard.renderFailureClusterPanel(failureRecords);
|
|
assert.ok(result.text.includes('Failure Patterns'));
|
|
assert.strictEqual(result.data.clusters.length, 2);
|
|
assert.strictEqual(result.data.clusters[0].pattern, 'regression');
|
|
assert.strictEqual(result.data.clusters[0].count, 2);
|
|
assert.strictEqual(result.data.total_failures, 3);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('renderAmendmentPanel lists pending amendments', () => {
|
|
const skillsById = new Map();
|
|
skillsById.set('alpha', { skill_id: 'alpha', skill_dir: alphaSkillDir });
|
|
|
|
const result = dashboard.renderAmendmentPanel(skillsById);
|
|
assert.ok(result.text.includes('Pending Amendments'));
|
|
assert.ok(result.data.total >= 1);
|
|
assert.ok(result.data.amendments.some(a => a.skill_id === 'alpha'));
|
|
})) passed++; else failed++;
|
|
|
|
if (test('renderVersionTimelinePanel shows version history', () => {
|
|
const skillsById = new Map();
|
|
skillsById.set('alpha', { skill_id: 'alpha', skill_dir: alphaSkillDir });
|
|
skillsById.set('beta', { skill_id: 'beta', skill_dir: betaSkillDir });
|
|
|
|
const result = dashboard.renderVersionTimelinePanel(skillsById);
|
|
assert.ok(result.text.includes('Version History'));
|
|
assert.ok(result.data.skills.length >= 1);
|
|
|
|
const alphaVersions = result.data.skills.find(s => s.skill_id === 'alpha');
|
|
assert.ok(alphaVersions);
|
|
assert.ok(alphaVersions.versions.length >= 2);
|
|
})) passed++; else failed++;
|
|
|
|
console.log('\nFull dashboard:');
|
|
|
|
if (test('renderDashboard produces all four panels', () => {
|
|
const result = dashboard.renderDashboard({
|
|
skillsRoot,
|
|
learnedRoot,
|
|
importedRoot,
|
|
homeDir,
|
|
runsFilePath: runsFile,
|
|
now,
|
|
warnThreshold: 0.1,
|
|
});
|
|
|
|
assert.ok(result.text.includes('ECC Skill Health Dashboard'));
|
|
assert.ok(result.text.includes('Success Rate'));
|
|
assert.ok(result.text.includes('Failure Patterns'));
|
|
assert.ok(result.text.includes('Pending Amendments'));
|
|
assert.ok(result.text.includes('Version History'));
|
|
assert.ok(result.data.generated_at === now);
|
|
assert.ok(result.data.summary);
|
|
assert.ok(result.data.panels['success-rate']);
|
|
assert.ok(result.data.panels['failures']);
|
|
assert.ok(result.data.panels['amendments']);
|
|
assert.ok(result.data.panels['versions']);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('renderDashboard supports single panel selection', () => {
|
|
const result = dashboard.renderDashboard({
|
|
skillsRoot,
|
|
learnedRoot,
|
|
importedRoot,
|
|
homeDir,
|
|
runsFilePath: runsFile,
|
|
now,
|
|
panel: 'failures',
|
|
});
|
|
|
|
assert.ok(result.text.includes('Failure Patterns'));
|
|
assert.ok(!result.text.includes('Version History'));
|
|
assert.ok(result.data.panels['failures']);
|
|
assert.ok(!result.data.panels['versions']);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('renderDashboard rejects unknown panel names', () => {
|
|
assert.throws(() => {
|
|
dashboard.renderDashboard({
|
|
skillsRoot,
|
|
learnedRoot,
|
|
importedRoot,
|
|
homeDir,
|
|
runsFilePath: runsFile,
|
|
now,
|
|
panel: 'nonexistent',
|
|
});
|
|
}, /Unknown panel/);
|
|
})) passed++; else failed++;
|
|
|
|
// A panel that reports every never-used skill as healthy is worse than an
|
|
// empty one: it looks like evidence and never raises an alert (#2463).
|
|
if (test('renderDashboard warns that unmeasured skills say nothing about health', () => {
|
|
const result = dashboard.renderDashboard({
|
|
skillsRoot,
|
|
learnedRoot,
|
|
importedRoot,
|
|
homeDir,
|
|
runsFilePath: path.join(homeDir, '.claude', 'state', 'no-such-runs.jsonl'),
|
|
now,
|
|
warnThreshold: 0.1,
|
|
});
|
|
|
|
const summary = result.data.summary;
|
|
assert.ok(summary.unmeasured_skills > 0, 'expected unmeasured skills');
|
|
assert.ok(
|
|
result.text.includes('WARNING'),
|
|
'expected an explicit warning when skills have no telemetry'
|
|
);
|
|
assert.match(result.text, /unmeasured/);
|
|
assert.strictEqual(
|
|
summary.healthy_skills,
|
|
0,
|
|
'skills without runs must not be counted as healthy'
|
|
);
|
|
})) passed++; else failed++;
|
|
|
|
// Isolated roots so the full-coverage case is guaranteed rather than
|
|
// conditional: every discovered skill needs a run inside the 30-day window.
|
|
if (test('renderDashboard omits the warning once every skill is measured', () => {
|
|
const coveredRoot = createTempDir('ecc-dashboard-covered-');
|
|
const coveredLearned = createTempDir('ecc-dashboard-covered-');
|
|
const coveredImported = createTempDir('ecc-dashboard-covered-');
|
|
const coveredRuns = path.join(homeDir, '.claude', 'state', 'covered-skill-runs.jsonl');
|
|
try {
|
|
for (const name of ['covered-a', 'covered-b']) {
|
|
createSkill(coveredRoot, name, `# ${name}\n`);
|
|
}
|
|
appendJsonl(coveredRuns, ['covered-a', 'covered-b'].map(skill_id => ({
|
|
skill_id,
|
|
skill_version: 'v1',
|
|
task_description: `Skill invocation: ${skill_id}`,
|
|
outcome: 'success',
|
|
recorded_at: now,
|
|
})));
|
|
|
|
const result = dashboard.renderDashboard({
|
|
skillsRoot: coveredRoot,
|
|
learnedRoot: coveredLearned,
|
|
importedRoot: coveredImported,
|
|
homeDir,
|
|
runsFilePath: coveredRuns,
|
|
now,
|
|
warnThreshold: 0.1,
|
|
});
|
|
|
|
assert.strictEqual(result.data.summary.unmeasured_skills, 0);
|
|
assert.strictEqual(result.data.summary.measured_skills, 2);
|
|
assert.ok(
|
|
!result.text.includes('WARNING'),
|
|
'no warning expected when every skill has a run in the window'
|
|
);
|
|
assert.strictEqual(
|
|
result.data.summary.total_skills,
|
|
result.data.summary.measured_skills + result.data.summary.unmeasured_skills,
|
|
'measured and unmeasured must partition the total'
|
|
);
|
|
} finally {
|
|
cleanupTempDir(coveredRoot);
|
|
cleanupTempDir(coveredLearned);
|
|
cleanupTempDir(coveredImported);
|
|
}
|
|
})) passed++; else failed++;
|
|
|
|
// The 30-day window can empty out while runs are still retained. Saying
|
|
// "nothing has ever been recorded" then is factually wrong.
|
|
if (test('the unmeasured warning distinguishes aged-out coverage from no history', () => {
|
|
const agedRuns = path.join(homeDir, '.claude', 'state', 'aged-skill-runs.jsonl');
|
|
const agedOut = new Date(Date.parse(now) - (60 * 24 * 60 * 60 * 1000)).toISOString();
|
|
appendJsonl(agedRuns, [{
|
|
skill_id: 'alpha',
|
|
skill_version: 'v1',
|
|
task_description: 'Skill invocation: alpha',
|
|
outcome: 'failure',
|
|
recorded_at: agedOut,
|
|
}]);
|
|
|
|
const result = dashboard.renderDashboard({
|
|
skillsRoot,
|
|
learnedRoot,
|
|
importedRoot,
|
|
homeDir,
|
|
runsFilePath: agedRuns,
|
|
now,
|
|
warnThreshold: 0.1,
|
|
});
|
|
|
|
assert.ok(result.data.summary.unmeasured_skills > 0, 'expected unmeasured skills');
|
|
assert.ok(result.text.includes('WARNING'), 'expected the warning');
|
|
assert.ok(
|
|
result.text.includes('aged out'),
|
|
'retained history outside the window must be described as aged out'
|
|
);
|
|
assert.ok(
|
|
!result.text.includes('Nothing has ever been recorded'),
|
|
'must not claim there is no history when runs are retained'
|
|
);
|
|
})) passed++; else failed++;
|
|
|
|
// Mixed coverage is the common case: some skills used recently, others not.
|
|
// It must not be described as aged out, nor as having no history at all.
|
|
if (test('the unmeasured warning describes partial coverage as partial', () => {
|
|
const mixedRuns = path.join(homeDir, '.claude', 'state', 'mixed-skill-runs.jsonl');
|
|
appendJsonl(mixedRuns, [
|
|
{
|
|
skill_id: 'alpha',
|
|
skill_version: 'v1',
|
|
task_description: 'Skill invocation: alpha',
|
|
outcome: 'success',
|
|
recorded_at: now,
|
|
},
|
|
{
|
|
skill_id: 'beta',
|
|
skill_version: 'v1',
|
|
task_description: 'Skill invocation: beta',
|
|
outcome: 'success',
|
|
recorded_at: new Date(Date.parse(now) - (60 * 24 * 60 * 60 * 1000)).toISOString(),
|
|
},
|
|
]);
|
|
|
|
const result = dashboard.renderDashboard({
|
|
skillsRoot,
|
|
learnedRoot,
|
|
importedRoot,
|
|
homeDir,
|
|
runsFilePath: mixedRuns,
|
|
now,
|
|
warnThreshold: 0.1,
|
|
});
|
|
|
|
assert.ok(result.text.includes('WARNING'), 'expected the warning');
|
|
assert.ok(
|
|
result.text.includes('Coverage is partial'),
|
|
'mixed recent and aged runs must be reported as partial coverage'
|
|
);
|
|
assert.ok(
|
|
!result.text.includes('aged out'),
|
|
'coverage is not aged out while recent runs exist'
|
|
);
|
|
assert.ok(
|
|
!result.text.includes('Nothing has ever been recorded'),
|
|
'runs are retained, so history is not missing'
|
|
);
|
|
})) passed++; else failed++;
|
|
|
|
// With no runs at all the stronger statement is the accurate one.
|
|
if (test('the unmeasured warning says nothing was ever recorded when there is no history', () => {
|
|
const result = dashboard.renderDashboard({
|
|
skillsRoot,
|
|
learnedRoot,
|
|
importedRoot,
|
|
homeDir,
|
|
runsFilePath: path.join(homeDir, '.claude', 'state', 'definitely-absent.jsonl'),
|
|
now,
|
|
warnThreshold: 0.1,
|
|
});
|
|
|
|
assert.ok(
|
|
result.text.includes('Nothing has ever been recorded'),
|
|
'an empty record set must be reported as missing telemetry'
|
|
);
|
|
})) passed++; else failed++;
|
|
|
|
console.log('\nCLI integration:');
|
|
|
|
if (test('CLI --dashboard --json returns valid JSON with all panels', () => {
|
|
const result = runCli([
|
|
'--dashboard',
|
|
'--json',
|
|
'--skills-root', skillsRoot,
|
|
'--learned-root', learnedRoot,
|
|
'--imported-root', importedRoot,
|
|
'--home', homeDir,
|
|
'--runs-file', runsFile,
|
|
'--now', now,
|
|
]);
|
|
|
|
assert.strictEqual(result.status, 0, result.stderr);
|
|
const payload = JSON.parse(result.stdout.trim());
|
|
assert.ok(payload.panels);
|
|
assert.ok(payload.panels['success-rate']);
|
|
assert.ok(payload.panels['failures']);
|
|
assert.ok(payload.summary);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('CLI --panel failures --json returns only the failures panel', () => {
|
|
const result = runCli([
|
|
'--dashboard',
|
|
'--panel', 'failures',
|
|
'--json',
|
|
'--skills-root', skillsRoot,
|
|
'--learned-root', learnedRoot,
|
|
'--imported-root', importedRoot,
|
|
'--home', homeDir,
|
|
'--runs-file', runsFile,
|
|
'--now', now,
|
|
]);
|
|
|
|
assert.strictEqual(result.status, 0, result.stderr);
|
|
const payload = JSON.parse(result.stdout.trim());
|
|
assert.ok(payload.panels['failures']);
|
|
assert.ok(!payload.panels['versions']);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('CLI --help mentions --dashboard', () => {
|
|
const result = runCli(['--help']);
|
|
assert.strictEqual(result.status, 0);
|
|
assert.match(result.stdout, /--dashboard/);
|
|
assert.match(result.stdout, /--panel/);
|
|
})) passed++; else failed++;
|
|
|
|
console.log('\nEdge cases:');
|
|
|
|
if (test('dashboard renders gracefully with no execution records', () => {
|
|
const emptyRunsFile = path.join(homeDir, '.claude', 'state', 'empty-runs.jsonl');
|
|
fs.mkdirSync(path.dirname(emptyRunsFile), { recursive: true });
|
|
fs.writeFileSync(emptyRunsFile, '', 'utf8');
|
|
|
|
const emptySkillsRoot = path.join(repoRoot, 'empty-skills');
|
|
fs.mkdirSync(emptySkillsRoot, { recursive: true });
|
|
|
|
const result = dashboard.renderDashboard({
|
|
skillsRoot: emptySkillsRoot,
|
|
learnedRoot: path.join(homeDir, '.claude', 'skills', 'empty-learned'),
|
|
importedRoot: path.join(homeDir, '.claude', 'skills', 'empty-imported'),
|
|
homeDir,
|
|
runsFilePath: emptyRunsFile,
|
|
now,
|
|
});
|
|
|
|
assert.ok(result.text.includes('ECC Skill Health Dashboard'));
|
|
assert.ok(result.text.includes('No failure patterns detected'));
|
|
assert.strictEqual(result.data.summary.total_skills, 0);
|
|
})) passed++; else failed++;
|
|
|
|
if (test('failure cluster panel handles all successes', () => {
|
|
const successRecords = [
|
|
{ skill_id: 'alpha', outcome: 'success', failure_reason: null },
|
|
{ skill_id: 'beta', outcome: 'success', failure_reason: null },
|
|
];
|
|
|
|
const result = dashboard.renderFailureClusterPanel(successRecords);
|
|
assert.strictEqual(result.data.clusters.length, 0);
|
|
assert.strictEqual(result.data.total_failures, 0);
|
|
assert.ok(result.text.includes('No failure patterns detected'));
|
|
})) passed++; else failed++;
|
|
|
|
if (test('chart helpers handle zero widths, truncation, and invalid buckets', () => {
|
|
assert.strictEqual(dashboard.horizontalBar(10, 0, 4), '\u2591\u2591\u2591\u2591');
|
|
assert.strictEqual(dashboard.horizontalBar(10, 10, 0), '');
|
|
|
|
const boxed = dashboard.panelBox('LongTitleForTinyPanel', ['abcdefghijklmnopqrstuvwxyz'], 10);
|
|
assert.ok(boxed.includes('abcdefgh'), 'long content should be truncated to inner width');
|
|
|
|
const defaultBox = dashboard.panelBox('Default Width', ['ok']);
|
|
assert.ok(defaultBox.split('\n')[0].length >= 60, 'omitted width should use default panel width');
|
|
|
|
const buckets = dashboard.bucketByDay([
|
|
{ skill_id: 'alpha', outcome: 'success', recorded_at: 'not-a-date' },
|
|
{ skill_id: 'alpha', outcome: 'success', recorded_at: now },
|
|
], Date.parse(now), 1);
|
|
assert.strictEqual(buckets[0].runs, 1, 'invalid dates should be ignored');
|
|
})) passed++; else failed++;
|
|
|
|
if (test('success rate panel handles no skills, missing records, and trend directions', () => {
|
|
const empty = dashboard.renderSuccessRatePanel([], [], { now });
|
|
assert.ok(empty.text.includes('No skill execution data available'));
|
|
assert.deepStrictEqual(empty.data.skills, []);
|
|
|
|
const orphan = dashboard.renderSuccessRatePanel([], [{ skill_id: 'orphan' }], { now });
|
|
assert.strictEqual(orphan.data.skills.length, 1);
|
|
assert.strictEqual(orphan.data.skills[0].current_7d, null);
|
|
assert.ok(orphan.text.includes('n/a'));
|
|
|
|
const declining = dashboard.renderSuccessRatePanel([
|
|
{ skill_id: 'gamma', outcome: 'failure', recorded_at: '2026-03-15T08:00:00.000Z' },
|
|
{ skill_id: 'gamma', outcome: 'success', recorded_at: '2026-02-28T08:00:00.000Z' },
|
|
{ skill_id: 'gamma', outcome: 'success', recorded_at: '2026-02-27T08:00:00.000Z' },
|
|
], [{ skill_id: 'gamma' }], { now });
|
|
assert.strictEqual(declining.data.skills[0].trend, '\u2198');
|
|
|
|
const flat = dashboard.renderSuccessRatePanel([
|
|
{ skill_id: 'delta', outcome: 'success', recorded_at: '2026-03-15T08:00:00.000Z' },
|
|
{ skill_id: 'delta', outcome: 'success', recorded_at: '2026-02-28T08:00:00.000Z' },
|
|
], [{ skill_id: 'delta' }], { now });
|
|
assert.strictEqual(flat.data.skills[0].trend, '\u2192');
|
|
})) passed++; else failed++;
|
|
|
|
if (test('failure cluster panel labels unknown single-skill failures', () => {
|
|
const result = dashboard.renderFailureClusterPanel([
|
|
{ skill_id: 'alpha', outcome: 'failure', failure_reason: '' },
|
|
]);
|
|
|
|
assert.strictEqual(result.data.clusters.length, 1);
|
|
assert.strictEqual(result.data.clusters[0].pattern, 'unknown');
|
|
assert.strictEqual(result.data.clusters[0].percentage, 100);
|
|
assert.ok(result.text.includes('(1 skill)'));
|
|
})) passed++; else failed++;
|
|
|
|
if (test('amendment panel handles missing dirs and pending proposal defaults', () => {
|
|
const proposalSkillDir = createSkill(skillsRoot, 'proposal-defaults', '# Proposal Defaults\n');
|
|
const proposalLog = path.join(proposalSkillDir, '.evolution', 'amendments.jsonl');
|
|
appendFile(proposalLog, JSON.stringify({ event: 'proposal' }) + '\n');
|
|
appendFile(proposalLog, JSON.stringify({ event: 'proposal', status: 'applied' }) + '\n');
|
|
|
|
const skillsById = new Map();
|
|
skillsById.set('missing-dir', { skill_id: 'missing-dir' });
|
|
skillsById.set('proposal-defaults', { skill_id: 'proposal-defaults', skill_dir: proposalSkillDir });
|
|
|
|
const result = dashboard.renderAmendmentPanel(skillsById, { width: 80 });
|
|
assert.strictEqual(result.data.total, 1);
|
|
assert.strictEqual(result.data.amendments[0].event, 'proposal');
|
|
assert.strictEqual(result.data.amendments[0].status, 'pending');
|
|
assert.strictEqual(result.data.amendments[0].created_at, null);
|
|
assert.ok(result.text.includes('1 amendment pending review'));
|
|
assert.ok(result.text.includes(' -'));
|
|
})) passed++; else failed++;
|
|
|
|
if (test('version timeline skips missing dirs and empty histories', () => {
|
|
const emptyVersionDir = createSkill(skillsRoot, 'empty-version-history', '# Empty Version History\n');
|
|
const skillsById = new Map();
|
|
skillsById.set('missing-dir', { skill_id: 'missing-dir' });
|
|
skillsById.set('empty-version-history', { skill_id: 'empty-version-history', skill_dir: emptyVersionDir });
|
|
|
|
const result = dashboard.renderVersionTimelinePanel(skillsById);
|
|
assert.deepStrictEqual(result.data.skills, []);
|
|
assert.ok(result.text.includes('No version history available'));
|
|
})) passed++; else failed++;
|
|
|
|
if (test('version timeline renders fallback date and reason values', () => {
|
|
const originalListVersions = versioning.listVersions;
|
|
const originalGetEvolutionLog = versioning.getEvolutionLog;
|
|
versioning.listVersions = () => [
|
|
{ version: 9, created_at: null },
|
|
];
|
|
versioning.getEvolutionLog = () => [
|
|
{ version: 9, reason: '' },
|
|
];
|
|
|
|
try {
|
|
const skillsById = new Map();
|
|
skillsById.set('fallback-version', { skill_id: 'fallback-version', skill_dir: skillsRoot });
|
|
const result = dashboard.renderVersionTimelinePanel(skillsById);
|
|
|
|
assert.strictEqual(result.data.skills.length, 1);
|
|
assert.strictEqual(result.data.skills[0].versions[0].reason, null);
|
|
assert.ok(result.text.includes('v9'));
|
|
assert.ok(result.text.includes(' - '));
|
|
} finally {
|
|
versioning.listVersions = originalListVersions;
|
|
versioning.getEvolutionLog = originalGetEvolutionLog;
|
|
}
|
|
})) passed++; else failed++;
|
|
|
|
if (test('renderDashboard rejects invalid timestamps', () => {
|
|
assert.throws(() => {
|
|
dashboard.renderDashboard({
|
|
skillsRoot,
|
|
learnedRoot,
|
|
importedRoot,
|
|
homeDir,
|
|
runsFilePath: runsFile,
|
|
now: 'not-a-timestamp',
|
|
});
|
|
}, /Invalid now timestamp/);
|
|
})) passed++; else failed++;
|
|
|
|
console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
|
|
} finally {
|
|
cleanupTempDir(repoRoot);
|
|
cleanupTempDir(homeDir);
|
|
}
|
|
|
|
process.exit(failed > 0 ? 1 : 0);
|
|
}
|
|
|
|
runTests();
|