/** * Contract tests for the curated TasteForge video skill. * No test contacts Fal, generates media, or mutates any provider account. */ "use strict"; const assert = require("assert"); const fs = require("fs"); const path = require("path"); const { spawnSync } = require("child_process"); const REPO_ROOT = path.join(__dirname, "..", ".."); function read(relativePath) { return fs.readFileSync(path.join(REPO_ROOT, relativePath), "utf8"); } function readJson(relativePath) { return JSON.parse(read(relativePath)); } function assertExactDryRunBoundary(payload, label) { assert.ok( Number.isInteger(payload.provider_calls) && payload.provider_calls === 0, `${label} must require exact integer provider_calls:0` ); assert.strictEqual(payload.provider_execution, false, `${label} must require provider_execution:false`); assert.strictEqual(payload.dry_run, true, `${label} must require dry_run:true`); assert.strictEqual(payload.submit, false, `${label} must require submit:false`); } function assertFiniteReal(value, label, { positive = false, nonnegative = false } = {}) { assert.strictEqual(typeof value, "number", `${label} must be a real number, not a boolean`); assert.ok(Number.isFinite(value), `${label} must be finite`); if (positive) assert.ok(value > 0, `${label} must be positive`); if (nonnegative) assert.ok(value >= 0, `${label} must be nonnegative`); } function assertFiniteEvidenceTree(value, label) { if (typeof value === "number" || typeof value === "boolean") { assertFiniteReal(value, label); } else if (Array.isArray(value)) { value.forEach((nested) => assertFiniteEvidenceTree(nested, label)); } else if (value && typeof value === "object") { Object.values(value).forEach((nested) => assertFiniteEvidenceTree(nested, label)); } } function validateFinalContractFixture(fixture) { for (const spec of fixture.genre_specs) { assert.strictEqual(spec.dry_run, true, `genre ${spec.number} must require dry_run:true`); } assertExactDryRunBoundary({ ...fixture.receipt, submit: false }, "receipt"); const durationByDigest = new Map(); for (const reference of fixture.receipt.references) { assertFiniteReal(reference.source_duration, "receipt source_duration", { positive: true }); const prior = durationByDigest.get(reference.sha256); assert.ok( prior === undefined || prior === reference.source_duration, "duplicate digest has conflicting source durations" ); durationByDigest.set(reference.sha256, reference.source_duration); assertFiniteEvidenceTree(reference.probe, "probe evidence"); assert.strictEqual(reference.probe.duration, reference.source_duration); for (const time of [ ...(reference.probe.sample_times || []), ...(reference.probe.scene_changes || []), ...(reference.probe.style_samples || []).map((sample) => sample.time), ]) { assertFiniteReal(time, "probe evidence time", { nonnegative: true }); assert.ok(time <= reference.source_duration, "probe evidence time exceeds source duration"); } } const recipe = fixture.effect_recipe; assertExactDryRunBoundary({ ...recipe, submit: false }, "effect recipe"); assertFiniteReal(recipe.timeline_duration, "timeline duration", { positive: true }); for (const event of recipe.events) { assertFiniteReal(event.time, "effect event time", { nonnegative: true }); assertFiniteReal(event.duration, "effect event duration", { positive: true }); assert.ok( event.time + event.duration <= recipe.timeline_duration, "effect event exceeds timeline duration" ); const evidence = event.evidence; const evidenceDuration = durationByDigest.get(evidence.reference_sha256); assert.ok(evidenceDuration !== undefined, "effect evidence cites unknown SHA-256"); assert.strictEqual( evidence.source_duration, evidenceDuration, "effect evidence duration must equal receipt reference duration" ); assertFiniteReal(evidence.time, "effect evidence time", { nonnegative: true }); assert.ok(evidence.time <= evidenceDuration, "effect evidence time exceeds source duration"); if (event.requires_subject_anchor) { const anchor = event.subject_anchor; const anchorDuration = durationByDigest.get(anchor.source_ref_sha256); assert.ok(anchorDuration !== undefined, "subject anchor cites unknown SHA-256"); assert.strictEqual( anchor.source_duration, anchorDuration, "subject-anchor duration must equal receipt reference duration" ); assertFiniteReal(anchor.evidence_time, "subject-anchor evidence time", { nonnegative: true }); assert.ok(anchor.evidence_time <= anchorDuration, "anchor evidence time exceeds source duration"); assert.strictEqual(anchor.lost_policy, "disable_effect_until_track_recovers"); } } for (const modality of ["image", "video", "3d_asset"]) { const manifest = fixture.manifests[modality]; assert.ok(manifest, `missing ${modality} manifest`); assertExactDryRunBoundary(manifest, `${modality} manifest`); assert.ok(manifest.requests.length > 0, `${modality} requests must not be empty`); for (const request of manifest.requests) { assertExactDryRunBoundary(request, `${modality} request`); assert.strictEqual(request.provider_call_mode, "disabled"); } } const binding = fixture.source_binding; assert.strictEqual(binding.probed_sha256, binding.before_probe_sha256); assert.strictEqual(binding.receipt_sha256, binding.before_probe_sha256); assert.strictEqual( binding.after_probe_sha256, binding.before_probe_sha256, "source mutation during probe must fail closed" ); assert.ok(fixture.missing_media_tool_error.exit_code > 0, "missing media tool must exit nonzero"); assert.ok(fixture.missing_media_tool_error.stderr.length < 256, "missing-tool error must stay bounded"); assert.doesNotMatch(fixture.missing_media_tool_error.stderr, /Traceback/i); for (const entry of fixture.output_entries) { assert.ok( entry.type === "regular_file" || entry.type === "directory", `output ${entry.path} must reject symlinks and special files` ); } } function validateRejectedContractFixture(fixture) { if (fixture.kind === "manifest") { assertExactDryRunBoundary(fixture.payload, "manifest"); for (const request of fixture.payload.requests || []) { assertExactDryRunBoundary(request, "request"); assert.strictEqual(request.provider_call_mode, "disabled"); } return; } if (fixture.kind === "effect_recipe") { for (const event of fixture.payload.events || []) { if (event.requires_subject_anchor) { assert.strictEqual( event.subject_anchor?.lost_policy, "disable_effect_until_track_recovers", "anchored CV effects must disable_effect_until_track_recovers" ); } } return; } assert.fail(`unknown fixture kind: ${fixture.kind}`); } const tests = []; function test(name, fn) { tests.push([name, fn]); } test("has valid discoverable frontmatter and trigger phrases", () => { const skill = read("skills/tasteforge-video/SKILL.md"); assert.match( skill, /^---\nname: tasteforge-video\ndescription: [^\n]+\nmetadata:\n {2}origin: ECC\n---\n/ ); for (const trigger of [ /interview .*video taste|video .*taste interview/i, /distill .*aesthetic .*structured/i, /validate a style pack/i, /apply a style pack to local footage/i, /export EDL\/FCPXML|export .*EDL.*FCPXML/i, /audit .*generated-media provenance|provenance audit/i, ]) assert.match(skill, trigger); const description = skill.match(/^description: ([^\n]+)$/m)?.[1] || ""; for (const discoveryTerm of ["multimodal", "image", "video", "3D", "file-driven", "distill", "apply"]) { assert.match(description, new RegExp(discoveryTerm, "i"), `frontmatter misses ${discoveryTerm}`); } const triggers = skill.match(/## When to Use\n([\s\S]*?)\n## /)?.[1] || ""; for (const discoveryTerm of ["multimodal", "image", "video", "3D", "file-driven", "distill", "apply"]) { assert.match(triggers, new RegExp(discoveryTerm, "i"), `triggers miss ${discoveryTerm}`); } }); test("distinguishes local deterministic operations from provider generation", () => { const skill = read("skills/tasteforge-video/SKILL.md"); assert.match(skill, /local, deterministic/i); assert.match(skill, /provider generation/i); assert.match(skill, /must fail closed/i); assert.match(skill, /explicit separately authorized execution/i); assert.match(skill, /ECC never calls Fal/i); assert.match( skill, /never\s+reads\s+any\s+API\s+key\s+or\s+other\s+credentials/i, "skill must state that no API key or credentials are read" ); }); test("never claims a Fal workflow is saved from a local reference", () => { const skill = read("skills/tasteforge-video/SKILL.md"); assert.match( skill, /never (?:claim|means|treat)[^.]*provider-side workflow (?:is|was) saved/i ); assert.match(skill, /reference[- ]only/i); assert.match(skill, /dry[- ]run|dry_run/i); }); test("links to the canonical ito-video implementation instead of duplicating it", () => { const skill = read("skills/tasteforge-video/SKILL.md"); assert.match(skill, /ito-video/i); assert.match(skill, /Ito-Markets\/ito-video/i); assert.match(skill, /python3 -m tasteforge/); assert.match(skill, /does not (?:vendor|duplicate|copy)/i); }); test("describes the deterministic workflow surface faithfully", () => { const skill = read("skills/tasteforge-video/SKILL.md"); for (const cmd of ["inspect", "validate", "interview", "distill", "apply", "export", "provenance"]) { assert.match(skill, new RegExp(`\\b${cmd}\\b`)); } assert.match(skill, /schema/i); assert.match(skill, /cadence/i); assert.match(skill, /style pack/i); }); test("defines the fail-closed file-driven multimodal contract", () => { const skill = read("skills/tasteforge-video/SKILL.md"); assert.match(skill, /python3 -m tasteforge multimodal --config/); for (const phrase of [ /Flash Ethereal/, /3D Cyber Glitch/, /Fluid Sketch/, /image.*video.*3D-asset/is, /seeded aperiodic/i, /subject anchor/i, /placement constraints/i, /provider_execution:\s*false/i, /path.*byte size.*SHA-256/is, /genre.*modality/is, /exact reference\/time provenance/i, /provider_calls:\s*0/i, /exact integer/i, /genre spec.*dry_run:\s*true/is, /effect recipe.*provider_calls:\s*0/is, /dry_run:\s*true/i, /submit:\s*false/i, /finite real/i, /booleans as invalid numbers/i, /duplicate occurrences.*digest.*agree.*duration/is, /effect evidence `source_duration`.*validated\s+receipt duration/is, /subject-anchor `source_duration`.*validated\s+receipt duration/is, /same stable bytes/i, /mutates while probing/i, /ffmpeg.*ffprobe.*bounded nonzero.*without a\s+Python traceback/is, /symlinks.*special files/is, /disable_effect_until_track_recovers/i, ]) assert.match(skill, phrase); assert.match(skill, /missing.*manifest.*fail closed/is); assert.match(skill, /tamper.*fail closed/is); }); test("executable fixture enforces the independently passed final contract", () => { const baseline = readJson("tests/fixtures/tasteforge-video/final-contract.json"); const clone = () => JSON.parse(JSON.stringify(baseline)); assert.doesNotThrow(() => validateFinalContractFixture(clone())); const mutations = [ ["genre dry_run false", (value) => { value.genre_specs[0].dry_run = false; }], ["receipt boolean provider_calls", (value) => { value.receipt.provider_calls = false; }], ["effect boolean provider_calls", (value) => { value.effect_recipe.provider_calls = false; }], ["manifest boolean provider_calls", (value) => { value.manifests.video.provider_calls = false; }], ["request boolean provider_calls", (value) => { value.manifests.video.requests[0].provider_calls = false; }], ["boolean timeline number", (value) => { value.effect_recipe.events[0].time = false; }], ["non-finite event duration", (value) => { value.effect_recipe.events[0].duration = NaN; }], ["non-finite evidence duration", (value) => { value.effect_recipe.events[0].evidence.source_duration = Infinity; }], ["boolean probe measurement", (value) => { value.receipt.references[0].probe.style_samples[0].luma = false; }], ["effect duration not bound to digest", (value) => { value.effect_recipe.events[0].evidence.source_duration = 5; }], ["anchor duration not bound to digest", (value) => { value.effect_recipe.events[0].subject_anchor.source_duration = 5; }], ["duplicate digest conflicting duration", (value) => { const duplicate = JSON.parse(JSON.stringify(value.receipt.references[0])); duplicate.source_duration = 7; duplicate.probe.duration = 7; value.receipt.references.push(duplicate); }], ["source mutation during probe", (value) => { value.source_binding.after_probe_sha256 = "b".repeat(64); }], ["missing-tool success exit", (value) => { value.missing_media_tool_error.exit_code = 0; }], ["missing-tool traceback", (value) => { value.missing_media_tool_error.stderr = "Traceback (most recent call last): secret\n"; }], ["symlink output", (value) => { value.output_entries.push({ path: "resolve", type: "symlink" }); }], ["special-file output", (value) => { value.output_entries.push({ path: "resolve/pipe", type: "fifo" }); }], ]; for (const [label, mutate] of mutations) { const fixture = clone(); mutate(fixture); assert.throws( () => validateFinalContractFixture(fixture), undefined, `${label} was not rejected` ); } }); test("executable fixtures reject dry_run:false and continue_without_anchor", () => { for (const fixtureName of [ "reject-dry-run-false.json", "reject-continue-without-anchor.json", ]) { const fixture = readJson(`tests/fixtures/tasteforge-video/${fixtureName}`); assert.strictEqual(fixture.expected, "reject"); assert.throws( () => validateRejectedContractFixture(fixture), undefined, `${fixtureName} was not rejected` ); } }); test("ships through the opt-in media-generation install module and npm package", () => { const modules = readJson("manifests/install-modules.json").modules; const module = modules.find((candidate) => candidate.id === "media-generation"); assert.ok(module, "media-generation install module is missing"); assert.ok( module.paths.includes("skills/tasteforge-video"), "skills/tasteforge-video missing from media-generation paths" ); assert.strictEqual(module.defaultInstall, false); const packed = readJson("package.json").files; assert.ok( packed.includes("skills/tasteforge-video/"), "skills/tasteforge-video/ missing from npm files" ); }); test("is discoverable in the source tree and in a simulated packed artifact", () => { const skillPath = path.join(REPO_ROOT, "skills", "tasteforge-video", "SKILL.md"); assert.ok(fs.existsSync(skillPath), "SKILL.md missing in source tree"); // Packed surface: npm includes the directory; the plugin manifest routes // ./skills/ wholesale; nothing ignores the directory. const npmignore = read(".npmignore"); const ignoresSkill = npmignore .split(/\r?\n/) .map((line) => line.trim()) .filter((line) => line && !line.startsWith("#")) .some((line) => { const normalized = line.replace(/\/+$/, ""); return ( normalized === "skills" || normalized === "skills/tasteforge-video" || normalized === "skills/tasteforge-video/SKILL.md" ); }); assert.ok(!ignoresSkill, ".npmignore must not exclude the skill"); const claudePlugin = readJson(".claude-plugin/plugin.json"); assert.ok( (claudePlugin.skills || []).includes("./skills/"), "claude plugin skills must route to the root skills/ directory" ); // Simulated installed layout: the files entry must name the skill dir and // the SKILL.md must exist beneath it with non-empty content. const stat = fs.statSync(path.join(REPO_ROOT, "skills", "tasteforge-video")); assert.ok(stat.isDirectory(), "skill must be a directory"); assert.ok(fs.readFileSync(skillPath, "utf8").trim().length > 200, "SKILL.md is empty-ish"); }); test("passes the curated skill validator", () => { const result = spawnSync( process.execPath, [path.join(REPO_ROOT, "scripts", "ci", "validate-skills.js")], { encoding: "utf8" } ); assert.strictEqual(result.status, 0, `validate-skills failed:\n${result.stdout}\n${result.stderr}`); assert.match(result.stdout + result.stderr, /skill director/i, "validator output unrecognized"); }); // Opt-in slow path: verifies the real npm tarball contents. Enabled with // ECC_TEST_NPM_PACK=1 (release/CI verification); the default suite relies on // the files-array assertions above. test("ships inside the real npm tarball (opt-in)", () => { if (process.env.ECC_TEST_NPM_PACK !== "1") { return { skipped: "set ECC_TEST_NPM_PACK=1 to run real npm pack inclusion" }; } const result = spawnSync("npm", ["pack", "--dry-run", "--ignore-scripts"], { cwd: REPO_ROOT, encoding: "utf8", }); assert.strictEqual(result.status, 0, `npm pack failed:\n${result.stderr}`); assert.match( result.stdout + result.stderr, /skills\/tasteforge-video\/SKILL\.md/, "SKILL.md missing from npm tarball contents" ); }); let failed = 0; let skipped = 0; console.log("\n=== Testing TasteForge video skill ===\n"); for (const [name, fn] of tests) { try { const result = fn(); if (result?.skipped) { skipped += 1; console.log(` - SKIP ${name}: ${result.skipped}`); continue; } console.log(` ✓ ${name}`); } catch (error) { failed += 1; console.log(` ✗ ${name}`); console.error(` ${error.message}`); } } if (failed) process.exit(1); console.log(`\n${tests.length - failed - skipped}/${tests.length} passed, ${skipped} skipped`);