Consolidate recovered eval framework and operator workflows (#3040)

* feat: consolidate offline eval and operator workflows

Compose the retained framework, operator skill, roadmap and cleanup ranges on current main. Preserve current release dependencies and keep candidate execution disabled pending OS containment. Repair draft/DOCX behavior, obligation uniqueness, trusted send and audience guidance, runner provenance and eval diagnostics.

Source-PR: 2930 0abe3727d2b500c6e4830bdeb47ed67cae3f4785
Source-PR: 2931 992b49c44ed872def49675b791168b8fcd091df6
Source-PR: 2932 4a193dd13041cb7a6bebf4d2e910a0cd32bcc797
Source-PR: 2933 59cdfe500a91949ba1415f1edd7279620f21e804
Source-Base: ca185ef5f7

* fix: repair foundation CI and update js-yaml

* fix: reconcile pending-delete capsule locks after close

---------

Co-authored-by: Claude Fable 5.1 <noreply@anthropic.com>

tests/scripts/eval-harness-package.test.js

@@ -0,0 +1,122 @@
+'use strict';
+
+// Dependency-free package contract only: never run prepack, build, or install.
+// Run serially: node tests/scripts/eval-harness-package.test.js
+const assert = require('assert');
+const fs = require('fs');
+const path = require('path');
+const { spawnSync } = require('child_process');
+const { getNpmPackEntry } = require('../lib/npm-pack-output');
+const { test, tempDir, cleanup, finish, runNpm } = require('../lib/eval-harness/helpers');
+
+const repo = path.resolve(__dirname, '../..');
+const work = tempDir('package smoke');
+const pkg = JSON.parse(fs.readFileSync(path.join(repo, 'package.json'), 'utf8'));
+const childEnv = { ...process.env, NODE_PATH: '', NODE_OPTIONS: '' };
+const command = (binary, args, options = {}) => spawnSync(binary, args, {
+  encoding: 'utf8', timeout: 60000, maxBuffer: 16 * 1024 * 1024,
+  env: childEnv, ...options,
+});
+
+function sourceFiles(relative) {
+  const dir = path.join(repo, relative);
+  return fs.readdirSync(dir, { withFileTypes: true }).flatMap(entry => {
+    const file = `${relative}/${entry.name}`;
+    assert.ok(!entry.isSymbolicLink(), `fixture must be a regular source tree: ${file}`);
+    return entry.isDirectory() ? sourceFiles(file) : [file];
+  }).sort();
+}
+
+// Match the actual aggregation contract in tests/run-all.js.
+function counts(stdout) {
+  const passed = stdout.match(/Passed:\s*(\d+)/);
+  const failed = stdout.match(/Failed:\s*(\d+)/);
+  assert.ok(passed && failed, 'result tokens must be parseable by tests/run-all.js');
+  return { passed: Number(passed[1]), failed: Number(failed[1]) };
+}
+
+let archive;
+try {
+  test('ignore-scripts tarball ships every example fixture and eval library file', () => {
+    const result = runNpm(['pack', '--ignore-scripts', '--offline', '--json',
+      '--pack-destination', work, '--cache', path.join(work, 'npm-cache')], {
+      cwd: repo, env: childEnv,
+    });
+    assert.strictEqual(result.status, 0, result.error?.message || result.stderr);
+    const entry = getNpmPackEntry(JSON.parse(result.stdout), pkg.name);
+    assert.ok(entry && typeof entry.filename === 'string');
+    assert.strictEqual(path.basename(entry.filename), entry.filename);
+    assert.ok(!entry.filename.startsWith('-'));
+    archive = path.join(work, entry.filename);
+    assert.ok(fs.statSync(archive).isFile());
+    const packed = new Set(entry.files.map(file => file.path));
+    const examples = sourceFiles('examples/eval-harness');
+    const required = ['scripts/eval-harness.js', ...sourceFiles('scripts/lib/eval-harness'), ...examples];
+    for (const file of required) assert.ok(packed.has(file), `package is missing ${file}`);
+    assert.ok(!packed.has('examples/CLAUDE.md'), 'do not publish unrelated examples');
+    console.log(`    package closure: ${examples.length} example files; prepack/build/install skipped`);
+  });
+
+  test('actual extracted CLI example runs from the package without installing dependencies', () => {
+    assert.ok(archive, 'packing must succeed before extracting');
+    const extract = path.join(work, 'extracted');
+    fs.mkdirSync(extract);
+    const unpack = command('tar', ['-xzf', archive, '-C', extract]);
+    assert.strictEqual(unpack.status, 0, unpack.error?.message || unpack.stderr);
+    const installed = path.join(extract, 'package');
+    assert.ok(!fs.existsSync(path.join(installed, 'node_modules')));
+    const runtimeTemp = path.join(work, 'example-runtime');
+    fs.mkdirSync(runtimeTemp);
+    const result = command(process.execPath, [path.join(installed, 'scripts/eval-harness.js'), 'example', '--keep'], {
+      cwd: installed,
+      env: { ...childEnv, TMPDIR: runtimeTemp, TMP: runtimeTemp, TEMP: runtimeTemp },
+    });
+    assert.strictEqual(result.status, 0, result.error?.message || result.stderr || result.stdout);
+    assert.match(result.stdout, /all steps passed/);
+    const runs = fs.readdirSync(runtimeTemp).filter(name => name.startsWith('ecc-eval-harness-example-'));
+    assert.strictEqual(runs.length, 1);
+    const run = path.join(runtimeTemp, runs[0]);
+    const journal = fs.readFileSync(path.join(run, 'capsule/journal.ndjson'), 'utf8').trim().split('\n').map(JSON.parse);
+    assert.ok(journal.some(entry => entry.kind === 'gate.unavailable' && entry.payload.status === 'blocked'));
+    assert.strictEqual(new Set(journal.map(entry => entry.lineage)).size, 5);
+    const receipt = JSON.parse(fs.readFileSync(path.join(run, 'bundle/receipt.json'), 'utf8'));
+    assert.strictEqual(receipt.gate_receipt_digest, null);
+    assert.strictEqual(receipt.gate_verdict, null);
+    assert.ok(!fs.existsSync(path.join(run, 'gate-candidate')));
+    assert.ok(journal.every(entry => entry.payload.verdict !== 'PROMOTE'));
+    for (const file of sourceFiles('examples/eval-harness')) {
+      assert.deepStrictEqual(fs.readFileSync(path.join(installed, file)), fs.readFileSync(path.join(repo, file)));
+    }
+    console.log('    extracted example: five lineages, no candidate execution or gate verdict');
+  });
+
+  test('aggregator regexes count every real framework check accurately', () => {
+    const suites = fs.readdirSync(path.join(repo, 'tests/lib/eval-harness')).filter(file => file.endsWith('.test.js')).sort();
+    let total = 0;
+    for (const suite of suites) {
+      const result = command(process.execPath, [path.join(repo, 'tests/lib/eval-harness', suite)], { cwd: work });
+      assert.strictEqual(result.status, 0, `${suite}: ${result.error?.message || result.stderr || result.stdout}`);
+      const parsed = counts(result.stdout);
+      const actualPassed = (result.stdout.match(/^\s*✓ /gm) || []).length;
+      const actualFailed = (result.stdout.match(/^\s*✗ /gm) || []).length;
+      assert.deepStrictEqual(parsed, { passed: actualPassed, failed: actualFailed }, suite);
+      assert.ok(actualPassed > 0, `${suite} must run actual checks`);
+      assert.strictEqual(parsed.failed, 0);
+      total += parsed.passed;
+    }
+    assert.ok(suites.length > 0);
+    console.log(`    framework aggregation: ${suites.length} suites, ${total} actual checks`);
+  });
+
+  test('failed checks remain visible to aggregation and return a failing exit', () => {
+    const helper = path.join(repo, 'tests/lib/eval-harness/helpers.js');
+    const script = `const h=require(${JSON.stringify(helper)});h.test('pass fixture',()=>{});h.test('failure fixture',()=>{throw new Error('synthetic failure');});h.finish('count fixture');`;
+    const result = command(process.execPath, ['-e', script], { cwd: work });
+    assert.strictEqual(result.status, 1);
+    assert.deepStrictEqual(counts(result.stdout), { passed: 1, failed: 1 });
+  });
+} finally {
+  cleanup(work);
+}
+
+finish('eval-harness package');

tests/scripts/install-readme-clarity.test.js

@@ -53,10 +53,10 @@ function runTests() {
 
   if (test('README leads with the idempotent guided plugin setup path', () => {
     const topClaudeSectionIndex = readme.indexOf('## Install with Claude Code');
-    const topGuidedCommandIndex = readme.indexOf('npx ecc-universal setup', topClaudeSectionIndex);
+    const topGuidedCommandIndex = readme.indexOf('npx ecc-universal@2.2.1 setup', topClaudeSectionIndex);
     const nativePluginCommandIndex = readme.indexOf('/plugin marketplace add', topClaudeSectionIndex);
     const installSectionIndex = readme.indexOf('## Install ECC');
-    const guidedCommandIndex = readme.indexOf('npx ecc-universal setup', installSectionIndex);
+    const guidedCommandIndex = readme.indexOf('npx ecc-universal@2.2.1 setup', installSectionIndex);
     const claudeDetailsIndex = readme.indexOf('### Claude Code details', installSectionIndex);
 
     assert.ok(
@@ -95,9 +95,9 @@ function runTests() {
   })) passed++; else failed++;
 
   if (test('README documents modern package-runner alternatives', () => {
-    assert.ok(readme.includes('pnpm dlx ecc-universal setup'));
-    assert.ok(readme.includes('yarn dlx ecc-universal setup'));
-    assert.ok(readme.includes('bunx ecc-universal setup'));
+    assert.ok(readme.includes('pnpm dlx ecc-universal@2.2.1 setup'));
+    assert.ok(readme.includes('yarn dlx ecc-universal@2.2.1 setup'));
+    assert.ok(readme.includes('bunx ecc-universal@2.2.1 setup'));
     assert.ok(
       readme.includes('Yarn Classic 1 does not provide `yarn dlx`'),
       'README should not advertise the modern Yarn command to Yarn Classic users'
@@ -122,10 +122,10 @@ function runTests() {
       'README should document doctor before reinstalling'
     );
     for (const command of [
-      'npx ecc-universal list-installed',
-      'npx ecc-universal doctor',
-      'npx ecc-universal repair',
-      'npx ecc-universal uninstall --dry-run',
+      'npx ecc-universal@2.2.1 list-installed',
+      'npx ecc-universal@2.2.1 doctor',
+      'npx ecc-universal@2.2.1 repair',
+      'npx ecc-universal@2.2.1 uninstall --dry-run',
     ]) {
       assert.ok(
         readme.includes(command),
@@ -148,7 +148,7 @@ function runTests() {
       'README should document the shell minimal profile command'
     );
     assert.ok(
-      readme.includes('npx ecc-universal install --profile minimal --target claude'),
+      readme.includes('npx ecc-universal@2.2.1 install --profile minimal --target claude'),
       'README should document the published universal-package minimal profile command'
     );
     assert.ok(
@@ -175,7 +175,7 @@ function runTests() {
       'README should surface component discovery before install steps'
     );
     assert.ok(
-      readme.includes('npx ecc-universal consult "security reviews" --target claude'),
+      readme.includes('npx ecc-universal@2.2.1 consult "security reviews" --target claude'),
       'README should document the packaged consult command'
     );
     assert.ok(
@@ -193,15 +193,15 @@ function runTests() {
 
   if (test('README gives the native guided Codex and managed Kimi dry-run paths', () => {
     assert.ok(
-      readme.includes('npx ecc-universal install --guided --harness codex --dry-run'),
+      readme.includes('npx ecc-universal@2.2.1 install --guided --harness codex --dry-run'),
       'README should verify Codex through the native guided reconciler'
     );
     assert.ok(
-      !readme.includes('npx ecc-universal install --profile core --target codex --dry-run'),
+      !readme.includes('npx ecc-universal@2.2.1 install --profile core --target codex --dry-run'),
       'README should not present the legacy managed Codex adapter as the native lifecycle'
     );
     assert.ok(
-      readme.includes('npx ecc-universal install --profile core --target kimi --dry-run')
+      readme.includes('npx ecc-universal@2.2.1 install --profile core --target kimi --dry-run')
     );
     for (const target of ['cursor', 'gemini', 'opencode', 'codebuddy', 'joycode', 'qwen', 'zed', 'hermes', 'openclaw']) {
       assert.ok(readme.includes(`\`${target}\``), `README should name the ${target} target`);
@@ -312,6 +312,18 @@ function runTests() {
     );
   })) passed++; else failed++;
 
+  if (test('README binds package runners to the release and avoids unaudited bootstraps', () => {
+    const version = JSON.parse(fs.readFileSync(path.join(__dirname, '..', '..', 'package.json'))).version;
+    const runners = [...readme.matchAll(/(?:npx |pnpm dlx |yarn dlx |bunx )(ecc-universal[^\s`]+)/g)];
+    assert.ok(runners.length >= 15);
+    for (const match of runners) assert.strictEqual(match[1], `ecc-universal@${version}`);
+    assert.ok(!/npx (?:-y )?(?:ecc-agentshield|ccg-workflow)/.test(readme));
+    assert.ok(!/npm install -g opencode(?:\s|$)/m.test(readme));
+    assert.match(readme, /version pin is not a security audit/i);
+    assert.match(readme, /already installed.*reviewed.*AgentShield/i);
+    assert.ok(readme.includes('https://www.npmjs.com/package/ecc-universal/v/2.2.1'));
+  })) passed++; else failed++;
+
   console.log(`\nResults: Passed: ${passed}, Failed: ${failed}`);
   process.exit(failed > 0 ? 1 : 0);
 }

tests/scripts/ito-compute-sponsor.test.js

@@ -237,7 +237,11 @@ function main() {
       assert.ok(localModelPath.includes('assets/images/sponsors/moonshot.png'));
       assert.ok(localModelPath.includes('assets/images/community/ecc-tools-mark.svg'));
       assert.match(readme, /install\.sh --target kimi --profile minimal/);
-      assert.match(readme, /npx ecc-universal doctor --target kimi/);
+      const version = JSON.parse(read('package.json')).version;
+      assert.ok(
+        readme.includes(`npx ecc-universal@${version} doctor --target kimi`),
+        'README must document the Kimi doctor command pinned to the ECC release'
+      );
       assert.match(readme, /\.kimi-code\/AGENTS\.md/);
       assert.match(readme, /\.kimi-code\/skills\//);
       assert.match(readme, /~\/\.kimi-code\/config\.toml/);

tests/scripts/npm-publish-surface.test.js

@@ -5,7 +5,8 @@
 const assert = require("assert")
 const fs = require("fs")
 const path = require("path")
-const { spawnSync } = require("child_process")
+const os = require("os")
+const { runNpm } = require("../lib/eval-harness/helpers")
 const { getNpmPackEntry } = require("../lib/npm-pack-output")
 
 function runTest(name, fn) {
@@ -43,6 +44,8 @@ function buildExpectedPublishPaths(repoRoot) {
   const extraPaths = [
     "manifests",
     "scripts/ecc.js",
+    "scripts/eval-harness.js",
+    "examples/eval-harness",
     "scripts/feedback.js",
     "scripts/catalog.js",
     "scripts/ci/scan-supply-chain-iocs.js",
@@ -103,6 +106,7 @@ function buildExpectedPublishPaths(repoRoot) {
     "assets/images/community",
     "docs/CODEX-NAVIGATION-GUIDE.md",
     "docs/COMMAND-AGENT-MAP.md",
+    "docs/ROADMAP.md",
     "docs/design/ecc-memory-vault.md",
     "assets/images/sponsors",
   ]
@@ -141,12 +145,20 @@ function main() {
     ["package.json files align to the module graph and explicit runtime allowlist", () => {
       assert.deepStrictEqual(actualPublishPaths, expectedPublishPaths)
     }],
-    ["npm pack publishes the reduced runtime surface", () => {
-      const result = spawnSync("npm", ["pack", "--dry-run", "--json"], {
-        cwd: repoRoot,
-        encoding: "utf8",
-        shell: process.platform === "win32",
-      })
+    ["npm pack --ignore-scripts publishes the reduced runtime surface (prepack not tested)", () => {
+      const cache = fs.mkdtempSync(path.join(os.tmpdir(), "ecc-pack-surface-"))
+      let result
+      try {
+        result = runNpm(["pack", "--dry-run", "--json", "--ignore-scripts", "--offline", "--cache", cache], {
+          cwd: repoRoot,
+          encoding: "utf8",
+          timeout: 60000,
+          maxBuffer: 16 * 1024 * 1024,
+          env: { ...process.env, NODE_PATH: "", NODE_OPTIONS: "" },
+        })
+      } finally {
+        fs.rmSync(cache, { recursive: true, force: true })
+      }
       assert.strictEqual(result.status, 0, result.error?.message || result.stderr)
 
       const packOutput = JSON.parse(result.stdout)
@@ -154,6 +166,17 @@ function main() {
       const packagedPaths = new Set(packEntry?.files?.map((file) => file.path) ?? [])
 
       for (const requiredPath of [
+        "scripts/eval-harness.js",
+        "scripts/lib/eval-harness/index.js",
+        "examples/eval-harness/run-example.js",
+        "examples/eval-harness/gate.config.json",
+        "examples/eval-harness/taskset.json",
+        "examples/eval-harness/variants/baseline/run.js",
+        "examples/eval-harness/variants/baseline/variant.json",
+        "examples/eval-harness/variants/candidate/run.js",
+        "examples/eval-harness/variants/candidate/variant.json",
+        "examples/eval-harness/variants/reward-hack/run.js",
+        "examples/eval-harness/variants/reward-hack/variant.json",
         "scripts/catalog.js",
         "scripts/ci/scan-supply-chain-iocs.js",
         "scripts/ci/supply-chain-advisory-sources.js",
@@ -199,6 +222,7 @@ function main() {
         "assets/images/community/heart.svg",
         "docs/CODEX-NAVIGATION-GUIDE.md",
         "docs/COMMAND-AGENT-MAP.md",
+        "docs/ROADMAP.md",
         "docs/design/ecc-memory-vault.md",
         "schemas/install-state.schema.json",
         "schemas/memory.schema.json",