ECC/docker/gateguard-effectiveness/run-intent.js

#!/usr/bin/env node
'use strict';

// Hidden-intent effectiveness run.
//
// Separate from run.js on purpose: run.js measures the gate's cost in one-shot
// autonomous sessions and remains the cost baseline. This runner measures
// whether the gate's questions obtain a fact the repository does not hold, and
// reports the holes rather than asserting coverage.

const fs = require('fs');
const crypto = require('crypto');
const os = require('os');
const path = require('path');
const { spawnSync } = require('child_process');
const { schedule, completedTrialKeys } = require('./lib');
const { ARMS, armSettings, resolveRef, materializeTree } = require('./arms');
const { loadIntentScenarios, graderIsSound, formatGraderScores, runIntentTrial, SUPPORTED_ARMS } = require('./intent-eval');
const { renderHoles, difficulty, isValidTrial } = require('./coverage');
const evidence = require('./evidence');

const REPO_ROOT = path.resolve(__dirname, '..', '..');

const USAGE = [
  'Usage: node docker/gateguard-effectiveness/run-intent.js --out <dir> --model <model> --allow-real-provider [options]',
  '',
  '  --out <dir>             results.jsonl, holes.md and transcripts/ (a rerun resumes)',
  '  --model <model>         Claude model under test (required unless --dry-run or --summarize)',
  '  --allow-real-provider   run real Claude sessions (each trial is several billed turns)',
  '  --judge-model <model>   model playing the user (default: haiku)',
  '  --claude <executable>   CLI executable for both agent and simulated user',
  `  --arms <a,b,...>        ${SUPPORTED_ARMS.join(', ')} (default: off,gate,placebo)`,
  '  --scenarios <a,b,...>   scenario ids (default: all)',
  '  --reps <n>              repetitions per scenario and arm (default: 3)',
  '  --user-turns <n>        how many times the user will answer (default: 3)',
  '  --max-turns <n>         agent turn limit per invocation (default: 40)',
  '  --timeout-min <n>       wall-clock limit per invocation (default: 15)',
  '  --seed <n>              arm order seed (default: 11)',
  '  --check-graders         confirm every grader separates start, trap and reference, then exit',
  '  --dry-run               print the schedule without running sessions',
  '  --summarize             rebuild the report from results.jsonl',
  '  --verify                verify evidence.json checksums and exit'
].join('\n');

function parseArgs(argv) {
  const options = {
    out: null, model: null, judgeModel: 'haiku', claude: 'claude',
    arms: ['off', 'gate', 'placebo'], scenarios: null, reps: 3, userTurns: 3,
    candidateRef: 'HEAD', mainRef: 'upstream/main', maxTurns: 40, timeoutMin: 15, seed: 11,
    allowRealProvider: false, dryRun: false, summarizeOnly: false, verifyOnly: false, checkGraders: false, help: false
  };
  const list = value => value.split(',').map(item => item.trim()).filter(Boolean);
  const positive = (value, name) => {
    if (!/^[1-9]\d*$/.test(value || '')) throw new Error(`${name} needs a positive integer\n\n${USAGE}`);
    return Number(value);
  };
  for (let i = 0; i < argv.length; i++) {
    const arg = argv[i];
    const next = () => {
      if (i + 1 >= argv.length) throw new Error(`${arg} needs a value\n\n${USAGE}`);
      return argv[++i];
    };
    if (arg === '--out') options.out = path.resolve(next());
    else if (arg === '--model') options.model = next();
    else if (arg === '--judge-model') options.judgeModel = next();
    else if (arg === '--claude') options.claude = next();
    else if (arg === '--arms') options.arms = list(next());
    else if (arg === '--scenarios') options.scenarios = list(next());
    else if (arg === '--reps') options.reps = positive(next(), '--reps');
    else if (arg === '--user-turns') options.userTurns = positive(next(), '--user-turns');
    else if (arg === '--candidate-ref') options.candidateRef = next();
    else if (arg === '--main-ref') options.mainRef = next();
    else if (arg === '--max-turns') options.maxTurns = positive(next(), '--max-turns');
    else if (arg === '--timeout-min') options.timeoutMin = positive(next(), '--timeout-min');
    else if (arg === '--seed') options.seed = positive(next(), '--seed');
    else if (arg === '--allow-real-provider') options.allowRealProvider = true;
    else if (arg === '--check-graders') options.checkGraders = true;
    else if (arg === '--dry-run') options.dryRun = true;
    else if (arg === '--summarize') options.summarizeOnly = true;
    else if (arg === '--verify') options.verifyOnly = true;
    else if (arg === '--help' || arg === '-h') options.help = true;
    else throw new Error(`unknown argument: ${arg}\n\n${USAGE}`);
  }
  const unsupported = options.arms.filter(arm => !SUPPORTED_ARMS.includes(arm));
  if (unsupported.length) throw new Error(`arm not supported for hidden-intent scenarios: ${unsupported.join(', ')}`);
  if (!options.verifyOnly && !options.help && !options.checkGraders &&
      (!options.arms.includes('gate') || options.arms.length < 2)) {
    throw new Error('the effectiveness comparison needs the gate arm and at least one comparator arm');
  }
  if (!options.help && !options.checkGraders && !options.out) throw new Error(`--out is required\n\n${USAGE}`);
  if (!options.help && !options.checkGraders && !options.dryRun && !options.summarizeOnly && !options.verifyOnly) {
    if (!options.model) throw new Error(`--model is required\n\n${USAGE}`);
    if (!options.allowRealProvider) throw new Error(`real sessions need --allow-real-provider\n\n${USAGE}`);
  }
  return options;
}

function readResults(file) {
  if (!fs.existsSync(file)) return [];
  return fs.readFileSync(file, 'utf8').split('\n').flatMap((line, index) => {
    if (!line) return [];
    try {
      return [JSON.parse(line)];
    } catch (error) {
      throw new Error(`results.jsonl line ${index + 1} is malformed; preserve the file for recovery (${error.message})`);
    }
  });
}

function writeAtomic(file, contents) {
  const temporary = path.join(path.dirname(file), `.${path.basename(file)}.${crypto.randomUUID()}.tmp`);
  try {
    fs.writeFileSync(temporary, contents, { flag: 'wx' });
    fs.renameSync(temporary, file);
  } catch (error) {
    if (fs.existsSync(temporary)) fs.rmSync(temporary, { force: true });
    throw error;
  }
}

// Resuming a session needs persistence, so Claude writes a transcript folder per
// trial workspace under its projects directory. A per-trial CLAUDE_CONFIG_DIR
// would isolate those but also hides the credentials, so they are removed here
// instead. Only folders this run created, named after its own temp work root,
// are touched.
function sessionsDir(env = process.env) {
  const home = env.CLAUDE_CONFIG_DIR || path.join(env.HOME || env.USERPROFILE || os.homedir(), '.claude');
  return path.join(home, 'projects');
}

function listSessions(env = process.env) {
  try {
    return new Set(fs.readdirSync(sessionsDir(env)));
  } catch (_) {
    return new Set();
  }
}

function projectSessionName(workspace) {
  const resolved = path.resolve(workspace);
  const encoded = resolved.replace(/[^A-Za-z0-9]/g, '-');
  if (encoded.length <= 200) return encoded;
  // Match the official Claude Agent SDK0.3.296 project storage key, including
  // its long-path suffix. Shared provider project-name overrides are not owned.
  let hash = 0;
  for (let index = 0; index < resolved.length; index += 1) {
    hash = ((hash << 5) - hash + resolved.charCodeAt(index)) | 0;
  }
  return `${encoded.slice(0, 200)}-${Math.abs(hash).toString(36)}`;
}

function removeTrialSessions(before, ownedWorkspaces, env = process.env) {
  if (env.CLAUDE_CODE_PROJECT_DIR_NAME) return;
  const dir = sessionsDir(env);
  const ownedNames = new Set([...ownedWorkspaces].map(projectSessionName));
  for (const entry of listSessions(env)) {
    if (before.has(entry) || !ownedNames.has(entry)) continue;
    fs.rmSync(path.join(dir, entry), { recursive: true, force: true });
  }
}

function writeReport(options, rows, meta) {
  const header = `Model: ${meta.model}; judge: ${meta.judgeModel}; hook commit: ${meta.sha}; reps: ${meta.reps}; user turns: ${meta.userTurns}.\n\n`;
  const configuration = meta.configuration;
  const expectedTrials = configuration
    ? schedule(configuration.scenarios.map(id => ({ id })), configuration.arms, configuration.reps, configuration.seed)
    : [];
  const validKeys = new Set(rows.filter(isValidTrial).map(row => row.key));
  const body = configuration
    ? renderHoles(rows, { expectedTrialKeys: expectedTrials.map(trial => trial.key) })
    : rows.length ? renderHoles(rows) : 'No trials recorded yet.\n';
  writeAtomic(path.join(options.out, 'holes.md'), header + body);
  writeAtomic(
    path.join(options.out, 'summary.json'),
    `${JSON.stringify({
      meta,
      difficulty: difficulty(rows.filter(isValidTrial)),
      rows: rows.length,
      expectedTrials: expectedTrials.length || null,
      validScheduledTrials: validKeys.size,
      complete: expectedTrials.length ? validKeys.size === expectedTrials.length : null,
      validRows: rows.filter(isValidTrial).length,
      invalidRows: rows.filter(row => !isValidTrial(row)).length
    }, null, 2)}\n`
  );
  writeAtomic(
    path.join(options.out, 'evidence.json'),
    `${JSON.stringify(evidence.buildManifest({
      meta,
      configuration: meta.configuration,
      scenarioFingerprints: meta.scenarioFingerprints,
      sourceFingerprints: meta.sourceFingerprints,
      outDir: options.out,
      rows
    }), null, 2)}\n`
  );
  return header + body;
}

function checkGraders(scenarios) {
  const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-grader-'));
  try {
    let bad = 0;
    for (const scenario of scenarios) {
      const { scores, sound } = graderIsSound(scenario, workRoot);
      const mark = sound ? 'ok  ' : 'BAD ';
      process.stdout.write(`${mark} ${scenario.id}: ${formatGraderScores(scores)}\n`);
      if (!sound) bad++;
    }
    if (bad) throw new Error(`${bad} grader(s) do not separate start, trap and reference, or pass a decoy`);
  } finally {
    fs.rmSync(workRoot, { recursive: true, force: true });
  }
}

function main() {
  const options = parseArgs(process.argv.slice(2));
  if (options.help) {
    process.stdout.write(`${USAGE}\n`);
    return;
  }
  if (options.verifyOnly) {
    const manifest = evidence.verifyManifest(options.out);
    process.stdout.write(`Evidence verified: ${Object.keys(manifest.files).length} artifact(s), schema ${manifest.schemaVersion}.\n`);
    return;
  }
  const savedMetaFile = options.out && path.join(options.out, 'meta.json');
  if (options.summarizeOnly && savedMetaFile && fs.existsSync(savedMetaFile)) {
    const saved = JSON.parse(fs.readFileSync(savedMetaFile, 'utf8'));
    const config = saved.configuration;
    if (config) Object.assign(options, {
      model: config.model, judgeModel: config.judgeModel, arms: config.arms,
      scenarios: config.scenarios, reps: config.reps, userTurns: config.userTurns,
      maxTurns: config.maxTurns, timeoutMin: config.timeoutMin, seed: config.seed
    });
  }
  const scenarios = loadIntentScenarios(undefined, options.scenarios);
  if (options.checkGraders) {
    checkGraders(scenarios);
    return;
  }

  fs.mkdirSync(path.join(options.out, 'transcripts'), { recursive: true });
  const resultsFile = path.join(options.out, 'results.jsonl');
  const metaFile = path.join(options.out, 'meta.json');
  const trees = { candidate: resolveRef(REPO_ROOT, options.candidateRef) };
  if (options.arms.includes('main')) trees.main = resolveRef(REPO_ROOT, options.mainRef);
  const configuration = {
    model: options.model, judgeModel: options.judgeModel, candidateSha: trees.candidate, mainSha: trees.main || null,
    reps: options.reps, userTurns: options.userTurns, arms: options.arms, scenarios: scenarios.map(scenario => scenario.id),
    maxTurns: options.maxTurns, timeoutMin: options.timeoutMin, seed: options.seed
  };
  const meta = {
    model: options.model, judgeModel: options.judgeModel, sha: trees.candidate, mainSha: trees.main || null,
    reps: options.reps, userTurns: options.userTurns, arms: options.arms,
    scenarios: scenarios.map(scenario => scenario.id), maxTurns: options.maxTurns, timeoutMin: options.timeoutMin,
    seed: options.seed, configuration,
    scenarioFingerprints: Object.fromEntries(scenarios.map(scenario => [scenario.id, evidence.hashDirectory(scenario.root)])),
    sourceFingerprints: Object.fromEntries([
      'run-intent.js', 'coverage.js', 'intent-eval.js', 'lib.js', 'session.js', 'user-sim.js', 'arms.js', 'arm-patch.js', 'evidence.js',
      '../context-profiles/ai-eval-lib.js'
    ].map(name => [name, evidence.hashFile(path.resolve(__dirname, name))]))
  };
  if (fs.existsSync(metaFile)) {
    const previous = JSON.parse(fs.readFileSync(metaFile, 'utf8'));
    if (!options.summarizeOnly && !options.dryRun && (
      JSON.stringify(previous.configuration) !== JSON.stringify(configuration) ||
      JSON.stringify(previous.scenarioFingerprints) !== JSON.stringify(meta.scenarioFingerprints) ||
      JSON.stringify(previous.sourceFingerprints) !== JSON.stringify(meta.sourceFingerprints)
    )) {
      throw new Error('--out was started with a different experiment configuration; use a new --out');
    }
  }
  const rows = readResults(resultsFile);
  const planned = schedule(scenarios, options.arms, options.reps, options.seed);
  evidence.validateRows(rows, planned);
  if (options.summarizeOnly) {
    const previous = JSON.parse(fs.readFileSync(metaFile, 'utf8'));
    if (JSON.stringify(previous.scenarioFingerprints) !== JSON.stringify(meta.scenarioFingerprints) ||
        JSON.stringify(previous.sourceFingerprints) !== JSON.stringify(meta.sourceFingerprints)) {
      throw new Error('--summarize cannot use changed scenario or harness inputs; preserve the original run inputs or use a new --out');
    }
    process.stdout.write(writeReport(options, rows, previous));
    return;
  }

  const workRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'gateguard-intent-'));
  try {
    // A grader that does not separate start, trap and reference cannot produce a
    // usable trial, so it is caught before any session is billed.
    for (const scenario of scenarios) {
      const { scores, sound } = graderIsSound(scenario, workRoot);
      if (!sound) {
        throw new Error(`${scenario.id}: grader scores ${formatGraderScores(scores)}; fix it before spending sessions`);
      }
    }
    const treeRoots = {};
    for (const [name, sha] of Object.entries(trees)) {
      treeRoots[name] = materializeTree(REPO_ROOT, sha, path.join(workRoot, 'trees', name));
    }
    const armSettingsByName = Object.fromEntries(
      options.arms.map(arm => [arm, armSettings(ARMS[arm].gate ? treeRoots[ARMS[arm].tree] : null)])
    );
    // Failed attempts remain in results.jsonl for audit, but the scheduled key
    // is complete only after a valid trial has been recorded.
    const done = completedTrialKeys(rows);
    const trials = planned.filter(trial => !done.has(trial.key));
    process.stderr.write(`${trials.length} trials to run (${done.size} already recorded)\n`);
    if (options.dryRun) {
      for (const trial of trials) process.stdout.write(`${trial.key}\n`);
      return;
    }
    writeAtomic(metaFile, `${JSON.stringify(meta, null, 2)}\n`);
    let judgeFailures = 0;
    for (const [index, trial] of trials.entries()) {
      const sessionsBefore = listSessions();
      const transcript = [];
      const ownedWorkspaces = new Set();
      const execute = (file, args, spawnOptions) => {
        if (spawnOptions.cwd) {
          const cwd = path.resolve(spawnOptions.cwd);
          const relative = path.relative(workRoot, cwd);
          if (relative && !relative.startsWith(`..${path.sep}`) && relative !== '..' && !path.isAbsolute(relative)) {
            ownedWorkspaces.add(cwd);
          }
        }
        const result = spawnSync(file, args, spawnOptions);
        if (result.stdout) transcript.push(result.stdout);
        return result;
      };
      let row;
      try {
        row = runIntentTrial(trial, {
          workRoot,
          armSettingsByName,
          executable: options.claude,
          model: options.model,
          judgeModel: options.judgeModel,
          maxTurns: options.maxTurns,
          timeoutMs: options.timeoutMin * 60000,
          userTurns: options.userTurns,
          execute
        });
      } finally {
        removeTrialSessions(sessionsBefore, ownedWorkspaces);
      }
      const attemptId = crypto.randomUUID();
      const transcriptName = `${trial.key.replace(/\//g, '__')}__${attemptId}.jsonl`;
      row.attemptId = attemptId;
      row.transcript = `transcripts/${transcriptName}`;
      fs.writeFileSync(
        path.join(options.out, row.transcript),
        transcript.join('\n')
      );
      row.transcriptSha256 = evidence.hashFile(path.join(options.out, row.transcript));
      fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
      // An unauthenticated or hookless run must not be recorded: graded as data it
      // looks like a finding (the first such trial was filed as a coverage hole).
      if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {
        throw new Error(`${trial.key}: Claude is not signed in (${row.providerMessage}); sign in with the claude CLI and rerun with the same --out`);
      }
      if (!row.hookObserved) {
        throw new Error(`${trial.key}: the gate did not run although the agent edited files; check the hook setup before spending more sessions`);
      }
      fs.appendFileSync(resultsFile, `${JSON.stringify(row)}\n`);
      rows.push(row);
      writeReport(options, rows, meta);
      const verdict = row.passed ? 'pass' : `score ${row.score}`;
      process.stderr.write(
        `[${index + 1}/${trials.length}] ${trial.key}: ${verdict}, ${row.outcomeClass}, asked=${row.asked}, decisive=${row.disclosedDecisive}, denials=${row.gateDenials}\n`
      );
      judgeFailures = row.judgeFailed ? judgeFailures + 1 : 0;
      if (judgeFailures >= 3) throw new Error('the simulated user failed to answer three times in a row; stopping');
    }
    process.stdout.write(writeReport(options, rows, meta));
  } finally {
    fs.rmSync(workRoot, { recursive: true, force: true });
  }
}

if (require.main === module) {
  try {
    main();
  } catch (error) {
    process.stderr.write(`[gateguard-intent] ${error.message}\n`);
    process.exitCode = 2;
  }
}

module.exports = { parseArgs, writeReport, checkGraders, completedTrialKeys, writeAtomic, sessionsDir, listSessions, projectSessionName, removeTrialSessions };