From 1f0d590da036f2e42ff2de843f278a3d2f8a9afd Mon Sep 17 00:00:00 2001 From: Affaan Mustafa Date: Wed, 5 Aug 2026 16:52:50 -0400 Subject: [PATCH] fix(ecc2): preserve harness evidence and legacy IDs --- README.md | 4 +- docs/design/ecc-ito-compute-integration.md | 21 +- docs/testing/ecc-ito-real-cli-bridge.tdd.md | 58 +- ecc2/src/harness_eval.rs | 205 +++- ecc2/src/main.rs | 145 ++- ecc2/src/session/store.rs | 1208 ++++++++++++++++--- mcp-configs/mcp-servers.json | 2 +- scripts/ecc.js | 8 +- scripts/ito.js | 25 +- scripts/lib/ito-environment.js | 5 +- skills/ito-compute/SKILL.md | 24 +- tests/ci/ito-compute-skill.test.js | 24 +- tests/scripts/ito-cli-bridge.test.js | 111 +- 13 files changed, 1551 insertions(+), 289 deletions(-) diff --git a/README.md b/README.md index bd91b353d..ccd26fad5 100644 --- a/README.md +++ b/README.md @@ -509,9 +509,9 @@ Kimi Code discovers the installed `.kimi/AGENTS.md` instructions and `.kimi/skil ### Itô compute CLI bridge -`ecc ito` delegates to the separately installed canonical Itô client; ECC does not maintain a second API client. `ecc ito auth` delegates to the canonical CLI's device authorization, opens the Itô verification page by default, and persists a device token in macOS Keychain; pass `--no-browser` to suppress the page handoff. ECC itself does no browser automation. The available operations are `ecc ito auth`, `ecc ito find`, `ecc ito status`, and the separately gated `ecc ito evals`. The matching MCP tools remain `ito_auth`, `ito_find`, and `ito_status`; node qualification is CLI-only. +`ecc ito` delegates to the separately installed canonical Itô client; ECC does not maintain a second API client. `ecc ito login [--no-browser]` performs device authorization, opens the Itô verification page by default, and persists a device token in macOS Keychain; `--no-browser` suppresses the page handoff. ECC itself does no browser automation. `ecc ito auth` is validation-only and rejects `--no-browser`. The available operations are `ecc ito login`, `ecc ito auth`, `ecc ito find`, `ecc ito status`, and the separately gated `ecc ito evals`. The matching MCP tools remain `ito_auth`, `ito_find`, and `ito_status`; `ito_auth` validates existing credentials and node qualification is CLI-only. -The `ito-compute-cli` package is currently unpublished. Build it locally from the Itô runtime repo (private while the desk hardens; design partners get access) under `cli/ito-compute-cli`, run `npm ci` and `npm run check`, then set `ECC_ITO_CLI_EXECUTABLE` to that build's absolute `dist/bin/ito.js` path. Device authorization is the default; keep legacy `ITO_API_KEY` only with explicit `ITO_AUTH_MODE=legacy`. ECC does not discover this credential-bearing client through `PATH`. See the [`ito-compute` skill](skills/ito-compute/SKILL.md) for the full RFQ authority and MCP setup contract. +The `ito-compute-cli` package is currently unpublished. Build it locally from the Itô runtime repo (private while the desk hardens; design partners get access) under `cli/ito-compute-cli`, run `npm ci` and `npm run check`, then set `ECC_ITO_CLI_EXECUTABLE` to that build's absolute `dist/bin/ito.js` path. Login never inherits `ITO_API_KEY`; auth, find, and status forward `ITO_API_KEY` directly when configured, and `ITO_AUTH_MODE=legacy` is not required. Device tokens use macOS Keychain by default; explicit file fallback must retain owner-only directory/file permissions. ECC does not discover this credential-bearing client through `PATH`. See the [`ito-compute` skill](skills/ito-compute/SKILL.md) for the full RFQ authority and MCP setup contract. `find` submits a live authenticated RFQ. It does not reserve capacity. `evals` requires both `ITO_ENABLE_SIXTYTWO_LIVE=1` and `--live-sixtytwo`, a separately installed `sixtytwo-cli==0.3.33`, an explicit node list, and an existing absolute configuration directory. It cannot rent, launch, recover, repair, or purchase. ECC exposes no quote lock, purchase, workload, or inference path, and it never replaces a missing client or failed live call with a local result. diff --git a/docs/design/ecc-ito-compute-integration.md b/docs/design/ecc-ito-compute-integration.md index 37a4d8e1e..7a4840032 100644 --- a/docs/design/ecc-ito-compute-integration.md +++ b/docs/design/ecc-ito-compute-integration.md @@ -24,9 +24,10 @@ ECC delegates to the canonical Itô package in `Ito-Markets/ito-cloud-runtime/cli/ito-compute-cli`. ECC does not maintain a second API client or response schema. -The wrapper exposes only the canonical CLI's `auth`, `find`, `status`, and `evals` +The wrapper exposes only the canonical CLI's `login`, `auth`, `find`, `status`, and `evals` operations: + ecc ito login [--no-browser] ecc ito auth ecc ito find ecc ito status @@ -36,11 +37,12 @@ The canonical MCP server exposes only `ito_auth`, `ito_find`, and `ito_status`. ECC includes an opt-in configuration template pointing to the local built MCP entry. It does not enable the server by default. -The former browser/manual-copy command is retired. `ecc ito auth` delegates to +The former browser/manual-copy command is retired. `ecc ito login` delegates to the canonical CLI's device authorization, which opens the Itô verification page by default and persists a device token in macOS Keychain. `--no-browser` suppresses that page handoff. ECC itself performs no browser automation and -stores no economic state. +stores no economic state. `ecc ito auth` is validation-only, never starts +device login, and rejects `--no-browser`. ## Local install @@ -63,15 +65,18 @@ For MCP, configure `node` with: /absolute/path/to/ito-cloud-runtime/cli/ito-compute-cli/dist/bin/ito-mcp.js -Device authorization is the default. ECC forwards only the required device -authorization settings, optional Itô endpoint overrides, and the minimum -process environment. Legacy `ITO_API_KEY` is forwarded only with explicit -`ITO_AUTH_MODE=legacy`; ECC does not inspect or log it. +Device login forwards only required authorization settings, optional Itô +endpoint overrides, and the minimum process environment; it never inherits +`ITO_API_KEY`. The `auth`, `find`, and `status` commands forward `ITO_API_KEY` +directly when configured; `ITO_AUTH_MODE=legacy` is not required. Device tokens +use macOS Keychain by default. Explicit file fallback retains owner-only 0700 +directory and 0600 token-file permissions. ECC does not inspect or log secrets. ## Authority and economics -- `auth` starts canonical device authorization, with `--no-browser` available +- `login` starts canonical device authorization, with `--no-browser` available when the operator does not want the CLI to open the verification page. +- `auth` validates existing credentials only. - `find` reads live inventory and submits a live authenticated RFQ. An operator or agent must gather every hard topology/economic constraint and obtain explicit buyer authority before invoking it. diff --git a/docs/testing/ecc-ito-real-cli-bridge.tdd.md b/docs/testing/ecc-ito-real-cli-bridge.tdd.md index a00328dc4..824d822d3 100644 --- a/docs/testing/ecc-ito-real-cli-bridge.tdd.md +++ b/docs/testing/ecc-ito-real-cli-bridge.tdd.md @@ -1,14 +1,14 @@ # ECC × Itô Real CLI Bridge — TDD Evidence -Date: 2026-07-23 +Date: 2026-08-05 Source plan: requirements were derived from the approved implementation handoff. No external plan file was executed. ## User journeys -1. As an ECC operator, I can invoke the canonical local Itô `auth`, `find`, and - `status` operations without a duplicate client or browser workflow. +1. As an ECC operator, I can explicitly invoke streaming device `login`, then + use validation-only `auth`, `find`, and `status` without a duplicate client. 2. As a security reviewer, I can prove unsupported operations, missing local installs, and ECC dry-run requests fail before any child process or network operation. @@ -21,62 +21,46 @@ Before production changes: ```text node tests/scripts/ito-cli-bridge.test.js -Passed: 0 -Failed: 9 +Passed: 13 +Failed: 8 node tests/ci/ito-compute-skill.test.js -Passed: 0 -Failed: 4 +Passed: 2 +Failed: 3 ``` -The failures were caused by the old browser-only `rent` command and the missing -real skill/install/MCP surfaces. +The failures captured the old combined auth/login surface, legacy-mode API-key +gate, buffered login output, and stale help, skill, MCP, and integration wording. ## GREEN evidence ```text node tests/scripts/ito-cli-bridge.test.js -Passed: 9 +Passed: 21 Failed: 0 node tests/ci/ito-compute-skill.test.js -Passed: 4 +Passed: 5 Failed: 0 -NODE_PATH=/node_modules \ - node scripts/ci/validate-install-manifests.js -Validated 33 install modules, 80 install components, and 7 profiles - -npm test -Total Tests: 3159 -Passed: 3159 -Failed: 0 - -npm run coverage -Statements: 89.21% -Branches: 79.71% -Functions: 93.96% -Lines: 89.21% - -npm run security:ioc-scan -Supply-chain IOC scan passed +node scripts/ci/validate-skills.js +Validated 281 skill directories ``` -The isolated worktree temporarily reused the canonical ECC checkout's existing -`node_modules` through an untracked local symlink. The symlink was removed -after validation; no dependency installation or source change was made in the -canonical checkout. -ESLint and Markdown lint also pass for every changed source file. The complete -package dry-run contains the wrapper, environment boundary, skill, and MCP -configuration. +`node tests/scripts/ito-compute-sponsor.test.js` reached 11 passes and 2 failures; +both failures are setup failures because the current worktree lacks `ajv`. +`node scripts/ci/validate-install-manifests.js` is blocked by the same missing +module. No dependency installation was performed. ## Test specification | Guarantee | Test | Type | Result | |---|---|---|---| -| Only `auth`, `find`, and `status` spawn | `tests/scripts/ito-cli-bridge.test.js` | end-to-end process contract | PASS | +| `login`, `auth`, `find`, and `status` forward only their reviewed surfaces | `tests/scripts/ito-cli-bridge.test.js` | end-to-end process contract | PASS | +| Login output streams before completion and its exit status propagates | `tests/scripts/ito-cli-bridge.test.js` | async process contract | PASS | +| `auth --no-browser` fails before spawn | `tests/scripts/ito-cli-bridge.test.js` | negative process contract | PASS | | Full RFQ arguments cross unchanged | `tests/scripts/ito-cli-bridge.test.js` | integration | PASS | -| Only required Itô settings cross the child boundary | `tests/scripts/ito-cli-bridge.test.js` | security integration | PASS | +| Login scrubs the API key; auth/find/status forward it directly; evals stays isolated | `tests/scripts/ito-cli-bridge.test.js` | security integration | PASS | | Unsupported and dry-run operations fail before spawn | `tests/scripts/ito-cli-bridge.test.js` | negative end-to-end | PASS | | Missing/relative executables fail with exact local guidance | `tests/scripts/ito-cli-bridge.test.js` | negative end-to-end | PASS | | Child output and exit code are preserved | `tests/scripts/ito-cli-bridge.test.js` | end-to-end process contract | PASS | diff --git a/ecc2/src/harness_eval.rs b/ecc2/src/harness_eval.rs index 81d08f3c4..641275970 100644 --- a/ecc2/src/harness_eval.rs +++ b/ecc2/src/harness_eval.rs @@ -5,22 +5,93 @@ mod tests { use std::collections::BTreeMap; #[test] - fn candidate_id_is_content_addressed_over_canonical_json() { + fn candidate_id_addresses_canonical_config_and_normalized_references() { let first = CandidateSpec::new( json!({"model": "fixed", "limits": {"steps": 3, "tools": ["read"]}}), - vec!["trace://one".into()], - vec!["evidence://one".into()], + vec![" trace://two ".into(), "trace://one".into()], + vec!["evidence://two".into(), " evidence://one ".into()], ) .unwrap(); let second = CandidateSpec::new( json!({"limits": {"tools": ["read"], "steps": 3}, "model": "fixed"}), - vec!["trace://two".into()], - vec!["evidence://two".into()], + vec!["trace://one".into(), "trace://two".into()], + vec!["evidence://one".into(), "evidence://two".into()], ) .unwrap(); assert_eq!(first.id, second.id); assert_eq!(first.canonical_config, second.canonical_config); + assert_eq!(first.trace_refs, vec!["trace://one", "trace://two"]); + assert_eq!( + first.evidence_refs, + vec!["evidence://one", "evidence://two"] + ); + } + + #[test] + fn candidate_id_changes_when_any_immutable_reference_changes() { + let original = CandidateSpec::new( + json!({"model": "fixed"}), + vec!["trace://one".into()], + vec!["evidence://one".into()], + ) + .unwrap(); + let changed_trace = CandidateSpec::new( + json!({"model": "fixed"}), + vec!["trace://two".into()], + vec!["evidence://one".into()], + ) + .unwrap(); + let changed_evidence = CandidateSpec::new( + json!({"model": "fixed"}), + vec!["trace://one".into()], + vec!["evidence://two".into()], + ) + .unwrap(); + + assert_ne!(original.id, changed_trace.id); + assert_ne!(original.id, changed_evidence.id); + } + + #[test] + fn candidate_integrity_rejects_reference_tampering() { + let mut candidate = CandidateSpec::new( + json!({"model": "fixed"}), + vec!["trace://one".into()], + vec!["evidence://one".into()], + ) + .unwrap(); + candidate.trace_refs = vec!["trace://tampered".into()]; + + assert!(candidate.verify_integrity().is_err()); + + let mut noncanonical = CandidateSpec::new( + json!({"model": "fixed"}), + vec!["trace://one".into(), "trace://two".into()], + vec!["evidence://one".into()], + ) + .unwrap(); + noncanonical.trace_refs.reverse(); + assert!(noncanonical.verify_integrity().is_err()); + } + + #[test] + fn persisted_candidate_integrity_accepts_only_exact_v1_or_v2_ids() { + let candidate = CandidateSpec::new( + json!({"model": "fixed", "limits": {"steps": 3}}), + vec!["trace://one".into()], + vec!["evidence://one".into()], + ) + .unwrap(); + let legacy_id = candidate.legacy_id(); + + candidate.verify_persisted_id(&candidate.id).unwrap(); + candidate.verify_persisted_id(&legacy_id).unwrap(); + assert!(candidate + .verify_persisted_id(&"a".repeat(64)) + .unwrap_err() + .to_string() + .contains("content address")); } #[test] @@ -120,14 +191,18 @@ pub struct CandidateSpec { impl CandidateSpec { pub fn new(config: Value, trace_refs: Vec, evidence_refs: Vec) -> Result { - validate_refs("trace", &trace_refs)?; - validate_refs("evidence", &evidence_refs)?; + let trace_refs = normalize_refs("trace", trace_refs)?; + let evidence_refs = normalize_refs("evidence", evidence_refs)?; let canonical_config = serde_json::to_string(&canonicalize(config))?; if canonical_config.len() > 1024 * 1024 { bail!("candidate configuration exceeds 1 MiB"); } - let digest = Sha256::digest(canonical_config.as_bytes()); - let id = digest.iter().map(|byte| format!("{byte:02x}")).collect(); + let artifact = serde_json::to_string(&CanonicalCandidateArtifact { + config: serde_json::from_str(&canonical_config)?, + trace_refs: &trace_refs, + evidence_refs: &evidence_refs, + })?; + let id = sha256_hex(artifact.as_bytes()); Ok(Self { id, canonical_config, @@ -137,23 +212,71 @@ impl CandidateSpec { } pub fn verify_integrity(&self) -> Result<()> { - let value: Value = serde_json::from_str(&self.canonical_config)?; - let rebuilt = Self::new(value, self.trace_refs.clone(), self.evidence_refs.clone())?; - if rebuilt.id != self.id || rebuilt.canonical_config != self.canonical_config { + self.verify_persisted_id(&self.id)?; + if self.id != self.id_for_v2()? { bail!("candidate content address or canonical configuration is invalid"); } Ok(()) } + + pub fn legacy_id(&self) -> String { + sha256_hex(self.canonical_config.as_bytes()) + } + + pub fn verify_persisted_id(&self, persisted_id: &str) -> Result<()> { + let value: Value = serde_json::from_str(&self.canonical_config)?; + let rebuilt = Self::new(value, self.trace_refs.clone(), self.evidence_refs.clone())?; + let is_v1 = persisted_id == self.legacy_id(); + let is_v2 = persisted_id == rebuilt.id; + if rebuilt.canonical_config != self.canonical_config + || (!is_v1 && !is_v2) + || (is_v2 + && (rebuilt.trace_refs != self.trace_refs + || rebuilt.evidence_refs != self.evidence_refs)) + { + bail!("candidate content address or canonical configuration is invalid"); + } + Ok(()) + } + + pub(crate) fn id_for_v2(&self) -> Result { + Ok(Self::new( + serde_json::from_str(&self.canonical_config)?, + self.trace_refs.clone(), + self.evidence_refs.clone(), + )? + .id) + } } -fn validate_refs(kind: &str, refs: &[String]) -> Result<()> { +#[derive(Serialize)] +struct CanonicalCandidateArtifact<'a> { + config: Value, + trace_refs: &'a [String], + evidence_refs: &'a [String], +} + +fn normalize_refs(kind: &str, refs: Vec) -> Result> { if refs.is_empty() || refs.iter().any(|reference| reference.trim().is_empty()) { bail!("at least one non-empty {kind} reference is required"); } if refs.len() > 100 || refs.iter().any(|reference| reference.len() > 4096) { bail!("{kind} references exceed bounded limits"); } - Ok(()) + let mut normalized = refs + .into_iter() + .map(|reference| reference.trim().to_string()) + .collect::>(); + normalized.sort(); + normalized.dedup(); + Ok(normalized) +} + +fn sha256_hex(bytes: &[u8]) -> String { + Sha256::digest(bytes) + .iter() + .map(|byte| format!("{byte:02x}")) + .collect() } fn canonicalize(value: Value) -> Value { @@ -184,6 +307,50 @@ pub struct RecordedEvidence { pub health: BTreeMap, } +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct HealthEvidenceSnapshot { + pub schema_version: u8, + pub evaluator: String, + pub candidate_id: String, + pub asserted_healthy: bool, +} + +impl HealthEvidenceSnapshot { + pub fn new(evaluator: &str, candidate_id: &str, asserted_healthy: bool) -> Result { + let snapshot = Self { + schema_version: 1, + evaluator: evaluator.to_string(), + candidate_id: candidate_id.to_string(), + asserted_healthy, + }; + snapshot.verify()?; + Ok(snapshot) + } + + pub fn canonical_json(&self) -> Result { + self.verify()?; + Ok(serde_json::to_string(self)?) + } + + pub fn digest(&self) -> Result { + Ok(sha256_hex(self.canonical_json()?.as_bytes())) + } + + pub fn verify(&self) -> Result<()> { + if self.schema_version != 1 + || self.evaluator != "recorded-v1" + || self.candidate_id.len() != 64 + || !self + .candidate_id + .bytes() + .all(|byte| byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte)) + { + bail!("invalid canonical health evidence snapshot"); + } + Ok(()) + } +} + pub struct RecordedEvaluator { name: String, scores: BTreeMap<(String, u64), f64>, @@ -238,6 +405,16 @@ impl RecordedEvaluator { }) } + pub fn health_evidence_snapshot(&self) -> Result { + HealthEvidenceSnapshot::new( + &self.name, + self.health_candidate + .as_deref() + .context("candidate-keyed health evidence is required")?, + self.health_ok, + ) + } + #[cfg(test)] pub fn calls(&self) -> &[(String, u64)] { &self.calls diff --git a/ecc2/src/main.rs b/ecc2/src/main.rs index db7b759f4..c4c078b88 100644 --- a/ecc2/src/main.rs +++ b/ecc2/src/main.rs @@ -447,25 +447,37 @@ enum Commands { enum HarnessEvalCommands { /// Record an immutable content-addressed candidate from a local JSON file Record { - #[arg(long)] config: PathBuf, - #[arg(long = "trace-ref", required = true)] trace_refs: Vec, - #[arg(long = "evidence-ref", required = true)] evidence_refs: Vec, + #[arg(long)] + config: PathBuf, + #[arg(long = "trace-ref", required = true)] + trace_refs: Vec, + #[arg(long = "evidence-ref", required = true)] + evidence_refs: Vec, }, /// Set the first baseline; subsequent changes require evaluation ActivateInitial { candidate_id: String, - #[arg(long)] evidence_ref: String, + #[arg(long)] + evidence_ref: String, }, /// Evaluate paired scores and conditionally promote with a health gate Run { - #[arg(long)] candidate: String, - #[arg(long)] baseline: String, - #[arg(long = "seed", required = true)] seeds: Vec, - #[arg(long)] measurements: PathBuf, - #[arg(long)] evidence_ref: String, - #[arg(long)] min_samples: usize, - #[arg(long)] min_mean_delta: f64, - #[arg(long)] min_win_rate: f64, + #[arg(long)] + candidate: String, + #[arg(long)] + baseline: String, + #[arg(long = "seed", required = true)] + seeds: Vec, + #[arg(long)] + measurements: PathBuf, + #[arg(long)] + evidence_ref: String, + #[arg(long)] + min_samples: usize, + #[arg(long)] + min_mean_delta: f64, + #[arg(long)] + min_win_rate: f64, }, /// Show append-only promotion audit entries Audit, @@ -1423,30 +1435,72 @@ async fn main() -> Result<()> { match cli.command { Some(Commands::HarnessEval { command }) => match command { - HarnessEvalCommands::Record { config, trace_refs, evidence_refs } => { - let value: serde_json::Value = serde_json::from_slice(&read_bounded_file(&config, 1_048_576, "candidate configuration")?) - .with_context(|| format!("Invalid JSON in {}", config.display()))?; + HarnessEvalCommands::Record { + config, + trace_refs, + evidence_refs, + } => { + let value: serde_json::Value = serde_json::from_slice(&read_bounded_file( + &config, + 1_048_576, + "candidate configuration", + )?) + .with_context(|| format!("Invalid JSON in {}", config.display()))?; let candidate = harness_eval::CandidateSpec::new(value, trace_refs, evidence_refs)?; db.record_harness_candidate(&candidate)?; println!("{}", candidate.id); } - HarnessEvalCommands::ActivateInitial { candidate_id, evidence_ref } => { + HarnessEvalCommands::ActivateInitial { + candidate_id, + evidence_ref, + } => { db.activate_initial_harness(&candidate_id, &evidence_ref)?; println!("Activated initial baseline: {candidate_id}"); } - HarnessEvalCommands::Run { candidate, baseline, seeds, measurements, evidence_ref, min_samples, min_mean_delta, min_win_rate } => { + HarnessEvalCommands::Run { + candidate, + baseline, + seeds, + measurements, + evidence_ref, + min_samples, + min_mean_delta, + min_win_rate, + } => { use harness_eval::Evaluator; - let evidence: harness_eval::RecordedEvidence = serde_json::from_slice(&read_bounded_file(&measurements, 8_388_608, "recorded measurements")?) - .with_context(|| format!("Invalid recorded evidence in {}", measurements.display()))?; + let evidence: harness_eval::RecordedEvidence = serde_json::from_slice( + &read_bounded_file(&measurements, 8_388_608, "recorded measurements")?, + ) + .with_context(|| { + format!("Invalid recorded evidence in {}", measurements.display()) + })?; let mut evaluator = harness_eval::RecordedEvaluator::from_evidence(evidence)?; let evaluator_name = evaluator.name().to_string(); - let samples = harness_eval::evaluate_paired(&mut evaluator, &candidate, &baseline, &seeds)?; - let policy = harness_eval::PromotionPolicy { min_samples, min_mean_delta, min_win_rate }; - let outcome = db.evaluate_promote_and_health_check(&candidate, &baseline, &evaluator_name, &samples, policy, &evidence_ref, |id| evaluator.health_check(id))?; + let health_evidence = evaluator.health_evidence_snapshot()?; + let samples = + harness_eval::evaluate_paired(&mut evaluator, &candidate, &baseline, &seeds)?; + let policy = harness_eval::PromotionPolicy { + min_samples, + min_mean_delta, + min_win_rate, + }; + let outcome = db.evaluate_promote_and_health_check( + &candidate, + &baseline, + &evaluator_name, + &samples, + policy, + &evidence_ref, + &health_evidence, + |id| evaluator.health_check(id), + )?; println!("{}", serde_json::to_string_pretty(&outcome)?); } HarnessEvalCommands::Audit => { - println!("{}", serde_json::to_string_pretty(&db.harness_audit_entries()?)?); + println!( + "{}", + serde_json::to_string_pretty(&db.harness_audit_entries()?)? + ); } }, Some(Commands::Dashboard) | None => { @@ -8628,18 +8682,51 @@ mod tests { #[test] fn harness_eval_cli_requires_explicit_bounded_inputs() { let cli = Cli::try_parse_from([ - "ecc", "harness-eval", "run", "--candidate", "candidate", "--baseline", "baseline", - "--seed", "1", "--seed", "2", "--measurements", "scores.json", "--evidence-ref", - "evidence://run", "--min-samples", "2", "--min-mean-delta", "0.1", "--min-win-rate", "0.5", - ]).expect("valid harness evaluation command"); + "ecc", + "harness-eval", + "run", + "--candidate", + "candidate", + "--baseline", + "baseline", + "--seed", + "1", + "--seed", + "2", + "--measurements", + "scores.json", + "--evidence-ref", + "evidence://run", + "--min-samples", + "2", + "--min-mean-delta", + "0.1", + "--min-win-rate", + "0.5", + ]) + .expect("valid harness evaluation command"); match cli.command { - Some(Commands::HarnessEval { command: HarnessEvalCommands::Run { seeds, min_samples, .. } }) => { + Some(Commands::HarnessEval { + command: + HarnessEvalCommands::Run { + seeds, min_samples, .. + }, + }) => { assert_eq!(seeds, vec![1, 2]); assert_eq!(min_samples, 2); } other => panic!("unexpected command: {other:?}"), } - assert!(Cli::try_parse_from(["ecc", "harness-eval", "run", "--candidate", "c", "--baseline", "b"]).is_err()); + assert!(Cli::try_parse_from([ + "ecc", + "harness-eval", + "run", + "--candidate", + "c", + "--baseline", + "b" + ]) + .is_err()); } #[test] diff --git a/ecc2/src/session/store.rs b/ecc2/src/session/store.rs index 145de8d3d..f71bb3640 100644 --- a/ecc2/src/session/store.rs +++ b/ecc2/src/session/store.rs @@ -10,7 +10,7 @@ use std::time::Duration; use crate::comms; use crate::config::Config; -use crate::harness_eval::{CandidateSpec, PairedSample, PromotionPolicy}; +use crate::harness_eval::{CandidateSpec, HealthEvidenceSnapshot, PairedSample, PromotionPolicy}; use crate::observability::{ToolCallEvent, ToolLogEntry, ToolLogPage}; use super::output::{OutputLine, OutputStream, OUTPUT_BUFFER_LIMIT}; @@ -36,6 +36,11 @@ pub struct HarnessAuditEntry { pub prior_candidate_id: Option, pub evaluation_id: Option, pub evidence_ref: String, + pub health_evidence_json: Option, + pub health_evidence_sha256: Option, + pub asserted_health: Option, + pub health_check_status: Option, + pub legacy_unverifiable: bool, pub created_at: String, } @@ -430,6 +435,12 @@ impl StateStore { evidence_refs_json TEXT NOT NULL, created_at TEXT NOT NULL ); + CREATE TABLE IF NOT EXISTS harness_candidate_aliases ( + alias_id TEXT PRIMARY KEY, + candidate_id TEXT NOT NULL REFERENCES harness_candidates(id), + id_version INTEGER NOT NULL CHECK(id_version = 2), + created_at TEXT NOT NULL + ); CREATE TABLE IF NOT EXISTS harness_evaluations ( id INTEGER PRIMARY KEY AUTOINCREMENT, candidate_id TEXT NOT NULL REFERENCES harness_candidates(id), @@ -439,6 +450,11 @@ impl StateStore { policy_json TEXT NOT NULL, comparison_json TEXT NOT NULL, evidence_ref TEXT NOT NULL, + health_evidence_json TEXT, + health_evidence_sha256 TEXT, + asserted_health INTEGER, + health_check_status TEXT, + legacy_unverifiable INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL ); CREATE TABLE IF NOT EXISTS active_harness_config ( @@ -453,10 +469,17 @@ impl StateStore { prior_candidate_id TEXT REFERENCES harness_candidates(id), evaluation_id INTEGER REFERENCES harness_evaluations(id), evidence_ref TEXT NOT NULL, + health_evidence_json TEXT, + health_evidence_sha256 TEXT, + asserted_health INTEGER, + health_check_status TEXT, + legacy_unverifiable INTEGER NOT NULL DEFAULT 0, created_at TEXT NOT NULL ); CREATE TRIGGER IF NOT EXISTS harness_candidates_no_update BEFORE UPDATE ON harness_candidates BEGIN SELECT RAISE(ABORT, 'harness candidates are immutable'); END; CREATE TRIGGER IF NOT EXISTS harness_candidates_no_delete BEFORE DELETE ON harness_candidates BEGIN SELECT RAISE(ABORT, 'harness candidates are immutable'); END; + CREATE TRIGGER IF NOT EXISTS harness_candidate_aliases_no_update BEFORE UPDATE ON harness_candidate_aliases BEGIN SELECT RAISE(ABORT, 'harness candidate aliases are immutable'); END; + CREATE TRIGGER IF NOT EXISTS harness_candidate_aliases_no_delete BEFORE DELETE ON harness_candidate_aliases BEGIN SELECT RAISE(ABORT, 'harness candidate aliases are immutable'); END; CREATE TRIGGER IF NOT EXISTS harness_evaluations_no_update BEFORE UPDATE ON harness_evaluations BEGIN SELECT RAISE(ABORT, 'harness evaluations are immutable'); END; CREATE TRIGGER IF NOT EXISTS harness_evaluations_no_delete BEFORE DELETE ON harness_evaluations BEGIN SELECT RAISE(ABORT, 'harness evaluations are immutable'); END; CREATE TRIGGER IF NOT EXISTS harness_eval_audit_no_update BEFORE UPDATE ON harness_eval_audit BEGIN SELECT RAISE(ABORT, 'harness audit is immutable'); END; @@ -493,6 +516,8 @@ impl StateStore { ", )?; self.ensure_session_columns()?; + self.ensure_harness_eval_columns()?; + self.ensure_harness_candidate_aliases()?; self.ensure_session_board_columns()?; self.refresh_session_board_meta()?; Ok(()) @@ -861,6 +886,109 @@ impl StateStore { Ok(()) } + fn ensure_harness_eval_columns(&self) -> Result<()> { + for (table, column, definition) in [ + ("harness_evaluations", "health_evidence_json", "TEXT"), + ("harness_evaluations", "health_evidence_sha256", "TEXT"), + ("harness_evaluations", "asserted_health", "INTEGER"), + ("harness_evaluations", "health_check_status", "TEXT"), + ("harness_eval_audit", "health_evidence_json", "TEXT"), + ("harness_eval_audit", "health_evidence_sha256", "TEXT"), + ("harness_eval_audit", "asserted_health", "INTEGER"), + ("harness_eval_audit", "health_check_status", "TEXT"), + ] { + if !self.has_column(table, column)? { + self.conn + .execute( + &format!("ALTER TABLE {table} ADD COLUMN {column} {definition}"), + [], + ) + .with_context(|| format!("Failed to add {column} column to {table}"))?; + } + } + for table in ["harness_evaluations", "harness_eval_audit"] { + if !self.has_column(table, "legacy_unverifiable")? { + self.conn.execute( + &format!("ALTER TABLE {table} ADD COLUMN legacy_unverifiable INTEGER NOT NULL DEFAULT 1"), + [], + ).with_context(|| format!("Failed to mark legacy rows in {table}"))?; + } + } + Ok(()) + } + + fn ensure_harness_candidate_aliases(&self) -> Result<()> { + let mut statement = self.conn.prepare( + "SELECT id, canonical_config_json, trace_refs_json, evidence_refs_json FROM harness_candidates ORDER BY id", + )?; + let rows = statement + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? + .collect::>>()?; + drop(statement); + let tx = self.conn.unchecked_transaction()?; + for (id, canonical_config, trace_json, evidence_json) in rows { + let candidate = CandidateSpec { + id: id.clone(), + canonical_config, + trace_refs: serde_json::from_str(&trace_json)?, + evidence_refs: serde_json::from_str(&evidence_json)?, + }; + candidate.verify_persisted_id(&id)?; + if id == candidate.legacy_id() && id != candidate.id_for_v2()? { + Self::register_harness_alias(&tx, &candidate.id_for_v2()?, &id)?; + } + } + let mut aliases = tx.prepare( + "SELECT alias_id, candidate_id, id_version FROM harness_candidate_aliases ORDER BY alias_id", + )?; + let alias_rows = aliases + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + )) + })? + .collect::>>()?; + drop(aliases); + for (alias_id, target_id, version) in alias_rows { + let (canonical_config, trace_json, evidence_json) = tx.query_row( + "SELECT canonical_config_json, trace_refs_json, evidence_refs_json FROM harness_candidates WHERE id = ?1", + [&target_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?)), + )?; + let target = CandidateSpec { + id: target_id.clone(), + canonical_config, + trace_refs: serde_json::from_str(&trace_json)?, + evidence_refs: serde_json::from_str(&evidence_json)?, + }; + if version != 2 + || target_id != target.legacy_id() + || alias_id != target.id_for_v2()? + || tx + .query_row( + "SELECT 1 FROM harness_candidates WHERE id = ?1", + [&alias_id], + |_| Ok(()), + ) + .optional()? + .is_some() + { + anyhow::bail!("candidate alias integrity verification failed"); + } + } + tx.commit()?; + Ok(()) + } + fn ensure_session_board_columns(&self) -> Result<()> { if !self.has_column("session_board", "row_label")? { self.conn @@ -870,13 +998,19 @@ impl StateStore { if !self.has_column("session_board", "previous_lane")? { self.conn - .execute("ALTER TABLE session_board ADD COLUMN previous_lane TEXT", []) + .execute( + "ALTER TABLE session_board ADD COLUMN previous_lane TEXT", + [], + ) .context("Failed to add previous_lane column to session_board table")?; } if !self.has_column("session_board", "previous_row_label")? { self.conn - .execute("ALTER TABLE session_board ADD COLUMN previous_row_label TEXT", []) + .execute( + "ALTER TABLE session_board ADD COLUMN previous_row_label TEXT", + [], + ) .context("Failed to add previous_row_label column to session_board table")?; } @@ -918,25 +1052,37 @@ impl StateStore { if !self.has_column("session_board", "status_detail")? { self.conn - .execute("ALTER TABLE session_board ADD COLUMN status_detail TEXT", []) + .execute( + "ALTER TABLE session_board ADD COLUMN status_detail TEXT", + [], + ) .context("Failed to add status_detail column to session_board table")?; } if !self.has_column("session_board", "movement_note")? { self.conn - .execute("ALTER TABLE session_board ADD COLUMN movement_note TEXT", []) + .execute( + "ALTER TABLE session_board ADD COLUMN movement_note TEXT", + [], + ) .context("Failed to add movement_note column to session_board table")?; } if !self.has_column("session_board", "activity_kind")? { self.conn - .execute("ALTER TABLE session_board ADD COLUMN activity_kind TEXT", []) + .execute( + "ALTER TABLE session_board ADD COLUMN activity_kind TEXT", + [], + ) .context("Failed to add activity_kind column to session_board table")?; } if !self.has_column("session_board", "activity_note")? { self.conn - .execute("ALTER TABLE session_board ADD COLUMN activity_note TEXT", []) + .execute( + "ALTER TABLE session_board ADD COLUMN activity_note TEXT", + [], + ) .context("Failed to add activity_note column to session_board table")?; } @@ -951,7 +1097,10 @@ impl StateStore { if !self.has_column("session_board", "conflict_signal")? { self.conn - .execute("ALTER TABLE session_board ADD COLUMN conflict_signal TEXT", []) + .execute( + "ALTER TABLE session_board ADD COLUMN conflict_signal TEXT", + [], + ) .context("Failed to add conflict_signal column to session_board table")?; } @@ -1121,9 +1270,7 @@ impl StateStore { permission_mode: row.get(4)?, add_dirs: serde_json::from_str(&add_dirs_json).unwrap_or_default(), max_budget_usd: row.get(6)?, - token_budget: row - .get::<_, Option>(7)? - .map(|tokens| tokens as u64), + token_budget: row.get::<_, Option>(7)?.map(|tokens| tokens as u64), append_system_prompt: row.get(8)?, agent: None, }) @@ -2319,13 +2466,14 @@ impl StateStore { let now = chrono::Utc::now().to_rfc3339(); for session in sessions { - let mut meta = board_meta - .get(&session.id) - .cloned() - .unwrap_or_else(|| SessionBoardMeta { - lane: board_lane_for_state(&session.state).to_string(), - ..SessionBoardMeta::default() - }); + let mut meta = + board_meta + .get(&session.id) + .cloned() + .unwrap_or_else(|| SessionBoardMeta { + lane: board_lane_for_state(&session.state).to_string(), + ..SessionBoardMeta::default() + }); if let Some(previous) = existing_meta.get(&session.id) { annotate_board_motion(&mut meta, previous); } @@ -2735,10 +2883,7 @@ impl StateStore { .map_err(Into::into) } - fn latest_task_handoff_activity( - &self, - session_id: &str, - ) -> Result> { + fn latest_task_handoff_activity(&self, session_id: &str) -> Result> { let latest_handoff = self .conn .query_row( @@ -2759,49 +2904,52 @@ impl StateStore { ) .optional()?; - Ok(latest_handoff.and_then(|(from_session, to_session, content)| { - let context = extract_task_handoff_context(&content)?; - let routing_suffix = routing_activity_suffix(&context); + Ok( + latest_handoff.and_then(|(from_session, to_session, content)| { + let context = extract_task_handoff_context(&content)?; + let routing_suffix = routing_activity_suffix(&context); - if session_id == to_session { - Some(( - "received".to_string(), - format!( - "Received from {}{}", - short_session_ref(&from_session), - routing_suffix - .map(|value| format!(" | {value}")) - .unwrap_or_default() - ), - )) - } else if session_id == from_session { - let (kind, base) = match routing_suffix { - Some("spawned") => { - ("spawned", format!("Spawned {}", short_session_ref(&to_session))) - } - Some("spawned fallback") => ( - "spawned_fallback", - format!("Spawned fallback {}", short_session_ref(&to_session)), - ), - _ => ( - "delegated", - format!("Delegated to {}", short_session_ref(&to_session)), - ), - }; - Some(( - kind.to_string(), - format!( - "{base}{}", - routing_suffix - .filter(|value| !value.starts_with("spawned")) - .map(|value| format!(" | {value}")) - .unwrap_or_default() - ), - )) - } else { - None - } - })) + if session_id == to_session { + Some(( + "received".to_string(), + format!( + "Received from {}{}", + short_session_ref(&from_session), + routing_suffix + .map(|value| format!(" | {value}")) + .unwrap_or_default() + ), + )) + } else if session_id == from_session { + let (kind, base) = match routing_suffix { + Some("spawned") => ( + "spawned", + format!("Spawned {}", short_session_ref(&to_session)), + ), + Some("spawned fallback") => ( + "spawned_fallback", + format!("Spawned fallback {}", short_session_ref(&to_session)), + ), + _ => ( + "delegated", + format!("Delegated to {}", short_session_ref(&to_session)), + ), + }; + Some(( + kind.to_string(), + format!( + "{base}{}", + routing_suffix + .filter(|value| !value.starts_with("spawned")) + .map(|value| format!(" | {value}")) + .unwrap_or_default() + ), + )) + } else { + None + } + }), + ) } pub fn insert_decision( @@ -3921,21 +4069,22 @@ impl StateStore { .query_map( rusqlite::params![session_id, page_size as i64, offset as i64], |row| { - Ok(ToolLogEntry { - id: row.get(0)?, - session_id: row.get(1)?, - tool_name: row.get(2)?, - input_summary: row.get::<_, Option>(3)?.unwrap_or_default(), - input_params_json: row - .get::<_, Option>(4)? - .unwrap_or_else(|| "{}".to_string()), - output_summary: row.get::<_, Option>(5)?.unwrap_or_default(), - trigger_summary: row.get::<_, Option>(6)?.unwrap_or_default(), - duration_ms: row.get::<_, Option>(7)?.unwrap_or_default() as u64, - risk_score: row.get::<_, Option>(8)?.unwrap_or_default(), - timestamp: row.get(9)?, - }) - })? + Ok(ToolLogEntry { + id: row.get(0)?, + session_id: row.get(1)?, + tool_name: row.get(2)?, + input_summary: row.get::<_, Option>(3)?.unwrap_or_default(), + input_params_json: row + .get::<_, Option>(4)? + .unwrap_or_else(|| "{}".to_string()), + output_summary: row.get::<_, Option>(5)?.unwrap_or_default(), + trigger_summary: row.get::<_, Option>(6)?.unwrap_or_default(), + duration_ms: row.get::<_, Option>(7)?.unwrap_or_default() as u64, + risk_score: row.get::<_, Option>(8)?.unwrap_or_default(), + timestamp: row.get(9)?, + }) + }, + )? .collect::, _>>()?; Ok(ToolLogPage { @@ -4381,7 +4530,11 @@ fn derive_board_meta_map(sessions: &[Session]) -> HashMap Option { for label in labels { if let Some(index) = lowered.find(label) { - let mut tail = task.get(index + label.len()..)?.trim_start_matches([' ', ':', '-', '#']); + let mut tail = task + .get(index + label.len()..)? + .trim_start_matches([' ', ':', '-', '#']); if tail.is_empty() { continue; } @@ -4596,7 +4751,10 @@ fn derive_board_conflict_signals(sessions: &[Session]) -> HashMap>(); @@ -4619,7 +4777,11 @@ fn derive_board_conflict_signals(sessions: &[Session]) -> HashMap Option<&'static str> { } fn extract_task_handoff_context(content: &str) -> Option { - if let Some(crate::comms::MessageType::TaskHandoff { context, .. }) = crate::comms::parse(content) + if let Some(crate::comms::MessageType::TaskHandoff { context, .. }) = + crate::comms::parse(content) { return Some(context); } @@ -5127,89 +5290,358 @@ fn overlap_state_priority(state: &SessionState) -> u8 { } impl StateStore { - pub fn record_harness_candidate(&self, candidate: &CandidateSpec) -> Result<()> { - candidate.verify_integrity()?; - let existing: Option<(String, String, String)> = self.conn.query_row( - "SELECT canonical_config_json, trace_refs_json, evidence_refs_json FROM harness_candidates WHERE id = ?1", - [&candidate.id], - |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), - ).optional()?; - let trace_json = serde_json::to_string(&candidate.trace_refs)?; - let evidence_json = serde_json::to_string(&candidate.evidence_refs)?; + fn register_harness_alias( + tx: &rusqlite::Transaction<'_>, + alias_id: &str, + candidate_id: &str, + ) -> Result<()> { + if tx + .query_row( + "SELECT 1 FROM harness_candidates WHERE id = ?1", + [alias_id], + |_| Ok(()), + ) + .optional()? + .is_some() + { + anyhow::bail!("candidate alias collision with physical candidate id"); + } + let existing = tx + .query_row( + "SELECT candidate_id FROM harness_candidate_aliases WHERE alias_id = ?1", + [alias_id], + |row| row.get::<_, String>(0), + ) + .optional()?; if let Some(existing) = existing { - if existing != (candidate.canonical_config.clone(), trace_json, evidence_json) { - anyhow::bail!("candidate id already exists with different immutable content or references"); + if existing != candidate_id { + anyhow::bail!("candidate alias collision with different immutable target"); } return Ok(()); } - self.conn.execute( - "INSERT INTO harness_candidates (id, canonical_config_json, trace_refs_json, evidence_refs_json, created_at) VALUES (?1, ?2, ?3, ?4, ?5)", - rusqlite::params![candidate.id, candidate.canonical_config, trace_json, evidence_json, chrono::Utc::now().to_rfc3339()], + tx.execute( + "INSERT INTO harness_candidate_aliases (alias_id, candidate_id, id_version, created_at) VALUES (?1, ?2, 2, ?3)", + rusqlite::params![alias_id, candidate_id, chrono::Utc::now().to_rfc3339()], )?; Ok(()) } + fn resolve_harness_candidate_id(connection: &Connection, candidate_id: &str) -> Result { + if let Some(target) = connection + .query_row( + "SELECT candidate_id FROM harness_candidate_aliases WHERE alias_id = ?1", + [candidate_id], + |row| row.get::<_, String>(0), + ) + .optional()? + { + return Ok(target); + } + connection + .query_row( + "SELECT id FROM harness_candidates WHERE id = ?1", + [candidate_id], + |row| row.get(0), + ) + .with_context(|| format!("unknown harness candidate id {candidate_id}")) + } + + pub fn record_harness_candidate(&self, candidate: &CandidateSpec) -> Result<()> { + candidate.verify_integrity()?; + let trace_json = serde_json::to_string(&candidate.trace_refs)?; + let evidence_json = serde_json::to_string(&candidate.evidence_refs)?; + let legacy_id = candidate.legacy_id(); + let legacy = self + .conn + .query_row( + "SELECT canonical_config_json, trace_refs_json, evidence_refs_json FROM harness_candidates WHERE id = ?1", + [&legacy_id], + |row| Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?, row.get::<_, String>(2)?)), + ) + .optional()?; + let expected = ( + candidate.canonical_config.clone(), + trace_json.clone(), + evidence_json.clone(), + ); + if let Some(stored) = legacy { + let legacy_candidate = CandidateSpec { + id: legacy_id.clone(), + canonical_config: stored.0, + trace_refs: serde_json::from_str(&stored.1)?, + evidence_refs: serde_json::from_str(&stored.2)?, + }; + legacy_candidate.verify_persisted_id(&legacy_id)?; + if legacy_candidate.id_for_v2()? != candidate.id { + anyhow::bail!("legacy candidate id collision with different immutable content"); + } + let tx = self.conn.unchecked_transaction()?; + Self::register_harness_alias(&tx, &candidate.id, &legacy_id)?; + tx.commit()?; + return Ok(()); + } + self.conn.execute( + "INSERT INTO harness_candidates (id, canonical_config_json, trace_refs_json, evidence_refs_json, created_at) + VALUES (?1, ?2, ?3, ?4, ?5) ON CONFLICT(id) DO NOTHING", + rusqlite::params![candidate.id, candidate.canonical_config, trace_json, evidence_json, chrono::Utc::now().to_rfc3339()], + )?; + let stored: (String, String, String) = self.conn.query_row( + "SELECT canonical_config_json, trace_refs_json, evidence_refs_json FROM harness_candidates WHERE id = ?1", + [&candidate.id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + if stored != expected { + anyhow::bail!("candidate id collision with different immutable content"); + } + Ok(()) + } + pub fn activate_initial_harness(&self, candidate_id: &str, evidence_ref: &str) -> Result<()> { - if candidate_id.len() != 64 || evidence_ref.trim().is_empty() || evidence_ref.len() > 4096 { anyhow::bail!("valid candidate id and bounded activation evidence reference are required"); } + if candidate_id.len() != 64 || evidence_ref.trim().is_empty() || evidence_ref.len() > 4096 { + anyhow::bail!( + "valid candidate id and bounded activation evidence reference are required" + ); + } let tx = self.conn.unchecked_transaction()?; - if tx.query_row("SELECT candidate_id FROM active_harness_config WHERE slot = 'default'", [], |row| row.get::<_, String>(0)).optional()?.is_some() { + let stored_candidate_id = Self::resolve_harness_candidate_id(&tx, candidate_id)?; + if tx + .query_row( + "SELECT candidate_id FROM active_harness_config WHERE slot = 'default'", + [], + |row| row.get::<_, String>(0), + ) + .optional()? + .is_some() + { anyhow::bail!("an active harness configuration already exists"); } let now = chrono::Utc::now().to_rfc3339(); - tx.execute("INSERT INTO active_harness_config (slot, candidate_id, updated_at) VALUES ('default', ?1, ?2)", rusqlite::params![candidate_id, now])?; - tx.execute("INSERT INTO harness_eval_audit (event_type, candidate_id, evidence_ref, created_at) VALUES ('initial_activation', ?1, ?2, ?3)", rusqlite::params![candidate_id, evidence_ref, now])?; + tx.execute("INSERT INTO active_harness_config (slot, candidate_id, updated_at) VALUES ('default', ?1, ?2)", rusqlite::params![stored_candidate_id, now])?; + tx.execute("INSERT INTO harness_eval_audit (event_type, candidate_id, evidence_ref, legacy_unverifiable, created_at) VALUES ('initial_activation', ?1, ?2, 0, ?3)", rusqlite::params![stored_candidate_id, evidence_ref, now])?; tx.commit()?; Ok(()) } #[cfg(test)] pub fn active_harness_id(&self) -> Result> { - Ok(self.conn.query_row("SELECT candidate_id FROM active_harness_config WHERE slot = 'default'", [], |row| row.get(0)).optional()?) + let stored = self + .conn + .query_row( + "SELECT candidate_id FROM active_harness_config WHERE slot = 'default'", + [], + |row| row.get(0), + ) + .optional()?; + if let Some(stored) = stored { + Ok(Some( + self.conn + .query_row( + "SELECT alias_id FROM harness_candidate_aliases WHERE candidate_id = ?1 AND id_version = 2", + [&stored], + |row| row.get(0), + ) + .optional()? + .unwrap_or(stored), + )) + } else { + Ok(None) + } } #[allow(clippy::too_many_arguments)] - pub fn evaluate_promote_and_health_check(&self, candidate_id: &str, baseline_id: &str, evaluator: &str, samples: &[PairedSample], policy: PromotionPolicy, evidence_ref: &str, health_check: F) -> Result - where F: FnOnce(&str) -> Result { - if candidate_id.len() != 64 || baseline_id.len() != 64 || evaluator != "recorded-v1" || evidence_ref.trim().is_empty() || evidence_ref.len() > 4096 { anyhow::bail!("valid candidate ids, recorded-v1 evaluator, and bounded evidence reference are required"); } + pub fn evaluate_promote_and_health_check( + &self, + candidate_id: &str, + baseline_id: &str, + evaluator: &str, + samples: &[PairedSample], + policy: PromotionPolicy, + evidence_ref: &str, + health_evidence: &HealthEvidenceSnapshot, + health_check: F, + ) -> Result + where + F: FnOnce(&str) -> Result, + { + if candidate_id.len() != 64 + || baseline_id.len() != 64 + || evaluator != "recorded-v1" + || evidence_ref.trim().is_empty() + || evidence_ref.len() > 4096 + { + anyhow::bail!("valid candidate ids, recorded-v1 evaluator, and bounded evidence reference are required"); + } + health_evidence.verify()?; + if health_evidence.candidate_id != candidate_id || health_evidence.evaluator != evaluator { + anyhow::bail!("health evidence does not match candidate and evaluator"); + } let comparison = policy.compare(samples)?; let tx = self.conn.unchecked_transaction()?; - let active: String = tx.query_row("SELECT candidate_id FROM active_harness_config WHERE slot = 'default'", [], |row| row.get(0)).context("no active baseline configuration")?; - if active != baseline_id { anyhow::bail!("baseline is not the active harness configuration"); } + let stored_candidate_id = Self::resolve_harness_candidate_id(&tx, candidate_id)?; + let stored_baseline_id = Self::resolve_harness_candidate_id(&tx, baseline_id)?; + let active: String = tx + .query_row( + "SELECT candidate_id FROM active_harness_config WHERE slot = 'default'", + [], + |row| row.get(0), + ) + .context("no active baseline configuration")?; + if active != stored_baseline_id { + anyhow::bail!("baseline is not the active harness configuration"); + } let now = chrono::Utc::now().to_rfc3339(); - tx.execute("INSERT INTO harness_evaluations (candidate_id, baseline_id, evaluator, samples_json, policy_json, comparison_json, evidence_ref, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", rusqlite::params![candidate_id, baseline_id, evaluator, serde_json::to_string(samples)?, serde_json::to_string(&policy)?, serde_json::to_string(&comparison)?, evidence_ref, now])?; - let evaluation_id = tx.last_insert_rowid(); if !comparison.passed { - tx.execute("INSERT INTO harness_eval_audit (event_type, candidate_id, prior_candidate_id, evaluation_id, evidence_ref, created_at) VALUES ('promotion_rejected', ?1, ?2, ?3, ?4, ?5)", rusqlite::params![candidate_id, baseline_id, evaluation_id, evidence_ref, now])?; + tx.execute("INSERT INTO harness_evaluations (candidate_id, baseline_id, evaluator, samples_json, policy_json, comparison_json, evidence_ref, legacy_unverifiable, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 0, ?8)", rusqlite::params![stored_candidate_id, stored_baseline_id, evaluator, serde_json::to_string(samples)?, serde_json::to_string(&policy)?, serde_json::to_string(&comparison)?, evidence_ref, now])?; + let evaluation_id = tx.last_insert_rowid(); + tx.execute("INSERT INTO harness_eval_audit (event_type, candidate_id, prior_candidate_id, evaluation_id, evidence_ref, legacy_unverifiable, created_at) VALUES ('promotion_rejected', ?1, ?2, ?3, ?4, 0, ?5)", rusqlite::params![stored_candidate_id, stored_baseline_id, evaluation_id, evidence_ref, now])?; tx.commit()?; - return Ok(HarnessPromotionOutcome { evaluation_id: Some(evaluation_id), promoted: false, rolled_back: false, failures: comparison.failures }); + return Ok(HarnessPromotionOutcome { + evaluation_id: Some(evaluation_id), + promoted: false, + rolled_back: false, + failures: comparison.failures, + }); } - let changed = tx.execute("UPDATE active_harness_config SET candidate_id = ?1, updated_at = ?2 WHERE slot = 'default' AND candidate_id = ?3", rusqlite::params![candidate_id, now, baseline_id])?; - if changed != 1 { anyhow::bail!("atomic promotion compare-and-swap failed"); } - let health_result = health_check(candidate_id); + let changed = tx.execute("UPDATE active_harness_config SET candidate_id = ?1, updated_at = ?2 WHERE slot = 'default' AND candidate_id = ?3", rusqlite::params![stored_candidate_id, now, stored_baseline_id])?; + if changed != 1 { + anyhow::bail!("atomic promotion compare-and-swap failed"); + } + let health_result = health_check(candidate_id).and_then(|healthy| { + if healthy != health_evidence.asserted_healthy { + anyhow::bail!("health check result does not match persisted assertion"); + } + Ok(healthy) + }); let healthy = matches!(health_result, Ok(true)); - let event_type = match &health_result { Ok(true) => "promoted", Ok(false) => "promotion_rolled_back", Err(_) => "health_check_error_rolled_back" }; + let event_type = match &health_result { + Ok(true) => "promoted", + Ok(false) => "promotion_rolled_back", + Err(_) => "health_check_error_rolled_back", + }; + let health_check_status = match &health_result { + Ok(true) => "healthy", + Ok(false) => "unhealthy", + Err(_) => "error", + }; if !healthy { - let restored = tx.execute("UPDATE active_harness_config SET candidate_id = ?1, updated_at = ?2 WHERE slot = 'default' AND candidate_id = ?3", rusqlite::params![baseline_id, now, candidate_id])?; - if restored != 1 { anyhow::bail!("atomic rollback compare-and-swap failed"); } + let restored = tx.execute("UPDATE active_harness_config SET candidate_id = ?1, updated_at = ?2 WHERE slot = 'default' AND candidate_id = ?3", rusqlite::params![stored_baseline_id, now, stored_candidate_id])?; + if restored != 1 { + anyhow::bail!("atomic rollback compare-and-swap failed"); + } } - tx.execute("INSERT INTO harness_eval_audit (event_type, candidate_id, prior_candidate_id, evaluation_id, evidence_ref, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6)", rusqlite::params![event_type, candidate_id, baseline_id, evaluation_id, evidence_ref, now])?; + let health_json = health_evidence.canonical_json()?; + let health_digest = health_evidence.digest()?; + tx.execute("INSERT INTO harness_evaluations (candidate_id, baseline_id, evaluator, samples_json, policy_json, comparison_json, evidence_ref, health_evidence_json, health_evidence_sha256, asserted_health, health_check_status, legacy_unverifiable, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, 0, ?12)", rusqlite::params![stored_candidate_id, stored_baseline_id, evaluator, serde_json::to_string(samples)?, serde_json::to_string(&policy)?, serde_json::to_string(&comparison)?, evidence_ref, health_json, health_digest, health_evidence.asserted_healthy, health_check_status, now])?; + let evaluation_id = tx.last_insert_rowid(); + tx.execute("INSERT INTO harness_eval_audit (event_type, candidate_id, prior_candidate_id, evaluation_id, evidence_ref, health_evidence_json, health_evidence_sha256, asserted_health, health_check_status, legacy_unverifiable, created_at) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 0, ?10)", rusqlite::params![event_type, stored_candidate_id, stored_baseline_id, evaluation_id, evidence_ref, health_json, health_digest, health_evidence.asserted_healthy, health_check_status, now])?; tx.commit()?; let failures = match health_result { Ok(true) => Vec::new(), Ok(false) => vec!["post-promotion health check returned false".to_string()], Err(error) => vec![format!("health check error: {error:#}")], }; - Ok(HarnessPromotionOutcome { evaluation_id: Some(evaluation_id), promoted: healthy, rolled_back: !healthy, failures }) + Ok(HarnessPromotionOutcome { + evaluation_id: Some(evaluation_id), + promoted: healthy, + rolled_back: !healthy, + failures, + }) } pub fn harness_audit_entries(&self) -> Result> { - let mut statement = self.conn.prepare("SELECT id, event_type, candidate_id, prior_candidate_id, evaluation_id, evidence_ref, created_at FROM harness_eval_audit ORDER BY id")?; - let entries = statement.query_map([], |row| Ok(HarnessAuditEntry { id: row.get(0)?, event_type: row.get(1)?, candidate_id: row.get(2)?, prior_candidate_id: row.get(3)?, evaluation_id: row.get(4)?, evidence_ref: row.get(5)?, created_at: row.get(6)? }))?.collect::>>()?; + let mut statement = self.conn.prepare("SELECT id, event_type, candidate_id, prior_candidate_id, evaluation_id, evidence_ref, health_evidence_json, health_evidence_sha256, asserted_health, health_check_status, legacy_unverifiable, created_at FROM harness_eval_audit ORDER BY id")?; + let entries = statement + .query_map([], |row| { + Ok(HarnessAuditEntry { + id: row.get(0)?, + event_type: row.get(1)?, + candidate_id: row.get(2)?, + prior_candidate_id: row.get(3)?, + evaluation_id: row.get(4)?, + evidence_ref: row.get(5)?, + health_evidence_json: row.get(6)?, + health_evidence_sha256: row.get(7)?, + asserted_health: row.get(8)?, + health_check_status: row.get(9)?, + legacy_unverifiable: row.get(10)?, + created_at: row.get(11)?, + }) + })? + .collect::>>()?; + for entry in &entries { + self.verify_harness_audit_entry(entry)?; + } Ok(entries) } + fn verify_harness_audit_entry(&self, entry: &HarnessAuditEntry) -> Result<()> { + let fields = ( + &entry.health_evidence_json, + &entry.health_evidence_sha256, + entry.asserted_health, + &entry.health_check_status, + ); + if matches!(fields, (None, None, None, None)) { + if entry.legacy_unverifiable + || matches!( + entry.event_type.as_str(), + "initial_activation" | "promotion_rejected" + ) + { + return Ok(()); + } + anyhow::bail!("missing harness health evidence integrity metadata"); + } + let (Some(json), Some(digest), Some(asserted), Some(status)) = fields else { + anyhow::bail!("incomplete harness health evidence integrity metadata"); + }; + if json.len() > 8192 { + anyhow::bail!("harness health evidence exceeds integrity verification bound"); + } + let snapshot: HealthEvidenceSnapshot = serde_json::from_str(json)?; + let snapshot_candidate_id = + Self::resolve_harness_candidate_id(&self.conn, &snapshot.candidate_id)?; + if snapshot.canonical_json()? != *json + || snapshot.digest()? != *digest + || snapshot.asserted_healthy != asserted + || snapshot_candidate_id != entry.candidate_id + { + anyhow::bail!("harness health evidence integrity verification failed"); + } + let event_consistent = match entry.event_type.as_str() { + "promoted" => status == "healthy" && asserted, + "promotion_rolled_back" => status == "unhealthy" && !asserted, + "health_check_error_rolled_back" => status == "error", + _ => false, + }; + if !event_consistent { + anyhow::bail!("harness health evidence is inconsistent with audit outcome"); + } + if let Some(evaluation_id) = entry.evaluation_id { + let evaluation: (Option, Option, Option, Option, bool) = self.conn.query_row( + "SELECT health_evidence_json, health_evidence_sha256, asserted_health, health_check_status, legacy_unverifiable FROM harness_evaluations WHERE id = ?1", + [evaluation_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?, row.get(3)?, row.get(4)?)), + )?; + if evaluation + != ( + Some(json.clone()), + Some(digest.clone()), + Some(asserted), + Some(status.clone()), + false, + ) + { + anyhow::bail!("audit health evidence does not match its evaluation"); + } + } + Ok(()) + } + #[cfg(test)] - fn connection_for_test(&self) -> &Connection { &self.conn } + fn connection_for_test(&self) -> &Connection { + &self.conn + } } #[cfg(test)] @@ -7262,29 +7694,415 @@ mod tests { use serde_json::json; let tempdir = TestDir::new("store-harness-eval")?; let db = StateStore::open(&tempdir.path().join("state.db"))?; - let baseline = CandidateSpec::new(json!({"prompt": "baseline"}), vec!["trace://b".into()], vec!["evidence://b".into()])?; - let candidate = CandidateSpec::new(json!({"prompt": "candidate"}), vec!["trace://c".into()], vec!["evidence://c".into()])?; + let baseline = CandidateSpec::new( + json!({"prompt": "baseline"}), + vec!["trace://b".into()], + vec!["evidence://b".into()], + )?; + let candidate = CandidateSpec::new( + json!({"prompt": "candidate"}), + vec!["trace://c".into()], + vec!["evidence://c".into()], + )?; db.record_harness_candidate(&baseline)?; db.record_harness_candidate(&candidate)?; db.activate_initial_harness(&baseline.id, "evidence://bootstrap")?; let samples = vec![ - PairedSample { seed: 1, candidate_score: 0.9, baseline_score: 0.5 }, - PairedSample { seed: 2, candidate_score: 0.8, baseline_score: 0.5 }, + PairedSample { + seed: 1, + candidate_score: 0.9, + baseline_score: 0.5, + }, + PairedSample { + seed: 2, + candidate_score: 0.8, + baseline_score: 0.5, + }, ]; - let policy = PromotionPolicy { min_samples: 2, min_mean_delta: 0.1, min_win_rate: 1.0 }; - let outcome = db.evaluate_promote_and_health_check(&candidate.id, &baseline.id, "recorded-v1", &samples, policy, "evidence://run", |_| Ok(false))?; + let policy = PromotionPolicy { + min_samples: 2, + min_mean_delta: 0.1, + min_win_rate: 1.0, + }; + let health = HealthEvidenceSnapshot::new("recorded-v1", &candidate.id, false)?; + let outcome = db.evaluate_promote_and_health_check( + &candidate.id, + &baseline.id, + "recorded-v1", + &samples, + policy, + "evidence://run", + &health, + |_| Ok(false), + )?; assert!(outcome.rolled_back); assert_eq!( outcome.failures, vec!["post-promotion health check returned false"] ); - assert_eq!(db.active_harness_id()?.as_deref(), Some(baseline.id.as_str())); + assert_eq!( + db.active_harness_id()?.as_deref(), + Some(baseline.id.as_str()) + ); let audit = db.harness_audit_entries()?; - assert_eq!(audit.iter().map(|entry| entry.event_type.as_str()).collect::>(), vec!["initial_activation", "promotion_rolled_back"]); - assert!(db.connection_for_test().execute("UPDATE harness_eval_audit SET event_type = 'tampered'", []).is_err()); - assert!(db.connection_for_test().execute("DELETE FROM harness_candidates", []).is_err()); + assert_eq!( + audit + .iter() + .map(|entry| entry.event_type.as_str()) + .collect::>(), + vec!["initial_activation", "promotion_rolled_back"] + ); + let rollback = audit.last().unwrap(); + assert_eq!(rollback.asserted_health, Some(false)); + assert_eq!(rollback.health_check_status.as_deref(), Some("unhealthy")); + assert!(rollback.health_evidence_json.is_some()); + assert_eq!( + rollback.health_evidence_sha256.as_deref().map(str::len), + Some(64) + ); + assert!(db + .connection_for_test() + .execute("UPDATE harness_eval_audit SET event_type = 'tampered'", []) + .is_err()); + assert!(db + .connection_for_test() + .execute("DELETE FROM harness_candidates", []) + .is_err()); + Ok(()) + } + + #[test] + fn record_harness_candidate_is_atomic_and_idempotent_across_connections() -> Result<()> { + use crate::harness_eval::CandidateSpec; + use serde_json::json; + use std::sync::{Arc, Barrier}; + + let tempdir = TestDir::new("store-harness-concurrent-record")?; + let db_path = tempdir.path().join("state.db"); + let first = StateStore::open(&db_path)?; + let second = StateStore::open(&db_path)?; + let candidate = CandidateSpec::new( + json!({"prompt": "candidate"}), + vec!["trace://one".into()], + vec!["evidence://one".into()], + )?; + let barrier = Arc::new(Barrier::new(2)); + let candidate_one = candidate.clone(); + let barrier_one = Arc::clone(&barrier); + let first_thread = std::thread::spawn(move || { + barrier_one.wait(); + first.record_harness_candidate(&candidate_one) + }); + let candidate_two = candidate.clone(); + let second_thread = std::thread::spawn(move || { + barrier.wait(); + second.record_harness_candidate(&candidate_two) + }); + + first_thread.join().unwrap()?; + second_thread.join().unwrap()?; + let reopened = StateStore::open(&db_path)?; + let count: i64 = reopened.connection_for_test().query_row( + "SELECT COUNT(*) FROM harness_candidates WHERE id = ?1", + [&candidate.id], + |row| row.get(0), + )?; + assert_eq!(count, 1); + reopened.record_harness_candidate(&candidate)?; + Ok(()) + } + + #[test] + fn record_harness_candidate_reports_deterministic_content_collision() -> Result<()> { + use crate::harness_eval::CandidateSpec; + use serde_json::json; + let tempdir = TestDir::new("store-harness-collision")?; + let db_path = tempdir.path().join("state.db"); + let db = StateStore::open(&db_path)?; + let candidate = CandidateSpec::new( + json!({"v": 1}), + vec!["trace://one".into()], + vec!["evidence://one".into()], + )?; + db.connection_for_test().execute( + "INSERT INTO harness_candidates (id, canonical_config_json, trace_refs_json, evidence_refs_json, created_at) VALUES (?1, '{}', '[\"trace://other\"]', '[\"evidence://other\"]', ?2)", + rusqlite::params![candidate.id, chrono::Utc::now().to_rfc3339()], + )?; + drop(db); + assert!(StateStore::open(&db_path) + .err() + .expect("mismatched v2 collision must be rejected") + .to_string() + .contains("candidate content address")); + Ok(()) + } + + #[test] + fn harness_health_evidence_integrity_detects_tampering() -> Result<()> { + use crate::harness_eval::{CandidateSpec, PairedSample, PromotionPolicy}; + use serde_json::json; + let tempdir = TestDir::new("store-harness-health-tamper")?; + let db = StateStore::open(&tempdir.path().join("state.db"))?; + let baseline = CandidateSpec::new( + json!({"v": 1}), + vec!["trace://b".into()], + vec!["evidence://b".into()], + )?; + let candidate = CandidateSpec::new( + json!({"v": 2}), + vec!["trace://c".into()], + vec!["evidence://c".into()], + )?; + db.record_harness_candidate(&baseline)?; + db.record_harness_candidate(&candidate)?; + db.activate_initial_harness(&baseline.id, "evidence://bootstrap")?; + let health = HealthEvidenceSnapshot::new("recorded-v1", &candidate.id, true)?; + db.evaluate_promote_and_health_check( + &candidate.id, + &baseline.id, + "recorded-v1", + &[PairedSample { + seed: 1, + candidate_score: 0.9, + baseline_score: 0.5, + }], + PromotionPolicy { + min_samples: 1, + min_mean_delta: 0.1, + min_win_rate: 1.0, + }, + "evidence://run", + &health, + |_| Ok(true), + )?; + assert!(db.harness_audit_entries().is_ok()); + + db.connection_for_test().execute_batch("DROP TRIGGER harness_eval_audit_no_update; UPDATE harness_eval_audit SET health_evidence_json = NULL, health_evidence_sha256 = NULL, asserted_health = NULL, health_check_status = NULL WHERE event_type = 'promoted';")?; + assert!(db + .harness_audit_entries() + .unwrap_err() + .to_string() + .contains("integrity")); + Ok(()) + } + + #[test] + fn open_adds_nullable_health_integrity_columns_to_legacy_schema() -> Result<()> { + let tempdir = TestDir::new("store-harness-legacy-migration")?; + let db_path = tempdir.path().join("state.db"); + let candidate = CandidateSpec::new( + serde_json::json!({}), + vec!["trace://legacy".into()], + vec!["evidence://legacy".into()], + )?; + let legacy_id = candidate.legacy_id(); + let legacy = Connection::open(&db_path)?; + legacy.execute_batch( + "CREATE TABLE harness_candidates (id TEXT PRIMARY KEY, canonical_config_json TEXT NOT NULL, trace_refs_json TEXT NOT NULL, evidence_refs_json TEXT NOT NULL, created_at TEXT NOT NULL); + CREATE TABLE harness_evaluations (id INTEGER PRIMARY KEY AUTOINCREMENT, candidate_id TEXT NOT NULL, baseline_id TEXT NOT NULL, evaluator TEXT NOT NULL, samples_json TEXT NOT NULL, policy_json TEXT NOT NULL, comparison_json TEXT NOT NULL, evidence_ref TEXT NOT NULL, created_at TEXT NOT NULL); + CREATE TABLE active_harness_config (slot TEXT PRIMARY KEY, candidate_id TEXT NOT NULL, updated_at TEXT NOT NULL); + CREATE TABLE harness_eval_audit (id INTEGER PRIMARY KEY AUTOINCREMENT, event_type TEXT NOT NULL, candidate_id TEXT NOT NULL, prior_candidate_id TEXT, evaluation_id INTEGER, evidence_ref TEXT NOT NULL, created_at TEXT NOT NULL);", + )?; + legacy.execute( + "INSERT INTO harness_candidates VALUES (?1, ?2, ?3, ?4, '2026-01-01T00:00:00Z')", + rusqlite::params![ + legacy_id, + candidate.canonical_config, + serde_json::to_string(&candidate.trace_refs)?, + serde_json::to_string(&candidate.evidence_refs)? + ], + )?; + legacy.execute( + "INSERT INTO active_harness_config VALUES ('default', ?1, '2026-01-01T00:00:00Z')", + [&legacy_id], + )?; + legacy.execute( + "INSERT INTO harness_eval_audit (event_type, candidate_id, evidence_ref, created_at) VALUES ('promoted', ?1, 'evidence://legacy', '2026-01-01T00:00:00Z')", + [&legacy_id], + )?; + drop(legacy); + + let db = StateStore::open(&db_path)?; + for table in ["harness_evaluations", "harness_eval_audit"] { + for column in [ + "health_evidence_json", + "health_evidence_sha256", + "asserted_health", + "health_check_status", + "legacy_unverifiable", + ] { + assert!(db.has_column(table, column)?); + } + } + let audit = db.harness_audit_entries()?; + assert_eq!(audit.len(), 1); + assert!(audit[0].legacy_unverifiable); + assert_eq!( + db.active_harness_id()?.as_deref(), + Some(candidate.id.as_str()) + ); + Ok(()) + } + + #[test] + fn open_aliases_exact_legacy_candidate_and_supports_v2_promotion_without_history_rewrite( + ) -> Result<()> { + use crate::harness_eval::{CandidateSpec, PairedSample, PromotionPolicy}; + use serde_json::json; + let tempdir = TestDir::new("store-harness-v1-alias-migration")?; + let db_path = tempdir.path().join("state.db"); + let baseline = CandidateSpec::new( + json!({"prompt": "legacy baseline"}), + vec!["trace://legacy".into()], + vec!["evidence://legacy".into()], + )?; + let legacy_id = baseline.legacy_id(); + let legacy = StateStore::open(&db_path)?; + legacy.connection_for_test().execute( + "INSERT INTO harness_candidates (id, canonical_config_json, trace_refs_json, evidence_refs_json, created_at) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![legacy_id, baseline.canonical_config, "[\" trace://legacy \",\"trace://legacy\"]", "[\" evidence://legacy \",\"evidence://legacy\"]", "2026-01-01T00:00:00Z"], + )?; + legacy.connection_for_test().execute( + "INSERT INTO active_harness_config (slot, candidate_id, updated_at) VALUES ('default', ?1, ?2)", + rusqlite::params![legacy_id, "2026-01-01T00:00:00Z"], + )?; + legacy.connection_for_test().execute( + "INSERT INTO harness_eval_audit (event_type, candidate_id, evidence_ref, legacy_unverifiable, created_at) VALUES ('initial_activation', ?1, 'evidence://legacy', 1, ?2)", + rusqlite::params![legacy_id, "2026-01-01T00:00:00Z"], + )?; + drop(legacy); + + let db = StateStore::open(&db_path)?; + db.record_harness_candidate(&baseline)?; + assert_eq!( + db.active_harness_id()?.as_deref(), + Some(baseline.id.as_str()) + ); + let candidate = CandidateSpec::new( + json!({"prompt": "v2 candidate"}), + vec!["trace://v2".into()], + vec!["evidence://v2".into()], + )?; + db.record_harness_candidate(&candidate)?; + let outcome = db.evaluate_promote_and_health_check( + &candidate.id, + &baseline.id, + "recorded-v1", + &[PairedSample { + seed: 1, + candidate_score: 1.0, + baseline_score: 0.5, + }], + PromotionPolicy { + min_samples: 1, + min_mean_delta: 0.1, + min_win_rate: 1.0, + }, + "evidence://v2-evaluation", + &HealthEvidenceSnapshot::new("recorded-v1", &candidate.id, true)?, + |_| Ok(true), + )?; + assert!(outcome.promoted); + assert_eq!( + db.active_harness_id()?.as_deref(), + Some(candidate.id.as_str()) + ); + let audit = db.harness_audit_entries()?; + assert_eq!(audit[0].candidate_id, legacy_id); + assert_eq!(audit[1].candidate_id, candidate.id); + assert_eq!( + audit[1].prior_candidate_id.as_deref(), + Some(legacy_id.as_str()) + ); + let legacy_backed_outcome = db.evaluate_promote_and_health_check( + &baseline.id, + &candidate.id, + "recorded-v1", + &[PairedSample { + seed: 2, + candidate_score: 1.0, + baseline_score: 0.5, + }], + PromotionPolicy { + min_samples: 1, + min_mean_delta: 0.1, + min_win_rate: 1.0, + }, + "evidence://legacy-backed-evaluation", + &HealthEvidenceSnapshot::new("recorded-v1", &baseline.id, true)?, + |_| Ok(true), + )?; + assert!(legacy_backed_outcome.promoted); + assert_eq!( + db.active_harness_id()?.as_deref(), + Some(baseline.id.as_str()) + ); + assert_eq!(db.harness_audit_entries()?.len(), 3); + drop(db); + + let reopened = StateStore::open(&db_path)?; + reopened.record_harness_candidate(&baseline)?; + assert_eq!( + reopened.active_harness_id()?.as_deref(), + Some(baseline.id.as_str()) + ); + assert_eq!(reopened.harness_audit_entries()?.len(), 3); + let alias_count: i64 = reopened.connection_for_test().query_row( + "SELECT COUNT(*) FROM harness_candidate_aliases WHERE alias_id = ?1 AND candidate_id = ?2", + rusqlite::params![baseline.id, legacy_id], + |row| row.get(0), + )?; + assert_eq!(alias_count, 1); + Ok(()) + } + + #[test] + fn open_rejects_tampered_legacy_candidate_and_alias_collisions() -> Result<()> { + use crate::harness_eval::CandidateSpec; + use serde_json::json; + let tempdir = TestDir::new("store-harness-v1-alias-tamper")?; + let db_path = tempdir.path().join("state.db"); + let candidate = CandidateSpec::new( + json!({"prompt": "legacy"}), + vec!["trace://legacy".into()], + vec!["evidence://legacy".into()], + )?; + let db = StateStore::open(&db_path)?; + db.connection_for_test().execute( + "INSERT INTO harness_candidates (id, canonical_config_json, trace_refs_json, evidence_refs_json, created_at) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![candidate.legacy_id(), "{\"prompt\":\"tampered\"}", serde_json::to_string(&candidate.trace_refs)?, serde_json::to_string(&candidate.evidence_refs)?, chrono::Utc::now().to_rfc3339()], + )?; + drop(db); + assert!(StateStore::open(&db_path) + .err() + .expect("tampered legacy candidate must be rejected") + .to_string() + .contains("candidate content address")); + + let collision_path = tempdir.path().join("collision.db"); + let db = StateStore::open(&collision_path)?; + db.connection_for_test().execute( + "INSERT INTO harness_candidates (id, canonical_config_json, trace_refs_json, evidence_refs_json, created_at) VALUES (?1, ?2, ?3, ?4, ?5)", + rusqlite::params![candidate.legacy_id(), candidate.canonical_config, serde_json::to_string(&candidate.trace_refs)?, serde_json::to_string(&candidate.evidence_refs)?, chrono::Utc::now().to_rfc3339()], + )?; + let other = CandidateSpec::new( + json!({"prompt": "other"}), + vec!["trace://other".into()], + vec!["evidence://other".into()], + )?; + db.record_harness_candidate(&other)?; + db.connection_for_test().execute( + "INSERT INTO harness_candidate_aliases (alias_id, candidate_id, id_version, created_at) VALUES (?1, ?2, 2, ?3)", + rusqlite::params![candidate.id, other.id, chrono::Utc::now().to_rfc3339()], + )?; + drop(db); + assert!(StateStore::open(&collision_path) + .err() + .expect("mismatched alias must be rejected") + .to_string() + .contains("alias collision")); Ok(()) } @@ -7294,8 +8112,16 @@ mod tests { use serde_json::json; let tempdir = TestDir::new("store-harness-health-error")?; let db = StateStore::open(&tempdir.path().join("state.db"))?; - let baseline = CandidateSpec::new(json!({"v": 1}), vec!["trace://b".into()], vec!["evidence://b".into()])?; - let candidate = CandidateSpec::new(json!({"v": 2}), vec!["trace://c".into()], vec!["evidence://c".into()])?; + let baseline = CandidateSpec::new( + json!({"v": 1}), + vec!["trace://b".into()], + vec!["evidence://b".into()], + )?; + let candidate = CandidateSpec::new( + json!({"v": 2}), + vec!["trace://c".into()], + vec!["evidence://c".into()], + )?; db.record_harness_candidate(&baseline)?; db.record_harness_candidate(&candidate)?; db.activate_initial_harness(&baseline.id, "evidence://bootstrap")?; @@ -7304,16 +8130,34 @@ mod tests { &candidate.id, &baseline.id, "recorded-v1", - &[PairedSample { seed: 1, candidate_score: 0.9, baseline_score: 0.5 }], - PromotionPolicy { min_samples: 1, min_mean_delta: 0.4, min_win_rate: 1.0 }, + &[PairedSample { + seed: 1, + candidate_score: 0.9, + baseline_score: 0.5, + }], + PromotionPolicy { + min_samples: 1, + min_mean_delta: 0.4, + min_win_rate: 1.0, + }, "evidence://run", + &HealthEvidenceSnapshot::new("recorded-v1", &candidate.id, true)?, |_| anyhow::bail!("probe unavailable"), )?; assert!(outcome.rolled_back); - assert_eq!(outcome.failures, vec!["health check error: probe unavailable"]); - assert_eq!(db.active_harness_id()?.as_deref(), Some(baseline.id.as_str())); - assert_eq!(db.harness_audit_entries()?.last().unwrap().event_type, "health_check_error_rolled_back"); + assert_eq!( + outcome.failures, + vec!["health check error: probe unavailable"] + ); + assert_eq!( + db.active_harness_id()?.as_deref(), + Some(baseline.id.as_str()) + ); + assert_eq!( + db.harness_audit_entries()?.last().unwrap().event_type, + "health_check_error_rolled_back" + ); Ok(()) } @@ -7323,15 +8167,47 @@ mod tests { use serde_json::json; let tempdir = TestDir::new("store-harness-gate")?; let db = StateStore::open(&tempdir.path().join("state.db"))?; - let baseline = CandidateSpec::new(json!({"v": 1}), vec!["trace://b".into()], vec!["evidence://b".into()])?; - let candidate = CandidateSpec::new(json!({"v": 2}), vec!["trace://c".into()], vec!["evidence://c".into()])?; + let baseline = CandidateSpec::new( + json!({"v": 1}), + vec!["trace://b".into()], + vec!["evidence://b".into()], + )?; + let candidate = CandidateSpec::new( + json!({"v": 2}), + vec!["trace://c".into()], + vec!["evidence://c".into()], + )?; db.record_harness_candidate(&baseline)?; db.record_harness_candidate(&candidate)?; db.activate_initial_harness(&baseline.id, "evidence://bootstrap")?; - let outcome = db.evaluate_promote_and_health_check(&candidate.id, &baseline.id, "recorded-v1", &[PairedSample { seed: 1, candidate_score: 0.6, baseline_score: 0.5 }], PromotionPolicy { min_samples: 2, min_mean_delta: 0.0, min_win_rate: 0.0 }, "evidence://run", |_| Ok(true))?; + let health = HealthEvidenceSnapshot::new("recorded-v1", &candidate.id, true)?; + let outcome = db.evaluate_promote_and_health_check( + &candidate.id, + &baseline.id, + "recorded-v1", + &[PairedSample { + seed: 1, + candidate_score: 0.6, + baseline_score: 0.5, + }], + PromotionPolicy { + min_samples: 2, + min_mean_delta: 0.0, + min_win_rate: 0.0, + }, + "evidence://run", + &health, + |_| Ok(true), + )?; assert!(!outcome.promoted); - assert_eq!(db.active_harness_id()?.as_deref(), Some(baseline.id.as_str())); - assert_eq!(db.harness_audit_entries()?.last().unwrap().event_type, "promotion_rejected"); + assert_eq!( + db.active_harness_id()?.as_deref(), + Some(baseline.id.as_str()) + ); + assert_eq!( + db.harness_audit_entries()?.last().unwrap().event_type, + "promotion_rejected" + ); Ok(()) } @@ -7342,17 +8218,49 @@ mod tests { let tempdir = TestDir::new("store-harness-success")?; let db_path = tempdir.path().join("state.db"); let db = StateStore::open(&db_path)?; - let baseline = CandidateSpec::new(json!({"v": 1}), vec!["trace://b".into()], vec!["evidence://b".into()])?; - let candidate = CandidateSpec::new(json!({"v": 2}), vec!["trace://c".into()], vec!["evidence://c".into()])?; + let baseline = CandidateSpec::new( + json!({"v": 1}), + vec!["trace://b".into()], + vec!["evidence://b".into()], + )?; + let candidate = CandidateSpec::new( + json!({"v": 2}), + vec!["trace://c".into()], + vec!["evidence://c".into()], + )?; db.record_harness_candidate(&baseline)?; db.record_harness_candidate(&candidate)?; db.activate_initial_harness(&baseline.id, "evidence://bootstrap")?; - let outcome = db.evaluate_promote_and_health_check(&candidate.id, &baseline.id, "recorded-v1", &[PairedSample { seed: 1, candidate_score: 0.9, baseline_score: 0.5 }], PromotionPolicy { min_samples: 1, min_mean_delta: 0.4, min_win_rate: 1.0 }, "evidence://run", |_| Ok(true))?; + let health = HealthEvidenceSnapshot::new("recorded-v1", &candidate.id, true)?; + let outcome = db.evaluate_promote_and_health_check( + &candidate.id, + &baseline.id, + "recorded-v1", + &[PairedSample { + seed: 1, + candidate_score: 0.9, + baseline_score: 0.5, + }], + PromotionPolicy { + min_samples: 1, + min_mean_delta: 0.4, + min_win_rate: 1.0, + }, + "evidence://run", + &health, + |_| Ok(true), + )?; assert!(outcome.promoted); drop(db); let reopened = StateStore::open(&db_path)?; - assert_eq!(reopened.active_harness_id()?.as_deref(), Some(candidate.id.as_str())); - assert_eq!(reopened.harness_audit_entries()?.last().unwrap().event_type, "promoted"); + assert_eq!( + reopened.active_harness_id()?.as_deref(), + Some(candidate.id.as_str()) + ); + assert_eq!( + reopened.harness_audit_entries()?.last().unwrap().event_type, + "promoted" + ); Ok(()) } } diff --git a/mcp-configs/mcp-servers.json b/mcp-configs/mcp-servers.json index 463e25864..49d91d6b9 100644 --- a/mcp-configs/mcp-servers.json +++ b/mcp-configs/mcp-servers.json @@ -8,7 +8,7 @@ "ito-compute": { "command": "node", "args": ["/absolute/path/to/ito-cloud-runtime/cli/ito-compute-cli/dist/bin/ito-mcp.js"], - "description": "Opt-in local Itô compute MCP. The canonical package is unpublished and must be built from Ito-Markets/ito-cloud-runtime/cli/ito-compute-cli. Exposes only ito_auth, ito_find, and ito_status. Authentication delegates to canonical CLI device authorization, which opens the Itô verification page by default and persists a device token in macOS Keychain; ecc ito auth --no-browser suppresses the CLI page handoff. ECC itself performs no browser automation. Legacy ITO_API_KEY requires explicit ITO_AUTH_MODE=legacy." + "description": "Opt-in local Itô compute MCP. The canonical package is unpublished and must be built from Ito-Markets/ito-cloud-runtime/cli/ito-compute-cli. Exposes only ito_auth, ito_find, and ito_status. ito_auth validates existing credentials; it does not start device login. Use ecc ito login [--no-browser] for device authorization, which stores tokens in macOS Keychain by default; explicit file fallback must retain owner-only settings. ECC itself performs no browser automation. ITO_API_KEY is forwarded directly to auth, find, and status when configured; ITO_AUTH_MODE=legacy is not required." }, "jira": { "command": "uvx", diff --git a/scripts/ecc.js b/scripts/ecc.js index c97b5289c..60890a2de 100755 --- a/scripts/ecc.js +++ b/scripts/ecc.js @@ -4,7 +4,7 @@ const { spawnSync } = require('child_process'); const path = require('path'); const { listAvailableLanguages } = require('./lib/install-executor'); const { getComputeSponsorCopy } = require('./lib/compute-sponsor'); -const { createSafeItoInvocationEnvironment } = require('./lib/ito-environment'); +const { createSafeItoInvocationEnvironment, getInvocationCommand } = require('./lib/ito-environment'); const COMMANDS = { install: { @@ -148,6 +148,7 @@ Examples: ecc catalog show framework:nextjs ecc consult "security reviews" ecc control-pane --port 8765 + ecc ito login [--no-browser] ecc ito auth ecc ito find --gpu h200 --count 8 --nodes 1 --gpus-per-node 8 --days 30 --storage-tb 1 --start-window 2099-08-15 --max-rate 3.00 --form-factor bare_metal --contract-type reservation --fabric infiniband --region us-east-1 ecc ito status --json @@ -241,6 +242,7 @@ function runCommand(commandName, args) { if (!command) { throw new Error(`Unknown command: ${commandName}`); } + const isItoLogin = commandName === 'ito' && getInvocationCommand(args) === 'login'; const result = spawnSync( process.execPath, [path.join(__dirname, command.script), ...args], @@ -253,7 +255,9 @@ function runCommand(commandName, args) { }), } : process.env, - stdio: commandName === 'memory' + stdio: isItoLogin + ? 'inherit' + : commandName === 'memory' ? ['inherit', 'pipe', 'pipe'] : ['pipe', 'pipe', 'pipe'], encoding: 'utf8', diff --git a/scripts/ito.js b/scripts/ito.js index cb448654e..592f9f2f0 100755 --- a/scripts/ito.js +++ b/scripts/ito.js @@ -10,7 +10,7 @@ const { getInvocationCommand, } = require("./lib/ito-environment"); -const SUPPORTED_COMMANDS = Object.freeze(["auth", "find", "status", "evals"]); +const SUPPORTED_COMMANDS = Object.freeze(["login", "auth", "find", "status", "evals"]); const CANONICAL_REPOSITORY = "https://github.com/Ito-Markets/ito-cloud-runtime.git"; const CANONICAL_PACKAGE_PATH = "cli/ito-compute-cli"; const CANONICAL_ENTRY_SEGMENTS = Object.freeze([ @@ -28,18 +28,20 @@ function showHelp() { ECC × Itô local CLI bridge Usage: - ecc ito auth [--no-browser] + ecc ito login [--no-browser] + ecc ito auth ecc ito find ecc ito status ecc ito evals --cluster --live-sixtytwo --nodes --config-dir - ecc ito --json + ecc ito --json The bridge invokes the separately installed canonical Itô CLI and returns its -real stdout, stderr, and exit code unchanged. "ecc ito auth" delegates to the +real stdout, stderr, and exit code unchanged. "ecc ito login" delegates to the canonical CLI's device authorization. It opens the Itô verification page by default and persists its device token in macOS Keychain. Pass --no-browser to suppress that handoff. ECC itself performs no browser automation and adds no lock, workload, inference, or purchase path. +"ecc ito auth" is validation-only and never starts device login. Important: - "find" reads live inventory and submits an authenticated RFQ. @@ -70,9 +72,11 @@ The same package's MCP server exposes only: Configure the MCP command as "node" with this absolute argument: /absolute/path/to/ito-cloud-runtime/${CANONICAL_PACKAGE_PATH}/dist/bin/ito-mcp.js -Device authorization is the default. Legacy ITO_API_KEY authentication is -forwarded only with explicit ITO_AUTH_MODE=legacy. Never put a key or token in -arguments, tracked files, or chat. +Device login never inherits ITO_API_KEY. The auth, find, and status commands +forward ITO_API_KEY directly when configured; ITO_AUTH_MODE=legacy is not +required. The canonical client stores device credentials in macOS Keychain by +default; file-token fallback remains explicit and must use restrictive settings. +Never put a key or token in arguments, tracked files, or chat. Live node qualification requires ITO_ENABLE_SIXTYTWO_LIVE=1, --live-sixtytwo, an explicit node list, and an existing absolute config @@ -157,9 +161,12 @@ function parseArgs(argv, environment = process.env) { const command = withoutJson.shift(); if (!SUPPORTED_COMMANDS.includes(command)) { throw new Error( - `Unsupported Itô command "${command || "(missing)"}"; ECC permits only auth, find, status, and evals.` + `Unsupported Itô command "${command || "(missing)"}"; ECC permits only login, auth, find, status, and evals.` ); } + if (command === "auth" && withoutJson.includes("--no-browser")) { + throw new Error("--no-browser is valid only for ecc ito login; auth is validation-only."); + } if (command === "evals") { validateNodeQualificationArgs(withoutJson, environment); } @@ -258,12 +265,14 @@ function invokeIto(executable, args, environment = process.env) { const invocation = buildInvocation(executable, args); const command = getInvocationCommand(args); const isNodeQualification = command === "evals"; + const isDeviceLogin = command === "login"; const result = spawnSync(invocation.executable, invocation.args, { cwd: process.cwd(), encoding: "utf8", // Keep policy helpers immutable for callers, but give child-process // instrumentation its own mutable copy (for example NODE_V8_COVERAGE). env: { ...createSafeItoInvocationEnvironment(environment, args) }, + stdio: isDeviceLogin ? "inherit" : ["pipe", "pipe", "pipe"], maxBuffer: MAX_OUTPUT_BYTES, timeout: isNodeQualification ? NODE_QUALIFICATION_TIMEOUT_MS : undefined, shell: false, diff --git a/scripts/lib/ito-environment.js b/scripts/lib/ito-environment.js index be42372ea..d23741c18 100644 --- a/scripts/lib/ito-environment.js +++ b/scripts/lib/ito-environment.js @@ -45,7 +45,7 @@ const ECC_ITO_CONTROL_KEYS = Object.freeze([ "ECC_ITO_CLI_EXECUTABLE", "NODE_ENV", ]); -const ITO_RUNTIME_COMMANDS = new Set(["auth", "find", "status"]); +const ITO_RUNTIME_COMMANDS = new Set(["login", "auth", "find", "status"]); function copyDefined(source, target, key) { if (typeof source[key] === "string") { @@ -64,7 +64,7 @@ function createSafeItoEnvironment(source = process.env, options = {}) { if (options.includeItoRuntime) { for (const key of ITO_RUNTIME_ENVIRONMENT_KEYS) { - if (key === "ITO_API_KEY" && source.ITO_AUTH_MODE !== "legacy") continue; + if (key === "ITO_API_KEY" && options.includeItoApiKey !== true) continue; copyDefined(source, safe, key); } } @@ -97,6 +97,7 @@ function createSafeItoInvocationEnvironment( return createSafeItoEnvironment(source, { includeControls: options.includeControls === true, includeItoRuntime: ITO_RUNTIME_COMMANDS.has(command), + includeItoApiKey: ["auth", "find", "status"].includes(command), includeItoEvals: command === "evals", }); } diff --git a/skills/ito-compute/SKILL.md b/skills/ito-compute/SKILL.md index 286a931fb..05c0c96d1 100644 --- a/skills/ito-compute/SKILL.md +++ b/skills/ito-compute/SKILL.md @@ -30,25 +30,30 @@ Set `ECC_ITO_CLI_EXECUTABLE` to the explicit absolute built entry: ``` ECC never discovers this credential-bearing client through `PATH`. -Device authorization is the default. Keep legacy `ITO_API_KEY` only with -explicit `ITO_AUTH_MODE=legacy`; never put it in arguments, tracked files, MCP -results, logs, or chat. +`ecc ito login` performs device authorization and never inherits `ITO_API_KEY`. +The validation-only `auth`, plus `find` and `status`, forward `ITO_API_KEY` +directly when configured; `ITO_AUTH_MODE=legacy` is not required. Never put a +key or token in arguments, tracked files, MCP results, logs, or chat. ## CLI workflow -1. Run `ecc ito auth` before the first operation. ECC delegates this to the +1. Run `ecc ito login` before the first operation. ECC delegates this to the canonical CLI's device authorization, which opens the Itô verification page by default and persists a device token in macOS Keychain. Use - `ecc ito auth --no-browser` to suppress the page handoff. ECC itself does no + `ecc ito login --no-browser` to suppress the page handoff. ECC itself does no browser automation. -2. Before `ecc ito find`, obtain explicit buyer authority to submit an RFQ. + Device tokens use macOS Keychain by default. File-token fallback is explicit + and its directory and token file must remain owner-only (0700 and 0600). +2. Run `ecc ito auth` to validate existing credentials; it never starts login + and rejects `--no-browser`. +3. Before `ecc ito find`, obtain explicit buyer authority to submit an RFQ. - Require `gpu`, `count`, whole `days`, `max-rate`, `nodes`, `gpus-per-node`, `storage-tb`, `start-window`, `form-factor`, `contract-type`, `fabric`, `region`, and the split-fill decision. - Require `count == nodes * gpus-per-node`; never derive topology. - Use `any` only when the buyer explicitly accepts any fabric or region. - Omitted `--allow-split` means false. -3. Run the live RFQ command: +4. Run the live RFQ command: ```sh ecc ito find \ @@ -66,7 +71,7 @@ results, logs, or chat. --region us-east-1 ``` -4. Run `ecc ito status` to inspect RFQs and procurement orders. +5. Run `ecc ito status` to inspect RFQs and procurement orders. After an ambiguous transport failure, check status before repeating `find`. Inventory prices are indicative. An RFQ is not reserved capacity. Treat a rate @@ -122,7 +127,8 @@ The server exposes only: - `ito_find` - `ito_status` -Use `ito_auth`, gather explicit buyer authority and every hard constraint, call +`ito_auth` validates existing credentials; it does not start device login. Use +`ito_auth`, gather explicit buyer authority and every hard constraint, call `ito_find`, then poll with `ito_status` when needed. ## Unsupported operations diff --git a/tests/ci/ito-compute-skill.test.js b/tests/ci/ito-compute-skill.test.js index 6535cb4d9..0f9997bba 100644 --- a/tests/ci/ito-compute-skill.test.js +++ b/tests/ci/ito-compute-skill.test.js @@ -35,6 +35,7 @@ function main() { ["documents only the real CLI commands and MCP tools", () => { const skill = read("skills/ito-compute/SKILL.md"); for (const command of [ + "ecc ito login", "ecc ito auth", "ecc ito find", "ecc ito status", @@ -57,10 +58,12 @@ function main() { assert.match(skill, /ECC_ITO_CLI_EXECUTABLE/); assert.match(skill, /explicit absolute built entry/); assert.match(skill, /never discovers[^\n]*through `PATH`/); - assert.match(skill, /opens the Itô verification page\s+by default/i); + assert.match(skill, /ecc ito login --no-browser/); + assert.match(skill, /auth.*validat/i); assert.match(skill, /--no-browser/); assert.match(skill, /macOS Keychain/i); - assert.match(skill, /ITO_AUTH_MODE=legacy/); + assert.match(skill, /(?:auth|find|status).*ITO_API_KEY/i); + assert.match(skill, /ITO_AUTH_MODE=legacy[^.]*not required/i); assert.match(skill, /ECC (?:itself )?(?:does|performs) no browser automation/i); assert.match(skill, /ITO_ENABLE_SIXTYTWO_LIVE/); assert.match(skill, /sixtytwo-cli==0\.3\.33/); @@ -68,6 +71,21 @@ function main() { assert.match(skill, /cannot (?:rent|launch|recover|repair)/i); assert.doesNotMatch(skill, /npm link/); }], + ["keeps README and integration docs aligned with the separated auth contract", () => { + for (const relativePath of [ + "README.md", + "docs/design/ecc-ito-compute-integration.md", + ]) { + const source = read(relativePath); + assert.match(source, /ecc ito login \[?--no-browser\]?/i, relativePath); + assert.match(source, /ecc ito auth/i, relativePath); + assert.match(source, /auth.*validat/i, relativePath); + assert.match(source, /login.*(?:Keychain|device authorization)/is, relativePath); + assert.doesNotMatch(source, /ecc ito auth --no-browser/i, relativePath); + assert.match(source, /ITO_API_KEY.*(?:auth|find|status)/is, relativePath); + assert.match(source, /ITO_AUTH_MODE=legacy[^.]*not required/i, relativePath); + } + }], ["registers one opt-in install module and capability", () => { const modules = readJson("manifests/install-modules.json").modules; const module = modules.find((candidate) => candidate.id === "ito-compute"); @@ -111,7 +129,7 @@ function main() { assert.doesNotMatch(JSON.stringify(server), /npx|ito_lock|ito_run|paper|simulat/i); assert.match(server.description, /ito_auth, ito_find, and ito_status/); assert.match(server.description, /unpublished/i); - assert.match(server.description, /device authorization/i); + assert.match(server.description, /ito_auth.*validat/i); assert.match(server.description, /macOS Keychain/i); assert.match(server.description, /no browser automation/i); }], diff --git a/tests/scripts/ito-cli-bridge.test.js b/tests/scripts/ito-cli-bridge.test.js index ff28b8eb3..f79634e7d 100644 --- a/tests/scripts/ito-cli-bridge.test.js +++ b/tests/scripts/ito-cli-bridge.test.js @@ -9,7 +9,7 @@ const assert = require("assert"); const fs = require("fs"); const os = require("os"); const path = require("path"); -const { spawnSync } = require("child_process"); +const { spawn, spawnSync } = require("child_process"); const REPO_ROOT = path.join(__dirname, "..", ".."); const ECC_SCRIPT = path.join(REPO_ROOT, "scripts", "ecc.js"); @@ -36,6 +36,34 @@ function runCli(args, environment = {}) { }); } +function runCliAndObserveFirstOutput(args, environment = {}) { + return new Promise((resolve, reject) => { + const child = spawn(process.execPath, [ECC_SCRIPT, ...args], { + cwd: REPO_ROOT, + env: { ...process.env, NODE_ENV: "test", ...environment }, + stdio: ["ignore", "pipe", "pipe"], + }); + let stdout = ""; + let stderr = ""; + let firstOutputAt; + const startedAt = Date.now(); + child.stdout.on("data", (chunk) => { + if (firstOutputAt === undefined) firstOutputAt = Date.now(); + stdout += chunk; + }); + child.stderr.on("data", (chunk) => { stderr += chunk; }); + child.once("error", reject); + child.once("close", (status) => resolve({ + status, + stdout, + stderr, + startedAt, + firstOutputAt, + closedAt: Date.now(), + })); + }); +} + function makeItoProbe(exitCode = 0) { const directory = fs.mkdtempSync(path.join(os.tmpdir(), "ecc-ito-cli-")); const log = path.join(directory, "invocation.json"); @@ -73,9 +101,9 @@ function readInvocation(probe) { return JSON.parse(fs.readFileSync(probe.log, "utf8")); } -function runTest(name, fn) { +async function runTest(name, fn) { try { - fn(); + await fn(); console.log(` ✓ ${name}`); return true; } catch (error) { @@ -85,12 +113,12 @@ function runTest(name, fn) { } } -function main() { +async function main() { console.log("\n=== Testing ECC × Itô real CLI bridge ===\n"); const tests = [ ["forwards only the reviewed RFQ CLI surface to an explicit local executable", () => { - for (const command of ["auth", "find", "status"]) { + for (const command of ["login", "auth", "find", "status"]) { const probe = makeItoProbe(); try { const result = runCli(["ito", command], { @@ -104,14 +132,27 @@ function main() { } } }], - ["forwards the canonical auth browser opt-out without performing browser automation", () => { + ["forwards the canonical login browser opt-out without performing browser automation", () => { + const probe = makeItoProbe(); + try { + const result = runCli(["ito", "login", "--no-browser"], { + ECC_ITO_CLI_EXECUTABLE: probe.executable, + }); + assert.strictEqual(result.status, 0, result.stderr); + assert.deepStrictEqual(readInvocation(probe).argv, ["login", "--no-browser"]); + } finally { + fs.rmSync(probe.directory, { recursive: true, force: true }); + } + }], + ["rejects --no-browser on validation-only auth before spawning", () => { const probe = makeItoProbe(); try { const result = runCli(["ito", "auth", "--no-browser"], { ECC_ITO_CLI_EXECUTABLE: probe.executable, }); - assert.strictEqual(result.status, 0, result.stderr); - assert.deepStrictEqual(readInvocation(probe).argv, ["auth", "--no-browser"]); + assert.notStrictEqual(result.status, 0); + assert.match(result.stderr, /--no-browser.*only.*login/i); + assert.ok(!fs.existsSync(probe.log)); } finally { fs.rmSync(probe.directory, { recursive: true, force: true }); } @@ -148,10 +189,10 @@ function main() { fs.rmSync(probe.directory, { recursive: true, force: true }); } }], - ["passes only the required device-auth runtime settings across the process boundary", () => { + ["login never inherits ITO_API_KEY but preserves secure token settings", () => { const probe = makeItoProbe(); try { - const result = runCli(["ito", "auth"], { + const result = runCli(["ito", "login"], { ECC_ITO_CLI_EXECUTABLE: probe.executable, ITO_API_KEY: "must-not-cross-without-legacy-mode", ITO_AUTH_MODE: "device", @@ -179,26 +220,46 @@ function main() { fs.rmSync(probe.directory, { recursive: true, force: true }); } }], - ["forwards the legacy API key only with explicit legacy auth mode", () => { - for (const [mode, expectedKey] of [ - [undefined, undefined], - ["device", undefined], - ["legacy", "ito_test_key"], - ]) { + ["forwards ITO_API_KEY directly to auth, find, and status without legacy mode", () => { + for (const command of ["auth", "find", "status"]) { const probe = makeItoProbe(); try { - const result = runCli(["ito", "status"], { + const result = runCli(["ito", command], { ECC_ITO_CLI_EXECUTABLE: probe.executable, ITO_API_KEY: "ito_test_key", - ...(mode ? { ITO_AUTH_MODE: mode } : {}), }); assert.strictEqual(result.status, 0, result.stderr); - assert.strictEqual(readInvocation(probe).env.ITO_API_KEY, expectedKey); + assert.strictEqual(readInvocation(probe).env.ITO_API_KEY, "ito_test_key"); } finally { fs.rmSync(probe.directory, { recursive: true, force: true }); } } }], + ["streams device login output before completion and propagates its exit status", async () => { + const probe = makeItoProbe(7); + try { + fs.writeFileSync( + probe.executable, + [ + '"use strict";', + 'process.stdout.write("device-code-now\\n");', + 'setTimeout(() => process.exit(7), 500);', + "", + ].join("\n") + ); + const result = await runCliAndObserveFirstOutput(["ito", "login"], { + ECC_ITO_CLI_EXECUTABLE: probe.executable, + }); + assert.strictEqual(result.status, 7, result.stderr); + assert.match(result.stdout, /device-code-now/); + assert.ok( + result.closedAt - result.firstOutputAt >= 350, + "login output was buffered until process completion", + ); + } finally { + fs.rmSync(probe.directory, { recursive: true, force: true }); + } + }], ["isolates live node qualification from Itô and unrelated credentials", () => { const probe = makeItoProbe(); try { @@ -398,7 +459,7 @@ function main() { ECC_ITO_CLI_EXECUTABLE: probe.executable, }); assert.notStrictEqual(result.status, 0, command); - assert.match(result.stderr, /only auth, find, status, and evals/i); + assert.match(result.stderr, /only login, auth, find, status, and evals/i); assert.ok(!fs.existsSync(probe.log), `${command} must not spawn the Itô CLI`); } finally { fs.rmSync(probe.directory, { recursive: true, force: true }); @@ -563,14 +624,15 @@ function main() { fs.rmSync(probe.directory, { recursive: true, force: true }); } }], - ["help exposes canonical device auth without claiming ECC browser automation", () => { + ["help separates device login from auth validation", () => { const probe = makeItoProbe(); try { const result = runCli(["ito", "--help"], { ECC_ITO_CLI_EXECUTABLE: probe.executable, }); assert.strictEqual(result.status, 0, result.stderr); - assert.match(result.stdout, /ecc ito auth \[--no-browser\]/); + assert.match(result.stdout, /ecc ito login \[--no-browser\]/); + assert.match(result.stdout, /ecc ito auth/); assert.match(result.stdout, /ecc ito find/); assert.match(result.stdout, /ecc ito status/); assert.match(result.stdout, /ecc ito evals/); @@ -585,7 +647,8 @@ function main() { assert.match(result.stdout, /opens the Itô verification page by default/i); assert.match(result.stdout, /macOS Keychain/i); assert.match(result.stdout, /ECC itself performs no browser automation/i); - assert.match(result.stdout, /ITO_AUTH_MODE=legacy/); + assert.match(result.stdout, /auth.*validat/i); + assert.match(result.stdout, /ITO_AUTH_MODE=legacy is not\s+required/i); assert.doesNotMatch( result.stdout, /manual copy|ito_lock|ito_run|npm link|paper|simulat/i @@ -600,7 +663,7 @@ function main() { let passed = 0; let failed = 0; for (const [name, fn] of tests) { - if (runTest(name, fn)) passed += 1; + if (await runTest(name, fn)) passed += 1; else failed += 1; }