From a973e331341d6b5a07a6e6575c2be6bd197932d2 Mon Sep 17 00:00:00 2001 From: montjeffrey Date: Thu, 8 Oct 2026 16:59:05 +0000 Subject: [PATCH] fix(context): recover retired routing indexes, validate entries and match advice to mode --- docs/design/context-profile-delivery.md | 6 +- docs/design/context-profile-delivery.tdd.md | 7 +++ node_modules | 1 + scripts/hooks/context-prompt-suggestions.js | 19 +++--- scripts/lib/context-routing-index.js | 69 +++++++++++++++------ scripts/lib/context-selection.js | 29 ++++++--- 6 files changed, 91 insertions(+), 40 deletions(-) create mode 120000 node_modules diff --git a/docs/design/context-profile-delivery.md b/docs/design/context-profile-delivery.md index 62acd9dfd..2960bff35 100644 --- a/docs/design/context-profile-delivery.md +++ b/docs/design/context-profile-delivery.md @@ -66,15 +66,15 @@ The task call sends the query and selected reference content on standard input t Ordinary interactive turns can receive advisory skill suggestions from a UserPromptSubmit hook. The hook prints up to three skill IDs with one-line descriptions and the `resolve` command to load one. Like implicit admission, a suggestion needs a matched term in the skill name or curated triggers, or an exact name; it never returns a skill body, changes the saved mode or selection, or grants authority. Loading still goes through `resolve`, which re-verifies canonical sources. -`ecc profile routing-index --state-root ` writes a metadata index for the current managed generation: the entries the resolver could suggest (not excluded, admissible without explicit selection), their curated triggers and precomputed retrieval vectors in a compact little-endian encoding. Building refuses a store whose generation no longer matches the canonical sources. A small pointer named by the generation binds the index to the state receipt; the entries file is named by the SHA-256 of its bytes. A `set`, `mode` or `rollback` retires the index until it is rebuilt, and the hook then stays silent with a stderr diagnostic. `--dry-run` reports whether the current index exists. +`ecc profile routing-index --state-root ` writes a metadata index for the current managed generation: the entries the resolver could suggest (not excluded, admissible without explicit selection), their curated triggers and precomputed retrieval vectors in a compact little-endian encoding. Building refuses a store whose generation no longer matches the canonical sources. A small pointer named by the generation binds the index to the state receipt; the entries file is named by the SHA-256 of its bytes. A `set`, `mode` or `rollback` retires the index until it is rebuilt, and the hook then stays silent with a stderr diagnostic; returning to an earlier generation reports its old index as missing, so the SessionStart builder rebuilds it. Reads validate every entry's fields and are bounded at 4 MiB and 2,048 entries. A build fails if skill sources change while it runs, and a non-UTF-8 skill source fails it instead of disappearing. `--dry-run` reports whether the current index exists. -The hook is inert unless `ECC_CONTEXT_STATE_ROOT` names the store, and it is not registered by default. It is also silent in manual mode, for slash commands, prompts under 12 characters and malformed input, and fails open when its 150 ms budget (`ECC_CONTEXT_SUGGEST_BUDGET_MS`) is spent. An optional async SessionStart hook builds a missing index. To opt in, add to Claude settings, with `` the ECC installation: +The hook is inert unless `ECC_CONTEXT_STATE_ROOT` names the store, and it is not registered by default. It is also silent in manual mode (checked before the index is read), for slash commands, prompts under 12 characters and malformed input, and suppresses output once its 150 ms budget (`ECC_CONTEXT_SUGGEST_BUDGET_MS`) is spent. The budget is checked between steps and cannot interrupt a stalled read, so the snippet sets Claude Code's own `timeout` as the hard bound. In suggest mode the hook names the `ecc profile mode auto` command instead of `resolve --load`, because suggest mode never returns bodies. An optional async SessionStart hook builds a missing index. To opt in, add to Claude settings, with `` the ECC installation: ```json { "env": { "ECC_CONTEXT_STATE_ROOT": "/absolute/dedicated/profile-store" }, "hooks": { - "UserPromptSubmit": [{ "hooks": [{ "type": "command", + "UserPromptSubmit": [{ "hooks": [{ "type": "command", "timeout": 2, "command": "node \"/scripts/hooks/run-with-flags.js\" user-prompt:context-suggestions scripts/hooks/context-prompt-suggestions.js" }] }], "SessionStart": [{ "hooks": [{ "type": "command", "async": true, "timeout": 30, "command": "node \"/scripts/hooks/run-with-flags.js\" session:context-routing-index scripts/hooks/context-routing-index.js" }] }] diff --git a/docs/design/context-profile-delivery.tdd.md b/docs/design/context-profile-delivery.tdd.md index 87a60dd6e..d475b0371 100644 --- a/docs/design/context-profile-delivery.tdd.md +++ b/docs/design/context-profile-delivery.tdd.md @@ -108,4 +108,11 @@ The suggest-only prompt hook from #2945 now reads a generation-bound metadata in - Selection, retrieval, profile CLI and hook-wrapper suites pass unchanged. - No interactive Claude or Codex session loaded the hook; suggestion quality is bounded by the resolver ranking measured above. +Review follow-up, October 8: + +- RED (15 index, hook and selection cases fail): a rollback to an indexed generation threw instead of reporting the index missing; digest-consistent entries with a non-string ID, name, description, owner or trigger, or a non-skill ID, were accepted; an oversized index was read in full; an anchored skill ranked fourth was hidden behind three unanchored matches; a skill edited mid-build was published under the stored generation; manual mode read the index before going silent; suggest mode advertised `resolve --load`, which returns no bodies there; admission denials were matched by message text, so a non-UTF-8 skill silently left the index. +- GREEN: a valid pointer for an earlier receipt is retired (`missing`) and the SessionStart builder rebuilds it; entries are validated against a closed key set before decoding and returned as new objects; pointer and index reads are bounded at 4 KiB, 4 MiB and 2,048 entries; suggestions filter for anchors before trimming to three; a build refuses plan or registry digests that differ from the checked carrier; the hook reads only the state binding before its manual-mode exit; suggest mode prints `ecc profile mode auto` with the current revision; denials carry `ECC_CONTEXT_*` codes and everything else propagates. +- Budget: a worker thread would make the 150 ms budget a hard bound but measured 135 to 150 ms for the same work (about 85 ms in process), so it would suppress most suggestions. The hook keeps in-process checks between steps, the read bounds cap the work at a real registry's size, and the documented snippet sets Claude Code's `timeout` as the outer bound. +- Full Claude index, auto mode, Linux x64 Node 22.22.0: wrapper invocations 108 to 122 ms against 29 to 45 ms with the hook disabled. + These boundaries keep the shipped behavior distinct from the M1 release gate. Authenticated outcome observations, a complete Tier 2 disk diff, live-install migration, other-provider activation, whole-context token truth and release defaults remain unverified until their explicit prerequisites are available. diff --git a/node_modules b/node_modules new file mode 120000 index 000000000..1af5d9a98 --- /dev/null +++ b/node_modules @@ -0,0 +1 @@ +/home/claude/affaan-m/ecc/node_modules \ No newline at end of file diff --git a/scripts/hooks/context-prompt-suggestions.js b/scripts/hooks/context-prompt-suggestions.js index ad2f74b57..8e3bf5697 100644 --- a/scripts/hooks/context-prompt-suggestions.js +++ b/scripts/hooks/context-prompt-suggestions.js @@ -29,11 +29,13 @@ function promptFrom(rawInput) { } } -function message(stateRoot, suggestions) { +function message(binding, suggestions) { + const root = JSON.stringify(binding.root); + const action = binding.selectionMode === 'suggest' + ? `Suggest mode never loads bodies. To load one, switch with ecc profile mode auto --state-root ${root} --expected-revision ${binding.revision}, then resolve it by ID.` + : `To load one, run ecc profile resolve --state-root ${root} --task-input - --load with its ID in explicitIds or proposedIds.`; return ['ECC context suggestions (advisory; nothing was loaded):', - ...suggestions.map(item => `- ${item.id}: ${item.description}`), - `To load one, run ecc profile resolve --state-root ${JSON.stringify(stateRoot)} --task-input - --load with its ID in explicitIds or proposedIds.`, - ''].join('\n'); + ...suggestions.map(item => `- ${item.id}: ${item.description}`), action, ''].join('\n'); } function run(rawInput, _context = {}, env = process.env) { @@ -46,15 +48,16 @@ function run(rawInput, _context = {}, env = process.env) { const budget = budgetMs(env); try { const routing = require('../lib/context-routing-index'); - if (Date.now() - started >= budget) return ''; - const index = routing.readRoutingIndex(stateRoot); + const binding = routing.readBinding(stateRoot); + if (binding.selectionMode === 'manual' || Date.now() - started >= budget) return ''; + const index = routing.readRoutingIndex(stateRoot, binding); if (!index) { return { stdout: '', stderr: '[ContextSuggest] No routing index for the current profile; run ecc profile routing-index --state-root .' }; } - if (index.selectionMode === 'manual' || Date.now() - started >= budget) return ''; + if (Date.now() - started >= budget) return ''; const suggestions = routing.suggestContext(index, prompt); if (!suggestions.length || Date.now() - started > budget) return ''; - return message(stateRoot, suggestions); + return message(binding, suggestions); } catch (error) { return { stdout: '', stderr: `[ContextSuggest] ${String(error.message).replace(/[^\x20-\x7E]/g, '?')}` }; } diff --git a/scripts/lib/context-routing-index.js b/scripts/lib/context-routing-index.js index 851d0d1b4..2d2b7103e 100644 --- a/scripts/lib/context-routing-index.js +++ b/scripts/lib/context-routing-index.js @@ -14,6 +14,13 @@ const POINTER_SCHEMA = 'ecc.context-routing-pointer.v1'; const DIGEST = /^[a-f0-9]{64}$/; const MAX_SUGGESTIONS = 3; const MAX_DESCRIPTION = 120; +// A Full index is under 1 MiB; these bounds keep a prompt-time read proportional +// to a real registry even when the store holds something else. +const MAX_POINTER_BYTES = 4096; +const MAX_INDEX_BYTES = 4 * 1024 * 1024; +const MAX_ENTRIES = 2048; +const SKILL_ID = /^skill:[a-z0-9]+(?:-[a-z0-9]+)*$/; +const ENTRY_KEYS = new Set(['id', 'name', 'description', 'ownerModuleId', 'packId', 'triggers', 'dense']); function checkedRoot(stateRoot) { if (typeof stateRoot !== 'string' || !path.isAbsolute(stateRoot) || path.resolve(stateRoot) !== stateRoot) { @@ -24,7 +31,7 @@ function checkedRoot(stateRoot) { // Reads only the ownership marker, state pointer and its receipt: enough to // bind an index to the current generation without rehashing the payload. -function currentBinding(stateRoot) { +function readBinding(stateRoot) { const root = checkedRoot(stateRoot); const marker = { schemaVersion: 'ecc.context-store.v1', destinationDigest: digestObject({ root }) }; const markerFile = path.join(root, 'store.json'); @@ -39,7 +46,8 @@ function currentBinding(stateRoot) { if (digestObject(receipt) !== state.receiptDigest || receipt.generationDigest !== state.generationDigest) { throw new Error('Managed receipt and state integrity mismatch'); } - return { root, generationDigest: state.generationDigest, receiptDigest: state.receiptDigest }; + return { root, generationDigest: state.generationDigest, receiptDigest: state.receiptDigest, + revision: receipt.revision, selectionMode: receipt.selection?.selectionMode || null }; } // Stored vectors are little-endian uint16 dimensions and float64 weights in @@ -77,7 +85,7 @@ function writeRoutingIndex({ stateRoot, repoRoot = DEFAULT_REPO_ROOT } = {}) { const root = checkedRoot(stateRoot); const status = require('./context-profile-store').getStoreStatus({ stateRoot: root }); if (!status.configured || status.recoveryRequired) throw new Error('Routing index requires a configured, recovered managed store'); - const binding = currentBinding(root); + const binding = readBinding(root); if (binding.receiptDigest !== status.receiptDigest) throw new Error('Managed state changed during routing index build'); const carrier = require('./context-carriers').planContextCarrier({ repoRoot, profileId: status.profileId, target: status.target, selectionMode: status.selectionMode, include: status.include, exclude: status.exclude }); @@ -85,6 +93,9 @@ function writeRoutingIndex({ stateRoot, repoRoot = DEFAULT_REPO_ROOT } = {}) { const { routingEntries } = require('./context-selection'); const routing = routingEntries({ repoRoot, profileId: status.profileId, target: status.target, selectionMode: status.selectionMode, include: status.include, exclude: status.exclude }); + if (routing.planDigest !== carrier.planDigest || routing.registryDigest !== carrier.registryDigest) { + throw new Error('Skill sources changed during routing index build; run it again'); + } const bytes = io.jsonBytes({ schemaVersion: SCHEMA, generationDigest: binding.generationDigest, receiptDigest: binding.receiptDigest, ...routing, entries: routing.entries.map(entry => ({ ...entry, dense: encodeDense(sparseDense(entry)) })) }); const digest = io.hash(bytes); @@ -92,37 +103,53 @@ function writeRoutingIndex({ stateRoot, repoRoot = DEFAULT_REPO_ROOT } = {}) { io.mkdir(path.join(root, 'routing')); io.mkdir(path.dirname(file)); if (!io.inspect(file, true).stat) io.writeExclusive(file, bytes); else if (!io.read(file).equals(bytes)) throw new Error('Routing index content changed under its digest'); - if (currentBinding(root).receiptDigest !== binding.receiptDigest) throw new Error('Managed state changed during routing index build'); + if (readBinding(root).receiptDigest !== binding.receiptDigest) throw new Error('Managed state changed during routing index build'); const pointer = { schemaVersion: POINTER_SCHEMA, generationDigest: binding.generationDigest, receiptDigest: binding.receiptDigest, indexDigest: digest, bytes: bytes.length }; io.atomicJson(pointerPath(root, binding.generationDigest), { ...pointer, pointerDigest: digestObject(pointer) }); return { status: 'written', path: file, generationDigest: binding.generationDigest, entries: routing.entries.length }; } -/** Return the index for the current generation, null when none was built. */ -function readRoutingIndex(stateRoot) { - const binding = currentBinding(stateRoot); +const optionalString = value => value === undefined || typeof value === 'string'; + +function validEntry(entry) { + return entry !== null && typeof entry === 'object' && !Array.isArray(entry) + && Object.keys(entry).every(key => ENTRY_KEYS.has(key)) + && typeof entry.id === 'string' && SKILL_ID.test(entry.id) && typeof entry.description === 'string' + && optionalString(entry.name) && optionalString(entry.ownerModuleId) && optionalString(entry.packId) + && (entry.triggers === undefined || (Array.isArray(entry.triggers) && entry.triggers.every(trigger => typeof trigger === 'string'))); +} + +/** Return the index for the current generation, or null when none was built + * for this state. A pointer left by an earlier state of the same generation + * (after a rollback or mode change) is retired, not corrupt. */ +function readRoutingIndex(stateRoot, binding = readBinding(stateRoot)) { + if (binding.root !== checkedRoot(stateRoot)) throw new Error('Routing binding belongs to a different store'); const file = pointerPath(binding.root, binding.generationDigest); if (!io.inspect(path.dirname(file), true).stat || !io.inspect(file, true).stat) return null; const invalid = () => new Error('Routing index integrity mismatch; rebuild it with ecc profile routing-index'); + if (io.inspect(file).stat.size > MAX_POINTER_BYTES) throw invalid(); const { pointerDigest, ...pointer } = io.readJson(file); if (pointer.schemaVersion !== POINTER_SCHEMA || digestObject(pointer) !== pointerDigest || !DIGEST.test(pointer.indexDigest || '') - || pointer.generationDigest !== binding.generationDigest || pointer.receiptDigest !== binding.receiptDigest) throw invalid(); - const bytes = io.read(entriesPath(binding.root, pointer.indexDigest)); + || !DIGEST.test(pointer.receiptDigest || '') || pointer.generationDigest !== binding.generationDigest + || !Number.isSafeInteger(pointer.bytes) || pointer.bytes <= 0 || pointer.bytes > MAX_INDEX_BYTES) throw invalid(); + if (pointer.receiptDigest !== binding.receiptDigest) return null; + const entriesFile = entriesPath(binding.root, pointer.indexDigest); + if (io.inspect(entriesFile).stat.size !== pointer.bytes) throw invalid(); + const bytes = io.read(entriesFile); if (bytes.length !== pointer.bytes || io.hash(bytes) !== pointer.indexDigest) throw invalid(); const index = JSON.parse(bytes.toString('utf8')); if (index.schemaVersion !== SCHEMA || index.generationDigest !== binding.generationDigest - || index.receiptDigest !== binding.receiptDigest || !Array.isArray(index.entries)) throw invalid(); - for (const entry of index.entries) { - entry.dense = decodeDense(entry.dense); - if (!entry.dense) throw invalid(); - } - return index; + || index.receiptDigest !== binding.receiptDigest || !Array.isArray(index.entries) || index.entries.length > MAX_ENTRIES + || !index.entries.every(validEntry)) throw invalid(); + const entries = index.entries.map(entry => ({ ...entry, dense: decodeDense(entry.dense) })); + if (entries.some(entry => !entry.dense)) throw invalid(); + return { ...index, entries }; } function routingIndexStatus(stateRoot) { - const binding = currentBinding(stateRoot); - const index = readRoutingIndex(stateRoot); + const binding = readBinding(stateRoot); + const index = readRoutingIndex(stateRoot, binding); return { status: index ? 'current' : 'missing', path: pointerPath(binding.root, binding.generationDigest), generationDigest: binding.generationDigest, entries: index ? index.entries.length : 0 }; } @@ -134,8 +161,10 @@ const singleLine = text => String(text).replace(CONTROL_CHARACTERS, ' ').replace /** Rank index entries for a prompt. IDs and one-line descriptions only; like * implicit admission, a suggestion needs a name or trigger anchor term. */ function suggestContext(index, prompt, { limit = MAX_SUGGESTIONS } = {}) { - return searchRetrieval(buildRetrievalIndex(index.entries), prompt, { limit: Math.min(limit, MAX_SUGGESTIONS) }) - .filter(candidate => /^skill:[a-z0-9]+(?:-[a-z0-9]+)*$/.test(candidate.id) && (candidate.exact || candidate.anchorTerms.length > 0)) + // Filter before trimming so unanchored matches cannot crowd out anchored ones. + return searchRetrieval(buildRetrievalIndex(index.entries), prompt, { limit: index.entries.length }) + .filter(candidate => SKILL_ID.test(candidate.id) && (candidate.exact || candidate.anchorTerms.length > 0)) + .slice(0, Math.min(limit, MAX_SUGGESTIONS)) .map(candidate => { const description = singleLine(candidate.description); return { id: candidate.id, @@ -143,4 +172,4 @@ function suggestContext(index, prompt, { limit = MAX_SUGGESTIONS } = {}) { }); } -module.exports = { readRoutingIndex, routingIndexStatus, suggestContext, writeRoutingIndex }; +module.exports = { readBinding, readRoutingIndex, routingIndexStatus, suggestContext, writeRoutingIndex }; diff --git a/scripts/lib/context-selection.js b/scripts/lib/context-selection.js index ca13133b5..e0e7f93cf 100644 --- a/scripts/lib/context-selection.js +++ b/scripts/lib/context-selection.js @@ -97,18 +97,26 @@ function policyFor(entry, reader) { dynamic: /!`/.test(source) }; } +// Admission denials carry stable codes so callers can tell a skill that may +// not be selected from a source that failed verification. +function denied(message, code) { + const error = new Error(message); + error.code = code; + return error; +} + function selectedClosure(ids, explicit, byId, excluded, reader) { const selected = new Set(); function visit(id) { if (!byId.has(id)) throw new Error(`Unknown context ID: ${id}`); - if (excluded.has(id)) throw new Error(`Context ID is excluded: ${id}`); + if (excluded.has(id)) throw denied(`Context ID is excluded: ${id}`, 'ECC_CONTEXT_EXCLUDED'); if (selected.has(id)) return; const entry = byId.get(id); const policy = policyFor(entry, reader); - if (policy.manualOnly && !explicit.has(id)) throw new Error(`Context ID is manual-only: ${id}`); - if (policy.authority || policy.dynamic) throw new Error(`Context requires native authority or dynamic-content review: ${id}`); + if (policy.manualOnly && !explicit.has(id)) throw denied(`Context ID is manual-only: ${id}`, 'ECC_CONTEXT_MANUAL_ONLY'); + if (policy.authority || policy.dynamic) throw denied(`Context requires native authority or dynamic-content review: ${id}`, 'ECC_CONTEXT_AUTHORITY'); selected.add(id); - if (selected.size > MAX_SELECTED) throw new Error('Task selection exceeds the skill limit'); + if (selected.size > MAX_SELECTED) throw denied('Task selection exceeds the skill limit', 'ECC_CONTEXT_SKILL_LIMIT'); entry.dependencies.forEach(visit); } ids.forEach(visit); @@ -122,7 +130,7 @@ function readSelected(ids, byId, reader) { return [...new Set([entry.sourcePath, ...entry.requiredResources])].map(sourcePath => { const actual = verifiedResource(entry, sourcePath, reader); total += actual.bytes; - if (total > MAX_CONTEXT_BYTES) throw new Error('Task context exceeds the 32000-byte budget; choose a narrower immediate step'); + if (total > MAX_CONTEXT_BYTES) throw denied('Task context exceeds the 32000-byte budget; choose a narrower immediate step', 'ECC_CONTEXT_BYTE_BUDGET'); const content = actual.content.toString('utf8'); if (!Buffer.from(content, 'utf8').equals(actual.content) || content.includes('\0')) throw new Error('Required context resource is not UTF-8 text'); return { id, path: sourcePath, digest: actual.digest, bytes: actual.bytes, content }; @@ -140,6 +148,9 @@ function validatePrevious(previous) { } } +const ADMISSION_DENIALS = new Set(['ECC_CONTEXT_EXCLUDED', 'ECC_CONTEXT_MANUAL_ONLY', 'ECC_CONTEXT_AUTHORITY', + 'ECC_CONTEXT_SKILL_LIMIT', 'ECC_CONTEXT_BYTE_BUDGET']); + function admissibility(byId, excluded, reader) { return id => { try { @@ -147,9 +158,9 @@ function admissibility(byId, excluded, reader) { readSelected(closure, byId, reader); return true; } catch (error) { - // Only known admission denials remove a suggestion. Source drift and - // malformed policy still fail closed instead of disappearing from view. - if (/manual-only|requires native authority|is excluded|exceeds the skill limit|32000-byte budget|not UTF-8 text/.test(error.message)) return false; + // Only coded admission denials remove a suggestion. Source drift, + // malformed policy and non-UTF-8 resources fail the build instead. + if (ADMISSION_DENIALS.has(error.code)) return false; throw error; } }; @@ -189,7 +200,7 @@ function resolveTaskContext({ repoRoot = DEFAULT_REPO_ROOT, task, profileId = 'l const proposedIds = [...(task.proposedIds || [])].sort(); [...explicitIds, ...proposedIds].forEach(id => { if (!byId.has(id)) throw new Error(`Unknown context ID: ${id}`); - if (excluded.has(id)) throw new Error(`Context ID is excluded: ${id}`); + if (excluded.has(id)) throw denied(`Context ID is excluded: ${id}`, 'ECC_CONTEXT_EXCLUDED'); }); const taskBinding = { sessionId: task.sessionId, taskId: task.taskId, revision: task.revision, phase: task.phase }; const bindingDigest = digestObject({ ...taskBinding, planDigest: plan.planDigest,