diff --git a/README.md b/README.md index 03d336224..442e220f8 100644 --- a/README.md +++ b/README.md @@ -1434,6 +1434,13 @@ export ECC_MAX_INJECTED_INSTINCTS=6 # Minimum confidence an instinct needs to be injected, 0-1 (default: 0.7) export ECC_INSTINCT_CONFIDENCE_THRESHOLD=0.7 +# SessionStart ranks injected instincts by confidence + project/stack relevance +# (default: on). Project-scoped instincts, and instincts whose domain/trigger +# matches the detected stack (languages, frameworks, plus terraform/dbt markers), +# get a small ranking boost so they surface above unrelated higher-confidence +# ones. Set to off/false/0/no to rank by confidence alone. +export ECC_INSTINCT_RELEVANCE_RANKING=on + # Keep context/scope/loop warnings but suppress API-rate cost estimates export ECC_CONTEXT_MONITOR_COST_WARNINGS=off ``` diff --git a/scripts/hooks/session-start.js b/scripts/hooks/session-start.js index 4cfc443ec..63854aff1 100644 --- a/scripts/hooks/session-start.js +++ b/scripts/hooks/session-start.js @@ -24,6 +24,11 @@ const { resolveProjectContext, writeSessionLease, resolveSessionId, getHomunculu const { getPackageManager, getSelectionPrompt } = require('../lib/package-manager'); const { listAliases } = require('../lib/session-aliases'); const { detectProjectType } = require('../lib/project-detect'); +const { + isRelevanceRankingEnabled, + detectStackKeywords, + computeRelevanceBoost, +} = require('../lib/instinct-relevance'); const path = require('path'); const fs = require('fs'); @@ -422,6 +427,20 @@ function summarizeActiveInstincts(observerContext) { const confidenceThreshold = getInstinctConfidenceThreshold(); const maxInjected = getMaxInjectedInstincts(); + // Relevance ranking (issue #2371 part b): at SessionStart there is no user + // task yet, so relevance is location/stack based. Project-scoped and + // stack-matching instincts get a small additive boost over their confidence. + // Gated by ECC_INSTINCT_RELEVANCE_RANKING (default on); when off, or when no + // stack is detected and nothing is project-scoped, every boost is 0 and the + // ranking collapses to confidence-only (unchanged behaviour). + // Detect the stack from the real project source tree (projectRoot), not the + // homunculus state dir (projectDir). In a global session projectRoot is empty, + // so detectStackKeywords falls back to process.cwd(). + const relevanceEnabled = isRelevanceRankingEnabled(); + const stackKeywords = relevanceEnabled + ? detectStackKeywords(observerContext.projectRoot || undefined) + : new Set(); + const deduped = new Map(); for (const instinct of scopedInstincts) { if (!instinct.id || instinct.confidence < confidenceThreshold) continue; @@ -435,10 +454,17 @@ function summarizeActiveInstincts(observerContext) { .map(instinct => ({ ...instinct, action: extractInstinctAction(instinct.content), + _relevance: relevanceEnabled ? computeRelevanceBoost(instinct, stackKeywords) : 0, })) .filter(instinct => instinct.action) .sort((left, right) => { - if (right.confidence !== left.confidence) return right.confidence - left.confidence; + // Primary: combined confidence + relevance. When relevance is off every + // _relevance is 0, so this reduces to the prior confidence-only ordering. + // Tie-breaks on a genuinely equal combined score: project scope first, + // then id (deterministic). + const leftScore = left.confidence + left._relevance; + const rightScore = right.confidence + right._relevance; + if (rightScore !== leftScore) return rightScore - leftScore; if (left._scopeLabel !== right._scopeLabel) return left._scopeLabel === 'project' ? -1 : 1; return String(left.id).localeCompare(String(right.id)); }) diff --git a/scripts/lib/instinct-relevance.js b/scripts/lib/instinct-relevance.js new file mode 100644 index 000000000..81dde7e79 --- /dev/null +++ b/scripts/lib/instinct-relevance.js @@ -0,0 +1,173 @@ +/** + * Instinct relevance ranking for SessionStart. + * + * At SessionStart there is no user task yet, so "relevance" is location/stack + * relevance: instincts scoped to the current project, or whose domain/trigger + * matches the detected stack, get a small additive boost on top of their + * confidence when ranking which instincts to inject. The confidence >= + * threshold floor and the injection cap are enforced by the caller; this + * module only computes the additive boost and the stack keyword set. When + * nothing is project-scoped and no stack is detected, every boost is 0 and the + * ranking degrades to confidence-only (unchanged behaviour). + * + * Resolves part (b) of: + * https://github.com/affaan-m/everything-claude-code/issues/2371 + */ + +const fs = require('fs'); +const path = require('path'); +const { detectProjectType } = require('./project-detect'); + +// Additive ranking boosts. These are intentionally NOT env-configurable: part +// (b) of the issue asks for relevance ranking, not more tunable knobs (part (a) +// already made the injection count + confidence threshold configurable). The +// values are chosen so a project-scoped 0.7 instinct (0.7 + 0.25 = 0.95) can +// surface above an unrelated global 0.9, and a stack-matching 0.75 instinct +// (0.75 + 0.2 = 0.95) can surface above an unrelated 0.9. +const DEFAULT_PROJECT_SCOPE_BOOST = 0.25; +const DEFAULT_STACK_MATCH_BOOST = 0.2; + +/** + * Whether a file with any of the given extensions exists directly in the root + * (non-recursive, top-level only — kept cheap for a blocking SessionStart hook). + * @param {string} root - Project root directory. + * @param {string[]} extensions - Extensions to look for (e.g. ['.tf']). + * @returns {boolean} + */ +function hasFileWithExtension(root, extensions) { + try { + return fs.readdirSync(root, { withFileTypes: true }).some( + (entry) => entry.isFile() && extensions.includes(path.extname(entry.name)) + ); + } catch { + return false; + } +} + +/** + * Whether a named file exists directly in the root. + * @param {string} root - Project root directory. + * @param {string} name - File name relative to root. + * @returns {boolean} + */ +function fileExists(root, name) { + try { + return fs.existsSync(path.join(root, name)); + } catch { + return false; + } +} + +/** + * Resolve whether relevance ranking is enabled. Default on; opt out by setting + * `ECC_INSTINCT_RELEVANCE_RANKING` to `off`, `false`, `0`, or `no` + * (case-insensitive). Any other value (including unset) keeps ranking on. + * @returns {boolean} + */ +function isRelevanceRankingEnabled() { + const raw = process.env.ECC_INSTINCT_RELEVANCE_RANKING; + if (raw === undefined || raw === null || raw === '') return true; + const normalized = String(raw).trim().toLowerCase(); + return !['off', 'false', '0', 'no'].includes(normalized); +} + +/** + * Cheap, non-recursive stack-keyword detection for the project root. Reuses + * detectProjectType (languages + frameworks) and layers the extra IaC/data + * markers issue #2371 calls out that detectProjectType does not cover + * (`*.tf` / `*.tfvars` -> terraform, `dbt_project.yml` -> dbt). + * @param {string} [projectRoot] - Defaults to process.cwd(). + * @param {{languages?: string[], frameworks?: string[]}} [projectInfo] - + * Optional precomputed detectProjectType() result, to avoid a second pass. + * @returns {Set} Lowercase keyword set (may be empty). + */ +function detectStackKeywords(projectRoot, projectInfo) { + const root = projectRoot || process.cwd(); + const keywords = new Set(); + + let info = projectInfo; + if (!info) { + try { + info = detectProjectType(root); + } catch { + info = { languages: [], frameworks: [] }; + } + } + for (const language of info.languages || []) keywords.add(String(language).toLowerCase()); + for (const framework of info.frameworks || []) keywords.add(String(framework).toLowerCase()); + + if (hasFileWithExtension(root, ['.tf', '.tfvars'])) keywords.add('terraform'); + if (fileExists(root, 'dbt_project.yml')) keywords.add('dbt'); + + return keywords; +} + +/** + * Tokenize a free-text field into lowercase word tokens (split on + * non-alphanumerics). Token-set matching avoids substring false positives such + * as the keyword `go` matching the word `good`. + * @param {string} value + * @returns {string[]} + */ +function tokenize(value) { + return String(value || '') + .toLowerCase() + .split(/[^a-z0-9]+/) + .filter(Boolean); +} + +/** + * Whether an instinct's domain/trigger/stack fields intersect the stack + * keywords by whole-token match. + * @param {object} instinct - Parsed instinct (frontmatter fields as properties). + * @param {Set} stackKeywords + * @returns {boolean} + */ +function instinctMatchesStack(instinct, stackKeywords) { + if (!instinct || !stackKeywords || stackKeywords.size === 0) return false; + const tokens = new Set([ + ...tokenize(instinct.domain), + ...tokenize(instinct.trigger), + ...tokenize(instinct.stack), + ]); + for (const keyword of stackKeywords) { + if (tokens.has(keyword)) return true; + } + return false; +} + +/** + * Additive relevance boost for ranking. Deterministic and pure. A + * project-scoped instinct (location-relevant by construction) and a + * stack-matching instinct each contribute their boost; both can apply. + * @param {object} instinct - Must carry `_scopeLabel` ('project'|'global') and + * optional `domain`/`trigger`/`stack` fields. + * @param {Set} stackKeywords + * @param {{projectBoost?: number, stackBoost?: number}} [opts] + * @returns {number} + */ +function computeRelevanceBoost(instinct, stackKeywords, opts) { + const options = opts || {}; + const projectBoost = Number.isFinite(options.projectBoost) + ? options.projectBoost + : DEFAULT_PROJECT_SCOPE_BOOST; + const stackBoost = Number.isFinite(options.stackBoost) + ? options.stackBoost + : DEFAULT_STACK_MATCH_BOOST; + + let boost = 0; + if (instinct && instinct._scopeLabel === 'project') boost += projectBoost; + if (instinctMatchesStack(instinct, stackKeywords)) boost += stackBoost; + return boost; +} + +module.exports = { + DEFAULT_PROJECT_SCOPE_BOOST, + DEFAULT_STACK_MATCH_BOOST, + isRelevanceRankingEnabled, + detectStackKeywords, + instinctMatchesStack, + computeRelevanceBoost, + // Exported for testing. + tokenize, +}; diff --git a/tests/hooks/hooks.test.js b/tests/hooks/hooks.test.js index 07d48dcd5..49d6f1e23 100644 --- a/tests/hooks/hooks.test.js +++ b/tests/hooks/hooks.test.js @@ -600,6 +600,64 @@ async function runTests() { passed++; else failed++; + if ( + await asyncTest('ranks stack-relevant instincts above higher-confidence unrelated ones (#2371)', async () => { + const isoHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-instinct-relevance-')); + const homunculusDir = path.join(isoHome, 'homunculus'); + const instinctsDir = path.join(homunculusDir, 'instincts', 'personal'); + fs.mkdirSync(instinctsDir, { recursive: true }); + // A stack-matching 0.75 instinct and an unrelated higher-confidence 0.9. + fs.writeFileSync( + path.join(instinctsDir, 'terraform-first.md'), + '---\nid: terraform-first\nconfidence: 0.75\ndomain: terraform\n---\n## Action\nRun terraform plan before every apply.\n' + ); + fs.writeFileSync( + path.join(instinctsDir, 'unrelated-high.md'), + '---\nid: unrelated-high\nconfidence: 0.9\ndomain: python\n---\n## Action\nPin Python dependencies in requirements.txt.\n' + ); + // A project root that detects as terraform via a *.tf marker. + const projectRoot = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-tf-project-')); + fs.writeFileSync(path.join(projectRoot, 'main.tf'), 'resource "null_resource" "x" {}\n'); + + const baseEnv = { + HOME: isoHome, + USERPROFILE: isoHome, + CLV2_HOMUNCULUS_DIR: homunculusDir, + CLAUDE_PROJECT_DIR: projectRoot, + ECC_INSTINCT_RELEVANCE_RANKING: 'on', + ECC_INSTINCT_CONFIDENCE_THRESHOLD: '0.7', + ECC_MAX_INJECTED_INSTINCTS: '6', + }; + + try { + const on = await runScript(path.join(scriptsDir, 'session-start.js'), '', baseEnv); + assert.strictEqual(on.code, 0); + const ctxOn = getSessionStartAdditionalContext(on.stdout); + const tfOn = ctxOn.indexOf('Run terraform plan before every apply.'); + const pyOn = ctxOn.indexOf('Pin Python dependencies in requirements.txt.'); + assert.ok(tfOn !== -1 && pyOn !== -1, `both instincts should inject, ctx: ${ctxOn}`); + assert.ok(tfOn < pyOn, `stack-matching 0.75 should rank above unrelated 0.9 when relevance is on, ctx: ${ctxOn}`); + + // Opting out restores pure confidence ordering (0.9 before 0.75). + const off = await runScript(path.join(scriptsDir, 'session-start.js'), '', { + ...baseEnv, + ECC_INSTINCT_RELEVANCE_RANKING: 'off', + }); + assert.strictEqual(off.code, 0); + const ctxOff = getSessionStartAdditionalContext(off.stdout); + const tfOff = ctxOff.indexOf('Run terraform plan before every apply.'); + const pyOff = ctxOff.indexOf('Pin Python dependencies in requirements.txt.'); + assert.ok(tfOff !== -1 && pyOff !== -1, `both instincts should still inject, ctx: ${ctxOff}`); + assert.ok(pyOff < tfOff, `with ranking off, higher-confidence 0.9 should rank first, ctx: ${ctxOff}`); + } finally { + fs.rmSync(isoHome, { recursive: true, force: true }); + fs.rmSync(projectRoot, { recursive: true, force: true }); + } + }) + ) + passed++; + else failed++; + if ( await asyncTest('disables session-start additional context when requested', async () => { const isoHome = fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-disabled-start-')); diff --git a/tests/lib/instinct-relevance.test.js b/tests/lib/instinct-relevance.test.js new file mode 100644 index 000000000..a3920c4cd --- /dev/null +++ b/tests/lib/instinct-relevance.test.js @@ -0,0 +1,231 @@ +/** + * Tests for scripts/lib/instinct-relevance.js + * + * Run with: node tests/lib/instinct-relevance.test.js + */ + +const assert = require('assert'); +const path = require('path'); +const fs = require('fs'); +const os = require('os'); + +const { + DEFAULT_PROJECT_SCOPE_BOOST, + DEFAULT_STACK_MATCH_BOOST, + isRelevanceRankingEnabled, + detectStackKeywords, + instinctMatchesStack, + computeRelevanceBoost, + tokenize, +} = require('../../scripts/lib/instinct-relevance'); + +function test(name, fn) { + try { + fn(); + console.log(` ✓ ${name}`); + return true; + } catch (err) { + console.log(` ✗ ${name}`); + console.log(` ${err.message}`); + return false; + } +} + +function createTempDir() { + return fs.mkdtempSync(path.join(os.tmpdir(), 'ecc-instinct-relevance-')); +} + +function cleanupDir(dir) { + try { + fs.rmSync(dir, { recursive: true, force: true }); + } catch { + /* ignore */ + } +} + +function writeFile(dir, name, content) { + fs.writeFileSync(path.join(dir, name), content); +} + +function runTests() { + let passed = 0; + let failed = 0; + + console.log('\nInstinct relevance ranking tests\n'); + + // --- tokenize --------------------------------------------------------- + if (test('tokenize splits on non-alphanumerics and lowercases', () => { + assert.deepStrictEqual(tokenize('Terraform-AWS_infra'), ['terraform', 'aws', 'infra']); + assert.deepStrictEqual(tokenize('when editing hooks'), ['when', 'editing', 'hooks']); + assert.deepStrictEqual(tokenize(''), []); + assert.deepStrictEqual(tokenize(undefined), []); + })) passed++; else failed++; + + // --- detectStackKeywords --------------------------------------------- + if (test('detectStackKeywords returns empty set for an empty directory', () => { + const dir = createTempDir(); + try { + const kw = detectStackKeywords(dir); + assert.ok(kw instanceof Set, 'should return a Set'); + assert.strictEqual(kw.size, 0); + } finally { + cleanupDir(dir); + } + })) passed++; else failed++; + + if (test('detectStackKeywords picks up a Rust project (Cargo.toml)', () => { + const dir = createTempDir(); + try { + writeFile(dir, 'Cargo.toml', '[package]\nname = "x"\n'); + const kw = detectStackKeywords(dir); + assert.ok(kw.has('rust'), `expected rust in ${[...kw].join(',')}`); + } finally { + cleanupDir(dir); + } + })) passed++; else failed++; + + if (test('detectStackKeywords picks up a Go project (go.mod)', () => { + const dir = createTempDir(); + try { + writeFile(dir, 'go.mod', 'module example.com/x\n\ngo 1.21\n'); + const kw = detectStackKeywords(dir); + assert.ok(kw.has('golang'), `expected golang in ${[...kw].join(',')}`); + } finally { + cleanupDir(dir); + } + })) passed++; else failed++; + + if (test('detectStackKeywords adds terraform for *.tf / *.tfvars files', () => { + const dir = createTempDir(); + try { + writeFile(dir, 'main.tf', 'resource "null_resource" "x" {}\n'); + const kw = detectStackKeywords(dir); + assert.ok(kw.has('terraform'), `expected terraform in ${[...kw].join(',')}`); + } finally { + cleanupDir(dir); + } + })) passed++; else failed++; + + if (test('detectStackKeywords adds dbt for dbt_project.yml', () => { + const dir = createTempDir(); + try { + writeFile(dir, 'dbt_project.yml', "name: 'demo'\n"); + const kw = detectStackKeywords(dir); + assert.ok(kw.has('dbt'), `expected dbt in ${[...kw].join(',')}`); + } finally { + cleanupDir(dir); + } + })) passed++; else failed++; + + if (test('detectStackKeywords accepts a precomputed projectInfo', () => { + const kw = detectStackKeywords('/nonexistent', { + languages: ['python'], + frameworks: ['django'], + }); + assert.ok(kw.has('python') && kw.has('django')); + })) passed++; else failed++; + + // --- instinctMatchesStack -------------------------------------------- + if (test('instinctMatchesStack matches on domain token', () => { + const kw = new Set(['terraform']); + assert.strictEqual(instinctMatchesStack({ domain: 'terraform' }, kw), true); + assert.strictEqual(instinctMatchesStack({ domain: 'terraform-aws' }, kw), true); + })) passed++; else failed++; + + if (test('instinctMatchesStack matches on trigger token', () => { + const kw = new Set(['python']); + assert.strictEqual( + instinctMatchesStack({ trigger: 'when writing python tests' }, kw), + true + ); + })) passed++; else failed++; + + if (test('instinctMatchesStack avoids substring false positives (go != good)', () => { + const kw = new Set(['go']); + assert.strictEqual(instinctMatchesStack({ domain: 'good practices' }, kw), false); + })) passed++; else failed++; + + if (test('instinctMatchesStack is false with empty keyword set or fields', () => { + assert.strictEqual(instinctMatchesStack({ domain: 'terraform' }, new Set()), false); + assert.strictEqual(instinctMatchesStack({}, new Set(['terraform'])), false); + assert.strictEqual(instinctMatchesStack(null, new Set(['terraform'])), false); + })) passed++; else failed++; + + // --- computeRelevanceBoost ------------------------------------------- + if (test('computeRelevanceBoost gives project boost only for project scope', () => { + const kw = new Set(); + assert.strictEqual( + computeRelevanceBoost({ _scopeLabel: 'project' }, kw), + DEFAULT_PROJECT_SCOPE_BOOST + ); + assert.strictEqual(computeRelevanceBoost({ _scopeLabel: 'global' }, kw), 0); + })) passed++; else failed++; + + if (test('computeRelevanceBoost gives stack boost only on a stack match', () => { + const kw = new Set(['rust']); + assert.strictEqual( + computeRelevanceBoost({ _scopeLabel: 'global', domain: 'rust' }, kw), + DEFAULT_STACK_MATCH_BOOST + ); + assert.strictEqual( + computeRelevanceBoost({ _scopeLabel: 'global', domain: 'python' }, kw), + 0 + ); + })) passed++; else failed++; + + if (test('computeRelevanceBoost stacks project + stack boosts', () => { + const kw = new Set(['rust']); + const boost = computeRelevanceBoost({ _scopeLabel: 'project', domain: 'rust' }, kw); + assert.strictEqual(boost, DEFAULT_PROJECT_SCOPE_BOOST + DEFAULT_STACK_MATCH_BOOST); + })) passed++; else failed++; + + if (test('computeRelevanceBoost honours custom boost overrides', () => { + const kw = new Set(['rust']); + const boost = computeRelevanceBoost( + { _scopeLabel: 'project', domain: 'rust' }, + kw, + { projectBoost: 1, stackBoost: 2 } + ); + assert.strictEqual(boost, 3); + })) passed++; else failed++; + + if (test('a project 0.7 instinct outranks an unrelated global 0.9 with boosts', () => { + // Confirms the boost magnitudes satisfy the issue's motivating example. + const kw = new Set(); + const projectScore = 0.7 + computeRelevanceBoost({ _scopeLabel: 'project' }, kw); + const globalScore = 0.9 + computeRelevanceBoost({ _scopeLabel: 'global' }, kw); + assert.ok(projectScore > globalScore, `${projectScore} !> ${globalScore}`); + })) passed++; else failed++; + + if (test('a stack-matching 0.75 instinct outranks an unrelated 0.9 with boosts', () => { + const kw = new Set(['terraform']); + const matchScore = 0.75 + computeRelevanceBoost({ _scopeLabel: 'global', domain: 'terraform' }, kw); + const otherScore = 0.9 + computeRelevanceBoost({ _scopeLabel: 'global', domain: 'python' }, kw); + assert.ok(matchScore > otherScore, `${matchScore} !> ${otherScore}`); + })) passed++; else failed++; + + // --- isRelevanceRankingEnabled --------------------------------------- + if (test('isRelevanceRankingEnabled defaults on and honours the opt-out toggle', () => { + const original = process.env.ECC_INSTINCT_RELEVANCE_RANKING; + try { + delete process.env.ECC_INSTINCT_RELEVANCE_RANKING; + assert.strictEqual(isRelevanceRankingEnabled(), true, 'unset should be on'); + for (const off of ['off', 'OFF', 'false', '0', 'no']) { + process.env.ECC_INSTINCT_RELEVANCE_RANKING = off; + assert.strictEqual(isRelevanceRankingEnabled(), false, `${off} should be off`); + } + for (const on of ['on', '1', 'true', 'yes', 'anything']) { + process.env.ECC_INSTINCT_RELEVANCE_RANKING = on; + assert.strictEqual(isRelevanceRankingEnabled(), true, `${on} should be on`); + } + } finally { + if (original === undefined) delete process.env.ECC_INSTINCT_RELEVANCE_RANKING; + else process.env.ECC_INSTINCT_RELEVANCE_RANKING = original; + } + })) passed++; else failed++; + + console.log(`\n=== Results: ${passed} passed, ${failed} failed ===\n`); + process.exit(failed > 0 ? 1 : 0); +} + +runTests();