From 161713edce61e326adcc6ffe442507f8ed6e0e94 Mon Sep 17 00:00:00 2001 From: montjeffrey Date: Sun, 11 Oct 2026 00:45:00 -0400 Subject: [PATCH 1/2] test(gateguard): pin the original four code questions and an opt-in external-contract question 8a7d6129 added the external-contract question for every code change that handles data and updated the pinned tests to match, including the legacy test "code condensed Edit hint for an unknown profile is unchanged". Its own evaluation then found the gated arm did not improve (25 to 23 of 48), so the question has no evidence behind it as a default. - Restore the test expectations 8a7d6129 changed: questionIdsFor, the byte-identical code text for src/a.py, the condensed hints, and the hook question lists default to data-schema without external-contract. - Add questionIdsFor/condensedHintFor cases with { externalContract: true }. - Add hook cases: GATEGUARD_EXTERNAL_CONTRACT_QUESTION=1/true/ON asks it beside data-schema and in the unknown-profile set, never without data; unset, 0, false and off keep the original four. Against main: target-class 5 failures, fact-force 10 failures. --- tests/hooks/gateguard-fact-force.test.js | 49 +++++++++++++++++----- tests/lib/gateguard-target-class.test.js | 52 +++++++++++++++++------- 2 files changed, 77 insertions(+), 24 deletions(-) diff --git a/tests/hooks/gateguard-fact-force.test.js b/tests/hooks/gateguard-fact-force.test.js index 1968cd5df..7a17ac150 100644 --- a/tests/hooks/gateguard-fact-force.test.js +++ b/tests/hooks/gateguard-fact-force.test.js @@ -4387,8 +4387,7 @@ function runTests() { '1. Name the file(s) and line(s) that will call this new file', '2. Confirm no existing file serves the same purpose (search the tree — Glob/Grep, or find/grep via Bash)', '3. If this file reads/writes data files, show field names, structure, and date format (use redacted or synthetic values, not raw production data)', - '4. Name what outside this repository decides the format, units, timezone or protocol semantics here (the consumer, the producer, or a stated convention), or state that the choice is unconstrained', - "5. Quote the user's current instruction verbatim", + "4. Quote the user's current instruction verbatim", '', 'If this call was sent in a parallel batch, other edits to src/a.py from that batch may already have been applied. Re-read the file before building on them.', '', @@ -4405,8 +4404,7 @@ function runTests() { '1. List ALL files that import/require this file (search the tree — Glob/Grep, or find/grep via Bash)', '2. List the public functions/classes affected by this change', '3. If this file reads/writes data files, show field names, structure, and date format (use redacted or synthetic values, not raw production data)', - '4. Name what outside this repository decides the format, units, timezone or protocol semantics here (the consumer, the producer, or a stated convention), or state that the choice is unconstrained', - "5. Quote the user's current instruction verbatim", + "4. Quote the user's current instruction verbatim", '', 'If this call was sent in a parallel batch, other edits to src/b.py from that batch may already have been applied. Re-read the file before building on them.', '', @@ -4468,12 +4466,10 @@ function runTests() { callers: 'Name the file(s) and line(s) that will call this new file', noDuplicate: 'Confirm no existing file serves the same purpose (search the tree — Glob/Grep, or find/grep via Bash)', dataSchema: 'If this file reads/writes data files, show field names, structure, and date format (use redacted or synthetic values, not raw production data)', - externalContract: - 'Name what outside this repository decides the format, units, timezone or protocol semantics here (the consumer, the producer, or a stated convention), or state that the choice is unconstrained', quote: QUOTE_LINE }; - const B2_FULL_EDIT = [QUESTION_LINES.importers, QUESTION_LINES.publicApi, QUESTION_LINES.dataSchema, QUESTION_LINES.externalContract, QUESTION_LINES.quote]; - const B2_FULL_WRITE = [QUESTION_LINES.callers, QUESTION_LINES.noDuplicate, QUESTION_LINES.dataSchema, QUESTION_LINES.externalContract, QUESTION_LINES.quote]; + const B2_FULL_EDIT = [QUESTION_LINES.importers, QUESTION_LINES.publicApi, QUESTION_LINES.dataSchema, QUESTION_LINES.quote]; + const B2_FULL_WRITE = [QUESTION_LINES.callers, QUESTION_LINES.noDuplicate, QUESTION_LINES.dataSchema, QUESTION_LINES.quote]; clearState(); if ( @@ -4501,12 +4497,45 @@ function runTests() { if ( test('an Edit that handles data keeps the data-schema question', () => { const reason = profileDenialReason('Edit', { file_path: 'src/calc.py', old_string: ' x = 1', new_string: ' rows = json.load(fh)' }); - assert.deepStrictEqual(listedQuestions(reason), [QUESTION_LINES.localCallers, QUESTION_LINES.dataSchema, QUESTION_LINES.externalContract, QUESTION_LINES.quote]); + assert.deepStrictEqual(listedQuestions(reason), [QUESTION_LINES.localCallers, QUESTION_LINES.dataSchema, QUESTION_LINES.quote]); }) ) passed++; else failed++; + const EXTERNAL_CONTRACT_LINE = + 'Name what outside this repository decides the format, units, timezone or protocol semantics here (the consumer, the producer, or a stated convention), or state that the choice is unconstrained'; + for (const value of ['1', 'true', 'ON']) { + clearState(); + if ( + test(`GATEGUARD_EXTERNAL_CONTRACT_QUESTION=${value} adds the external-contract question beside data-schema`, () => { + const env = { GATEGUARD_EXTERNAL_CONTRACT_QUESTION: value }; + const data = profileDenialReason('Edit', { file_path: 'src/calc.py', old_string: ' x = 1', new_string: ' rows = json.load(fh)' }, env); + assert.deepStrictEqual(listedQuestions(data), [QUESTION_LINES.localCallers, QUESTION_LINES.dataSchema, EXTERNAL_CONTRACT_LINE, QUESTION_LINES.quote]); + clearState(); + const unknown = profileDenialReason('Edit', { file_path: 'src/a.rb', old_string: 'a', new_string: 'b' }, env); + assert.deepStrictEqual(listedQuestions(unknown), [...B2_FULL_EDIT.slice(0, 3), EXTERNAL_CONTRACT_LINE, QUESTION_LINES.quote]); + clearState(); + const noData = profileDenialReason('Edit', { file_path: 'src/calc.js', old_string: ' return a + 1;', new_string: ' return a * 2;' }, env); + assert.deepStrictEqual(listedQuestions(noData), [QUESTION_LINES.localCallers, QUESTION_LINES.quote], 'no data, no external-contract'); + }) + ) + passed++; + else failed++; + } + + for (const value of ['', '0', 'false', 'off']) { + clearState(); + if ( + test(`GATEGUARD_EXTERNAL_CONTRACT_QUESTION=${JSON.stringify(value)} keeps the original four code questions`, () => { + const reason = profileDenialReason('Edit', { file_path: 'src/a.rb', old_string: 'a', new_string: 'b' }, { GATEGUARD_EXTERNAL_CONTRACT_QUESTION: value }); + assert.deepStrictEqual(listedQuestions(reason), B2_FULL_EDIT); + }) + ) + passed++; + else failed++; + } + clearState(); if ( test('a Write of a new code file without data handling drops the data-schema question', () => { @@ -4608,7 +4637,7 @@ function runTests() { assert.deepStrictEqual(listedQuestions(exported), [QUESTION_LINES.importers, QUESTION_LINES.publicApi, QUESTION_LINES.quote]); clearState(); const fetch = profileDenialReason('Edit', { file_path: 'scripts/q4.sh', old_string: ' run', new_string: ' curl -s "$URL" > out.json' }); - assert.deepStrictEqual(listedQuestions(fetch), [QUESTION_LINES.localCallers, QUESTION_LINES.dataSchema, QUESTION_LINES.externalContract, QUESTION_LINES.quote]); + assert.deepStrictEqual(listedQuestions(fetch), [QUESTION_LINES.localCallers, QUESTION_LINES.dataSchema, QUESTION_LINES.quote]); clearState(); const ps = profileDenialReason('Edit', { file_path: 'scripts/q5.ps1', old_string: 'function Get-A {', new_string: 'function Get-B {' }); assert.deepStrictEqual(listedQuestions(ps), [QUESTION_LINES.importers, QUESTION_LINES.publicApi, QUESTION_LINES.quote]); diff --git a/tests/lib/gateguard-target-class.test.js b/tests/lib/gateguard-target-class.test.js index 06a3cf7c0..3d77a52fc 100644 --- a/tests/lib/gateguard-target-class.test.js +++ b/tests/lib/gateguard-target-class.test.js @@ -347,21 +347,21 @@ test('every non-code class has questions and a condensed hint; code has neither' test('questionIdsFor gives stable ids per class, action and change profile', () => { const known = (touchesPublicSurface, touchesData) => ({ known: true, language: 'js', touchesPublicSurface, touchesData, trivial: false }); const table = [ - ['code', false, null, ['importers', 'public-api', 'data-schema', 'external-contract', 'quote-instruction']], - ['code', true, null, ['callers', 'no-duplicate', 'data-schema', 'external-contract', 'quote-instruction']], - ['code', false, { known: false, touchesPublicSurface: false, touchesData: false }, ['importers', 'public-api', 'data-schema', 'external-contract', 'quote-instruction']], - ['code', false, known(true, true), ['importers', 'public-api', 'data-schema', 'external-contract', 'quote-instruction']], + ['code', false, null, ['importers', 'public-api', 'data-schema', 'quote-instruction']], + ['code', true, null, ['callers', 'no-duplicate', 'data-schema', 'quote-instruction']], + ['code', false, { known: false, touchesPublicSurface: false, touchesData: false }, ['importers', 'public-api', 'data-schema', 'quote-instruction']], + ['code', false, known(true, true), ['importers', 'public-api', 'data-schema', 'quote-instruction']], ['code', false, known(true, false), ['importers', 'public-api', 'quote-instruction']], - ['code', false, known(false, true), ['local-callers', 'data-schema', 'external-contract', 'quote-instruction']], + ['code', false, known(false, true), ['local-callers', 'data-schema', 'quote-instruction']], ['code', false, known(false, false), ['local-callers', 'quote-instruction']], ['code', true, known(false, false), ['callers', 'no-duplicate', 'quote-instruction']], - ['code', true, known(true, true), ['callers', 'no-duplicate', 'data-schema', 'external-contract', 'quote-instruction']], + ['code', true, known(true, true), ['callers', 'no-duplicate', 'data-schema', 'quote-instruction']], ['instruction', false, known(false, false), ['loader', 'behaviour-change', 'no-duplicate-instruction', 'quote-instruction']], ['test', true, known(false, false), ['under-test', 'existing-tests', 'quote-instruction']], ['prose', true, null, ['supersedes', 'linked-from', 'why-new-file', 'quote-instruction']], ['prose', false, null, ['references', 'corrects-or-adds', 'quote-instruction']], ['config', false, known(false, false), ['config-reader', 'config-effect', 'no-plaintext-secrets', 'quote-instruction']], - ['__proto__', false, null, ['importers', 'public-api', 'data-schema', 'external-contract', 'quote-instruction']] + ['__proto__', false, null, ['importers', 'public-api', 'data-schema', 'quote-instruction']] ]; for (const [cls, isWrite, profile, expected] of table) { const ids = questionIdsFor(cls, isWrite, profile); @@ -386,7 +386,7 @@ test('condensedHintFor matches the question set', () => { test('code condensed hints name exactly the questions of the change profile', () => { const known = (touchesPublicSurface, touchesData) => ({ known: true, language: 'js', touchesPublicSurface, touchesData, trivial: false }); - const ids = ['importers', 'public-api', 'local-callers', 'callers', 'no-duplicate', 'data-schema', 'external-contract', 'quote-instruction']; + const ids = ['importers', 'public-api', 'local-callers', 'callers', 'no-duplicate', 'data-schema', 'quote-instruction']; for (const id of ids) assert.ok(condensedQuestionPhrase(id), `${id} has a phrase`); for (const isWrite of [false, true]) { for (const [s, d] of [[true, true], [true, false], [false, true], [false, false]]) { @@ -427,7 +427,7 @@ test('code condensed hints read as one sentence', () => { ); assert.strictEqual( condensedHintFor('code', false, known(true, true)), - "briefly state the files that import this file, the public functions/classes affected, the data schemas it reads or writes, what outside the repository fixes the format or semantics, and the user's verbatim instruction, then retry." + "briefly state the files that import this file, the public functions/classes affected, the data schemas it reads or writes, and the user's verbatim instruction, then retry." ); assert.strictEqual( condensedHintFor('code', true, known(true, false)), @@ -445,11 +445,11 @@ test('code condensed hints keep the data and duplicate checks when the profile a } }); -test('code condensed Edit hint for an unknown profile names every question asked', () => { - const expected = "briefly state importers/callers, affected API, data schemas if any, what outside the repository fixes the format or semantics, and the user's verbatim instruction, then retry."; +test('code condensed Edit hint for an unknown profile is unchanged', () => { + const legacy = "briefly state importers/callers, affected API, data schemas if any, and the user's verbatim instruction, then retry."; for (const profile of [null, undefined, { known: false, touchesPublicSurface: false, touchesData: false }, 'x']) { - assert.strictEqual(condensedHintFor('code', false, profile), expected); - assert.strictEqual(condensedHintFor('__proto__', false, profile), expected); + assert.strictEqual(condensedHintFor('code', false, profile), legacy); + assert.strictEqual(condensedHintFor('__proto__', false, profile), legacy); } }); @@ -457,13 +457,37 @@ test('code condensed Write hint for an unknown profile asks every creation quest for (const profile of [null, undefined, { known: false, touchesPublicSurface: false, touchesData: false }, 'x']) { assert.strictEqual( condensedHintFor('code', true, profile), - "briefly state the file(s) and line(s) that will call it, that no existing file serves the same purpose, the data schemas it reads or writes, what outside the repository fixes the format or semantics, and the user's verbatim instruction, then retry." + "briefly state the file(s) and line(s) that will call it, that no existing file serves the same purpose, the data schemas it reads or writes, and the user's verbatim instruction, then retry." ); } assert.strictEqual(condensedQuestionPhrase('constructor'), ''); assert.strictEqual(condensedQuestionPhrase('loader'), ''); }); +test('external-contract joins data-schema only when opted in', () => { + const known = (touchesPublicSurface, touchesData) => ({ known: true, language: 'js', touchesPublicSurface, touchesData, trivial: false }); + const on = { externalContract: true }; + const table = [ + ['code', false, null, ['importers', 'public-api', 'data-schema', 'external-contract', 'quote-instruction']], + ['code', true, null, ['callers', 'no-duplicate', 'data-schema', 'external-contract', 'quote-instruction']], + ['code', false, known(false, true), ['local-callers', 'data-schema', 'external-contract', 'quote-instruction']], + ['code', false, known(true, false), ['importers', 'public-api', 'quote-instruction']], + ['config', false, known(false, true), ['config-reader', 'config-effect', 'no-plaintext-secrets', 'quote-instruction']] + ]; + for (const [cls, isWrite, profile, expected] of table) { + assert.deepStrictEqual(questionIdsFor(cls, isWrite, profile, on), expected, `${cls} ${isWrite} ${JSON.stringify(profile)}`); + } + for (const off of [undefined, {}, { externalContract: false }, { externalContract: 'yes' }]) { + assert.ok(!questionIdsFor('code', false, null, off).includes('external-contract'), JSON.stringify(off)); + } + assert.strictEqual( + condensedHintFor('code', false, null, on), + "briefly state importers/callers, affected API, data schemas if any, what outside the repository fixes the format or semantics, and the user's verbatim instruction, then retry." + ); + assert.ok(condensedHintFor('code', true, known(true, true), on).includes(condensedQuestionPhrase('external-contract'))); + assert.ok(!condensedHintFor('code', true, known(true, true)).includes(condensedQuestionPhrase('external-contract'))); +}); + // ── isSensitiveTarget ── test('isSensitiveTarget matches every listed basename, segment and prefix rule', () => { From f7f47679b68a593c8c56f6b3354999379b143974 Mon Sep 17 00:00:00 2001 From: montjeffrey Date: Sun, 11 Oct 2026 00:45:01 -0400 Subject: [PATCH 2/2] fix(gateguard): make the external-contract question opt-in - questionIdsFor and condensedHintFor take an optional { externalContract: true }; without it a code change that handles data asks data-schema only, and an unknown profile gets the original four questions and condensed hint. - The hook reads GATEGUARD_EXTERNAL_CONTRACT_QUESTION with the same spellings as GATEGUARD_METRICS and passes it to the full denial, the condensed denial and the metrics record, so all three agree. - The effectiveness harness's arm patch forwards the option, so an arm never drops it silently. Docs: change-profile.md gains an External-contract question section and qualifies "byte-identical to the fixed four questions"; design-notes.md, SKILL.md (switch table, data rule, word-for-word set) and CHANGELOG say the fifth question is opt-in; question-effectiveness.md and the harness README note that the runner clears GATEGUARD_* variables, so a gate arm from a current tree measures the gate without the question. skill-triggers@1.json: registryDigest refreshed for the SKILL.md body change; the description and triggers are unchanged. --- CHANGELOG.md | 1 + docker/gateguard-effectiveness/README.md | 7 ++-- docker/gateguard-effectiveness/arm-patch.js | 8 ++--- docs/gateguard/change-profile.md | 18 +++++++++- docs/gateguard/design-notes.md | 4 ++- docs/gateguard/question-effectiveness.md | 8 +++++ manifests/context-packs/skill-triggers@1.json | 2 +- scripts/hooks/gateguard-fact-force.js | 11 +++++-- scripts/lib/gateguard-target-class.js | 33 ++++++++++++++----- skills/gateguard/SKILL.md | 20 +++++++---- 10 files changed, 84 insertions(+), 28 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9b279b0e1..4f0531fa4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Changed - GateGuard first-touch denials ask questions that fit the target: by class (instruction, test, prose, config, code) on its project-relative path, and for code by what the change touches. An edit that changes no public declaration asks for the in-module call sites instead of importers and public API, and the data-schema question is dropped when the change handles no data. Changes the hook cannot read with confidence keep the original four questions word for word. +- GateGuard's external-contract question ("name what outside this repository decides the format, units, timezone or protocol semantics") is opt-in through `GATEGUARD_EXTERNAL_CONTRACT_QUESTION`. Hidden-intent trials did not show it helps, so by default a code change that handles data asks only the data-schema question, and an unreadable change gets the original four questions and condensed hint. - GateGuard credits investigation that already happened: a completed, non-error `Glob`, `Grep`, `LS`, or search-style Bash/PowerShell command in the current human turn that names the target and whose scope contains it passes the first touch with a note. `Read` never counts, nor do searches in the same batch, before a compaction, with error results, or whose filters exclude the target or limit the search to another directory. When a denial follows a search that nearly counted, it names that search and why it did not count. - GateGuard passes comment and whitespace-only edits of code, test and prose files with a note, without marking the file checked, so the next change that alters code is still gated. - GateGuard asks once for a batch of new files: after a new file is denied, later new files of the same class in the same real directory within the same human turn pass with a note. diff --git a/docker/gateguard-effectiveness/README.md b/docker/gateguard-effectiveness/README.md index e4aed748a..646446118 100644 --- a/docker/gateguard-effectiveness/README.md +++ b/docker/gateguard-effectiveness/README.md @@ -21,8 +21,11 @@ where the deciding fact exists only in a simulated user's hidden intent, so an agent that does not ask can only guess. Scenario files omit declared gate question targets, but this does not establish blind or independent authorship: the authors knew the taxonomy, and the external-contract question was added -after inspecting this corpus. Treat those results as exploratory. The report -classifies behavior into descriptive hole categories. +after inspecting this corpus. Treat those results as exploratory. That question +is now opt-in (`GATEGUARD_EXTERNAL_CONTRACT_QUESTION`), and the runner clears +every `GATEGUARD_*` variable from the agent's environment, so a `gate` arm +built from a current candidate tree runs without it. The report classifies +behavior into descriptive hole categories. Each run writes `meta.json`, raw `results.jsonl`, transcripts, `summary.json`, `holes.md`, and `evidence.json`. The evidence manifest fingerprints the frozen diff --git a/docker/gateguard-effectiveness/arm-patch.js b/docker/gateguard-effectiveness/arm-patch.js index 8f2798523..dc59b2848 100644 --- a/docker/gateguard-effectiveness/arm-patch.js +++ b/docker/gateguard-effectiveness/arm-patch.js @@ -43,16 +43,16 @@ function applyArm(arm, targetClass) { const originalText = targetClass.questionText; const originalPhrase = targetClass.condensedQuestionPhrase; const originalHint = targetClass.condensedHintFor; - const questionIdsFor = (cls, isWrite, profile) => armQuestionIds(arm, originalIds(cls, isWrite, profile)); + const questionIdsFor = (cls, isWrite, profile, options) => armQuestionIds(arm, originalIds(cls, isWrite, profile, options)); const questionText = id => (Object.hasOwn(PLACEBO_TEXT, id) ? PLACEBO_TEXT[id] : originalText(id)); const phraseFor = id => { if (Object.hasOwn(PLACEBO_PHRASES, id)) return PLACEBO_PHRASES[id]; if (id === KEPT_ID) return VERBATIM_PHRASE; return originalPhrase(id) || originalText(id).replace(/ \(search the tree.*\)$/, '').replace(/^\w/, c => c.toLowerCase()); }; - const condensedHintFor = (cls, isWrite, profile) => { - const ids = questionIdsFor(cls, isWrite, profile); - if (ids.join('\n') === originalIds(cls, isWrite, profile).join('\n')) return originalHint(cls, isWrite, profile); + const condensedHintFor = (cls, isWrite, profile, options) => { + const ids = questionIdsFor(cls, isWrite, profile, options); + if (ids.join('\n') === originalIds(cls, isWrite, profile, options).join('\n')) return originalHint(cls, isWrite, profile, options); return `briefly state ${joinPhrases(ids.map(phraseFor).filter(Boolean))}, then retry.`; }; Object.assign(targetClass, { questionIdsFor, questionText, condensedHintFor }); diff --git a/docs/gateguard/change-profile.md b/docs/gateguard/change-profile.md index 1bb8d2c1c..653fabfdb 100644 --- a/docs/gateguard/change-profile.md +++ b/docs/gateguard/change-profile.md @@ -116,6 +116,21 @@ or a PowerShell file or web cmdlet (`Out-File`, `Get-Content`, `*-Clixml`; `Import-Csv` matches `csv`). Redirection is found without reading quotes, so a `>` inside a string also counts. +## External-contract question + +`GATEGUARD_EXTERNAL_CONTRACT_QUESTION` (`1`, `true`, `on`, `enabled`, +`enable` or `yes`) adds `external-contract` right after every `data-schema` +in the table above: "Name what outside this repository decides the format, +units, timezone or protocol semantics here (the consumer, the producer, or a +stated convention), or state that the choice is unconstrained". The condensed +hint and the decision metrics name it too. It is off by default because it +has no evidence behind it: it was designed after inspecting the +hidden-intent corpus, and in the rerun that added it the gated arm did not +improve (see +[question-effectiveness.md](question-effectiveness.md#adding-an-external-contract-question-did-not-help)). +`questionIdsFor(class, isWrite, profile, { externalContract: true })` returns +the ids with it. + ## Trivial edits An Edit is trivial when, for every entry, old and new have the same code once @@ -293,7 +308,8 @@ questions were asked without storing text. | config | any | `config-reader`, `config-effect`, `no-plaintext-secrets`, `quote-instruction` | `[data-schema]` is asked only when `touchesData`. With an unknown profile the -code text is byte-identical to the fixed four questions. Sensitive and +code text is byte-identical to the fixed four questions, unless the opt-in +[external-contract question](#external-contract-question) is on. Sensitive and hard-linked targets and NotebookEdit calls are never profiled, so they always get the full code questions. Instruction and config questions carry no change-dependent item, and the test and prose items diff --git a/docs/gateguard/design-notes.md b/docs/gateguard/design-notes.md index 5809dc9aa..c16e036f9 100644 --- a/docs/gateguard/design-notes.md +++ b/docs/gateguard/design-notes.md @@ -207,7 +207,9 @@ their absolute path. A Windows-style path has no real location on a POSIX host The first matching class wins: instruction, test, prose, config, code (the fallback). Cursor `.mdc` rule files are instructions wherever they live. A basename starting with `.env` is config unless it has a code extension -(`.env.example.ts` is source). Code targets ask the original four questions, +(`.env.example.ts` is source). Code targets ask the original four questions +(five with the opt-in +[external-contract question](change-profile.md#external-contract-question)), narrowed by the change profile (see [Questions from the change profile](change-profile.md#questions-from-the-change-profile)); other classes get class-specific questions and a class-specific condensed hint. diff --git a/docs/gateguard/question-effectiveness.md b/docs/gateguard/question-effectiveness.md index f7ab9259a..3135ebca5 100644 --- a/docs/gateguard/question-effectiveness.md +++ b/docs/gateguard/question-effectiveness.md @@ -529,6 +529,14 @@ and the corpus suggests that kind of answer matters. What is not claimed is that adding it improved anything measurable here. A fair test needs scenarios written by independent authors who have not seen that question or these results. +Because nothing here shows it helps, the question is now opt-in: it is asked +only with `GATEGUARD_EXTERNAL_CONTRACT_QUESTION` on (see +[change-profile.md](change-profile.md#external-contract-question)). The +harness clears every `GATEGUARD_*` variable from the agent's environment, so a +`gate` arm built from a candidate tree that includes this change measures the +gate without it, the "before the question" condition above. The runs reported +here used candidate trees from before the change, where it was always asked. + **Standing conclusion.** Across 336 billed trials the gate's benefit is not established, its direction has been positive in the unflagged runs, and its cost is a consistent 35 to 45% in turns and spend. "Not confirmed" is not "no diff --git a/manifests/context-packs/skill-triggers@1.json b/manifests/context-packs/skill-triggers@1.json index ed3be0d40..928c94131 100644 --- a/manifests/context-packs/skill-triggers@1.json +++ b/manifests/context-packs/skill-triggers@1.json @@ -10,7 +10,7 @@ "id": "hand-seeded", "source": "manual-curation-pending-regeneration" }, - "registryDigest": "16549b53055bf5c106e51ca7fb2fe68f8a705b5f6bad40d388349320caedfa63", + "registryDigest": "0136fd919b9b7ffe92e70b3ce28c6d21bd78b658387c0030d9a40eb63e3eddb9", "schemaVersion": 1, "triggers": { "skill:agent-runtime-gateway-smoke-test": [ diff --git a/scripts/hooks/gateguard-fact-force.js b/scripts/hooks/gateguard-fact-force.js index 977ec90d1..ad1fe74de 100644 --- a/scripts/hooks/gateguard-fact-force.js +++ b/scripts/hooks/gateguard-fact-force.js @@ -1945,9 +1945,14 @@ function batchSiblingWarning(safePath) { ); } +// see docs/gateguard/change-profile.md#external-contract-question +function questionOptions() { + return { externalContract: ECC_ENABLE_VALUES.has(normalizeEnvValue(process.env.GATEGUARD_EXTERNAL_CONTRACT_QUESTION)) }; +} + function firstTouchGateMsg(filePath, isWrite, cls, profile) { const safe = sanitizePath(filePath); - const questions = questionIdsFor(cls, isWrite, profile).map(questionText); + const questions = questionIdsFor(cls, isWrite, profile, questionOptions()).map(questionText); return [ '[Fact-Forcing Gate]', '', @@ -1968,7 +1973,7 @@ function firstTouchGateMsg(filePath, isWrite, cls, profile) { */ function condensedGateMsg(action, filePath, ordinal, cls = 'code', targetNote = '', profile = null, missNote = '') { const safe = sanitizePath(filePath); - const hint = condensedHintFor(cls, action === 'creation', profile); + const hint = condensedHintFor(cls, action === 'creation', profile, questionOptions()); return ( `[Fact-Forcing Gate] (denial #${ordinal} this session) First ${action} of ${safe}: ` + `${hint} ` + @@ -2009,7 +2014,7 @@ function firstTouchDenial(filePath, { isWrite, denials, cls, sensitive, linked = cls, sensitive, profile, - questions: questionIdsFor(cls, isWrite, profile) + questions: questionIdsFor(cls, isWrite, profile, questionOptions()) }); if (denials > getFullDenialBudget()) { const action = isWrite ? 'creation' : 'edit'; diff --git a/scripts/lib/gateguard-target-class.js b/scripts/lib/gateguard-target-class.js index 0f2e3a4db..d140bed1c 100644 --- a/scripts/lib/gateguard-target-class.js +++ b/scripts/lib/gateguard-target-class.js @@ -186,7 +186,7 @@ const CLASS_QUESTIONS = Object.fromEntries( ); // see docs/gateguard/change-profile.md#questions-from-the-change-profile -function codeQuestionIds(isWrite, profile) { +function codeQuestionIds(isWrite, profile, externalContract) { const known = Boolean(profile) && profile.known === true; const surface = !known || profile.touchesPublicSurface !== false; const data = !known || profile.touchesData !== false; @@ -194,8 +194,14 @@ function codeQuestionIds(isWrite, profile) { // A change that handles data is usually constrained by something outside the // tree - a consumer's dialect, a reporting timezone, a protocol's semantics - // and the repository cannot settle it. data-schema asks what the change - // touches; external-contract asks who decides what it has to be. - return data ? [...opening, 'data-schema', 'external-contract'] : opening; + // touches; external-contract asks who decides what it has to be. It is + // opt-in: see docs/gateguard/change-profile.md#external-contract-question. + if (!data) return opening; + return externalContract ? [...opening, 'data-schema', 'external-contract'] : [...opening, 'data-schema']; +} + +function asksExternalContract(options) { + return Boolean(options) && options.externalContract === true; } /** True when a class's questions depend on the change profile. */ @@ -203,9 +209,14 @@ function questionsUseProfile(cls) { return !Object.hasOwn(CLASS_QUESTION_IDS, cls); } -/** Stable ids of the first-touch questions for a class, action and change profile. */ -function questionIdsFor(cls, isWrite, profile) { - const ids = Object.hasOwn(CLASS_QUESTION_IDS, cls) ? CLASS_QUESTION_IDS[cls](Boolean(isWrite)) : codeQuestionIds(Boolean(isWrite), profile); +/** + * Stable ids of the first-touch questions for a class, action and change profile. + * `options.externalContract === true` adds the opt-in external-contract question. + */ +function questionIdsFor(cls, isWrite, profile, options) { + const ids = Object.hasOwn(CLASS_QUESTION_IDS, cls) + ? CLASS_QUESTION_IDS[cls](Boolean(isWrite)) + : codeQuestionIds(Boolean(isWrite), profile, asksExternalContract(options)); return [...ids, 'quote-instruction']; } @@ -215,6 +226,8 @@ function questionText(id) { } const CODE_CONDENSED_HINT = + "briefly state importers/callers, affected API, data schemas if any, and the user's verbatim instruction, then retry."; +const CODE_CONDENSED_HINT_EXTERNAL_CONTRACT = "briefly state importers/callers, affected API, data schemas if any, what outside the repository fixes the format or semantics, and the user's verbatim instruction, then retry."; const CONDENSED_PHRASES = Object.freeze({ @@ -252,10 +265,12 @@ const CLASS_CONDENSED_HINTS = { }; /** One-line condensed hint matching `questionIdsFor`. */ -function condensedHintFor(cls, isWrite, profile) { +function condensedHintFor(cls, isWrite, profile, options) { if (Object.hasOwn(CLASS_CONDENSED_HINTS, cls)) return CLASS_CONDENSED_HINTS[cls](Boolean(isWrite)); - if (!isWrite && !(profile && profile.known === true)) return CODE_CONDENSED_HINT; - const phrases = questionIdsFor(cls, isWrite, profile).map(condensedQuestionPhrase).filter(Boolean); + if (!isWrite && !(profile && profile.known === true)) { + return asksExternalContract(options) ? CODE_CONDENSED_HINT_EXTERNAL_CONTRACT : CODE_CONDENSED_HINT; + } + const phrases = questionIdsFor(cls, isWrite, profile, options).map(condensedQuestionPhrase).filter(Boolean); return `briefly state ${joinPhrases(phrases)}, then retry.`; } diff --git a/skills/gateguard/SKILL.md b/skills/gateguard/SKILL.md index e6ebcc1f7..7cda6449b 100644 --- a/skills/gateguard/SKILL.md +++ b/skills/gateguard/SKILL.md @@ -117,19 +117,23 @@ For **code** targets the gate reads the change it is about to allow (Edit and clock reads on either side; in shell scripts also redirection to a file, `curl`, `wget`, `jq` and PowerShell file and web cmdlets. Without them the data - question is dropped. + question is dropped. With `GATEGUARD_EXTERNAL_CONTRACT_QUESTION` on, a + change that keeps the data question is also asked what outside the + repository decides its format or semantics. - **Trivial:** see [Comment and whitespace-only edits](#comment-and-whitespace-only-edits). Supported extensions: `.js .mjs .cjs .jsx .ts .tsx .mts .cts .py .pyi .go .rs .java .kt .kts .cs .c .h .cc .cpp .cxx .hpp .hh .hxx .sh .bash .zsh .ps1 .psm1 .bat .cmd`. Any other extension, a missing or non-string field, more than 64 MultiEdit entries, a side over -64 KiB, or 256 KiB in total means the full questions, word for word. Sensitive +64 KiB, or 256 KiB in total means the full questions, word for word: the +original four (importers, public API, data schemas, the user's instruction), +or five with `GATEGUARD_EXTERNAL_CONTRACT_QUESTION` on. Sensitive and hard-linked targets and NotebookEdit calls always get the full questions. Other classes keep their class questions. Each question has a stable id (`importers`, `public-api`, `local-callers`, `callers`, `no-duplicate`, `data-schema`, -`quote-instruction`, and one per class question), listed in -`docs/gateguard/change-profile.md`. +`external-contract` when opted in, `quote-instruction`, and one per class +question), listed in `docs/gateguard/change-profile.md`. ### Questions by target class @@ -583,10 +587,12 @@ load-bearing destructive-Bash checks keep running: | `GATEGUARD_BASH_EXTRA_DESTRUCTIVE` | unset | Extra destructive-command patterns, as regex source, added to the built-in set. A malformed regex is treated as unset (built-ins still apply) and logged once to stderr. | | `GATEGUARD_STATE_DIR` | `~/.gateguard` | Where per-session gate state is kept. If state cannot be persisted the gate allows the operation rather than looping, and names this variable in the warning. | | `GATEGUARD_METRICS` | unset (off) | Records one line per Edit/Write/MultiEdit/NotebookEdit/Bash/PowerShell decision in `/metrics.jsonl` (see [Decision metrics](#decision-metrics)). Never changes a decision. | +| `GATEGUARD_EXTERNAL_CONTRACT_QUESTION` | unset (off) | Adds a code question after the data-schema question: name what outside the repository decides the format, units, timezone or protocol semantics. Experimental: in hidden-intent trials it did not change outcomes (`docs/gateguard/question-effectiveness.md`). Never changes a decision. | -`GATEGUARD_BASH_ROUTINE_DISABLED` and `GATEGUARD_METRICS` accept `1`, `true`, -`on`, `enabled`, `enable`, or `yes` (case- and whitespace-insensitive); any -other value leaves the routine gate on and metrics off. +`GATEGUARD_BASH_ROUTINE_DISABLED`, `GATEGUARD_METRICS` and +`GATEGUARD_EXTERNAL_CONTRACT_QUESTION` accept `1`, `true`, `on`, `enabled`, +`enable`, or `yes` (case- and whitespace-insensitive); any other value leaves +the routine gate on, metrics off, and the question unasked. #### Decision metrics