From deb5d0a1d020d7519746ebcbcb6821eeb0f6a168 Mon Sep 17 00:00:00 2001 From: montjeffrey Date: Sun, 11 Oct 2026 14:14:13 -0400 Subject: [PATCH 1/2] test(gateguard): pin a CSV decoy that overwrites its input rows The CSV grader reads its expected values from the same ROWS objects it passes to exportCsv, so an exporter that overwrites the input fields and exports them scores 1 (Greptile on #3519). decoy-mutates does exactly that; graderIsSound reports the grader unsound on this head (decoy-mutates=1). The grader-soundness test now prints every decoy score through formatGraderScores. --- .../csv-export-delimiter/decoy-mutates/src/export.js | 11 +++++++++++ tests/docker/gateguard-intent.test.js | 7 ++----- 2 files changed, 13 insertions(+), 5 deletions(-) create mode 100644 docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-mutates/src/export.js diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-mutates/src/export.js b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-mutates/src/export.js new file mode 100644 index 000000000..2e2f06df5 --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-mutates/src/export.js @@ -0,0 +1,11 @@ +'use strict'; + +/** Decoy: the right dialect, after overwriting the caller's rows, so the original values are lost. */ +function exportCsv(rows) { + for (const row of rows) { + for (const key of Object.keys(row)) row[key] = 'x'; + } + return rows.map(row => Object.values(row).map(value => `"${value}"`).join(';')).join('\r\n') + '\r\n'; +} + +module.exports = { exportCsv }; diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js index a0ea86f99..d4395af50 100644 --- a/tests/docker/gateguard-intent.test.js +++ b/tests/docker/gateguard-intent.test.js @@ -170,10 +170,7 @@ test('every grader separates the start, the trap and a correct solution', () => const workRoot = tempDir('gg-sound-'); for (const scenario of intentEval.loadIntentScenarios()) { const { scores, sound } = intentEval.graderIsSound(scenario, workRoot); - assert.ok( - sound, - `${scenario.id}: start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}` - ); + assert.ok(sound, `${scenario.id}: ${intentEval.formatGraderScores(scores)}`); } }); @@ -182,7 +179,7 @@ test('the CSV grader rejects the right dialect around the wrong values', () => { const workRoot = tempDir('gg-decoy-'); const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter'); const decoys = fs.readdirSync(csv.root).filter(name => name.startsWith('decoy-')).sort(); - assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys']); + assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys', 'decoy-mutates']); for (const decoy of decoys) { const dest = path.join(workRoot, decoy); lib.prepareWorkspace(csv, dest, decoy); From 58f7021a64f817392cda86ddd6bd7d6bfb2b249d Mon Sep 17 00:00:00 2001 From: montjeffrey Date: Sun, 11 Oct 2026 14:14:13 -0400 Subject: [PATCH 2/2] fix(gateguard): fix the CSV grader's expected values before the export runs Snapshot the header and expected rows when the grader loads and pass exportCsv copies, so an exporter that overwrites its input cannot change what it is compared with. decoy-mutates now scores 0.75 with the other two decoys; reference still scores 1. --- .../scenarios-intent/csv-export-delimiter/grader.cjs | 11 +++++++---- docs/gateguard/question-effectiveness.md | 7 ++++--- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs index 37129807c..e3f5fe91d 100644 --- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs +++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs @@ -6,20 +6,23 @@ const path = require('path'); const ROWS = [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }]; +// Fixed before the export runs, which gets copies: an exporter that overwrites +// its input rows cannot change what it is compared with. +const HEADER = JSON.stringify(Object.keys(ROWS[0])); +const EXPECTED = JSON.stringify(ROWS.map(row => Object.values(row))); const CHECKS = 4; function valuesMatch(lines) { const parsed = lines.map(line => line.slice(1, -1).split('";"')); - const header = Object.keys(ROWS[0]); - const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === JSON.stringify(header) ? parsed.slice(1) : parsed; - return JSON.stringify(data) === JSON.stringify(ROWS.map(row => Object.values(row))); + const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === HEADER ? parsed.slice(1) : parsed; + return JSON.stringify(data) === EXPECTED; } let checks = []; try { const { exportCsv } = require(path.resolve('src/export.js')); if (typeof exportCsv !== 'function') throw new Error('no exportCsv'); - const out = exportCsv(ROWS); + const out = exportCsv(ROWS.map(row => ({ ...row }))); if (typeof out !== 'string') throw new Error('not a string'); const lines = out.split('\r\n').filter(Boolean); const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line)); diff --git a/docs/gateguard/question-effectiveness.md b/docs/gateguard/question-effectiveness.md index 7ac2d7355..e20b860c7 100644 --- a/docs/gateguard/question-effectiveness.md +++ b/docs/gateguard/question-effectiveness.md @@ -259,9 +259,10 @@ the ungated arm failing is what a working trap looks like. Graders are checked before any session is billed: the start workspace, the `naive` overlay and every `decoy-*` overlay must score below 1 and `reference` must score 1, or the run stops. A decoy has the right shape and the wrong -content. `csv-export-delimiter` has two (a constant in the right dialect, and -the column names instead of the values), and its grader now checks the -exported values as well as the dialect. Its grader checked only the dialect +content. `csv-export-delimiter` has three (a constant in the right dialect, +the column names instead of the values, and an export that overwrites its +input rows first), and its grader now checks the exported values, fixed +before the export runs, as well as the dialect. Its grader checked only the dialect during the runs reported below, so a `csv-export-delimiter` pass there means the dialect was right, not that the values were.