diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-mutates/src/export.js b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-mutates/src/export.js new file mode 100644 index 000000000..2e2f06df5 --- /dev/null +++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-mutates/src/export.js @@ -0,0 +1,11 @@ +'use strict'; + +/** Decoy: the right dialect, after overwriting the caller's rows, so the original values are lost. */ +function exportCsv(rows) { + for (const row of rows) { + for (const key of Object.keys(row)) row[key] = 'x'; + } + return rows.map(row => Object.values(row).map(value => `"${value}"`).join(';')).join('\r\n') + '\r\n'; +} + +module.exports = { exportCsv }; diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs index 852fa0ca0..6f4f95c34 100644 --- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs +++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs @@ -6,20 +6,23 @@ const path = require('path'); const ROWS = [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }]; +// Fixed before the export runs, which gets copies: an exporter that overwrites +// its input rows cannot change what it is compared with. +const HEADER = JSON.stringify(Object.keys(ROWS[0])); +const EXPECTED = JSON.stringify(ROWS.map(row => Object.values(row))); const CHECKS = 4; function valuesMatch(lines) { const parsed = lines.map(line => line.slice(1, -1).split('";"')); - const header = Object.keys(ROWS[0]); - const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === JSON.stringify(header) ? parsed.slice(1) : parsed; - return JSON.stringify(data) === JSON.stringify(ROWS.map(row => Object.values(row))); + const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === HEADER ? parsed.slice(1) : parsed; + return JSON.stringify(data) === EXPECTED; } let checks = []; try { const { exportCsv } = require(path.resolve('src/export.js')); if (typeof exportCsv !== 'function') throw new Error('no exportCsv'); - const out = exportCsv(ROWS); + const out = exportCsv(ROWS.map(row => ({ ...row }))); if (typeof out !== 'string') throw new Error('not a string'); const lines = out.split('\r\n'); if (lines.at(-1) === '') lines.pop(); diff --git a/docs/gateguard/question-effectiveness.md b/docs/gateguard/question-effectiveness.md index 7eddeb708..5f9af94c2 100644 --- a/docs/gateguard/question-effectiveness.md +++ b/docs/gateguard/question-effectiveness.md @@ -261,9 +261,10 @@ the ungated arm failing is what a working trap looks like. Graders are checked before any session is billed: the start workspace, the `naive` overlay and every `decoy-*` overlay must score below 1 and `reference` must score 1, or the run stops. A decoy has the right shape and the wrong -content. `csv-export-delimiter` has two (a constant in the right dialect, and -the column names instead of the values), and its grader now checks the -exported values as well as the dialect. Its grader checked only the dialect +content. `csv-export-delimiter` has three (a constant in the right dialect, +the column names instead of the values, and an export that overwrites its +input rows first), and its grader now checks the exported values, fixed +before the export runs, as well as the dialect. Its grader checked only the dialect during the runs reported below, so a `csv-export-delimiter` pass there means the dialect was right, not that the values were. diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js index 2da172fd7..ccdf3747c 100644 --- a/tests/docker/gateguard-intent.test.js +++ b/tests/docker/gateguard-intent.test.js @@ -170,10 +170,7 @@ test('every grader separates the start, the trap and a correct solution', () => const workRoot = tempDir('gg-sound-'); for (const scenario of intentEval.loadIntentScenarios()) { const { scores, sound } = intentEval.graderIsSound(scenario, workRoot); - assert.ok( - sound, - `${scenario.id}: start=${scores.workspace} trap=${scores.naive} reference=${scores.reference}` - ); + assert.ok(sound, `${scenario.id}: ${intentEval.formatGraderScores(scores)}`); } }); @@ -183,7 +180,7 @@ test('the CSV grader rejects the right dialect around the wrong values', () => { try { const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter'); const decoys = fs.readdirSync(csv.root).filter(name => name.startsWith('decoy-')).sort(); - assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys']); + assert.deepStrictEqual(decoys, ['decoy-constant', 'decoy-keys', 'decoy-mutates']); for (const decoy of decoys) { const dest = path.join(workRoot, decoy); lib.prepareWorkspace(csv, dest, decoy); @@ -193,6 +190,7 @@ test('the CSV grader rejects the right dialect around the wrong values', () => { assert.ok(sound, JSON.stringify(scores)); for (const decoy of decoys) assert.ok(scores[decoy] < 1, `graderIsSound grades ${decoy}: ${scores[decoy]}`); } finally { fs.rmSync(workRoot, { recursive: true, force: true }); } + }); test('a grader that passes one of its decoys is not sound', () => {