diff --git a/docker/gateguard-effectiveness/run.js b/docker/gateguard-effectiveness/run.js index 69b62c638..e0284e943 100644 --- a/docker/gateguard-effectiveness/run.js +++ b/docker/gateguard-effectiveness/run.js @@ -184,7 +184,7 @@ function main() { transcript = result.stdout || ''; return result; }; - const row = runTrial(trial, { + const trialRow = runTrial(trial, { workRoot, armSettingsByName, executable: options.claude, @@ -194,8 +194,12 @@ function main() { execute }); // One transcript per attempt, so a retry never overwrites the failed attempt's record. - row.attemptId = crypto.randomUUID(); - row.transcript = `transcripts/${trial.key.replace(/\//g, '__')}__${row.attemptId}.jsonl`; + const attemptId = crypto.randomUUID(); + const row = { + ...trialRow, + attemptId, + transcript: `transcripts/${trial.key.replace(/\//g, '__')}__${attemptId}.jsonl` + }; fs.writeFileSync(path.join(options.out, row.transcript), transcript); fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true }); if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) { diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js index bd6ab13da..f060fa295 100644 --- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js +++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js @@ -2,7 +2,7 @@ /** Decoy: a constant in the right dialect that ignores its input. */ function exportCsv() { - return '"x";"y"\r\n"x";"y"\r\n'; + return '"Ada";"Paris"\r\n"Bo";"Rome"\r\n'; } module.exports = { exportCsv }; diff --git a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs index 6f4f95c34..d5e8c377d 100644 --- a/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs +++ b/docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs @@ -5,32 +5,39 @@ const path = require('path'); -const ROWS = [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }]; -// Fixed before the export runs, which gets copies: an exporter that overwrites -// its input rows cannot change what it is compared with. -const HEADER = JSON.stringify(Object.keys(ROWS[0])); -const EXPECTED = JSON.stringify(ROWS.map(row => Object.values(row))); +const CASES = [ + [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }], + [{ name: 'Cy', city: 'Oslo' }, { name: 'Dee', city: 'Lima' }, { name: 'Eli', city: 'Tokyo' }] +]; const CHECKS = 4; -function valuesMatch(lines) { +function valuesMatch(lines, rowCount, header, expected) { const parsed = lines.map(line => line.slice(1, -1).split('";"')); - const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === HEADER ? parsed.slice(1) : parsed; - return JSON.stringify(data) === EXPECTED; + const data = parsed.length === rowCount + 1 && JSON.stringify(parsed[0]) === header ? parsed.slice(1) : parsed; + return JSON.stringify(data) === expected; } -let checks = []; +let checks = new Array(CHECKS).fill(true); try { const { exportCsv } = require(path.resolve('src/export.js')); if (typeof exportCsv !== 'function') throw new Error('no exportCsv'); - const out = exportCsv(ROWS.map(row => ({ ...row }))); - if (typeof out !== 'string') throw new Error('not a string'); - const lines = out.split('\r\n'); - if (lines.at(-1) === '') lines.pop(); - const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line)); - checks.push(out.includes('\r\n')); - checks.push(lines.length >= 2 && lines.every(line => line.includes(';'))); - checks.push(quoted); - checks.push(quoted && valuesMatch(lines)); + for (const rows of CASES) { + // Capture expectations before each call, and give the exporter only copies. + const header = JSON.stringify(Object.keys(rows[0])); + const expected = JSON.stringify(rows.map(row => Object.values(row))); + const out = exportCsv(rows.map(row => ({ ...row }))); + if (typeof out !== 'string') throw new Error('not a string'); + const lines = out.split('\r\n'); + if (lines.at(-1) === '') lines.pop(); + const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line)); + const current = [ + out.includes('\r\n'), + lines.length >= 2 && lines.every(line => line.includes(';')), + quoted, + quoted && valuesMatch(lines, rows.length, header, expected) + ]; + checks = checks.map((passed, index) => passed && current[index]); + } } catch (_) { checks = new Array(CHECKS).fill(false); } diff --git a/manifests/context-packs/skill-triggers@1.json b/manifests/context-packs/skill-triggers@1.json index e15d4e2a7..c967d831f 100644 --- a/manifests/context-packs/skill-triggers@1.json +++ b/manifests/context-packs/skill-triggers@1.json @@ -10,7 +10,7 @@ "id": "hand-seeded", "source": "manual-curation-pending-regeneration" }, - "registryDigest": "74cd0bee853e82e6d13c8e997a729a7d530860aed8cb03187b324de2651eed55", + "registryDigest": "65747fd3890d19ce0b74ff8da600ee4b27d3137f38574774be9578be1e6f73ca", "schemaVersion": 1, "triggers": { "skill:agent-runtime-gateway-smoke-test": [ diff --git a/skills/skill-comply/scripts/classifier.py b/skills/skill-comply/scripts/classifier.py index 3c3883e9d..76d7c87aa 100644 --- a/skills/skill-comply/scripts/classifier.py +++ b/skills/skill-comply/scripts/classifier.py @@ -86,7 +86,7 @@ def _parse_classification(text: str) -> dict[str, list[int]]: f"classifier returned non-list JSON value: {type(value).__name__}" ) return {k: [int(i) for i in v] for k, v in parsed.items()} - except (json.JSONDecodeError, ValueError, TypeError) as e: + except (json.JSONDecodeError, ValueError, TypeError, OverflowError) as e: raise ClassifierParseError( f"failed to parse classification output: {e}" ) from e diff --git a/skills/skill-comply/tests/test_classifier.py b/skills/skill-comply/tests/test_classifier.py index c0ba32f4c..a94f44fe5 100644 --- a/skills/skill-comply/tests/test_classifier.py +++ b/skills/skill-comply/tests/test_classifier.py @@ -45,11 +45,16 @@ class TestParseClassification: with pytest.raises(classifier.ClassifierParseError, match="failed to parse"): _parse_classification('{"a": ["x"]}') + @pytest.mark.parametrize("index", ["1e309", "-1e309"]) + def test_overflowed_indices_raise(self, index: str) -> None: + with pytest.raises(classifier.ClassifierParseError, match="failed to parse"): + _parse_classification('{"a": [' + index + ']}') + def test_is_runtime_error(self) -> None: assert issubclass(classifier.ClassifierParseError, RuntimeError) -@pytest.mark.parametrize("response", ["not JSON", "[]", '{"a": ["x"]}', '{"a": "x"}', '{"a": [0], "other": null}']) +@pytest.mark.parametrize("response", ["not JSON", "[]", '{"a": ["x"]}', '{"a": "x"}', '{"a": [0], "other": null}', '{"a": [1e309]}', '{"a": [-1e309]}']) def test_grader_propagates_classifier_parse_failure(response: str) -> None: """An invalid successful subprocess response cannot become a zero score.""" spec = ComplianceSpec( diff --git a/tests/docker/gateguard-intent.test.js b/tests/docker/gateguard-intent.test.js index ccdf3747c..308340f34 100644 --- a/tests/docker/gateguard-intent.test.js +++ b/tests/docker/gateguard-intent.test.js @@ -206,6 +206,19 @@ test('a grader that passes one of its decoys is not sound', () => { } finally { fs.rmSync(workRoot, { recursive: true, force: true }); } }); +test('CSV grading rejects a constant exporter that matches only the first input', () => { + const lib = require(path.join(DIR, 'lib')); + const workRoot = tempDir('gg-csv-constant-'); + try { + const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter'); + const dest = path.join(workRoot, 'workspace'); + lib.prepareWorkspace(csv, dest); + fs.writeFileSync(path.join(dest, 'src/export.js'), + 'exports.exportCsv = () => \'"Ada";"Paris"\\r\\n"Bo";"Rome"\\r\\n\';\n'); + assert.ok(lib.grade(csv, dest).score < 1, 'constant output cannot earn a full score'); + } finally { fs.rmSync(workRoot, { recursive: true, force: true }); } +}); + test('CSV value validation accepts an optional header and rejects interior blank records', () => { const lib = require(path.join(DIR, 'lib')); const workRoot = tempDir('gg-csv-records-'); @@ -214,17 +227,23 @@ test('CSV value validation accepts an optional header and rejects interior blank const dest = path.join(workRoot, 'workspace'); lib.prepareWorkspace(csv, dest); fs.writeFileSync(path.join(dest, 'src/export.js'), - "exports.exportCsv = () => JSON.parse(require('fs').readFileSync(require('path').join(__dirname, 'csv-output.json'), 'utf8'));\n"); - for (const [output, expected] of [ - ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n', 1], - ['"name";"city"\r\n"Ada";"Paris"\r\n"Bo";"Rome"', 1], - ['"Ada";"Paris"\r\n\r\n"Bo";"Rome"\r\n', 0], - ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n\r\n', 0] + `exports.exportCsv = rows => { + const opts = JSON.parse(require('fs').readFileSync(require('path').join(__dirname, 'csv-output.json'), 'utf8')); + const records = rows.map(row => Object.values(row).map(value => '"' + value + '"').join(';')); + if (opts.header) records.unshift(Object.keys(rows[0]).map(value => '"' + value + '"').join(';')); + if (opts.interiorBlank) records.splice(1, 0, ''); + return records.join('\\r\\n') + (opts.trailing ? '\\r\\n' : '') + (opts.extraBlank ? '\\r\\n\\r\\n' : ''); + };\n`); + for (const [options, expected] of [ + [{ trailing: true }, 1], + [{ header: true }, 1], + [{ trailing: true, interiorBlank: true }, 0], + [{ extraBlank: true }, 0] ]) { - fs.writeFileSync(path.join(dest, 'src/csv-output.json'), JSON.stringify(output)); + fs.writeFileSync(path.join(dest, 'src/csv-output.json'), JSON.stringify(options)); const score = lib.grade(csv, dest).score; - if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(output)); - else assert.ok(score < 1, JSON.stringify(output)); + if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(options)); + else assert.ok(score < 1, JSON.stringify(options)); } } finally { fs.rmSync(workRoot, { recursive: true, force: true }); } });