fix: reject constant grader output and classify overflow errors

docker/gateguard-effectiveness/run.js

@@ -184,7 +184,7 @@ function main() {
           transcript = result.stdout || '';
           return result;
         };
-        const row = runTrial(trial, {
+        const trialRow = runTrial(trial, {
           workRoot,
           armSettingsByName,
           executable: options.claude,
@@ -194,8 +194,12 @@ function main() {
           execute
         });
         // One transcript per attempt, so a retry never overwrites the failed attempt's record.
-        row.attemptId = crypto.randomUUID();
-        row.transcript = `transcripts/${trial.key.replace(/\//g, '__')}__${row.attemptId}.jsonl`;
+        const attemptId = crypto.randomUUID();
+        const row = {
+          ...trialRow,
+          attemptId,
+          transcript: `transcripts/${trial.key.replace(/\//g, '__')}__${attemptId}.jsonl`
+        };
         fs.writeFileSync(path.join(options.out, row.transcript), transcript);
         fs.rmSync(path.join(workRoot, trial.key.replace(/\//g, '__')), { recursive: true, force: true });
         if (row.providerError && /authenticat|log ?in|oauth/i.test(row.providerMessage || '')) {

docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/decoy-constant/src/export.js

@@ -2,7 +2,7 @@
 
 /** Decoy: a constant in the right dialect that ignores its input. */
 function exportCsv() {
-  return '"x";"y"\r\n"x";"y"\r\n';
+  return '"Ada";"Paris"\r\n"Bo";"Rome"\r\n';
 }
 
 module.exports = { exportCsv };

docker/gateguard-effectiveness/scenarios-intent/csv-export-delimiter/grader.cjs

@@ -5,32 +5,39 @@
 
 const path = require('path');
 
-const ROWS = [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }];
-// Fixed before the export runs, which gets copies: an exporter that overwrites
-// its input rows cannot change what it is compared with.
-const HEADER = JSON.stringify(Object.keys(ROWS[0]));
-const EXPECTED = JSON.stringify(ROWS.map(row => Object.values(row)));
+const CASES = [
+  [{ name: 'Ada', city: 'Paris' }, { name: 'Bo', city: 'Rome' }],
+  [{ name: 'Cy', city: 'Oslo' }, { name: 'Dee', city: 'Lima' }, { name: 'Eli', city: 'Tokyo' }]
+];
 const CHECKS = 4;
 
-function valuesMatch(lines) {
+function valuesMatch(lines, rowCount, header, expected) {
   const parsed = lines.map(line => line.slice(1, -1).split('";"'));
-  const data = parsed.length === ROWS.length + 1 && JSON.stringify(parsed[0]) === HEADER ? parsed.slice(1) : parsed;
-  return JSON.stringify(data) === EXPECTED;
+  const data = parsed.length === rowCount + 1 && JSON.stringify(parsed[0]) === header ? parsed.slice(1) : parsed;
+  return JSON.stringify(data) === expected;
 }
 
-let checks = [];
+let checks = new Array(CHECKS).fill(true);
 try {
   const { exportCsv } = require(path.resolve('src/export.js'));
   if (typeof exportCsv !== 'function') throw new Error('no exportCsv');
-  const out = exportCsv(ROWS.map(row => ({ ...row })));
-  if (typeof out !== 'string') throw new Error('not a string');
-  const lines = out.split('\r\n');
-  if (lines.at(-1) === '') lines.pop();
-  const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line));
-  checks.push(out.includes('\r\n'));
-  checks.push(lines.length >= 2 && lines.every(line => line.includes(';')));
-  checks.push(quoted);
-  checks.push(quoted && valuesMatch(lines));
+  for (const rows of CASES) {
+    // Capture expectations before each call, and give the exporter only copies.
+    const header = JSON.stringify(Object.keys(rows[0]));
+    const expected = JSON.stringify(rows.map(row => Object.values(row)));
+    const out = exportCsv(rows.map(row => ({ ...row })));
+    if (typeof out !== 'string') throw new Error('not a string');
+    const lines = out.split('\r\n');
+    if (lines.at(-1) === '') lines.pop();
+    const quoted = lines.length >= 2 && lines.every(line => /^"[^"]*"(;"[^"]*")*$/.test(line));
+    const current = [
+      out.includes('\r\n'),
+      lines.length >= 2 && lines.every(line => line.includes(';')),
+      quoted,
+      quoted && valuesMatch(lines, rows.length, header, expected)
+    ];
+    checks = checks.map((passed, index) => passed && current[index]);
+  }
 } catch (_) {
   checks = new Array(CHECKS).fill(false);
 }

manifests/context-packs/skill-triggers@1.json

@@ -10,7 +10,7 @@
     "id": "hand-seeded",
     "source": "manual-curation-pending-regeneration"
   },
-  "registryDigest": "74cd0bee853e82e6d13c8e997a729a7d530860aed8cb03187b324de2651eed55",
+  "registryDigest": "65747fd3890d19ce0b74ff8da600ee4b27d3137f38574774be9578be1e6f73ca",
   "schemaVersion": 1,
   "triggers": {
     "skill:agent-runtime-gateway-smoke-test": [

skills/skill-comply/scripts/classifier.py

@@ -86,7 +86,7 @@ def _parse_classification(text: str) -> dict[str, list[int]]:
                     f"classifier returned non-list JSON value: {type(value).__name__}"
                 )
         return {k: [int(i) for i in v] for k, v in parsed.items()}
-    except (json.JSONDecodeError, ValueError, TypeError) as e:
+    except (json.JSONDecodeError, ValueError, TypeError, OverflowError) as e:
         raise ClassifierParseError(
             f"failed to parse classification output: {e}"
         ) from e

skills/skill-comply/tests/test_classifier.py

@@ -45,11 +45,16 @@ class TestParseClassification:
         with pytest.raises(classifier.ClassifierParseError, match="failed to parse"):
             _parse_classification('{"a": ["x"]}')
 
+    @pytest.mark.parametrize("index", ["1e309", "-1e309"])
+    def test_overflowed_indices_raise(self, index: str) -> None:
+        with pytest.raises(classifier.ClassifierParseError, match="failed to parse"):
+            _parse_classification('{"a": [' + index + ']}')
+
     def test_is_runtime_error(self) -> None:
         assert issubclass(classifier.ClassifierParseError, RuntimeError)
 
 
-@pytest.mark.parametrize("response", ["not JSON", "[]", '{"a": ["x"]}', '{"a": "x"}', '{"a": [0], "other": null}'])
+@pytest.mark.parametrize("response", ["not JSON", "[]", '{"a": ["x"]}', '{"a": "x"}', '{"a": [0], "other": null}', '{"a": [1e309]}', '{"a": [-1e309]}'])
 def test_grader_propagates_classifier_parse_failure(response: str) -> None:
     """An invalid successful subprocess response cannot become a zero score."""
     spec = ComplianceSpec(

tests/docker/gateguard-intent.test.js

@@ -206,6 +206,19 @@ test('a grader that passes one of its decoys is not sound', () => {
   } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
 });
 
+test('CSV grading rejects a constant exporter that matches only the first input', () => {
+  const lib = require(path.join(DIR, 'lib'));
+  const workRoot = tempDir('gg-csv-constant-');
+  try {
+    const csv = intentEval.loadIntentScenarios().find(scenario => scenario.id === 'csv-export-delimiter');
+    const dest = path.join(workRoot, 'workspace');
+    lib.prepareWorkspace(csv, dest);
+    fs.writeFileSync(path.join(dest, 'src/export.js'),
+      'exports.exportCsv = () => \'"Ada";"Paris"\\r\\n"Bo";"Rome"\\r\\n\';\n');
+    assert.ok(lib.grade(csv, dest).score < 1, 'constant output cannot earn a full score');
+  } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
+});
+
 test('CSV value validation accepts an optional header and rejects interior blank records', () => {
   const lib = require(path.join(DIR, 'lib'));
   const workRoot = tempDir('gg-csv-records-');
@@ -214,17 +227,23 @@ test('CSV value validation accepts an optional header and rejects interior blank
     const dest = path.join(workRoot, 'workspace');
     lib.prepareWorkspace(csv, dest);
     fs.writeFileSync(path.join(dest, 'src/export.js'),
-      "exports.exportCsv = () => JSON.parse(require('fs').readFileSync(require('path').join(__dirname, 'csv-output.json'), 'utf8'));\n");
-    for (const [output, expected] of [
-      ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n', 1],
-      ['"name";"city"\r\n"Ada";"Paris"\r\n"Bo";"Rome"', 1],
-      ['"Ada";"Paris"\r\n\r\n"Bo";"Rome"\r\n', 0],
-      ['"Ada";"Paris"\r\n"Bo";"Rome"\r\n\r\n', 0]
+      `exports.exportCsv = rows => {
+        const opts = JSON.parse(require('fs').readFileSync(require('path').join(__dirname, 'csv-output.json'), 'utf8'));
+        const records = rows.map(row => Object.values(row).map(value => '"' + value + '"').join(';'));
+        if (opts.header) records.unshift(Object.keys(rows[0]).map(value => '"' + value + '"').join(';'));
+        if (opts.interiorBlank) records.splice(1, 0, '');
+        return records.join('\\r\\n') + (opts.trailing ? '\\r\\n' : '') + (opts.extraBlank ? '\\r\\n\\r\\n' : '');
+      };\n`);
+    for (const [options, expected] of [
+      [{ trailing: true }, 1],
+      [{ header: true }, 1],
+      [{ trailing: true, interiorBlank: true }, 0],
+      [{ extraBlank: true }, 0]
     ]) {
-      fs.writeFileSync(path.join(dest, 'src/csv-output.json'), JSON.stringify(output));
+      fs.writeFileSync(path.join(dest, 'src/csv-output.json'), JSON.stringify(options));
       const score = lib.grade(csv, dest).score;
-      if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(output));
-      else assert.ok(score < 1, JSON.stringify(output));
+      if (expected === 1) assert.strictEqual(score, 1, JSON.stringify(options));
+      else assert.ok(score < 1, JSON.stringify(options));
     }
   } finally { fs.rmSync(workRoot, { recursive: true, force: true }); }
 });