From 7b34638500dc0b9964555698e33e70dd37277a28 Mon Sep 17 00:00:00 2001
From: drakeo338 <paranoyouz@gmail.com>
Date: Sun, 11 Oct 2026 19:21:15 +0000
Subject: [PATCH] fix: raise on unparseable skill-comply classifier output

_parse_classification returned {} for garbage or non-dict output, so a failed classification looked like zero matched steps and graded as non-compliant. Raise ClassifierParseError (a RuntimeError, like the non-zero-rc path) instead.
---
 skills/skill-comply/scripts/classifier.py    | 14 ++++++---
 skills/skill-comply/tests/test_classifier.py | 32 ++++++++++++++++++++
 2 files changed, 42 insertions(+), 4 deletions(-)
 create mode 100644 skills/skill-comply/tests/test_classifier.py

diff --git a/skills/skill-comply/scripts/classifier.py b/skills/skill-comply/scripts/classifier.py
index 2e4207c82..fdec3b7dc 100644
--- a/skills/skill-comply/scripts/classifier.py
+++ b/skills/skill-comply/scripts/classifier.py
@@ -14,6 +14,10 @@ from scripts.parser import ComplianceSpec, ObservationEvent
 PROMPTS_DIR = Path(__file__).parent.parent / "prompts"
 
 
+class ClassifierParseError(RuntimeError):
+    """Raised when the classifier output cannot be parsed."""
+
+
 def classify_events(
     spec: ComplianceSpec,
     trace: list[ObservationEvent],
@@ -73,13 +77,15 @@ def _parse_classification(text: str) -> dict[str, list[int]]:
     try:
         parsed = json.loads(cleaned)
         if not isinstance(parsed, dict):
-            logger.warning("Classifier returned non-dict JSON: %s", type(parsed).__name__)
-            return {}
+            raise ClassifierParseError(
+                f"classifier returned non-dict JSON: {type(parsed).__name__}"
+            )
         return {
             k: [int(i) for i in v]
             for k, v in parsed.items()
             if isinstance(v, list)
         }
     except (json.JSONDecodeError, ValueError, TypeError) as e:
-        logger.warning("Failed to parse classification output: %s", e)
-        return {}
+        raise ClassifierParseError(
+            f"failed to parse classification output: {e}"
+        ) from e
diff --git a/skills/skill-comply/tests/test_classifier.py b/skills/skill-comply/tests/test_classifier.py
new file mode 100644
index 000000000..bcc44c905
--- /dev/null
+++ b/skills/skill-comply/tests/test_classifier.py
@@ -0,0 +1,32 @@
+"""Tests for scripts.classifier._parse_classification."""
+
+import pytest
+
+from scripts import classifier
+from scripts.classifier import _parse_classification
+
+
+class TestParseClassification:
+    def test_plain_json(self):
+        assert _parse_classification('{"a": [0, 2]}') == {"a": [0, 2]}
+
+    def test_fenced_json(self):
+        assert _parse_classification('```json\n{"a": [1]}\n```') == {"a": [1]}
+
+    def test_dict_without_list_values_is_empty(self):
+        assert _parse_classification('{"a": "x"}') == {}
+
+    def test_garbage_raises(self):
+        with pytest.raises(RuntimeError):
+            _parse_classification("sorry, I cannot classify these")
+
+    def test_non_dict_json_raises(self):
+        with pytest.raises(RuntimeError):
+            _parse_classification("[1, 2, 3]")
+
+    def test_non_integer_indices_raise(self):
+        with pytest.raises(RuntimeError):
+            _parse_classification('{"a": ["x"]}')
+
+    def test_is_runtime_error(self):
+        assert issubclass(classifier.ClassifierParseError, RuntimeError)