merge: reviewed classifier CLI failure boundary

manifests/context-packs/skill-triggers@1.json

@@ -10,7 +10,7 @@
     "id": "hand-seeded",
     "source": "manual-curation-pending-regeneration"
   },
-  "registryDigest": "47ac81f7220b4237db249d4e70e926b57bcf444ca6557102c0718aceef860ce0",
+  "registryDigest": "edfc258e6fdf863ed802aa985f5ae2b623b10b060a7c7c7bada329648f04031d",
   "schemaVersion": 1,
   "triggers": {
     "skill:agent-runtime-gateway-smoke-test": [

skills/skill-comply/scripts/run.py

@@ -10,6 +10,7 @@ from typing import Any
 
 import yaml
 
+from scripts.classifier import ClassifierParseError
 from scripts.grader import grade
 from scripts.report import generate_report
 from scripts.runner import run_scenario
@@ -95,7 +96,14 @@ def main() -> None:
     for scenario in scenarios:
         logger.info("       Running %s...", scenario.level_name)
         run = run_scenario(scenario, model=args.model)
-        result = grade(spec, list(run.observations))
+        try:
+            result = grade(spec, list(run.observations))
+        except ClassifierParseError:
+            logger.error(
+                "ClassifierParseError: Classification failed; "
+                "invalid classifier output. No report generated."
+            )
+            sys.exit(1)
         graded_results.append((scenario.level_name, result, list(run.observations)))
         logger.info("       %s: %.0f%%", scenario.level_name, result.compliance_rate * 100)
 

skills/skill-comply/tests/test_run.py

@@ -0,0 +1,96 @@
+"""Tests for the CLI's classification failure boundary."""
+
+from pathlib import Path
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+import pytest
+from scripts import run
+from scripts.classifier import ClassifierParseError
+from scripts.parser import parse_spec, parse_trace
+
+FIXTURES = Path(__file__).parent.parent / "fixtures"
+
+
+@pytest.fixture
+def cli(monkeypatch, tmp_path):
+    skill = tmp_path / "fixture.md"
+    skill.write_text("Read a file before editing it.\n")
+    output = tmp_path / "report.md"
+    spec = parse_spec(FIXTURES / "tdd_spec.yaml")
+    observations = tuple(parse_trace(FIXTURES / "compliant_trace.jsonl"))
+    scenarios = [
+        SimpleNamespace(level_name=name, description="Controlled scenario")
+        for name in ("supportive", "neutral")
+    ]
+    monkeypatch.setattr("sys.argv", ["skill-comply", str(skill), "--output", str(output)])
+    monkeypatch.setattr(run, "__file__", str(tmp_path / "scripts" / "run.py"))
+    monkeypatch.setattr(run, "generate_spec", Mock(return_value=spec))
+    monkeypatch.setattr(run, "generate_scenarios", Mock(return_value=scenarios))
+    execute = Mock(return_value=SimpleNamespace(observations=observations))
+    monkeypatch.setattr(run, "run_scenario", execute)
+    report = Mock(return_value="Controlled compliance report\n")
+    monkeypatch.setattr(run, "generate_report", report)
+    return SimpleNamespace(output=output, report=report, execute=execute)
+
+
+@pytest.mark.parametrize("failure_index", [0, 1])
+def test_classifier_failure_exits_without_report_or_raw_output(
+    cli, monkeypatch, caplog, failure_index
+):
+    error = ClassifierParseError("raw-classifier-private-marker")
+    results = [SimpleNamespace(compliance_rate=1.0)] * failure_index + [error]
+    monkeypatch.setattr(run, "grade", Mock(side_effect=results))
+
+    with pytest.raises(SystemExit) as exited:
+        run.main()
+
+    assert exited.value.code == 1
+    assert "ClassifierParseError" in caplog.text
+    assert "Classification failed" in caplog.text
+    assert caplog.records[-1].message == (
+        "ClassifierParseError: Classification failed; "
+        "invalid classifier output. No report generated."
+    )
+    assert "raw-classifier-private-marker" not in caplog.text
+    assert "Traceback" not in caplog.text
+    assert "Overall Compliance" not in caplog.text
+    assert not cli.output.exists()
+    cli.report.assert_not_called()
+    assert cli.execute.call_count == failure_index + 1
+
+
+@pytest.mark.parametrize("error_type", [RuntimeError, ValueError, OSError])
+def test_unrelated_grading_exception_propagates(cli, monkeypatch, error_type):
+    error = error_type("unrelated failure")
+    monkeypatch.setattr(run, "grade", Mock(side_effect=error))
+
+    with pytest.raises(error_type) as raised:
+        run.main()
+
+    assert raised.value is error
+    assert not cli.output.exists()
+    cli.report.assert_not_called()
+
+
+def test_classifier_error_outside_grading_propagates(cli):
+    error = ClassifierParseError("execution failure")
+    cli.execute.side_effect = error
+
+    with pytest.raises(ClassifierParseError) as raised:
+        run.main()
+
+    assert raised.value is error
+    cli.report.assert_not_called()
+
+
+def test_valid_positive_grading_writes_report(cli, monkeypatch, caplog):
+    monkeypatch.setattr(run, "grade", Mock(return_value=SimpleNamespace(compliance_rate=1.0)))
+    caplog.set_level("INFO")
+
+    run.main()
+
+    assert cli.output.read_text() == "Controlled compliance report\n"
+    cli.report.assert_called_once()
+    assert cli.execute.call_count == 2
+    assert "Overall Compliance: 100%" in caplog.text