diff --git a/manifests/context-packs/skill-triggers@1.json b/manifests/context-packs/skill-triggers@1.json index b02f9173d..3fc2738cd 100644 --- a/manifests/context-packs/skill-triggers@1.json +++ b/manifests/context-packs/skill-triggers@1.json @@ -10,7 +10,7 @@ "id": "hand-seeded", "source": "manual-curation-pending-regeneration" }, - "registryDigest": "47ac81f7220b4237db249d4e70e926b57bcf444ca6557102c0718aceef860ce0", + "registryDigest": "edfc258e6fdf863ed802aa985f5ae2b623b10b060a7c7c7bada329648f04031d", "schemaVersion": 1, "triggers": { "skill:agent-runtime-gateway-smoke-test": [ diff --git a/skills/skill-comply/scripts/run.py b/skills/skill-comply/scripts/run.py index 3e4695e51..e67fdcaf6 100644 --- a/skills/skill-comply/scripts/run.py +++ b/skills/skill-comply/scripts/run.py @@ -10,6 +10,7 @@ from typing import Any import yaml +from scripts.classifier import ClassifierParseError from scripts.grader import grade from scripts.report import generate_report from scripts.runner import run_scenario @@ -95,7 +96,14 @@ def main() -> None: for scenario in scenarios: logger.info(" Running %s...", scenario.level_name) run = run_scenario(scenario, model=args.model) - result = grade(spec, list(run.observations)) + try: + result = grade(spec, list(run.observations)) + except ClassifierParseError: + logger.error( + "ClassifierParseError: Classification failed; " + "invalid classifier output. No report generated." + ) + sys.exit(1) graded_results.append((scenario.level_name, result, list(run.observations))) logger.info(" %s: %.0f%%", scenario.level_name, result.compliance_rate * 100) diff --git a/skills/skill-comply/tests/test_run.py b/skills/skill-comply/tests/test_run.py new file mode 100644 index 000000000..f96cd1397 --- /dev/null +++ b/skills/skill-comply/tests/test_run.py @@ -0,0 +1,96 @@ +"""Tests for the CLI's classification failure boundary.""" + +from pathlib import Path +from types import SimpleNamespace +from unittest.mock import Mock + +import pytest +from scripts import run +from scripts.classifier import ClassifierParseError +from scripts.parser import parse_spec, parse_trace + +FIXTURES = Path(__file__).parent.parent / "fixtures" + + +@pytest.fixture +def cli(monkeypatch, tmp_path): + skill = tmp_path / "fixture.md" + skill.write_text("Read a file before editing it.\n") + output = tmp_path / "report.md" + spec = parse_spec(FIXTURES / "tdd_spec.yaml") + observations = tuple(parse_trace(FIXTURES / "compliant_trace.jsonl")) + scenarios = [ + SimpleNamespace(level_name=name, description="Controlled scenario") + for name in ("supportive", "neutral") + ] + monkeypatch.setattr("sys.argv", ["skill-comply", str(skill), "--output", str(output)]) + monkeypatch.setattr(run, "__file__", str(tmp_path / "scripts" / "run.py")) + monkeypatch.setattr(run, "generate_spec", Mock(return_value=spec)) + monkeypatch.setattr(run, "generate_scenarios", Mock(return_value=scenarios)) + execute = Mock(return_value=SimpleNamespace(observations=observations)) + monkeypatch.setattr(run, "run_scenario", execute) + report = Mock(return_value="Controlled compliance report\n") + monkeypatch.setattr(run, "generate_report", report) + return SimpleNamespace(output=output, report=report, execute=execute) + + +@pytest.mark.parametrize("failure_index", [0, 1]) +def test_classifier_failure_exits_without_report_or_raw_output( + cli, monkeypatch, caplog, failure_index +): + error = ClassifierParseError("raw-classifier-private-marker") + results = [SimpleNamespace(compliance_rate=1.0)] * failure_index + [error] + monkeypatch.setattr(run, "grade", Mock(side_effect=results)) + + with pytest.raises(SystemExit) as exited: + run.main() + + assert exited.value.code == 1 + assert "ClassifierParseError" in caplog.text + assert "Classification failed" in caplog.text + assert caplog.records[-1].message == ( + "ClassifierParseError: Classification failed; " + "invalid classifier output. No report generated." + ) + assert "raw-classifier-private-marker" not in caplog.text + assert "Traceback" not in caplog.text + assert "Overall Compliance" not in caplog.text + assert not cli.output.exists() + cli.report.assert_not_called() + assert cli.execute.call_count == failure_index + 1 + + +@pytest.mark.parametrize("error_type", [RuntimeError, ValueError, OSError]) +def test_unrelated_grading_exception_propagates(cli, monkeypatch, error_type): + error = error_type("unrelated failure") + monkeypatch.setattr(run, "grade", Mock(side_effect=error)) + + with pytest.raises(error_type) as raised: + run.main() + + assert raised.value is error + assert not cli.output.exists() + cli.report.assert_not_called() + + +def test_classifier_error_outside_grading_propagates(cli): + error = ClassifierParseError("execution failure") + cli.execute.side_effect = error + + with pytest.raises(ClassifierParseError) as raised: + run.main() + + assert raised.value is error + cli.report.assert_not_called() + + +def test_valid_positive_grading_writes_report(cli, monkeypatch, caplog): + monkeypatch.setattr(run, "grade", Mock(return_value=SimpleNamespace(compliance_rate=1.0))) + caplog.set_level("INFO") + + run.main() + + assert cli.output.read_text() == "Controlled compliance report\n" + cli.report.assert_called_once() + assert cli.execute.call_count == 2 + assert "Overall Compliance: 100%" in caplog.text