diff --git a/docker/gateguard-effectiveness/README.md b/docker/gateguard-effectiveness/README.md index c8886fea0..35ea5d1e8 100644 --- a/docker/gateguard-effectiveness/README.md +++ b/docker/gateguard-effectiveness/README.md @@ -37,9 +37,12 @@ cryptographic signature or proof of who ran the experiment. grader to score `reference` 1 and the start workspace, the `naive` trap and every `decoy-*` overlay below 1. A decoy has the right shape and the wrong content, so a grader that checks only the form fails the check. Both runners count only valid -trials: an attempt that ended in a provider error or timeout stays in -`results.jsonl` with its own transcript, is reported apart, and is retried on -resume. A gated trial whose hook never ran stops the run and is not recorded. +trials: authentication-related provider errors stop the run before the attempt +is appended to `results.jsonl`. The transcript is retained, but there is no +attempt record. Other provider-error or timeout attempts stay in `results.jsonl` +with their own transcripts, are reported apart, and are retried on resume. +A gated trial whose hook never ran also stops the run without a `results.jsonl` +record. ```bash node docker/gateguard-effectiveness/run-intent.js --check-graders diff --git a/docs/gateguard/question-effectiveness.md b/docs/gateguard/question-effectiveness.md index 7ac2d7355..7eddeb708 100644 --- a/docs/gateguard/question-effectiveness.md +++ b/docs/gateguard/question-effectiveness.md @@ -104,10 +104,12 @@ transcript and decision metrics from the hook; the questions asked; how many of the scenario's `evidence` patterns the agent stated in its own text (for example the three importers); turns, tokens, cost and wall time as reported by Claude; and provider errors or timeouts. The full transcript of each attempt is -kept in `transcripts/`. An attempt that ends in a provider error or timeout -stays in `results.jsonl` but is not a trial: summaries count valid trials only, +kept in `transcripts/`. An authentication-related provider error stops the run +before the attempt is appended to `results.jsonl`; its transcript is retained, +but there is no attempt record. Other provider-error or timeout attempts +stay in `results.jsonl` but are not trials: summaries count valid trials only, report the failed attempts apart, and a resumed run retries their keys. A gated -trial whose hook never ran is not recorded at all; the run stops instead. +trial whose hook never ran also has no `results.jsonl` record; the run stops instead. ## Analysis diff --git a/pi/core/skills/agent-self-evaluation/scripts/evaluate.py b/pi/core/skills/agent-self-evaluation/scripts/evaluate.py index afedd37a9..e231ac3ce 100755 --- a/pi/core/skills/agent-self-evaluation/scripts/evaluate.py +++ b/pi/core/skills/agent-self-evaluation/scripts/evaluate.py @@ -370,7 +370,8 @@ def _read_file_or_text(path: Optional[str], *, required: bool = False) -> Option with open(path) as f: return f.read() except OSError as error: - if error.errno not in (errno.ENOENT, errno.ENAMETOOLONG): + windows_inline_task = error.errno == errno.EINVAL and sys.platform == "win32" and not required + if error.errno not in (errno.ENOENT, errno.ENAMETOOLONG) and not windows_inline_task: raise if required: print(f"Error: output file '{path}' not found", file=sys.stderr) diff --git a/skills/agent-self-evaluation/scripts/evaluate.py b/skills/agent-self-evaluation/scripts/evaluate.py index afedd37a9..e231ac3ce 100755 --- a/skills/agent-self-evaluation/scripts/evaluate.py +++ b/skills/agent-self-evaluation/scripts/evaluate.py @@ -370,7 +370,8 @@ def _read_file_or_text(path: Optional[str], *, required: bool = False) -> Option with open(path) as f: return f.read() except OSError as error: - if error.errno not in (errno.ENOENT, errno.ENAMETOOLONG): + windows_inline_task = error.errno == errno.EINVAL and sys.platform == "win32" and not required + if error.errno not in (errno.ENOENT, errno.ENAMETOOLONG) and not windows_inline_task: raise if required: print(f"Error: output file '{path}' not found", file=sys.stderr) diff --git a/tests/test_agent_self_evaluation.py b/tests/test_agent_self_evaluation.py index 515256f96..bc31f0d34 100644 --- a/tests/test_agent_self_evaluation.py +++ b/tests/test_agent_self_evaluation.py @@ -8,22 +8,20 @@ import re import subprocess import sys from pathlib import Path -from types import SimpleNamespace +from types import ModuleType, SimpleNamespace +from typing import NoReturn import pytest SCRIPT = Path(__file__).resolve().parents[1] / "skills/agent-self-evaluation/scripts/evaluate.py" -SPEC = importlib.util.spec_from_file_location("agent_self_evaluation", SCRIPT) -evaluator = importlib.util.module_from_spec(SPEC) -sys.modules[SPEC.name] = evaluator -SPEC.loader.exec_module(evaluator) +PI_SCRIPT = SCRIPT.parents[3] / "pi/core/skills/agent-self-evaluation/scripts/evaluate.py" -def args(**overrides): +def args(**overrides: object) -> argparse.Namespace: return argparse.Namespace(**{"interactive": False, "task": None, "output": None, **overrides}) -def test_output_file_uses_piped_task(tmp_path, monkeypatch): +def test_output_file_uses_piped_task(evaluator: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: output = tmp_path / "response.txt" output.write_text("All tests passed.", encoding="utf-8") monkeypatch.setattr(sys, "stdin", io.StringIO("Add retry logic")) @@ -31,7 +29,7 @@ def test_output_file_uses_piped_task(tmp_path, monkeypatch): assert evaluator._read_input(args(output=str(output))) == ("Add retry logic", "All tests passed.") -def test_explicit_task_takes_precedence_over_pipe(tmp_path, monkeypatch): +def test_explicit_task_takes_precedence_over_pipe(evaluator: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: output = tmp_path / "response.txt" output.write_text("Answer", encoding="utf-8") stream = io.StringIO("Ignored pipe") @@ -41,7 +39,7 @@ def test_explicit_task_takes_precedence_over_pipe(tmp_path, monkeypatch): assert stream.tell() == 0 -def test_output_file_without_task_does_not_read_terminal(tmp_path, monkeypatch): +def test_output_file_without_task_does_not_read_terminal(evaluator: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: output = tmp_path / "response.txt" output.write_text("Answer", encoding="utf-8") monkeypatch.setattr(sys, "stdin", SimpleNamespace(isatty=lambda: True)) @@ -49,36 +47,36 @@ def test_output_file_without_task_does_not_read_terminal(tmp_path, monkeypatch): assert evaluator._read_input(args(output=str(output))) == (None, "Answer") -def test_no_output_file_keeps_stdin_as_output(monkeypatch): +def test_no_output_file_keeps_stdin_as_output(evaluator: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr(sys, "stdin", io.StringIO("Piped answer")) assert evaluator._read_input(args(task="Inline task")) == ("Inline task", "Piped answer") -def test_interactive_mode_keeps_prompt_and_stdin(monkeypatch): +def test_interactive_mode_keeps_prompt_and_stdin(evaluator: ModuleType, monkeypatch: pytest.MonkeyPatch) -> None: monkeypatch.setattr("builtins.input", lambda _prompt: " Interactive task ") monkeypatch.setattr(sys, "stdin", io.StringIO("Interactive answer")) assert evaluator._read_input(args(interactive=True)) == ("Interactive task", "Interactive answer") -def test_task_can_still_be_read_from_file(tmp_path, monkeypatch): +def test_task_can_still_be_read_from_file(evaluator: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: task = tmp_path / "task.txt" task.write_text("File task", encoding="utf-8") monkeypatch.setattr(sys, "stdin", io.StringIO("Piped answer")) assert evaluator._read_input(args(task=str(task))) == ("File task", "Piped answer") -def test_missing_output_file_still_fails(tmp_path, monkeypatch, capsys): +def test_missing_output_file_still_fails(evaluator: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch, capsys: pytest.CaptureFixture[str]) -> None: monkeypatch.setattr(sys, "stdin", io.StringIO("Task")) with pytest.raises(SystemExit, match="1"): evaluator._read_input(args(output=str(tmp_path / "missing.txt"))) assert "not found" in capsys.readouterr().err -def test_cli_uses_piped_task_for_conciseness_score(tmp_path): +def test_cli_uses_piped_task_for_conciseness_score(evaluator: ModuleType, tmp_path: Path) -> None: output = tmp_path / "response.txt" output.write_text("Verified with tests. " * 25, encoding="utf-8") result = subprocess.run( - [sys.executable, "-X", "utf8", str(SCRIPT), "--output", str(output)], + [sys.executable, "-X", "utf8", str(evaluator.__file__), "--output", str(output)], input="Fix bug", capture_output=True, encoding="utf-8", @@ -88,7 +86,7 @@ def test_cli_uses_piped_task_for_conciseness_score(tmp_path): assert re.search(r"Conciseness[^\n]*3/5", result.stdout) -def test_long_inline_task_remains_text(tmp_path, monkeypatch): +def test_long_inline_task_remains_text(evaluator: ModuleType, tmp_path: Path, monkeypatch: pytest.MonkeyPatch) -> None: output = tmp_path / "response.txt" output.write_text("Answer", encoding="utf-8") task = "Long explicit task description. " * 60 @@ -97,11 +95,58 @@ def test_long_inline_task_remains_text(tmp_path, monkeypatch): @pytest.mark.parametrize("error_code", [errno.EACCES, errno.EISDIR]) -def test_task_read_does_not_hide_other_file_errors(monkeypatch, error_code): - def fail_open(_path): +def test_task_read_does_not_hide_other_file_errors(evaluator: ModuleType, monkeypatch: pytest.MonkeyPatch, error_code: int) -> None: + def fail_open(_path: str) -> NoReturn: raise OSError(error_code, "Cannot read task") monkeypatch.setattr("builtins.open", fail_open) with pytest.raises(OSError) as captured: evaluator._read_file_or_text("task-file") assert captured.value.errno == error_code + + +@pytest.fixture(params=[SCRIPT, PI_SCRIPT], ids=["canonical", "pi"]) +def evaluator(request: pytest.FixtureRequest) -> ModuleType: + spec = importlib.util.spec_from_file_location("task_input_evaluator", request.param) + assert spec is not None and spec.loader is not None + module = importlib.util.module_from_spec(spec) + sys.modules[spec.name] = module + spec.loader.exec_module(module) + return module + + +@pytest.mark.parametrize("platform", ["win32", "linux", "darwin"]) +@pytest.mark.parametrize("required", [False, True], ids=["optional", "required"]) +@pytest.mark.parametrize( + "error_code", [errno.EINVAL, errno.ENOENT, errno.ENAMETOOLONG, errno.EACCES, errno.EISDIR, errno.EIO] +) +def test_file_error_fallback_is_platform_and_mode_scoped( + evaluator: ModuleType, + monkeypatch: pytest.MonkeyPatch, + capsys: pytest.CaptureFixture[str], + platform: str, + required: bool, + error_code: int, +) -> None: + task = "Long inline task. " * 60 + error = OSError(error_code, "Cannot read file") + + def fail_open(_path: str) -> NoReturn: + raise error + + monkeypatch.setattr(evaluator.sys, "platform", platform) + monkeypatch.setattr("builtins.open", fail_open) + can_fallback = error_code in (errno.ENOENT, errno.ENAMETOOLONG) or ( + error_code == errno.EINVAL and platform == "win32" and not required + ) + if can_fallback and not required: + assert evaluator._read_file_or_text(task) == task + elif can_fallback: + with pytest.raises(SystemExit) as captured_exit: + evaluator._read_file_or_text(task, required=True) + assert captured_exit.value.code == 1 + assert "not found" in capsys.readouterr().err + else: + with pytest.raises(OSError) as captured_error: + evaluator._read_file_or_text(task, required=required) + assert captured_error.value is error