mirror of
https://github.com/affaan-m/ECC.git
synced 2026-10-19 22:41:45 +02:00
fix: isolate sandbox tests and clarify evaluator pipe input
This commit is contained in:
@@ -10,7 +10,7 @@
|
||||
"id": "hand-seeded",
|
||||
"source": "manual-curation-pending-regeneration"
|
||||
},
|
||||
"registryDigest": "6373302fba7684a29f4287625420fc4d7d2b3a989b23373e0f9a6f14e5c9b55e",
|
||||
"registryDigest": "80fbd629779375546bc17b0d5a49c08cb4e22491e03e80f0590ba75aedf9315c",
|
||||
"schemaVersion": 1,
|
||||
"triggers": {
|
||||
"skill:agent-runtime-gateway-smoke-test": [
|
||||
|
||||
@@ -398,7 +398,13 @@ def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Evaluate agent output against the 5-axis rubric"
|
||||
)
|
||||
parser.add_argument("--task", help="Task description (file path or inline text)")
|
||||
parser.add_argument(
|
||||
"--task",
|
||||
help=(
|
||||
"Task description (file path or inline text); when omitted with --output and non-TTY "
|
||||
"stdin, read task text until EOF and wait if the input remains open"
|
||||
),
|
||||
)
|
||||
parser.add_argument("--output", help="Agent output to evaluate (file path)")
|
||||
parser.add_argument("--interactive", action="store_true", help="Prompt for task and read output from stdin")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -398,7 +398,13 @@ def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Evaluate agent output against the 5-axis rubric"
|
||||
)
|
||||
parser.add_argument("--task", help="Task description (file path or inline text)")
|
||||
parser.add_argument(
|
||||
"--task",
|
||||
help=(
|
||||
"Task description (file path or inline text); when omitted with --output and non-TTY "
|
||||
"stdin, read task text until EOF and wait if the input remains open"
|
||||
),
|
||||
)
|
||||
parser.add_argument("--output", help="Agent output to evaluate (file path)")
|
||||
parser.add_argument("--interactive", action="store_true", help="Prompt for task and read output from stdin")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -12,7 +12,6 @@ from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
from scripts.runner import (
|
||||
SANDBOX_BASE,
|
||||
_parse_stream_json,
|
||||
_safe_sandbox_dir,
|
||||
_setup_sandbox,
|
||||
@@ -30,8 +29,18 @@ class _FakeScenario:
|
||||
setup_commands: tuple[str, ...] = ()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def isolated_sandbox_base(
|
||||
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr("scripts.runner.SANDBOX_BASE", tmp_path / "sandbox")
|
||||
|
||||
|
||||
class TestSafeSandboxDirectory:
|
||||
def test_empty_id_cannot_delete_the_shared_base(self) -> None:
|
||||
def test_empty_id_cannot_delete_the_shared_base(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
monkeypatch.setattr("scripts.runner.SANDBOX_BASE", tmp_path / "sandbox")
|
||||
with (
|
||||
patch("scripts.runner.Path.exists", return_value=True),
|
||||
patch("scripts.runner.shutil.rmtree") as remove,
|
||||
@@ -45,13 +54,21 @@ class TestSafeSandboxDirectory:
|
||||
mkdir.assert_not_called()
|
||||
execute.assert_not_called()
|
||||
|
||||
def test_resolved_base_is_not_a_scenario_directory(self) -> None:
|
||||
with patch("scripts.runner.Path.resolve", return_value=SANDBOX_BASE):
|
||||
def test_resolved_base_is_not_a_scenario_directory(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
base = tmp_path / "sandbox"
|
||||
monkeypatch.setattr("scripts.runner.SANDBOX_BASE", base)
|
||||
with patch("scripts.runner.Path.resolve", return_value=base):
|
||||
with pytest.raises(ValueError, match="sandbox"):
|
||||
_safe_sandbox_dir("alias")
|
||||
|
||||
def test_valid_id_keeps_existing_sanitization(self) -> None:
|
||||
assert _safe_sandbox_dir("scenario/one") == SANDBOX_BASE / "scenario_one"
|
||||
def test_valid_id_keeps_existing_sanitization(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
) -> None:
|
||||
base = tmp_path / "sandbox"
|
||||
monkeypatch.setattr("scripts.runner.SANDBOX_BASE", base)
|
||||
assert _safe_sandbox_dir("scenario/one") == base / "scenario_one"
|
||||
|
||||
def test_resolved_escape_is_still_rejected(
|
||||
self, tmp_path: Path, monkeypatch: pytest.MonkeyPatch
|
||||
|
||||
Reference in New Issue
Block a user