diff --git a/backend/tests/test_api_outputs.py b/backend/tests/test_api_outputs.py new file mode 100644 index 00000000..a964b437 --- /dev/null +++ b/backend/tests/test_api_outputs.py @@ -0,0 +1,993 @@ +"""Integration tests for /api/outputs. + +Boots the real FastAPI app through the shared `client` fixture so every +route exercises auth middleware + lifespan. Anthropic + the model +registry are monkeypatched per-test for the LLM-driven endpoints. The +agent_manager-spawning endpoint reuses the existing `stub_agent_loop` +fixture so the real `launch_agent` runs (creating an in-memory session) +without spawning the SDK. + +Layout mirrors `outputs.py`: + - CRUD (/list, /create, /{id}, PUT, DELETE) + legacy migration + - Workspace seed / read / write / delete + - File serve (workspace + saved output, with token rewrite + _d + payload injection) + - Backend execute + - vibe-code + auto-run (LLM-mocked) + - auto-run-agent (stub_agent_loop + AgentConfig spy) + - Auth control mirroring test_api_agents.test_protected_route_requires_auth +""" + +from __future__ import annotations + +import base64 +import json +import os +import sys +from unittest.mock import AsyncMock, MagicMock + +import pytest + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _create_output(client, **overrides) -> dict: + """POST /create with sensible defaults, return the persisted output dict.""" + payload = { + "name": "Test Output", + "description": "test", + "input_schema": { + "type": "object", + "properties": {"x": {"type": "integer"}}, + "required": ["x"], + }, + "files": {"index.html": ""}, + } + payload.update(overrides) + resp = client.post("/api/outputs/create", json=payload) + assert resp.status_code == 200, resp.text + return resp.json()["output"] + + +def _make_mock_anthropic(text_response: str) -> MagicMock: + """Build a mock Anthropic client. + + The route does `await client.messages.create(...)` then reads + `resp.content[0].text`. Mirror that shape. + """ + fake_resp = MagicMock() + fake_resp.content = [MagicMock(text=text_response)] + client_mock = MagicMock() + client_mock.messages.create = AsyncMock(return_value=fake_resp) + return client_mock + + +def _patch_anthropic(monkeypatch, text_response: str) -> MagicMock: + """Hook `_get_anthropic_client` in outputs.py to return a mock that + responds with `text_response` on every messages.create call. + + Returns the inner mock client so tests can assert call args. + """ + from backend.apps.outputs import outputs as outputs_mod + + mock_client = _make_mock_anthropic(text_response) + monkeypatch.setattr(outputs_mod, "_get_anthropic_client", lambda: mock_client) + return mock_client + + +def _patch_aux_model(monkeypatch, model_id: str = "claude-haiku-fake") -> None: + """Stub `resolve_aux_model` on the registry so vibe-code/auto-run never + try to inspect the user's actual model connections.""" + from backend.apps.agents.providers import registry + + async def _fake(_settings, preferred_tier="haiku"): + return (model_id, None) + + monkeypatch.setattr(registry, "resolve_aux_model", _fake) + + +# --------------------------------------------------------------------------- +# CRUD + legacy migration +# --------------------------------------------------------------------------- + + +def test_list_empty_on_fresh_dir(client): + resp = client.get("/api/outputs/list") + assert resp.status_code == 200 + assert resp.json() == {"outputs": []} + + +def test_create_get_update_delete_round_trip(client): + created = _create_output(client, name="Alpha") + output_id = created["id"] + assert created["name"] == "Alpha" + + listed = client.get("/api/outputs/list").json()["outputs"] + assert any(o["id"] == output_id for o in listed) + + fetched = client.get(f"/api/outputs/{output_id}") + assert fetched.status_code == 200 + assert fetched.json()["name"] == "Alpha" + + upd = client.put( + f"/api/outputs/{output_id}", + json={ + "name": "Beta", + "auto_run_config": { + "enabled": True, + "prompt": "fetch X", + "mode": "agent", + "model": "sonnet", + }, + }, + ) + assert upd.status_code == 200 + body = upd.json()["output"] + assert body["name"] == "Beta" + assert body["auto_run_config"]["enabled"] is True + assert body["auto_run_config"]["prompt"] == "fetch X" + + deleted = client.delete(f"/api/outputs/{output_id}") + assert deleted.status_code == 200 + + from backend.config.paths import OUTPUTS_DIR + assert not os.path.exists(os.path.join(OUTPUTS_DIR, f"{output_id}.json")) + + gone = client.get(f"/api/outputs/{output_id}") + assert gone.status_code == 404 + + +def test_get_unknown_output_returns_404(client): + resp = client.get("/api/outputs/does-not-exist") + assert resp.status_code == 404 + + +def test_create_migrates_legacy_frontend_backend_code(client): + resp = client.post( + "/api/outputs/create", + json={ + "name": "Legacy", + "frontend_code": "old", + "backend_code": "result = {}", + }, + ) + assert resp.status_code == 200 + output = resp.json()["output"] + assert output["files"]["index.html"] == "old" + assert output["files"]["backend.py"] == "result = {}" + + +def test_update_unknown_output_returns_404(client): + resp = client.put("/api/outputs/missing", json={"name": "x"}) + assert resp.status_code == 404 + + +def test_delete_unknown_output_returns_404(client): + resp = client.delete("/api/outputs/missing") + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# Workspace seed +# --------------------------------------------------------------------------- + + +def test_workspace_seed_with_explicit_files(client): + from backend.config.paths import OUTPUTS_WORKSPACE_DIR + + workspace_id = "ws-explicit" + resp = client.post( + "/api/outputs/workspace/seed", + json={ + "workspace_id": workspace_id, + "files": { + "index.html": "seeded", + "schema.json": '{"type":"object"}', + }, + "meta": {"name": "X", "description": "y"}, + }, + ) + assert resp.status_code == 200 + + folder = os.path.join(OUTPUTS_WORKSPACE_DIR, workspace_id) + assert os.path.isfile(os.path.join(folder, "index.html")) + assert os.path.isfile(os.path.join(folder, "schema.json")) + assert os.path.isfile(os.path.join(folder, "SKILL.md")) + + with open(os.path.join(folder, "meta.json")) as f: + meta = json.load(f) + assert meta["name"] == "X" + + +def test_workspace_seed_empty_uses_default_template(client): + from backend.apps.outputs.view_builder_templates import VIEW_TEMPLATE_FILES + from backend.config.paths import OUTPUTS_WORKSPACE_DIR + + workspace_id = "ws-default" + resp = client.post( + "/api/outputs/workspace/seed", + json={"workspace_id": workspace_id}, + ) + assert resp.status_code == 200 + + folder = os.path.join(OUTPUTS_WORKSPACE_DIR, workspace_id) + for rel_path in VIEW_TEMPLATE_FILES: + assert os.path.isfile(os.path.join(folder, rel_path)), rel_path + assert os.path.isfile(os.path.join(folder, "SKILL.md")) + + +def test_workspace_seed_drops_path_traversal_keys(client): + """Keys that escape the workspace folder via `..` are silently + skipped (continue branch in seed_workspace).""" + from backend.config.paths import OUTPUTS_WORKSPACE_DIR + + workspace_id = "ws-traversal" + resp = client.post( + "/api/outputs/workspace/seed", + json={ + "workspace_id": workspace_id, + "files": { + "ok.txt": "kept", + "../escape.html": "should-not-write", + }, + }, + ) + assert resp.status_code == 200 + + folder = os.path.join(OUTPUTS_WORKSPACE_DIR, workspace_id) + assert os.path.isfile(os.path.join(folder, "ok.txt")) + parent = os.path.dirname(os.path.normpath(folder)) + assert not os.path.exists(os.path.join(parent, "escape.html")) + + +# --------------------------------------------------------------------------- +# Workspace read +# --------------------------------------------------------------------------- + + +def test_workspace_read_returns_files_and_meta(client): + workspace_id = "ws-read" + client.post( + "/api/outputs/workspace/seed", + json={ + "workspace_id": workspace_id, + "files": {"index.html": ""}, + "meta": {"name": "Read me"}, + }, + ) + + resp = client.get(f"/api/outputs/workspace/{workspace_id}") + assert resp.status_code == 200 + body = resp.json() + assert body["files"]["index.html"] == "" + assert body["meta"] == {"name": "Read me"} + assert body["path"].endswith(workspace_id) + + +def test_workspace_read_returns_none_meta_for_bad_meta_json(client): + """Garbage meta.json triggers the JSONDecodeError swallow branch + in `read_workspace`, returning meta=None.""" + from backend.config.paths import OUTPUTS_WORKSPACE_DIR + + workspace_id = "ws-bad-meta" + folder = os.path.join(OUTPUTS_WORKSPACE_DIR, workspace_id) + os.makedirs(folder, exist_ok=True) + with open(os.path.join(folder, "meta.json"), "w") as f: + f.write("{not valid json") + + resp = client.get(f"/api/outputs/workspace/{workspace_id}") + assert resp.status_code == 200 + assert resp.json()["meta"] is None + + +def test_workspace_read_missing_returns_404(client): + resp = client.get("/api/outputs/workspace/does-not-exist") + assert resp.status_code == 404 + + +# --------------------------------------------------------------------------- +# Workspace file write / delete +# --------------------------------------------------------------------------- + + +def test_workspace_write_and_delete_file(client): + from backend.config.paths import OUTPUTS_WORKSPACE_DIR + + workspace_id = "ws-write" + client.post("/api/outputs/workspace/seed", json={"workspace_id": workspace_id}) + + write = client.put( + f"/api/outputs/workspace/{workspace_id}/file/sub/dir/app.css", + json={"content": "body { color: red; }"}, + ) + assert write.status_code == 200 + assert write.json() == {"ok": True} + + full = os.path.join(OUTPUTS_WORKSPACE_DIR, workspace_id, "sub", "dir", "app.css") + assert os.path.isfile(full) + + delete = client.delete(f"/api/outputs/workspace/{workspace_id}/file/sub/dir/app.css") + assert delete.status_code == 200 + assert not os.path.exists(full) + # Empty parent dirs collapse up to the workspace root. + assert not os.path.exists(os.path.join(OUTPUTS_WORKSPACE_DIR, workspace_id, "sub")) + + +def test_workspace_write_traversal_rejected(client): + workspace_id = "ws-write-trav" + client.post("/api/outputs/workspace/seed", json={"workspace_id": workspace_id}) + + resp = client.put( + f"/api/outputs/workspace/{workspace_id}/file/..%2Fescape.html", + json={"content": "x"}, + ) + assert resp.status_code == 403 + + +def test_workspace_write_missing_workspace_404(client): + resp = client.put( + "/api/outputs/workspace/missing/file/foo.txt", + json={"content": "x"}, + ) + assert resp.status_code == 404 + + +def test_workspace_delete_traversal_rejected(client): + workspace_id = "ws-del-trav" + client.post("/api/outputs/workspace/seed", json={"workspace_id": workspace_id}) + + resp = client.delete(f"/api/outputs/workspace/{workspace_id}/file/..%2Fescape.html") + assert resp.status_code == 403 + + +def test_workspace_delete_missing_workspace_404(client): + resp = client.delete("/api/outputs/workspace/missing/file/foo.txt") + assert resp.status_code == 404 + + +def test_workspace_delete_missing_file_is_idempotent(client): + """DELETE on an existing workspace but missing file still returns + {"ok": True} (no-op branch).""" + workspace_id = "ws-del-idem" + client.post("/api/outputs/workspace/seed", json={"workspace_id": workspace_id}) + + resp = client.delete(f"/api/outputs/workspace/{workspace_id}/file/nope.txt") + assert resp.status_code == 200 + assert resp.json() == {"ok": True} + + +# --------------------------------------------------------------------------- +# Serve endpoints +# --------------------------------------------------------------------------- + + +def test_workspace_serve_non_html_is_raw(client): + workspace_id = "ws-serve-css" + client.post( + "/api/outputs/workspace/seed", + json={ + "workspace_id": workspace_id, + "files": {"app.css": "body { color: red; }"}, + }, + ) + resp = client.get(f"/api/outputs/workspace/{workspace_id}/serve/app.css") + assert resp.status_code == 200 + assert resp.text == "body { color: red; }" + assert resp.headers["content-type"].startswith("text/css") + + +def test_workspace_serve_index_html_injects_default_globals(client, auth_token): + workspace_id = "ws-serve-html" + html = ( + 'x' + '' + '' + "" + ) + client.post( + "/api/outputs/workspace/seed", + json={"workspace_id": workspace_id, "files": {"index.html": html}}, + ) + + resp = client.get(f"/api/outputs/workspace/{workspace_id}/serve/index.html") + assert resp.status_code == 200 + body = resp.text + assert "window.OUTPUT_INPUT = {}" in body + assert "window.OUTPUT_BACKEND_RESULT = null" in body + # Relative got the token; absolute ' + out = _inject_token_into_relative_urls(html, "tok123") + assert 'href="styles.css?token=tok123"' in out + assert 'src="app.js?token=tok123"' in out + + +def test_inject_token_appends_with_amp_when_query_present(): + html = '' + out = _inject_token_into_relative_urls(html, "tok") + assert 'src="app.js?v=1&token=tok"' in out + + +def test_inject_token_preserves_fragment(): + html = '' + out = _inject_token_into_relative_urls(html, "tok") + assert 'href="page.html?v=1&token=tok#sec"' in out + + +def test_inject_token_preserves_fragment_no_query(): + html = '' + out = _inject_token_into_relative_urls(html, "tok") + assert 'href="page.html?token=tok#sec"' in out + + +@pytest.mark.parametrize("prefix", _ABSOLUTE_URL_PREFIXES) +def test_inject_token_skips_absolute_urls(prefix): + """Every prefix in _ABSOLUTE_URL_PREFIXES must be left untouched.""" + url = f"{prefix}foo" + html = f'' + out = _inject_token_into_relative_urls(html, "tok") + assert f'src="{url}"' in out + assert "token=tok" not in out + + +def test_inject_token_skips_urls_with_existing_token(): + html = '' + out = _inject_token_into_relative_urls(html, "newtok") + assert 'href="styles.css?token=existing"' in out + assert "newtok" not in out + + +def test_inject_token_noop_when_token_empty(): + html = '' + assert _inject_token_into_relative_urls(html, "") == html + + +def test_inject_token_handles_single_quotes(): + html = "" + out = _inject_token_into_relative_urls(html, "tok") + assert "styles.css?token=tok" in out + + +def test_inject_token_requires_whitespace_before_attr(): + """The regex matches `\\shref=...`, so attr-like substrings without + leading whitespace are NOT touched (defensive: no false positives in + user-supplied JSON / inline scripts).""" + html = 'data-href="x.css"' + out = _inject_token_into_relative_urls(html, "tok") + assert out == html + + +# --------------------------------------------------------------------------- +# _decode_data_param +# --------------------------------------------------------------------------- + + +def test_decode_data_param_round_trip(): + payload = {"i": {"k": 1}, "r": {"v": 2}} + encoded = base64.b64encode(json.dumps(payload).encode()).decode() + input_json, result_json = _decode_data_param(encoded) + assert json.loads(input_json) == {"k": 1} + assert json.loads(result_json) == {"v": 2} + + +def test_decode_data_param_missing_keys_default(): + encoded = base64.b64encode(b"{}").decode() + input_json, result_json = _decode_data_param(encoded) + assert input_json == "{}" + assert result_json == "null" + + +def test_decode_data_param_malformed_returns_defaults(): + assert _decode_data_param("not-base64!") == ("{}", "null") + assert _decode_data_param("") == ("{}", "null") + + +# --------------------------------------------------------------------------- +# _walk_directory +# --------------------------------------------------------------------------- + + +def test_walk_directory_nonexistent_returns_empty(tmp_path): + assert _walk_directory(str(tmp_path / "nope")) == {} + + +def test_walk_directory_returns_relative_paths(tmp_path): + (tmp_path / "a.txt").write_text("A") + nested = tmp_path / "sub" / "deep" + nested.mkdir(parents=True) + (nested / "b.txt").write_text("B") + + result = _walk_directory(str(tmp_path)) + assert result["a.txt"] == "A" + assert result[os.path.join("sub", "deep", "b.txt")] == "B" + + +def test_walk_directory_skips_unreadable(tmp_path): + """Binary files that fail UTF-8 decode are silently skipped — the + `except Exception: pass` swallow path.""" + (tmp_path / "ok.txt").write_text("hello") + (tmp_path / "binary.dat").write_bytes(bytes([0xFF, 0xFE, 0x00, 0x80])) + + result = _walk_directory(str(tmp_path)) + assert result["ok.txt"] == "hello" + assert "binary.dat" not in result + + +# --------------------------------------------------------------------------- +# _load_all / _save / _load / load_output +# --------------------------------------------------------------------------- + + +def test_save_load_round_trip(tmp_data_dirs): + out = Output(name="round-trip", description="d", icon="x") + _save(out) + loaded = _load(out.id) + assert loaded.name == "round-trip" + assert loaded.description == "d" + assert loaded.id == out.id + + +def test_load_missing_raises_404(tmp_data_dirs): + with pytest.raises(HTTPException) as exc: + _load("does-not-exist") + assert exc.value.status_code == 404 + + +def test_load_output_returns_none_for_missing(tmp_data_dirs): + assert load_output("does-not-exist") is None + + +def test_load_output_returns_resolved(tmp_data_dirs): + out = Output(name="x") + _save(out) + fetched = load_output(out.id) + assert fetched is not None + assert fetched.name == "x" + + +def test_load_all_picks_up_saved(tmp_data_dirs): + a = Output(name="a") + b = Output(name="b") + _save(a) + _save(b) + names = sorted(o.name for o in _load_all()) + assert names == ["a", "b"] + + +def test_load_all_empty_when_dir_missing(monkeypatch, tmp_path): + """If DATA_DIR doesn't exist, _load_all returns [].""" + monkeypatch.setattr(outputs_mod, "DATA_DIR", str(tmp_path / "nope")) + assert _load_all() == [] + + +# --------------------------------------------------------------------------- +# Models — legacy field migration + properties +# --------------------------------------------------------------------------- + + +def test_output_migrates_frontend_and_backend_code(): + out = Output( + name="legacy", + frontend_code="x", + backend_code="result = {}", + ) + assert out.files == { + "index.html": "x", + "backend.py": "result = {}", + } + assert out.frontend_code == "x" + assert out.backend_code == "result = {}" + + +def test_output_already_has_files_drops_legacy_fields(): + out = Output( + name="ok", + files={"index.html": "

kept

"}, + frontend_code="", + backend_code="dropped", + ) + assert out.files == {"index.html": "

kept

"} + + +def test_output_frontend_backend_properties_default_to_empty(): + out = Output(name="empty") + assert out.frontend_code == "" + assert out.backend_code is None + + +def test_output_create_migrates_legacy_fields(): + create = OutputCreate( + name="x", + frontend_code="", + backend_code="result = {}", + ) + assert create.files["index.html"] == "" + assert create.files["backend.py"] == "result = {}" + + +def test_output_update_partial_excludes_none(): + upd = OutputUpdate(name="renamed") + dumped = upd.model_dump(exclude_none=True) + assert dumped == {"name": "renamed"} + + +def test_output_update_migrates_legacy_fields(): + upd = OutputUpdate(frontend_code="") + dumped = upd.model_dump(exclude_none=True) + assert dumped["files"] == {"index.html": ""} + + +def test_workspace_seed_migrates_schema_json_field(): + seed = WorkspaceSeedRequest( + workspace_id="ws-1", + frontend_code="", + backend_code="result = {}", + schema_json='{"type":"object"}', + ) + assert seed.files is not None + assert seed.files["index.html"] == "" + assert seed.files["backend.py"] == "result = {}" + assert seed.files["schema.json"] == '{"type":"object"}' + + +def test_workspace_seed_files_already_set_drops_legacy(): + seed = WorkspaceSeedRequest( + workspace_id="ws-2", + files={"index.html": ""}, + frontend_code="", + ) + assert seed.files == {"index.html": ""} + + +def test_auto_run_config_defaults(): + cfg = AutoRunConfig() + assert cfg.enabled is False + assert cfg.mode == "agent" + assert cfg.model == "sonnet" + assert cfg.context_paths == [] + assert cfg.forced_tools == [] + + +# --------------------------------------------------------------------------- +# executor.execute_backend_code +# --------------------------------------------------------------------------- + + +async def test_execute_backend_happy_path(): + code = "result['x'] = input_data['y'] + 1" + res = await execute_backend_code(code, {"y": 41}) + assert isinstance(res, BackendExecResult) + assert res.result == {"x": 42} + assert res.stdout == "" + + +async def test_execute_backend_captures_stdout(): + code = "print('hello world'); result['ok'] = True" + res = await execute_backend_code(code, {}) + assert res.result == {"ok": True} + assert "hello world" in res.stdout + + +async def test_execute_backend_syntax_error_raises(): + """SyntaxError during compile bubbles up as RuntimeError with the + nonzero exit code.""" + with pytest.raises(RuntimeError) as exc: + await execute_backend_code("def : bad", {}) + assert "Backend code error" in str(exc.value) + + +async def test_execute_backend_runtime_error_raises(): + with pytest.raises(RuntimeError) as exc: + await execute_backend_code("raise ValueError('boom')", {}) + msg = str(exc.value) + assert "Backend code error" in msg + assert "ValueError" in msg or "boom" in msg + + +async def test_execute_backend_timeout(monkeypatch): + from backend.apps.outputs import executor as exec_mod + + monkeypatch.setattr(exec_mod, "TIMEOUT_SECONDS", 0.1) + with pytest.raises(RuntimeError) as exc: + await execute_backend_code("import time; time.sleep(5)", {}) + assert "timed out" in str(exc.value) + + +async def test_execute_backend_non_json_output(): + """Corrupt the stdout JSON by writing extra bytes BEFORE the + postamble's json.dump runs. Subprocess exits 0 but stdout no + longer parses → JSONDecodeError → RuntimeError 'did not produce + valid JSON'.""" + code = ( + "_orig_stdout.write('not json prefix ')\n" + "_orig_stdout.flush()\n" + ) + with pytest.raises(RuntimeError) as exc: + await execute_backend_code(code, {}) + assert "did not produce valid JSON" in str(exc.value)