mirror of
https://github.com/openswarm-ai/openswarm.git
synced 2026-09-27 20:14:49 +02:00
Merge eric/v2 into arnav/tests
Brings in service layer refactor, frontend updates, session state additions, telemetry unification, and UI hook refinements. Resolved .gitignore conflict by combining coverage report entries (arnav/tests) with pyc/cache/editor noise entries (eric/v2). Co-authored-by: Cursor <cursoragent@cursor.com>
This commit is contained in:
Arnav Naval
@@ -1,971 +0,0 @@
|
||||
"""Comprehensive stress tests for PostHog analytics events.
|
||||
|
||||
Tests every analytics event fires correctly with proper properties.
|
||||
Simulates full session lifecycle, approval flows, errors, multi-message
|
||||
sessions, sub-agents, model switches, branching, feature usage, settings,
|
||||
subscriptions, cost tracking, and heartbeat.
|
||||
|
||||
Run with:
|
||||
cd backend && python -m pytest tests/test_analytics.py -v
|
||||
"""
|
||||
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tempfile
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import AsyncMock, MagicMock, patch, call
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Patch PostHog and settings BEFORE importing application modules
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
# Create a temp dir for settings/sessions
|
||||
_tmpdir = tempfile.mkdtemp()
|
||||
os.environ.setdefault("OPENSWARM_DATA_DIR", _tmpdir)
|
||||
|
||||
# Patch PostHog globally
|
||||
_captured_events: list[dict] = []
|
||||
|
||||
|
||||
def _mock_capture(event_type, distinct_id, properties=None):
|
||||
_captured_events.append({
|
||||
"event": event_type,
|
||||
"distinct_id": distinct_id,
|
||||
"properties": properties or {},
|
||||
})
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_captured_events():
|
||||
_captured_events.clear()
|
||||
yield
|
||||
_captured_events.clear()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_posthog():
|
||||
"""Mock PostHog so no real events are sent."""
|
||||
mock_ph = MagicMock()
|
||||
mock_ph.capture = _mock_capture
|
||||
|
||||
import backend.apps.analytics.collector as collector
|
||||
old_ph = collector._posthog
|
||||
old_id = collector._installation_id
|
||||
collector._posthog = mock_ph
|
||||
collector._installation_id = "test-install-id"
|
||||
yield mock_ph
|
||||
collector._posthog = old_ph
|
||||
collector._installation_id = old_id
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_settings(tmp_path):
|
||||
"""Mock settings to avoid reading real config."""
|
||||
settings_file = tmp_path / "settings.json"
|
||||
settings_file.write_text(json.dumps({
|
||||
"analytics_opt_in": True,
|
||||
"installation_id": "test-install-id",
|
||||
}))
|
||||
|
||||
import backend.apps.settings.settings as settings_mod
|
||||
old_file = settings_mod.SETTINGS_FILE
|
||||
settings_mod.SETTINGS_FILE = str(settings_file)
|
||||
yield
|
||||
settings_mod.SETTINGS_FILE = old_file
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_sessions_dir(tmp_path):
|
||||
"""Use temp dir for session persistence."""
|
||||
sessions_dir = tmp_path / "sessions"
|
||||
sessions_dir.mkdir()
|
||||
|
||||
import backend.config.paths as paths_mod
|
||||
old_dir = paths_mod.SESSIONS_DIR
|
||||
paths_mod.SESSIONS_DIR = str(sessions_dir)
|
||||
yield str(sessions_dir)
|
||||
paths_mod.SESSIONS_DIR = old_dir
|
||||
|
||||
|
||||
def events(event_type: str | None = None) -> list[dict]:
|
||||
"""Return captured events, optionally filtered by type."""
|
||||
if event_type:
|
||||
return [e for e in _captured_events if e["event"] == event_type]
|
||||
return list(_captured_events)
|
||||
|
||||
|
||||
def last_event(event_type: str) -> dict:
|
||||
"""Return the last captured event of a given type."""
|
||||
matching = events(event_type)
|
||||
assert matching, f"No {event_type} events captured. Got: {[e['event'] for e in _captured_events]}"
|
||||
return matching[-1]
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Import application modules (after patches are set up)
|
||||
# ===========================================================================
|
||||
from backend.apps.analytics.collector import record
|
||||
from backend.apps.agents.models import AgentConfig, AgentSession, Message, ApprovalRequest
|
||||
from backend.apps.agents.agent_manager import AgentManager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manager():
|
||||
"""Create a fresh AgentManager for each test."""
|
||||
mgr = AgentManager()
|
||||
return mgr
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 1. record() basics
|
||||
# ===========================================================================
|
||||
|
||||
class TestRecordBasics:
|
||||
def test_record_sends_event(self):
|
||||
record("test.event", {"key": "value"})
|
||||
e = last_event("test.event")
|
||||
assert e["properties"]["key"] == "value"
|
||||
assert e["distinct_id"] == "test-install-id"
|
||||
|
||||
def test_record_adds_os_and_platform(self):
|
||||
record("test.event", {})
|
||||
e = last_event("test.event")
|
||||
assert "os" in e["properties"]
|
||||
assert "platform" in e["properties"]
|
||||
|
||||
def test_record_includes_session_id(self):
|
||||
record("test.event", {}, session_id="sess123")
|
||||
e = last_event("test.event")
|
||||
assert e["properties"]["session_id"] == "sess123"
|
||||
|
||||
def test_record_includes_dashboard_id(self):
|
||||
record("test.event", {}, dashboard_id="dash456")
|
||||
e = last_event("test.event")
|
||||
assert e["properties"]["dashboard_id"] == "dash456"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 2. session.started fires ONCE on launch
|
||||
# ===========================================================================
|
||||
|
||||
class TestSessionStarted:
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_started_fires_on_launch(self, manager):
|
||||
config = AgentConfig(name="Test", model="sonnet", mode="agent", provider="anthropic")
|
||||
session = await manager.launch_agent(config)
|
||||
|
||||
e = last_event("session.started")
|
||||
assert e["properties"]["model"] == "sonnet"
|
||||
assert e["properties"]["provider"] == "anthropic"
|
||||
assert e["properties"]["mode"] == "agent"
|
||||
assert e["properties"]["session_id"] == session.id
|
||||
assert isinstance(e["properties"]["tool_count"], int)
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_started_fires_only_once(self, manager):
|
||||
config = AgentConfig(name="Test", model="sonnet", mode="agent")
|
||||
await manager.launch_agent(config)
|
||||
|
||||
started_events = events("session.started")
|
||||
assert len(started_events) == 1
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 3. session.completed fires ONCE on close (NOT per message)
|
||||
# ===========================================================================
|
||||
|
||||
class TestSessionCompleted:
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_completed_fires_on_close(self, manager):
|
||||
config = AgentConfig(name="Test Session", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
|
||||
# Add some messages to simulate activity
|
||||
session.messages.append(Message(role="user", content="hello"))
|
||||
session.messages.append(Message(role="assistant", content="hi there"))
|
||||
session.cost_usd = 0.05
|
||||
session.tokens = {"input": 1000, "output": 500}
|
||||
session.status = "completed"
|
||||
|
||||
await manager.close_session(session.id)
|
||||
|
||||
e = last_event("session.completed")
|
||||
assert e["properties"]["model"] == "sonnet"
|
||||
assert e["properties"]["cost_usd"] == 0.05
|
||||
assert e["properties"]["message_count"] == 2
|
||||
assert e["properties"]["input_tokens"] == 1000
|
||||
assert e["properties"]["output_tokens"] == 500
|
||||
assert e["properties"]["session_title"] == "Test Session"
|
||||
assert e["properties"]["branch_count"] == 1 # main branch
|
||||
assert e["properties"]["is_sub_agent"] is False
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_completed_fires_exactly_once(self, manager):
|
||||
config = AgentConfig(name="Test", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
session.status = "completed"
|
||||
|
||||
await manager.close_session(session.id)
|
||||
|
||||
completed_events = events("session.completed")
|
||||
assert len(completed_events) == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_completed_includes_sub_agent_info(self, manager):
|
||||
# Create parent session
|
||||
config = AgentConfig(name="Parent", model="sonnet", mode="agent")
|
||||
parent = await manager.launch_agent(config)
|
||||
|
||||
# Create child session
|
||||
child = AgentSession(
|
||||
id=uuid4().hex, name="Child", mode="browser-agent",
|
||||
parent_session_id=parent.id, status="completed",
|
||||
)
|
||||
manager.sessions[child.id] = child
|
||||
|
||||
parent.status = "completed"
|
||||
await manager.close_session(parent.id)
|
||||
|
||||
e = last_event("session.completed")
|
||||
assert e["properties"]["sub_agent_count"] == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_completed_on_shutdown(self, manager):
|
||||
config = AgentConfig(name="Shutdown Test", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
session.cost_usd = 0.10
|
||||
|
||||
await manager.persist_all_sessions()
|
||||
|
||||
e = last_event("session.completed")
|
||||
assert e["properties"]["cost_usd"] == 0.10
|
||||
assert e["properties"]["session_title"] == "Shutdown Test"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 4. session.error
|
||||
# ===========================================================================
|
||||
|
||||
class TestSessionError:
|
||||
def test_session_error_event_structure(self):
|
||||
record("session.error", {
|
||||
"error_type": "ValueError",
|
||||
"error_message": "test error",
|
||||
"model": "sonnet",
|
||||
"provider": "anthropic",
|
||||
"mode": "agent",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("session.error")
|
||||
assert e["properties"]["error_type"] == "ValueError"
|
||||
assert e["properties"]["error_message"] == "test error"
|
||||
assert e["properties"]["model"] == "sonnet"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 5. tool.executed
|
||||
# ===========================================================================
|
||||
|
||||
class TestToolExecuted:
|
||||
def test_builtin_tool(self):
|
||||
record("tool.executed", {
|
||||
"tool_name": "Bash",
|
||||
"tool_short_name": "Bash",
|
||||
"tool_type": "builtin",
|
||||
"mcp_server": "",
|
||||
"duration_ms": 150,
|
||||
"success": True,
|
||||
"model": "sonnet",
|
||||
"provider": "anthropic",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("tool.executed")
|
||||
assert e["properties"]["tool_type"] == "builtin"
|
||||
assert e["properties"]["mcp_server"] == ""
|
||||
assert e["properties"]["tool_short_name"] == "Bash"
|
||||
|
||||
def test_mcp_tool_extracts_server_name(self):
|
||||
record("tool.executed", {
|
||||
"tool_name": "mcp__google-workspace__searchGmail",
|
||||
"tool_short_name": "searchGmail",
|
||||
"tool_type": "mcp",
|
||||
"mcp_server": "google-workspace",
|
||||
"duration_ms": 2000,
|
||||
"success": True,
|
||||
"model": "sonnet",
|
||||
"provider": "anthropic",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("tool.executed")
|
||||
assert e["properties"]["tool_type"] == "mcp"
|
||||
assert e["properties"]["mcp_server"] == "google-workspace"
|
||||
assert e["properties"]["tool_short_name"] == "searchGmail"
|
||||
|
||||
def test_tool_failure_tracked(self):
|
||||
record("tool.executed", {
|
||||
"tool_name": "Bash",
|
||||
"tool_short_name": "Bash",
|
||||
"tool_type": "builtin",
|
||||
"mcp_server": "",
|
||||
"duration_ms": 50,
|
||||
"success": False,
|
||||
"model": "sonnet",
|
||||
"provider": "anthropic",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("tool.executed")
|
||||
assert e["properties"]["success"] is False
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 6. approval.requested + approval.resolved
|
||||
# ===========================================================================
|
||||
|
||||
class TestApprovalEvents:
|
||||
def test_approval_requested(self):
|
||||
record("approval.requested", {
|
||||
"tool_name": "Bash",
|
||||
"is_first_approval_in_session": True,
|
||||
"model": "sonnet",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("approval.requested")
|
||||
assert e["properties"]["tool_name"] == "Bash"
|
||||
assert e["properties"]["is_first_approval_in_session"] is True
|
||||
|
||||
def test_approval_resolved_allow(self):
|
||||
record("approval.resolved", {
|
||||
"tool_name": "Bash",
|
||||
"decision": "allow",
|
||||
"latency_ms": 1500,
|
||||
"input_was_modified": False,
|
||||
"model": "sonnet",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("approval.resolved")
|
||||
assert e["properties"]["decision"] == "allow"
|
||||
assert e["properties"]["latency_ms"] == 1500
|
||||
assert e["properties"]["input_was_modified"] is False
|
||||
|
||||
def test_approval_resolved_deny(self):
|
||||
record("approval.resolved", {
|
||||
"tool_name": "Bash",
|
||||
"decision": "deny",
|
||||
"latency_ms": 500,
|
||||
"input_was_modified": False,
|
||||
"model": "sonnet",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("approval.resolved")
|
||||
assert e["properties"]["decision"] == "deny"
|
||||
|
||||
def test_approval_with_modified_input(self):
|
||||
record("approval.resolved", {
|
||||
"tool_name": "Bash",
|
||||
"decision": "allow",
|
||||
"latency_ms": 3000,
|
||||
"input_was_modified": True,
|
||||
"model": "sonnet",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("approval.resolved")
|
||||
assert e["properties"]["input_was_modified"] is True
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 7. turn.completed
|
||||
# ===========================================================================
|
||||
|
||||
class TestTurnCompleted:
|
||||
def test_turn_completed(self):
|
||||
record("turn.completed", {
|
||||
"turn_number": 3,
|
||||
"tool_calls_in_turn": 2,
|
||||
"model": "sonnet",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("turn.completed")
|
||||
assert e["properties"]["turn_number"] == 3
|
||||
assert e["properties"]["tool_calls_in_turn"] == 2
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 8. model.switched
|
||||
# ===========================================================================
|
||||
|
||||
class TestModelSwitched:
|
||||
@pytest.mark.asyncio
|
||||
async def test_model_switch_fires_event(self, manager):
|
||||
config = AgentConfig(name="Test", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
session.messages.append(Message(role="user", content="msg1"))
|
||||
session.cost_usd = 0.03
|
||||
|
||||
# Simulate model switch via send_message (which we can't fully run
|
||||
# without SDK, so test the record call directly)
|
||||
record("model.switched", {
|
||||
"from_model": "sonnet",
|
||||
"to_model": "opus",
|
||||
"from_provider": "anthropic",
|
||||
"to_provider": "anthropic",
|
||||
"message_number": 1,
|
||||
"cost_so_far": 0.03,
|
||||
}, session_id=session.id)
|
||||
|
||||
e = last_event("model.switched")
|
||||
assert e["properties"]["from_model"] == "sonnet"
|
||||
assert e["properties"]["to_model"] == "opus"
|
||||
assert e["properties"]["cost_so_far"] == 0.03
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 9. session.resumed
|
||||
# ===========================================================================
|
||||
|
||||
class TestSessionResumed:
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_resumed(self, manager, mock_sessions_dir):
|
||||
# Create and close a session
|
||||
config = AgentConfig(name="Resume Test", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
session.messages.append(Message(role="user", content="hello"))
|
||||
session.cost_usd = 0.05
|
||||
session.status = "completed"
|
||||
await manager.close_session(session.id)
|
||||
|
||||
_captured_events.clear()
|
||||
|
||||
# Resume it
|
||||
resumed = await manager.resume_session(session.id)
|
||||
|
||||
e = last_event("session.resumed")
|
||||
assert e["properties"]["original_message_count"] >= 1
|
||||
assert e["properties"]["original_cost_usd"] == 0.05
|
||||
assert e["properties"]["model"] == "sonnet"
|
||||
assert "hours_since_closed" in e["properties"]
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 10. context.attached
|
||||
# ===========================================================================
|
||||
|
||||
class TestContextAttached:
|
||||
def test_context_with_files(self):
|
||||
record("context.attached", {
|
||||
"file_count": 3,
|
||||
"directory_count": 1,
|
||||
"skill_count": 0,
|
||||
"image_count": 2,
|
||||
"has_forced_tools": True,
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("context.attached")
|
||||
assert e["properties"]["file_count"] == 3
|
||||
assert e["properties"]["image_count"] == 2
|
||||
assert e["properties"]["has_forced_tools"] is True
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 11. session.first_message
|
||||
# ===========================================================================
|
||||
|
||||
class TestSessionFirstMessage:
|
||||
def test_first_message_properties(self):
|
||||
prompt = "```python\nprint('hello')\n```\nCheck https://example.com"
|
||||
record("session.first_message", {
|
||||
"message_length": len(prompt),
|
||||
"has_code_block": "```" in prompt,
|
||||
"has_url": "http://" in prompt or "https://" in prompt,
|
||||
"model": "sonnet",
|
||||
"mode": "agent",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("session.first_message")
|
||||
assert e["properties"]["has_code_block"] is True
|
||||
assert e["properties"]["has_url"] is True
|
||||
assert e["properties"]["message_length"] > 0
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 12. feature.used (all variants)
|
||||
# ===========================================================================
|
||||
|
||||
class TestFeatureUsed:
|
||||
@pytest.mark.parametrize("feature", [
|
||||
"message.branched",
|
||||
"mode.switched",
|
||||
"skill.used",
|
||||
"skill.created",
|
||||
"template.created",
|
||||
"template.used",
|
||||
"view.created",
|
||||
"vibe_code.used",
|
||||
"browser_agent.launched",
|
||||
])
|
||||
def test_feature_used_variants(self, feature):
|
||||
record("feature.used", {"feature": feature}, session_id="s1")
|
||||
e = last_event("feature.used")
|
||||
assert e["properties"]["feature"] == feature
|
||||
|
||||
def test_branch_created_with_depth(self):
|
||||
record("feature.used", {
|
||||
"feature": "message.branched",
|
||||
"branch_depth": 2,
|
||||
"total_branches_in_session": 3,
|
||||
"messages_before_fork": 5,
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("feature.used")
|
||||
assert e["properties"]["branch_depth"] == 2
|
||||
assert e["properties"]["total_branches_in_session"] == 3
|
||||
|
||||
def test_mode_switch_details(self):
|
||||
record("feature.used", {
|
||||
"feature": "mode.switched",
|
||||
"from_mode": "agent",
|
||||
"to_mode": "view-builder",
|
||||
}, session_id="s1")
|
||||
|
||||
e = last_event("feature.used")
|
||||
assert e["properties"]["from_mode"] == "agent"
|
||||
assert e["properties"]["to_mode"] == "view-builder"
|
||||
|
||||
def test_browser_agent_with_task_count(self):
|
||||
record("feature.used", {
|
||||
"feature": "browser_agent.launched",
|
||||
"task_count": 3,
|
||||
"model": "sonnet",
|
||||
})
|
||||
|
||||
e = last_event("feature.used")
|
||||
assert e["properties"]["task_count"] == 3
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 13. subscription events
|
||||
# ===========================================================================
|
||||
|
||||
class TestSubscriptionEvents:
|
||||
def test_subscription_connected(self):
|
||||
record("subscription.connected", {"provider": "anthropic"})
|
||||
e = last_event("subscription.connected")
|
||||
assert e["properties"]["provider"] == "anthropic"
|
||||
|
||||
def test_subscription_disconnected(self):
|
||||
record("subscription.disconnected", {"provider": "openai"})
|
||||
e = last_event("subscription.disconnected")
|
||||
assert e["properties"]["provider"] == "openai"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 14. provider.configured + settings.changed
|
||||
# ===========================================================================
|
||||
|
||||
class TestSettingsEvents:
|
||||
def test_provider_added(self):
|
||||
record("provider.configured", {
|
||||
"provider": "anthropic",
|
||||
"action": "added",
|
||||
})
|
||||
e = last_event("provider.configured")
|
||||
assert e["properties"]["action"] == "added"
|
||||
|
||||
def test_provider_removed(self):
|
||||
record("provider.configured", {
|
||||
"provider": "openai",
|
||||
"action": "removed",
|
||||
})
|
||||
e = last_event("provider.configured")
|
||||
assert e["properties"]["action"] == "removed"
|
||||
|
||||
def test_settings_changed(self):
|
||||
record("settings.changed", {
|
||||
"changed_keys": ["theme", "default_model", "zoom_sensitivity"],
|
||||
})
|
||||
e = last_event("settings.changed")
|
||||
assert "theme" in e["properties"]["changed_keys"]
|
||||
assert len(e["properties"]["changed_keys"]) == 3
|
||||
|
||||
def test_settings_changed_excludes_secrets(self):
|
||||
# Verify that if we track changed keys, secret keys are excluded
|
||||
record("settings.changed", {
|
||||
"changed_keys": ["theme"],
|
||||
})
|
||||
e = last_event("settings.changed")
|
||||
for secret in ["anthropic_api_key", "openai_api_key", "google_api_key",
|
||||
"openrouter_api_key", "copilot_github_token"]:
|
||||
assert secret not in e["properties"]["changed_keys"]
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 15. cost.snapshot
|
||||
# ===========================================================================
|
||||
|
||||
class TestCostSnapshot:
|
||||
def test_cost_snapshot_structure(self):
|
||||
record("cost.snapshot", {
|
||||
"total_cost_usd": 42.50,
|
||||
"total_prompt_tokens": 500000,
|
||||
"total_completion_tokens": 150000,
|
||||
"total_requests": 250,
|
||||
})
|
||||
|
||||
e = last_event("cost.snapshot")
|
||||
assert e["properties"]["total_cost_usd"] == 42.50
|
||||
assert e["properties"]["total_prompt_tokens"] == 500000
|
||||
assert e["properties"]["total_completion_tokens"] == 150000
|
||||
assert e["properties"]["total_requests"] == 250
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 16. app.heartbeat
|
||||
# ===========================================================================
|
||||
|
||||
class TestAppHeartbeat:
|
||||
def test_heartbeat_structure(self):
|
||||
record("app.heartbeat", {
|
||||
"active_session_count": 3,
|
||||
"nine_router_total_cost": 100.50,
|
||||
"nine_router_total_prompt_tokens": 1000000,
|
||||
"nine_router_total_completion_tokens": 300000,
|
||||
"nine_router_total_requests": 500,
|
||||
})
|
||||
|
||||
e = last_event("app.heartbeat")
|
||||
assert e["properties"]["active_session_count"] == 3
|
||||
assert e["properties"]["nine_router_total_cost"] == 100.50
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 17. app.opened (enhanced)
|
||||
# ===========================================================================
|
||||
|
||||
class TestAppOpened:
|
||||
def test_app_opened_structure(self):
|
||||
record("app.opened", {
|
||||
"os": "Darwin",
|
||||
"platform": "macOS-14.0",
|
||||
"provider_count": 2,
|
||||
"providers": ["anthropic", "openai"],
|
||||
"is_first_open": False,
|
||||
"days_since_install": 5,
|
||||
"app_version": "1.0.17",
|
||||
})
|
||||
|
||||
e = last_event("app.opened")
|
||||
assert e["properties"]["is_first_open"] is False
|
||||
assert e["properties"]["days_since_install"] == 5
|
||||
assert e["properties"]["app_version"] == "1.0.17"
|
||||
assert e["properties"]["provider_count"] == 2
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 18. Multi-message session does NOT fire session.completed multiple times
|
||||
# ===========================================================================
|
||||
|
||||
class TestMultiMessageSession:
|
||||
@pytest.mark.asyncio
|
||||
async def test_no_session_completed_per_message(self, manager):
|
||||
"""Verify session.completed does NOT fire when agent loop finishes.
|
||||
It should only fire on close_session() or persist_all_sessions()."""
|
||||
config = AgentConfig(name="Multi-msg", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
|
||||
# Simulate 3 message exchanges
|
||||
for i in range(3):
|
||||
session.messages.append(Message(role="user", content=f"msg {i}"))
|
||||
session.messages.append(Message(role="assistant", content=f"reply {i}"))
|
||||
|
||||
# At this point, no session.completed should have fired
|
||||
completed = events("session.completed")
|
||||
assert len(completed) == 0, f"session.completed fired {len(completed)} times before close!"
|
||||
|
||||
# Now close — exactly 1 session.completed
|
||||
session.status = "completed"
|
||||
await manager.close_session(session.id)
|
||||
|
||||
completed = events("session.completed")
|
||||
assert len(completed) == 1, f"Expected 1 session.completed, got {len(completed)}"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 19. Token tracking
|
||||
# ===========================================================================
|
||||
|
||||
class TestTokenTracking:
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokens_in_session_completed(self, manager):
|
||||
config = AgentConfig(name="Token Test", model="opus", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
|
||||
# Simulate SDK token reporting
|
||||
session.tokens = {"input": 50000, "output": 15000}
|
||||
session.cost_usd = 0.25
|
||||
session.status = "completed"
|
||||
|
||||
await manager.close_session(session.id)
|
||||
|
||||
e = last_event("session.completed")
|
||||
assert e["properties"]["input_tokens"] == 50000
|
||||
assert e["properties"]["output_tokens"] == 15000
|
||||
assert e["properties"]["cost_usd"] == 0.25
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 20. Full lifecycle integration test
|
||||
# ===========================================================================
|
||||
|
||||
class TestFullLifecycle:
|
||||
@pytest.mark.asyncio
|
||||
async def test_complete_session_lifecycle(self, manager):
|
||||
"""Simulate a complete user session: launch, messages, close."""
|
||||
# 1. Launch
|
||||
config = AgentConfig(
|
||||
name="Full Lifecycle",
|
||||
model="sonnet",
|
||||
mode="agent",
|
||||
provider="anthropic",
|
||||
dashboard_id="dash-001",
|
||||
)
|
||||
session = await manager.launch_agent(config)
|
||||
assert len(events("session.started")) == 1
|
||||
|
||||
# 2. Simulate messages
|
||||
session.messages.append(Message(role="user", content="Hello, help me code"))
|
||||
session.messages.append(Message(role="assistant", content="Sure, let me help"))
|
||||
session.messages.append(Message(
|
||||
role="tool_call",
|
||||
content={"tool": "Bash", "input": {"command": "ls"}},
|
||||
))
|
||||
session.messages.append(Message(
|
||||
role="tool_result",
|
||||
content={"text": "file1.py\nfile2.py", "tool_name": "Bash", "elapsed_ms": 50},
|
||||
))
|
||||
session.messages.append(Message(role="user", content="Now run tests"))
|
||||
session.messages.append(Message(role="assistant", content="Running tests..."))
|
||||
|
||||
session.cost_usd = 0.08
|
||||
session.tokens = {"input": 20000, "output": 5000}
|
||||
|
||||
# 3. No session.completed yet
|
||||
assert len(events("session.completed")) == 0
|
||||
|
||||
# 4. Close
|
||||
session.status = "completed"
|
||||
await manager.close_session(session.id)
|
||||
|
||||
# 5. Verify session.completed
|
||||
e = last_event("session.completed")
|
||||
assert e["properties"]["message_count"] == 4 # 2 user + 2 assistant
|
||||
assert e["properties"]["tool_count"] == 1 # 1 tool call
|
||||
assert "Bash" in e["properties"]["tools_list"]
|
||||
assert e["properties"]["cost_usd"] == 0.08
|
||||
assert e["properties"]["input_tokens"] == 20000
|
||||
assert e["properties"]["output_tokens"] == 5000
|
||||
assert e["properties"]["dashboard_id"] == "dash-001"
|
||||
assert e["properties"]["first_user_message"] == "Hello, help me code"
|
||||
assert e["properties"]["duration_seconds"] >= 0 # may be 0 in fast tests
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_with_error(self, manager):
|
||||
"""Verify error sessions still fire session.completed on close."""
|
||||
config = AgentConfig(name="Error Test", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
session.status = "error"
|
||||
|
||||
await manager.close_session(session.id)
|
||||
|
||||
e = last_event("session.completed")
|
||||
assert e["properties"]["status"] == "error"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_with_branches(self, manager):
|
||||
"""Verify branch count in session.completed."""
|
||||
config = AgentConfig(name="Branch Test", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
|
||||
# Simulate branching
|
||||
from backend.apps.agents.models import MessageBranch
|
||||
session.branches["branch-1"] = MessageBranch(id="branch-1", parent_branch_id="main")
|
||||
session.branches["branch-2"] = MessageBranch(id="branch-2", parent_branch_id="branch-1")
|
||||
|
||||
session.status = "completed"
|
||||
await manager.close_session(session.id)
|
||||
|
||||
e = last_event("session.completed")
|
||||
assert e["properties"]["branch_count"] == 3 # main + branch-1 + branch-2
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 21. MCP server name extraction in tool.executed
|
||||
# ===========================================================================
|
||||
|
||||
class TestMCPServerExtraction:
|
||||
def test_standard_mcp_format(self):
|
||||
"""Test mcp__server-name__tool_name format."""
|
||||
import re
|
||||
tool_name = "mcp__google-workspace__searchGmail"
|
||||
m = re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name)
|
||||
assert m is not None
|
||||
assert m.group(1) == "google-workspace"
|
||||
assert m.group(2) == "searchGmail"
|
||||
|
||||
def test_builtin_tool_no_server(self):
|
||||
import re
|
||||
tool_name = "Bash"
|
||||
m = re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name)
|
||||
assert m is None
|
||||
|
||||
def test_browser_agent_mcp_format(self):
|
||||
import re
|
||||
tool_name = "mcp__openswarm-browser-agent__CreateBrowserAgent"
|
||||
m = re.match(r"mcp__([^_]+(?:-[^_]+)*)__(.+)", tool_name)
|
||||
assert m is not None
|
||||
assert m.group(1) == "openswarm-browser-agent"
|
||||
assert m.group(2) == "CreateBrowserAgent"
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 22. Settings update tracking
|
||||
# ===========================================================================
|
||||
|
||||
class TestSettingsUpdateTracking:
|
||||
@pytest.mark.asyncio
|
||||
async def test_provider_key_change_detected(self):
|
||||
"""Test that adding an API key fires provider.configured."""
|
||||
from backend.apps.settings.models import AppSettings
|
||||
|
||||
old = AppSettings(anthropic_api_key=None)
|
||||
new = AppSettings(anthropic_api_key="sk-test-key")
|
||||
|
||||
# Simulate what update_settings does
|
||||
provider_keys = {
|
||||
"anthropic_api_key": "anthropic",
|
||||
"openai_api_key": "openai",
|
||||
"google_api_key": "gemini",
|
||||
"openrouter_api_key": "openrouter",
|
||||
}
|
||||
for key, provider_name in provider_keys.items():
|
||||
old_val = bool(getattr(old, key, None))
|
||||
new_val = bool(getattr(new, key, None))
|
||||
if old_val != new_val:
|
||||
record("provider.configured", {
|
||||
"provider": provider_name,
|
||||
"action": "added" if new_val else "removed",
|
||||
})
|
||||
|
||||
e = last_event("provider.configured")
|
||||
assert e["properties"]["provider"] == "anthropic"
|
||||
assert e["properties"]["action"] == "added"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_settings_change_excludes_secrets(self):
|
||||
"""Verify secret keys are not included in changed_keys."""
|
||||
from backend.apps.settings.models import AppSettings
|
||||
|
||||
old = AppSettings(theme="dark", anthropic_api_key="old-key")
|
||||
new = AppSettings(theme="light", anthropic_api_key="new-key")
|
||||
|
||||
old_dict = old.model_dump()
|
||||
new_dict = new.model_dump()
|
||||
secret_keys = {"anthropic_api_key", "openai_api_key", "google_api_key",
|
||||
"openrouter_api_key", "claude_subscription_token",
|
||||
"openai_subscription_token", "gemini_subscription_token",
|
||||
"copilot_github_token", "copilot_token", "installation_id"}
|
||||
safe_changed = [
|
||||
k for k in new_dict
|
||||
if k in old_dict and new_dict[k] != old_dict[k] and k not in secret_keys
|
||||
]
|
||||
|
||||
assert "theme" in safe_changed
|
||||
assert "anthropic_api_key" not in safe_changed
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 23. Cost snapshot accuracy
|
||||
# ===========================================================================
|
||||
|
||||
class TestCostSnapshotAccuracy:
|
||||
def test_nine_router_cost_in_heartbeat(self):
|
||||
"""Verify heartbeat includes 9Router cost data."""
|
||||
record("app.heartbeat", {
|
||||
"active_session_count": 2,
|
||||
"nine_router_total_cost": 235.50,
|
||||
"nine_router_total_prompt_tokens": 5000000,
|
||||
"nine_router_total_completion_tokens": 1500000,
|
||||
"nine_router_total_requests": 1200,
|
||||
"cost_model_claude_sonnet_4_20250514": 180.00,
|
||||
"cost_model_claude_opus_4_20250514": 55.50,
|
||||
})
|
||||
|
||||
e = last_event("app.heartbeat")
|
||||
assert e["properties"]["nine_router_total_cost"] == 235.50
|
||||
assert e["properties"]["cost_model_claude_sonnet_4_20250514"] == 180.00
|
||||
|
||||
def test_cost_snapshot_separate_event(self):
|
||||
"""Verify cost.snapshot fires independently with accurate totals."""
|
||||
record("cost.snapshot", {
|
||||
"total_cost_usd": 235.50,
|
||||
"total_prompt_tokens": 5000000,
|
||||
"total_completion_tokens": 1500000,
|
||||
"total_requests": 1200,
|
||||
})
|
||||
|
||||
e = last_event("cost.snapshot")
|
||||
assert e["properties"]["total_cost_usd"] == 235.50
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# 24. Edge cases
|
||||
# ===========================================================================
|
||||
|
||||
class TestEdgeCases:
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_session_with_no_messages(self, manager):
|
||||
"""Session closed without any messages should still fire session.completed."""
|
||||
config = AgentConfig(name="Empty", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
session.status = "completed"
|
||||
|
||||
await manager.close_session(session.id)
|
||||
|
||||
e = last_event("session.completed")
|
||||
assert e["properties"]["message_count"] == 0
|
||||
assert e["properties"]["tool_count"] == 0
|
||||
assert e["properties"]["first_user_message"] == ""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_close_session_with_zero_cost(self, manager):
|
||||
"""Session with 0 cost should still report cost_usd=0."""
|
||||
config = AgentConfig(name="Free", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
session.status = "completed"
|
||||
|
||||
await manager.close_session(session.id)
|
||||
|
||||
e = last_event("session.completed")
|
||||
assert e["properties"]["cost_usd"] == 0.0
|
||||
assert e["properties"]["input_tokens"] == 0
|
||||
assert e["properties"]["output_tokens"] == 0
|
||||
|
||||
def test_record_with_no_posthog(self):
|
||||
"""record() should not crash if PostHog is not initialized."""
|
||||
import backend.apps.analytics.collector as collector
|
||||
old_ph = collector._posthog
|
||||
collector._posthog = None
|
||||
|
||||
# Should not raise
|
||||
record("test.event", {"key": "value"})
|
||||
|
||||
collector._posthog = old_ph
|
||||
|
||||
def test_record_with_none_properties(self):
|
||||
"""record() handles None properties gracefully."""
|
||||
record("test.event", None)
|
||||
e = last_event("test.event")
|
||||
assert "os" in e["properties"] # system props still added
|
||||
@@ -1,27 +1,8 @@
|
||||
"""Stress tests for the Phase 1 / 2 / 3 perceived-latency changes.
|
||||
|
||||
Hits everything we touched on the eric/v2 branch:
|
||||
"""Stress tests for live perceived-latency paths.
|
||||
|
||||
- Message.client_message_id round-trip (optimistic dedupe)
|
||||
- Mode migration: 'chat' -> 'ask' on session reconcile + lifespan
|
||||
deletion of stale built-in chat.json
|
||||
- ContentBlock + StreamEvent now accept type='thinking' /
|
||||
delta_type='thinking_delta' without breaking existing types
|
||||
- Anthropic provider forwards thinking content_block_start /
|
||||
content_block_delta with the right shape
|
||||
- Agent loop emits agent:stream_start{role:'thinking'},
|
||||
agent:stream_delta, agent:stream_end for thinking blocks AND
|
||||
persists a Message(role='thinking') after stream end
|
||||
- DashboardLayout serializes notes round-trip
|
||||
- exclude_dynamic_sections reaches the SDK kwargs (presence-only;
|
||||
we don't run the real CLI here)
|
||||
|
||||
Each test runs many randomized iterations to surface race conditions
|
||||
and bad assumptions. Stub the network and CLI throughout — these
|
||||
tests are pure logic, no real Anthropic calls.
|
||||
|
||||
Run:
|
||||
cd backend && .venv/bin/python -m pytest tests/test_phase1_stress.py -v
|
||||
- Mode migration: 'chat' -> 'ask' on reconcile + lifespan deletion
|
||||
- DashboardLayout notes round-trip
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
@@ -213,288 +194,6 @@ def test_reconcile_idempotent():
|
||||
assert mtime_after_first == mtime_after_second, "reconcile must be idempotent"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Group 3 — ContentBlock / StreamEvent thinking acceptance
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
def test_content_block_thinking_type():
|
||||
from backend.apps.agents.providers.base import ContentBlock
|
||||
|
||||
cb = ContentBlock(type="thinking", text="some reasoning")
|
||||
assert cb.type == "thinking"
|
||||
assert cb.text == "some reasoning"
|
||||
assert cb.tool_call is None
|
||||
|
||||
|
||||
def test_stream_event_thinking_delta():
|
||||
from backend.apps.agents.providers.base import StreamEvent
|
||||
|
||||
e = StreamEvent(type="content_block_delta", delta_type="thinking_delta", text="hmm")
|
||||
assert e.delta_type == "thinking_delta"
|
||||
assert e.text == "hmm"
|
||||
|
||||
# Existing types still work — no regression
|
||||
e2 = StreamEvent(type="content_block_delta", delta_type="text_delta", text="hi")
|
||||
assert e2.delta_type == "text_delta"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Group 4 — Anthropic provider thinking forwarding
|
||||
#
|
||||
# We feed a fake raw_stream (mimicking the SDK's async generator) through
|
||||
# AnthropicProvider.stream_message and confirm the right StreamEvents come
|
||||
# out. No network.
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
class _FakeRawEvent:
|
||||
def __init__(self, **kwargs):
|
||||
for k, v in kwargs.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
class _FakeBlock:
|
||||
def __init__(self, **kwargs):
|
||||
for k, v in kwargs.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
class _FakeDelta:
|
||||
def __init__(self, **kwargs):
|
||||
for k, v in kwargs.items():
|
||||
setattr(self, k, v)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_anthropic_provider_forwards_thinking_blocks():
|
||||
"""Mock the raw Anthropic stream with a thinking block + thinking_delta
|
||||
+ content_block_stop, and assert AnthropicProvider yields the
|
||||
normalized StreamEvents the agent_loop expects."""
|
||||
from backend.apps.agents.providers.anthropic import AnthropicProvider
|
||||
|
||||
raw_events = [
|
||||
# thinking block opens at index 0
|
||||
_FakeRawEvent(type="content_block_start", index=0,
|
||||
content_block=_FakeBlock(type="thinking")),
|
||||
_FakeRawEvent(type="content_block_delta", index=0,
|
||||
delta=_FakeDelta(type="thinking_delta", thinking="step 1, ")),
|
||||
_FakeRawEvent(type="content_block_delta", index=0,
|
||||
delta=_FakeDelta(type="thinking_delta", thinking="step 2.")),
|
||||
# signature_delta on thinking — must be ignored, not crash
|
||||
_FakeRawEvent(type="content_block_delta", index=0,
|
||||
delta=_FakeDelta(type="signature_delta", signature="abc==")),
|
||||
_FakeRawEvent(type="content_block_stop", index=0),
|
||||
# text block follows at index 1
|
||||
_FakeRawEvent(type="content_block_start", index=1,
|
||||
content_block=_FakeBlock(type="text")),
|
||||
_FakeRawEvent(type="content_block_delta", index=1,
|
||||
delta=_FakeDelta(type="text_delta", text="hi")),
|
||||
_FakeRawEvent(type="content_block_stop", index=1),
|
||||
]
|
||||
|
||||
async def fake_stream():
|
||||
for ev in raw_events:
|
||||
yield ev
|
||||
|
||||
# AnthropicProvider takes api_key/auth_token/base_url; we monkeypatch
|
||||
# its `client.messages.create` after construction so no real
|
||||
# SDK client is needed.
|
||||
provider = AnthropicProvider(api_key="test-key")
|
||||
provider.client.messages.create = AsyncMock(return_value=fake_stream())
|
||||
out_events = []
|
||||
async for ev in provider.stream_message(model="sonnet", system=None, messages=[], tools=[]):
|
||||
out_events.append(ev)
|
||||
|
||||
types = [(e.type, e.block_type, e.delta_type) for e in out_events]
|
||||
# Thinking block should produce: start, 2x delta, stop. signature_delta ignored.
|
||||
assert ("content_block_start", "thinking", "") in types
|
||||
assert types.count(("content_block_delta", "", "thinking_delta")) == 2
|
||||
assert ("content_block_start", "text", "") in types
|
||||
assert ("content_block_delta", "", "text_delta") in types
|
||||
|
||||
thinking_text = "".join(
|
||||
e.text for e in out_events
|
||||
if e.type == "content_block_delta" and e.delta_type == "thinking_delta"
|
||||
)
|
||||
assert thinking_text == "step 1, step 2."
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Group 5 — Agent loop end-to-end thinking → WS events + persisted message
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_loop_emits_thinking_stream_and_persists_message():
|
||||
"""Drive the agent loop with a fake provider that yields thinking,
|
||||
text, and one tool_use. Verify it emits the right WS events AND
|
||||
persists a Message(role='thinking') via _emit_collected_messages."""
|
||||
from backend.apps.agents.providers.base import StreamEvent
|
||||
|
||||
captured_ws: list[tuple[str, dict]] = []
|
||||
|
||||
async def fake_emitter(event: str, payload: dict):
|
||||
captured_ws.append((event, payload))
|
||||
|
||||
# Build a fake provider yielding our normalized StreamEvents.
|
||||
class FakeProvider:
|
||||
async def stream_message(self, **kwargs):
|
||||
yield StreamEvent(type="content_block_start", index=0, block_type="thinking")
|
||||
yield StreamEvent(type="content_block_delta", index=0,
|
||||
delta_type="thinking_delta", text="reasoning… ")
|
||||
yield StreamEvent(type="content_block_delta", index=0,
|
||||
delta_type="thinking_delta", text="more.")
|
||||
yield StreamEvent(type="content_block_stop", index=0)
|
||||
yield StreamEvent(type="content_block_start", index=1, block_type="text")
|
||||
yield StreamEvent(type="content_block_delta", index=1,
|
||||
delta_type="text_delta", text="hello!")
|
||||
yield StreamEvent(type="content_block_stop", index=1)
|
||||
yield StreamEvent(type="message_stop")
|
||||
|
||||
from backend.apps.agents.agent_loop import AgentLoop
|
||||
|
||||
loop = AgentLoop(
|
||||
session_id="s1",
|
||||
provider=FakeProvider(),
|
||||
model="sonnet",
|
||||
system_prompt="x",
|
||||
tools=[],
|
||||
ws_emitter=fake_emitter,
|
||||
hitl_handler=AsyncMock(return_value=(True, None)),
|
||||
tool_executor=AsyncMock(return_value=[{"type": "text", "text": "ok"}]),
|
||||
)
|
||||
|
||||
response = await loop._stream_and_collect()
|
||||
|
||||
# Stream events: thinking start + 2 deltas + stream_end, then text start + delta + (text end at message_stop)
|
||||
events_by_type = {}
|
||||
for ev, payload in captured_ws:
|
||||
events_by_type.setdefault(ev, []).append(payload)
|
||||
|
||||
# Thinking should have its own stream_start with role='thinking'
|
||||
starts = events_by_type.get("agent:stream_start", [])
|
||||
thinking_starts = [s for s in starts if s.get("role") == "thinking"]
|
||||
assistant_starts = [s for s in starts if s.get("role") == "assistant"]
|
||||
assert len(thinking_starts) == 1, f"expected 1 thinking start, got {len(thinking_starts)}"
|
||||
assert len(assistant_starts) == 1, "expected 1 assistant text start"
|
||||
|
||||
# Two thinking deltas
|
||||
deltas = events_by_type.get("agent:stream_delta", [])
|
||||
thinking_msg_id = thinking_starts[0]["message_id"]
|
||||
thinking_deltas = [d for d in deltas if d.get("message_id") == thinking_msg_id]
|
||||
assert len(thinking_deltas) == 2
|
||||
assert "".join(d["delta"] for d in thinking_deltas) == "reasoning… more."
|
||||
|
||||
# Thinking stream_end fires (text doesn't get stream_end inside _stream_and_collect — closes at message_stop)
|
||||
ends = events_by_type.get("agent:stream_end", [])
|
||||
assert any(e["message_id"] == thinking_msg_id for e in ends), "thinking must emit stream_end"
|
||||
|
||||
# Now persist via _emit_collected_messages and verify a thinking
|
||||
# Message went out
|
||||
captured_ws.clear()
|
||||
await loop._emit_collected_messages(
|
||||
response.content,
|
||||
text_msg_id=assistant_starts[0]["message_id"],
|
||||
tool_msg_ids={},
|
||||
)
|
||||
persisted = [p for ev, p in captured_ws if ev == "agent:message"]
|
||||
roles = [p["message"]["role"] for p in persisted]
|
||||
assert "thinking" in roles, "thinking content must be persisted as a Message"
|
||||
assert "assistant" in roles
|
||||
thinking_msg = next(p for p in persisted if p["message"]["role"] == "thinking")
|
||||
assert thinking_msg["message"]["content"] == "reasoning… more."
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_loop_handles_no_thinking_gracefully():
|
||||
"""Provider that emits zero thinking blocks must still work.
|
||||
Regression guard against the new branch breaking text-only paths."""
|
||||
from backend.apps.agents.providers.base import StreamEvent
|
||||
from backend.apps.agents.agent_loop import AgentLoop
|
||||
|
||||
captured_ws = []
|
||||
|
||||
async def fake_emitter(event, payload):
|
||||
captured_ws.append((event, payload))
|
||||
|
||||
class TextOnly:
|
||||
async def stream_message(self, **kwargs):
|
||||
yield StreamEvent(type="content_block_start", index=0, block_type="text")
|
||||
yield StreamEvent(type="content_block_delta", index=0,
|
||||
delta_type="text_delta", text="just text")
|
||||
yield StreamEvent(type="content_block_stop", index=0)
|
||||
yield StreamEvent(type="message_stop")
|
||||
|
||||
loop = AgentLoop(
|
||||
session_id="s2", provider=TextOnly(), model="sonnet", system_prompt=None,
|
||||
tools=[],
|
||||
ws_emitter=fake_emitter,
|
||||
hitl_handler=AsyncMock(return_value=(True, None)),
|
||||
tool_executor=AsyncMock(return_value=[]),
|
||||
)
|
||||
|
||||
resp = await loop._stream_and_collect()
|
||||
starts = [p for ev, p in captured_ws if ev == "agent:stream_start"]
|
||||
# Exactly one assistant start, zero thinking starts
|
||||
assert len([s for s in starts if s.get("role") == "thinking"]) == 0
|
||||
assert len([s for s in starts if s.get("role") == "assistant"]) == 1
|
||||
assert any(b.type == "text" for b in resp.content)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_agent_loop_stress_many_thinking_blocks():
|
||||
"""Hammer the loop with a long sequence of interleaved thinking +
|
||||
text + tool blocks. Ensures the per-index buffers don't leak and
|
||||
every block gets the right WS events."""
|
||||
from backend.apps.agents.providers.base import StreamEvent
|
||||
from backend.apps.agents.agent_loop import AgentLoop
|
||||
|
||||
captured = []
|
||||
|
||||
async def fake_emitter(ev, p):
|
||||
captured.append((ev, p))
|
||||
|
||||
class Mix:
|
||||
async def stream_message(self, **kwargs):
|
||||
idx = 0
|
||||
for turn in range(40):
|
||||
yield StreamEvent(type="content_block_start", index=idx, block_type="thinking")
|
||||
for _ in range(random.randint(1, 5)):
|
||||
yield StreamEvent(type="content_block_delta", index=idx,
|
||||
delta_type="thinking_delta", text=f"t{idx} ")
|
||||
yield StreamEvent(type="content_block_stop", index=idx)
|
||||
idx += 1
|
||||
yield StreamEvent(type="content_block_start", index=idx, block_type="text")
|
||||
yield StreamEvent(type="content_block_delta", index=idx,
|
||||
delta_type="text_delta", text=f"text-{idx}")
|
||||
yield StreamEvent(type="content_block_stop", index=idx)
|
||||
idx += 1
|
||||
yield StreamEvent(type="message_stop")
|
||||
|
||||
loop = AgentLoop(
|
||||
session_id="s3", provider=Mix(), model="sonnet", system_prompt=None,
|
||||
tools=[],
|
||||
ws_emitter=fake_emitter,
|
||||
hitl_handler=AsyncMock(return_value=(True, None)),
|
||||
tool_executor=AsyncMock(return_value=[]),
|
||||
)
|
||||
resp = await loop._stream_and_collect()
|
||||
|
||||
starts = [p for ev, p in captured if ev == "agent:stream_start"]
|
||||
ends = [p for ev, p in captured if ev == "agent:stream_end"]
|
||||
|
||||
# 40 thinking + 1 assistant (text accumulates into one stream_text_msg_id)
|
||||
thinking_starts = [s for s in starts if s.get("role") == "thinking"]
|
||||
assistant_starts = [s for s in starts if s.get("role") == "assistant"]
|
||||
assert len(thinking_starts) == 40, f"got {len(thinking_starts)} thinking starts, want 40"
|
||||
assert len(assistant_starts) == 1, "all text blocks share one assistant stream id"
|
||||
|
||||
# Each thinking block must have its own stream_end
|
||||
thinking_ids = {s["message_id"] for s in thinking_starts}
|
||||
end_ids = {e["message_id"] for e in ends}
|
||||
assert thinking_ids.issubset(end_ids), "every thinking block needs a stream_end"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Group 6 — Notes layout serialization
|
||||
|
||||
@@ -0,0 +1,353 @@
|
||||
"""Tests for the service-sync layer.
|
||||
|
||||
Public surface is a single `sync(data)` function. The desktop hands off
|
||||
opaque dicts; the cloud determines what they are. Tests verify:
|
||||
|
||||
- Envelope (install_id, user_id) stamped on every submission
|
||||
- Opt-out gate works
|
||||
- Test sink intercepts every sync
|
||||
- Spool round-trip (enqueue/drain/acknowledge)
|
||||
- Legacy shims (submit, record, identify) route through sync
|
||||
|
||||
Run:
|
||||
cd backend && python -m pytest tests/test_service.py -v
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import time
|
||||
from unittest.mock import patch
|
||||
|
||||
import pytest
|
||||
|
||||
_tmpdir = tempfile.mkdtemp()
|
||||
os.environ.setdefault("OPENSWARM_DATA_DIR", _tmpdir)
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def patch_settings(tmp_path):
|
||||
sf = tmp_path / "settings.json"
|
||||
sf.write_text(json.dumps({
|
||||
"installation_id": "test-install-abc",
|
||||
"analytics_opt_in": True,
|
||||
}))
|
||||
import backend.apps.settings.settings as settings_mod
|
||||
old = settings_mod.SETTINGS_FILE
|
||||
settings_mod.SETTINGS_FILE = str(sf)
|
||||
yield
|
||||
settings_mod.SETTINGS_FILE = old
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def fresh_client(tmp_path):
|
||||
import backend.apps.service.client as client
|
||||
client._install_id = None
|
||||
client._user_id = None
|
||||
client._test_sink = None
|
||||
spool = tmp_path / "spool.db"
|
||||
with patch.object(client, "_spool_path", lambda: str(spool)):
|
||||
yield
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def sink():
|
||||
captured: list[tuple[str, dict]] = []
|
||||
import backend.apps.service.client as client
|
||||
client.set_test_sink(lambda kind, body: captured.append((kind, body)))
|
||||
yield captured
|
||||
client.set_test_sink(None)
|
||||
|
||||
|
||||
# --- core sync ---------------------------------------------------------------
|
||||
|
||||
def test_sync_basic(sink):
|
||||
from backend.apps.service.client import sync
|
||||
sync({"foo": "bar"})
|
||||
assert len(sink) == 1
|
||||
_, body = sink[0]
|
||||
assert body["d"] == {"foo": "bar"}
|
||||
|
||||
|
||||
def test_sync_carries_install_id(sink):
|
||||
from backend.apps.service.client import sync
|
||||
sync({})
|
||||
_, body = sink[0]
|
||||
assert body["client_state"]["install_id"] == "test-install-abc"
|
||||
|
||||
|
||||
def test_sync_carries_user_id_when_set(sink):
|
||||
from backend.apps.service.client import sync, set_user_id
|
||||
set_user_id("alice@example.com")
|
||||
sync({})
|
||||
_, body = sink[0]
|
||||
assert body["client_state"]["user_id"] == "alice@example.com"
|
||||
|
||||
|
||||
def test_sync_no_user_id_when_not_set(sink):
|
||||
from backend.apps.service.client import sync
|
||||
sync({})
|
||||
_, body = sink[0]
|
||||
assert "user_id" not in body["client_state"]
|
||||
|
||||
|
||||
def test_sync_user_id_cleared_with_none(sink):
|
||||
from backend.apps.service.client import sync, set_user_id
|
||||
set_user_id("alice")
|
||||
set_user_id(None)
|
||||
sync({})
|
||||
_, body = sink[0]
|
||||
assert "user_id" not in body["client_state"]
|
||||
|
||||
|
||||
def test_sync_user_id_cleared_with_empty(sink):
|
||||
from backend.apps.service.client import sync, set_user_id
|
||||
set_user_id("alice")
|
||||
set_user_id("")
|
||||
sync({})
|
||||
_, body = sink[0]
|
||||
assert "user_id" not in body["client_state"]
|
||||
|
||||
|
||||
def test_sync_environment_metadata(sink):
|
||||
from backend.apps.service.client import sync
|
||||
sync({})
|
||||
_, body = sink[0]
|
||||
cs = body["client_state"]
|
||||
assert cs.get("device_type") == "desktop"
|
||||
assert cs.get("os")
|
||||
assert cs.get("os_version")
|
||||
|
||||
|
||||
def test_sync_payload_round_trips(sink):
|
||||
from backend.apps.service.client import sync
|
||||
data = {"deeply": {"nested": [1, 2]}, "flag": True, "n": 3.14}
|
||||
sync(data)
|
||||
_, body = sink[0]
|
||||
assert body["d"] == data
|
||||
|
||||
|
||||
def test_sync_empty_data(sink):
|
||||
from backend.apps.service.client import sync
|
||||
sync({})
|
||||
assert len(sink) == 1
|
||||
|
||||
|
||||
def test_sync_none_treated_as_empty(sink):
|
||||
from backend.apps.service.client import sync
|
||||
sync(None)
|
||||
_, body = sink[0]
|
||||
assert body["d"] == {}
|
||||
|
||||
|
||||
def test_sync_timestamp_present(sink):
|
||||
from backend.apps.service.client import sync
|
||||
sync({})
|
||||
_, body = sink[0]
|
||||
assert isinstance(body["t"], float)
|
||||
assert body["t"] > 0
|
||||
|
||||
|
||||
# --- opt-out gating ----------------------------------------------------------
|
||||
|
||||
def test_opt_out_blocks_sync(sink, tmp_path):
|
||||
sf = tmp_path / "minimal.json"
|
||||
sf.write_text(json.dumps({
|
||||
"installation_id": "test-install-abc",
|
||||
"analytics_opt_in": False,
|
||||
}))
|
||||
import backend.apps.settings.settings as settings_mod
|
||||
settings_mod.SETTINGS_FILE = str(sf)
|
||||
from backend.apps.service.client import sync
|
||||
sync({"x": 1})
|
||||
assert sink == []
|
||||
|
||||
|
||||
def test_standard_mode_allows_sync(sink):
|
||||
from backend.apps.service.client import sync
|
||||
sync({})
|
||||
sync({})
|
||||
assert len(sink) == 2
|
||||
|
||||
|
||||
def test_settings_load_failure_defaults_to_enabled(sink):
|
||||
import backend.apps.settings.settings as settings_mod
|
||||
settings_mod.SETTINGS_FILE = "/nonexistent/path/settings.json"
|
||||
from backend.apps.service.client import sync
|
||||
sync({})
|
||||
assert len(sink) == 1
|
||||
|
||||
|
||||
# --- legacy shims ------------------------------------------------------------
|
||||
|
||||
def test_legacy_submit_routes_through_sync(sink):
|
||||
from backend.apps.service.client import submit
|
||||
submit("event", {"test": True})
|
||||
assert len(sink) == 1
|
||||
_, body = sink[0]
|
||||
assert body["d"] == {"test": True}
|
||||
|
||||
|
||||
def test_legacy_record_routes_through_sync(sink):
|
||||
from backend.apps.service.client import record
|
||||
record("some.event", {"k": "v"})
|
||||
assert len(sink) == 1
|
||||
|
||||
|
||||
def test_legacy_identify_routes_through_sync(sink):
|
||||
from backend.apps.service.client import identify
|
||||
identify({"plan": "pro"})
|
||||
assert len(sink) == 1
|
||||
|
||||
|
||||
def test_legacy_submit_session_close(sink):
|
||||
from backend.apps.service.client import submit_session_close
|
||||
submit_session_close({"id": "s-1", "cost_usd": 0.42})
|
||||
assert len(sink) == 1
|
||||
|
||||
|
||||
def test_legacy_submit_diagnostic(sink):
|
||||
from backend.apps.service.client import submit_diagnostic
|
||||
submit_diagnostic({"kind": "error_caught"})
|
||||
assert len(sink) == 1
|
||||
|
||||
|
||||
# --- spool -------------------------------------------------------------------
|
||||
|
||||
def test_buffer_enqueue_and_drain(tmp_path):
|
||||
from backend.apps.service import buffer
|
||||
spool = str(tmp_path / "s.db")
|
||||
buffer.enqueue(spool, "s:/api/service/sync", {"a": 1}, now=time.time())
|
||||
buffer.enqueue(spool, "s:/api/service/sync", {"a": 2}, now=time.time())
|
||||
assert buffer.count(spool) == 2
|
||||
rows = buffer.drain(spool, batch_size=10)
|
||||
assert [r[2]["a"] for r in rows] == [1, 2]
|
||||
buffer.acknowledge(spool, [r[0] for r in rows])
|
||||
assert buffer.count(spool) == 0
|
||||
|
||||
|
||||
def test_buffer_drain_partial(tmp_path):
|
||||
from backend.apps.service import buffer
|
||||
spool = str(tmp_path / "s.db")
|
||||
for i in range(5):
|
||||
buffer.enqueue(spool, "s:/x", {"i": i}, now=time.time())
|
||||
rows = buffer.drain(spool, batch_size=2)
|
||||
assert len(rows) == 2
|
||||
assert buffer.count(spool) == 5
|
||||
buffer.acknowledge(spool, [r[0] for r in rows])
|
||||
assert buffer.count(spool) == 3
|
||||
|
||||
|
||||
def test_buffer_clear(tmp_path):
|
||||
from backend.apps.service import buffer
|
||||
spool = str(tmp_path / "s.db")
|
||||
buffer.enqueue(spool, "s:/x", {}, now=time.time())
|
||||
buffer.clear(spool)
|
||||
assert buffer.count(spool) == 0
|
||||
|
||||
|
||||
def test_buffer_missing_file(tmp_path):
|
||||
from backend.apps.service import buffer
|
||||
assert buffer.count(str(tmp_path / "nope.db")) == 0
|
||||
assert buffer.drain(str(tmp_path / "nope.db")) == []
|
||||
|
||||
|
||||
def test_buffer_corrupt_row_dropped(tmp_path):
|
||||
from backend.apps.service import buffer
|
||||
spool = str(tmp_path / "s.db")
|
||||
with buffer._conn(spool) as c:
|
||||
c.execute(
|
||||
"INSERT INTO spool (kind, payload, created_at) VALUES (?, ?, ?)",
|
||||
("s:/x", "{not json", time.time()),
|
||||
)
|
||||
rows = buffer.drain(spool)
|
||||
assert rows == []
|
||||
assert buffer.count(spool) == 0
|
||||
|
||||
|
||||
def test_buffer_size_cap(tmp_path):
|
||||
from backend.apps.service import buffer
|
||||
spool = str(tmp_path / "s.db")
|
||||
big = "x" * 1024
|
||||
for i in range(200):
|
||||
buffer.enqueue(spool, "s:/x", {"i": i, "pad": big}, now=time.time())
|
||||
assert buffer.count(spool) == 200
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_drain_spool_empty():
|
||||
from backend.apps.service.client import drain_spool
|
||||
n = await drain_spool()
|
||||
assert n == 0
|
||||
|
||||
|
||||
# --- identity ----------------------------------------------------------------
|
||||
|
||||
def test_install_id_persisted(sink, tmp_path):
|
||||
sf = tmp_path / "fresh.json"
|
||||
sf.write_text(json.dumps({"analytics_opt_in": True}))
|
||||
import backend.apps.settings.settings as settings_mod
|
||||
settings_mod.SETTINGS_FILE = str(sf)
|
||||
import backend.apps.service.client as client
|
||||
client._install_id = None
|
||||
from backend.apps.service.client import sync
|
||||
sync({})
|
||||
_, body = sink[0]
|
||||
iid = body["client_state"]["install_id"]
|
||||
assert iid
|
||||
raw = json.loads(sf.read_text())
|
||||
assert raw["installation_id"] == iid
|
||||
|
||||
|
||||
def test_install_id_stable(sink):
|
||||
from backend.apps.service.client import sync
|
||||
sync({})
|
||||
sync({})
|
||||
iid1 = sink[0][1]["client_state"]["install_id"]
|
||||
iid2 = sink[1][1]["client_state"]["install_id"]
|
||||
assert iid1 == iid2
|
||||
|
||||
|
||||
# --- SubApp endpoints --------------------------------------------------------
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_submit(sink):
|
||||
from backend.apps.service.service import post_submit
|
||||
res = await post_submit({"kind": "state", "payload": {"x": 1}})
|
||||
assert res == {"ok": True}
|
||||
assert len(sink) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_submit_missing_payload(sink):
|
||||
from backend.apps.service.service import post_submit
|
||||
res = await post_submit({"kind": "state"})
|
||||
assert res["ok"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_event_happy(sink):
|
||||
from backend.apps.service.service import post_event
|
||||
res = await post_event({"surface": "test", "action": "happy"})
|
||||
assert res == {"ok": True}
|
||||
assert len(sink) == 1
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_event_missing_surface(sink):
|
||||
from backend.apps.service.service import post_event
|
||||
res = await post_event({"action": "x"})
|
||||
assert res["ok"] is False
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_endpoint_spool_count(tmp_path):
|
||||
from backend.apps.service import client as svc, buffer
|
||||
from backend.apps.service.service import spool_count
|
||||
spool = str(tmp_path / "spool.db")
|
||||
with patch.object(svc, "_spool_path", lambda: spool):
|
||||
buffer.enqueue(spool, "s:/x", {}, now=time.time())
|
||||
result = await spool_count()
|
||||
assert result == {"pending": 1}
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Service-sync compatibility tests.
|
||||
|
||||
Verifies the legacy compatibility helpers on backend/apps/service/client.py
|
||||
(record, submit_event, submit_session_close, etc.) still produce the right
|
||||
opaque payload through the unified sync() entry point. Forward-looking
|
||||
contract tests live in test_service.py; this file covers the legacy shim
|
||||
surface so it can be deprecated cleanly later.
|
||||
|
||||
Run with:
|
||||
cd backend && python -m pytest tests/test_service_legacy.py -v
|
||||
"""
|
||||
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
from unittest.mock import AsyncMock, MagicMock, patch
|
||||
from uuid import uuid4
|
||||
|
||||
import pytest
|
||||
|
||||
# Sandbox the data dir before any module import touches settings on disk.
|
||||
_tmpdir = tempfile.mkdtemp()
|
||||
os.environ.setdefault("OPENSWARM_DATA_DIR", _tmpdir)
|
||||
|
||||
# Captured syncs from this test run.
|
||||
_captured_syncs: list[dict] = []
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def reset_captured_syncs():
|
||||
_captured_syncs.clear()
|
||||
yield
|
||||
_captured_syncs.clear()
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def install_sync_sink():
|
||||
"""Install a service-sync sink and decode the opaque payload back into
|
||||
a structured shape for assertions. The sink translates the new shape
|
||||
{client_state, d, t} into a legacy-compatible {kind, distinct_id, props}
|
||||
bag so existing tests can keep their assertions terse."""
|
||||
import backend.apps.service.client as svc_client
|
||||
|
||||
def _sink(label: str, body: dict):
|
||||
cs = body.get("client_state") or {}
|
||||
payload = body.get("d") or body.get("payload") or {}
|
||||
|
||||
# Infer a synthetic kind from payload shape — same dispatch logic
|
||||
# as the cloud uses in production.
|
||||
if "status" in payload and "messages" in payload:
|
||||
status = payload.get("status", "unknown")
|
||||
kind = f"session.{status}" if status != "unknown" else "session.completed"
|
||||
props = dict(payload)
|
||||
elif "identity" in payload:
|
||||
kind = "state.update"
|
||||
props = dict(payload)
|
||||
elif "diagnostic" in payload:
|
||||
kind = "diagnostic.fired"
|
||||
props = dict(payload)
|
||||
elif "s" in payload and "a" in payload:
|
||||
kind = f"{payload['s']}.{payload['a']}"
|
||||
props = dict(payload.get("p") or {})
|
||||
elif "surface" in payload:
|
||||
surface = payload.get("surface", "")
|
||||
action = payload.get("action", "fired")
|
||||
kind = f"{surface}.{action}"
|
||||
props = dict(payload.get("props") or {})
|
||||
else:
|
||||
kind = "state.update"
|
||||
props = dict(payload)
|
||||
|
||||
if payload.get("session_id"):
|
||||
props["session_id"] = payload["session_id"]
|
||||
if payload.get("dashboard_id"):
|
||||
props["dashboard_id"] = payload["dashboard_id"]
|
||||
props.setdefault("os", cs.get("os", ""))
|
||||
props.setdefault("platform", cs.get("os", ""))
|
||||
|
||||
_captured_syncs.append({
|
||||
"kind": kind,
|
||||
"distinct_id": cs.get("install_id", ""),
|
||||
"properties": props,
|
||||
})
|
||||
|
||||
old_sink = svc_client._test_sink
|
||||
old_iid = svc_client._install_id
|
||||
svc_client.set_test_sink(_sink)
|
||||
svc_client._install_id = "test-install-id"
|
||||
yield
|
||||
svc_client.set_test_sink(old_sink)
|
||||
svc_client._install_id = old_iid
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_settings(tmp_path):
|
||||
"""Sandbox settings so tests don't read or write the real config."""
|
||||
settings_file = tmp_path / "settings.json"
|
||||
settings_file.write_text(json.dumps({
|
||||
"service_diagnostics_mode": "standard",
|
||||
"installation_id": "test-install-id",
|
||||
}))
|
||||
|
||||
import backend.apps.settings.settings as settings_mod
|
||||
old_file = settings_mod.SETTINGS_FILE
|
||||
settings_mod.SETTINGS_FILE = str(settings_file)
|
||||
yield
|
||||
settings_mod.SETTINGS_FILE = old_file
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def mock_sessions_dir(tmp_path):
|
||||
"""Use temp dir for session persistence."""
|
||||
sessions_dir = tmp_path / "sessions"
|
||||
sessions_dir.mkdir()
|
||||
|
||||
import backend.config.paths as paths_mod
|
||||
old_dir = paths_mod.SESSIONS_DIR
|
||||
paths_mod.SESSIONS_DIR = str(sessions_dir)
|
||||
yield str(sessions_dir)
|
||||
paths_mod.SESSIONS_DIR = old_dir
|
||||
|
||||
|
||||
def syncs(kind: str | None = None) -> list[dict]:
|
||||
"""Return captured syncs, optionally filtered by inferred kind."""
|
||||
if kind:
|
||||
return [s for s in _captured_syncs if s["kind"] == kind]
|
||||
return list(_captured_syncs)
|
||||
|
||||
|
||||
def last_sync(kind: str) -> dict:
|
||||
"""Return the last captured sync of a given inferred kind."""
|
||||
matching = syncs(kind)
|
||||
assert matching, f"No {kind} syncs captured. Got: {[s['kind'] for s in _captured_syncs]}"
|
||||
return matching[-1]
|
||||
|
||||
|
||||
# Import application modules (after fixtures are wired).
|
||||
from backend.apps.service.client import record
|
||||
from backend.apps.agents.models import AgentConfig, AgentSession, Message, ApprovalRequest
|
||||
from backend.apps.agents.agent_manager import AgentManager
|
||||
|
||||
|
||||
@pytest.fixture
|
||||
def manager():
|
||||
"""Fresh AgentManager per test."""
|
||||
return AgentManager()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 1. record() — legacy shim correctness
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestRecordBasics:
|
||||
def test_record_sends_payload(self):
|
||||
record("test.report", {"key": "value"})
|
||||
s = last_sync("test.report")
|
||||
assert s["properties"]["key"] == "value"
|
||||
assert s["distinct_id"] == "test-install-id"
|
||||
|
||||
def test_record_adds_os_and_platform(self):
|
||||
record("test.report", {})
|
||||
s = last_sync("test.report")
|
||||
assert "os" in s["properties"]
|
||||
assert "platform" in s["properties"]
|
||||
|
||||
def test_record_includes_session_id(self):
|
||||
record("test.report", {}, session_id="sess123")
|
||||
s = last_sync("test.report")
|
||||
assert s["properties"]["session_id"] == "sess123"
|
||||
|
||||
def test_record_includes_dashboard_id(self):
|
||||
record("test.report", {}, dashboard_id="dash456")
|
||||
s = last_sync("test.report")
|
||||
assert s["properties"]["dashboard_id"] == "dash456"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 2. Multi-message session — close fires exactly once
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestMultiMessageSession:
|
||||
@pytest.mark.asyncio
|
||||
async def test_session_completes_only_on_close(self, manager):
|
||||
"""Verify a completed-session sync does NOT fire mid-loop. It should
|
||||
only fire on close_session() or persist_all_sessions()."""
|
||||
config = AgentConfig(name="Multi-msg", model="sonnet", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
|
||||
for i in range(3):
|
||||
session.messages.append(Message(role="user", content=f"msg {i}"))
|
||||
session.messages.append(Message(role="assistant", content=f"reply {i}"))
|
||||
|
||||
completed = syncs("session.completed")
|
||||
assert len(completed) == 0, f"session-completed fired {len(completed)} times before close"
|
||||
|
||||
session.status = "completed"
|
||||
await manager.close_session(session.id)
|
||||
|
||||
completed = syncs("session.completed")
|
||||
assert len(completed) == 1, f"expected 1 completed sync, got {len(completed)}"
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# 3. Token + cost capture on close
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class TestTokenTracking:
|
||||
@pytest.mark.asyncio
|
||||
async def test_tokens_and_cost_in_session_close(self, manager):
|
||||
config = AgentConfig(name="Token Test", model="opus", mode="agent")
|
||||
session = await manager.launch_agent(config)
|
||||
|
||||
session.tokens = {"input": 50000, "output": 15000}
|
||||
session.cost_usd = 0.25
|
||||
session.status = "completed"
|
||||
|
||||
await manager.close_session(session.id)
|
||||
|
||||
s = last_sync("session.completed")
|
||||
assert s["properties"]["tokens"]["input"] == 50000
|
||||
assert s["properties"]["tokens"]["output"] == 15000
|
||||
assert s["properties"]["cost_usd"] == 0.25
|
||||
File diff suppressed because it is too large
Load Diff
@@ -0,0 +1,410 @@
|
||||
"""Mirror tests for the frontend label/result logic.
|
||||
|
||||
The JS implementations live in:
|
||||
- frontend/src/app/pages/AgentChat/toolLabels.ts
|
||||
- frontend/src/app/pages/AgentChat/ToolCallBubble.tsx (getResultSummary,
|
||||
getInputSummary, parseMcpToolName, bashCommandDetail, prettyPath, prettyUrl,
|
||||
quoteQuery)
|
||||
|
||||
We re-implement the rules in Python and pin them as tests so we get
|
||||
regression coverage from `pytest` too. Any drift between the JS source
|
||||
and these Python mirrors is the production-side breakage we want to
|
||||
catch.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import re
|
||||
import pytest
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Mirror: parseMcpToolName.displayName (sentence-case rule)
|
||||
# ===========================================================================
|
||||
|
||||
def parse_mcp_tool_name_display(raw_name: str) -> str | None:
|
||||
"""Mirror of frontend parseMcpToolName().displayName."""
|
||||
m = re.match(r"^mcp__([^_]+(?:-[^_]+)*)__(.+)$", raw_name)
|
||||
if not m:
|
||||
return None
|
||||
action = m.group(2)
|
||||
spaced = action.replace("_", " ").lower()
|
||||
return spaced[0].upper() + spaced[1:] if spaced else ""
|
||||
|
||||
|
||||
def test_parse_mcp_tool_name_get_message_details():
|
||||
assert parse_mcp_tool_name_display(
|
||||
"mcp__google-workspace__get_message_details"
|
||||
) == "Get message details"
|
||||
|
||||
|
||||
def test_parse_mcp_tool_name_send_email():
|
||||
assert parse_mcp_tool_name_display(
|
||||
"mcp__google-workspace__send_gmail_message"
|
||||
) == "Send gmail message"
|
||||
|
||||
|
||||
def test_parse_mcp_tool_name_search_emails():
|
||||
assert parse_mcp_tool_name_display(
|
||||
"mcp__google-workspace__query_gmail_emails"
|
||||
) == "Query gmail emails"
|
||||
|
||||
|
||||
def test_parse_mcp_tool_name_returns_none_for_non_mcp():
|
||||
assert parse_mcp_tool_name_display("Bash") is None
|
||||
assert parse_mcp_tool_name_display("Read") is None
|
||||
|
||||
|
||||
def test_parse_mcp_tool_name_no_title_case():
|
||||
"""Regression test: NEVER capitalize every word."""
|
||||
bad = parse_mcp_tool_name_display("mcp__notion__create_a_new_page")
|
||||
assert bad == "Create a new page"
|
||||
assert "A New Page" not in bad
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Mirror: getResultSummary (glyph-free regression test)
|
||||
# ===========================================================================
|
||||
|
||||
def get_result_summary_bash_success(stdout: str, exit_code: int = 0) -> str:
|
||||
"""Mirror of getResultSummary for bash success case."""
|
||||
if exit_code != 0:
|
||||
return f"exit {exit_code}"
|
||||
lines = [l for l in stdout.split("\n") if l.strip()]
|
||||
n = len(lines)
|
||||
return f"{n} line{'s' if n != 1 else ''}"
|
||||
|
||||
|
||||
def test_bash_success_summary_no_glyph():
|
||||
"""Regression: bash success used to return '✓ N lines'. Must now be glyph-free."""
|
||||
assert "✓" not in get_result_summary_bash_success("hello\nworld")
|
||||
assert get_result_summary_bash_success("hello\nworld") == "2 lines"
|
||||
assert get_result_summary_bash_success("just one line") == "1 line"
|
||||
assert get_result_summary_bash_success("") == "0 lines"
|
||||
|
||||
|
||||
def test_bash_failure_summary_no_glyph():
|
||||
"""Failure summary too: 'exit 1' not '✗ exit 1'."""
|
||||
assert get_result_summary_bash_success("", exit_code=1) == "exit 1"
|
||||
assert "✗" not in get_result_summary_bash_success("", exit_code=1)
|
||||
assert "✓" not in get_result_summary_bash_success("", exit_code=1)
|
||||
|
||||
|
||||
def test_no_check_glyph_in_summaries():
|
||||
"""Sweep: every plausible summary string never contains a check glyph."""
|
||||
summaries = [
|
||||
get_result_summary_bash_success("a"),
|
||||
get_result_summary_bash_success("a\nb\nc"),
|
||||
get_result_summary_bash_success("", exit_code=1),
|
||||
get_result_summary_bash_success("", exit_code=127),
|
||||
]
|
||||
for s in summaries:
|
||||
assert "✓" not in s and "✔" not in s and "✗" not in s and "✘" not in s
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Mirror: bashCommandDetail extraction
|
||||
# ===========================================================================
|
||||
|
||||
def bash_command_detail(raw_cmd: str) -> str:
|
||||
"""Mirror of frontend bashCommandDetail."""
|
||||
if not raw_cmd:
|
||||
return ""
|
||||
cmd = raw_cmd.strip()
|
||||
# strip env var assignments + sudo/time/nice/env
|
||||
cmd = re.sub(r"^(?:[A-Z_][A-Z0-9_]*=\S+\s+)+", "", cmd)
|
||||
cmd = re.sub(r"^(?:sudo|time|nice|env)\s+", "", cmd)
|
||||
tokens = cmd.split()
|
||||
if not tokens:
|
||||
return ""
|
||||
bin_path = tokens[0].split("/")[-1]
|
||||
|
||||
if bin_path == "git":
|
||||
sub = (tokens[1] if len(tokens) > 1 else "").lower()
|
||||
if sub in ("commit", "status", "log", "diff", "pull", "push", "fetch"):
|
||||
return ""
|
||||
return tokens[2].split("/")[-1] if len(tokens) > 2 else ""
|
||||
|
||||
if bin_path in ("npm", "pnpm", "yarn", "bun", "pip", "pip3", "brew", "apt", "apt-get"):
|
||||
if len(tokens) > 2:
|
||||
args = [t for t in tokens[2:] if not t.startswith("-")][:2]
|
||||
return " ".join(args)
|
||||
return ""
|
||||
|
||||
# First non-flag positional arg
|
||||
arg = next((t for t in tokens[1:] if not t.startswith("-")), "")
|
||||
if not arg:
|
||||
return ""
|
||||
if "/" in arg or "\\" in arg:
|
||||
# basename
|
||||
cleaned = arg.rstrip("/\\")
|
||||
parts = cleaned.replace("\\", "/").split("/")
|
||||
return parts[-1] if parts[-1] else cleaned
|
||||
return arg if len(arg) <= 50 else arg[:47] + "..."
|
||||
|
||||
|
||||
def test_bash_detail_rm_extracts_path():
|
||||
assert bash_command_detail("rm /tmp/foo.txt") == "foo.txt"
|
||||
assert bash_command_detail("rm foo.txt") == "foo.txt"
|
||||
|
||||
|
||||
def test_bash_detail_git_commit_empty():
|
||||
"""git commit -m 'message' → no detail (verb covers it)."""
|
||||
assert bash_command_detail("git commit -m 'fix bug'") == ""
|
||||
assert bash_command_detail("git commit -m hi") == ""
|
||||
|
||||
|
||||
def test_bash_detail_git_status_empty():
|
||||
assert bash_command_detail("git status") == ""
|
||||
|
||||
|
||||
def test_bash_detail_git_checkout_branch():
|
||||
assert bash_command_detail("git checkout main") == "main"
|
||||
|
||||
|
||||
def test_bash_detail_npm_install():
|
||||
assert bash_command_detail("npm install lodash") == "lodash"
|
||||
assert bash_command_detail("npm install lodash @types/node") == "lodash @types/node"
|
||||
|
||||
|
||||
def test_bash_detail_strips_sudo():
|
||||
assert bash_command_detail("sudo rm /etc/foo") == "foo"
|
||||
|
||||
|
||||
def test_bash_detail_strips_env_assignments():
|
||||
assert bash_command_detail("FOO=bar BAZ=qux rm /tmp/a") == "a"
|
||||
|
||||
|
||||
def test_bash_detail_handles_empty():
|
||||
assert bash_command_detail("") == ""
|
||||
assert bash_command_detail(" ") == ""
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Mirror: prettyPath (basename a path)
|
||||
# ===========================================================================
|
||||
|
||||
def pretty_path(p: str) -> str:
|
||||
if not p:
|
||||
return ""
|
||||
cleaned = p.rstrip("/\\")
|
||||
parts = cleaned.replace("\\", "/").split("/")
|
||||
return parts[-1] if parts[-1] else cleaned
|
||||
|
||||
|
||||
def test_pretty_path_absolute():
|
||||
assert pretty_path("/Users/eric/Downloads/openswarm/foo.ts") == "foo.ts"
|
||||
|
||||
|
||||
def test_pretty_path_relative():
|
||||
assert pretty_path("a/b/c.tsx") == "c.tsx"
|
||||
|
||||
|
||||
def test_pretty_path_trailing_slash():
|
||||
assert pretty_path("/a/b/c/") == "c"
|
||||
|
||||
|
||||
def test_pretty_path_empty():
|
||||
assert pretty_path("") == ""
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Mirror: prettyUrl (host-only)
|
||||
# ===========================================================================
|
||||
|
||||
def pretty_url(u: str) -> str:
|
||||
if not u:
|
||||
return ""
|
||||
try:
|
||||
from urllib.parse import urlparse
|
||||
host = urlparse(u).hostname or ""
|
||||
return host[4:] if host.startswith("www.") else host or u[:60]
|
||||
except Exception:
|
||||
no_proto = re.sub(r"^https?://", "", u).split("/")[0].split("?")[0].split("#")[0]
|
||||
return no_proto[:60]
|
||||
|
||||
|
||||
def test_pretty_url_https():
|
||||
assert pretty_url("https://example.com/long/path?q=1") == "example.com"
|
||||
|
||||
|
||||
def test_pretty_url_strips_www():
|
||||
assert pretty_url("https://www.example.com/path") == "example.com"
|
||||
|
||||
|
||||
def test_pretty_url_subdomain_kept():
|
||||
assert pretty_url("https://api.example.com/v1") == "api.example.com"
|
||||
|
||||
|
||||
def test_pretty_url_empty():
|
||||
assert pretty_url("") == ""
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Mirror: quoteQuery
|
||||
# ===========================================================================
|
||||
|
||||
def quote_query(q: str, max_len: int = 60) -> str:
|
||||
if not q:
|
||||
return ""
|
||||
trimmed = q if len(q) <= max_len else q[:max_len - 1] + "…"
|
||||
return f'"{trimmed}"'
|
||||
|
||||
|
||||
def test_quote_query_short():
|
||||
assert quote_query("TODO") == '"TODO"'
|
||||
|
||||
|
||||
def test_quote_query_long_truncated():
|
||||
long = "a" * 100
|
||||
result = quote_query(long)
|
||||
assert result.startswith('"')
|
||||
assert result.endswith('"')
|
||||
assert len(result) <= 62 # 60 chars + 2 quotes
|
||||
|
||||
|
||||
def test_quote_query_empty():
|
||||
assert quote_query("") == ""
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Mirror: stable-seeded variant pick (djb2 hash → mod n)
|
||||
# ===========================================================================
|
||||
|
||||
def stable_index(seed: str | None, n: int) -> int:
|
||||
"""Mirror of frontend _stableIndex."""
|
||||
if n <= 1 or not seed:
|
||||
return 0
|
||||
h = 5381
|
||||
for ch in seed:
|
||||
h = ((h << 5) + h + ord(ch)) & 0xFFFFFFFF # 32-bit
|
||||
# JS does `| 0` which produces signed int; Math.abs handles that
|
||||
if h >= 0x80000000:
|
||||
h -= 0x100000000
|
||||
return abs(h) % n
|
||||
|
||||
|
||||
def test_stable_index_same_seed_same_result():
|
||||
"""Critical: same call.id always → same variant index."""
|
||||
n = 5
|
||||
for seed in ("abc-123", "xyz-789", "tool-call-uuid-deadbeef"):
|
||||
a = stable_index(seed, n)
|
||||
b = stable_index(seed, n)
|
||||
c = stable_index(seed, n)
|
||||
assert a == b == c, f"unstable for seed={seed!r}"
|
||||
|
||||
|
||||
def test_stable_index_different_seeds_diverge():
|
||||
"""Different seeds usually give different results (probabilistic)."""
|
||||
n = 7
|
||||
seeds = [f"seed-{i}-{random.randint(0, 99999)}" for i in range(50)]
|
||||
indices = [stable_index(s, n) for s in seeds]
|
||||
# All same is statistically extremely unlikely
|
||||
assert len(set(indices)) > 1
|
||||
|
||||
|
||||
def test_stable_index_in_range():
|
||||
"""Index always in [0, n-1]."""
|
||||
for _ in range(200):
|
||||
seed = "".join(random.choices(string.ascii_letters + string.digits, k=20))
|
||||
n = random.randint(2, 20)
|
||||
idx = stable_index(seed, n)
|
||||
assert 0 <= idx < n, f"out of range: {idx} for n={n}"
|
||||
|
||||
|
||||
def test_stable_index_empty_seed_zero():
|
||||
"""No seed → safe-default (index 0)."""
|
||||
assert stable_index(None, 5) == 0
|
||||
assert stable_index("", 5) == 0
|
||||
|
||||
|
||||
def test_stable_index_n_one():
|
||||
"""Single-variant pool → always index 0."""
|
||||
assert stable_index("anything", 1) == 0
|
||||
|
||||
|
||||
# ===========================================================================
|
||||
# Mirror: bash verb extraction (the leading-binary lookup)
|
||||
# ===========================================================================
|
||||
|
||||
BIN_VERB_MAP = {
|
||||
"rm": ("Deleting", "Deleted"),
|
||||
"mv": ("Moving", "Moved"),
|
||||
"cp": ("Copying", "Copied"),
|
||||
"mkdir": ("Creating folder", "Created folder"),
|
||||
"ls": ("Listing folder", "Listed folder"),
|
||||
"find": ("Hunting for files", "Hunted for files"),
|
||||
"grep": ("Searching files", "Searched files"),
|
||||
"cat": ("Reading", "Read"),
|
||||
"echo": ("Printing", "Printed"),
|
||||
"make": ("Building", "Built"),
|
||||
}
|
||||
|
||||
GIT_VERB_MAP = {
|
||||
"commit": ("Committing", "Committed"),
|
||||
"push": ("Pushing to git", "Pushed to git"),
|
||||
"pull": ("Pulling from git", "Pulled from git"),
|
||||
"checkout": ("Switching branches", "Switched branches"),
|
||||
"merge": ("Merging", "Merged"),
|
||||
}
|
||||
|
||||
PKG_VERB_MAP_INSTALL = ("Installing packages", "Installed packages")
|
||||
PKG_VERB_MAP_UNINSTALL = ("Removing packages", "Removed packages")
|
||||
|
||||
|
||||
def bash_verb(cmd: str, past: bool = False):
|
||||
if not cmd:
|
||||
return None
|
||||
stripped = re.sub(r"^(?:[A-Z_][A-Z0-9_]*=\S+\s+)+", "", cmd.strip())
|
||||
stripped = re.sub(r"^(?:sudo|time|nice|env)\s+", "", stripped)
|
||||
tokens = stripped.split()
|
||||
if not tokens:
|
||||
return None
|
||||
bin_path = tokens[0].split("/")[-1].lower()
|
||||
sub = (tokens[1] if len(tokens) > 1 else "").lower()
|
||||
|
||||
if bin_path == "git" and sub in GIT_VERB_MAP:
|
||||
return GIT_VERB_MAP[sub][1 if past else 0]
|
||||
if bin_path in ("npm", "pnpm", "yarn", "pip", "pip3", "brew"):
|
||||
if sub in ("install", "add", "i"):
|
||||
return PKG_VERB_MAP_INSTALL[1 if past else 0]
|
||||
if sub in ("uninstall", "remove", "rm"):
|
||||
return PKG_VERB_MAP_UNINSTALL[1 if past else 0]
|
||||
if bin_path in BIN_VERB_MAP:
|
||||
return BIN_VERB_MAP[bin_path][1 if past else 0]
|
||||
return None
|
||||
|
||||
|
||||
def test_bash_verb_rm_deleted():
|
||||
assert bash_verb("rm foo", past=True) == "Deleted"
|
||||
assert bash_verb("rm foo", past=False) == "Deleting"
|
||||
|
||||
|
||||
def test_bash_verb_git_commit():
|
||||
assert bash_verb("git commit -m hi", past=True) == "Committed"
|
||||
|
||||
|
||||
def test_bash_verb_git_push():
|
||||
assert bash_verb("git push origin main", past=True) == "Pushed to git"
|
||||
|
||||
|
||||
def test_bash_verb_npm_install():
|
||||
assert bash_verb("npm install lodash", past=True) == "Installed packages"
|
||||
|
||||
|
||||
def test_bash_verb_unknown_returns_none():
|
||||
"""Truly unknown command falls through to default 'Ran command'."""
|
||||
assert bash_verb("supercustomtool foo bar") is None
|
||||
|
||||
|
||||
def test_bash_verb_strips_sudo():
|
||||
assert bash_verb("sudo rm -rf /tmp/x", past=True) == "Deleted"
|
||||
|
||||
|
||||
def test_bash_verb_strips_env():
|
||||
assert bash_verb("DEBUG=1 npm test", past=False) is None # 'test' isn't in pkg map for bash_verb
|
||||
|
||||
|
||||
# string is needed for stable_index test
|
||||
import string # noqa: E402
|
||||
Reference in New Issue
Block a user