diff --git a/libs/checkpoint-postgres/tests/test_store.py b/libs/checkpoint-postgres/tests/test_store.py index f7c7a1b2c..42bf50419 100644 --- a/libs/checkpoint-postgres/tests/test_store.py +++ b/libs/checkpoint-postgres/tests/test_store.py @@ -861,3 +861,46 @@ def test_store_ttl(store): # Now has been (TTL_SECONDS-2)*2 > TTL_SECONDS + TTL_SECONDS/2 res = store.search(ns, query="bar", refresh_ttl=False) assert len(res) == 0 + + +@pytest.mark.parametrize( + "vector_type,distance_type", + [ + ("vector", "cosine"), + ("vector", "inner_product"), + ("halfvec", "cosine"), + ("halfvec", "inner_product"), + ], +) +def test_non_ascii( + request: Any, + fake_embeddings: CharacterEmbeddings, + vector_type: str, + distance_type: str, +) -> None: + """Test support for non-ascii characters""" + with _create_vector_store( + vector_type, + distance_type, + fake_embeddings + ) as store: + + store.put(("user_123", "memories"), "1", {"text": "这是中文"}) # Chinese + store.put( + ("user_123", "memories"), "2", {"text": "これは日本語です"} + ) # Japanese + store.put(("user_123", "memories"), "3", {"text": "이건 한국어야"}) # Korean + store.put(("user_123", "memories"), "4", {"text": "Это русский"}) # Russian + store.put(("user_123", "memories"), "5", {"text": "यह रूसी है"}) # Hindi + + result1 = store.search(("user_123", "memories"), query="这是中文") + result2 = store.search(("user_123", "memories"), query="これは日本語です") + result3 = store.search(("user_123", "memories"), query="이건 한국어야") + result4 = store.search(("user_123", "memories"), query="Это русский") + result5 = store.search(("user_123", "memories"), query="यह रूसी है") + + assert result1[0].key == "1" + assert result2[0].key == "2" + assert result3[0].key == "3" + assert result4[0].key == "4" + assert result5[0].key == "5" diff --git a/libs/checkpoint-sqlite/tests/test_store.py b/libs/checkpoint-sqlite/tests/test_store.py index 590394375..844493fdc 100644 --- a/libs/checkpoint-sqlite/tests/test_store.py +++ b/libs/checkpoint-sqlite/tests/test_store.py @@ -1067,3 +1067,31 @@ def test_sql_injection_vulnerability(store: SqliteStore) -> None: with pytest.raises(ValueError, match="Invalid filter key"): store.search(("docs",), filter={malicious_key: "dummy"}) + + +@pytest.mark.parametrize("distance_type", VECTOR_TYPES) +def test_non_ascii( + fake_embeddings: CharacterEmbeddings, + distance_type: str, +) -> None: + """Test support for non-ascii characters""" + with create_vector_store(fake_embeddings, distance_type=distance_type) as store: + store.put(("user_123", "memories"), "1", {"text": "这是中文"}) # Chinese + store.put( + ("user_123", "memories"), "2", {"text": "これは日本語です"} + ) # Japanese + store.put(("user_123", "memories"), "3", {"text": "이건 한국어야"}) # Korean + store.put(("user_123", "memories"), "4", {"text": "Это русский"}) # Russian + store.put(("user_123", "memories"), "5", {"text": "यह रूसी है"}) # Hindi + + result1 = store.search(("user_123", "memories"), query="这是中文") + result2 = store.search(("user_123", "memories"), query="これは日本語です") + result3 = store.search(("user_123", "memories"), query="이건 한국어야") + result4 = store.search(("user_123", "memories"), query="Это русский") + result5 = store.search(("user_123", "memories"), query="यह रूसी है") + + assert result1[0].key == "1" + assert result2[0].key == "2" + assert result3[0].key == "3" + assert result4[0].key == "4" + assert result5[0].key == "5" diff --git a/libs/checkpoint/langgraph/store/base/embed.py b/libs/checkpoint/langgraph/store/base/embed.py index 738a9e53f..afaa4d135 100644 --- a/libs/checkpoint/langgraph/store/base/embed.py +++ b/libs/checkpoint/langgraph/store/base/embed.py @@ -238,7 +238,7 @@ def get_text_at_path(obj: Any, path: str | list[str]) -> list[str]: - Nested paths in multi-field: "{field1,nested.field2}" """ if not path or path == "$": - return [json.dumps(obj, sort_keys=True)] + return [json.dumps(obj, sort_keys=True, ensure_ascii=False)] tokens = tokenize_path(path) if isinstance(path, str) else path @@ -249,7 +249,7 @@ def get_text_at_path(obj: Any, path: str | list[str]) -> list[str]: elif obj is None: return [] elif isinstance(obj, (list, dict)): - return [json.dumps(obj, sort_keys=True)] + return [json.dumps(obj, sort_keys=True, ensure_ascii=False)] return [] token = tokens[pos] @@ -295,7 +295,11 @@ def get_text_at_path(obj: Any, path: str | list[str]) -> list[str]: if isinstance(current_obj, (str, int, float, bool)): results.append(str(current_obj)) elif isinstance(current_obj, (list, dict)): - results.append(json.dumps(current_obj, sort_keys=True)) + results.append( + json.dumps( + current_obj, sort_keys=True, ensure_ascii=False + ) + ) # Handle wildcard elif token == "*": diff --git a/libs/checkpoint/tests/test_redis_cache.py b/libs/checkpoint/tests/test_redis_cache.py index bec001e9b..ca1fce09c 100644 --- a/libs/checkpoint/tests/test_redis_cache.py +++ b/libs/checkpoint/tests/test_redis_cache.py @@ -5,12 +5,13 @@ import time import pytest import redis +from langgraph.cache.base import FullKey from langgraph.cache.redis import RedisCache class TestRedisCache: @pytest.fixture(autouse=True) - def setup(self): + def setup(self) -> None: """Set up test Redis client and cache.""" self.client = redis.Redis( host="localhost", port=6379, db=0, decode_responses=False @@ -20,21 +21,21 @@ class TestRedisCache: except redis.ConnectionError: pytest.skip("Redis server not available") - self.cache = RedisCache(self.client, prefix="test:cache:") + self.cache: RedisCache = RedisCache(self.client, prefix="test:cache:") # Clean up before each test self.client.flushdb() - def teardown_method(self): + def teardown_method(self) -> None: """Clean up after each test.""" try: self.client.flushdb() except Exception: pass - def test_basic_set_and_get(self): + def test_basic_set_and_get(self) -> None: """Test basic set and get operations.""" - keys = [(("graph", "node"), "key1")] + keys: list[FullKey] = [(("graph", "node"), "key1")] values = {keys[0]: ({"result": 42}, None)} # Set value @@ -45,9 +46,9 @@ class TestRedisCache: assert len(result) == 1 assert result[keys[0]] == {"result": 42} - def test_batch_operations(self): + def test_batch_operations(self) -> None: """Test batch set and get operations.""" - keys = [ + keys: list[FullKey] = [ (("graph", "node1"), "key1"), (("graph", "node2"), "key2"), (("other", "node"), "key3"), @@ -68,9 +69,9 @@ class TestRedisCache: assert result[keys[1]] == {"result": 2} assert result[keys[2]] == {"result": 3} - def test_ttl_behavior(self): + def test_ttl_behavior(self) -> None: """Test TTL (time-to-live) functionality.""" - key = (("graph", "node"), "ttl_key") + key: FullKey = (("graph", "node"), "ttl_key") values = {key: ({"data": "expires_soon"}, 1)} # 1 second TTL # Set with TTL @@ -88,10 +89,10 @@ class TestRedisCache: result = self.cache.get([key]) assert len(result) == 0 - def test_namespace_isolation(self): + def test_namespace_isolation(self) -> None: """Test that different namespaces are isolated.""" - key1 = (("graph1", "node"), "same_key") - key2 = (("graph2", "node"), "same_key") + key1: FullKey = (("graph1", "node"), "same_key") + key2: FullKey = (("graph2", "node"), "same_key") values = {key1: ({"graph": 1}, None), key2: ({"graph": 2}, None)} @@ -101,9 +102,12 @@ class TestRedisCache: assert result[key1] == {"graph": 1} assert result[key2] == {"graph": 2} - def test_clear_all(self): + def test_clear_all(self) -> None: """Test clearing all cached values.""" - keys = [(("graph", "node1"), "key1"), (("graph", "node2"), "key2")] + keys: list[FullKey] = [ + (("graph", "node1"), "key1"), + (("graph", "node2"), "key2"), + ] values = {keys[0]: ({"result": 1}, None), keys[1]: ({"result": 2}, None)} self.cache.set(values) @@ -119,9 +123,9 @@ class TestRedisCache: result = self.cache.get(keys) assert len(result) == 0 - def test_clear_by_namespace(self): + def test_clear_by_namespace(self) -> None: """Test clearing cached values by namespace.""" - keys = [ + keys: list[FullKey] = [ (("graph1", "node"), "key1"), (("graph2", "node"), "key2"), (("graph1", "other"), "key3"), @@ -142,7 +146,7 @@ class TestRedisCache: assert len(result) == 1 assert result[keys[1]] == {"result": 2} - def test_empty_operations(self): + def test_empty_operations(self) -> None: """Test behavior with empty keys/values.""" # Empty get result = self.cache.get([]) @@ -151,14 +155,14 @@ class TestRedisCache: # Empty set self.cache.set({}) # Should not raise error - def test_nonexistent_keys(self): + def test_nonexistent_keys(self) -> None: """Test getting keys that don't exist.""" - keys = [(("graph", "node"), "nonexistent")] + keys: list[FullKey] = [(("graph", "node"), "nonexistent")] result = self.cache.get(keys) assert len(result) == 0 @pytest.mark.asyncio - async def test_async_operations(self): + async def test_async_operations(self) -> None: """Test async set and get operations with sync Redis client.""" # Create sync Redis client and cache (like main integration tests) client = redis.Redis(host="localhost", port=6379, db=1, decode_responses=False) @@ -167,9 +171,9 @@ class TestRedisCache: except Exception: pytest.skip("Redis not available") - cache = RedisCache(client, prefix="test:async:") + cache: RedisCache = RedisCache(client, prefix="test:async:") - keys = [(("graph", "node"), "async_key")] + keys: list[FullKey] = [(("graph", "node"), "async_key")] values = {keys[0]: ({"async": True}, None)} # Async set (delegates to sync) @@ -184,7 +188,7 @@ class TestRedisCache: client.flushdb() @pytest.mark.asyncio - async def test_async_clear(self): + async def test_async_clear(self) -> None: """Test async clear operations with sync Redis client.""" # Create sync Redis client and cache (like main integration tests) client = redis.Redis(host="localhost", port=6379, db=1, decode_responses=False) @@ -193,9 +197,9 @@ class TestRedisCache: except Exception: pytest.skip("Redis not available") - cache = RedisCache(client, prefix="test:async:") + cache: RedisCache = RedisCache(client, prefix="test:async:") - keys = [(("graph", "node"), "key")] + keys: list[FullKey] = [(("graph", "node"), "key")] values = {keys[0]: ({"data": "test"}, None)} await cache.aset(values) @@ -214,44 +218,44 @@ class TestRedisCache: # Cleanup client.flushdb() - def test_redis_unavailable_get(self): + def test_redis_unavailable_get(self) -> None: """Test behavior when Redis is unavailable during get operations.""" # Create cache with non-existent Redis server bad_client = redis.Redis( host="nonexistent", port=9999, socket_connect_timeout=0.1 ) - cache = RedisCache(bad_client, prefix="test:cache:") + cache: RedisCache = RedisCache(bad_client, prefix="test:cache:") - keys = [(("graph", "node"), "key")] + keys: list[FullKey] = [(("graph", "node"), "key")] result = cache.get(keys) # Should return empty dict when Redis unavailable assert result == {} - def test_redis_unavailable_set(self): + def test_redis_unavailable_set(self) -> None: """Test behavior when Redis is unavailable during set operations.""" # Create cache with non-existent Redis server bad_client = redis.Redis( host="nonexistent", port=9999, socket_connect_timeout=0.1 ) - cache = RedisCache(bad_client, prefix="test:cache:") + cache: RedisCache = RedisCache(bad_client, prefix="test:cache:") - keys = [(("graph", "node"), "key")] + keys: list[FullKey] = [(("graph", "node"), "key")] values = {keys[0]: ({"data": "test"}, None)} # Should not raise exception when Redis unavailable cache.set(values) # Should silently fail @pytest.mark.asyncio - async def test_redis_unavailable_async(self): + async def test_redis_unavailable_async(self) -> None: """Test async behavior when Redis is unavailable.""" # Create sync cache with non-existent Redis server (like main integration tests) bad_client = redis.Redis( host="nonexistent", port=9999, socket_connect_timeout=0.1 ) - cache = RedisCache(bad_client, prefix="test:cache:") + cache: RedisCache = RedisCache(bad_client, prefix="test:cache:") - keys = [(("graph", "node"), "key")] + keys: list[FullKey] = [(("graph", "node"), "key")] values = {keys[0]: ({"data": "test"}, None)} # Should return empty dict for get (delegates to sync) @@ -261,10 +265,10 @@ class TestRedisCache: # Should not raise exception for set (delegates to sync) await cache.aset(values) # Should silently fail - def test_corrupted_data_handling(self): + def test_corrupted_data_handling(self) -> None: """Test handling of corrupted data in Redis.""" # Set some valid data first - keys = [(("graph", "node"), "valid_key")] + keys: list[FullKey] = [(("graph", "node"), "valid_key")] values = {keys[0]: ({"data": "valid"}, None)} self.cache.set(values) @@ -273,33 +277,36 @@ class TestRedisCache: self.client.set(corrupted_key, b"invalid:data:format:too:many:colons") # Should skip corrupted entry and return only valid ones - all_keys = [keys[0], (("graph", "node"), "corrupted_key")] + all_keys: list[FullKey] = [keys[0], (("graph", "node"), "corrupted_key")] result = self.cache.get(all_keys) assert len(result) == 1 assert result[keys[0]] == {"data": "valid"} - def test_key_parsing_edge_cases(self): + def test_key_parsing_edge_cases(self) -> None: """Test key parsing with edge cases.""" # Test empty namespace - key1 = ((), "empty_ns") + key1: FullKey = ((), "empty_ns") values = {key1: ({"data": "empty_ns"}, None)} self.cache.set(values) result = self.cache.get([key1]) assert result[key1] == {"data": "empty_ns"} # Test namespace with special characters - key2 = (("graph:with:colons", "node-with-dashes"), "key_with_underscores") + key2: FullKey = ( + ("graph:with:colons", "node-with-dashes"), + "key_with_underscores", + ) values = {key2: ({"data": "special_chars"}, None)} self.cache.set(values) result = self.cache.get([key2]) assert result[key2] == {"data": "special_chars"} - def test_large_data_serialization(self): + def test_large_data_serialization(self) -> None: """Test handling of large data objects.""" # Create a large data structure large_data = {"large_list": list(range(1000)), "nested": {"data": "x" * 1000}} - key = (("graph", "node"), "large_key") + key: FullKey = (("graph", "node"), "large_key") values = {key: (large_data, None)} self.cache.set(values) diff --git a/libs/checkpoint/tests/test_store.py b/libs/checkpoint/tests/test_store.py index 589eec0e4..1bdfc0408 100644 --- a/libs/checkpoint/tests/test_store.py +++ b/libs/checkpoint/tests/test_store.py @@ -1021,3 +1021,27 @@ async def test_embed_with_path(fake_embeddings: CharacterEmbeddings) -> None: assert len(results) == 3 doc5_result = next(r for r in results if r.key == "doc5") assert doc5_result.score is None + + +def test_non_ascii(fake_embeddings: CharacterEmbeddings) -> None: + """Test support for non-ascii characters""" + store = InMemoryStore( + index={"dims": fake_embeddings.dims, "embed": fake_embeddings} + ) + store.put(("user_123", "memories"), "1", {"text": "这是中文"}) # Chinese + store.put(("user_123", "memories"), "2", {"text": "これは日本語です"}) # Japanese + store.put(("user_123", "memories"), "3", {"text": "이건 한국어야"}) # Korean + store.put(("user_123", "memories"), "4", {"text": "Это русский"}) # Russian + store.put(("user_123", "memories"), "5", {"text": "यह रूसी है"}) # Hindi + + result1 = store.search(("user_123", "memories"), query="这是中文") + result2 = store.search(("user_123", "memories"), query="これは日本語です") + result3 = store.search(("user_123", "memories"), query="이건 한국어야") + result4 = store.search(("user_123", "memories"), query="Это русский") + result5 = store.search(("user_123", "memories"), query="यह रूसी है") + + assert result1[0].key == "1" + assert result2[0].key == "2" + assert result3[0].key == "3" + assert result4[0].key == "4" + assert result5[0].key == "5"