mirror of
https://github.com/langchain-ai/langgraph.git
synced 2026-08-28 10:49:56 +02:00
fix(checkpoint): preserve non-ascii text in InMemoryStore embeddings (#6111)
### Description
* Set `ensure_ascii=False` for all `json.dumps` calls in
`get_text_at_path`. Preserves non-ASCII text instead of embedding
`\uXXXX` escapes.
**Before**
```python
store.put(("user_123", "memories"), "1", {"text": "这是中文"})
# embeds {"text": "\\u8fd9\\u662f\\u4e2d\\u6587"}
```
**After**
```python
store.put(("user_123", "memories"), "1", {"text": "这是中文"})
# embeds {"text": "这是中文"}
```
### Tests & Docs
* Add unit test `test_non_ascii` that writes three records (Chinese,
Japanese, Korean) to an `InMemoryStore`, searches with the same strings,
and asserts the correct top hit with a score >= 0.15 for each.
### Issue
Fixes #5946
This commit is contained in:
Caspar Broekhuizen
@@ -861,3 +861,46 @@ def test_store_ttl(store):
|
||||
# Now has been (TTL_SECONDS-2)*2 > TTL_SECONDS + TTL_SECONDS/2
|
||||
res = store.search(ns, query="bar", refresh_ttl=False)
|
||||
assert len(res) == 0
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"vector_type,distance_type",
|
||||
[
|
||||
("vector", "cosine"),
|
||||
("vector", "inner_product"),
|
||||
("halfvec", "cosine"),
|
||||
("halfvec", "inner_product"),
|
||||
],
|
||||
)
|
||||
def test_non_ascii(
|
||||
request: Any,
|
||||
fake_embeddings: CharacterEmbeddings,
|
||||
vector_type: str,
|
||||
distance_type: str,
|
||||
) -> None:
|
||||
"""Test support for non-ascii characters"""
|
||||
with _create_vector_store(
|
||||
vector_type,
|
||||
distance_type,
|
||||
fake_embeddings
|
||||
) as store:
|
||||
|
||||
store.put(("user_123", "memories"), "1", {"text": "这是中文"}) # Chinese
|
||||
store.put(
|
||||
("user_123", "memories"), "2", {"text": "これは日本語です"}
|
||||
) # Japanese
|
||||
store.put(("user_123", "memories"), "3", {"text": "이건 한국어야"}) # Korean
|
||||
store.put(("user_123", "memories"), "4", {"text": "Это русский"}) # Russian
|
||||
store.put(("user_123", "memories"), "5", {"text": "यह रूसी है"}) # Hindi
|
||||
|
||||
result1 = store.search(("user_123", "memories"), query="这是中文")
|
||||
result2 = store.search(("user_123", "memories"), query="これは日本語です")
|
||||
result3 = store.search(("user_123", "memories"), query="이건 한국어야")
|
||||
result4 = store.search(("user_123", "memories"), query="Это русский")
|
||||
result5 = store.search(("user_123", "memories"), query="यह रूसी है")
|
||||
|
||||
assert result1[0].key == "1"
|
||||
assert result2[0].key == "2"
|
||||
assert result3[0].key == "3"
|
||||
assert result4[0].key == "4"
|
||||
assert result5[0].key == "5"
|
||||
|
||||
@@ -1067,3 +1067,31 @@ def test_sql_injection_vulnerability(store: SqliteStore) -> None:
|
||||
|
||||
with pytest.raises(ValueError, match="Invalid filter key"):
|
||||
store.search(("docs",), filter={malicious_key: "dummy"})
|
||||
|
||||
|
||||
@pytest.mark.parametrize("distance_type", VECTOR_TYPES)
|
||||
def test_non_ascii(
|
||||
fake_embeddings: CharacterEmbeddings,
|
||||
distance_type: str,
|
||||
) -> None:
|
||||
"""Test support for non-ascii characters"""
|
||||
with create_vector_store(fake_embeddings, distance_type=distance_type) as store:
|
||||
store.put(("user_123", "memories"), "1", {"text": "这是中文"}) # Chinese
|
||||
store.put(
|
||||
("user_123", "memories"), "2", {"text": "これは日本語です"}
|
||||
) # Japanese
|
||||
store.put(("user_123", "memories"), "3", {"text": "이건 한국어야"}) # Korean
|
||||
store.put(("user_123", "memories"), "4", {"text": "Это русский"}) # Russian
|
||||
store.put(("user_123", "memories"), "5", {"text": "यह रूसी है"}) # Hindi
|
||||
|
||||
result1 = store.search(("user_123", "memories"), query="这是中文")
|
||||
result2 = store.search(("user_123", "memories"), query="これは日本語です")
|
||||
result3 = store.search(("user_123", "memories"), query="이건 한국어야")
|
||||
result4 = store.search(("user_123", "memories"), query="Это русский")
|
||||
result5 = store.search(("user_123", "memories"), query="यह रूसी है")
|
||||
|
||||
assert result1[0].key == "1"
|
||||
assert result2[0].key == "2"
|
||||
assert result3[0].key == "3"
|
||||
assert result4[0].key == "4"
|
||||
assert result5[0].key == "5"
|
||||
|
||||
@@ -238,7 +238,7 @@ def get_text_at_path(obj: Any, path: str | list[str]) -> list[str]:
|
||||
- Nested paths in multi-field: "{field1,nested.field2}"
|
||||
"""
|
||||
if not path or path == "$":
|
||||
return [json.dumps(obj, sort_keys=True)]
|
||||
return [json.dumps(obj, sort_keys=True, ensure_ascii=False)]
|
||||
|
||||
tokens = tokenize_path(path) if isinstance(path, str) else path
|
||||
|
||||
@@ -249,7 +249,7 @@ def get_text_at_path(obj: Any, path: str | list[str]) -> list[str]:
|
||||
elif obj is None:
|
||||
return []
|
||||
elif isinstance(obj, (list, dict)):
|
||||
return [json.dumps(obj, sort_keys=True)]
|
||||
return [json.dumps(obj, sort_keys=True, ensure_ascii=False)]
|
||||
return []
|
||||
|
||||
token = tokens[pos]
|
||||
@@ -295,7 +295,11 @@ def get_text_at_path(obj: Any, path: str | list[str]) -> list[str]:
|
||||
if isinstance(current_obj, (str, int, float, bool)):
|
||||
results.append(str(current_obj))
|
||||
elif isinstance(current_obj, (list, dict)):
|
||||
results.append(json.dumps(current_obj, sort_keys=True))
|
||||
results.append(
|
||||
json.dumps(
|
||||
current_obj, sort_keys=True, ensure_ascii=False
|
||||
)
|
||||
)
|
||||
|
||||
# Handle wildcard
|
||||
elif token == "*":
|
||||
|
||||
@@ -5,12 +5,13 @@ import time
|
||||
import pytest
|
||||
import redis
|
||||
|
||||
from langgraph.cache.base import FullKey
|
||||
from langgraph.cache.redis import RedisCache
|
||||
|
||||
|
||||
class TestRedisCache:
|
||||
@pytest.fixture(autouse=True)
|
||||
def setup(self):
|
||||
def setup(self) -> None:
|
||||
"""Set up test Redis client and cache."""
|
||||
self.client = redis.Redis(
|
||||
host="localhost", port=6379, db=0, decode_responses=False
|
||||
@@ -20,21 +21,21 @@ class TestRedisCache:
|
||||
except redis.ConnectionError:
|
||||
pytest.skip("Redis server not available")
|
||||
|
||||
self.cache = RedisCache(self.client, prefix="test:cache:")
|
||||
self.cache: RedisCache = RedisCache(self.client, prefix="test:cache:")
|
||||
|
||||
# Clean up before each test
|
||||
self.client.flushdb()
|
||||
|
||||
def teardown_method(self):
|
||||
def teardown_method(self) -> None:
|
||||
"""Clean up after each test."""
|
||||
try:
|
||||
self.client.flushdb()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def test_basic_set_and_get(self):
|
||||
def test_basic_set_and_get(self) -> None:
|
||||
"""Test basic set and get operations."""
|
||||
keys = [(("graph", "node"), "key1")]
|
||||
keys: list[FullKey] = [(("graph", "node"), "key1")]
|
||||
values = {keys[0]: ({"result": 42}, None)}
|
||||
|
||||
# Set value
|
||||
@@ -45,9 +46,9 @@ class TestRedisCache:
|
||||
assert len(result) == 1
|
||||
assert result[keys[0]] == {"result": 42}
|
||||
|
||||
def test_batch_operations(self):
|
||||
def test_batch_operations(self) -> None:
|
||||
"""Test batch set and get operations."""
|
||||
keys = [
|
||||
keys: list[FullKey] = [
|
||||
(("graph", "node1"), "key1"),
|
||||
(("graph", "node2"), "key2"),
|
||||
(("other", "node"), "key3"),
|
||||
@@ -68,9 +69,9 @@ class TestRedisCache:
|
||||
assert result[keys[1]] == {"result": 2}
|
||||
assert result[keys[2]] == {"result": 3}
|
||||
|
||||
def test_ttl_behavior(self):
|
||||
def test_ttl_behavior(self) -> None:
|
||||
"""Test TTL (time-to-live) functionality."""
|
||||
key = (("graph", "node"), "ttl_key")
|
||||
key: FullKey = (("graph", "node"), "ttl_key")
|
||||
values = {key: ({"data": "expires_soon"}, 1)} # 1 second TTL
|
||||
|
||||
# Set with TTL
|
||||
@@ -88,10 +89,10 @@ class TestRedisCache:
|
||||
result = self.cache.get([key])
|
||||
assert len(result) == 0
|
||||
|
||||
def test_namespace_isolation(self):
|
||||
def test_namespace_isolation(self) -> None:
|
||||
"""Test that different namespaces are isolated."""
|
||||
key1 = (("graph1", "node"), "same_key")
|
||||
key2 = (("graph2", "node"), "same_key")
|
||||
key1: FullKey = (("graph1", "node"), "same_key")
|
||||
key2: FullKey = (("graph2", "node"), "same_key")
|
||||
|
||||
values = {key1: ({"graph": 1}, None), key2: ({"graph": 2}, None)}
|
||||
|
||||
@@ -101,9 +102,12 @@ class TestRedisCache:
|
||||
assert result[key1] == {"graph": 1}
|
||||
assert result[key2] == {"graph": 2}
|
||||
|
||||
def test_clear_all(self):
|
||||
def test_clear_all(self) -> None:
|
||||
"""Test clearing all cached values."""
|
||||
keys = [(("graph", "node1"), "key1"), (("graph", "node2"), "key2")]
|
||||
keys: list[FullKey] = [
|
||||
(("graph", "node1"), "key1"),
|
||||
(("graph", "node2"), "key2"),
|
||||
]
|
||||
values = {keys[0]: ({"result": 1}, None), keys[1]: ({"result": 2}, None)}
|
||||
|
||||
self.cache.set(values)
|
||||
@@ -119,9 +123,9 @@ class TestRedisCache:
|
||||
result = self.cache.get(keys)
|
||||
assert len(result) == 0
|
||||
|
||||
def test_clear_by_namespace(self):
|
||||
def test_clear_by_namespace(self) -> None:
|
||||
"""Test clearing cached values by namespace."""
|
||||
keys = [
|
||||
keys: list[FullKey] = [
|
||||
(("graph1", "node"), "key1"),
|
||||
(("graph2", "node"), "key2"),
|
||||
(("graph1", "other"), "key3"),
|
||||
@@ -142,7 +146,7 @@ class TestRedisCache:
|
||||
assert len(result) == 1
|
||||
assert result[keys[1]] == {"result": 2}
|
||||
|
||||
def test_empty_operations(self):
|
||||
def test_empty_operations(self) -> None:
|
||||
"""Test behavior with empty keys/values."""
|
||||
# Empty get
|
||||
result = self.cache.get([])
|
||||
@@ -151,14 +155,14 @@ class TestRedisCache:
|
||||
# Empty set
|
||||
self.cache.set({}) # Should not raise error
|
||||
|
||||
def test_nonexistent_keys(self):
|
||||
def test_nonexistent_keys(self) -> None:
|
||||
"""Test getting keys that don't exist."""
|
||||
keys = [(("graph", "node"), "nonexistent")]
|
||||
keys: list[FullKey] = [(("graph", "node"), "nonexistent")]
|
||||
result = self.cache.get(keys)
|
||||
assert len(result) == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_operations(self):
|
||||
async def test_async_operations(self) -> None:
|
||||
"""Test async set and get operations with sync Redis client."""
|
||||
# Create sync Redis client and cache (like main integration tests)
|
||||
client = redis.Redis(host="localhost", port=6379, db=1, decode_responses=False)
|
||||
@@ -167,9 +171,9 @@ class TestRedisCache:
|
||||
except Exception:
|
||||
pytest.skip("Redis not available")
|
||||
|
||||
cache = RedisCache(client, prefix="test:async:")
|
||||
cache: RedisCache = RedisCache(client, prefix="test:async:")
|
||||
|
||||
keys = [(("graph", "node"), "async_key")]
|
||||
keys: list[FullKey] = [(("graph", "node"), "async_key")]
|
||||
values = {keys[0]: ({"async": True}, None)}
|
||||
|
||||
# Async set (delegates to sync)
|
||||
@@ -184,7 +188,7 @@ class TestRedisCache:
|
||||
client.flushdb()
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_async_clear(self):
|
||||
async def test_async_clear(self) -> None:
|
||||
"""Test async clear operations with sync Redis client."""
|
||||
# Create sync Redis client and cache (like main integration tests)
|
||||
client = redis.Redis(host="localhost", port=6379, db=1, decode_responses=False)
|
||||
@@ -193,9 +197,9 @@ class TestRedisCache:
|
||||
except Exception:
|
||||
pytest.skip("Redis not available")
|
||||
|
||||
cache = RedisCache(client, prefix="test:async:")
|
||||
cache: RedisCache = RedisCache(client, prefix="test:async:")
|
||||
|
||||
keys = [(("graph", "node"), "key")]
|
||||
keys: list[FullKey] = [(("graph", "node"), "key")]
|
||||
values = {keys[0]: ({"data": "test"}, None)}
|
||||
|
||||
await cache.aset(values)
|
||||
@@ -214,44 +218,44 @@ class TestRedisCache:
|
||||
# Cleanup
|
||||
client.flushdb()
|
||||
|
||||
def test_redis_unavailable_get(self):
|
||||
def test_redis_unavailable_get(self) -> None:
|
||||
"""Test behavior when Redis is unavailable during get operations."""
|
||||
# Create cache with non-existent Redis server
|
||||
bad_client = redis.Redis(
|
||||
host="nonexistent", port=9999, socket_connect_timeout=0.1
|
||||
)
|
||||
cache = RedisCache(bad_client, prefix="test:cache:")
|
||||
cache: RedisCache = RedisCache(bad_client, prefix="test:cache:")
|
||||
|
||||
keys = [(("graph", "node"), "key")]
|
||||
keys: list[FullKey] = [(("graph", "node"), "key")]
|
||||
result = cache.get(keys)
|
||||
|
||||
# Should return empty dict when Redis unavailable
|
||||
assert result == {}
|
||||
|
||||
def test_redis_unavailable_set(self):
|
||||
def test_redis_unavailable_set(self) -> None:
|
||||
"""Test behavior when Redis is unavailable during set operations."""
|
||||
# Create cache with non-existent Redis server
|
||||
bad_client = redis.Redis(
|
||||
host="nonexistent", port=9999, socket_connect_timeout=0.1
|
||||
)
|
||||
cache = RedisCache(bad_client, prefix="test:cache:")
|
||||
cache: RedisCache = RedisCache(bad_client, prefix="test:cache:")
|
||||
|
||||
keys = [(("graph", "node"), "key")]
|
||||
keys: list[FullKey] = [(("graph", "node"), "key")]
|
||||
values = {keys[0]: ({"data": "test"}, None)}
|
||||
|
||||
# Should not raise exception when Redis unavailable
|
||||
cache.set(values) # Should silently fail
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_redis_unavailable_async(self):
|
||||
async def test_redis_unavailable_async(self) -> None:
|
||||
"""Test async behavior when Redis is unavailable."""
|
||||
# Create sync cache with non-existent Redis server (like main integration tests)
|
||||
bad_client = redis.Redis(
|
||||
host="nonexistent", port=9999, socket_connect_timeout=0.1
|
||||
)
|
||||
cache = RedisCache(bad_client, prefix="test:cache:")
|
||||
cache: RedisCache = RedisCache(bad_client, prefix="test:cache:")
|
||||
|
||||
keys = [(("graph", "node"), "key")]
|
||||
keys: list[FullKey] = [(("graph", "node"), "key")]
|
||||
values = {keys[0]: ({"data": "test"}, None)}
|
||||
|
||||
# Should return empty dict for get (delegates to sync)
|
||||
@@ -261,10 +265,10 @@ class TestRedisCache:
|
||||
# Should not raise exception for set (delegates to sync)
|
||||
await cache.aset(values) # Should silently fail
|
||||
|
||||
def test_corrupted_data_handling(self):
|
||||
def test_corrupted_data_handling(self) -> None:
|
||||
"""Test handling of corrupted data in Redis."""
|
||||
# Set some valid data first
|
||||
keys = [(("graph", "node"), "valid_key")]
|
||||
keys: list[FullKey] = [(("graph", "node"), "valid_key")]
|
||||
values = {keys[0]: ({"data": "valid"}, None)}
|
||||
self.cache.set(values)
|
||||
|
||||
@@ -273,33 +277,36 @@ class TestRedisCache:
|
||||
self.client.set(corrupted_key, b"invalid:data:format:too:many:colons")
|
||||
|
||||
# Should skip corrupted entry and return only valid ones
|
||||
all_keys = [keys[0], (("graph", "node"), "corrupted_key")]
|
||||
all_keys: list[FullKey] = [keys[0], (("graph", "node"), "corrupted_key")]
|
||||
result = self.cache.get(all_keys)
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[keys[0]] == {"data": "valid"}
|
||||
|
||||
def test_key_parsing_edge_cases(self):
|
||||
def test_key_parsing_edge_cases(self) -> None:
|
||||
"""Test key parsing with edge cases."""
|
||||
# Test empty namespace
|
||||
key1 = ((), "empty_ns")
|
||||
key1: FullKey = ((), "empty_ns")
|
||||
values = {key1: ({"data": "empty_ns"}, None)}
|
||||
self.cache.set(values)
|
||||
result = self.cache.get([key1])
|
||||
assert result[key1] == {"data": "empty_ns"}
|
||||
|
||||
# Test namespace with special characters
|
||||
key2 = (("graph:with:colons", "node-with-dashes"), "key_with_underscores")
|
||||
key2: FullKey = (
|
||||
("graph:with:colons", "node-with-dashes"),
|
||||
"key_with_underscores",
|
||||
)
|
||||
values = {key2: ({"data": "special_chars"}, None)}
|
||||
self.cache.set(values)
|
||||
result = self.cache.get([key2])
|
||||
assert result[key2] == {"data": "special_chars"}
|
||||
|
||||
def test_large_data_serialization(self):
|
||||
def test_large_data_serialization(self) -> None:
|
||||
"""Test handling of large data objects."""
|
||||
# Create a large data structure
|
||||
large_data = {"large_list": list(range(1000)), "nested": {"data": "x" * 1000}}
|
||||
key = (("graph", "node"), "large_key")
|
||||
key: FullKey = (("graph", "node"), "large_key")
|
||||
values = {key: (large_data, None)}
|
||||
|
||||
self.cache.set(values)
|
||||
|
||||
@@ -1021,3 +1021,27 @@ async def test_embed_with_path(fake_embeddings: CharacterEmbeddings) -> None:
|
||||
assert len(results) == 3
|
||||
doc5_result = next(r for r in results if r.key == "doc5")
|
||||
assert doc5_result.score is None
|
||||
|
||||
|
||||
def test_non_ascii(fake_embeddings: CharacterEmbeddings) -> None:
|
||||
"""Test support for non-ascii characters"""
|
||||
store = InMemoryStore(
|
||||
index={"dims": fake_embeddings.dims, "embed": fake_embeddings}
|
||||
)
|
||||
store.put(("user_123", "memories"), "1", {"text": "这是中文"}) # Chinese
|
||||
store.put(("user_123", "memories"), "2", {"text": "これは日本語です"}) # Japanese
|
||||
store.put(("user_123", "memories"), "3", {"text": "이건 한국어야"}) # Korean
|
||||
store.put(("user_123", "memories"), "4", {"text": "Это русский"}) # Russian
|
||||
store.put(("user_123", "memories"), "5", {"text": "यह रूसी है"}) # Hindi
|
||||
|
||||
result1 = store.search(("user_123", "memories"), query="这是中文")
|
||||
result2 = store.search(("user_123", "memories"), query="これは日本語です")
|
||||
result3 = store.search(("user_123", "memories"), query="이건 한국어야")
|
||||
result4 = store.search(("user_123", "memories"), query="Это русский")
|
||||
result5 = store.search(("user_123", "memories"), query="यह रूसी है")
|
||||
|
||||
assert result1[0].key == "1"
|
||||
assert result2[0].key == "2"
|
||||
assert result3[0].key == "3"
|
||||
assert result4[0].key == "4"
|
||||
assert result5[0].key == "5"
|
||||
|
||||
Reference in New Issue
Block a user