fix(checkpoint): preserve non-ascii text in InMemoryStore embeddings (#6111)

### Description
* Set `ensure_ascii=False` for all `json.dumps` calls in
`get_text_at_path`. Preserves non-ASCII text instead of embedding
`\uXXXX` escapes.

**Before**
```python
store.put(("user_123", "memories"), "1", {"text": "这是中文"})
# embeds {"text": "\\u8fd9\\u662f\\u4e2d\\u6587"}
```

**After**
```python
store.put(("user_123", "memories"), "1", {"text": "这是中文"})
# embeds {"text": "这是中文"}
```

### Tests & Docs

* Add unit test `test_non_ascii` that writes three records (Chinese,
Japanese, Korean) to an `InMemoryStore`, searches with the same strings,
and asserts the correct top hit with a score >= 0.15 for each.

### Issue
Fixes #5946
This commit is contained in:
Caspar Broekhuizen
2025-09-09 17:52:11 +00:00
committed by GitHub
parent 7bbe8d8628
commit 682f39e0d3
5 changed files with 151 additions and 45 deletions
@@ -861,3 +861,46 @@ def test_store_ttl(store):
# Now has been (TTL_SECONDS-2)*2 > TTL_SECONDS + TTL_SECONDS/2
res = store.search(ns, query="bar", refresh_ttl=False)
assert len(res) == 0
@pytest.mark.parametrize(
"vector_type,distance_type",
[
("vector", "cosine"),
("vector", "inner_product"),
("halfvec", "cosine"),
("halfvec", "inner_product"),
],
)
def test_non_ascii(
request: Any,
fake_embeddings: CharacterEmbeddings,
vector_type: str,
distance_type: str,
) -> None:
"""Test support for non-ascii characters"""
with _create_vector_store(
vector_type,
distance_type,
fake_embeddings
) as store:
store.put(("user_123", "memories"), "1", {"text": "这是中文"}) # Chinese
store.put(
("user_123", "memories"), "2", {"text": "これは日本語です"}
) # Japanese
store.put(("user_123", "memories"), "3", {"text": "이건 한국어야"}) # Korean
store.put(("user_123", "memories"), "4", {"text": "Это русский"}) # Russian
store.put(("user_123", "memories"), "5", {"text": "यह रूसी है"}) # Hindi
result1 = store.search(("user_123", "memories"), query="这是中文")
result2 = store.search(("user_123", "memories"), query="これは日本語です")
result3 = store.search(("user_123", "memories"), query="이건 한국어야")
result4 = store.search(("user_123", "memories"), query="Это русский")
result5 = store.search(("user_123", "memories"), query="यह रूसी है")
assert result1[0].key == "1"
assert result2[0].key == "2"
assert result3[0].key == "3"
assert result4[0].key == "4"
assert result5[0].key == "5"