Add optoin to not index

This commit is contained in:
William Fu-Hinthorn
2024-11-25 16:45:23 -08:00
parent 37f09cee5a
commit 03023a968e
5 changed files with 30 additions and 3 deletions
@@ -350,6 +350,8 @@ class BasePostgresStore(Generic[C]):
elif text_fields is None:
text_fields = ["__root__"]
for op in inserts:
if op.index is False:
continue
value = op.value
ns = _namespace_to_text(op.namespace)
k = op.key
@@ -307,6 +307,15 @@ async def test_vector_update_with_embedding(vector_store: AsyncPostgresStore) ->
if r.key == "doc1":
assert r.response_metadata["score"] > after_score
# Don't index this one
await vector_store.aput(
("test",), "doc4", {"text": "new text about dogs"}, index=False
)
results_new = await vector_store.asearch(
("test",), query="new text about dogs", limit=3
)
assert not any(r.key == "doc4" for r in results_new)
async def test_vector_search_with_filters(vector_store: AsyncPostgresStore) -> None:
"""Test combining vector search with filters."""
@@ -458,6 +458,11 @@ def test_vector_update_with_embedding(vector_store: PostgresStore) -> None:
if r.key == "doc1":
assert r.response_metadata["score"] > after_score
# Don't index this one
vector_store.put(("test",), "doc4", {"text": "new text about dogs"}, index=False)
results_new = vector_store.search(("test",), query="new text about dogs", limit=3)
assert not any(r.key == "doc4" for r in results_new)
def test_vector_search_with_filters(vector_store: PostgresStore) -> None:
"""Test combining vector search with filters."""
@@ -176,6 +176,12 @@ class PutOp(NamedTuple):
- Values can be of any serializable type
- If None, it indicates that the item should be deleted
"""
index: Optional[bool] = None
"""Whether to index the item (if supported by the store).
Defaults to True if the store supports indexing. This will embed the document
so it can be queried using search.
"""
NameSpacePath = tuple[Union[str, Literal["*"]], ...]
@@ -351,9 +357,11 @@ class BaseStore(ABC):
namespace: Hierarchical path for the item.
key: Unique identifier within the namespace.
value: Dictionary containing the item's data.
index: Whether to index the item (if supported by the store).
Defaults to True if the store supports indexing.
"""
_validate_namespace(namespace)
self.batch([PutOp(namespace, key, value)])
self.batch([PutOp(namespace, key, value, index=index)])
def delete(self, namespace: tuple[str, ...], key: str) -> None:
"""Delete an item.
@@ -468,9 +476,11 @@ class BaseStore(ABC):
namespace: Hierarchical path for the item.
key: Unique identifier within the namespace.
value: Dictionary containing the item's data.
index: Whether to index the item (if supported by the store).
Defaults to True if the store supports indexing.
"""
_validate_namespace(namespace)
await self.abatch([PutOp(namespace, key, value)])
await self.abatch([PutOp(namespace, key, value, index)])
async def adelete(self, namespace: tuple[str, ...], key: str) -> None:
"""Asynchronously delete an item.
@@ -57,10 +57,11 @@ class AsyncBatchedBaseStore(BaseStore):
namespace: tuple[str, ...],
key: str,
value: dict[str, Any],
index: Optional[bool] = None,
) -> None:
_validate_namespace(namespace)
fut = self._loop.create_future()
self._aqueue[fut] = PutOp(namespace, key, value)
self._aqueue[fut] = PutOp(namespace, key, value, index)
return await fut
async def adelete(