mirror of
https://github.com/langchain-ai/langgraph.git
synced 2026-08-17 21:25:46 +02:00
fix(checkpoint-postgres,checkpoint-sqlite): scope namespace matching to segment boundaries (#8478)
## Summary
Namespace scoping in the Postgres and SQLite stores matched the
dot-joined prefix with `LIKE '<path>%'`, which does not respect the `.`
separator — a search scoped to `("foo",)` also returned rows under
`("foobar",)`. Scoping now matches the namespace exactly or requires the
separator before any remainder, and pattern metacharacters in labels are
escaped.
`list_namespaces` moves to segment-aware matching for prefix and suffix
conditions, since neither `LIKE` nor `GLOB` can express "any character
except the separator".
Per-package reasoning is in the commit message.
## Compatibility
`*` in a `list_namespaces` match path now spans exactly one segment,
restoring the documented behavior (`NamespacePath` documents `("cache",
"*", "v1")` as "any cache category with v1 version") and matching
`InMemoryStore`. To match at any depth, combine both conditions, which
are ANDed: `list_namespaces(prefix=["uid"], suffix=["alice"])`.
## Test plan
- [x] `make format` / `make lint` / `make test` from
`libs/checkpoint-postgres` (224 passed) and `libs/checkpoint-sqlite`
(112 passed, 3 skipped)
This commit is contained in:
@@ -4,6 +4,7 @@ import asyncio
|
||||
import concurrent.futures
|
||||
import json
|
||||
import logging
|
||||
import re
|
||||
import threading
|
||||
from collections import defaultdict
|
||||
from collections.abc import Callable, Iterable, Iterator, Sequence
|
||||
@@ -463,8 +464,9 @@ class BasePostgresStore(Generic[C]):
|
||||
ns_condition = "TRUE"
|
||||
ns_param: Sequence[str] | None = None
|
||||
if op.namespace_prefix:
|
||||
ns_condition = "store.prefix LIKE %s"
|
||||
ns_param = (f"{_namespace_to_text(op.namespace_prefix)}%",)
|
||||
ns_condition, ns_param = _namespace_prefix_condition(
|
||||
op.namespace_prefix
|
||||
)
|
||||
else:
|
||||
ns_param = ()
|
||||
|
||||
@@ -617,15 +619,17 @@ class BasePostgresStore(Generic[C]):
|
||||
conditions.append("(expires_at IS NULL OR expires_at > NOW())")
|
||||
if op.match_conditions:
|
||||
for condition in op.match_conditions:
|
||||
if condition.match_type == "prefix":
|
||||
conditions.append("prefix LIKE %s")
|
||||
if condition.match_type in ("prefix", "suffix"):
|
||||
if not condition.path:
|
||||
# An empty path constrains nothing; skipping keeps it a
|
||||
# no-op rather than emitting a pattern that matches no
|
||||
# namespace at all.
|
||||
continue
|
||||
conditions.append("prefix ~ %s")
|
||||
params.append(
|
||||
f"{_namespace_to_text(condition.path, handle_wildcards=True)}%"
|
||||
)
|
||||
elif condition.match_type == "suffix":
|
||||
conditions.append("prefix LIKE %s")
|
||||
params.append(
|
||||
f"%{_namespace_to_text(condition.path, handle_wildcards=True)}"
|
||||
_namespace_match_pattern(
|
||||
condition.path, condition.match_type
|
||||
)
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
@@ -1271,15 +1275,59 @@ def _get_index_params(store: Any) -> tuple[str, dict[str, Any]]:
|
||||
return kind, sanitized
|
||||
|
||||
|
||||
def _namespace_to_text(
|
||||
namespace: tuple[str, ...], handle_wildcards: bool = False
|
||||
) -> str:
|
||||
def _namespace_to_text(namespace: tuple[str, ...]) -> str:
|
||||
"""Convert namespace tuple to text string."""
|
||||
if handle_wildcards:
|
||||
namespace = tuple("%" if val == "*" else val for val in namespace)
|
||||
return ".".join(namespace)
|
||||
|
||||
|
||||
def _escape_like_literal(text: str) -> str:
|
||||
"""Escape LIKE metacharacters so `text` is matched literally.
|
||||
|
||||
Namespace labels may contain `_` and `%`, which would otherwise act as
|
||||
wildcards: `("user_1",)` would match `("userX1",)`. Backslash is escaped
|
||||
first so it cannot escape the following character. Requires an explicit
|
||||
`ESCAPE '\\'` clause on the pattern.
|
||||
"""
|
||||
return text.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
|
||||
|
||||
|
||||
def _namespace_prefix_condition(namespace_prefix: tuple[str, ...]) -> tuple[str, tuple]:
|
||||
"""Build the SQL scoping a search to a namespace and its descendants.
|
||||
|
||||
Matches the namespace exactly or requires the `.` separator before any
|
||||
remainder, so a prefix of `("foo",)` does not also match `("foobar",)`.
|
||||
|
||||
Both arms stay index-friendly: equality on the `(prefix, key)` primary key,
|
||||
the anchored LIKE on the `prefix text_pattern_ops` index.
|
||||
|
||||
Only the LIKE arm is escaped -- equality does not interpret metacharacters,
|
||||
so escaping it would stop `("user_1",)` from matching itself.
|
||||
"""
|
||||
path = _namespace_to_text(namespace_prefix)
|
||||
condition = r"(store.prefix = %s OR store.prefix LIKE %s ESCAPE '\')"
|
||||
return condition, (path, f"{_escape_like_literal(path)}.%")
|
||||
|
||||
|
||||
def _namespace_match_pattern(path: tuple[str, ...], match_type: str) -> str:
|
||||
"""Build a POSIX regex matching the dot-joined prefix on whole segments.
|
||||
|
||||
Needed because `LIKE` cannot express "any character except the separator".
|
||||
Matches how `InMemoryStore` compares namespaces element-wise.
|
||||
|
||||
`*` matches exactly one segment. Prefix matches stay open-ended but must end
|
||||
on a separator; suffix matches anchor at the end and begin on one.
|
||||
|
||||
Examples:
|
||||
prefix ("uid", "*", "alice") -> ^uid\\.[^.]+\\.alice(\\.|\\Z)
|
||||
suffix ("alice",) -> (^|\\.)alice\\Z
|
||||
"""
|
||||
segments = ("[^.]+" if part == "*" else re.escape(part) for part in path)
|
||||
body = r"\.".join(segments)
|
||||
if match_type == "suffix":
|
||||
return rf"(^|\.){body}\Z"
|
||||
return rf"^{body}(\.|\Z)"
|
||||
|
||||
|
||||
def _row_to_item(
|
||||
namespace: tuple[str, ...],
|
||||
row: Row,
|
||||
|
||||
@@ -20,6 +20,10 @@ from langgraph.store.base import (
|
||||
from psycopg import Connection
|
||||
|
||||
from langgraph.store.postgres import PostgresStore
|
||||
from langgraph.store.postgres.base import (
|
||||
_escape_like_literal,
|
||||
_namespace_match_pattern,
|
||||
)
|
||||
from tests.conftest import (
|
||||
DEFAULT_URI,
|
||||
VECTOR_TYPES,
|
||||
@@ -326,6 +330,127 @@ def test_list_namespaces(store) -> None:
|
||||
store.delete(namespace, "dummy")
|
||||
|
||||
|
||||
def test_escape_like_literal() -> None:
|
||||
assert _escape_like_literal("users.alice") == "users.alice"
|
||||
assert _escape_like_literal("user_1") == r"user\_1"
|
||||
assert _escape_like_literal("100%") == r"100\%"
|
||||
assert _escape_like_literal("a\\b") == "a\\\\b"
|
||||
assert _escape_like_literal("") == ""
|
||||
|
||||
|
||||
def test_namespace_match_pattern() -> None:
|
||||
assert _namespace_match_pattern(("foo",), "prefix") == r"^foo(\.|\Z)"
|
||||
assert _namespace_match_pattern(("uid", "users"), "prefix") == r"^uid\.users(\.|\Z)"
|
||||
assert (
|
||||
_namespace_match_pattern(("uid", "*", "alice"), "prefix")
|
||||
== r"^uid\.[^.]+\.alice(\.|\Z)"
|
||||
)
|
||||
assert _namespace_match_pattern(("alice",), "suffix") == r"(^|\.)alice\Z"
|
||||
|
||||
# Regex metacharacters in a label are quoted, not interpreted.
|
||||
pattern = _namespace_match_pattern(("a.b+c",), "prefix")
|
||||
assert re.match(pattern, "a.b+c.child")
|
||||
assert not re.match(pattern, "axbbbc")
|
||||
|
||||
|
||||
def test_search_namespace_segment_boundary(store) -> None:
|
||||
"""Prefix scoping must stop at namespace segment boundaries.
|
||||
|
||||
Namespaces are stored dot-joined, so matching the raw text also returns
|
||||
siblings sharing leading characters. Callers isolate tenants by namespace,
|
||||
so prefix-shaped ids (1 vs 12) would cross-read.
|
||||
"""
|
||||
for namespace in [
|
||||
("foo",),
|
||||
("foo", "child"),
|
||||
("foo", "child", "deep"),
|
||||
("foobar",),
|
||||
("foobar", "baz"),
|
||||
("foo2",),
|
||||
]:
|
||||
store.put(namespace, "k", {"v": 1})
|
||||
|
||||
def _namespaces(prefix: tuple[str, ...]) -> set[tuple[str, ...]]:
|
||||
return {item.namespace for item in store.search(prefix, limit=100)}
|
||||
|
||||
assert _namespaces(("foo",)) == {
|
||||
("foo",),
|
||||
("foo", "child"),
|
||||
("foo", "child", "deep"),
|
||||
}
|
||||
# The sibling scope is independent, not merely narrower.
|
||||
assert _namespaces(("foobar",)) == {("foobar",), ("foobar", "baz")}
|
||||
assert _namespaces(("foo", "child")) == {("foo", "child"), ("foo", "child", "deep")}
|
||||
assert _namespaces(("foo2",)) == {("foo2",)}
|
||||
assert _namespaces(("fo",)) == set()
|
||||
|
||||
|
||||
def test_search_empty_prefix_is_unconstrained(store) -> None:
|
||||
"""An empty prefix constrains nothing and must return every namespace."""
|
||||
for namespace in [("a",), ("b", "c"), ("d", "e", "f")]:
|
||||
store.put(namespace, "k", {"v": 1})
|
||||
|
||||
assert {item.namespace for item in store.search((), limit=100)} == {
|
||||
("a",),
|
||||
("b", "c"),
|
||||
("d", "e", "f"),
|
||||
}
|
||||
|
||||
|
||||
def test_search_namespace_like_metacharacters(store) -> None:
|
||||
"""`_` and `%` are legal namespace labels, not LIKE wildcards."""
|
||||
for namespace in [
|
||||
("user_1",),
|
||||
("user_1", "child"),
|
||||
("userX1",),
|
||||
("a%b",),
|
||||
("axxb",),
|
||||
]:
|
||||
store.put(namespace, "k", {"v": 1})
|
||||
|
||||
def _namespaces(prefix: tuple[str, ...]) -> set[tuple[str, ...]]:
|
||||
return {item.namespace for item in store.search(prefix, limit=100)}
|
||||
|
||||
# Also asserts the namespace still matches itself, which catches escaping
|
||||
# the equality arm by mistake.
|
||||
assert _namespaces(("user_1",)) == {("user_1",), ("user_1", "child")}
|
||||
assert _namespaces(("a%b",)) == {("a%b",)}
|
||||
|
||||
|
||||
def test_list_namespaces_segment_boundary(store) -> None:
|
||||
for namespace in [
|
||||
("foo",),
|
||||
("foo", "child"),
|
||||
("foobar",),
|
||||
("foobar", "baz"),
|
||||
("uid", "users", "alice"),
|
||||
("uid", "users", "malice"),
|
||||
("uid", "a", "b", "alice"),
|
||||
]:
|
||||
store.put(namespace, "k", {"v": 1})
|
||||
|
||||
assert set(store.list_namespaces(prefix=["foo"], limit=100)) == {
|
||||
("foo",),
|
||||
("foo", "child"),
|
||||
}
|
||||
# Suffix must align to a segment: "malice" does not end with the "alice"
|
||||
# segment.
|
||||
assert set(store.list_namespaces(suffix=["alice"], limit=100)) == {
|
||||
("uid", "users", "alice"),
|
||||
("uid", "a", "b", "alice"),
|
||||
}
|
||||
# "*" spans exactly one segment.
|
||||
assert set(store.list_namespaces(prefix=["uid", "*", "alice"], limit=100)) == {
|
||||
("uid", "users", "alice"),
|
||||
}
|
||||
# Prefix matching stays open-ended across depth.
|
||||
assert set(store.list_namespaces(prefix=["uid"], limit=100)) == {
|
||||
("uid", "users", "alice"),
|
||||
("uid", "users", "malice"),
|
||||
("uid", "a", "b", "alice"),
|
||||
}
|
||||
|
||||
|
||||
def test_search(store) -> None:
|
||||
# Create test data
|
||||
test_data = [
|
||||
@@ -1026,3 +1151,16 @@ def test_non_ascii(
|
||||
assert result3[0].key == "3"
|
||||
assert result4[0].key == "4"
|
||||
assert result5[0].key == "5"
|
||||
|
||||
|
||||
def test_namespace_labels_with_trailing_newline(store) -> None:
|
||||
"""Labels may contain newlines, and must not match a differently-named label."""
|
||||
store.put(("users", "alice"), "k", {"v": 1})
|
||||
store.put(("users", "alice\n"), "k", {"v": 2})
|
||||
|
||||
assert set(store.list_namespaces(suffix=["alice"], limit=100)) == {
|
||||
("users", "alice"),
|
||||
}
|
||||
assert set(store.list_namespaces(prefix=["users", "alice"], limit=100)) == {
|
||||
("users", "alice"),
|
||||
}
|
||||
|
||||
@@ -24,11 +24,13 @@ from langgraph.store.base.batch import AsyncBatchedBaseStore
|
||||
|
||||
from langgraph.store.sqlite.base import (
|
||||
_PLACEHOLDER,
|
||||
NS_MATCH_FUNCTION,
|
||||
BaseSqliteStore,
|
||||
SqliteIndexConfig,
|
||||
_decode_ns_text,
|
||||
_ensure_index_config,
|
||||
_group_ops,
|
||||
_namespace_match,
|
||||
_row_to_item,
|
||||
_row_to_search_item,
|
||||
)
|
||||
@@ -150,6 +152,13 @@ class AsyncSqliteStore(AsyncBatchedBaseStore, BaseSqliteStore):
|
||||
if self.is_setup:
|
||||
return
|
||||
|
||||
# list_namespaces needs segment-aware matching, which SQLite cannot
|
||||
# express in LIKE or GLOB. Registered here rather than in __init__
|
||||
# because aiosqlite's create_function is a coroutine.
|
||||
await self.conn.create_function(
|
||||
NS_MATCH_FUNCTION, 2, _namespace_match, deterministic=True
|
||||
)
|
||||
|
||||
# Create migrations table if it doesn't exist
|
||||
await self.conn.execute(
|
||||
"""
|
||||
|
||||
@@ -93,12 +93,74 @@ class SqliteIndexConfig(IndexConfig):
|
||||
pass
|
||||
|
||||
|
||||
def _namespace_to_text(
|
||||
namespace: tuple[str, ...], handle_wildcards: bool = False
|
||||
) -> str:
|
||||
NS_MATCH_FUNCTION = "_langgraph_namespace_match"
|
||||
"""SQLite user function backing segment-aware namespace matching.
|
||||
|
||||
Registered under a private name rather than overriding `REGEXP`, so a caller's
|
||||
own `REGEXP` is left untouched.
|
||||
"""
|
||||
|
||||
|
||||
def _namespace_match(prefix: str | None, pattern: str) -> int:
|
||||
"""Backing implementation of `NS_MATCH_FUNCTION`."""
|
||||
if prefix is None:
|
||||
return 0
|
||||
return 1 if re.search(pattern, prefix) else 0
|
||||
|
||||
|
||||
def _escape_glob_literal(text: str) -> str:
|
||||
"""Escape GLOB metacharacters so `text` is matched literally.
|
||||
|
||||
GLOB has no `ESCAPE` clause, so metacharacters are wrapped in a character
|
||||
class instead. `]` is literal outside a class and needs no escaping.
|
||||
"""
|
||||
return text.replace("[", "[[]").replace("*", "[*]").replace("?", "[?]")
|
||||
|
||||
|
||||
def _namespace_prefix_condition(
|
||||
namespace_prefix: tuple[str, ...], column: str = "prefix"
|
||||
) -> tuple[str, tuple[str, ...]]:
|
||||
"""Build the SQL scoping a search to a namespace and its descendants.
|
||||
|
||||
Matches the namespace exactly or requires the `.` separator before any
|
||||
remainder, so a prefix of `("foo",)` does not also match `("foobar",)`.
|
||||
|
||||
Uses GLOB rather than LIKE because SQLite's LIKE is case-insensitive for
|
||||
ASCII, which would match `("FOO",)` for a prefix of `("foo",)` even though
|
||||
`get`/`put`/`delete` compare with `=` and treat those as distinct.
|
||||
|
||||
An empty prefix is unconstrained and matches every namespace.
|
||||
"""
|
||||
if not namespace_prefix:
|
||||
return "TRUE", ()
|
||||
path = _namespace_to_text(namespace_prefix)
|
||||
condition = f"({column} = ? OR {column} GLOB ?)"
|
||||
return condition, (path, f"{_escape_glob_literal(path)}.*")
|
||||
|
||||
|
||||
def _namespace_match_pattern(path: tuple[str, ...], match_type: str) -> str:
|
||||
"""Build a regex matching the dot-joined prefix on whole namespace segments.
|
||||
|
||||
Needed because neither LIKE nor GLOB can express "any character except the
|
||||
separator": GLOB has character classes but no quantifier, so `[^.]*` still
|
||||
crosses `.`. Matches how `InMemoryStore` compares namespaces element-wise.
|
||||
|
||||
`*` matches exactly one segment. Prefix matches stay open-ended but must end
|
||||
on a separator; suffix matches anchor at the end and begin on one.
|
||||
|
||||
Examples:
|
||||
prefix ("uid", "*", "alice") -> ^uid\\.[^.]+\\.alice(\\.|\\Z)
|
||||
suffix ("alice",) -> (^|\\.)alice\\Z
|
||||
"""
|
||||
segments = ("[^.]+" if part == "*" else re.escape(part) for part in path)
|
||||
body = r"\.".join(segments)
|
||||
if match_type == "suffix":
|
||||
return rf"(^|\.){body}\Z"
|
||||
return rf"^{body}(\.|\Z)"
|
||||
|
||||
|
||||
def _namespace_to_text(namespace: tuple[str, ...]) -> str:
|
||||
"""Convert namespace tuple to text string."""
|
||||
if handle_wildcards:
|
||||
namespace = tuple("%" if val == "*" else val for val in namespace)
|
||||
return ".".join(namespace)
|
||||
|
||||
|
||||
@@ -461,8 +523,11 @@ class BaseSqliteStore:
|
||||
else " AND " + " AND ".join(filter_conditions)
|
||||
)
|
||||
if op.namespace_prefix:
|
||||
prefix_filter_str = f"WHERE s.prefix LIKE ? {filter_str} "
|
||||
ns_args: Sequence = (f"{_namespace_to_text(op.namespace_prefix)}%",)
|
||||
ns_condition, ns_args_tuple = _namespace_prefix_condition(
|
||||
op.namespace_prefix, column="s.prefix"
|
||||
)
|
||||
prefix_filter_str = f"WHERE {ns_condition} {filter_str} "
|
||||
ns_args: Sequence = ns_args_tuple
|
||||
else:
|
||||
ns_args = ()
|
||||
if filter_str:
|
||||
@@ -503,12 +568,15 @@ class BaseSqliteStore:
|
||||
]
|
||||
# Regular search branch (no vector search)
|
||||
else:
|
||||
base_query = """
|
||||
ns_condition, ns_args_tuple = _namespace_prefix_condition(
|
||||
op.namespace_prefix
|
||||
)
|
||||
base_query = f"""
|
||||
SELECT prefix, key, value, created_at, updated_at, expires_at, ttl_minutes, NULL as score
|
||||
FROM store
|
||||
WHERE prefix LIKE ?
|
||||
WHERE {ns_condition}
|
||||
"""
|
||||
params = [f"{_namespace_to_text(op.namespace_prefix)}%"]
|
||||
params = list(ns_args_tuple)
|
||||
|
||||
if filter_conditions:
|
||||
params.extend(filter_params)
|
||||
@@ -549,16 +617,28 @@ class BaseSqliteStore:
|
||||
|
||||
if op.match_conditions:
|
||||
for cond in op.match_conditions:
|
||||
if cond.match_type == "prefix":
|
||||
where_clauses.append("prefix LIKE ?")
|
||||
params.append(
|
||||
f"{_namespace_to_text(cond.path, handle_wildcards=True)}%"
|
||||
)
|
||||
elif cond.match_type == "suffix":
|
||||
where_clauses.append("prefix LIKE ?")
|
||||
params.append(
|
||||
f"%{_namespace_to_text(cond.path, handle_wildcards=True)}"
|
||||
)
|
||||
if cond.match_type in ("prefix", "suffix"):
|
||||
if not cond.path:
|
||||
# An empty path constrains nothing; skipping keeps it a
|
||||
# no-op rather than emitting a pattern that matches no
|
||||
# namespace at all.
|
||||
continue
|
||||
if cond.match_type == "prefix" and "*" not in cond.path:
|
||||
# Equivalent to the anchored pattern, but SQLite can
|
||||
# satisfy `=` and a trailing-wildcard GLOB from
|
||||
# store_prefix_idx. The user function is opaque to the
|
||||
# planner, so it would scan every row and call back
|
||||
# into Python for each one.
|
||||
condition, args = _namespace_prefix_condition(
|
||||
tuple(cond.path)
|
||||
)
|
||||
where_clauses.append(condition)
|
||||
params.extend(args)
|
||||
else:
|
||||
where_clauses.append(f"{NS_MATCH_FUNCTION}(prefix, ?) = 1")
|
||||
params.append(
|
||||
_namespace_match_pattern(cond.path, cond.match_type)
|
||||
)
|
||||
else:
|
||||
logger.warning(
|
||||
"Unknown match_type in list_namespaces: %s", cond.match_type
|
||||
@@ -785,6 +865,9 @@ class SqliteStore(BaseSqliteStore, BaseStore):
|
||||
super().__init__()
|
||||
self._deserializer = deserializer
|
||||
self.conn = conn
|
||||
# Registered here rather than in from_conn_string so a caller-supplied
|
||||
# connection also gets it.
|
||||
conn.create_function(NS_MATCH_FUNCTION, 2, _namespace_match, deterministic=True)
|
||||
self.lock = threading.Lock()
|
||||
self.is_setup = False
|
||||
self.index_config = index
|
||||
|
||||
@@ -716,3 +716,32 @@ async def test_search_items(
|
||||
for ns in test_namespaces:
|
||||
key = f"item_{ns[-1]}"
|
||||
await store.adelete(ns, key)
|
||||
|
||||
|
||||
async def test_async_namespace_segment_boundary(store: AsyncSqliteStore) -> None:
|
||||
"""Segment-aware scoping on the async path.
|
||||
|
||||
Also covers that the namespace-match SQLite function is registered on the
|
||||
async connection -- aiosqlite's create_function is a coroutine, so it is
|
||||
registered in setup() rather than __init__.
|
||||
"""
|
||||
for namespace in [
|
||||
("foo",),
|
||||
("foo", "child"),
|
||||
("foobar",),
|
||||
("uid", "users", "alice"),
|
||||
("uid", "users", "malice"),
|
||||
("user_1",),
|
||||
("userX1",),
|
||||
]:
|
||||
await store.aput(namespace, "k", {"v": 1})
|
||||
|
||||
found = {item.namespace for item in await store.asearch(("foo",), limit=100)}
|
||||
assert found == {("foo",), ("foo", "child")}
|
||||
|
||||
found = {item.namespace for item in await store.asearch(("user_1",), limit=100)}
|
||||
assert found == {("user_1",)}
|
||||
|
||||
assert set(await store.alist_namespaces(suffix=["alice"], limit=100)) == {
|
||||
("uid", "users", "alice"),
|
||||
}
|
||||
|
||||
@@ -18,7 +18,13 @@ from langgraph.store.base import (
|
||||
)
|
||||
|
||||
from langgraph.store.sqlite import SqliteStore
|
||||
from langgraph.store.sqlite.base import SqliteIndexConfig
|
||||
from langgraph.store.sqlite.base import (
|
||||
NS_MATCH_FUNCTION,
|
||||
BaseSqliteStore,
|
||||
SqliteIndexConfig,
|
||||
_escape_glob_literal,
|
||||
_namespace_match_pattern,
|
||||
)
|
||||
|
||||
|
||||
# Local embeddings implementation for testing vector search
|
||||
@@ -1229,3 +1235,208 @@ def test_non_ascii(
|
||||
assert result3[0].key == "3"
|
||||
assert result4[0].key == "4"
|
||||
assert result5[0].key == "5"
|
||||
|
||||
|
||||
def test_escape_glob_literal() -> None:
|
||||
assert _escape_glob_literal("users.alice") == "users.alice"
|
||||
# "_" and "%" are LIKE wildcards but literal in GLOB, so they are left alone.
|
||||
assert _escape_glob_literal("user_1") == "user_1"
|
||||
assert _escape_glob_literal("100%") == "100%"
|
||||
assert _escape_glob_literal("a*b") == "a[*]b"
|
||||
assert _escape_glob_literal("a?b") == "a[?]b"
|
||||
assert _escape_glob_literal("a[b") == "a[[]b"
|
||||
|
||||
|
||||
def test_namespace_match_pattern() -> None:
|
||||
assert _namespace_match_pattern(("foo",), "prefix") == r"^foo(\.|\Z)"
|
||||
assert (
|
||||
_namespace_match_pattern(("uid", "*", "alice"), "prefix")
|
||||
== r"^uid\.[^.]+\.alice(\.|\Z)"
|
||||
)
|
||||
assert _namespace_match_pattern(("alice",), "suffix") == r"(^|\.)alice\Z"
|
||||
|
||||
|
||||
def test_search_namespace_segment_boundary(store: SqliteStore) -> None:
|
||||
"""Prefix scoping must stop at namespace segment boundaries.
|
||||
|
||||
Namespaces are stored dot-joined, so matching the raw text also returns
|
||||
siblings sharing leading characters.
|
||||
"""
|
||||
for namespace in [
|
||||
("foo",),
|
||||
("foo", "child"),
|
||||
("foo", "child", "deep"),
|
||||
("foobar",),
|
||||
("foobar", "baz"),
|
||||
("foo2",),
|
||||
]:
|
||||
store.put(namespace, "k", {"v": 1})
|
||||
|
||||
def _namespaces(prefix: tuple[str, ...]) -> set[tuple[str, ...]]:
|
||||
return {item.namespace for item in store.search(prefix, limit=100)}
|
||||
|
||||
assert _namespaces(("foo",)) == {
|
||||
("foo",),
|
||||
("foo", "child"),
|
||||
("foo", "child", "deep"),
|
||||
}
|
||||
# The sibling scope is independent, not merely narrower.
|
||||
assert _namespaces(("foobar",)) == {("foobar",), ("foobar", "baz")}
|
||||
assert _namespaces(("foo2",)) == {("foo2",)}
|
||||
assert _namespaces(("fo",)) == set()
|
||||
|
||||
|
||||
def test_search_namespace_wildcard_chars_are_literal(store: SqliteStore) -> None:
|
||||
"""LIKE and GLOB metacharacters in labels must be matched literally."""
|
||||
for namespace in [
|
||||
("user_1",),
|
||||
("user_1", "child"),
|
||||
("userX1",),
|
||||
("a%b",),
|
||||
("axxb",),
|
||||
("star*",),
|
||||
("starX",),
|
||||
]:
|
||||
store.put(namespace, "k", {"v": 1})
|
||||
|
||||
def _namespaces(prefix: tuple[str, ...]) -> set[tuple[str, ...]]:
|
||||
return {item.namespace for item in store.search(prefix, limit=100)}
|
||||
|
||||
# Also asserts each namespace still matches itself, which catches escaping
|
||||
# the equality arm by mistake.
|
||||
assert _namespaces(("user_1",)) == {("user_1",), ("user_1", "child")}
|
||||
assert _namespaces(("a%b",)) == {("a%b",)}
|
||||
assert _namespaces(("star*",)) == {("star*",)}
|
||||
|
||||
|
||||
def test_search_namespace_is_case_sensitive(store: SqliteStore) -> None:
|
||||
"""Search must agree with get/put, which compare namespaces with `=`.
|
||||
|
||||
SQLite's LIKE is case-insensitive for ASCII, so matching with it conflated
|
||||
namespaces that every other operation treats as distinct.
|
||||
"""
|
||||
store.put(("Foo",), "k", {"v": "upper"})
|
||||
store.put(("foo",), "k", {"v": "lower"})
|
||||
|
||||
assert {item.namespace for item in store.search(("foo",), limit=100)} == {("foo",)}
|
||||
assert {item.namespace for item in store.search(("Foo",), limit=100)} == {("Foo",)}
|
||||
|
||||
|
||||
def test_list_namespaces_segment_boundary(store: SqliteStore) -> None:
|
||||
for namespace in [
|
||||
("foo",),
|
||||
("foo", "child"),
|
||||
("foobar",),
|
||||
("foobar", "baz"),
|
||||
("uid", "users", "alice"),
|
||||
("uid", "users", "malice"),
|
||||
("uid", "a", "b", "alice"),
|
||||
]:
|
||||
store.put(namespace, "k", {"v": 1})
|
||||
|
||||
assert set(store.list_namespaces(prefix=["foo"], limit=100)) == {
|
||||
("foo",),
|
||||
("foo", "child"),
|
||||
}
|
||||
# Suffix must align to a segment: "malice" does not end with the "alice"
|
||||
# segment.
|
||||
assert set(store.list_namespaces(suffix=["alice"], limit=100)) == {
|
||||
("uid", "users", "alice"),
|
||||
("uid", "a", "b", "alice"),
|
||||
}
|
||||
# "*" spans exactly one segment.
|
||||
assert set(store.list_namespaces(prefix=["uid", "*", "alice"], limit=100)) == {
|
||||
("uid", "users", "alice"),
|
||||
}
|
||||
# Prefix matching stays open-ended across depth.
|
||||
assert set(store.list_namespaces(prefix=["uid"], limit=100)) == {
|
||||
("uid", "users", "alice"),
|
||||
("uid", "users", "malice"),
|
||||
("uid", "a", "b", "alice"),
|
||||
}
|
||||
|
||||
|
||||
def test_search_empty_prefix_is_unconstrained(store: SqliteStore) -> None:
|
||||
"""An empty prefix constrains nothing and must return every namespace."""
|
||||
for namespace in [("a",), ("b", "c"), ("d", "e", "f")]:
|
||||
store.put(namespace, "k", {"v": 1})
|
||||
|
||||
assert {item.namespace for item in store.search((), limit=100)} == {
|
||||
("a",),
|
||||
("b", "c"),
|
||||
("d", "e", "f"),
|
||||
}
|
||||
|
||||
|
||||
def test_namespace_labels_with_trailing_newline(store: SqliteStore) -> None:
|
||||
"""Labels may contain newlines, and must not match a differently-named label.
|
||||
|
||||
Python's `$` also matches just before a trailing newline, so the patterns use
|
||||
`\\Z` to anchor at the true end of the string.
|
||||
"""
|
||||
store.put(("users", "alice"), "k", {"v": 1})
|
||||
store.put(("users", "alice\n"), "k", {"v": 2})
|
||||
|
||||
assert set(store.list_namespaces(suffix=["alice"], limit=100)) == {
|
||||
("users", "alice"),
|
||||
}
|
||||
assert set(store.list_namespaces(prefix=["users", "alice"], limit=100)) == {
|
||||
("users", "alice"),
|
||||
}
|
||||
|
||||
|
||||
def test_list_namespaces_prefix_uses_indexable_condition() -> None:
|
||||
"""Plain prefixes must use the indexable condition, not the match function.
|
||||
|
||||
A user function is opaque to the query planner, so it scans every row and
|
||||
calls back into Python for each one. Only suffix and wildcard paths, which
|
||||
no SQLite operator can express, need it.
|
||||
"""
|
||||
store = BaseSqliteStore()
|
||||
|
||||
def where(match_type: str, path: tuple[str, ...]) -> str:
|
||||
op = ListNamespacesOp(
|
||||
match_conditions=(MatchCondition(match_type=match_type, path=path),),
|
||||
max_depth=None,
|
||||
limit=10,
|
||||
offset=0,
|
||||
)
|
||||
query, _ = store._get_batch_list_namespaces_queries([(0, op)])[0]
|
||||
return " ".join(query.split())
|
||||
|
||||
assert "GLOB" in where("prefix", ("uid", "users"))
|
||||
assert NS_MATCH_FUNCTION not in where("prefix", ("uid", "users"))
|
||||
# A label that merely contains "*" is not the wildcard.
|
||||
assert "GLOB" in where("prefix", ("star*",))
|
||||
# Wildcard and suffix cannot be expressed by GLOB, so they keep the function.
|
||||
assert NS_MATCH_FUNCTION in where("prefix", ("uid", "*", "alice"))
|
||||
assert NS_MATCH_FUNCTION in where("suffix", ("alice",))
|
||||
|
||||
|
||||
def test_list_namespaces_metacharacter_labels(store: SqliteStore) -> None:
|
||||
"""Metacharacters in labels are literal on both matching paths.
|
||||
|
||||
Plain prefixes take the `= OR GLOB` condition and wildcard/suffix paths take
|
||||
the regex function, so escaping has to hold in two different syntaxes.
|
||||
"""
|
||||
pairs = [
|
||||
("star*", "starX"),
|
||||
("q?m", "qXm"),
|
||||
("br[ack]et", "brXacXket"),
|
||||
("user_1", "userX1"),
|
||||
("a%b", "axxb"),
|
||||
("plus+", "plusX"),
|
||||
]
|
||||
for label, decoy in pairs:
|
||||
store.put((label,), "k", {"v": 1})
|
||||
store.put((decoy,), "k", {"v": 1})
|
||||
store.put((label, "child"), "k", {"v": 1})
|
||||
|
||||
for label, decoy in pairs:
|
||||
found = set(store.list_namespaces(prefix=[label], limit=100))
|
||||
assert found == {(label,), (label, "child")}
|
||||
# The decoy differs only where the metacharacter would have matched.
|
||||
assert (decoy,) not in found
|
||||
assert set(store.list_namespaces(prefix=[label, "child"], limit=100)) == {
|
||||
(label, "child"),
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user