This commit is contained in:
Eugene Yurtsev
2025-12-09 11:28:15 -05:00
parent 706f3e981e
commit b7d068677c
2 changed files with 156 additions and 92 deletions
@@ -3,6 +3,7 @@ from __future__ import annotations
import concurrent.futures
import datetime
import logging
import math
import re
import sqlite3
import threading
@@ -404,12 +405,9 @@ class BaseSqliteStore:
# SQLite json_extract returns unquoted string values
if isinstance(value, str):
filter_conditions.append(
"json_extract(value, '$."
+ key
+ "') = '"
+ value.replace("'", "''")
+ "'"
"json_extract(value, '$." + key + "') = ?"
)
filter_params.append(value)
elif value is None:
filter_conditions.append(
"json_extract(value, '$." + key + "') IS NULL"
@@ -423,9 +421,11 @@ class BaseSqliteStore:
+ ("1" if value else "0")
)
elif isinstance(value, (int, float)):
# Use parameterized query to handle special floats and large integers
filter_conditions.append(
"json_extract(value, '$." + key + "') = " + str(value)
"json_extract(value, '$." + key + "') = ?"
)
filter_params.append(float(value))
else:
# Complex objects (list, dict, …) compare JSON text
filter_conditions.append(
@@ -636,85 +636,66 @@ class BaseSqliteStore:
# We need to properly format values for SQLite JSON extraction comparison
if op == "$eq":
if isinstance(value, str):
# Direct string comparison with proper quoting for unquoted json_extract result
return (
f"json_extract(value, '$.{key}') = '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') = ?", [value]
elif value is None:
return f"json_extract(value, '$.{key}') IS NULL", []
elif isinstance(value, bool):
# SQLite JSON stores booleans as integers
return f"json_extract(value, '$.{key}') = {1 if value else 0}", []
elif isinstance(value, (int, float)):
return f"json_extract(value, '$.{key}') = {value}", []
# Convert to float to handle inf, -inf, nan, and very large integers
# SQLite REAL can handle these cases better than INTEGER
return f"json_extract(value, '$.{key}') = ?", [float(value)]
else:
return f"json_extract(value, '$.{key}') = ?", [orjson.dumps(value)]
elif op == "$gt":
# For numeric values, SQLite needs to compare as numbers, not strings
if isinstance(value, (int, float)):
return f"CAST(json_extract(value, '$.{key}') AS REAL) > {value}", []
# Convert to float to handle special values and very large integers
return f"CAST(json_extract(value, '$.{key}') AS REAL) > ?", [
float(value)
]
elif isinstance(value, str):
return (
f"json_extract(value, '$.{key}') > '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') > ?", [value]
else:
return f"json_extract(value, '$.{key}') > ?", [orjson.dumps(value)]
elif op == "$gte":
if isinstance(value, (int, float)):
return f"CAST(json_extract(value, '$.{key}') AS REAL) >= {value}", []
return f"CAST(json_extract(value, '$.{key}') AS REAL) >= ?", [
float(value)
]
elif isinstance(value, str):
return (
f"json_extract(value, '$.{key}') >= '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') >= ?", [value]
else:
return f"json_extract(value, '$.{key}') >= ?", [orjson.dumps(value)]
elif op == "$lt":
if isinstance(value, (int, float)):
return f"CAST(json_extract(value, '$.{key}') AS REAL) < {value}", []
return f"CAST(json_extract(value, '$.{key}') AS REAL) < ?", [
float(value)
]
elif isinstance(value, str):
return (
f"json_extract(value, '$.{key}') < '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') < ?", [value]
else:
return f"json_extract(value, '$.{key}') < ?", [orjson.dumps(value)]
elif op == "$lte":
if isinstance(value, (int, float)):
return f"CAST(json_extract(value, '$.{key}') AS REAL) <= {value}", []
return f"CAST(json_extract(value, '$.{key}') AS REAL) <= ?", [
float(value)
]
elif isinstance(value, str):
return (
f"json_extract(value, '$.{key}') <= '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') <= ?", [value]
else:
return f"json_extract(value, '$.{key}') <= ?", [orjson.dumps(value)]
elif op == "$ne":
if isinstance(value, str):
return (
f"json_extract(value, '$.{key}') != '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') != ?", [value]
elif value is None:
return f"json_extract(value, '$.{key}') IS NOT NULL", []
elif isinstance(value, bool):
return f"json_extract(value, '$.{key}') != {1 if value else 0}", []
elif isinstance(value, (int, float)):
return f"json_extract(value, '$.{key}') != {value}", []
# Convert to float for consistency
return f"json_extract(value, '$.{key}') != ?", [float(value)]
else:
return f"json_extract(value, '$.{key}') != ?", [orjson.dumps(value)]
else:
@@ -874,85 +855,66 @@ class SqliteStore(BaseSqliteStore, BaseStore):
# We need to properly format values for SQLite JSON extraction comparison
if op == "$eq":
if isinstance(value, str):
# Direct string comparison with proper quoting for unquoted json_extract result
return (
f"json_extract(value, '$.{key}') = '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') = ?", [value]
elif value is None:
return f"json_extract(value, '$.{key}') IS NULL", []
elif isinstance(value, bool):
# SQLite JSON stores booleans as integers
return f"json_extract(value, '$.{key}') = {1 if value else 0}", []
elif isinstance(value, (int, float)):
return f"json_extract(value, '$.{key}') = {value}", []
# Convert to float to handle inf, -inf, nan, and very large integers
# SQLite REAL can handle these cases better than INTEGER
return f"json_extract(value, '$.{key}') = ?", [float(value)]
else:
return f"json_extract(value, '$.{key}') = ?", [orjson.dumps(value)]
elif op == "$gt":
# For numeric values, SQLite needs to compare as numbers, not strings
if isinstance(value, (int, float)):
return f"CAST(json_extract(value, '$.{key}') AS REAL) > {value}", []
# Convert to float to handle special values and very large integers
return f"CAST(json_extract(value, '$.{key}') AS REAL) > ?", [
float(value)
]
elif isinstance(value, str):
return (
f"json_extract(value, '$.{key}') > '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') > ?", [value]
else:
return f"json_extract(value, '$.{key}') > ?", [orjson.dumps(value)]
elif op == "$gte":
if isinstance(value, (int, float)):
return f"CAST(json_extract(value, '$.{key}') AS REAL) >= {value}", []
return f"CAST(json_extract(value, '$.{key}') AS REAL) >= ?", [
float(value)
]
elif isinstance(value, str):
return (
f"json_extract(value, '$.{key}') >= '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') >= ?", [value]
else:
return f"json_extract(value, '$.{key}') >= ?", [orjson.dumps(value)]
elif op == "$lt":
if isinstance(value, (int, float)):
return f"CAST(json_extract(value, '$.{key}') AS REAL) < {value}", []
return f"CAST(json_extract(value, '$.{key}') AS REAL) < ?", [
float(value)
]
elif isinstance(value, str):
return (
f"json_extract(value, '$.{key}') < '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') < ?", [value]
else:
return f"json_extract(value, '$.{key}') < ?", [orjson.dumps(value)]
elif op == "$lte":
if isinstance(value, (int, float)):
return f"CAST(json_extract(value, '$.{key}') AS REAL) <= {value}", []
return f"CAST(json_extract(value, '$.{key}') AS REAL) <= ?", [
float(value)
]
elif isinstance(value, str):
return (
f"json_extract(value, '$.{key}') <= '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') <= ?", [value]
else:
return f"json_extract(value, '$.{key}') <= ?", [orjson.dumps(value)]
elif op == "$ne":
if isinstance(value, str):
return (
f"json_extract(value, '$.{key}') != '"
+ value.replace("'", "''")
+ "'",
[],
)
return f"json_extract(value, '$.{key}') != ?", [value]
elif value is None:
return f"json_extract(value, '$.{key}') IS NOT NULL", []
elif isinstance(value, bool):
return f"json_extract(value, '$.{key}') != {1 if value else 0}", []
elif isinstance(value, (int, float)):
return f"json_extract(value, '$.{key}') != {value}", []
# Convert to float for consistency
return f"json_extract(value, '$.{key}') != ?", [float(value)]
else:
return f"json_extract(value, '$.{key}') != ?", [orjson.dumps(value)]
else: