checkpoint-postgres: handle null chars in metadata (#1885)

This commit is contained in:
Vadym Barda
2024-09-27 16:02:03 +00:00
committed by GitHub
parent 947c74c10d
commit bf19dc7d08
3 changed files with 21 additions and 1 deletions
@@ -229,7 +229,8 @@ class BasePostgresSaver(BaseCheckpointSaver[str]):
def _dump_metadata(self, metadata: CheckpointMetadata) -> str:
serialized_metadata = self.jsonplus_serde.dumps(metadata)
return serialized_metadata.decode()
# NOTE: we're using JSON serializer (not msgpack), so we need to remove null characters before writing
return serialized_metadata.decode().replace("\\u0000", "")
def get_next_version(self, current: Optional[str], channel: ChannelProtocol) -> str:
if current is None:
@@ -101,3 +101,13 @@ class TestAsyncPostgresSaver:
} == {"", "inner"}
# TODO: test before and limit params
async def test_null_chars(self) -> None:
async with AsyncPostgresSaver.from_conn_string(DEFAULT_URI) as saver:
config = await saver.aput(
self.config_1, self.chkpnt_1, {"my_key": "\x00abc"}, {}
)
assert (await saver.aget_tuple(config)).metadata["my_key"] == "abc"
assert [c async for c in saver.alist(None, filter={"my_key": "abc"})][
0
].metadata["my_key"] == "abc"
@@ -101,3 +101,12 @@ class TestPostgresSaver:
} == {"", "inner"}
# TODO: test before and limit params
def test_null_chars(self) -> None:
with PostgresSaver.from_conn_string(DEFAULT_URI) as saver:
config = saver.put(self.config_1, self.chkpnt_1, {"my_key": "\x00abc"}, {})
assert saver.get_tuple(config).metadata["my_key"] == "abc"
assert (
list(saver.list(None, filter={"my_key": "abc"}))[0].metadata["my_key"]
== "abc"
)