fix: order delta channel replay by task path

DeltaChannel reconstructs its value by replaying ancestor writes through
the reducer. Every saver ordered a checkpoint's writes by (task_id, idx),
but live execution applies them in task-path order: apply_writes sorts a
super-step's tasks by task_path_str(task.path[:3]) before calling
channel.update. task_id is a hash of the path, so the two orders are
unrelated, and two or more tasks writing one DeltaChannel in a single
super-step replayed in an arbitrary permutation.

Reducers are only required to be batching-invariant, not order-invariant,
so the permutation changes the value: get_state disagreed with what invoke
returned, and continuing the thread persisted the reordered replay as the
base for later writes.

Replay now orders by (task_path, task_id, idx), following the precedent
already set for the Send channel by SELECT_PENDING_SENDS_SQL. InMemorySaver
and the postgres savers already persisted task_path and only needed the
sort key; sqlite accepted task_path on put_writes and dropped it, so the
writes table gains the column, added by setup() to databases created by
earlier versions.

Writes stored without a task_path sort first within their checkpoint, which
is where live execution applies the task-less input writes that carry "".

Co-authored-by: ErenAta16 <149434812+ErenAta16@users.noreply.github.com>
Co-authored-by: ragnarok268 <58264829+ragnarok268@users.noreply.github.com>
This commit is contained in:
Elior Nataf Lackritz
2026-09-23 10:55:18 -04:00
co-authored by ErenAta16 ragnarok268
parent bdb85b5aa8
commit cdd02084e9
9 changed files with 307 additions and 31 deletions
@@ -168,6 +168,7 @@ class _DeltaStage2Row(TypedDict, total=False):
type: str | None
blob: bytes | None
task_id: str | None # "w" rows only
task_path: str | None # "w" rows only
idx: int | None # "w" rows only
version: str | None # "b" rows only
@@ -319,7 +320,7 @@ def _build_delta_stage2_sql(
branches.append(
"SELECT 'w'::text AS _kind, "
"checkpoint_id, channel, "
"type, blob, task_id, idx, NULL::text AS version "
"type, blob, task_id, task_path, idx, NULL::text AS version "
"FROM checkpoint_writes "
"WHERE thread_id = %s AND checkpoint_ns = %s AND channel = %s "
"AND checkpoint_id = ANY(%s)"
@@ -327,7 +328,8 @@ def _build_delta_stage2_sql(
for _ in channels_with_seed:
branches.append(
"SELECT 'b'::text AS _kind, NULL::text AS checkpoint_id, channel, "
"type, blob, NULL::text AS task_id, NULL::int AS idx, version "
"type, blob, NULL::text AS task_id, NULL::text AS task_path, "
"NULL::int AS idx, version "
"FROM checkpoint_blobs "
"WHERE thread_id = %s AND checkpoint_ns = %s AND channel = %s "
"AND version = %s"
@@ -492,10 +494,11 @@ class BasePostgresSaver(BaseCheckpointSaver[str]):
stored value, or when the seed blob is sentinel "empty" — in both cases
the consumer treats absence as "start empty".
"""
# writes_by_ch_by_cid[channel][cid] = list of (type, blob, task_id, idx)
writes_by_ch_by_cid: dict[str, dict[str, list[tuple[str, bytes, str, int]]]] = {
ch: {} for ch in channels
}
# writes_by_ch_by_cid[channel][cid] = list of
# (type, blob, task_id, idx, task_path)
writes_by_ch_by_cid: dict[
str, dict[str, list[tuple[str, bytes, str, int, str]]]
] = {ch: {} for ch in channels}
# seed_blob_by_ver[(channel, version)] = (type, blob)
seed_blob_by_ver: dict[tuple[str, str], tuple[str, bytes]] = {}
@@ -506,8 +509,14 @@ class BasePostgresSaver(BaseCheckpointSaver[str]):
cid = cast(str, r["checkpoint_id"])
writes_by_ch_by_cid.setdefault(ch, {}).setdefault(cid, []).append(
cast(
"tuple[str, bytes, str, int]",
(r["type"], r["blob"], r["task_id"], r["idx"]),
"tuple[str, bytes, str, int, str]",
(
r["type"],
r["blob"],
r["task_id"],
r["idx"],
r["task_path"],
),
)
)
else: # kind == "b"
@@ -516,10 +525,10 @@ class BasePostgresSaver(BaseCheckpointSaver[str]):
"tuple[str, bytes]", (r["type"], r["blob"])
)
# Sort writes per (channel, cid) newest-first by (task_id, idx)
# Sort writes per (channel, cid) newest-first by (task_path, task_id, idx)
for cid_map in writes_by_ch_by_cid.values():
for ws in cid_map.values():
ws.sort(key=lambda w: (w[2], w[3]), reverse=True)
ws.sort(key=lambda w: (w[4], w[2], w[3]), reverse=True)
result: dict[str, DeltaChannelHistory] = {}
for ch in channels:
@@ -529,7 +538,9 @@ class BasePostgresSaver(BaseCheckpointSaver[str]):
collected: list[PendingWrite] = []
cid_writes = writes_by_ch_by_cid.get(ch, {})
for cid in chain_cids:
for type_tag, write_blob, task_id, _idx in cid_writes.get(cid, []):
for type_tag, write_blob, task_id, _idx, _path in cid_writes.get(
cid, []
):
val = self.serde.loads_typed((type_tag, write_blob))
collected.append((task_id, ch, val))
collected.reverse()